diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..4c82899 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +target +.travis? +pom.xml +*.md +.gitignore +.idea +code/tests +code/requirements.tests.txt +__pycache__ diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 21a96d3..69aea17 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,6 +1,4 @@ -# This is a basic workflow to help you get started with Actions - -name: CI +name: "Test and build notification scripts." on: push: @@ -13,18 +11,34 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - - name: Run a one-line script - run: echo Hello, world! + - uses: actions/setup-python@v2 + with: + python-version: '3.8' - - name: Run a multi-line script + - name: Run unit tests run: | - echo Add other actions to build, - echo test, and deploy your project. + pip install -r requirements.tests.txt + pytest tests/ --junitxml=test-report.xml -v + + - name: Multi-arch docker image build prerequired + run: sudo docker run --privileged linuxkit/binfmt:v0.7 + + - name: Build and deploy on architecture + env: + DOCKER_USERNAME: ${{ secrets.SIXSQ_DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.SIXSQ_DOCKER_PASSWORD }} + run: ./container-release.sh + + - name: Publish Unit Test Results + uses: EnricoMi/publish-unit-test-result-action@v1 + if: always() + with: + files: test-report.xml notify: if: always() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b790f23 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,41 @@ +# Automatic release for every new tag +name: "Release" + +on: + push: + tags: + - "*.*.*" + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v2 + + - name: Multi-arch docker image build prerequired + run: sudo docker run --privileged linuxkit/binfmt:v0.7 + + - name: Build and deploy on architecture + env: + DOCKER_USERNAME: ${{ secrets.SIXSQ_DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.SIXSQ_DOCKER_PASSWORD }} + DOCKER_ORG: nuvla + run: ./container-release.sh + + + notify: + if: always() + name: Post Workflow Status To Slack + needs: + - build + runs-on: ubuntu-latest + steps: + - name: Slack Workflow Notification + uses: Gamesight/slack-workflow-status@master + with: + # Required Input + repo_token: ${{secrets.GITHUB_TOKEN}} + slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}} + # Optional Input + icon_emoji: ':rocket:' diff --git a/container-release.sh b/container-release.sh new file mode 100755 index 0000000..586f422 --- /dev/null +++ b/container-release.sh @@ -0,0 +1,88 @@ +#!/bin/bash -xe + +############################### +# CHANGE THIS ON EVERY REPO # +DOCKER_IMAGE=$(basename `git rev-parse --show-toplevel`) +############################### + +# default env vars in GH actions +GIT_BRANCH=$(echo ${GITHUB_REF} | awk -F'/' '{print $(NF)}' | sed -e 's/[^a-z0-9\._-]/-/g') + +# non-tagged builds are not releases, so they always go on nuvladev +DOCKER_ORG=${DOCKER_ORG:-nuvladev} + +MANIFEST=${DOCKER_ORG}/${DOCKER_IMAGE}:${GIT_BRANCH} + +platforms=(amd64 arm64) + + +# +# remove any previous builds +# + +rm -Rf target/*.tar +mkdir -p target + +# +# generate image for each platform +# + +for platform in "${platforms[@]}"; do + GIT_BUILD_TIME=$(date --utc +%FT%T.%3NZ) + docker run --rm --privileged -v ${PWD}:/tmp/work --entrypoint buildctl-daemonless.sh moby/buildkit:master \ + build \ + --frontend dockerfile.v0 \ + --opt platform=linux/${platform} \ + --opt filename=./Dockerfile \ + --opt build-arg:GIT_BRANCH=${GIT_BRANCH} \ + --opt build-arg:GIT_BUILD_TIME=${GIT_BUILD_TIME} \ + --opt build-arg:GIT_COMMIT_ID=${GITHUB_SHA} \ + --opt build-arg:GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER} \ + --opt build-arg:GITHUB_RUN_ID=${GITHUB_RUN_ID} \ + --opt build-arg:PROJECT_URL=${GIHUB_SERVER_URL}/${GITHUB_REPOSITORY} \ + --output type=docker,name=${MANIFEST}-${platform},dest=/tmp/work/target/${DOCKER_IMAGE}-${platform}.docker.tar \ + --local context=/tmp/work \ + --local dockerfile=/tmp/work \ + --progress plain + +done + +# +# load all generated images +# + +for platform in "${platforms[@]}"; do + docker load --input ./target/${DOCKER_IMAGE}-${platform}.docker.tar +done + + +manifest_args=(${MANIFEST}) + +# +# login to docker hub +# + +unset HISTFILE +echo ${DOCKER_PASSWORD} | docker login -u ${DOCKER_USERNAME} --password-stdin + +# +# push all generated images +# + +for platform in "${platforms[@]}"; do + docker push ${MANIFEST}-${platform} + manifest_args+=("${MANIFEST}-${platform}") +done + +# +# create manifest, update, and push +# + +export DOCKER_CLI_EXPERIMENTAL=enabled +docker manifest create "${manifest_args[@]}" + +for platform in "${platforms[@]}"; do + docker manifest annotate ${MANIFEST} ${MANIFEST}-${platform} --arch ${platform} +done + +docker manifest push --purge ${MANIFEST} \ No newline at end of file diff --git a/requirements.tests.txt b/requirements.tests.txt new file mode 100644 index 0000000..25b96d4 --- /dev/null +++ b/requirements.tests.txt @@ -0,0 +1,2 @@ +-r requirements.txt +pytest==7.0.1 diff --git a/requirements.txt b/requirements.txt index 1dd4ad1..00c8c38 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ kafka-python==2.0.2 requests==2.25.0 urllib3==1.26.2 Jinja2==2.11.2 +# To fix https://github.com/aws/aws-sam-cli/issues/3661 +markupsafe==2.0.1 diff --git a/src/notify-email.py b/src/notify-email.py index c6c8ba4..ef4ed0f 100755 --- a/src/notify-email.py +++ b/src/notify-email.py @@ -1,14 +1,18 @@ #!/usr/bin/env python3 -from notify_deps import * - -import smtplib +import multiprocessing +import os import requests +import smtplib +import time from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from jinja2 import Template from datetime import datetime +from notify_deps import get_logger, timestamp_convert, main +from notify_deps import NUVLA_ENDPOINT + log_local = get_logger('email') @@ -33,7 +37,7 @@ def get_nuvla_config(): headers = {'nuvla-authn-info': nuvla_api_authn_header} resp = requests.get(config_url, headers=headers) if resp.status_code != 200: - raise Exception(f'Failed to get response from server: status {resp.status_code}') + raise EnvironmentError(f'Failed to get response from server: status {resp.status_code}') return resp.json() @@ -47,7 +51,7 @@ def set_smpt_params(): try: SMTP_PORT = int(os.environ['SMTP_PORT']) except ValueError: - raise Exception(f"Incorrect value for SMTP_PORT number: {os.environ['SMTP_PORT']}") + raise ValueError(f"Incorrect value for SMTP_PORT number: {os.environ['SMTP_PORT']}") SMTP_SSL = os.environ['SMTP_SSL'].lower() in ['true', 'True'] else: nuvla_config = get_nuvla_config() @@ -59,7 +63,7 @@ def set_smpt_params(): except Exception as ex: msg = f'Provide full SMTP config either via env vars or in configuration/nuvla: {ex}' log_local.error(msg) - raise Exception(msg) + raise ValueError(msg) KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_EMAIL_S' @@ -81,7 +85,6 @@ def get_smtp_server(debug_level=0) -> smtplib.SMTP: def html_content(values: dict): - # subs_config_id = values.get('SUBS_ID') subs_config_link = f'Notification configuration' r_uri = values.get('RESOURCE_URI') @@ -135,12 +138,19 @@ def send(server: smtplib.SMTP, recipients, subject, html, attempts=SEND_EMAIL_AT raise SendFailedMaxAttempts(f'Failed sending email after {attempts} attempts.') +def get_recipients(v: dict): + return list(filter(lambda x: x != '', v.get('DESTINATION', '').split(' '))) + + def worker(workq: multiprocessing.Queue): smtp_server = get_smtp_server() while True: msg = workq.get() if msg: - recipients = msg.value['DESTINATION'].split(',') + recipients = get_recipients(msg.value) + if len(recipients) == 0: + log_local.warning(f'No recipients provided in: {msg.value}') + continue r_id = msg.value.get('RESOURCE_ID') r_name = msg.value.get('NAME') subject = msg.value.get('SUBS_NAME') or f'{r_name or r_id} alert' diff --git a/src/notify-slack.py b/src/notify-slack.py index e24183a..7b1c820 100755 --- a/src/notify-slack.py +++ b/src/notify-slack.py @@ -1,9 +1,14 @@ #!/usr/bin/env python3 +import json +import datetime +import multiprocessing import requests +import os import re -from notify_deps import * +from notify_deps import get_logger, timestamp_convert, main +from notify_deps import NUVLA_ENDPOINT KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S' KAFKA_GROUP_ID = 'nuvla-notification-slack' @@ -18,7 +23,6 @@ def message_content(values: dict): - # subs_config_id = values.get('SUBS_ID') subs_name = lt.sub('<', gt.sub('>', values.get('SUBS_NAME', ''))) subs_config_txt = f'<{NUVLA_ENDPOINT}/ui/notifications|{subs_name}>' diff --git a/tests/notify_deps.py b/tests/notify_deps.py new file mode 120000 index 0000000..b26ed92 --- /dev/null +++ b/tests/notify_deps.py @@ -0,0 +1 @@ +../src/notify_deps.py \ No newline at end of file diff --git a/tests/test_notify_email.py b/tests/test_notify_email.py new file mode 100644 index 0000000..fcd4b26 --- /dev/null +++ b/tests/test_notify_email.py @@ -0,0 +1,14 @@ +import unittest + +from notify_email import get_recipients + + +class NotifyEmail(unittest.TestCase): + + def test_get_recipients(self): + assert 0 == len(get_recipients({})) + e1 = 'a@b.c' + e2 = 'a@b.c' + assert [e1] == get_recipients({'DESTINATION': e1}) + assert [e1, e2] == get_recipients({'DESTINATION': f'{e1} {e2}'}) + assert [e1, e2] == get_recipients({'DESTINATION': f' {e1} {e2} '})