Merge pull request #13 from nuvla/fix-12

* fix issue
* test and build in CI
* minor
* typo
* fix for aws/aws-sam-cli#3661
* exec permission for container-release.sh
* multi-arch docker image build required

.dockerignore

@@ -0,0 +1,9 @@
+target
+.travis?
+pom.xml
+*.md
+.gitignore
+.idea
+code/tests
+code/requirements.tests.txt
+__pycache__

.github/workflows/main.yml

@@ -1,6 +1,4 @@
-# This is a basic workflow to help you get started with Actions
-
-name: CI
+name: "Test and build notification scripts."
 
 on:
   push:
@@ -13,18 +11,34 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     steps:
       - uses: actions/checkout@v2
 
-      - name: Run a one-line script
-        run: echo Hello, world!
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.8'
 
-      - name: Run a multi-line script
+      - name: Run unit tests
         run: |
-          echo Add other actions to build,
-          echo test, and deploy your project.
+          pip install -r requirements.tests.txt
+          pytest tests/ --junitxml=test-report.xml -v
+
+      - name: Multi-arch docker image build prerequired
+        run: sudo docker run --privileged linuxkit/binfmt:v0.7
+
+      - name: Build and deploy on architecture
+        env:
+          DOCKER_USERNAME: ${{ secrets.SIXSQ_DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.SIXSQ_DOCKER_PASSWORD }}
+        run: ./container-release.sh
+
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v1
+        if: always()
+        with:
+          files: test-report.xml
 
   notify:
     if: always()

.github/workflows/release.yml

@@ -0,0 +1,41 @@
+# Automatic release for every new tag
+name: "Release"
+
+on:
+  push:
+    tags:
+      - "*.*.*"
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Multi-arch docker image build prerequired
+        run: sudo docker run --privileged linuxkit/binfmt:v0.7
+
+      - name: Build and deploy on architecture
+        env:
+          DOCKER_USERNAME: ${{ secrets.SIXSQ_DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.SIXSQ_DOCKER_PASSWORD }}
+          DOCKER_ORG: nuvla
+        run: ./container-release.sh
+
+
+  notify:
+    if: always()
+    name: Post Workflow Status To Slack
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Slack Workflow Notification
+        uses: Gamesight/slack-workflow-status@master
+        with:
+          # Required Input
+          repo_token: ${{secrets.GITHUB_TOKEN}}
+          slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}}
+          # Optional Input
+          icon_emoji: ':rocket:'

container-release.sh

@@ -0,0 +1,88 @@
+#!/bin/bash -xe
+
+###############################
+#  CHANGE THIS ON EVERY REPO  #
+DOCKER_IMAGE=$(basename `git rev-parse --show-toplevel`)
+###############################
+
+# default env vars in GH actions
+GIT_BRANCH=$(echo ${GITHUB_REF} | awk -F'/' '{print $(NF)}' | sed -e 's/[^a-z0-9\._-]/-/g')
+
+# non-tagged builds are not releases, so they always go on nuvladev
+DOCKER_ORG=${DOCKER_ORG:-nuvladev}
+
+MANIFEST=${DOCKER_ORG}/${DOCKER_IMAGE}:${GIT_BRANCH}
+
+platforms=(amd64 arm64)
+
+
+#
+# remove any previous builds
+#
+
+rm -Rf target/*.tar
+mkdir -p target
+
+#
+# generate image for each platform
+#
+
+for platform in "${platforms[@]}"; do
+    GIT_BUILD_TIME=$(date --utc +%FT%T.%3NZ)
+    docker run --rm --privileged -v ${PWD}:/tmp/work --entrypoint buildctl-daemonless.sh moby/buildkit:master \
+           build \
+           --frontend dockerfile.v0 \
+           --opt platform=linux/${platform} \
+           --opt filename=./Dockerfile \
+           --opt build-arg:GIT_BRANCH=${GIT_BRANCH} \
+           --opt build-arg:GIT_BUILD_TIME=${GIT_BUILD_TIME} \
+           --opt build-arg:GIT_COMMIT_ID=${GITHUB_SHA} \
+           --opt build-arg:GITHUB_RUN_NUMBER=${GITHUB_RUN_NUMBER} \
+           --opt build-arg:GITHUB_RUN_ID=${GITHUB_RUN_ID} \
+           --opt build-arg:PROJECT_URL=${GIHUB_SERVER_URL}/${GITHUB_REPOSITORY} \
+           --output type=docker,name=${MANIFEST}-${platform},dest=/tmp/work/target/${DOCKER_IMAGE}-${platform}.docker.tar \
+           --local context=/tmp/work \
+           --local dockerfile=/tmp/work \
+           --progress plain
+
+done
+
+#
+# load all generated images
+#
+
+for platform in "${platforms[@]}"; do
+    docker load --input ./target/${DOCKER_IMAGE}-${platform}.docker.tar
+done
+
+
+manifest_args=(${MANIFEST})
+
+#
+# login to docker hub
+#
+
+unset HISTFILE
+echo ${DOCKER_PASSWORD} | docker login -u ${DOCKER_USERNAME} --password-stdin
+
+#
+# push all generated images
+#
+
+for platform in "${platforms[@]}"; do
+    docker push ${MANIFEST}-${platform}
+    manifest_args+=("${MANIFEST}-${platform}")
+done
+
+#
+# create manifest, update, and push
+#
+
+export DOCKER_CLI_EXPERIMENTAL=enabled
+docker manifest create "${manifest_args[@]}"
+
+for platform in "${platforms[@]}"; do
+    docker manifest annotate ${MANIFEST} ${MANIFEST}-${platform} --arch ${platform}
+done
+
+docker manifest push --purge ${MANIFEST}

requirements.tests.txt

@@ -0,0 +1,2 @@
+-r requirements.txt
+pytest==7.0.1

requirements.txt

@@ -2,3 +2,5 @@ kafka-python==2.0.2
 requests==2.25.0
 urllib3==1.26.2
 Jinja2==2.11.2
+# To fix https://github.com/aws/aws-sam-cli/issues/3661
+markupsafe==2.0.1

src/notify-email.py

@@ -1,14 +1,18 @@
 #!/usr/bin/env python3
 
-from notify_deps import *
-
-import smtplib
+import multiprocessing
+import os
 import requests
+import smtplib
+import time
 from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 from jinja2 import Template
 from datetime import datetime
 
+from notify_deps import get_logger, timestamp_convert, main
+from notify_deps import NUVLA_ENDPOINT
+
 
 log_local = get_logger('email')
 
@@ -33,7 +37,7 @@ def get_nuvla_config():
     headers = {'nuvla-authn-info': nuvla_api_authn_header}
     resp = requests.get(config_url, headers=headers)
     if resp.status_code != 200:
-        raise Exception(f'Failed to get response from server: status {resp.status_code}')
+        raise EnvironmentError(f'Failed to get response from server: status {resp.status_code}')
     return resp.json()
 
 
@@ -47,7 +51,7 @@ def set_smpt_params():
             try:
                 SMTP_PORT = int(os.environ['SMTP_PORT'])
             except ValueError:
-                raise Exception(f"Incorrect value for SMTP_PORT number: {os.environ['SMTP_PORT']}")
+                raise ValueError(f"Incorrect value for SMTP_PORT number: {os.environ['SMTP_PORT']}")
             SMTP_SSL = os.environ['SMTP_SSL'].lower() in ['true', 'True']
         else:
             nuvla_config = get_nuvla_config()
@@ -59,7 +63,7 @@ def set_smpt_params():
     except Exception as ex:
         msg = f'Provide full SMTP config either via env vars or in configuration/nuvla: {ex}'
         log_local.error(msg)
-        raise Exception(msg)
+        raise ValueError(msg)
 
 
 KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_EMAIL_S'
@@ -81,7 +85,6 @@ def get_smtp_server(debug_level=0) -> smtplib.SMTP:
 
 
 def html_content(values: dict):
-    # subs_config_id = values.get('SUBS_ID')
     subs_config_link = f'<a href="{NUVLA_ENDPOINT}/ui/notifications">Notification configuration</a>'
 
     r_uri = values.get('RESOURCE_URI')
@@ -135,12 +138,19 @@ def send(server: smtplib.SMTP, recipients, subject, html, attempts=SEND_EMAIL_AT
     raise SendFailedMaxAttempts(f'Failed sending email after {attempts} attempts.')
 
 
+def get_recipients(v: dict):
+    return list(filter(lambda x: x != '', v.get('DESTINATION', '').split(' ')))
+
+
 def worker(workq: multiprocessing.Queue):
     smtp_server = get_smtp_server()
     while True:
         msg = workq.get()
         if msg:
-            recipients = msg.value['DESTINATION'].split(',')
+            recipients = get_recipients(msg.value)
+            if len(recipients) == 0:
+                log_local.warning(f'No recipients provided in: {msg.value}')
+                continue
             r_id = msg.value.get('RESOURCE_ID')
             r_name = msg.value.get('NAME')
             subject = msg.value.get('SUBS_NAME') or f'{r_name or r_id} alert'

src/notify-slack.py

@@ -1,9 +1,14 @@
 #!/usr/bin/env python3
 
+import json
+import datetime
+import multiprocessing
 import requests
+import os
 import re
 
-from notify_deps import *
+from notify_deps import get_logger, timestamp_convert, main
+from notify_deps import NUVLA_ENDPOINT
 
 KAFKA_TOPIC = os.environ.get('KAFKA_TOPIC') or 'NOTIFICATIONS_SLACK_S'
 KAFKA_GROUP_ID = 'nuvla-notification-slack'
@@ -18,7 +23,6 @@
 
 
 def message_content(values: dict):
-    # subs_config_id = values.get('SUBS_ID')
     subs_name = lt.sub('<', gt.sub('>', values.get('SUBS_NAME', '')))
     subs_config_txt = f'<{NUVLA_ENDPOINT}/ui/notifications|{subs_name}>'
 

tests/notify_deps.py

@@ -0,0 +1 @@
+../src/notify_deps.py

tests/test_notify_email.py

@@ -0,0 +1,14 @@
+import unittest
+
+from notify_email import get_recipients
+
+
+class NotifyEmail(unittest.TestCase):
+
+    def test_get_recipients(self):
+        assert 0 == len(get_recipients({}))
+        e1 = 'a@b.c'
+        e2 = 'a@b.c'
+        assert [e1] == get_recipients({'DESTINATION': e1})
+        assert [e1, e2] == get_recipients({'DESTINATION': f'{e1} {e2}'})
+        assert [e1, e2] == get_recipients({'DESTINATION': f' {e1}   {e2}   '})