Prepare for release of MCP production deployment (#269)

* Fix id update typing for sentinel_ac_logger.
* Use github vars for mcp-dev environment
* Fix flake8 errors
* Bump actions versions to remove node12 deprecation warnings
* Fix deprecation warning for applyAspect

  The API @aws-cdk/core.ConstructNode.applyAspect is deprecated:
  Use "Aspects.of(construct).add(aspect)" instead. This API will
  be removed in the next major release

* Use OIDC role for mcp-dev deployment
* Upgrade Node and CDK

  - upgrade node to 16.x
  - upgrade to latest version of CDK v1
  - fix dependency list error (appending should be extending)

* Remove buckets on destroy
* Bump Aurora Serverless to v11.18 (10.7 obsolete)
* Restore subscription to Landsat SNS
* Add mcp-production-deployment (#267)
* Update Landsat SNS topic
* Add mcp-production-deployment
* Add HLS_LANDSAT_SNS_TOPIC_ENABLED env var (default: "true") to allow disabling of trigger

---------

Co-authored-by: Sean Harkins <sharkinsgis@gmail.com>

.github/workflows/mcp_dev_deployment.yml

@@ -0,0 +1,82 @@
+name: mcp dev deployment
+
+on:
+  push:
+    branches: [dev]
+  workflow_dispatch:
+
+env:
+  AWS_DEFAULT_REGION: us-west-2
+
+# See https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services#updating-your-github-actions-workflow
+permissions:
+  id-token: write # required for requesting the JWT
+  contents: read  # required for actions/checkout
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python: [3.8]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Tox
+        run: pip install tox
+      - name: Run Tox test environment
+        # Run tox using the version of Python in `PATH`
+        run: tox -e py
+  mcp-dev-deployment:
+    strategy:
+      matrix:
+        python: [3.8]
+    runs-on: ubuntu-20.04
+    needs: [unit-tests]
+    environment:
+      name: mcp-dev
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install tox
+        run: pip install tox
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME_ARN }}
+          role-session-name: ${{ github.actor }}
+          aws-region: us-west-2
+      - name: Convert secrets to environment variables
+        env:
+          SECRETS_JSON: ${{ toJson(secrets) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$SECRETS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Convert vars to environment variables
+        env:
+          VARS_JSON: ${{ toJson(vars) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$VARS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Deploy with tox
+        run: tox -v -e dev -r -- deploy --require-approval never
+      - name: Run DB setup
+        run: |
+          setupdb=$(aws cloudformation describe-stacks \
+            --stack-name $HLS_STACKNAME \
+            --query "Stacks[0].Outputs[?OutputKey=='setupdbexport'].OutputValue" \
+            --output=text
+          )
+          aws lambda invoke --function-name=$setupdb response.json

.github/workflows/mcp_production_deployment.yml

@@ -0,0 +1,81 @@
+name: MCP production deployment
+
+on:
+  release:
+    types: [ published ]
+
+env:
+  AWS_DEFAULT_REGION: us-west-2
+
+# See https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/configuring-openid-connect-in-amazon-web-services#updating-your-github-actions-workflow
+permissions:
+  id-token: write # required for requesting the JWT
+  contents: read  # required for actions/checkout
+
+jobs:
+  unit-tests:
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python: [3.8]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install Tox
+        run: pip install tox
+      - name: Run Tox test environment
+        # Run tox using the version of Python in `PATH`
+        run: tox -e py
+  mcp-production-deployment:
+    strategy:
+      matrix:
+        python: [3.8]
+    runs-on: ubuntu-20.04
+    needs: [unit-tests]
+    environment:
+      name: mcp-production
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: Install tox
+        run: pip install tox
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v3
+        with:
+          role-to-assume: ${{ vars.AWS_ROLE_TO_ASSUME_ARN }}
+          role-session-name: ${{ github.actor }}
+          aws-region: us-west-2
+      - name: Convert secrets to environment variables
+        env:
+          SECRETS_JSON: ${{ toJson(secrets) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$SECRETS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Convert vars to environment variables
+        env:
+          VARS_JSON: ${{ toJson(vars) }}
+        run: |
+          while read -rd $'' line; do
+            echo "$line" >> $GITHUB_ENV
+          done < <(
+            jq -r <<<"$VARS_JSON" 'to_entries|map("\(.key)=\(.value)\u0000")[]'
+          )
+      - name: Deploy with tox
+        run: tox -v -e dev -r -- deploy --require-approval never
+      - name: Run DB setup
+        run: |
+          setupdb=$(aws cloudformation describe-stacks \
+            --stack-name $HLS_STACKNAME \
+            --query "Stacks[0].Outputs[?OutputKey=='setupdbexport'].OutputValue" \
+            --output=text
+          )
+          aws lambda invoke --function-name=$setupdb response.json

.github/workflows/tox_tests.yml

@@ -12,9 +12,9 @@ jobs:
         python: [3.8]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Setup Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
       - name: Install Tox and any other packages

README.md

@@ -1,78 +1,106 @@
-# hls-orchestration
+# HLS Orchestration
 
 AWS Stack for processing HLS data.
 
 ![Alt text](/docs/HLS_architecture.png)
 
-For more detailed data flow diagrams and architecture see [architecture](/docs/architecture.md).
+For more detailed data flow diagrams and architecture see
+[architecture](/docs/architecture.md).
 
-For more details about all of the HLS project's components see [hls-project](https://github.com/nasa-impact/hls-project).
+For more details about all of the HLS project's components see
+[hls-project](https://github.com/nasa-impact/hls-project).
 
+## Requirements
 
-### Requirements
-Python>=3.7 \
-tox \
-aws-cli \
-jq \
-An IAM role with sufficient permissions for creating, destroying and modifying the relevant stack resources.
+- Python>=3.7
+- tox
+- aws-cli
+- jq
+- An IAM role with sufficient permissions for creating, destroying, and
+  modifying the relevant stack resources.
 
-### Environment Settings
-Environment variables are set in `environment.sh`. Copy `environment.sh.sample` to `environment.sh` and update the settings prior to running any commands.  The following variables can be overridden from the calling shell's environment
-```
-$ export HLS_STACKNAME=<Name of your stack>
-$ export HLS_LAADS_TOKEN=<Token used for accessing the Laads Data>
-$ export HLS_SENTINEL_OUTPUT_BUCKET_ROLE_ARN=<GCC Role for accessing output bucket>
+## Environment Settings
+
+Environment variables are set in `environment.sh`.  Copy `environment.sh.sample`
+to `environment.sh` and update the settings prior to running any commands.  The
+following variables can be overridden from the calling shell's environment:
+
+```plain
+export HLS_STACKNAME=<Name of your stack>
+export HLS_LAADS_TOKEN=<Token used for accessing the Laads Data>
+export HLS_SENTINEL_OUTPUT_BUCKET_ROLE_ARN=<GCC Role for accessing output bucket>
 ```
 
-### Synth
+## Synth
+
 Display generated cloud formation template that will be used to deploy.
+
+```plain
+source environment.sh && tox -e dev -r -- synth
 ```
-$ source ./environment.sh && tox -e dev -r -- synth
-```
 
-### Diff
+## Diff
+
 Display a diff of the current deployment and any changes created.
-```
-$ source ./environment.sh && tox -e dev -r -- diff
-```
 
-### Deploy
-Deploy current version of stack.
-```
-$ source ./environment.sh && tox -e dev -r -- deploy
+```plain
+source environment.sh && tox -e dev -r -- diff
 ```
 
-The repository is configured to create automatic deployments to the `hls-development` stack when PRs are merged into the `dev` branch.  This deployment uses [Github Actions Environments](https://docs.github.com/en/actions/reference/environments) to manage the environment configuration rather than the `environment.sh`.
+## Deploy
 
-Deployments to GCC have restrictions over creating VPCs and the types of AMIs which can be utilized. To deploy to GCC your shell will require the following environment settings.
+Deploy current version of stack:
+
+```plain
+source environment.sh && tox -e dev -r -- deploy
 ```
+
+The repository is configured to create automatic deployments to the
+`hls-development` stack when PRs are merged into the `dev` branch.  This
+deployment uses
+[Github Actions Environments](https://docs.github.com/en/actions/reference/environments)
+to manage the environment configuration rather than the `environment.sh`.
+
+Deployments to GCC have restrictions over creating VPCs and the types of AMIs
+which can be utilized.  To deploy to GCC your shell will require the following
+environment settings:
+
+```plain
 export GCC=true
 export AWS_DEFAULT_REGION=us-west-2
 export HLS_GCC_ACCOUNT=<The GCC account id>
 export HLS_GCC_VPCID=<The vpc id provided by GCC administrators>
 export HLS_GCC_BOUNDARY_ARN=<The boundary policy arn>
 ```
 
+## Setup Logging Database
 
-### Setup Logging Database
-After `deploy` is run and the stack is created run
-```
-$ source ./environment.sh && ./scripts/setupdb.sh
+After `deploy` is run and the stack is created run:
+
+```plain
+source environment.sh && scripts/setupdb.sh
 ```
+
 To bootstrap the logging database.
 
-### Development
-For active stack development run
-```
-$ source ./environment.sh && tox -e dev -r -- version
-```
-This creates a local virtualenv in the directory `devenv`.  To use it for development
+## Development
+
+For active stack development run:
+
+```plain
+source environment.sh && tox -e dev -r -- version
 ```
-$ source devenv/bin/activate
+
+This creates a local virtualenv in the directory `devenv`.  To use it for development:
+
+```plain
+source devenv/bin/activate
 ```
 
-### Tests
+## Tests
+
 To run unit test for all included Lambda functions
-```
+
+```plain
 tox -r
 ```

cdk.context.json

@@ -0,0 +1,5 @@
+{
+  "acknowledged-issue-numbers": [
+    19836
+  ]
+}

setup.py

@@ -1,12 +1,10 @@
 """Setup for hls-orchestration"""
-import os
-import subprocess
-import sys
+
 
 from setuptools import find_packages, setup
 
 # Runtime requirements.
-aws_cdk_version = "1.65.0"
+aws_cdk_version = "1.203.0"
 aws_cdk_reqs = [
     "core",
     "aws-s3",
@@ -31,10 +29,9 @@
 
 inst_reqs = [
     "boto3",
+    *[f"aws_cdk.{x}=={aws_cdk_version}" for x in aws_cdk_reqs],
 ]
 
-inst_reqs.append([f"aws_cdk.{x}=={aws_cdk_version}" for x in aws_cdk_reqs])
-
 extra_reqs = {
     "test": [
         "pytest",
@@ -49,6 +46,7 @@
         "flake8",
         "nodeenv",
         "isort",
+        "mypy",
         "pre-commit",
         "pre-commit-hooks",
     ],

stack/hlsconstructs/landsat_incomplete_step_function.py

@@ -83,4 +83,4 @@ def __init__(
             )
         )
 
-        self.addLambdasToRole(locals())
+        self.add_lambdas_to_role(locals())

stack/hlsconstructs/landsat_mgrs_partials_step_function.py

@@ -136,4 +136,4 @@ def __init__(
             role_arn=self.steps_role.role_arn,
         )
 
-        self.addLambdasToRole(locals())
+        self.add_lambdas_to_role(locals())

stack/hlsconstructs/landsat_mgrs_step_function.py

@@ -134,4 +134,4 @@ def __init__(
             role_arn=self.steps_role.role_arn,
         )
 
-        self.addLambdasToRole(locals())
+        self.add_lambdas_to_role(locals())

stack/hlsconstructs/landsat_step_function.py

@@ -254,4 +254,4 @@ def __init__(
             )
         )
 
-        self.addLambdasToRole(locals())
+        self.add_lambdas_to_role(locals())

stack/hlsconstructs/rds.py

@@ -59,7 +59,7 @@ def __init__(
             "RdsCluster",
             engine="aurora-postgresql",
             engine_mode="serverless",
-            engine_version="10.7",
+            engine_version="11.18",
             database_name="hls",
             db_subnet_group_name=self.subnet_group.ref,
             enable_http_endpoint=True,