Merge pull request #133 from vincentclaes/updgrade-cdk

upgrade cdk

.devcontainer/devcontainer.Dockerfile

@@ -4,7 +4,7 @@
 #-------------------------------------------------------------------------------------------------------------
 
 # Update the VARIANT arg in devcontainer.json to pick a Python version: 3, 3.8, 3.7, 3.6
-ARG VARIANT=3.6
+ARG VARIANT=3.8
 FROM python:${VARIANT}
 
 # If you would prefer to have multiple Python versions in your container,

.github/workflows/pr.yml

@@ -1,22 +1,33 @@
-name: Build-Test-RunExamples
-on: [pull_request]
-
-env:
-  IMAGE_TAG: local
+name: build-python
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
-  Build-Test-RunExamples:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: ["3.8"]
+    env:
+      OS: ${{ matrix.os }}
+      PYTHON: ${{ matrix.python-version }}
 
     steps:
-    - name: Checking out
-      uses: actions/checkout@v2
+    - uses: actions/checkout@v3
+
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
       with:
-        ref: ${{ github.ref }}
+        python-version: ${{ matrix.python-version }}
 
-    - name: Build Container
-      run: |
-         cat .devcontainer/devcontainer.Dockerfile | docker build -t $IMAGE_TAG -
+    - name: Setup project
+      run: make setup
 
-    - name: Run Tests and Examples
-      run: docker run -v $PWD:/workspace/vscode -w /workspace/vscode $IMAGE_TAG make gh-actions
+    - name: Run test suite
+      run: make test

Makefile

@@ -1,17 +1,18 @@
-install:
+setup:
+	pip3 install pip --upgrade
 	pip3 install poetry
 	poetry config virtualenvs.create true
 	poetry install
 
 
-install-dev:
+setup-dev:
 	make install
 	# install pre commit hooks to check the code
 	# before committing.
 	poetry run pre-commit install
 
-tests:
-	poetry run pytest
+test:
+	AWS_DEFAULT_REGION=eu-west-1 poetry run pytest --disable-warnings
 
 run-examples:
 	cd "${CURDIR}/examples/data_pipeline_simple" && poetry run cdk synth --app "python datajob_stack.py"

datajob/datajob_context.py

@@ -4,7 +4,6 @@
 from aws_cdk import aws_s3
 from aws_cdk import aws_s3_deployment
 from aws_cdk import core
-from aws_empty_bucket.empty_s3_bucket import EmptyS3Bucket
 
 from datajob import logger
 
@@ -76,12 +75,13 @@ def _create_data_bucket(self) -> tuple:
         data_bucket_name = self._get_unique_bucket_name()
         # todo - can we validate the bucket name?
         logger.debug(f"creating deployment bucket {data_bucket_name}")
-        data_bucket = EmptyS3Bucket(
+        data_bucket = aws_s3.Bucket(
             self,
             data_bucket_name,
             bucket_name=data_bucket_name,
             # todo - we might want to refine the removal policy.
             #  Might not be wise to destroy it after we destroy the stack.
+            auto_delete_objects=True,
             removal_policy=core.RemovalPolicy.DESTROY,
         )
         return data_bucket, data_bucket_name
@@ -100,10 +100,11 @@ def _create_deployment_bucket(self) -> tuple:
         deployment_bucket_name = f"{unique_bucket_name}-deployment-bucket"
         # todo - can we validate the bucket name?
         logger.debug(f"creating deployment bucket {deployment_bucket_name}")
-        deployment_bucket = EmptyS3Bucket(
+        deployment_bucket = aws_s3.Bucket(
             self,
             deployment_bucket_name,
             bucket_name=deployment_bucket_name,
+            auto_delete_objects=True,
             removal_policy=core.RemovalPolicy.DESTROY,
         )
         return deployment_bucket, deployment_bucket_name

examples/data_pipeline_simple/create_100_simple.py

@@ -0,0 +1,25 @@
+from aws_cdk import core
+
+from datajob.datajob_stack import DataJobStack
+from datajob.glue.glue_job import GlueJob
+from datajob.stepfunctions.stepfunctions_workflow import StepfunctionsWorkflow
+
+app = core.App()
+
+# The datajob_stack is the instance that will result in a cloudformation stack.
+# We inject the datajob_stack object through all the resources that we want to add.
+with DataJobStack(scope=app, id="data-pipeline-simple") as datajob_stack:
+
+    # We define 2 glue jobs with the relative path to the source code.
+    task = GlueJob(
+        datajob_stack=datajob_stack, name="task1", job_path="glue_jobs/task1.py"
+    )
+
+    for i in range(100, 250):
+        # We instantiate a step functions workflow and orchestrate the glue jobs.
+        with StepfunctionsWorkflow(
+            datajob_stack=datajob_stack, name=f"workflow-{i}"
+        ) as sfn:
+            task >> ...
+
+app.synth()