From 994dfee64bd76d914e0c162ee1e4f1b527437c82 Mon Sep 17 00:00:00 2001 From: Kuan Fan <31664961+kuanfandevops@users.noreply.github.com> Date: Tue, 25 Jul 2023 14:39:12 -0700 Subject: [PATCH] Add pull request build for Jan release (#2463) --- .github/readme.md | 4 +- .github/workflows/branch-deploy-template.yaml | 28 ++- ...uild-template.yaml => build-template.yaml} | 80 +++++--- .github/workflows/dev-jan-release.yaml | 7 +- .github/workflows/pr-dev-cicd.yaml | 54 +++++ .../workflows/pr-dev-database-template.yaml | 69 +++++++ .github/workflows/pr-dev-deploy-template.yaml | 191 ++++++++++++++++++ .github/workflows/pr-teardown.yaml | 41 ++++ README.md | 1 + charts/tfrs-spilo/values-dev.yaml | 2 +- openshift-v4/templates/celery/Dockerfile | 14 ++ .../templates/celery/celery-bc-docker.yaml | 89 ++++++++ .../templates/scan-handler/Dockerfile | 14 ++ .../scan-handler/scan-handler-bc-docker.yaml | 89 ++++++++ 14 files changed, 643 insertions(+), 40 deletions(-) rename .github/workflows/{branch-build-template.yaml => build-template.yaml} (60%) create mode 100644 .github/workflows/pr-dev-cicd.yaml create mode 100644 .github/workflows/pr-dev-database-template.yaml create mode 100644 .github/workflows/pr-dev-deploy-template.yaml create mode 100644 .github/workflows/pr-teardown.yaml create mode 100644 openshift-v4/templates/celery/Dockerfile create mode 100644 openshift-v4/templates/celery/celery-bc-docker.yaml create mode 100644 openshift-v4/templates/scan-handler/Dockerfile create mode 100644 openshift-v4/templates/scan-handler/scan-handler-bc-docker.yaml diff --git a/.github/readme.md b/.github/readme.md index 6133d5db2..55b8fe416 100644 --- a/.github/readme.md +++ b/.github/readme.md @@ -40,7 +40,5 @@ * tfrs-release.yaml (TFRS release-2.10.0): the pipeline builds the release and deploys on Test and Prod, it needs to be manually triggered * create-release.yaml (Create Release after merging to master): tag and create the release after merging release branch to master. The description of the tracking pull request becomes release notes -## Other Pipelines -* cleanup-cron-workflow-runs.yaml (Scheduled cleanup old workflow runs): a cron job to cleanup the old workflows -* cleanup-workflow-runs.yaml (Cleanup old workflow runs): manually cleanup teh workflow runs + diff --git a/.github/workflows/branch-deploy-template.yaml b/.github/workflows/branch-deploy-template.yaml index 74648e14a..8d2a2e8cc 100644 --- a/.github/workflows/branch-deploy-template.yaml +++ b/.github/workflows/branch-deploy-template.yaml @@ -161,4 +161,30 @@ jobs: --set databaseServiceHostName=${{ inputs.database-service-host-name }} \ --set rabbitmqVHost=${{ inputs.rabbitmq-vhost }} \ -n ${{ secrets.namespace }} -f ./values${{ inputs.suffix }}.yaml tfrs-scan-handler${{ inputs.suffix }} . - fi \ No newline at end of file + fi + + - name: Deploy tfrs-scan-coordinator + shell: bash {0} + run: | + oc tag ${{ secrets.tools-namespace }}/tfrs-scan-hacoordinatorndler:build-${{ inputs.branch-name }} ${{ secrets.namespace }}/tfrs-scan-coordinator:${{ inputs.env-name }}-${{ inputs.branch-name }} + cd charts/tfrs-apps/charts/tfrs-scan-coordinator + helm status -n ${{ secrets.namespace }} tfrs-scan-coordinator${{ inputs.suffix }} + if [ $? -eq 0 ]; then + echo "tfrs-scan-coordinator${{ inputs.suffix }} release exists already" + helm upgrade \ + --set scanCoordinatorImageTagName=${{ inputs.env-name }}-${{ inputs.branch-name }} \ + --set suffix=${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=${{ inputs.env-Name }} \ + --set rabbitmqVHost=${{ inputs.rabbitmq-vhost }} \ + -n ${{ secrets.namespace }} -f ./values${{ inputs.suffix }}.yaml tfrs-scan-coordinator${{ inputs.suffix }} . + else + echo "tfrs-scan-coordinator${{ inputs.suffix }} release does not exist" + helm install \ + --set scanCoordinatorImageTagName=${{ inputs.env-name }}-${{ inputs.branch-name }} \ + --set suffix=${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=${{ inputs.env-Name }} \ + --set rabbitmqVHost=${{ inputs.rabbitmq-vhost }} \ + -n ${{ secrets.namespace }} -f ./values${{ inputs.suffix }}.yaml tfrs-scan-coordinator${{ inputs.suffix }} . + fi \ No newline at end of file diff --git a/.github/workflows/branch-build-template.yaml b/.github/workflows/build-template.yaml similarity index 60% rename from .github/workflows/branch-build-template.yaml rename to .github/workflows/build-template.yaml index d3370f3ba..07a8e0b4a 100644 --- a/.github/workflows/branch-build-template.yaml +++ b/.github/workflows/build-template.yaml @@ -1,10 +1,18 @@ +# This template supports both pr build and branch build name: Branch Build Template on: workflow_call: inputs: - branch-name: # sample value: main-release-jan-2024 + # when build branch, the sample value is -main-release-jan-2024 + # when build pull request, the sample value is -jan-2024 + suffix: + required: true + type: string + # when build branch, the sample value is main-release-jan-2024 + # when build pull request, the sample value is refs/pull/2024/head + checkout-ref: required: true type: string secrets: @@ -31,7 +39,7 @@ jobs: - name: Check out repository uses: actions/checkout@v3.5.3 with: - ref: ${{ inputs.branch-name }} + ref: ${{ inputs.checkout-ref }} - name: Log in to Openshift uses: redhat-actions/oc-login@v1.2 @@ -45,13 +53,13 @@ jobs: run: | cd openshift-v4/templates/backend oc process -f ./backend-bc.yaml NAME=tfrs \ - SUFFIX=-build-${{ inputs.branch-name}} \ - VERSION=build-${{ inputs.branch-name }} \ + SUFFIX=-build${{ inputs.suffix}} \ + VERSION=build${{ inputs.suffix }} \ GIT_URL=${{ env.GIT_URL }} \ - GIT_REF=${{ inputs.branch-name }} \ + GIT_REF=${{ inputs.checkout-ref }} \ | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }} - oc cancel-build bc/tfrs-backend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true - oc start-build --wait=true tfrs-backend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} + oc cancel-build bc/tfrs-backend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true + oc start-build --wait=true tfrs-backend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} build-frontend: @@ -64,7 +72,7 @@ jobs: - name: Check out repository uses: actions/checkout@v3.5.3 with: - ref: ${{ inputs.branch-name }} + ref: ${{ inputs.checkout-ref }} - name: Log in to Openshift uses: redhat-actions/oc-login@v1.2 @@ -78,17 +86,18 @@ jobs: run: | cd openshift-v4/templates/frontend oc process -f ./frontend-bc-docker.yaml NAME=tfrs \ - SUFFIX=-build-${{ inputs.branch-name}} \ - VERSION=build-${{ inputs.branch-name }} \ + SUFFIX=-build${{ inputs.suffix}} \ + VERSION=build${{ inputs.suffix }} \ GIT_URL=${{ env.GIT_URL }} \ - GIT_REF=${{ inputs.branch-name }} \ + GIT_REF=${{ inputs.checkout-ref }} \ | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }} - oc cancel-build bc/tfrs-frontend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true - oc start-build --wait=true tfrs-frontend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} + oc cancel-build bc/tfrs-frontend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true + oc start-build --wait=true tfrs-frontend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} build-celery: name: Build TFRS Celery on Openshift + needs: [build-frontend, build-backend] runs-on: ubuntu-latest timeout-minutes: 60 @@ -97,7 +106,7 @@ jobs: - name: Check out repository uses: actions/checkout@v3.5.3 with: - ref: ${{ inputs.branch-name }} + ref: ${{ inputs.checkout-ref }} - name: Log in to Openshift uses: redhat-actions/oc-login@v1.2 @@ -110,18 +119,21 @@ jobs: - name: Build TFRS Celery run: | cd openshift-v4/templates/celery - oc process -f ./celery-bc.yaml NAME=tfrs \ - SUFFIX=-build-${{ inputs.branch-name}} \ - VERSION=build-${{ inputs.branch-name }} \ + pwd + ls -l + oc process -f ./celery-bc-docker.yaml NAME=tfrs \ + SUFFIX=-build${{ inputs.suffix}} \ + VERSION=build${{ inputs.suffix }} \ GIT_URL=${{ env.GIT_URL }} \ - RELEASE_BRANCH=${{ inputs.branch-name}} \ + GIT_REF=${{ inputs.checkout-ref }} \ | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }} - oc cancel-build bc/tfrs-celery-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true - oc start-build --wait=true tfrs-celery-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} + oc cancel-build bc/tfrs-celery-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true + oc start-build --wait=true tfrs-celery-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} build-scan-coordinator: name: Build TFRS Scan Coordinator on Openshift + needs: [build-frontend, build-backend] runs-on: ubuntu-latest timeout-minutes: 60 @@ -130,7 +142,7 @@ jobs: - name: Check out repository uses: actions/checkout@v3.5.3 with: - ref: ${{ inputs.branch-name }} + ref: ${{ inputs.checkout-ref }} - name: Log in to Openshift uses: redhat-actions/oc-login@v1.2 @@ -144,17 +156,18 @@ jobs: run: | cd openshift-v4/templates/scan-coordinator oc process -f ./scan-coordinator-bc.yaml NAME=tfrs \ - SUFFIX=-build-${{ inputs.branch-name}} \ - VERSION=build-${{ inputs.branch-name }} \ + SUFFIX=-build${{ inputs.suffix}} \ + VERSION=build${{ inputs.suffix }} \ GIT_URL=${{ env.GIT_URL }} \ - GIT_REF=${{ inputs.branch-name }} \ + GIT_REF=${{ inputs.checkout-ref }} \ | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }} - oc cancel-build bc/tfrs-scan-coordinator-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true - oc start-build --wait=true tfrs-scan-coordinator-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} + oc cancel-build bc/tfrs-scan-coordinator-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true + oc start-build --wait=true tfrs-scan-coordinator-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} build-scan-handler: name: Build TFRS Scan Handler on Openshift + needs: [build-scan-coordinator, build-celery] runs-on: ubuntu-latest timeout-minutes: 60 @@ -163,7 +176,7 @@ jobs: - name: Check out repository uses: actions/checkout@v3.5.3 with: - ref: ${{ inputs.branch-name }} + ref: ${{ inputs.checkout-ref }} - name: Log in to Openshift uses: redhat-actions/oc-login@v1.2 @@ -176,10 +189,11 @@ jobs: - name: Build TFRS Scan-Handler run: | cd openshift-v4/templates/scan-handler - oc process -f ./scan-handler-bc.yaml NAME=tfrs \ - SUFFIX=-build-${{ inputs.branch-name}} \ - VERSION=build-${{ inputs.branch-name }} \ - RELEASE_BRANCH=${{ inputs.branch-name }} \ + oc process -f ./scan-handler-bc-docker.yaml NAME=tfrs \ + SUFFIX=-build${{ inputs.suffix}} \ + VERSION=build${{ inputs.suffix }} \ + GIT_URL=${{ env.GIT_URL }} \ + GIT_REF=${{ inputs.checkout-ref }} \ | oc apply --wait=true -f - -n ${{ secrets.tools-namespace }} - oc cancel-build bc/tfrs-scan-handler-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true - oc start-build --wait=true tfrs-scan-handler-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} \ No newline at end of file + oc cancel-build bc/tfrs-scan-handler-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true + oc start-build --wait=true tfrs-scan-handler-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} \ No newline at end of file diff --git a/.github/workflows/dev-jan-release.yaml b/.github/workflows/dev-jan-release.yaml index 3ed6dbd48..2d93a482e 100644 --- a/.github/workflows/dev-jan-release.yaml +++ b/.github/workflows/dev-jan-release.yaml @@ -65,12 +65,15 @@ jobs: VALIDATE_PYTHON_PYLINT: true LOG_LEVEL: WARN + # when build branch, the suffix sample is -main-release-jan-2024 + # the checkout-ref sample is main-release-jan-2024 build: name: Build needs: [unit-test, lint] - uses: ./.github/workflows/branch-build-template.yaml + uses: ./.github/workflows/build-template.yaml with: - branch-name: ${{ github.ref_name }} + suffix: -${{ github.ref_name }} + checkout-ref: ${{ github.ref_name }} secrets: tools-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools openshift-server: ${{ secrets.OPENSHIFT_SERVER }} diff --git a/.github/workflows/pr-dev-cicd.yaml b/.github/workflows/pr-dev-cicd.yaml new file mode 100644 index 000000000..ce6128835 --- /dev/null +++ b/.github/workflows/pr-dev-cicd.yaml @@ -0,0 +1,54 @@ +# Please refer to ./readme.md for how to build single pull request + +# Update this workflow name per pull request +name: TFRS Dev Jan PR CICD +on: + workflow_dispatch: + pull_request: + types: [opened, edited, synchronize, reopened] + branches: + - 'main-release-jan-2024' + +jobs: + + setup-database: + if: endsWith( github.event.pull_request.title, 'build-on-dev' ) + uses: ./.github/workflows/pr-dev-database-template.yaml + with: + pr-number: ${{ github.event.pull_request.number }} + dev-suffix: -jan-${{ github.event.pull_request.number }} + secrets: + dev-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev + tfrs-dev-username: ${{ secrets.TFRS_DEV_USERNAME }} + tfrs-dev-password: ${{ secrets.TFRS_DEV_PASSWORD }} + openshift-server: ${{ secrets.OPENSHIFT_SERVER }} + openshift-token: ${{ secrets.OPENSHIFT_TOKEN }} + + # when build pull reuqest, the suffix sample is -jan-1234 + # the checkout-ref is in the format of refs/pull/1234/head + build: + if: endsWith( github.event.pull_request.title, 'build-on-dev' ) + name: Build Pull Request + uses: ./.github/workflows/build-template.yaml + with: + suffix: -jan-${{ github.event.pull_request.number }} + checkout-ref: refs/pull/${{ github.event.pull_request.number }}/head + secrets: + tools-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools + openshift-server: ${{ secrets.OPENSHIFT_SERVER }} + openshift-token: ${{ secrets.OPENSHIFT_TOKEN }} + + deploy: + if: endsWith( github.event.pull_request.title, 'build-on-dev' ) + needs: [setup-database, build] + uses: ./.github/workflows/pr-dev-deploy-template.yaml + with: + suffix: -jan-${{ github.event.pull_request.number }} + checkout-ref: refs/pull/${{ github.event.pull_request.number }}/head + database-service-host-name: tfrs-spilo-jan-${{ github.event.pull_request.number }} + secrets: + tools-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools + namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev + openshift-server: ${{ secrets.OPENSHIFT_SERVER }} + openshift-token: ${{ secrets.OPENSHIFT_TOKEN }} + \ No newline at end of file diff --git a/.github/workflows/pr-dev-database-template.yaml b/.github/workflows/pr-dev-database-template.yaml new file mode 100644 index 000000000..458d8b62b --- /dev/null +++ b/.github/workflows/pr-dev-database-template.yaml @@ -0,0 +1,69 @@ +name: PR Dev Database Template + +on: + workflow_call: + inputs: + # pull request number + pr-number: + required: true + type: string + # the suffix will be appended to tfrs-spilo, same values: -1234, -jan-1242 + dev-suffix: + required: true + type: string + secrets: + dev-namespace: + required: true + tfrs-dev-username: + required: true + tfrs-dev-password: + required: true + openshift-server: + required: true + openshift-token: + required: true + +jobs: + + database: + + name: Start Database + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + + - name: Check out repository + uses: actions/checkout@v3 + with: + ref: refs/pull/${{ inputs.pr-number }}/head + + - name: Log in to Openshift + uses: redhat-actions/oc-login@v1.2 + with: + openshift_server_url: ${{ secrets.openshift-server }} + openshift_token: ${{ secrets.openshift-token }} + insecure_skip_tls_verify: true + namespace: ${{ secrets.dev-namespace }} + + - name: Setup Database + shell: bash {0} + run: | + cd charts/tfrs-spilo + helm dependency build + helm status -n ${{ secrets.dev-namespace }} tfrs-spilo${{ inputs.dev-suffix }} + if [ $? -eq 0 ]; then + echo "tfrs-spilo${{ inputs.dev-suffix }} exists already" + else + echo "Installing tfrs-spilo${{ inputs.dev-suffix }}" + helm install -n ${{ secrets.dev-namespace }} -f ./values-dev.yaml --wait tfrs-spilo${{ inputs.dev-suffix }} . + oc -n ${{ secrets.dev-namespace }} wait --for=condition=Ready pod/tfrs-spilo${{ inputs.dev-suffix }}-0 + oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "create user \"${{ secrets.tfrs-dev-username }}\" WITH PASSWORD '${{ secrets.tfrs-dev-password }}'" || true + oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "create database tfrs owner \"${{ secrets.tfrs-dev-username }}\" ENCODING 'UTF8' LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8'" || true + oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_filename='postgresql-%H.log'" || true + oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_connections='off'" || true + oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_disconnections='off'" || true + oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_checkpoints='off'" || true + oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "select pg_reload_conf()" || true + fi + diff --git a/.github/workflows/pr-dev-deploy-template.yaml b/.github/workflows/pr-dev-deploy-template.yaml new file mode 100644 index 000000000..fbedd4885 --- /dev/null +++ b/.github/workflows/pr-dev-deploy-template.yaml @@ -0,0 +1,191 @@ + + +name: PR Dev Deploy Template + +on: + workflow_call: + inputs: + # suffix is in format of -jan-1923 + suffix: + required: true + type: string + # when build pull request, the sample value is refs/pull/2023/head + checkout-ref: + required: true + type: string + # database-service-host-name, sample tfrs-spilo-dev-1988 + database-service-host-name: + required: true + type: string + secrets: + tools-namespace: + required: true + namespace: + required: true + openshift-server: + required: true + openshift-token: + required: true + +jobs: + + deploy: + + name: Deploy tfrs + runs-on: ubuntu-latest + timeout-minutes: 60 + + steps: + + - name: Check out repository + uses: actions/checkout@v3 + with: + ref: ${{ inputs.checkout-ref }} + + - name: Log in to Openshift + uses: redhat-actions/oc-login@v1.2 + with: + openshift_server_url: ${{ secrets.openshift-server }} + openshift_token: ${{ secrets.openshift-token }} + insecure_skip_tls_verify: true + namespace: ${{ secrets.tools-namespace }} + + - name: Create vhost on Rabbitmq Dev + shell: bash {0} + run: | + oc -n ${{ secrets.namespace }} exec tfrs-rabbitmq-0 -- rabbitmqctl add_vhost tfrs-dev${{ inputs.suffix }}-vhost + oc -n ${{ secrets.namespace }} exec tfrs-rabbitmq-0 -- rabbitmqctl set_permissions --vhost tfrs-dev${{ inputs.suffix }}-vhost tfrs ".*" ".*" ".*" + + - name: Deploy tfrs-frontend + shell: bash {0} + run: | + oc tag ${{ secrets.tools-namespace }}/tfrs-frontend:build${{ inputs.suffix }} ${{ secrets.namespace }}/tfrs-frontend:dev${{ inputs.suffix }} + cd charts/tfrs-apps/charts/tfrs-frontend + helm status -n ${{ secrets.namespace }} tfrs-frontend-dev${{ inputs.suffix }} + if [ $? -eq 0 ]; then + echo "tfrs-frontend-dev${{ inputs.suffix }} release exists already" + helm upgrade \ + --set frontendImageTagName=dev${{ inputs.suffix }} \ + --set suffix=-dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-frontend-dev${{ inputs.suffix }} . + else + echo "tfrs-frontend-dev${{ inputs.suffix }} release does not exist" + helm install \ + --set frontendImageTagName=dev${{ inputs.suffix }} \ + --set suffix=-dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-frontend-dev${{ inputs.suffix }} . + fi + + - name: Deploy tfrs-backend + shell: bash {0} + run: | + oc tag ${{ secrets.tools-namespace }}/tfrs-backend:build${{ inputs.suffix }} ${{ secrets.namespace }}/tfrs-backend:dev${{ inputs.suffix }} + cd charts/tfrs-apps/charts/tfrs-backend + helm status -n ${{ secrets.namespace }} tfrs-backend-dev${{ inputs.suffix }} + if [ $? -eq 0 ]; then + echo "tfrs-backend-dev${{ inputs.suffix }} release exists already" + helm upgrade \ + --set backendImageTagName=dev${{ inputs.suffix }} \ + --set suffix=-dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + --set databaseServiceHostName=${{ inputs.database-service-host-name }} \ + --set rabbitmqVHost=tfrs-dev${{ inputs.suffix }}-vhost \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-backend-dev${{ inputs.suffix }} . + else + echo "tfrs-backend-dev${{ inputs.suffix }} release does not exist" + helm install \ + --set backendImageTagName=dev${{ inputs.suffix }} \ + --set suffix=dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + --set databaseServiceHostName=${{ inputs.database-service-host-name }} \ + --set rabbitmqVHost=tfrs-dev${{ inputs.suffix }}-vhost \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-backend-dev${{ inputs.suffix }} . + fi + + - name: Deploy tfrs-celery + shell: bash {0} + run: | + oc tag ${{ secrets.tools-namespace }}/tfrs-celery:build${{ inputs.suffix }} ${{ secrets.namespace }}/tfrs-celery:dev${{ inputs.suffix }} + cd charts/tfrs-apps/charts/tfrs-celery + helm status -n ${{ secrets.namespace }} tfrs-celery-dev${{ inputs.suffix }} + if [ $? -eq 0 ]; then + echo "tfrs-celery-dev${{ inputs.suffix }} release exists already" + helm upgrade \ + --set celeryImageTagName=dev${{ inputs.suffix }} \ + --set suffix=-dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + --set databaseServiceHostName=${{ inputs.database-service-host-name }} \ + --set rabbitmqVHost=tfrs-dev${{ inputs.suffix }}-vhost \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-celery-dev${{ inputs.suffix }} . + else + echo "tfrs-celery-dev${{ inputs.suffix }} release does not exist" + helm install \ + --set celeryImageTagName=${{ inputs.env-name }}-${{ inputs.suffix }} \ + --set suffix=-dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + --set databaseServiceHostName=${{ inputs.database-service-host-name }} \ + --set rabbitmqVHost=tfrs-dev${{ inputs.suffix }}-vhost \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-celery-dev${{ inputs.suffix }} . + fi + + - name: Deploy tfrs-scan-handler + shell: bash {0} + run: | + oc tag ${{ secrets.tools-namespace }}/tfrs-scan-handler:build${{ inputs.suffix }} ${{ secrets.namespace }}/tfrs-scan-handler:dev${{ inputs.suffix }} + cd charts/tfrs-apps/charts/tfrs-scan-handler + helm status -n ${{ secrets.namespace }} tfrs-scan-handler-dev${{ inputs.suffix }} + if [ $? -eq 0 ]; then + echo "tfrs-scan-handler-dev${{ inputs.suffix }} release exists already" + helm upgrade \ + --set scanHandlerImageTagName=dev${{ inputs.suffix }} \ + --set suffix=-dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + --set databaseServiceHostName=${{ inputs.database-service-host-name }} \ + --set rabbitmqVHost=tfrs-dev${{ inputs.suffix }}-vhost \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-scan-handler-dev${{ inputs.suffix }} . + else + echo "tfrs-scan-handler-dev${{ inputs.suffix }} release does not exist" + helm install \ + --set scanHandlerImageTagName=dev${{ inputs.suffix }} \ + --set suffix=dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + --set databaseServiceHostName=${{ inputs.database-service-host-name }} \ + --set rabbitmqVHost=tfrs-dev${{ inputs.suffix }}-vhost \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-scan-handler-dev${{ inputs.suffix }} . + fi + + - name: Deploy tfrs-scan-coordinator + shell: bash {0} + run: | + oc tag ${{ secrets.tools-namespace }}/tfrs-scan-coordinator:build${{ inputs.suffix}} ${{ secrets.namespace }}/tfrs-scan-coordinator:dev${{ inputs.suffix}} + cd charts/tfrs-apps/charts/tfrs-scan-coordinator + helm status -n ${{ secrets.namespace }} tfrs-scan-coordinator-dev${{ inputs.suffix }} + if [ $? -eq 0 ]; then + echo "tfrs-scan-coordinator-dev${{ inputs.suffix }} release exists already" + helm upgrade \ + --set scanCoordinatorImageTagName=dev${{ inputs.suffix}} \ + --set suffix=-dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + --set rabbitmqVHost=tfrs-dev${{ inputs.suffix }}-vhost \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-scan-coordinator-dev${{ inputs.suffix }} . + else + echo "tfrs-scan-coordinator${{ inputs.suffix }} release does not exist" + helm install \ + --set scanCoordinatorImageTagName=dev${{ inputs.suffix}} \ + --set suffix=-dev${{ inputs.suffix }} \ + --set namespace=${{ secrets.namespace }} \ + --set envName=dev \ + --set rabbitmqVHost=tfrs-dev${{ inputs.suffix }}-vhost \ + -n ${{ secrets.namespace }} -f ./values-dev-jan.yaml tfrs-scan-coordinator-dev${{ inputs.suffix }} . + fi \ No newline at end of file diff --git a/.github/workflows/pr-teardown.yaml b/.github/workflows/pr-teardown.yaml new file mode 100644 index 000000000..f6e426cbd --- /dev/null +++ b/.github/workflows/pr-teardown.yaml @@ -0,0 +1,41 @@ +name: TFRS Dev Jan PR Teardown + +on: + pull_request: + types: closed + branches: + - 'main-release-jan-2024' + +env: + TOOLS_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools + DEV_NAMESPACE: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev + +jobs: + + teardown-on-dev: + if: endsWith( github.event.pull_request.title, 'build-on-dev' ) + name: Tear TFRS down on Dev + runs-on: ubuntu-latest + timeout-minutes: 20 + + steps: + + - name: Log in to Openshift + uses: redhat-actions/oc-login@v1.2 + with: + openshift_server_url: ${{ secrets.OPENSHIFT_SERVER }} + openshift_token: ${{ secrets.OPENSHIFT_TOKEN }} + insecure_skip_tls_verify: true + namespace: ${{ env.TOOLS_NAMESPACE }} + + - name: Undeploy on Dev + shell: bash {0} + run: | + oc -n ${{ env.DEV_NAMESPACE }} exec tfrs-rabbitmq-0 -- rabbitmqctl delete_vhost tfrs-dev-jan-${{ github.event.pull_request.number }}-vhost + helm -n ${{ env.DEV_NAMESPACE }} uninstall tfrs-spilo-jan-${{ github.event.pull_request.number }} || true + helm -n ${{ env.DEV_NAMESPACE }} uninstall tfrs-backend-dev-jan-${{ github.event.pull_request.number }} || true + helm -n ${{ env.DEV_NAMESPACE }} uninstall tfrs-frontend-dev-jan-${{ github.event.pull_request.number }} || true + helm -n ${{ env.DEV_NAMESPACE }} uninstall tfrs-celery-dev-jan-${{ github.event.pull_request.number }} || true + helm -n ${{ env.DEV_NAMESPACE }} uninstall tfrs-scan-handler-dev-jan-${{ github.event.pull_request.number }} || true + helm -n ${{ env.DEV_NAMESPACE }} uninstall tfrs-scan-coordinator-dev-jan-${{ github.event.pull_request.number }} || true + diff --git a/README.md b/README.md index e05c7acc3..c6a689b58 100644 --- a/README.md +++ b/README.md @@ -120,3 +120,4 @@ This is a list that was created on 2023-02-01 with all Zelda Devs to provide alt - New learning and applying it to our work - Innovation work + diff --git a/charts/tfrs-spilo/values-dev.yaml b/charts/tfrs-spilo/values-dev.yaml index a51647b97..be1c1bfbd 100644 --- a/charts/tfrs-spilo/values-dev.yaml +++ b/charts/tfrs-spilo/values-dev.yaml @@ -1,6 +1,6 @@ spilo: - replicaCount: 2 + replicaCount: 1 credentials: useExistingSecret: true diff --git a/openshift-v4/templates/celery/Dockerfile b/openshift-v4/templates/celery/Dockerfile new file mode 100644 index 000000000..b303b6d63 --- /dev/null +++ b/openshift-v4/templates/celery/Dockerfile @@ -0,0 +1,14 @@ +FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.9.15 +RUN apt-get update \ + && apt-get install -y git \ + && apt-get install -y supervisor +WORKDIR /app/tfrs +COPY . . +COPY ./security-scan/scan-handler/celery.conf /etc/supervisor/conf.d +RUN pip install --upgrade pip \ + && pip install -r backend/requirements.txt \ + && chgrp -R root /var/log/supervisor \ + && chmod -R g+w /var/log/supervisor \ + && chmod -R g+w /run || : \ + && chmod -R g+w /app +CMD ["supervisord"] \ No newline at end of file diff --git a/openshift-v4/templates/celery/celery-bc-docker.yaml b/openshift-v4/templates/celery/celery-bc-docker.yaml new file mode 100644 index 000000000..55919621f --- /dev/null +++ b/openshift-v4/templates/celery/celery-bc-docker.yaml @@ -0,0 +1,89 @@ +--- +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: celery-bc + creationTimestamp: +parameters: + - name: NAME + displayName: + description: the module name entered when run yo bcdk:pipeline, which is tfrs + required: true + - name: SUFFIX + displayName: + description: sample is -pr-0 + required: true + - name: VERSION + displayName: + description: image tag name for output + required: true + - name: GIT_URL + displayName: + description: tfrs repo + required: true + - name: GIT_REF + displayName: + description: tfrs repo ref + required: true +objects: + - apiVersion: image.openshift.io/v1 + kind: ImageStream + metadata: + annotations: + description: Keeps track of changes in the celery image + labels: + shared: "true" + creationTimestamp: null + name: ${NAME}-celery + spec: + lookupPolicy: + local: false + status: + dockerImageRepository: "" + - kind: BuildConfig + apiVersion: build.openshift.io/v1 + metadata: + name: ${NAME}-celery${SUFFIX} + creationTimestamp: + spec: + triggers: [] + runPolicy: Serial + source: + git: + uri: ${GIT_URL} + ref: ${GIT_REF} + type: Git + strategy: + type: Docker + dockerStrategy: + dockerfilePath: openshift-v4/templates/celery/Dockerfile + noCache: true + env: + - name: ARTIFACTORY_USER + valueFrom: + secretKeyRef: + name: artifacts-default-cgcynz + key: username + - name: ARTIFACTORY_PASSWORD + valueFrom: + secretKeyRef: + name: artifacts-default-cgcynz + key: password + forcePull: true + output: + to: + kind: ImageStreamTag + name: ${NAME}-celery:${VERSION} + resources: + limits: + cpu: 1500m + memory: 1300Mi + requests: + cpu: 750m + memory: 650Mi + postCommit: {} + nodeSelector: + successfulBuildsHistoryLimit: 5 + failedBuildsHistoryLimit: 5 + status: + lastVersion: 0 diff --git a/openshift-v4/templates/scan-handler/Dockerfile b/openshift-v4/templates/scan-handler/Dockerfile new file mode 100644 index 000000000..c57fb5c7b --- /dev/null +++ b/openshift-v4/templates/scan-handler/Dockerfile @@ -0,0 +1,14 @@ +FROM artifacts.developer.gov.bc.ca/docker-remote/python:3.9.15 +RUN apt-get update \ + && apt-get install -y git \ + && apt-get install -y supervisor +WORKDIR /app/tfrs +COPY . . +COPY security-scan/scan-handler/scan-handler.conf /etc/supervisor/conf.d +RUN pip install --upgrade pip \ + && pip install -r backend/requirements.txt \ + && chgrp -R root /var/log/supervisor \ + && chmod -R g+w /var/log/supervisor \ + && chmod -R g+w /run || : \ + && chmod -R g+w /app +CMD ["supervisord"] \ No newline at end of file diff --git a/openshift-v4/templates/scan-handler/scan-handler-bc-docker.yaml b/openshift-v4/templates/scan-handler/scan-handler-bc-docker.yaml new file mode 100644 index 000000000..d25dc1e1d --- /dev/null +++ b/openshift-v4/templates/scan-handler/scan-handler-bc-docker.yaml @@ -0,0 +1,89 @@ +--- +kind: Template +apiVersion: template.openshift.io/v1 +metadata: + name: scan-handler-bc + creationTimestamp: +parameters: +- name: NAME + displayName: + description: the module name entered when run yo bcdk:pipeline, which is tfrs + required: true +- name: SUFFIX + displayName: + description: sample is -pr-0 + required: true +- name: VERSION + displayName: + description: image tag name for output + required: true +- name: GIT_URL + displayName: + description: tfrs repo + required: true +- name: GIT_REF + displayName: + description: tfrs repo ref + required: true +objects: +- apiVersion: image.openshift.io/v1 + kind: ImageStream + metadata: + annotations: + description: Keeps track of changes in the celery image + labels: + shared: "true" + creationTimestamp: null + name: ${NAME}-scan-handler + spec: + lookupPolicy: + local: false + status: + dockerImageRepository: "" +- kind: BuildConfig + apiVersion: build.openshift.io/v1 + metadata: + name: ${NAME}-scan-handler${SUFFIX} + creationTimestamp: + spec: + triggers: [] + runPolicy: Serial + source: + type: Git + git: + uri: ${GIT_URL} + ref: ${GIT_REF} + strategy: + type: Docker + dockerStrategy: + dockerfilePath: openshift-v4/templates/scan-handler/Dockerfile + noCache: true + env: + - name: ARTIFACTORY_USER + valueFrom: + secretKeyRef: + name: artifacts-default-cgcynz + key: username + - name: ARTIFACTORY_PASSWORD + valueFrom: + secretKeyRef: + name: artifacts-default-cgcynz + key: password + forcePull: true + output: + to: + kind: ImageStreamTag + name: ${NAME}-scan-handler:${VERSION} + resources: + limits: + cpu: 1500m + memory: 1300Mi + requests: + cpu: 750m + memory: 650Mi + postCommit: {} + nodeSelector: + successfulBuildsHistoryLimit: 5 + failedBuildsHistoryLimit: 5 + status: + lastVersion: 0