Skip to content

Commit

Permalink
Add pull request build for Jan release (#2463)
Browse files Browse the repository at this point in the history
  • Loading branch information
kuanfandevops authored and Your Name committed Aug 30, 2023
1 parent cb0fd15 commit 994dfee
Show file tree
Hide file tree
Showing 14 changed files with 643 additions and 40 deletions.
4 changes: 1 addition & 3 deletions .github/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,5 @@
* tfrs-release.yaml (TFRS release-2.10.0): the pipeline builds the release and deploys on Test and Prod, it needs to be manually triggered
* create-release.yaml (Create Release after merging to master): tag and create the release after merging release branch to master. The description of the tracking pull request becomes release notes

## Other Pipelines

* cleanup-cron-workflow-runs.yaml (Scheduled cleanup old workflow runs): a cron job to cleanup the old workflows
* cleanup-workflow-runs.yaml (Cleanup old workflow runs): manually cleanup teh workflow runs

28 changes: 27 additions & 1 deletion .github/workflows/branch-deploy-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -161,4 +161,30 @@ jobs:
--set databaseServiceHostName=${{ inputs.database-service-host-name }} \
--set rabbitmqVHost=${{ inputs.rabbitmq-vhost }} \
-n ${{ secrets.namespace }} -f ./values${{ inputs.suffix }}.yaml tfrs-scan-handler${{ inputs.suffix }} .
fi
fi
- name: Deploy tfrs-scan-coordinator
shell: bash {0}
run: |
oc tag ${{ secrets.tools-namespace }}/tfrs-scan-hacoordinatorndler:build-${{ inputs.branch-name }} ${{ secrets.namespace }}/tfrs-scan-coordinator:${{ inputs.env-name }}-${{ inputs.branch-name }}
cd charts/tfrs-apps/charts/tfrs-scan-coordinator
helm status -n ${{ secrets.namespace }} tfrs-scan-coordinator${{ inputs.suffix }}
if [ $? -eq 0 ]; then
echo "tfrs-scan-coordinator${{ inputs.suffix }} release exists already"
helm upgrade \
--set scanCoordinatorImageTagName=${{ inputs.env-name }}-${{ inputs.branch-name }} \
--set suffix=${{ inputs.suffix }} \
--set namespace=${{ secrets.namespace }} \
--set envName=${{ inputs.env-Name }} \
--set rabbitmqVHost=${{ inputs.rabbitmq-vhost }} \
-n ${{ secrets.namespace }} -f ./values${{ inputs.suffix }}.yaml tfrs-scan-coordinator${{ inputs.suffix }} .
else
echo "tfrs-scan-coordinator${{ inputs.suffix }} release does not exist"
helm install \
--set scanCoordinatorImageTagName=${{ inputs.env-name }}-${{ inputs.branch-name }} \
--set suffix=${{ inputs.suffix }} \
--set namespace=${{ secrets.namespace }} \
--set envName=${{ inputs.env-Name }} \
--set rabbitmqVHost=${{ inputs.rabbitmq-vhost }} \
-n ${{ secrets.namespace }} -f ./values${{ inputs.suffix }}.yaml tfrs-scan-coordinator${{ inputs.suffix }} .
fi
Original file line number Diff line number Diff line change
@@ -1,10 +1,18 @@

# This template supports both pr build and branch build
name: Branch Build Template

on:
workflow_call:
inputs:
branch-name: # sample value: main-release-jan-2024
# when build branch, the sample value is -main-release-jan-2024
# when build pull request, the sample value is -jan-2024
suffix:
required: true
type: string
# when build branch, the sample value is main-release-jan-2024
# when build pull request, the sample value is refs/pull/2024/head
checkout-ref:
required: true
type: string
secrets:
Expand All @@ -31,7 +39,7 @@ jobs:
- name: Check out repository
uses: actions/checkout@v3.5.3
with:
ref: ${{ inputs.branch-name }}
ref: ${{ inputs.checkout-ref }}

- name: Log in to Openshift
uses: redhat-actions/oc-login@v1.2
Expand All @@ -45,13 +53,13 @@ jobs:
run: |
cd openshift-v4/templates/backend
oc process -f ./backend-bc.yaml NAME=tfrs \
SUFFIX=-build-${{ inputs.branch-name}} \
VERSION=build-${{ inputs.branch-name }} \
SUFFIX=-build${{ inputs.suffix}} \
VERSION=build${{ inputs.suffix }} \
GIT_URL=${{ env.GIT_URL }} \
GIT_REF=${{ inputs.branch-name }} \
GIT_REF=${{ inputs.checkout-ref }} \
| oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-backend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-backend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-backend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-backend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}
build-frontend:

Expand All @@ -64,7 +72,7 @@ jobs:
- name: Check out repository
uses: actions/checkout@v3.5.3
with:
ref: ${{ inputs.branch-name }}
ref: ${{ inputs.checkout-ref }}

- name: Log in to Openshift
uses: redhat-actions/oc-login@v1.2
Expand All @@ -78,17 +86,18 @@ jobs:
run: |
cd openshift-v4/templates/frontend
oc process -f ./frontend-bc-docker.yaml NAME=tfrs \
SUFFIX=-build-${{ inputs.branch-name}} \
VERSION=build-${{ inputs.branch-name }} \
SUFFIX=-build${{ inputs.suffix}} \
VERSION=build${{ inputs.suffix }} \
GIT_URL=${{ env.GIT_URL }} \
GIT_REF=${{ inputs.branch-name }} \
GIT_REF=${{ inputs.checkout-ref }} \
| oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-frontend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-frontend-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-frontend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-frontend-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}
build-celery:

name: Build TFRS Celery on Openshift
needs: [build-frontend, build-backend]
runs-on: ubuntu-latest
timeout-minutes: 60

Expand All @@ -97,7 +106,7 @@ jobs:
- name: Check out repository
uses: actions/checkout@v3.5.3
with:
ref: ${{ inputs.branch-name }}
ref: ${{ inputs.checkout-ref }}

- name: Log in to Openshift
uses: redhat-actions/oc-login@v1.2
Expand All @@ -110,18 +119,21 @@ jobs:
- name: Build TFRS Celery
run: |
cd openshift-v4/templates/celery
oc process -f ./celery-bc.yaml NAME=tfrs \
SUFFIX=-build-${{ inputs.branch-name}} \
VERSION=build-${{ inputs.branch-name }} \
pwd
ls -l
oc process -f ./celery-bc-docker.yaml NAME=tfrs \
SUFFIX=-build${{ inputs.suffix}} \
VERSION=build${{ inputs.suffix }} \
GIT_URL=${{ env.GIT_URL }} \
RELEASE_BRANCH=${{ inputs.branch-name}} \
GIT_REF=${{ inputs.checkout-ref }} \
| oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-celery-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-celery-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-celery-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-celery-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}
build-scan-coordinator:

name: Build TFRS Scan Coordinator on Openshift
needs: [build-frontend, build-backend]
runs-on: ubuntu-latest
timeout-minutes: 60

Expand All @@ -130,7 +142,7 @@ jobs:
- name: Check out repository
uses: actions/checkout@v3.5.3
with:
ref: ${{ inputs.branch-name }}
ref: ${{ inputs.checkout-ref }}

- name: Log in to Openshift
uses: redhat-actions/oc-login@v1.2
Expand All @@ -144,17 +156,18 @@ jobs:
run: |
cd openshift-v4/templates/scan-coordinator
oc process -f ./scan-coordinator-bc.yaml NAME=tfrs \
SUFFIX=-build-${{ inputs.branch-name}} \
VERSION=build-${{ inputs.branch-name }} \
SUFFIX=-build${{ inputs.suffix}} \
VERSION=build${{ inputs.suffix }} \
GIT_URL=${{ env.GIT_URL }} \
GIT_REF=${{ inputs.branch-name }} \
GIT_REF=${{ inputs.checkout-ref }} \
| oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-scan-coordinator-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-scan-coordinator-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-scan-coordinator-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-scan-coordinator-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}
build-scan-handler:

name: Build TFRS Scan Handler on Openshift
needs: [build-scan-coordinator, build-celery]
runs-on: ubuntu-latest
timeout-minutes: 60

Expand All @@ -163,7 +176,7 @@ jobs:
- name: Check out repository
uses: actions/checkout@v3.5.3
with:
ref: ${{ inputs.branch-name }}
ref: ${{ inputs.checkout-ref }}

- name: Log in to Openshift
uses: redhat-actions/oc-login@v1.2
Expand All @@ -176,10 +189,11 @@ jobs:
- name: Build TFRS Scan-Handler
run: |
cd openshift-v4/templates/scan-handler
oc process -f ./scan-handler-bc.yaml NAME=tfrs \
SUFFIX=-build-${{ inputs.branch-name}} \
VERSION=build-${{ inputs.branch-name }} \
RELEASE_BRANCH=${{ inputs.branch-name }} \
oc process -f ./scan-handler-bc-docker.yaml NAME=tfrs \
SUFFIX=-build${{ inputs.suffix}} \
VERSION=build${{ inputs.suffix }} \
GIT_URL=${{ env.GIT_URL }} \
GIT_REF=${{ inputs.checkout-ref }} \
| oc apply --wait=true -f - -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-scan-handler-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-scan-handler-build-${{ inputs.branch-name}} -n ${{ secrets.tools-namespace }}
oc cancel-build bc/tfrs-scan-handler-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }} || true
oc start-build --wait=true tfrs-scan-handler-build${{ inputs.suffix}} -n ${{ secrets.tools-namespace }}
7 changes: 5 additions & 2 deletions .github/workflows/dev-jan-release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,15 @@ jobs:
VALIDATE_PYTHON_PYLINT: true
LOG_LEVEL: WARN

# when build branch, the suffix sample is -main-release-jan-2024
# the checkout-ref sample is main-release-jan-2024
build:
name: Build
needs: [unit-test, lint]
uses: ./.github/workflows/branch-build-template.yaml
uses: ./.github/workflows/build-template.yaml
with:
branch-name: ${{ github.ref_name }}
suffix: -${{ github.ref_name }}
checkout-ref: ${{ github.ref_name }}
secrets:
tools-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools
openshift-server: ${{ secrets.OPENSHIFT_SERVER }}
Expand Down
54 changes: 54 additions & 0 deletions .github/workflows/pr-dev-cicd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Please refer to ./readme.md for how to build single pull request

# Update this workflow name per pull request
name: TFRS Dev Jan PR CICD
on:
workflow_dispatch:
pull_request:
types: [opened, edited, synchronize, reopened]
branches:
- 'main-release-jan-2024'

jobs:

setup-database:
if: endsWith( github.event.pull_request.title, 'build-on-dev' )
uses: ./.github/workflows/pr-dev-database-template.yaml
with:
pr-number: ${{ github.event.pull_request.number }}
dev-suffix: -jan-${{ github.event.pull_request.number }}
secrets:
dev-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev
tfrs-dev-username: ${{ secrets.TFRS_DEV_USERNAME }}
tfrs-dev-password: ${{ secrets.TFRS_DEV_PASSWORD }}
openshift-server: ${{ secrets.OPENSHIFT_SERVER }}
openshift-token: ${{ secrets.OPENSHIFT_TOKEN }}

# when build pull reuqest, the suffix sample is -jan-1234
# the checkout-ref is in the format of refs/pull/1234/head
build:
if: endsWith( github.event.pull_request.title, 'build-on-dev' )
name: Build Pull Request
uses: ./.github/workflows/build-template.yaml
with:
suffix: -jan-${{ github.event.pull_request.number }}
checkout-ref: refs/pull/${{ github.event.pull_request.number }}/head
secrets:
tools-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools
openshift-server: ${{ secrets.OPENSHIFT_SERVER }}
openshift-token: ${{ secrets.OPENSHIFT_TOKEN }}

deploy:
if: endsWith( github.event.pull_request.title, 'build-on-dev' )
needs: [setup-database, build]
uses: ./.github/workflows/pr-dev-deploy-template.yaml
with:
suffix: -jan-${{ github.event.pull_request.number }}
checkout-ref: refs/pull/${{ github.event.pull_request.number }}/head
database-service-host-name: tfrs-spilo-jan-${{ github.event.pull_request.number }}
secrets:
tools-namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-tools
namespace: ${{ secrets.OPENSHIFT_NAMESPACE_PLATE }}-dev
openshift-server: ${{ secrets.OPENSHIFT_SERVER }}
openshift-token: ${{ secrets.OPENSHIFT_TOKEN }}

69 changes: 69 additions & 0 deletions .github/workflows/pr-dev-database-template.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
name: PR Dev Database Template

on:
workflow_call:
inputs:
# pull request number
pr-number:
required: true
type: string
# the suffix will be appended to tfrs-spilo, same values: -1234, -jan-1242
dev-suffix:
required: true
type: string
secrets:
dev-namespace:
required: true
tfrs-dev-username:
required: true
tfrs-dev-password:
required: true
openshift-server:
required: true
openshift-token:
required: true

jobs:

database:

name: Start Database
runs-on: ubuntu-latest
timeout-minutes: 60

steps:

- name: Check out repository
uses: actions/checkout@v3
with:
ref: refs/pull/${{ inputs.pr-number }}/head

- name: Log in to Openshift
uses: redhat-actions/oc-login@v1.2
with:
openshift_server_url: ${{ secrets.openshift-server }}
openshift_token: ${{ secrets.openshift-token }}
insecure_skip_tls_verify: true
namespace: ${{ secrets.dev-namespace }}

- name: Setup Database
shell: bash {0}
run: |
cd charts/tfrs-spilo
helm dependency build
helm status -n ${{ secrets.dev-namespace }} tfrs-spilo${{ inputs.dev-suffix }}
if [ $? -eq 0 ]; then
echo "tfrs-spilo${{ inputs.dev-suffix }} exists already"
else
echo "Installing tfrs-spilo${{ inputs.dev-suffix }}"
helm install -n ${{ secrets.dev-namespace }} -f ./values-dev.yaml --wait tfrs-spilo${{ inputs.dev-suffix }} .
oc -n ${{ secrets.dev-namespace }} wait --for=condition=Ready pod/tfrs-spilo${{ inputs.dev-suffix }}-0
oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "create user \"${{ secrets.tfrs-dev-username }}\" WITH PASSWORD '${{ secrets.tfrs-dev-password }}'" || true
oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "create database tfrs owner \"${{ secrets.tfrs-dev-username }}\" ENCODING 'UTF8' LC_COLLATE = 'en_US.UTF-8' LC_CTYPE = 'en_US.UTF-8'" || true
oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_filename='postgresql-%H.log'" || true
oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_connections='off'" || true
oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_disconnections='off'" || true
oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "ALTER SYSTEM SET log_checkpoints='off'" || true
oc -n ${{ secrets.dev-namespace }} exec tfrs-spilo${{ inputs.dev-suffix }}-0 -- psql -c "select pg_reload_conf()" || true
fi
Loading

0 comments on commit 994dfee

Please sign in to comment.