BCDA-8301: Fix large object S3 trigger + update logging #32
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Integration test that triggers a CCLF import in the test environment and | |
| # verifies the imported data in the test database for ACO A0001. | |
| name: cclf-import test integration | |
| on: | |
| pull_request: | |
| paths: | |
| - .github/workflows/cclf-import-test-integration.yml | |
| - .github/workflows/cclf-import-test-deploy.yml | |
| - bcda/cclf/** | |
| - bcda/lambda/cclf/** | |
| workflow_dispatch: | |
| # Ensure we have only one integration test running at a time | |
| concurrency: | |
| group: cclf-import-test-integration | |
| jobs: | |
| # Deploy first if triggered by pull_request | |
| deploy: | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: ./.github/workflows/cclf-import-test-deploy.yml | |
| secrets: inherit | |
| trigger: | |
| if: ${{ always() }} | |
| needs: deploy | |
| permissions: | |
| contents: read | |
| id-token: write | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: bcda | |
| outputs: | |
| filename: ${{ steps.createfile.outputs.FILENAME }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ vars.AWS_REGION }} | |
| role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-cclf-import-function | |
| - uses: aws-actions/configure-aws-credentials@v4 | |
| with: | |
| aws-region: ${{ vars.AWS_REGION }} | |
| # Note that we use the BFD role with access to the bucket | |
| role-to-assume: arn:aws:iam::${{ secrets.BFD_ACCOUNT_ID }}:role/bfd-test-eft-bcda-bucket-role | |
| role-chaining: true | |
| role-skip-session-tagging: true | |
| - name: Upload test file to the BFD bucket to trigger lambda function via SNS message | |
| id: createfile | |
| run: | | |
| year=$(date +'%y') | |
| date=$(date +'%y%m%d') | |
| time=$(date +'%H%M%S') | |
| fname=T.BCD.A0001.ZCY24.D${date}.T${time}1 | |
| cclf8_fname=T.BCD.A0001.ZC8Y24.D${date}.T${time}1 | |
| echo "FILENAME=$cclf8_fname" >> "$GITHUB_OUTPUT" | |
| unzip ../shared_files/cclf/archives/valid2/T.BCD.A0001.ZCY18.D181120.T1000000 | |
| # Clone the synthetic zip file with updated years | |
| new_dir="D${date}.T${time}" | |
| mkdir $new_dir | |
| mv T.BCD.A0001.ZC8Y18.D181120.T1000009 $new_dir/T.BCD.A0001.ZC8Y${year}.D${date}.T${time}1 | |
| mv T.BCD.A0001.ZC9Y18.D181120.T1000010 $new_dir/T.BCD.A0001.ZC9Y${year}.D${date}.T${time}1 | |
| mv T.BCD.A0001.ZC0Y18.D181120.T1000011 $new_dir/T.BCD.A0001.ZC0Y${year}.D${date}.T${time}1 | |
| zip -jr $fname $new_dir | |
| aws s3 cp --no-progress $fname \ | |
| s3://bfd-test-eft/bfdeft01/bcda/in/test/$fname | |
| verify: | |
| needs: trigger | |
| runs-on: self-hosted | |
| env: | |
| ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: "true" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: aws-actions/configure-aws-credentials@v3 | |
| with: | |
| aws-region: ${{ vars.AWS_REGION }} | |
| role-to-assume: arn:aws:iam::${{ secrets.ACCOUNT_ID }}:role/delegatedadmin/developer/bcda-test-github-actions | |
| - name: Install psql | |
| run: | | |
| sudo amazon-linux-extras install postgresql14 | |
| - name: Get database credentials | |
| uses: cmsgov/ab2d-bcda-dpc-platform/actions/aws-params-env-action@main | |
| env: | |
| AWS_REGION: ${{ vars.AWS_REGION }} | |
| with: | |
| params: | | |
| CONNECTION_INFO=/bcda/test/api/DATABASE_URL | |
| - name: Verify CCLF file was ingested | |
| env: | |
| FILENAME: ${{needs.trigger.outputs.filename}} | |
| # CAUTION: if changing the script below, validate that sensitive information is not printed in the workflow | |
| run: | | |
| HOST=$(aws rds describe-db-instances --db-instance-identifier bcda-test-rds 2>&1 | jq -r '.DBInstances[0].Endpoint.Address' 2>&1) | |
| CONNECTION_URL=$(echo $CONNECTION_INFO 2>&1 | sed -E "s/@.*\/bcda/\@$HOST\/bcda/" 2>&1) | |
| # Verify that we have a record of the CCLF file in the database | |
| CCLF_FILE=`psql -t "$CONNECTION_URL" -c "SELECT id FROM cclf_files WHERE name = '$FILENAME' LIMIT 1" 2>&1` | |
| if [[ $? -ne 0 || -z $CCLF_FILE ]]; then | |
| echo "cclf_file query returned zero results or command failed" | |
| exit 1 | |
| else | |
| # Verify that the correct number of benes were imported into the database. | |
| CCLF_BENES=`psql -t "$CONNECTION_URL" -c "SELECT count(mbi) FROM cclf_beneficiaries WHERE file_id = $CCLF_FILE" 2>&1` | |
| if [[ $? -ne 0 || -z $CCLF_BENES ]]; then | |
| echo "CCLF beneficiaries query returned zero results or command failed" | |
| exit 1 | |
| fi | |
| if [[ $(echo $CCLF_BENES | xargs) != "6" ]]; then | |
| echo "expected 6 beneficiaries imported from file, received $CCLF_BENES". | |
| exit 1 | |
| fi | |
| fi |