Solve CI vulnerability (dmlc#1581)

* test * test * test * test * test * test (#14) * test * test * test * test * test * test * test * test repo name * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test * test (#15) * test * test * test * test (#16) * test * test * test (#17) * test * test * test * test * test (#18) * test * test * test (#19) * test * test * test * test (#20) * test * test * full test * full test * full test * merge bac * fix * fix * fix * full test * fix * test pr (#21) * test pr * fix * test pr (#22) * test pr * test pr * test pr (#23) * test pr * test pr * rerun * rerun * update batch * full test (#24) * full test * full test * rename * finish testing * readme * trigger workflow
yinweisu · Jan 6, 2021 · 094dbba · 094dbba
1 parent 134ede8
commit 094dbba
Show file tree

Hide file tree

Showing 6 changed files with 381 additions and 278 deletions.
diff --git a/.github/workflows/build_docs.sh b/.github/workflows/build_docs.sh
@@ -10,7 +10,7 @@ EFS=/mnt/efs
 mkdir -p ~/.mxnet/datasets
 for f in $EFS/.mxnet/datasets/*; do
     if [ -d "$f" ]; then
-        # Will not run if no directories are available
+        # Will not run if no directories are available 
         ln -s $f ~/.mxnet/datasets/$(basename "$f")
     fi
 done
@@ -26,13 +26,10 @@ sed -i.bak 's/2196f3/178dc9/g' build/html/_static/sphinx_materialdesign_theme.cs
 sed -i.bak 's/pre{padding:1rem;margin:1.5rem\\s0;overflow:auto;overflow-y:hidden}/pre{padding:1rem;margin:1.5rem 0;overflow:auto;overflow-y:scroll}/g' build/html/_static/sphinx_materialdesign_theme.css
 
 if [[ ($BRANCH == master) && ($GIT_REPO == dmlc/gluon-cv) ]]; then
-	# aws s3 cp s3://gluon-cv.mxnet.io/coverage.svg build/html/coverage.svg
 	aws s3 sync --delete build/html/ s3://gluon-cv.mxnet.io/ --acl public-read --cache-control max-age=7200
-	# aws s3 cp build/html/coverage.svg s3://gluon-cv.mxnet.io/coverage.svg --acl public-read --cache-control max-age=300
 	echo "Uploaded doc to http://gluon-cv.mxnet.io"
 	echo master
 else
-	# aws s3 cp s3://gluon-vision-staging/$PR_NUMBER/$COMMIT_SHA/coverage.svg build/html/coverage.svg
 	aws s3 sync --delete build/html/ s3://gluon-vision-staging/PR-$PR_NUMBER/$COMMIT_SHA/ --acl public-read
 	echo "Uploaded doc to http://gluon-vision-staging.s3-website-us-west-2.amazonaws.com/PR-$PR_NUMBER/$COMMIT_SHA/index.html"
 	echo $GIT_REPO: $BRANCH

diff --git a/.github/workflows/build_test.yml b/.github/workflows/build_test.yml
@@ -0,0 +1,283 @@
+name: Build Test
+on: 
+  workflow_run:
+    workflows: ["Unit Test"]
+    types:
+      - completed
+jobs:
+  unittest-check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: haya14busa/action-workflow_run-status@v1
+      - name: Fail if unittests failed
+        if: ${{ github.event.workflow_run.conclusion != 'success' }}
+        run: exit 1
+  model_zoo_mxnet:
+    needs: unittest-check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: haya14busa/action-workflow_run-status@v1
+      - name: Log
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: |
+          echo "$GITHUB_CONTEXT"
+      - name: Download artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: unittest.yml
+          run_id: ${{ github.event.workflow_run.id }}
+      - name: Get git context(For push)
+        if: ${{ github.event.workflow_run.event == 'push' }}
+        run: |
+          ref=$(head -n 1 artifact/ref.atf)
+          repo=$(head -n 1 artifact/repo.atf)
+          echo "REF=$ref" >> $GITHUB_ENV
+          echo "REPO=$repo" >> $GITHUB_ENV
+      - name: Get git context(For pull request)
+        if: ${{ github.event.workflow_run.event == 'pull_request' }}
+        run: |
+          ref=$(head -n 1 artifact/ref.atf)
+          repo=$(head -n 1 artifact/repo.atf)
+          pr=$(head -n 1 artifact/pr.atf)
+          echo "REF=$ref" >> $GITHUB_ENV
+          echo "REPO=$repo" >> $GITHUB_ENV
+          echo "PR=$pr" >> $GITHUB_ENV
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }}
+          aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+      - name: Install dependencies
+        run: |
+          pip install --upgrade --force-reinstall --no-deps .
+          pip install boto3
+      - name: Test model_zoo_mxnet on AWS Batch(For push)
+        shell: bash -l {0}
+        if: ${{ github.event.workflow_run.event == 'push' }}
+        run: |
+          echo "Start submitting job"
+          python ./tools/batch/submit-job.py --region us-east-1 \
+                                             --job-type ci \
+                                             --name GluonCV-GPU-ModelZooMxnet-${{ env.REF }} \
+                                             --source-ref ${{ env.REF }} \
+                                             --work-dir . \
+                                             --remote https://github.com/${{ env.REPO }} \
+                                             --command "chmod +x ./.github/workflows/gpu_test.sh && ./.github/workflows/gpu_test.sh gluoncv tests/model_zoo" \
+                                             --wait
+      - name: Test model_zoo_mxnet on AWS Batch(For pull request)
+        if: ${{ github.event.workflow_run.event == 'pull_request' }}
+        run: |
+          echo "Start submitting job"
+          python ./tools/batch/submit-job.py --region us-east-1 \
+                                             --job-type ci \
+                                             --name GluonCV-GPU-ModelZooMxnet-PR#${{ env.PR }} \
+                                             --source-ref ${{ env.REF }} \
+                                             --work-dir . \
+                                             --remote https://github.com/${{ env.REPO }} \
+                                             --command "chmod +x ./.github/workflows/gpu_test.sh && ./.github/workflows/gpu_test.sh gluoncv tests/model_zoo" \
+                                             --wait
+  model_zoo_torch:
+    needs: unittest-check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: haya14busa/action-workflow_run-status@v1
+      - name: Download artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: unittest.yml
+          run_id: ${{ github.event.workflow_run.id }}
+      - name: Get git context(For push)
+        if: ${{ github.event.workflow_run.event == 'push' }}
+        run: |
+          ref=$(head -n 1 artifact/ref.atf)
+          repo=$(head -n 1 artifact/repo.atf)
+          echo "REF=$ref" >> $GITHUB_ENV
+          echo "REPO=$repo" >> $GITHUB_ENV
+      - name: Get git context(For pull request)
+        if: ${{ github.event.workflow_run.event == 'pull_request' }}
+        run: |
+          ref=$(head -n 1 artifact/ref.atf)
+          repo=$(head -n 1 artifact/repo.atf)
+          pr=$(head -n 1 artifact/pr.atf)
+          echo "REF=$ref" >> $GITHUB_ENV
+          echo "REPO=$repo" >> $GITHUB_ENV
+          echo "PR=$pr" >> $GITHUB_ENV
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }}
+          aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+      - name: Install dependencies
+        run: |
+          pip install --upgrade --force-reinstall --no-deps .
+          pip install boto3
+      - name: Test model_zoo_torch on AWS Batch(For push)
+        shell: bash -l {0}
+        if: ${{ github.event.workflow_run.event == 'push' }}
+        run: |
+          echo "Start submitting job"
+          python ./tools/batch/submit-job.py --region us-east-1 \
+                                             --job-type ci \
+                                             --name GluonCV-GPU-ModelZooTorch-${{ env.REF }} \
+                                             --source-ref ${{ env.REF }} \
+                                             --work-dir . \
+                                             --remote https://github.com/${{ env.REPO }} \
+                                             --command "chmod +x ./.github/workflows/gpu_test.sh && ./.github/workflows/gpu_test.sh gluoncv/torch tests/model_zoo_torch" \
+                                             --wait
+      - name: Test model_zoo_torch on AWS Batch(For pull request)
+        if: ${{ github.event.workflow_run.event == 'pull_request' }}
+        run: |
+          echo "Start submitting job"
+          python ./tools/batch/submit-job.py --region us-east-1 \
+                                             --job-type ci \
+                                             --name GluonCV-GPU-ModelZooTorch-PR#${{ env.PR }} \
+                                             --source-ref ${{ env.REF }} \
+                                             --work-dir . \
+                                             --remote https://github.com/${{ env.REPO }} \
+                                             --command "chmod +x ./.github/workflows/gpu_test.sh && ./.github/workflows/gpu_test.sh gluoncv/torch tests/model_zoo_torch" \
+                                             --wait
+  auto:
+    needs: unittest-check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: haya14busa/action-workflow_run-status@v1
+      - name: Download artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: unittest.yml
+          run_id: ${{ github.event.workflow_run.id }}
+      - name: Get git context(For push)
+        if: ${{ github.event.workflow_run.event == 'push' }}
+        run: |
+          ref=$(head -n 1 artifact/ref.atf)
+          repo=$(head -n 1 artifact/repo.atf)
+          echo "REF=$ref" >> $GITHUB_ENV
+          echo "REPO=$repo" >> $GITHUB_ENV
+      - name: Get git context(For pull request)
+        if: ${{ github.event.workflow_run.event == 'pull_request' }}
+        run: |
+          ref=$(head -n 1 artifact/ref.atf)
+          repo=$(head -n 1 artifact/repo.atf)
+          pr=$(head -n 1 artifact/pr.atf)
+          echo "REF=$ref" >> $GITHUB_ENV
+          echo "REPO=$repo" >> $GITHUB_ENV
+          echo "PR=$pr" >> $GITHUB_ENV
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }}
+          aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+      - name: Install dependencies
+        run: |
+          pip install --upgrade --force-reinstall --no-deps .
+          pip install boto3
+      - name: Test model_zoo_torch on AWS Batch(For push)
+        shell: bash -l {0}
+        if: ${{ github.event.workflow_run.event == 'push' }}
+        run: |
+          echo "Start submitting job"
+          python ./tools/batch/submit-job.py --region us-east-1 \
+                                             --job-type ci \
+                                             --name GluonCV-GPU-Auto-${{ env.REF }} \
+                                             --source-ref ${{ env.REF }} \
+                                             --work-dir . \
+                                             --remote https://github.com/${{ env.REPO }} \
+                                             --command "chmod +x ./.github/workflows/gpu_test.sh && ./.github/workflows/gpu_test.sh gluoncv tests/auto" \
+                                             --wait
+      - name: Test model_zoo_torch on AWS Batch(For pull request)
+        if: ${{ github.event.workflow_run.event == 'pull_request' }}
+        run: |
+          echo "Start submitting job"
+          python ./tools/batch/submit-job.py --region us-east-1 \
+                                             --job-type ci \
+                                             --name GluonCV-GPU-Auto-PR#${{ env.PR }} \
+                                             --source-ref ${{ env.REF }} \
+                                             --work-dir . \
+                                             --remote https://github.com/${{ env.REPO }} \
+                                             --command "chmod +x ./.github/workflows/gpu_test.sh && ./.github/workflows/gpu_test.sh gluoncv tests/auto" \
+                                             --wait
+  build-docs:
+    needs: [unittest-check, model_zoo_mxnet, model_zoo_torch, auto]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: haya14busa/action-workflow_run-status@v1
+      - name: Checkout repository
+        uses: actions/checkout@v2
+      - name: Download artifact
+        uses: dawidd6/action-download-artifact@v2
+        with:
+          workflow: unittest.yml
+          run_id: ${{ github.event.workflow_run.id }}
+      - name: Get git context(For push)
+        if: ${{ github.event.workflow_run.event == 'push' }}
+        run: |
+          ref=$(head -n 1 artifact/ref.atf)
+          repo=$(head -n 1 artifact/repo.atf)
+          echo "REF=$ref" >> $GITHUB_ENV
+          echo "REPO=$repo" >> $GITHUB_ENV
+      - name: Get git context(For pull request)
+        if: ${{ github.event.workflow_run.event == 'pull_request' }}
+        run: |
+          ref=$(head -n 1 artifact/ref.atf)
+          repo=$(head -n 1 artifact/repo.atf)
+          pr=$(head -n 1 artifact/pr.atf)
+          sha=$(head -n 1 artifact/sha.atf)
+          short_sha=$(git rev-parse --short "$sha")
+          echo "REF=$ref" >> $GITHUB_ENV
+          echo "REPO=$repo" >> $GITHUB_ENV
+          echo "PR=$pr" >> $GITHUB_ENV
+          echo "$SHORT_SHA"
+          echo "SHORT_SHA=$short_sha" >> $GITHUB_ENV
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.GLUONCV_DEV_ACCESS_ID }}
+          aws-secret-access-key: ${{ secrets.GLUONCV_DEV_SECRET_ACCESS_KEY }}
+          aws-region: us-east-1
+      - name: Install dependencies
+        run: |
+          pip install --upgrade --force-reinstall --no-deps .
+          pip install boto3
+      - name: Build docs on AWS Batch(For push)
+        shell: bash -l {0}
+        if: ${{ github.event.workflow_run.event == 'push' }}
+        run: |
+          echo "Start submitting job"
+          python ./tools/batch/submit-job.py --region us-east-1 \
+                                             --job-type ci \
+                                             --name GluonCV-GPU-BuildDocs-${{ env.REF }} \
+                                             --source-ref ${{ env.REF }} \
+                                             --work-dir . \
+                                             --remote https://github.com/${{ env.REPO }} \
+                                             --command "chmod +x ./.github/workflows/build_docs.sh && ./.github/workflows/build_docs.sh ${{ env.REF }} ${{ env.SHORT_SHA }} ${{ env.REPO }} ${{ env.PR }}" \
+                                             --wait
+      - name: Build docs on AWS Batch(For pull request)
+        if: ${{ github.event.workflow_run.event == 'pull_request' }}
+        run: |
+          echo "Start submitting job"
+          python ./tools/batch/submit-job.py --region us-east-1 \
+                                             --job-type ci \
+                                             --name GluonCV-GPU-BuildDocs-PR#${{ env.PR }} \
+                                             --source-ref ${{ env.REF }} \
+                                             --work-dir . \
+                                             --remote https://github.com/${{ env.REPO }} \
+                                             --command "chmod +x ./.github/workflows/build_docs.sh && ./.github/workflows/build_docs.sh ${{ env.REF }} ${{ env.SHORT_SHA }} ${{ env.REPO }} ${{ env.PR }} " \
+                                             --wait
+      - name: Comment on PR
+        if: ${{ github.event.workflow_run.event == 'pull_request' }}
+        uses: peter-evans/create-or-update-comment@v1.4.3
+        with:
+          issue-number: ${{ env.PR }}
+          body: |
+            Job PR-${{ env.PR }}-${{ env.SHORT_SHA }} is done.
+            Docs are uploaded to http://gluon-vision-staging.s3-website-us-west-2.amazonaws.com/PR-${{ env.PR }}/${{ env.SHORT_SHA }}/index.html