run tests on GitHub actions

.github/workflows/ci-test-pr.yml

@@ -0,0 +1,272 @@
+name: Build and Run Tests
+
+on:
+  push
+
+jobs:
+  setup:
+    runs-on: ubuntu-latest
+
+    env:
+      AWS_ACCOUNT_NUMBER: '848416313321'
+      AWS_REGION: 'eu-west-1'
+
+    outputs:
+      image: ${{ steps.build-image.outputs.image }}
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+        aws-region: eu-west-1
+
+    - name: Login to Amazon ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v1
+
+    - name: Login to ECR
+      uses: docker/login-action@v1
+      with:
+        registry: ${{ env.AWS_ACCOUNT_NUMBER }}.dkr.ecr.${{ env.AWS_REGION }}.amazonaws.com
+
+    - name: Set up Docker Buildx
+      id: buildx
+      uses: docker/setup-buildx-action@v1
+
+    - name: Cache Docker layers
+      uses: actions/cache@v2
+      id: cache
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
+
+    - name: Docker Build And Push To ECR
+      id: docker-build-ecr
+      uses: docker/build-push-action@v2
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        ECR_REPOSITORY: ${{ github.event.repository.name }}
+        IMAGE_TAG: ${{ github.sha }}
+      with:
+        push: true
+        tags: |
+          ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}
+        builder: ${{ steps.buildx.outputs.name }}
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+    -
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      name: Move cache
+      run: |
+        rm -rf /tmp/.buildx-cache
+        mv /tmp/.buildx-cache-new /tmp/.buildx-cache
+
+
+  test-functional:
+    runs-on: ubuntu-latest
+
+    needs: setup
+
+    env:
+      AWS_DEFAULT_REGION: 'eu-west-1'
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+        aws-region: eu-west-1
+
+    - name: Login to Amazon ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v1
+
+    - name: Setup Functional Tests
+      id: setup-functional-tests
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        ECR_REPOSITORY: ${{ github.event.repository.name }}
+        IMAGE_TAG: ${{ github.sha }}
+      run: |
+        docker pull ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}
+        docker tag ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} api:latest
+
+    - name: Run Functional Tests
+      id: run-functional-tests
+      env:
+        GITHUB_JOB_NAME: functional-tests
+      run: |
+        DEPLOY_ENV=test docker-compose -f docker-compose.yml -f docker-test.yml up -d
+        until curl --silent -I -f --fail http://localhost:3000 ; do printf .; sleep 1; done
+        until curl -I -f --silent -XGET 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=120s'; do printf '.'; sleep 1; done
+        docker-compose exec -T api rm -r tmp/cache tmp/cache1 tmp/cache2 tmp/cache3 tmp/cache4 tmp/cache5 || true
+        docker-compose exec -T api mkdir -p tmp/cache tmp/cache1 tmp/cache2 tmp/cache3 tmp/cache4 tmp/cache5
+        docker-compose exec -T api test/setup-parallel-env.sh
+        docker-compose exec -T api bundle exec rake assets:precompile
+        docker-compose exec -T -e GITHUB_JOB_NAME=${{ env.GITHUB_JOB_NAME }} api test/run-tests.sh
+
+    - name: Save Coverage
+      id: functional-coverage
+      env:
+        CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+        GIT_COMMIT_SHA: ${{ github.sha }}
+        GIT_BRANCH: ${{ github.ref }}
+        GITHUB_JOB_NAME: functional-tests
+        GITHUB_RUN_ID: ${{ github.run_id }}
+      run: |
+        GIT_COMMITTED_AT=$(git log -1 --format=%ct)
+        docker-compose exec -T \
+        -e GIT_COMMIT_SHA=${{ env.GIT_COMMIT_SHA }} \
+        -e GIT_BRANCH=${{ env.GIT_BRANCH }} \
+        -e GITHUB_JOB_NAME=${{ env.GITHUB_JOB_NAME }} \
+        -e GITHUB_RUN_ID=${{ env.GITHUB_RUN_ID }} \
+        -e CC_TEST_REPORTER_ID=${{ env.CC_TEST_REPORTER_ID }} \
+        -e GIT_COMMITTED_AT=${GIT_COMMITTED_AT} \
+        -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
+        -e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
+        -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
+        -e AWS_REGION=${{ env.AWS_REGION }} \
+        api test/test-coverage.sh
+
+
+  test-unit:
+    runs-on: ubuntu-latest
+
+    needs: setup
+
+    env:
+      AWS_DEFAULT_REGION: 'eu-west-1'
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+        aws-region: eu-west-1
+
+    - name: Login to Amazon ECR
+      id: login-ecr
+      uses: aws-actions/amazon-ecr-login@v1
+
+    - name: Setup Unit Tests
+      id: setup-unit-tests
+      env:
+        ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+        ECR_REPOSITORY: ${{ github.event.repository.name }}
+        IMAGE_TAG: ${{ github.sha }}
+      run: |
+        docker pull ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }}
+        docker tag ${{ env.ECR_REGISTRY }}/${{ env.ECR_REPOSITORY }}:${{ env.IMAGE_TAG }} api:latest
+
+    - name: Run Unit Tests
+      id: run-unit-tests
+      env:
+        GITHUB_JOB_NAME: unit-tests
+      run: |
+        DEPLOY_ENV=test docker-compose -f docker-compose.yml -f docker-test.yml up -d
+        until curl --silent -I -f --fail http://localhost:3000 ; do printf .; sleep 1; done
+        until curl -I -f --silent  -XGET 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=120s'; do printf '.'; sleep 1; done
+        docker-compose exec -T api mkdir -p tmp/cache tmp/cache1 tmp/cache2 tmp/cache3 tmp/cache4 tmp/cache5
+        docker-compose exec -T api test/setup-parallel-env.sh
+        docker-compose exec -T api bundle exec rake assets:precompile
+        docker-compose exec -T -e GITHUB_JOB_NAME=${{ env.GITHUB_JOB_NAME }} api test/run-tests.sh
+
+    - name: Save Coverage
+      id: unit-coverage
+      env:
+        CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+        GIT_COMMIT_SHA: ${{ github.sha }}
+        GIT_BRANCH: ${{ github.ref }}
+        GITHUB_JOB_NAME: unit-tests
+        GITHUB_RUN_ID: ${{ github.run_id }}
+      run: |
+        GIT_COMMITTED_AT=$(git log -1 --format=%ct)
+        docker-compose exec -T \
+        -e GIT_COMMIT_SHA=${{ env.GIT_COMMIT_SHA }} \
+        -e GIT_BRANCH=${{ env.GIT_BRANCH }} \
+        -e GITHUB_JOB_NAME=${{ env.GITHUB_JOB_NAME }} \
+        -e GITHUB_RUN_ID=${{ env.GITHUB_RUN_ID }} \
+        -e CC_TEST_REPORTER_ID=${{ env.CC_TEST_REPORTER_ID }} \
+        -e GIT_COMMITTED_AT=${GIT_COMMITTED_AT} \
+        -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
+        -e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
+        -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
+        -e AWS_REGION=${{ env.AWS_REGION }} \
+        api test/test-coverage.sh
+
+  cleanup-and-notify:
+    runs-on: ubuntu-latest
+
+    needs:
+      - test-unit
+      - test-functional
+
+    env:
+      AWS_DEFAULT_REGION: 'eu-west-1'
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        role-to-assume: ${{ secrets.AWS_ROLE_TO_ASSUME }}
+        aws-region: eu-west-1
+
+    - name: Upload Coverage
+      id: upload-coverage
+      env:
+        CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}
+        GIT_COMMIT_SHA: ${{ github.sha }}
+        GIT_BRANCH: ${{ github.ref }}
+        GITHUB_RUN_ID: ${{ github.run_id }}
+      run: |
+        GIT_COMMITTED_AT=$(git log -1 --format=%ct)
+        docker-compose exec -T \
+        -e GIT_COMMIT_SHA=${{ env.GIT_COMMIT_SHA }} \
+        -e GIT_BRANCH=${{ env.GIT_BRANCH }} \
+        -e GITHUB_RUN_ID=${{ env.GITHUB_RUN_ID }} \
+        -e CC_TEST_REPORTER_ID=${{ env.CC_TEST_REPORTER_ID }} \
+        -e GIT_COMMITTED_AT=${GIT_COMMITTED_AT} \
+        -e AWS_ACCESS_KEY_ID=${{ env.AWS_ACCESS_KEY_ID }} \
+        -e AWS_SECRET_ACCESS_KEY=${{ env.AWS_SECRET_ACCESS_KEY }} \
+        -e AWS_SESSION_TOKEN=${{ env.AWS_SESSION_TOKEN }} \
+        -e AWS_REGION=${{ env.AWS_REGION }} \
+        api test/upload-coverage.sh
+
+    - name: Notify Slack
+      id: slack-notify
+      uses: 8398a7/action-slack@v3
+      if: ${{ always() }}
+      with:
+        status: custom
+        fields: workflow,job,commit,repo,ref,author,took,message
+        custom_payload: |
+          {
+            attachments: [{
+              color: '${{ job.status }}' === 'success' ? 'good' : '${{ job.status }}' === 'failure' ? 'danger' : 'warning',
+              text: `${process.env.AS_REPO} @ ${process.env.AS_REF}: ${process.env.AS_WORKFLOW}\n${process.env.AS_MESSAGE} (${process.env.AS_COMMIT})\nby ${process.env.AS_AUTHOR}`,
+            }]
+          }
+      env:
+        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

Dockerfile

@@ -22,16 +22,18 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
     ffmpegthumbnailer \
     git \
     graphicsmagick \
-    libidn11-dev \
     inotify-tools \
+    libidn11-dev \
     libpq-dev \
-    libtag1-dev \
     libsqlite3-dev \
+    libtag1-dev \
+    lsof \
     lz4 \
     nodejs \
+    python-pip \
+    rename \
     sqlite3 \
-    lsof \
-    python-pip
+    unzip
 
 RUN python -m pip install -U setuptools wheel
 

docker-compose.yml

@@ -65,10 +65,8 @@ services:
       - postgres
       - redis
       - minio
-    volumes:
-      - ".:/app"
     environment:
       RAILS_ENV: development
       SERVER_PORT: 3000
     networks:
-      - dev
+      - dev

docker-entrypoint.sh

@@ -3,6 +3,16 @@
 # Wait for Elasticsearch
 until curl --silent -XGET --fail http://elasticsearch:9200; do printf '.'; sleep 1; done
 
+set_config() {
+  find config/ -iname \*.example | rename -v "s/.example//g"
+}
+
+if [[ "${DEPLOY_ENV}" == "test" ]]; then
+  set_config
+  touch tmp/parallel_runtime_test.log
+  chmod +w tmp/parallel_runtime_test.log
+fi
+
 # Rake tasks
 if [ "$RAILS_ENV" == "test" ]
 then
@@ -25,4 +35,4 @@ else
   echo "pidfile '/app/tmp/pids/server-$RAILS_ENV.pid'" >> $puma
   echo "port $SERVER_PORT" >> $puma
   bundle exec puma -C $puma
-fi
+fi

docker-test.yml

@@ -1,5 +1,7 @@
 version: "2.2"
 services:
   api:
+    image: api
     environment:
       RAILS_ENV: test
+      DEPLOY_ENV: test