CI stuff

circle: Run unit tests + postgres on circle for outside contributors automatically circle+azure: skip non-postgres integration tests on for outside contributors circle+azure: build wheels after tests pass Update the docker image so venvs can pip install -U pip
dbt-labs · Feb 6, 2020 · f171dd3 · f171dd3
1 parent 032d77d
commit f171dd3
Show file tree

Hide file tree

Showing 5 changed files with 145 additions and 43 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,8 +1,34 @@
 version: 2
 jobs:
   unit:
+    docker: &test_only
+      - image: fishtownjacob/test-container:3
+        environment:
+          DBT_INVOCATION_ENV: circle
+    steps:
+      - checkout
+      - run: tox -e flake8,mypy,unit-py36,unit-py38
+  build-wheels:
+    docker: *test_only
+    steps:
+      - checkout
+      - run:
+          name: Build wheels
+          command: |
+            $PYTHON_BIN -m pip install -U pip setuptools
+            $PYTHON_BIN -m pip install -r requirements.txt
+            $PYTHON_BIN -m pip install -r dev_requirements.txt
+            /bin/bash ./scripts/build-wheels.sh
+          environment:
+            PYTHON_BIN: /home/tox/venv3.8/bin/python
+      - store_artifacts:
+          path: ./dist
+          destination: dist
+  integration-postgres-py36:
     docker: &test_and_postgres
-      - image: fishtownjacob/test-container:2
+      - image: fishtownjacob/test-container:3
+        environment:
+          DBT_INVOCATION_ENV: circle
       - image: postgres
         name: database
         environment: &pgenv
@@ -19,20 +45,13 @@ jobs:
             PGUSER: root
             PGPASSWORD: password
             PGDATABASE: postgres
-      - run: tox -e flake8,mypy,unit-py36,unit-py38
-  integration-postgres-py36:
-    docker: *test_and_postgres
-    steps:
-      - checkout
-      - run: *setupdb
       - run:
           name: Run tests
           command: tox -e integration-postgres-py36
       - store_artifacts:
           path: ./logs
   integration-snowflake-py36:
-    docker: &test_only
-      - image: fishtownjacob/test-container:2
+    docker: *test_only
     steps:
       - checkout
       - run:
@@ -109,21 +128,42 @@ workflows:
       - integration-redshift-py36:
           requires:
             - integration-postgres-py36
+          filters: &no_forks
+            branches:
+              # Forked pull requests have CIRCLE_BRANCH set to pull/XXX
+              ignore: /pull\/[0-9]+/
       - integration-bigquery-py36:
           requires:
             - integration-postgres-py36
+          filters: *no_forks
       - integration-snowflake-py36:
           requires:
             - integration-postgres-py36
+          filters: *no_forks
       - integration-postgres-py38:
           requires:
             - unit
       - integration-redshift-py38:
           requires:
             - integration-postgres-py38
+          filters: *no_forks
       - integration-bigquery-py38:
           requires:
             - integration-postgres-py38
+          filters: *no_forks
       - integration-snowflake-py38:
           requires:
             - integration-postgres-py38
+          filters: *no_forks
+      - build-wheels:
+          requires:
+            - unit
+            - integration-postgres-py36
+            - integration-redshift-py36
+            - integration-bigquery-py36
+            - integration-snowflake-py36
+            - integration-postgres-py38
+            - integration-redshift-py38
+            - integration-bigquery-py38
+            - integration-snowflake-py38
+          filters: *no_forks
diff --git a/Dockerfile b/Dockerfile
@@ -26,19 +26,23 @@ RUN pip3 install tox wheel
 
 RUN python2.7 -m pip install virtualenv wheel && \
     python2.7 -m virtualenv /home/tox/venv2.7 && \
-    /home/tox/venv2.7/bin/python -m pip install -U pip tox
+    /home/tox/venv2.7/bin/python -m pip install -U pip tox && \
+    chown -R dbt_test_user /home/tox/venv2.7
 
 RUN python3.6 -m pip install -U pip wheel && \
     python3.6 -m venv /home/tox/venv3.6 && \
-    /home/tox/venv3.6/bin/python -m pip install -U pip tox
+    /home/tox/venv3.6/bin/python -m pip install -U pip tox && \
+    chown -R dbt_test_user /home/tox/venv3.6
 
 RUN python3.7 -m pip install -U pip wheel && \
     python3.7 -m venv /home/tox/venv3.7 && \
-    /home/tox/venv3.7/bin/python -m pip install -U pip tox
+    /home/tox/venv3.7/bin/python -m pip install -U pip tox && \
+    chown -R dbt_test_user /home/tox/venv3.7
 
 RUN python3.8 -m pip install -U pip wheel && \
     python3.8 -m venv /home/tox/venv3.8 && \
-    /home/tox/venv3.8/bin/python -m pip install -U pip tox
+    /home/tox/venv3.8/bin/python -m pip install -U pip tox && \
+    chown -R dbt_test_user /home/tox/venv3.8
 
 USER dbt_test_user
 

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -32,40 +32,42 @@ jobs:
   dependsOn: UnitTest
 
   steps:
-  - pwsh: |
-      choco install postgresql --params '/Password:password' --params-global --version 10.6
-      Set-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host   all   all   ::1/128        trust"
-      Add-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host   all   all   127.0.0.1/32   trust"
-      # the service name is "postgresql-x64-10", conveniently it's both the display name and the actual name
-      Restart-Service postgresql-x64-10
-
-      & "C:\program files\postgresql\10\bin\createdb.exe" -U postgres dbt
-      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "CREATE ROLE root WITH PASSWORD 'password';"
-      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "ALTER ROLE root WITH LOGIN;"
-      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
-      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;"
-      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
-      & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
-    displayName: Install postgresql and set up database
-
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.7'
-      architecture: 'x64'
-
-  - script: python -m pip install --upgrade pip && pip install tox
-    displayName: 'Install dependencies'
-
-  - script: python -m tox -e pywin-postgres
-    displayName: Run integration tests
+  - pwsh: echo "True"
+    displayName: Skip everything
+  # - pwsh: |
+  #     choco install postgresql --params '/Password:password' --params-global --version 10.6
+  #     Set-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host   all   all   ::1/128        trust"
+  #     Add-Content "c:\program files\postgresql\10\data\pg_hba.conf" "host   all   all   127.0.0.1/32   trust"
+  #     # the service name is "postgresql-x64-10", conveniently it's both the display name and the actual name
+  #     Restart-Service postgresql-x64-10
+
+  #     & "C:\program files\postgresql\10\bin\createdb.exe" -U postgres dbt
+  #     & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "CREATE ROLE root WITH PASSWORD 'password';"
+  #     & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "ALTER ROLE root WITH LOGIN;"
+  #     & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "GRANT CREATE, CONNECT ON DATABASE dbt TO root WITH GRANT OPTION;"
+  #     & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "CREATE ROLE noaccess WITH PASSWORD 'password' NOSUPERUSER;"
+  #     & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "ALTER ROLE noaccess WITH LOGIN;"
+  #     & "C:\program files\postgresql\10\bin\psql.exe"  -U postgres -c "GRANT CONNECT ON DATABASE dbt TO noaccess;"
+  #   displayName: Install postgresql and set up database
+
+  # - task: UsePythonVersion@0
+  #   inputs:
+  #     versionSpec: '3.7'
+  #     architecture: 'x64'
+
+  # - script: python -m pip install --upgrade pip && pip install tox
+  #   displayName: 'Install dependencies'
+
+  # - script: python -m tox -e pywin-postgres
+  #   displayName: Run integration tests
 
 # These three are all similar except secure environment variables, which MUST be passed along to their tasks,
 # but there's probably a better way to do this!
 - job: SnowflakeIntegrationTest
   pool:
     vmImage: 'vs2017-win2016'
   dependsOn: PostgresIntegrationTest
-
+  condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true'))
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -87,13 +89,12 @@ jobs:
   pool:
     vmImage: 'vs2017-win2016'
   dependsOn: PostgresIntegrationTest
-
+  condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true'))
   steps:
   - task: UsePythonVersion@0
     inputs:
       versionSpec: '3.7'
       architecture: 'x64'
-
   - script: python -m pip install --upgrade pip && pip install tox
     displayName: 'Install dependencies'
   - script: python -m tox -e pywin-bigquery
@@ -105,7 +106,7 @@ jobs:
   pool:
     vmImage: 'vs2017-win2016'
   dependsOn: PostgresIntegrationTest
-
+  condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true'))
   steps:
   - task: UsePythonVersion@0
     inputs:
@@ -123,3 +124,33 @@ jobs:
       REDSHIFT_TEST_PORT: $(REDSHIFT_TEST_PORT)
       REDSHIFT_TEST_HOST: $(REDSHIFT_TEST_HOST)
     displayName: Run integration tests
+
+- job: BuildWheel
+  pool:
+    vmImage: 'vs2017-win2016'
+  dependsOn:
+    - UnitTest
+    - PostgresIntegrationTest
+    - SnowflakeIntegrationTest
+    - BigQueryIntegrationTest
+    - RedshiftIntegrationTest
+  condition: and(succeeded(), ne(variables['SYSTEM.PULLREQUEST.ISFORK'], 'true'))
+  steps:
+    - task: UsePythonVersion@0
+      inputs:
+        versionSpec: '3.7'
+        architecture: 'x64'
+    - script: python -m pip install --upgrade pip setuptools && python -m pip install -r requirements.txt && python -m pip install -r dev_requirements.txt
+      displayName: Install dependencies
+    - task: ShellScript@2
+      inputs:
+        scriptPath: scripts/build-wheels.sh
+    - task: CopyFiles@2
+      inputs:
+        sourceFolder: '$(Build.SourcesDirectory)\dist\'
+        contents: '(*.whl|*.tar.gz)'
+        TargetFolder: '$(Build.ArtifactStagingDirectory)'
+    - task: PublishBuildArtifacts@1
+      inputs:
+        pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+        artifactName: dists
diff --git a/dev_requirements.txt b/dev_requirements.txt
@@ -8,3 +8,4 @@ ipdb
 pytest-xdist>=1.28.0,<2
 flaky>=3.5.3,<4
 mypy==0.761
+wheel
diff --git a/scripts/build-wheels.sh b/scripts/build-wheels.sh
@@ -0,0 +1,26 @@
+#!/bin/bash -eo pipefail
+
+DBT_PATH="$( cd "$(dirname "$0")/.." ; pwd -P )"
+
+PYTHON_BIN=${PYTHON_BIN:-python}
+
+echo $SCRIPTPATH
+echo $PYTHON_BIN
+
+set -x
+
+rm -rf "$DBT_PATH"/dist
+mkdir -p "$DBT_PATH"/dist
+
+for SUBPATH in core plugins/postgres plugins/redshift plugins/bigquery plugins/snowflake
+do
+    rm -rf "$DBT_PATH"/"$SUBPATH"/dist
+    cd "$DBT_PATH"/"$SUBPATH"
+    $PYTHON_BIN setup.py sdist bdist_wheel
+    cp -r "$DBT_PATH"/"$SUBPATH"/dist/* "$DBT_PATH"/dist/
+done
+
+cd "$DBT_PATH"
+$PYTHON_BIN setup.py sdist bdist_wheel
+
+set +x