build-and-test.yml

name: Build and Test
on: [push, pull_request, workflow_dispatch]
jobs:
  build-and-test:
    runs-on: ubuntu-20.04
    env:
      SPARK_LOCAL_IP: localhost
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AZURE_TEST_ACCOUNT_KEY: ${{ secrets.AZURE_TEST_ACCOUNT_KEY }}
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_service_account_key.json
      GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2
      - name: Cache Scala, SBT
        uses: actions/cache@v2
        with:
          path: |
            ~/.sbt
            ~/.ivy2
            ~/.cache/coursier
          key: build-and-test-scala
      - name: Install Java 8
        uses: actions/setup-java@v1
        with:
          java-version: '8'
      - run: ./build/sbt test

  python:
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.7, 3.8, 3.9]
        include:
          - pandas-version: 1.2.4
            pyarrow-version: 4.0.0
    env:
      PYTHON_VERSION: ${{ matrix.python-version }}
      PANDAS_VERSION: ${{ matrix.pandas-version }}
      PYARROW_VERSION: ${{ matrix.pyarrow-version }}
      SPARK_LOCAL_IP: localhost
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AZURE_TEST_ACCOUNT_KEY: ${{ secrets.AZURE_TEST_ACCOUNT_KEY }}
      GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_service_account_key.json
      GOOGLE_SERVICE_ACCOUNT_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_KEY }}
      # Github Actions' default miniconda
      CONDA_PREFIX: /usr/share/miniconda
    steps:
      - name: Checkout repository
        uses: actions/checkout@v2
      - name: Cache Scala, SBT
        uses: actions/cache@v2
        with:
          path: |
            ~/.sbt
            ~/.ivy2
            ~/.cache/coursier
          key: build-and-test-python
      - name: Install Java 8
        uses: actions/setup-java@v1
        with:
          java-version: '8'
      - name: Install dependencies
        run: |
          # See also https://github.com/conda/conda/issues/7980
          source "$CONDA_PREFIX/etc/profile.d/conda.sh"
          conda update -q conda
          conda create -c conda-forge -q -n test-environment python=$PYTHON_VERSION
          conda activate test-environment
          conda config --env --add pinned_packages python=$PYTHON_VERSION
          conda config --env --add pinned_packages pandas==$PANDAS_VERSION
          conda config --env --add pinned_packages pyarrow==$PYARROW_VERSION
          conda install -c conda-forge --yes pandas==$PANDAS_VERSION pyarrow==$PYARROW_VERSION
          sed -i -e "/pandas/d" -e "/pyarrow/d" python/requirements-dev.txt
          conda install -c conda-forge --yes --file python/requirements-dev.txt
          conda list
      - name: Build Server
        run: ./build/sbt package
      - name: Run tests
        run: |
          # See also https://github.com/conda/conda/issues/7980
          source "$CONDA_PREFIX/etc/profile.d/conda.sh"
          conda activate test-environment
          ./python/dev/lint-python
          ./python/dev/pytest