Refactor PullRequest Image Building and Testing GitHub Actions CI (#118)

enables building and testing images from PRs.

.github/workflows/pull_request.yml

@@ -18,6 +18,15 @@ jobs:
     steps:
     - name: Checkout Repository
       uses: actions/checkout@v2
+    - name: Set Job Environment Variables
+      run: |
+        CALVER="$( date -u '+%Y.%m.%d' )"
+        SHA="$( git rev-parse --short ${GITHUB_SHA} )"
+        DOCKER_TAG="${CALVER}-${SHA}"
+        IMAGE_SPEC="${IMAGE_PREFIX}${IMAGE}:${DOCKER_TAG}"
+        ONBUILD_IMAGE_SPEC=${IMAGE_PREFIX}${IMAGE}-onbuild:${DOCKER_TAG}
+        echo "::set-env name=IMAGE_SPEC::${IMAGE_SPEC}"
+        echo "::set-env name=ONBUILD_IMAGE_SPEC::${ONBUILD_IMAGE_SPEC}"
     - name: Set Up Python 3.7
       uses: actions/setup-python@v1
       with:
@@ -37,19 +46,25 @@ jobs:
         docker version
     - name: Build Image
       run: |
-        python3 build.py --pr --image-prefix=${IMAGE_PREFIX} ${IMAGE}
-    - name: Authenticate with DockerHub
+        python3 build.py --image-prefix=${IMAGE_PREFIX} ${IMAGE}
+    - name: Test Image
       run: |
-        echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-    - name: Push Image to DockerHub
-      # NOTE: could export tag names from build.py?
+        docker run ${IMAGE_SPEC} binder/verify
+        docker run ${ONBUILD_IMAGE_SPEC} binder/verify
+    - name: Save Docker Image
       run: |
-        CALVER="$( date -u '+%Y.%m.%d' )"
-        SHA="$( git rev-parse --short ${GITHUB_SHA} )"
-        DOCKER_TAG=${CALVER}-${SHA}-PR
-
-        docker push ${IMAGE_PREFIX}${IMAGE}:${DOCKER_TAG}
-        docker push ${IMAGE_PREFIX}${IMAGE}-onbuild:${DOCKER_TAG}
+        docker save ${ONBUILD_IMAGE_SPEC} | gzip > base-image.tar.gz
+        docker run ${ONBUILD_IMAGE_SPEC} conda list > conda-packages.txt
+    - name: Archive Conda Package List
+      uses: actions/upload-artifact@v1
+      with:
+        name: ${{env.IMAGE}}-packages
+        path: conda-packages.txt
+    - name: Upload Base-Notebook Image
+      uses: actions/upload-artifact@v1
+      with:
+        name: base-image
+        path: base-image.tar.gz
 
   onbuild-images:
     needs: base-image
@@ -62,6 +77,23 @@ jobs:
     steps:
     - name: Checkout Repo
       uses: actions/checkout@v2
+    - name: Set Job Environment Variables
+      run: |
+        CALVER="$( date -u '+%Y.%m.%d' )"
+        SHA="$( git rev-parse --short ${GITHUB_SHA} )"
+        DOCKER_TAG="${CALVER}-${SHA}"
+        IMAGE_SPEC="${IMAGE_PREFIX}${{ matrix.IMAGE }}:${DOCKER_TAG}"
+        echo "::set-env name=IMAGE_SPEC::${IMAGE_SPEC}"
+    - name: Download Base-Noteook Docker Image
+      uses: actions/download-artifact@v1
+      with:
+        name: base-image
+        path: ./artifact
+    - name: Load Docker Image
+      run: |
+        ls -l *
+        docker load < ./artifact/base-image.tar.gz
+        docker images
     - name: Set Up Python 3.7
       uses: actions/setup-python@v1
       with:
@@ -79,18 +111,19 @@ jobs:
         pip install -r requirements.txt
         repo2docker --version
         docker version
-    - name: Authenticate with DockerHub
-      run: |
-        echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
     - name: Build Image
       run: |
-        python3 build.py --pr --image-prefix=${IMAGE_PREFIX} ${{ matrix.IMAGE }}
-    - name: Push Image to DockerHub
+        python3 build.py --image-prefix=${IMAGE_PREFIX} ${{ matrix.IMAGE }}
+    - name: Test Image
       run: |
-        IMAGE=${{ matrix.IMAGE }}
-        CALVER="$( date -u '+%Y.%m.%d' )"
-        SHA="$( git rev-parse --short ${GITHUB_SHA} )"
-        DOCKER_TAG=${CALVER}-${SHA}-PR
-
-        docker push ${IMAGE_PREFIX}${IMAGE}:${DOCKER_TAG}
-        docker push ${IMAGE_PREFIX}${IMAGE}-onbuild:${DOCKER_TAG}
+        docker run ${IMAGE_SPEC} binder/verify
+        docker run ${IMAGE_SPEC} conda list > conda-packages.txt
+    - name: Archive Conda Package List
+      uses: actions/upload-artifact@v1
+      with:
+        name: ${{matrix.IMAGE}}-packages
+        path: conda-packages.txt
+      # NOTE: test onbuild images as well?
+      # NOTE: push to image registry for testing?
+      # NOTE: should just re-tag image and push to dockerhub when merged
+      # NOTE: Add cleanup step to delete artifacts

README.md

@@ -2,7 +2,7 @@ Pangeo Stacks
 =============
 *Currated Docker images for use with Jupyter and [Pangeo](http://pangeo.io/)*
 
-[![Build Status](https://travis-ci.org/pangeo-data/pangeo-stacks.svg?branch=master)](https://travis-ci.org/pangeo-data/pangeo-stacks)
+![Action Status](https://github.com/pangeo-data/pangeo-stacks/workflows/MasterBuild/badge.svg)
 
 This repository contains a few currated Docker images that can be used with deployments of the [Pangeo Helm Chart](https://github.com/pangeo-data/helm-chart). Each of the images in this repository are configured and built using [repo2docker](https://repo2docker.readthedocs.io) and are continuously deployed to DockerHub. Importantly, each image built in this repo includes the minimum required libraries to do scalable computations with Pangeo (via dask-kubernetes).
 

base-notebook/README.md

@@ -1,3 +1,3 @@
 # Pangeo/Base-Notebook
 
-A bare-bones image with Jupyter and Dask.
+A bare-bones image with Jupyter and Dask

build.py

@@ -47,7 +47,6 @@ def image_exists_in_registry(client, image_spec):
 def docker_build(image_spec, path, build_args):
     pwd = os.getcwd()
     print(f'Building {image_spec}')
-    print(f'PWD={pwd}')
     os.system('docker images')
 
     if os.path.exists(os.path.join(path, 'Dockerfile')):
@@ -60,16 +59,6 @@ def docker_build(image_spec, path, build_args):
         cmd += f' --build-arg {k}={v}'
     print(cmd)
     os.system(cmd)
-    # NOTE: not sure why this is failing to run, but no Error
-    #cmd = [
-    #    'docker', 'build',
-    #    '-t', image_spec,
-    #    '-f', df_path
-    #]
-    #for k, v in build_args.items():
-    #    cmd += [' --build-arg', f'{k}={v}']
-    #command.append(path)
-    #subprocess.check_call(cmd, shell=True)
 
 
 def r2d_build(image, image_spec, cache_from):
@@ -83,19 +72,6 @@ def r2d_build(image, image_spec, cache_from):
 
     r2d.initialize()
     r2d.build()
-    # NOTE: not sure why this is failing to run, but no Error
-    # if os.path.exists(os.path.join(r2d.subdir, 'binder/verify')):
-    #     print(f'Validating {image_spec}')
-    #     # Validate the built image
-    #     subprocess.check_call([
-    #         'docker',
-    #         'run',
-    #         '-i', '-t',
-    #         f'{r2d.output_image_spec}',
-    #         'binder/verify'
-    #     ], shell=True)
-    # else:
-    #     print(f'No verify script found for {image_spec}')
 
 
 def main():
@@ -104,12 +80,6 @@ def main():
         'image',
         help='Image to build. Subdirectory with this name must exist'
     )
-    argparser.add_argument(
-        '--pr',
-        action="store_true",
-        default=False,
-        help='Append PR to image names if run in pull request workflow'
-    )
     argparser.add_argument(
         '--image-prefix',
         help='Prefix for image to be built. Usually contains registry url and name',
@@ -132,8 +102,6 @@ def main():
         # Stick to UTC for calver
         existing_calver = date.astimezone(pytz.utc).strftime('%Y.%m.%d')
         existing_image_spec = f'{image_name}:{existing_calver}-{sha}'
-        if args.pr:
-            existing_image_spec+='-PR'
         if image_exists_in_registry(client, existing_image_spec):
             print(f'Re-using cache from {existing_image_spec}')
             cache_from = [existing_image_spec]
@@ -146,8 +114,7 @@ def main():
     calver = datetime.utcnow().strftime('%Y.%m.%d')
     sha = next(iter(sha_date))
     tag = f'{calver}-{sha}'
-    if args.pr:
-        tag+='-PR'
+    image_spec = f'{image_name}:{tag}'
     dockerfile_paths = [
         os.path.join(args.image, 'binder', 'Dockerfile'),
         os.path.join(args.image, 'Dockerfile')
@@ -157,7 +124,7 @@ def main():
         # Can be just r2d once we can pass arbitrary BUILD ARGs to it
         # https://github.com/jupyter/repo2docker/issues/645
         docker_build(
-            f'{image_name}:{tag}',
+            image_spec,
             args.image,
             {
                 'VERSION': tag
@@ -167,15 +134,15 @@ def main():
         # Build regular image
         r2d_build(
             args.image,
-            f'{image_name}:{tag}',
+            image_spec,
             cache_from
         )
 
     # Build onbuild image
     docker_build(
         f'{image_name}-onbuild:{tag}',
         'onbuild',
-        {'BASE_IMAGE_SPEC': f'{image_name}:{tag}'}
+        {'BASE_IMAGE_SPEC': image_spec}
     )
 
 

onbuild/Dockerfile

@@ -13,8 +13,7 @@ ONBUILD COPY . ${REPO_DIR}
 ONBUILD COPY . ${REPO_DIR}/.onbuild-child
 ONBUILD RUN chown -R 1000:1000 ${REPO_DIR}
 ONBUILD RUN /usr/local/bin/r2d_overlay.py build
-ONBUILD RUN rm -rf ${REPO_DIR}/.onbuild-child
 
 ONBUILD USER ${NB_USER}
 
-ENTRYPOINT /usr/local/bin/r2d_overlay.py start
+ENTRYPOINT ["/usr/local/bin/r2d_overlay.py", "start"]

onbuild/r2d_overlay.py

@@ -23,24 +23,11 @@ def _pre_exec():
         return func
     return wrap
 
-def binder_path(path):
-    has_binder = os.path.isdir(os.path.join(ONBUILD_CONTENTS_DIR, "binder"))
-    has_dotbinder = os.path.isdir(os.path.join(ONBUILD_CONTENTS_DIR, ".binder"))
-
-    if has_binder and has_dotbinder:
-        raise RuntimeError(
-            "The repository contains both a 'binder' and a '.binder' "
-            "directory. However they are exclusive."
-        )
-
-    if has_dotbinder:
-        binder_dir =  ".binder"
-    elif has_binder:
-        binder_dir = "binder"
-    else:
-        binder_dir = ""
 
-    return os.path.join(ONBUILD_CONTENTS_DIR, binder_dir, path)
+def binder_path(path):
+    if os.path.exists(os.path.join(ONBUILD_CONTENTS_DIR, 'binder')):
+        return os.path.join(ONBUILD_CONTENTS_DIR, 'binder', path)
+    return os.path.join(ONBUILD_CONTENTS_DIR, path)
 
 
 @become(NB_UID)
@@ -122,45 +109,38 @@ def build():
                     ['/bin/bash', '-c', command], preexec_fn=applicator._pre_exec
                 )
 
-def start():
-    #Enable additional actions in the future
-    applicators = [apply_start]
-
-    for applicator in applicators:
-        commands = applicator()
-
-        if commands:
-            for command in commands:
-                subprocess.check_call(
-                    ['/bin/bash', '-c', command], preexec_fn=applicator._pre_exec
-                )
-
 @become(NB_UID)
-def apply_start():
+def start(args):
     st_path = binder_path('start')
 
     if os.path.exists(st_path):
-        return [
-            f'chmod +x {st_path}',
-            # since st_path is a fully qualified path, no need to add a ./
-            f'{st_path}'
-        ]
+        subprocess.check_call(['chmod', '+x', st_path])
     else:
-        return [f'/usr/local/bin/repo2docker-entrypoint']
+        st_path = '/usr/local/bin/repo2docker-entrypoint'
+
+    os.execv(st_path, [st_path] + args)
 
 def main():
     argparser = argparse.ArgumentParser()
-    argparser.add_argument(
-        'action',
-        choices=('build', 'start')
+    subparsers = argparser.add_subparsers(dest='action')
+
+    build_parser = subparsers.add_parser('build')
+
+    start_parser = subparsers.add_parser('start')
+    start_parser.add_argument(
+        'args',
+        # We want REMAINDER instead '*' here so argparse passes through args starting with '-'
+        # Without this, if you try to do `r2d_overlay.py start /bin/bash -c "echo hi"`
+        # will fail, since argparse will try to parse the '-c'
+        nargs=argparse.REMAINDER
     )
 
     args = argparser.parse_args()
 
     if args.action == 'build':
         build()
     elif args.action == 'start':
-        start()
+        start(args.args)
 
 
 if __name__ == '__main__':

pangeo-esip/binder/tests/test_dask_env.py

@@ -0,0 +1,20 @@
+import pytest
+
+
+@pytest.fixture(scope='module')
+def client():
+    from dask.distributed import Client
+    with Client(n_workers=4) as dask_client:
+        yield dask_client
+
+def test_check_dask_version(client):
+    print(client)
+    versions = client.get_versions(check=True)
+
+
+def test_dask_config():
+    import dask
+
+    assert '/srv/conda/envs/notebook/etc/dask' in dask.config.paths
+    assert dask.config.config['labextension']['factory']['class'] == 'KubeCluster'
+    assert 'worker-template' in dask.config.config['kubernetes']

pangeo-esip/binder/tests/test_imports.py

@@ -0,0 +1,15 @@
+import importlib
+
+import pytest
+
+packages = [
+    # these are problem libraries that don't always seem to import, mostly due
+    # to dependencies outside the python world
+    'netCDF4', 'h5py', 'rasterio', 'geopandas', 'cartopy', 'geoviews',
+    # these are things we can't live without, just to be safe
+    'gcsfs', 's3fs', 'xarray', 'intake', 'dask', 'distributed',
+    ]
+
+@pytest.mark.parametrize('package_name', packages, ids=packages)
+def test_import(package_name):
+    importlib.import_module(package_name)