feat: adds superset docker services and open edx integrations

* install custom requirements for mysql and OAuth2
* install custom oauth2 manager and course permissions plugin, which
  uses the Open edX mysql db to fetch user staff/superuser status
* init OAuth2 application on LMS

README.rst

@@ -6,17 +6,28 @@ Installation
 
 ::
 
-    pip install git+https://github.com/opencraft/tutor-contrib-superset
+    pip install git+https://github.com/open-craft/tutor-contrib-superset
 
 Usage
 -----
 
 ::
 
+    # Enable this plugin
     tutor plugins enable superset
+    tutor config save
+
+    # Set up SSO with Open edX
+    # (only needed if you enabled this plugin _after_ running `tutor quickstart`)
+    tutor [dev|local] init
+
+
+Connect to Superset's UI on the configured port (default is `:8088`):
+
+  http://local.overhang.io:8088
 
 
 License
 -------
 
-This software is licensed under the terms of the AGPLv3.
+This software is licensed under the terms of the AGPLv3.

tutorsuperset/plugin.py

@@ -17,6 +17,26 @@
         # Each new setting is a pair: (setting_name, default_value).
         # Prefix your setting names with 'SUPERSET_'.
         ("SUPERSET_VERSION", __version__),
+        ("SUPERSET_TAG", "latest-dev"),
+        ("SUPERSET_HOST", "{{ LMS_HOST }}"),
+        ("SUPERSET_PORT", "8088"),
+        # TODO: use our mysql database instead?
+        ("SUPERSET_DB_DIALECT", "postgresql"),
+        ("SUPERSET_DB_HOST", "superset_db"),
+        ("SUPERSET_DB_PORT", "5432"),
+        ("SUPERSET_DB_NAME", "superset"),
+        ("SUPERSET_DB_USERNAME", "superset"),
+        ("SUPERSET_OAUTH2_BASE_URL", "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}:8000"),
+        ("SUPERSET_OAUTH2_ACCESS_TOKEN_URL", "{{ SUPERSET_OAUTH2_BASE_URL }}/oauth2/access_token/"),
+        ("SUPERSET_OAUTH2_AUTHORIZE_URL", "{{ SUPERSET_OAUTH2_BASE_URL }}/oauth2/authorize/"),
+        ("SUPERSET_OPENEDX_USERNAME_URL", "{{ SUPERSET_OAUTH2_BASE_URL }}/api/user/v1/me"),
+        ("SUPERSET_OPENEDX_USER_PROFILE_URL", "{{ SUPERSET_OAUTH2_BASE_URL }}/api/user/v1/accounts/{username}"),
+        ("SUPERSET_OPENEDX_COURSES_LIST_URL",
+         "{{ SUPERSET_OAUTH2_BASE_URL }}/api/courses/v1/courses/?permissions={permission}&username={username}"),
+        # This admin account is unusable when SSO is enabled,
+        # but it's used by the "load examples" optional script.
+        ("SUPERSET_ADMIN_USERNAME", "admin"),
+        ("SUPERSET_LOAD_EXAMPLES", 0),
     ]
 )
 
@@ -27,7 +47,11 @@
         # Each new setting is a pair: (setting_name, unique_generated_value).
         # Prefix your setting names with 'SUPERSET_'.
         # For example:
-        # ("SUPERSET_SECRET_KEY", "{{ 24|random_string }}"),
+        ("SUPERSET_SECRET_KEY", "{{ 24|random_string }}"),
+        ("SUPERSET_DB_PASSWORD", "{{ 24|random_string }}"),
+        ("SUPERSET_OAUTH2_CLIENT_ID", "{{ 16|random_string }}"),
+        ("SUPERSET_OAUTH2_CLIENT_SECRET", "{{ 16|random_string }}"),
+        ("SUPERSET_ADMIN_PASSWORD", "{{ 10|random_string }}"),
     ]
 )
 
@@ -46,10 +70,14 @@
 ########################################
 
 # To run the script from templates/superset/tasks/myservice/init, add:
-# hooks.Filters.COMMANDS_INIT.add_item((
-#     "myservice",
-#     ("superset", "tasks", "myservice", "init"),
-# ))
+hooks.Filters.COMMANDS_INIT.add_item((
+    "superset",
+    ("superset", "tasks", "init-superset.sh"),
+));
+hooks.Filters.COMMANDS_INIT.add_item((
+    "lms",
+    ("superset", "tasks", "init-openedx.sh"),
+))
 
 
 ########################################
@@ -100,6 +128,119 @@
     ],
 )
 
+# docker-compose statements shared between the superset services
+SUPERSET_DOCKER_COMPOSE_SHARED = """image: apache/superset:{{ SUPERSET_TAG }}
+  environment:
+    DATABASE_DIALECT: {{ SUPERSET_DB_DIALECT }}
+    DATABASE_HOST: {{ SUPERSET_DB_HOST }}
+    DATABASE_PORT: {{ SUPERSET_DB_PORT }}
+    DATABASE_DB: {{ SUPERSET_DB_NAME }}
+    DATABASE_HOST: {{ SUPERSET_DB_HOST }}
+    DATABASE_PASSWORD: {{ SUPERSET_DB_PASSWORD }}
+    DATABASE_USER: {{ SUPERSET_DB_USERNAME }}
+    OPENEDX_MYSQL_HOST: {{ MYSQL_HOST }}
+    OPENEDX_MYSQL_PORT: {{ MYSQL_PORT }}
+    OPENEDX_MYSQL_DATABASE: {{ OPENEDX_MYSQL_DATABASE }}
+    OPENEDX_MYSQL_USERNAME: {{ OPENEDX_MYSQL_USERNAME }}
+    OPENEDX_MYSQL_PASSWORD: {{ OPENEDX_MYSQL_PASSWORD }}
+    OAUTH2_CLIENT_ID: {{ SUPERSET_OAUTH2_CLIENT_ID }}
+    OAUTH2_CLIENT_SECRET: {{ SUPERSET_OAUTH2_CLIENT_SECRET }}
+    OAUTH2_BASE_URL: {{ SUPERSET_OAUTH2_BASE_URL }}
+    OAUTH2_ACCESS_TOKEN_URL: {{ SUPERSET_OAUTH2_ACCESS_TOKEN_URL }}
+    OAUTH2_AUTHORIZE_URL: {{ SUPERSET_OAUTH2_AUTHORIZE_URL }}
+    OPENEDX_USERNAME_URL: {{ SUPERSET_OPENEDX_USERNAME_URL }}
+    OPENEDX_USER_PROFILE_URL: {{ SUPERSET_OPENEDX_USER_PROFILE_URL }}
+    OPENEDX_COURSES_LIST_URL: {{ SUPERSET_OPENEDX_COURSES_LIST_URL }}
+    SECRET_KEY: {{ SUPERSET_SECRET_KEY }}
+    PYTHONPATH: /app/pythonpath:/app/docker/pythonpath_dev
+    REDIS_HOST: superset_cache
+    REDIS_PORT: 6379
+    FLASK_ENV: production
+    SUPERSET_ENV: production
+    SUPERSET_LOAD_EXAMPLES: {{ SUPERSET_LOAD_EXAMPLES }}
+    SUPERSET_PORT: {{ SUPERSET_PORT }}
+    ADMIN_USERNAME: {{ SUPERSET_ADMIN_USERNAME }}
+    ADMIN_PASSWORD: {{ SUPERSET_ADMIN_PASSWORD }}
+    CYPRESS_CONFIG: 0
+  user: root
+  depends_on:
+    - superset-db
+    - superset-cache
+  volumes:
+    - ../../env/plugins/superset/apps/docker:/app/docker
+    - ../../env/plugins/superset/apps/pythonpath:/app/pythonpath
+    - ../../env/plugins/superset/apps/superset_home:/app/superset_home
+  restart: unless-stopped"""
+
+# Modified from https://github.com/apache/superset/blob/969c963/docker-compose-non-dev.yml
+hooks.Filters.ENV_PATCHES.add_item(
+    (
+        "local-docker-compose-services",
+        f"""
+superset-cache:
+  image: redis:latest
+  container_name: superset_cache
+  restart: unless-stopped
+  volumes:
+    - ../../data/superset/redis:/data
+
+superset-db:
+  environment:
+    POSTGRES_DB: {{{{ SUPERSET_DB_NAME }}}}
+    POSTGRES_USER: {{{{ SUPERSET_DB_USERNAME }}}}
+    POSTGRES_PASSWORD: {{{{ SUPERSET_DB_PASSWORD }}}}
+  image: postgres:10
+  container_name: superset_db
+  restart: unless-stopped
+  volumes:
+    - ../../data/superset/postgresql:/var/lib/postgresql/data
+
+superset-app:
+  {SUPERSET_DOCKER_COMPOSE_SHARED}
+  container_name: superset_app
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+  ports:
+    - 8088:{{{{ SUPERSET_PORT }}}}
+
+superset-worker:
+  {SUPERSET_DOCKER_COMPOSE_SHARED}
+  container_name: superset_worker
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "worker"]
+  healthcheck:
+    test: ["CMD-SHELL", "celery inspect ping -A superset.tasks.celery_app:app -d celery@$$HOSTNAME"]
+
+superset-worker-beat:
+  {SUPERSET_DOCKER_COMPOSE_SHARED}
+  container_name: superset_worker_beat
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "worker"]
+  healthcheck:
+    disable: true
+
+# All the superset services we need to run together
+superset:
+  {SUPERSET_DOCKER_COMPOSE_SHARED}
+  command: ["bash"]
+  depends_on:
+    - superset-app
+    - superset-worker
+    - superset-worker-beat
+        """
+    )
+)
+
+# Initialization jobs
+hooks.Filters.ENV_PATCHES.add_item(
+    (
+        "local-docker-compose-jobs-services",
+        f"""
+superset-job:
+  {SUPERSET_DOCKER_COMPOSE_SHARED}
+  depends_on:
+    - superset
+        """
+    )
+)
+
 
 ########################################
 # PATCH LOADING

tutorsuperset/templates/superset/apps/docker/docker-bootstrap.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Copied from original:
+#
+# https://github.com/apache/superset/blob/969c963/docker/docker-bootstrap.sh
+
+set -eo pipefail
+
+REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
+#
+# Make sure we have dev requirements installed
+#
+if [ -f "${REQUIREMENTS_LOCAL}" ]; then
+  echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
+  pip install -r "${REQUIREMENTS_LOCAL}"
+else
+  echo "Skipping local overrides"
+fi
+
+if [[ "${1}" == "worker" ]]; then
+  echo "Starting Celery worker..."
+  celery --app=superset.tasks.celery_app:app worker -Ofair -l INFO
+elif [[ "${1}" == "beat" ]]; then
+  echo "Starting Celery beat..."
+  celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
+elif [[ "${1}" == "app" ]]; then
+  echo "Starting web app..."
+  flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+elif [[ "${1}" == "app-gunicorn" ]]; then
+  echo "Starting web app..."
+  /usr/bin/run-server.sh
+fi

tutorsuperset/templates/superset/apps/docker/requirements-local.txt

@@ -0,0 +1,2 @@
+authlib  # OAuth2
+mysqlclient