feat: adds superset docker services and open edx integrations (#1)

feat: adds superset docker services and open edx integrations

* install custom requirements for mysql and OAuth2
* install custom oauth2 manager and course permissions plugin, which
  uses the Open edX mysql db to fetch user staff/superuser status
* init OAuth2 application on LMS
* replaces deprecated Tutor init commands with latest version
* uses open edx redis for superset caching + mysql for the backend (instead of postgres)
* handles LMS running on port :8000 in dev, or default ports in local

README.rst

@@ -6,17 +6,27 @@ Installation
 
 ::
 
-    pip install git+https://github.com/opencraft/tutor-contrib-superset
+    pip install git+https://github.com/open-craft/tutor-contrib-superset
 
 Usage
 -----
 
 ::
 
+    # Enable this plugin
     tutor plugins enable superset
+    tutor config save
+
+    # Set up SSO with Open edX
+    tutor [dev|local] do init --limit superset
+
+
+Connect to Superset's UI on the configured port (default is `:8088`):
+
+  http://local.overhang.io:8088
 
 
 License
 -------
 
-This software is licensed under the terms of the AGPLv3.
+This software is licensed under the terms of the AGPLv3.

tutorsuperset/plugin.py

@@ -17,6 +17,19 @@
         # Each new setting is a pair: (setting_name, default_value).
         # Prefix your setting names with 'SUPERSET_'.
         ("SUPERSET_VERSION", __version__),
+        ("SUPERSET_TAG", "2.0.1"),
+        ("SUPERSET_HOST", "{{ LMS_HOST }}"),
+        ("SUPERSET_PORT", "8088"),
+        ("SUPERSET_DB_DIALECT", "mysql"),
+        ("SUPERSET_DB_HOST", "{{ MYSQL_HOST }}"),
+        ("SUPERSET_DB_PORT", "{{ MYSQL_PORT }}"),
+        ("SUPERSET_DB_NAME", "superset"),
+        ("SUPERSET_DB_USERNAME", "superset"),
+        ("SUPERSET_OAUTH2_ACCESS_TOKEN_PATH", "/oauth2/access_token/"),
+        ("SUPERSET_OAUTH2_AUTHORIZE_PATH", "/oauth2/authorize/"),
+        ("SUPERSET_OPENEDX_USERNAME_PATH", "/api/user/v1/me"),
+        ("SUPERSET_OPENEDX_USER_PROFILE_PATH", "/api/user/v1/accounts/{username}"),
+        ("SUPERSET_OPENEDX_COURSES_LIST_PATH", "/api/courses/v1/courses/?permissions={permission}&username={username}"),
     ]
 )
 
@@ -27,7 +40,10 @@
         # Each new setting is a pair: (setting_name, unique_generated_value).
         # Prefix your setting names with 'SUPERSET_'.
         # For example:
-        # ("SUPERSET_SECRET_KEY", "{{ 24|random_string }}"),
+        ("SUPERSET_SECRET_KEY", "{{ 24|random_string }}"),
+        ("SUPERSET_DB_PASSWORD", "{{ 24|random_string }}"),
+        ("SUPERSET_OAUTH2_CLIENT_ID", "{{ 16|random_string }}"),
+        ("SUPERSET_OAUTH2_CLIENT_SECRET", "{{ 16|random_string }}"),
     ]
 )
 
@@ -45,11 +61,30 @@
 # INITIALIZATION TASKS
 ########################################
 
-# To run the script from templates/superset/tasks/myservice/init, add:
-# hooks.Filters.COMMANDS_INIT.add_item((
-#     "myservice",
-#     ("superset", "tasks", "myservice", "init"),
-# ))
+# To add a custom initialization task, create a bash script template under:
+# tutorsuperset/templates/superset/jobs/init/
+# and then add it to the MY_INIT_TASKS list. Each task is in the format:
+# ("<service>", ("<path>", "<to>", "<script>", "<template>"))
+MY_INIT_TASKS = [
+    # For example, to add LMS initialization steps, you could add the script template at:
+    # tutorsuperset/templates/superset/jobs/init/lms.sh
+    # And then add the line:
+    ### ("lms", ("superset", "jobs", "init", "lms.sh")),
+    ("mysql", ("superset", "jobs", "init", "init-mysql.sh")),
+    ("superset", ("superset", "jobs", "init", "init-superset.sh")),
+    ("lms", ("superset", "jobs", "init", "init-openedx.sh")),
+]
+
+# For each task added to MY_INIT_TASKS, we load the task template
+# and add it to the CLI_DO_INIT_TASKS filter, which tells Tutor to
+# run it as part of the `init` job.
+for service, template_path in MY_INIT_TASKS:
+    full_path: str = pkg_resources.resource_filename(
+        "tutorsuperset", os.path.join("templates", *template_path)
+    )
+    with open(full_path, encoding="utf-8") as init_task_file:
+        init_task: str = init_task_file.read()
+    hooks.Filters.CLI_DO_INIT_TASKS.add_item((service, init_task))
 
 
 ########################################
@@ -100,6 +135,153 @@
     ],
 )
 
+# docker-compose statements shared between the superset services
+SUPERSET_DOCKER_COMPOSE_COMMON = """image: apache/superset:{{ SUPERSET_TAG }}
+  user: root
+  depends_on:
+    - mysql
+    - redis
+  volumes:
+    - ../../env/plugins/superset/apps/docker:/app/docker
+    - ../../env/plugins/superset/apps/pythonpath:/app/pythonpath
+    - ../../env/plugins/superset/apps/superset_home:/app/superset_home
+  restart: unless-stopped
+  environment:
+    DATABASE_DIALECT: {{ SUPERSET_DB_DIALECT }}
+    DATABASE_HOST: {{ SUPERSET_DB_HOST }}
+    DATABASE_PORT: {{ SUPERSET_DB_PORT }}
+    DATABASE_DB: {{ SUPERSET_DB_NAME }}
+    DATABASE_HOST: {{ SUPERSET_DB_HOST }}
+    DATABASE_PASSWORD: {{ SUPERSET_DB_PASSWORD }}
+    DATABASE_USER: {{ SUPERSET_DB_USERNAME }}
+    OPENEDX_MYSQL_HOST: {{ MYSQL_HOST }}
+    OPENEDX_MYSQL_PORT: {{ MYSQL_PORT }}
+    OPENEDX_MYSQL_DATABASE: {{ OPENEDX_MYSQL_DATABASE }}
+    OPENEDX_MYSQL_USERNAME: {{ OPENEDX_MYSQL_USERNAME }}
+    OPENEDX_MYSQL_PASSWORD: {{ OPENEDX_MYSQL_PASSWORD }}
+    OAUTH2_CLIENT_ID: {{ SUPERSET_OAUTH2_CLIENT_ID }}
+    OAUTH2_CLIENT_SECRET: {{ SUPERSET_OAUTH2_CLIENT_SECRET }}
+    SECRET_KEY: {{ SUPERSET_SECRET_KEY }}
+    PYTHONPATH: /app/pythonpath:/app/docker/pythonpath_dev
+    REDIS_HOST: {{ REDIS_HOST }}
+    REDIS_PORT: {{ REDIS_PORT }}
+    REDIS_PASSWORD: {{ REDIS_PASSWORD }}
+    FLASK_ENV: production
+    SUPERSET_ENV: production
+    SUPERSET_HOST: {{ SUPERSET_HOST }}
+    SUPERSET_PORT: {{ SUPERSET_PORT }}
+    OAUTH2_ACCESS_TOKEN_PATH: "{{ SUPERSET_OAUTH2_ACCESS_TOKEN_PATH }}"
+    OAUTH2_AUTHORIZE_PATH: "{{ SUPERSET_OAUTH2_AUTHORIZE_PATH }}"
+    OPENEDX_USERNAME_PATH: "{{ SUPERSET_OPENEDX_USERNAME_PATH }}"
+    OPENEDX_USER_PROFILE_PATH: "{{ SUPERSET_OPENEDX_USER_PROFILE_PATH }}"
+    OPENEDX_COURSES_LIST_PATH: "{{ SUPERSET_OPENEDX_COURSES_LIST_PATH }}" """
+
+########################################
+# LOCAL services
+# Run with `tutor local ...`
+########################################
+
+# OPENEDX_LMS_ROOT_URL for local runs on default port (:80 for http, :443 for https)
+SUPERSET_DOCKER_COMPOSE_COMMON_LOCAL = (
+    SUPERSET_DOCKER_COMPOSE_COMMON +
+    '\n    OPENEDX_LMS_ROOT_URL: "{% if ENABLE_HTTPS %}https{% else %}http{% endif %}://{{ LMS_HOST }}"'
+)
+
+# Modified from https://github.com/apache/superset/blob/969c963/docker-compose-non-dev.yml
+
+hooks.Filters.ENV_PATCHES.add_item(
+    (
+        "local-docker-compose-services",
+        f"""
+superset-app:
+  {SUPERSET_DOCKER_COMPOSE_COMMON_LOCAL}
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+  ports:
+    - 8088:{{{{ SUPERSET_PORT }}}}
+  depends_on:
+    - superset-worker
+    - superset-worker-beat
+
+superset-worker:
+  {SUPERSET_DOCKER_COMPOSE_COMMON_LOCAL}
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "worker"]
+  healthcheck:
+    test: ["CMD-SHELL", "celery inspect ping -A superset.tasks.celery_app:app -d celery@$$HOSTNAME"]
+
+superset-worker-beat:
+  {SUPERSET_DOCKER_COMPOSE_COMMON_LOCAL}
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "worker"]
+  healthcheck:
+    disable: true
+        """
+    )
+)
+
+# Initialization jobs
+hooks.Filters.ENV_PATCHES.add_item(
+    (
+        "local-docker-compose-jobs-services",
+        f"""
+superset-job:
+  {SUPERSET_DOCKER_COMPOSE_COMMON_LOCAL}
+  depends_on:
+    - superset-app
+        """
+    )
+)
+
+########################################
+# DEV services
+# Run with `tutor dev ...`
+########################################
+
+# OPENEDX_LMS_ROOT_URL for dev must include the LMS dev port
+SUPERSET_DOCKER_COMPOSE_COMMON_DEV = (
+    SUPERSET_DOCKER_COMPOSE_COMMON +
+    '\n    OPENEDX_LMS_ROOT_URL: "http://{{ LMS_HOST }}:8000"'
+)
+
+# Modified from https://github.com/apache/superset/blob/969c963/docker-compose-non-dev.yml
+hooks.Filters.ENV_PATCHES.add_item(
+    (
+        "local-docker-compose-dev-services",
+        f"""
+superset-app:
+  {SUPERSET_DOCKER_COMPOSE_COMMON_DEV}
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+  ports:
+    - 8088:{{{{ SUPERSET_PORT }}}}
+  depends_on:
+    - superset-worker
+    - superset-worker-beat
+
+superset-worker:
+  {SUPERSET_DOCKER_COMPOSE_COMMON_DEV}
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "worker"]
+  healthcheck:
+    test: ["CMD-SHELL", "celery inspect ping -A superset.tasks.celery_app:app -d celery@$$HOSTNAME"]
+
+superset-worker-beat:
+  {SUPERSET_DOCKER_COMPOSE_COMMON_DEV}
+  command: ["bash", "/app/docker/docker-bootstrap.sh", "worker"]
+  healthcheck:
+    disable: true
+        """
+    )
+)
+
+# Initialization jobs
+hooks.Filters.ENV_PATCHES.add_item(
+    (
+        "local-docker-compose-dev-jobs-services",
+        f"""
+superset-job:
+  {SUPERSET_DOCKER_COMPOSE_COMMON_DEV}
+  depends_on:
+    - superset-app
+        """
+    )
+)
 
 ########################################
 # PATCH LOADING

tutorsuperset/templates/superset/apps/docker/docker-bootstrap.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Copied from original:
+#
+# https://github.com/apache/superset/blob/969c963/docker/docker-bootstrap.sh
+
+set -eo pipefail
+
+REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
+#
+# Make sure we have dev requirements installed
+#
+if [ -f "${REQUIREMENTS_LOCAL}" ]; then
+  echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
+  pip install -r "${REQUIREMENTS_LOCAL}"
+else
+  echo "Skipping local overrides"
+fi
+
+if [[ "${1}" == "worker" ]]; then
+  echo "Starting Celery worker..."
+  celery --app=superset.tasks.celery_app:app worker -Ofair -l INFO
+elif [[ "${1}" == "beat" ]]; then
+  echo "Starting Celery beat..."
+  celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
+elif [[ "${1}" == "app" ]]; then
+  echo "Starting web app..."
+  flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+elif [[ "${1}" == "app-gunicorn" ]]; then
+  echo "Starting web app..."
+  /usr/bin/run-server.sh
+fi

tutorsuperset/templates/superset/apps/docker/requirements-local.txt

@@ -0,0 +1,2 @@
+authlib  # OAuth2
+mysqlclient