Merge pull request #764 from jhamrick/migrate-db

Migrate database with alembic

.coveragerc

@@ -7,4 +7,5 @@ source = nbgrader
 omit =
     /home/travis/virtualenv/python3.4.2/lib/python3.4/site-packages/*
     nbgrader/tests/*
-    nbgrader/apps/notebookapp.py
+    nbgrader/apps/notebookapp.py
+    nbgrader/alembic

nbgrader/alembic.ini

@@ -0,0 +1,67 @@
+# A generic, single database configuration.
+
+[alembic]
+# path to migration scripts
+script_location = {alembic_dir}
+sqlalchemy.url = {db_url}
+
+# template used to generate migration files
+# file_template = %%(rev)s_%%(slug)s
+
+# max length of characters to apply to the
+# "slug" field
+#truncate_slug_length = 40
+
+# set to 'true' to run the environment during
+# the 'revision' command, regardless of autogenerate
+# revision_environment = false
+
+# set to 'true' to allow .pyc and .pyo files without
+# a source .py file to be detected as revisions in the
+# versions/ directory
+# sourceless = false
+
+# version location specification; this defaults
+# to alembic/versions.  When using multiple version
+# directories, initial revisions must be specified with --version-path
+# version_locations = %(here)s/bar %(here)s/bat alembic/versions
+
+# the output encoding used when revision files
+# are written from script.py.mako
+# output_encoding = utf-8
+
+
+# Logging configuration
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S

nbgrader/alembic/README

@@ -0,0 +1 @@
+This is the alembic configuration for nbgrader database migrations.

nbgrader/alembic/env.py

@@ -0,0 +1,70 @@
+from __future__ import with_statement
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+from logging.config import fileConfig
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+target_metadata = None
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url, target_metadata=target_metadata, literal_binds=True)
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata
+        )
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

nbgrader/alembic/script.py.mako

@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

nbgrader/alembic/versions/50a4d84c131a_add_kernelspecs.py

@@ -0,0 +1,26 @@
+"""add kernelspecs
+
+Revision ID: 50a4d84c131a
+Revises: b6d005d67074
+Create Date: 2017-06-01 16:48:02.243764
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '50a4d84c131a'
+down_revision = 'b6d005d67074'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('notebook', sa.Column(
+        'kernelspec', sa.String(1024), nullable=False,
+        server_default='{}'))
+
+
+def downgrade():
+    op.drop_column('notebook', 'kernelspec')

nbgrader/alembic/versions/b6d005d67074_initial_alembic_revision.py

@@ -0,0 +1,24 @@
+"""Initial alembic revision
+
+Revision ID: b6d005d67074
+Revises: 
+Create Date: 2017-06-01 16:05:35.868678
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'b6d005d67074'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

nbgrader/api.py

@@ -3,6 +3,7 @@
 from . import utils
 
 import contextlib
+import subprocess as sp
 
 from sqlalchemy import (create_engine, ForeignKey, Column, String, Text,
     DateTime, Interval, Float, Enum, UniqueConstraint, Boolean)
@@ -15,6 +16,7 @@
 from sqlalchemy import select, func, exists, case, literal_column
 
 from uuid import uuid4
+from .dbutil import _temp_alembic_ini
 
 Base = declarative_base()
 
@@ -23,6 +25,13 @@ def new_uuid():
     return uuid4().hex
 
 
+def get_alembic_version():
+    with _temp_alembic_ini('sqlite:////tmp/gradebook.db') as alembic_ini:
+        output = sp.check_output(['alembic', '-c', alembic_ini, 'heads'])
+        head = output.decode().split("\n")[0].split(" ")[0]
+        return head
+
+
 class InvalidEntry(ValueError):
     pass
 
@@ -1029,8 +1038,16 @@ def __init__(self, db_url):
         self.db = scoped_session(sessionmaker(autoflush=True, bind=self.engine))
 
         # this creates all the tables in the database if they don't already exist
+        db_exists = len(self.engine.table_names()) > 0
         Base.metadata.create_all(bind=self.engine)
 
+        # set the alembic version if it doesn't exist
+        if not db_exists:
+            alembic_version = get_alembic_version()
+            self.db.execute("CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL);")
+            self.db.execute("INSERT INTO alembic_version (version_num) VALUES ('{}');".format(alembic_version))
+            self.db.commit()
+
     def __enter__(self):
         return self
 

nbgrader/apps/baseapp.py

@@ -10,6 +10,7 @@
 import traceback
 import logging
 import shutil
+import sqlalchemy
 
 from jupyter_core.application import JupyterApp
 from nbconvert.exporters.export import exporter_map
@@ -580,6 +581,16 @@ def _handle_failure(gd):
                 errors.append((gd['assignment_id'], gd['student_id']))
                 _handle_failure(gd)
 
+            except sqlalchemy.exc.OperationalError:
+                _handle_failure(gd)
+                self.log.error(traceback.format_exc())
+                self.fail(
+                    "There was an error accessing the nbgrader database. This "
+                    "may occur if you recently upgraded nbgrader. To resolve "
+                    "the issue, first BACK UP your database and then run the "
+                    "command `nbgrader db upgrade`."
+                )
+
             except Exception:
                 self.log.error("There was an error processing assignment: %s", assignment)
                 self.log.error(traceback.format_exc())

nbgrader/apps/dbapp.py

@@ -3,12 +3,15 @@
 
 import csv
 import os
+import shutil
 
 from textwrap import dedent
 from traitlets import default, Unicode, Bool
+from datetime import datetime
 
 from . import NbGrader
 from ..api import Gradebook, MissingEntry
+from .. import dbutil
 
 aliases = {
     'log-level': 'Application.log_level',
@@ -382,6 +385,35 @@ def start(self):
         super(DbAssignmentApp, self).start()
 
 
+class DbUpgradeApp(NbGrader):
+    """Based on the `jupyterhub upgrade-db` command found in jupyterhub.app.UpgradeDB"""
+
+    name = 'nbgrader-db-upgrade'
+    description = u'Upgrade the database schema to the latest version'
+
+    def _backup_db_file(self, db_file):
+        """Backup a database file"""
+        if not os.path.exists(db_file):
+            with Gradebook("sqlite:///{}".format(db_file)):
+                pass
+
+        timestamp = datetime.now().strftime('.%Y-%m-%d-%H%M%S.%f')
+        backup_db_file = db_file + timestamp
+        if os.path.exists(backup_db_file):
+            self.fail("backup db file already exists: %s" % backup_db_file)
+
+        self.log.info("Backing up %s => %s", db_file, backup_db_file)
+        shutil.copy(db_file, backup_db_file)
+
+    def start(self):
+        super(DbUpgradeApp, self).start()
+        if (self.coursedir.db_url.startswith('sqlite:///')):
+            db_file = self.coursedir.db_url.split(':///', 1)[1]
+            self._backup_db_file(db_file)
+        self.log.info("Upgrading %s", self.coursedir.db_url)
+        dbutil.upgrade(self.coursedir.db_url)
+
+
 class DbApp(NbGrader):
 
     name = 'nbgrader-db'
@@ -404,6 +436,14 @@ class DbApp(NbGrader):
                 """
             ).strip()
         ),
+        upgrade=(
+            DbUpgradeApp,
+            dedent(
+                """
+                Upgrade database schema to latest version.
+                """
+            ).strip()
+        ),
     )
 
     @default("classes")