Database split part III: Use two different postgresql databases.

.travis.yml

@@ -9,6 +9,7 @@ python:
   - "2.7"
 services:
   - postgresql
+  - redis-server
 addons:
   postgresql: "9.5"
   chrome: stable
@@ -44,6 +45,7 @@ env:
        DJANGO_SETTINGS_MODULE=kobo.settings.testing
        DJANGO_LANGUAGE_CODES="en ar es fr hi ku pl pt zh-hans"
        DATABASE_URL="postgres://postgres@localhost:5432/travis_ci_test"
+       REDIS_SESSION_URL="redis://localhost:6379"
        TRAVIS_NODE_VERSION="8"
        PATH=$PATH:$HOME/build/kobotoolbox/kpi/node_modules/.bin/
 install:

dependencies/pip/dev_requirements.txt

@@ -49,6 +49,7 @@ django-mptt==0.8.7
 django-oauth-toolkit==0.10.0
 django-picklefield==1.0.0  # via django-constance
 django-private-storage==2.1.2
+django-redis-sessions==0.6.1
 django-registration-redux==1.3
 django-reversion==2.0.8
 django-ses==0.8.9
@@ -137,6 +138,6 @@ werkzeug==0.14.1
 whitenoise==3.3.1
 whoosh==2.7.4
 xlrd==1.1.0
-xlsxwriter==1.0.4
+xlsxwriter==1.1.2
 xlwt==1.3.0
 zipp==0.3.3               # via importlib-metadata

dependencies/pip/external_services.txt

@@ -44,6 +44,7 @@ django-mptt==0.8.7
 django-oauth-toolkit==0.10.0
 django-picklefield==1.0.0  # via django-constance
 django-private-storage==2.1.2
+django-redis-sessions==0.6.1
 django-registration-redux==1.3
 django-reversion==2.0.8
 django-ses==0.8.9

dependencies/pip/requirements.in

@@ -45,6 +45,7 @@ django-storages
 django-private-storage
 djangorestframework
 djangorestframework-xml
+django-redis-sessions
 drf-extensions
 gunicorn
 jsonfield

dependencies/pip/requirements.txt

@@ -44,6 +44,7 @@ django-mptt==0.8.7
 django-oauth-toolkit==0.10.0
 django-picklefield==1.0.0  # via django-constance
 django-private-storage==2.1.2
+django-redis-sessions==0.6.1
 django-registration-redux==1.3
 django-reversion==2.0.8
 django-ses==0.8.9

kobo/settings/base.py

@@ -16,6 +16,7 @@
 from pymongo import MongoClient
 
 from ..static_lists import EXTRA_LANG_INFO
+from kpi.utils.redis_helper import RedisHelper
 
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -198,9 +199,17 @@ def __init__(self, *args, **kwargs):
 
 # Database
 # https://docs.djangoproject.com/en/1.7/ref/settings/#databases
+
+# @TODO add `KC_DATABASE_URL` and `KPI_DATABASE_URL`:
+#  - `kobo-install` templates
+#  - `kobo-docker` templates
+#  - `kobo-deployments` templates`
+
+kobocat_database_url = os.getenv("KC_DATABASE_URL", "sqlite:///%s/db.sqlite3" % BASE_DIR)
+
 DATABASES = {
     'default': dj_database_url.config(default="sqlite:///%s/db.sqlite3" % BASE_DIR),
-    'kobocat': dj_database_url.config(default="sqlite:///%s/db.sqlite3" % BASE_DIR),
+    'kobocat': dj_database_url.parse(kobocat_database_url)
 }
 
 DATABASE_ROUTERS = ["kpi.db_routers.DefaultDatabaseRouter"]
@@ -694,3 +703,9 @@ def __init__(self, *args, **kwargs):
 MONGO_CONNECTION = MongoClient(
     MONGO_CONNECTION_URL, j=True, tz_aware=True, connect=False)
 MONGO_DB = MONGO_CONNECTION[MONGO_DATABASE['NAME']]
+
+
+SESSION_ENGINE = "redis_sessions.session"
+SESSION_REDIS = RedisHelper.config(default="redis://redis_cache:6380/2")
+
+TESTING = False

kobo/settings/testing.py

@@ -8,5 +8,9 @@
     'default': dj_database_url.config(default="sqlite:///%s/db.sqlite3" % BASE_DIR),
 }
 
+DATABASE_ROUTERS = ["kpi.db_routers.TestingDatabaseRouter"]
+
+TESTING = True
+
 if 'KPI_AWS_STORAGE_BUCKET_NAME' in os.environ:
     PRIVATE_STORAGE_S3_REVERSE_PROXY = False

kpi/db_routers.py

@@ -35,3 +35,12 @@ def allow_migrate(self, db, app_label, model=None, **hints):
         if app_label == SHADOW_MODEL_APP_LABEL:
             return False
         return True
+
+
+class TestingDatabaseRouter(DefaultDatabaseRouter):
+
+    def db_for_read(self, model, **hints):
+        return "default"
+
+    def db_for_write(self, model, **hints):
+        return "default"

kpi/deployment_backends/kc_access/shadow_models.py

@@ -7,6 +7,7 @@
 from django.utils.translation import ugettext_lazy as _
 from django.contrib.auth.models import User, Permission
 from django.contrib.contenttypes.models import ContentType
+from django.utils import timezone
 
 from kpi.constants import SHADOW_MODEL_APP_LABEL
 
@@ -23,8 +24,9 @@ class ReadOnlyModelError(ValueError):
 
 
 class ShadowModel(models.Model):
-    ''' Allows identification of writeable and read-only shadow models'''
-
+    """
+    Allows identification of writeable and read-only shadow models
+    """
     class Meta:
         managed = False
         abstract = True
@@ -150,7 +152,7 @@ class Meta(ShadowModel.Meta):
 
 
 class UserObjectPermission(ShadowModel):
-    '''
+    """
     For the _sole purpose_ of letting us manipulate KoBoCAT
     permissions, this comprises the following django-guardian classes
     all condensed into one:
@@ -162,7 +164,7 @@ class UserObjectPermission(ShadowModel):
 
     CAVEAT LECTOR: The django-guardian custom manager,
     UserObjectPermissionManager, is NOT included!
-    '''
+    """
     permission = models.ForeignKey(Permission)
     content_type = models.ForeignKey(ContentType)
     object_pk = models.CharField(_('object ID'), max_length=255)
@@ -202,6 +204,66 @@ def save(self, *args, **kwargs):
         return super(UserObjectPermission, self).save(*args, **kwargs)
 
 
+class KCUser(ShadowModel):
+
+    username = models.CharField(_("username"), max_length=30)
+    password = models.CharField(_("password"), max_length=128)
+    last_login = models.DateTimeField(_("last login"), blank=True, null=True)
+    is_superuser = models.BooleanField(_('superuser status'), default=False)
+    first_name = models.CharField(_('first name'), max_length=30, blank=True)
+    last_name = models.CharField(_('last name'), max_length=150, blank=True)
+    email = models.EmailField(_('email address'), blank=True)
+    is_staff = models.BooleanField(_('staff status'), default=False)
+    is_active = models.BooleanField(_('active'), default=True)
+    date_joined = models.DateTimeField(_('date joined'), default=timezone.now)
+
+    class Meta(ShadowModel.Meta):
+        db_table = "auth_user"
+
+    @classmethod
+    def sync(cls, auth_user):
+        try:
+            kc_auth_user = cls.objects.get(pk=auth_user.pk)
+            assert kc_auth_user.username == auth_user.username
+        except KCUser.DoesNotExist:
+            kc_auth_user = cls(pk=auth_user.pk, username=auth_user.username)
+
+        kc_auth_user.password = auth_user.password
+        kc_auth_user.last_login = auth_user.last_login
+        kc_auth_user.is_superuser = auth_user.is_superuser
+        kc_auth_user.first_name = auth_user.first_name
+        kc_auth_user.last_name = auth_user.last_name
+        kc_auth_user.email = auth_user.email
+        kc_auth_user.is_staff = auth_user.is_staff
+        kc_auth_user.is_active = auth_user.is_active
+        kc_auth_user.date_joined = auth_user.date_joined
+
+        kc_auth_user.save()
+
+
+class KCToken(ShadowModel):
+
+    key = models.CharField(_("Key"), max_length=40, primary_key=True)
+    user = models.OneToOneField(getattr(settings, 'AUTH_USER_MODEL', 'auth.User'),
+                                related_name='auth_token',
+                                on_delete=models.CASCADE, verbose_name=_("User"))
+    created = models.DateTimeField(_("Created"), auto_now_add=True)
+
+    class Meta(ShadowModel.Meta):
+        db_table = "authtoken_token"
+
+    @classmethod
+    def sync(cls, auth_token):
+        try:
+            # Token use a One-to-One relationship on User.
+            # Thus, we can retrieve tokens from users' id. 
+            kc_auth_token = cls.objects.get(user_id=auth_token.user_id)
+        except KCToken.DoesNotExist:
+            kc_auth_token = cls(pk=auth_token.pk, user=auth_token.user)
+
+        kc_auth_token.save()
+
+
 def safe_kc_read(func):
     def _wrapper(*args, **kwargs):
         try:

kpi/deployment_backends/kobocat_backend.py

@@ -2,31 +2,25 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, unicode_literals
 
-import cStringIO
 import json
+import posixpath
 import re
-import requests
-import unicodecsv
 import urlparse
-import posixpath
 
-from bson import json_util
+import requests
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
-from django.http import HttpResponse
 from django.utils.translation import ugettext_lazy as _
-from pyxform.xls2json_backends import xls_to_dict
-from rest_framework import exceptions, status, serializers
-from rest_framework.request import Request
+from rest_framework import status
 from rest_framework.authtoken.models import Token
 
-from ..exceptions import BadFormatException, KobocatDeploymentException
-from .base_backend import BaseDeploymentBackend
-from .kc_access.utils import instance_count, last_submission_time
-from .kc_access.shadow_models import ReadOnlyInstance, ReadOnlyXForm
 from kpi.constants import INSTANCE_FORMAT_TYPE_JSON, INSTANCE_FORMAT_TYPE_XML
-from kpi.utils.mongo_helper import MongoHelper
 from kpi.utils.log import logging
+from kpi.utils.mongo_helper import MongoHelper
+from .base_backend import BaseDeploymentBackend
+from .kc_access.shadow_models import ReadOnlyInstance, ReadOnlyXForm
+from .kc_access.utils import instance_count, last_submission_time
+from ..exceptions import BadFormatException, KobocatDeploymentException
 
 
 class KobocatDeploymentBackend(BaseDeploymentBackend):

kpi/signals.py

@@ -1,20 +1,24 @@
 # -*- coding: utf-8 -*-
+from django.conf import settings
 from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
 from django.contrib.auth.models import User
+from rest_framework.authtoken.models import Token
 
 from kobo.apps.hook.models.hook import Hook
 from taggit.models import Tag
 from .models import TagUid
 from .model_utils import grant_default_model_level_perms
+from kpi.deployment_backends.kc_access.shadow_models import KCUser, KCToken
+
 
 @receiver(post_save, sender=User)
 def default_permissions_post_save(sender, instance, created, raw, **kwargs):
-    '''
+    """
     Users must have both model-level and object-level permissions to satisfy
     DRF, so assign the newly-created user all available collection and asset
     permissions at the model level
-    '''
+    """
     if raw:
         # `raw` means we can't touch (so make sure your fixtures include
         # all necessary permissions!)
@@ -25,9 +29,37 @@ def default_permissions_post_save(sender, instance, created, raw, **kwargs):
         return
     grant_default_model_level_perms(instance)
 
+
+@receiver(post_save, sender=User)
+def save_kobocat_user(sender, instance, **kwargs):
+    """
+    Sync Auth User table between KPI and KC
+    """
+    if not settings.TESTING:
+        KCUser.sync(instance)
+
+
+@receiver(post_save, sender=Token)
+def save_kobocat_token(sender, instance, **kwargs):
+    """
+    Sync AuthToken table between KPI and KC
+    """
+    if not settings.TESTING:
+        KCToken.sync(instance)
+
+
+@receiver(post_delete, sender=Token)
+def delete_kobocat_token(sender, instance, **kwargs):
+    """
+    Delete corresponding record from KC AuthToken table
+    """
+    if not settings.TESTING:
+        KCToken.objects.filter(pk=instance.pk).delete()
+
+
 @receiver(post_save, sender=Tag)
 def tag_uid_post_save(sender, instance, created, raw, **kwargs):
-    ''' Make sure we have a TagUid object for each newly-created Tag '''
+    """ Make sure we have a TagUid object for each newly-created Tag """
     if raw or not created:
         return
     TagUid.objects.get_or_create(tag=instance)
@@ -40,4 +72,4 @@ def update_kc_xform_has_kpi_hooks(sender, instance, **kwargs):
     """
     asset = instance.asset
     if asset.has_deployment:
-        asset.deployment.set_has_kpi_hooks()
+        asset.deployment.set_has_kpi_hooks()

kpi/tests/test_token.py

@@ -3,6 +3,7 @@
 from rest_framework import status
 from rest_framework.authtoken.models import Token
 
+
 class UserListTests(APITestCase):
     fixtures = ['test_data']
 

kpi/utils/redis_helper.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+import os
+import re
+
+from django.core.exceptions import ImproperlyConfigured
+
+
+class RedisHelper(object):
+    """
+    Redis's helper.
+
+    Mimics dj_database_url
+
+    """
+
+    @staticmethod
+    def config(default=None):
+        """
+        Parses `REDIS_SESSION_URL` environment variable to return a dict with
+        expected attributes for django redis session.
+
+        :return: dict
+        """
+
+        redis_connection_url = os.getenv("REDIS_SESSION_URL", default)
+        match = re.match(r"redis://(:(?P<password>[^@]*)@)?(?P<host>[^:]+):(?P<port>\d+)(/(?P<index>\d+))?",
+                        redis_connection_url)
+        if not match:
+            raise ImproperlyConfigured("Could not parse Redis session URL. Please verify 'REDIS_SESSION_URL' value")
+
+        redis_connection_dict = {
+            "host": match.group("host"),
+            "port": match.group("port"),
+            "db": match.group("index") or 0,
+            "password": match.group("password"),
+            "prefix": os.getenv("REDIS_SESSION_PREFIX", "session"),
+            "socket_timeout": os.getenv("REDIS_SESSION_SOCKET_TIMEOUT", 1),
+        }
+        return redis_connection_dict