diff --git a/.travis.yml b/.travis.yml index 4593ff52ee..47e01ae50c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,7 @@ python: - "2.7" services: - postgresql + - redis-server addons: postgresql: "9.5" chrome: stable @@ -44,6 +45,7 @@ env: DJANGO_SETTINGS_MODULE=kobo.settings.testing DJANGO_LANGUAGE_CODES="en ar es fr hi ku pl pt zh-hans" DATABASE_URL="postgres://postgres@localhost:5432/travis_ci_test" + REDIS_SESSION_URL="redis://localhost:6379" TRAVIS_NODE_VERSION="8" PATH=$PATH:$HOME/build/kobotoolbox/kpi/node_modules/.bin/ install: diff --git a/dependencies/pip/dev_requirements.txt b/dependencies/pip/dev_requirements.txt index f53b10f6a2..3ad1d0238b 100644 --- a/dependencies/pip/dev_requirements.txt +++ b/dependencies/pip/dev_requirements.txt @@ -49,6 +49,7 @@ django-mptt==0.8.7 django-oauth-toolkit==0.10.0 django-picklefield==1.0.0 # via django-constance django-private-storage==2.1.2 +django-redis-sessions==0.6.1 django-registration-redux==1.3 django-reversion==2.0.8 django-ses==0.8.9 @@ -137,6 +138,6 @@ werkzeug==0.14.1 whitenoise==3.3.1 whoosh==2.7.4 xlrd==1.1.0 -xlsxwriter==1.0.4 +xlsxwriter==1.1.2 xlwt==1.3.0 zipp==0.3.3 # via importlib-metadata diff --git a/dependencies/pip/external_services.txt b/dependencies/pip/external_services.txt index e097ab4732..a02c10aed4 100644 --- a/dependencies/pip/external_services.txt +++ b/dependencies/pip/external_services.txt @@ -44,6 +44,7 @@ django-mptt==0.8.7 django-oauth-toolkit==0.10.0 django-picklefield==1.0.0 # via django-constance django-private-storage==2.1.2 +django-redis-sessions==0.6.1 django-registration-redux==1.3 django-reversion==2.0.8 django-ses==0.8.9 diff --git a/dependencies/pip/requirements.in b/dependencies/pip/requirements.in index 383f084125..88f3ccb9c4 100644 --- a/dependencies/pip/requirements.in +++ b/dependencies/pip/requirements.in @@ -45,6 +45,7 @@ django-storages django-private-storage djangorestframework djangorestframework-xml +django-redis-sessions drf-extensions gunicorn jsonfield diff --git a/dependencies/pip/requirements.txt b/dependencies/pip/requirements.txt index 55c59e07c1..b217ec2c68 100644 --- a/dependencies/pip/requirements.txt +++ b/dependencies/pip/requirements.txt @@ -44,6 +44,7 @@ django-mptt==0.8.7 django-oauth-toolkit==0.10.0 django-picklefield==1.0.0 # via django-constance django-private-storage==2.1.2 +django-redis-sessions==0.6.1 django-registration-redux==1.3 django-reversion==2.0.8 django-ses==0.8.9 diff --git a/kobo/settings/base.py b/kobo/settings/base.py index ee9ab6c769..91d95f0bd1 100644 --- a/kobo/settings/base.py +++ b/kobo/settings/base.py @@ -16,6 +16,7 @@ from pymongo import MongoClient from ..static_lists import EXTRA_LANG_INFO +from kpi.utils.redis_helper import RedisHelper # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -198,9 +199,17 @@ def __init__(self, *args, **kwargs): # Database # https://docs.djangoproject.com/en/1.7/ref/settings/#databases + +# @TODO add `KC_DATABASE_URL` and `KPI_DATABASE_URL`: +# - `kobo-install` templates +# - `kobo-docker` templates +# - `kobo-deployments` templates` + +kobocat_database_url = os.getenv("KC_DATABASE_URL", "sqlite:///%s/db.sqlite3" % BASE_DIR) + DATABASES = { 'default': dj_database_url.config(default="sqlite:///%s/db.sqlite3" % BASE_DIR), - 'kobocat': dj_database_url.config(default="sqlite:///%s/db.sqlite3" % BASE_DIR), + 'kobocat': dj_database_url.parse(kobocat_database_url) } DATABASE_ROUTERS = ["kpi.db_routers.DefaultDatabaseRouter"] @@ -694,3 +703,9 @@ def __init__(self, *args, **kwargs): MONGO_CONNECTION = MongoClient( MONGO_CONNECTION_URL, j=True, tz_aware=True, connect=False) MONGO_DB = MONGO_CONNECTION[MONGO_DATABASE['NAME']] + + +SESSION_ENGINE = "redis_sessions.session" +SESSION_REDIS = RedisHelper.config(default="redis://redis_cache:6380/2") + +TESTING = False diff --git a/kobo/settings/testing.py b/kobo/settings/testing.py index 3be4b0fc82..8eb0f1c886 100644 --- a/kobo/settings/testing.py +++ b/kobo/settings/testing.py @@ -8,5 +8,9 @@ 'default': dj_database_url.config(default="sqlite:///%s/db.sqlite3" % BASE_DIR), } +DATABASE_ROUTERS = ["kpi.db_routers.TestingDatabaseRouter"] + +TESTING = True + if 'KPI_AWS_STORAGE_BUCKET_NAME' in os.environ: PRIVATE_STORAGE_S3_REVERSE_PROXY = False diff --git a/kpi/db_routers.py b/kpi/db_routers.py index 795ad24bb9..8017e05f2c 100644 --- a/kpi/db_routers.py +++ b/kpi/db_routers.py @@ -35,3 +35,12 @@ def allow_migrate(self, db, app_label, model=None, **hints): if app_label == SHADOW_MODEL_APP_LABEL: return False return True + + +class TestingDatabaseRouter(DefaultDatabaseRouter): + + def db_for_read(self, model, **hints): + return "default" + + def db_for_write(self, model, **hints): + return "default" diff --git a/kpi/deployment_backends/kc_access/shadow_models.py b/kpi/deployment_backends/kc_access/shadow_models.py index 98666f9258..f3fecd4d44 100644 --- a/kpi/deployment_backends/kc_access/shadow_models.py +++ b/kpi/deployment_backends/kc_access/shadow_models.py @@ -7,6 +7,7 @@ from django.utils.translation import ugettext_lazy as _ from django.contrib.auth.models import User, Permission from django.contrib.contenttypes.models import ContentType +from django.utils import timezone from kpi.constants import SHADOW_MODEL_APP_LABEL @@ -23,8 +24,9 @@ class ReadOnlyModelError(ValueError): class ShadowModel(models.Model): - ''' Allows identification of writeable and read-only shadow models''' - + """ + Allows identification of writeable and read-only shadow models + """ class Meta: managed = False abstract = True @@ -150,7 +152,7 @@ class Meta(ShadowModel.Meta): class UserObjectPermission(ShadowModel): - ''' + """ For the _sole purpose_ of letting us manipulate KoBoCAT permissions, this comprises the following django-guardian classes all condensed into one: @@ -162,7 +164,7 @@ class UserObjectPermission(ShadowModel): CAVEAT LECTOR: The django-guardian custom manager, UserObjectPermissionManager, is NOT included! - ''' + """ permission = models.ForeignKey(Permission) content_type = models.ForeignKey(ContentType) object_pk = models.CharField(_('object ID'), max_length=255) @@ -202,6 +204,66 @@ def save(self, *args, **kwargs): return super(UserObjectPermission, self).save(*args, **kwargs) +class KCUser(ShadowModel): + + username = models.CharField(_("username"), max_length=30) + password = models.CharField(_("password"), max_length=128) + last_login = models.DateTimeField(_("last login"), blank=True, null=True) + is_superuser = models.BooleanField(_('superuser status'), default=False) + first_name = models.CharField(_('first name'), max_length=30, blank=True) + last_name = models.CharField(_('last name'), max_length=150, blank=True) + email = models.EmailField(_('email address'), blank=True) + is_staff = models.BooleanField(_('staff status'), default=False) + is_active = models.BooleanField(_('active'), default=True) + date_joined = models.DateTimeField(_('date joined'), default=timezone.now) + + class Meta(ShadowModel.Meta): + db_table = "auth_user" + + @classmethod + def sync(cls, auth_user): + try: + kc_auth_user = cls.objects.get(pk=auth_user.pk) + assert kc_auth_user.username == auth_user.username + except KCUser.DoesNotExist: + kc_auth_user = cls(pk=auth_user.pk, username=auth_user.username) + + kc_auth_user.password = auth_user.password + kc_auth_user.last_login = auth_user.last_login + kc_auth_user.is_superuser = auth_user.is_superuser + kc_auth_user.first_name = auth_user.first_name + kc_auth_user.last_name = auth_user.last_name + kc_auth_user.email = auth_user.email + kc_auth_user.is_staff = auth_user.is_staff + kc_auth_user.is_active = auth_user.is_active + kc_auth_user.date_joined = auth_user.date_joined + + kc_auth_user.save() + + +class KCToken(ShadowModel): + + key = models.CharField(_("Key"), max_length=40, primary_key=True) + user = models.OneToOneField(getattr(settings, 'AUTH_USER_MODEL', 'auth.User'), + related_name='auth_token', + on_delete=models.CASCADE, verbose_name=_("User")) + created = models.DateTimeField(_("Created"), auto_now_add=True) + + class Meta(ShadowModel.Meta): + db_table = "authtoken_token" + + @classmethod + def sync(cls, auth_token): + try: + # Token use a One-to-One relationship on User. + # Thus, we can retrieve tokens from users' id. + kc_auth_token = cls.objects.get(user_id=auth_token.user_id) + except KCToken.DoesNotExist: + kc_auth_token = cls(pk=auth_token.pk, user=auth_token.user) + + kc_auth_token.save() + + def safe_kc_read(func): def _wrapper(*args, **kwargs): try: diff --git a/kpi/deployment_backends/kobocat_backend.py b/kpi/deployment_backends/kobocat_backend.py index 9e7bfd1060..ff9558b372 100644 --- a/kpi/deployment_backends/kobocat_backend.py +++ b/kpi/deployment_backends/kobocat_backend.py @@ -2,31 +2,25 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, unicode_literals -import cStringIO import json +import posixpath import re -import requests -import unicodecsv import urlparse -import posixpath -from bson import json_util +import requests from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.http import HttpResponse from django.utils.translation import ugettext_lazy as _ -from pyxform.xls2json_backends import xls_to_dict -from rest_framework import exceptions, status, serializers -from rest_framework.request import Request +from rest_framework import status from rest_framework.authtoken.models import Token -from ..exceptions import BadFormatException, KobocatDeploymentException -from .base_backend import BaseDeploymentBackend -from .kc_access.utils import instance_count, last_submission_time -from .kc_access.shadow_models import ReadOnlyInstance, ReadOnlyXForm from kpi.constants import INSTANCE_FORMAT_TYPE_JSON, INSTANCE_FORMAT_TYPE_XML -from kpi.utils.mongo_helper import MongoHelper from kpi.utils.log import logging +from kpi.utils.mongo_helper import MongoHelper +from .base_backend import BaseDeploymentBackend +from .kc_access.shadow_models import ReadOnlyInstance, ReadOnlyXForm +from .kc_access.utils import instance_count, last_submission_time +from ..exceptions import BadFormatException, KobocatDeploymentException class KobocatDeploymentBackend(BaseDeploymentBackend): diff --git a/kpi/signals.py b/kpi/signals.py index ba09eea812..bda3bc0390 100644 --- a/kpi/signals.py +++ b/kpi/signals.py @@ -1,20 +1,24 @@ # -*- coding: utf-8 -*- +from django.conf import settings from django.db.models.signals import post_save, post_delete from django.dispatch import receiver from django.contrib.auth.models import User +from rest_framework.authtoken.models import Token from kobo.apps.hook.models.hook import Hook from taggit.models import Tag from .models import TagUid from .model_utils import grant_default_model_level_perms +from kpi.deployment_backends.kc_access.shadow_models import KCUser, KCToken + @receiver(post_save, sender=User) def default_permissions_post_save(sender, instance, created, raw, **kwargs): - ''' + """ Users must have both model-level and object-level permissions to satisfy DRF, so assign the newly-created user all available collection and asset permissions at the model level - ''' + """ if raw: # `raw` means we can't touch (so make sure your fixtures include # all necessary permissions!) @@ -25,9 +29,37 @@ def default_permissions_post_save(sender, instance, created, raw, **kwargs): return grant_default_model_level_perms(instance) + +@receiver(post_save, sender=User) +def save_kobocat_user(sender, instance, **kwargs): + """ + Sync Auth User table between KPI and KC + """ + if not settings.TESTING: + KCUser.sync(instance) + + +@receiver(post_save, sender=Token) +def save_kobocat_token(sender, instance, **kwargs): + """ + Sync AuthToken table between KPI and KC + """ + if not settings.TESTING: + KCToken.sync(instance) + + +@receiver(post_delete, sender=Token) +def delete_kobocat_token(sender, instance, **kwargs): + """ + Delete corresponding record from KC AuthToken table + """ + if not settings.TESTING: + KCToken.objects.filter(pk=instance.pk).delete() + + @receiver(post_save, sender=Tag) def tag_uid_post_save(sender, instance, created, raw, **kwargs): - ''' Make sure we have a TagUid object for each newly-created Tag ''' + """ Make sure we have a TagUid object for each newly-created Tag """ if raw or not created: return TagUid.objects.get_or_create(tag=instance) @@ -40,4 +72,4 @@ def update_kc_xform_has_kpi_hooks(sender, instance, **kwargs): """ asset = instance.asset if asset.has_deployment: - asset.deployment.set_has_kpi_hooks() \ No newline at end of file + asset.deployment.set_has_kpi_hooks() diff --git a/kpi/tests/test_token.py b/kpi/tests/test_token.py index 81240e54a4..545ae04ce6 100644 --- a/kpi/tests/test_token.py +++ b/kpi/tests/test_token.py @@ -3,6 +3,7 @@ from rest_framework import status from rest_framework.authtoken.models import Token + class UserListTests(APITestCase): fixtures = ['test_data'] diff --git a/kpi/utils/redis_helper.py b/kpi/utils/redis_helper.py new file mode 100644 index 0000000000..8615141e66 --- /dev/null +++ b/kpi/utils/redis_helper.py @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +import os +import re + +from django.core.exceptions import ImproperlyConfigured + + +class RedisHelper(object): + """ + Redis's helper. + + Mimics dj_database_url + + """ + + @staticmethod + def config(default=None): + """ + Parses `REDIS_SESSION_URL` environment variable to return a dict with + expected attributes for django redis session. + + :return: dict + """ + + redis_connection_url = os.getenv("REDIS_SESSION_URL", default) + match = re.match(r"redis://(:(?P[^@]*)@)?(?P[^:]+):(?P\d+)(/(?P\d+))?", + redis_connection_url) + if not match: + raise ImproperlyConfigured("Could not parse Redis session URL. Please verify 'REDIS_SESSION_URL' value") + + redis_connection_dict = { + "host": match.group("host"), + "port": match.group("port"), + "db": match.group("index") or 0, + "password": match.group("password"), + "prefix": os.getenv("REDIS_SESSION_PREFIX", "session"), + "socket_timeout": os.getenv("REDIS_SESSION_SOCKET_TIMEOUT", 1), + } + return redis_connection_dict