Merge pull request #2015 from kobotoolbox/rest_email_notification

Rest email notification

.gitignore

@@ -24,15 +24,4 @@ test/compiled/*
 celerybeat-schedule
 deployments.json
 .DS_Store
-jsapp/fonts/*.eot
-jsapp/fonts/*.svg
-jsapp/fonts/*.ttf
-jsapp/fonts/*.woff
-jsapp/fonts/*.woff2
-jsapp/fonts/*.scss
-jsapp/fonts/*.css
-jsapp/fonts/*.md
-jsapp/fonts/*.ijmap
-jsapp/fonts/*.otf
-jsapp/fonts/codepoints
-jsapp/fonts/*.html
+jsapp/fonts

Dockerfile

@@ -127,6 +127,8 @@ RUN ln -s "${KPI_SRC_DIR}/docker/init.bash" /etc/my_init.d/10_init_kpi.bash && \
     ln -s "${KPI_SRC_DIR}/docker/run_uwsgi.bash" /etc/service/uwsgi/run && \
     mkdir -p /etc/service/celery && \
     ln -s "${KPI_SRC_DIR}/docker/run_celery.bash" /etc/service/celery/run && \
+    mkdir -p /etc/service/celery_beat && \
+    ln -s "${KPI_SRC_DIR}/docker/run_celery_beat.bash" /etc/service/celery_beat/run && \
     mkdir -p /etc/service/celery_sync_kobocat_xforms && \
     ln -s "${KPI_SRC_DIR}/docker/run_celery_sync_kobocat_xforms.bash" /etc/service/celery_sync_kobocat_xforms/run
 

dependencies/pip/dev_requirements.txt

@@ -25,6 +25,7 @@ defusedxml==0.5.0         # via djangorestframework-xml
 dj-database-url==0.4.2
 dj-static==0.0.6
 django-braces==1.11.0
+django-celery-beat==1.1.1
 django-constance[database]==2.2.0
 django-debug-toolbar==1.6
 django-extensions==1.7.6

dependencies/pip/external_services.txt

@@ -6,16 +6,16 @@
 #
 -e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest
 -e git+https://github.com/kobotoolbox/formpack.git@c94751b25f315e15db6d32fa2ebf816263cb6c56#egg=formpack
-amqp==1.4.9
+amqp==2.3.2
 anyjson==0.3.3
 argparse==1.4.0           # via unittest2
 asn1crypto==0.24.0        # via cryptography
 begins==0.9
-billiard==3.3.0.23
+billiard==3.5.0.4
 boto3==1.5.8
 boto==2.40.0
 botocore==1.8.22          # via boto3, s3transfer
-celery==3.1.23
+celery==4.2.1
 cffi==1.8.3               # via cryptography
 contextlib2==0.5.4        # via raven
 cookies==2.2.1            # via responses
@@ -26,6 +26,7 @@ defusedxml==0.5.0         # via djangorestframework-xml
 dj-database-url==0.4.1
 dj-static==0.0.6
 django-braces==1.8.1
+django-celery-beat==1.1.1
 django-constance[database]==2.2.0
 django-debug-toolbar==1.4
 django-extensions==1.6.7
@@ -61,7 +62,7 @@ ipaddress==1.0.17         # via cryptography
 jmespath==0.9.3           # via boto3, botocore
 jsonfield==1.0.3
 jsonschema==2.6.0
-kombu==3.0.35
+kombu==4.2.1
 linecache2==1.0.0         # via traceback2
 lxml==4.2.1
 markdown==2.6.6
@@ -101,6 +102,7 @@ unicodecsv==0.14.1
 unittest2==1.1.0          # via pyxform
 urllib3==1.15.1           # via transifex-client
 uwsgi==2.0.17
+vine==1.1.4               # via amqp
 whitenoise==3.3.1
 whoosh==2.7.4
 xlrd==1.1.0

dependencies/pip/requirements.in

@@ -21,10 +21,11 @@ anyjson
 billiard
 boto
 boto3
-celery
+celery>=4.0,<5.0
 dj-static
 dj-database-url
 django-braces
+django-celery-beat
 django-constance[database]
 django-debug-toolbar
 django-extensions

dependencies/pip/requirements.txt

@@ -2,20 +2,20 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    pip-compile --output-file dependencies/pip/requirements.txt dependencies/pip/requirements.in
+#    pip-compile --output-file requirements.txt requirements.in
 #
 -e git+https://github.com/dimagi/django-digest@0eb1c921329dd187c343b61acfbec4e98450136e#egg=django_digest
 -e git+https://github.com/kobotoolbox/formpack.git@c94751b25f315e15db6d32fa2ebf816263cb6c56#egg=formpack
-amqp==1.4.9
+amqp==2.3.2
 anyjson==0.3.3
 argparse==1.4.0           # via unittest2
 asn1crypto==0.24.0        # via cryptography
 begins==0.9
-billiard==3.3.0.23
+billiard==3.5.0.4
 boto3==1.5.8
 boto==2.40.0
 botocore==1.8.22          # via boto3, s3transfer
-celery==3.1.23
+celery==4.2.1
 cffi==1.8.3               # via cryptography
 cookies==2.2.1            # via responses
 cryptography==2.2.2       # via pyopenssl
@@ -25,6 +25,7 @@ defusedxml==0.5.0         # via djangorestframework-xml
 dj-database-url==0.4.1
 dj-static==0.0.6
 django-braces==1.8.1
+django-celery-beat==1.1.1
 django-constance[database]==2.2.0
 django-debug-toolbar==1.4
 django-extensions==1.6.7
@@ -60,7 +61,7 @@ ipaddress==1.0.17         # via cryptography
 jmespath==0.9.3           # via boto3, botocore
 jsonfield==1.0.3
 jsonschema==2.6.0
-kombu==3.0.35
+kombu==4.2.1
 linecache2==1.0.0         # via traceback2
 lxml==4.2.1
 markdown==2.6.6
@@ -96,6 +97,7 @@ traceback2==1.4.0         # via unittest2
 unicodecsv==0.14.1
 unittest2==1.1.0          # via pyxform
 uwsgi==2.0.17
+vine==1.1.4               # via amqp
 whitenoise==3.3.1
 whoosh==2.7.4
 xlrd==1.1.0

docker/run_celery.bash

@@ -4,7 +4,7 @@ source /etc/profile
 
 # Run the main Celery worker (will not process `sync_kobocat_xforms` jobs).
 cd "${KPI_SRC_DIR}"
-exec celery worker -A kobo --beat --loglevel=info \
+exec celery worker -A kobo --loglevel=info \
     --hostname=main_worker@%h \
     --logfile=${KPI_LOGS_DIR}/celery.log \
     --pidfile=/tmp/celery.pid \

docker/run_celery_beat.bash

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+source /etc/profile
+
+# Run the main Celery worker (will not process `sync_kobocat_xforms` jobs).
+cd "${KPI_SRC_DIR}"
+exec celery beat -A kobo --loglevel=info \
+    --logfile=${KPI_LOGS_DIR}/celery_beat.log \
+    --pidfile=/tmp/celery_beat.pid \
+    --scheduler django_celery_beat.schedulers:DatabaseScheduler

docker/run_celery_sync_kobocat_xforms.bash

@@ -12,6 +12,4 @@ exec celery worker -A kobo --loglevel=info \
     --pidfile=/tmp/celery_sync_kobocat_xforms.pid \
     --queues=sync_kobocat_xforms_queue \
     --concurrency=1 \
-    --maxtasksperchild=1
-    # Watch out: this may be changed in 4.x to `--max-tasks-per-child` per
-    # http://docs.celeryproject.org/en/latest/reference/celery.bin.worker.html#cmdoption-celery-worker-max-tasks-per-child
+    --max-tasks-per-child=1

jsapp/js/components/RESTServices/RESTServicesForm.es6

@@ -1,6 +1,7 @@
 import $ from 'jquery';
 import React from 'react';
 import autoBind from 'react-autobind';
+import TagsInput from 'react-tagsinput';
 import alertify from 'alertifyjs';
 import bem from '../../bem';
 import {dataInterface} from '../../dataInterface';
@@ -42,24 +43,22 @@ export default class RESTServicesForm extends React.Component {
       endpointError: null,
       type: EXPORT_TYPES.JSON,
       isActive: true,
+      emailNotification: true,
       authLevel: null,
       authOptions: [
         AUTH_OPTIONS.no_auth,
         AUTH_OPTIONS.basic_auth
       ],
       authUsername: '',
       authPassword: '',
+      selectedFields: [],
       customHeaders: [
         this.getEmptyHeaderRow()
       ]
     };
     autoBind(this);
   }
 
-  /*
-   * initialization
-   */
-
   componentDidMount() {
     if (this.state.hookUid) {
       dataInterface.getHook(this.state.assetUid, this.state.hookUid)
@@ -69,6 +68,7 @@ export default class RESTServicesForm extends React.Component {
             name: data.name,
             endpoint: data.endpoint,
             isActive: data.active,
+            emailNotification: data.email_notification || true,
             type: data.export_type,
             authLevel: AUTH_OPTIONS[data.auth_level] || null,
             customHeaders: this.headersObjToArr(data.settings.custom_headers)
@@ -151,6 +151,10 @@ export default class RESTServicesForm extends React.Component {
   handleAuthPasswordChange(newPassword) {this.setState({authPassword: newPassword});}
 
   handleActiveChange(isChecked) {this.setState({isActive: isChecked});}
+
+  handleEmailNotificationChange(isChecked) {
+    this.setState({emailNotification: isChecked});
+  }
 
   handleTypeRadioChange(name, value) {this.setState({[name]: value});}
 
@@ -182,6 +186,7 @@ export default class RESTServicesForm extends React.Component {
       name: this.state.name,
       endpoint: this.state.endpoint,
       active: this.state.isActive,
+      email_notification: this.state.emailNotification,
       export_type: this.state.type,
       auth_level: authLevel,
       settings: {
@@ -249,7 +254,7 @@ export default class RESTServicesForm extends React.Component {
   }
 
   /*
-   * rendering custom headers
+   * handle custom headers
    */
 
  onCustomHeaderInputKeyPress(evt) {
@@ -341,7 +346,36 @@ export default class RESTServicesForm extends React.Component {
   }
 
   /*
-   * initialization
+   * handle fields
+   */
+
+  onSelectedFieldsChange(evt) {
+    this.setState({selectedFields: evt});
+  }
+
+  renderFieldsSelector() {
+    const inputProps = {
+      placeholder: t('Add field(s)'),
+      id: 'selected-fields-input'
+    };
+
+    return (
+      <bem.FormModal__item>
+        <label htmlFor='selected-fields-input'>
+          {t('Select fields')}
+        </label>
+
+        <TagsInput
+          value={this.state.selectedFields}
+          onChange={this.onSelectedFieldsChange.bind(this)}
+          inputProps={inputProps}
+        />
+      </bem.FormModal__item>
+    )
+  }
+
+  /*
+   * rendering
    */
 
   render() {
@@ -389,7 +423,16 @@ export default class RESTServicesForm extends React.Component {
                 onChange={this.handleActiveChange.bind(this)}
                 checked={this.state.isActive}
                 label={t('Enabled')}
-                errors={['test', 'secn']}
+              />
+            </bem.FormModal__item>
+
+            <bem.FormModal__item>
+              <Checkbox
+                name='emailNotification'
+                id='email-checkbox'
+                onChange={this.handleEmailNotificationChange.bind(this)}
+                checked={this.state.emailNotification}
+                label={t('Receive emails notifications')}
               />
             </bem.FormModal__item>
 
@@ -434,6 +477,8 @@ export default class RESTServicesForm extends React.Component {
               />
             </bem.FormModal__item>
 
+            {this.renderFieldsSelector()}
+
             {this.state.authLevel && this.state.authLevel.value === AUTH_OPTIONS.basic_auth.value &&
               <bem.FormModal__item>
                 <TextBox

kobo/apps/hook/migrations/0002_add_email_notifications_field.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('hook', '0001_tables_creation'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='hook',
+            name='email_notification',
+            field=models.BooleanField(default=True),
+        ),
+    ]

kobo/apps/hook/models/hook.py

@@ -41,6 +41,7 @@ class Hook(models.Model):
     settings = JSONBField(default=dict)
     date_created = models.DateTimeField(default=timezone.now)
     date_modified = models.DateTimeField(default=timezone.now)
+    email_notification = models.BooleanField(default=True)
 
     class Meta:
         ordering = ["name"]

kobo/apps/hook/models/service_definition_interface.py

@@ -124,7 +124,7 @@ def save_log(self, status_code, message, success=False):
         try:
             # Try to load the log with a multiple field FK because
             # we don't know the log `uid` in this context, but we do know
-            # its `hook` FK and its `instance.uuid
+            # its `hook` FK and its `instance.id
             log = HookLog.objects.get(**fields)
         except HookLog.DoesNotExist:
             log = HookLog(**fields)

kobo/apps/hook/serializers/hook.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+import constance
+from django.utils.translation import ugettext as _
 from rest_framework import serializers
 from rest_framework.reverse import reverse
 
@@ -13,7 +15,7 @@ class Meta:
         model = Hook
         fields = ("url", "logs_url", "asset", "uid", "name", "endpoint", "active", "export_type",
                   "auth_level", "success_count", "failed_count", "pending_count", "settings",
-                  "date_modified")
+                  "date_modified", "email_notification")
 
         read_only_fields = ("asset", "uid", "date_modified", "success_count", "failed_count", "pending_count")
 
@@ -26,4 +28,15 @@ def get_url(self, hook):
 
     def get_logs_url(self, hook):
         return reverse("hook-log-list", args=(hook.asset.uid, hook.uid),
-                       request=self.context.get("request", None))
+                       request=self.context.get("request", None))
+
+    def validate_endpoint(self, value):
+        """
+        Check if endpoint is valid
+        """
+        if not value.startswith("http"):
+            raise serializers.ValidationError(_("Invalid scheme"))
+        elif not constance.config.ALLOW_UNSECURED_HOOK_ENDPOINTS and \
+            value.startswith("http:"):
+            raise serializers.ValidationError(_("Unsecured endpoint is not allowed"))
+        return value