From 2e51046cf34539dbf084c2a60bf6804902629218 Mon Sep 17 00:00:00 2001
From: karimkawambwa <4308339+karimkawambwa@users.noreply.github.com>
Date: Wed, 5 Feb 2020 09:23:45 +0300
Subject: [PATCH 1/7] Create the json cache files when starting the api

---
 contrib/start.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/contrib/start.sh b/contrib/start.sh
index eb8b924..f3fa59e 100755
--- a/contrib/start.sh
+++ b/contrib/start.sh
@@ -2,6 +2,11 @@
 python manage.py migrate --noinput                # Apply database migrations
 python manage.py collectstatic --clear --noinput  # Collect static files
 
+# recreate the cached static json files
+python manage.py cache_static_json_data --interval 5m
+python manage.py cache_static_json_data --interval 1h
+python manage.py cache_static_json_data --interval 24h
+
 # Prepare log files and start outputting logs to stdout
 touch /src/logs/celery.log
 touch /src/logs/gunicorn.log

From 8b80c79a6f6ae5647463def0ebd6c04dcc35c912 Mon Sep 17 00:00:00 2001
From: karimkawambwa <4308339+karimkawambwa@users.noreply.github.com>
Date: Tue, 11 Feb 2020 15:47:12 +0300
Subject: [PATCH 2/7] Use raw sql query with a faster db + Remove
 SensorDataStat

---
 contrib/start.sh                              |   2 +
 sensorsafrica/admin.py                        |  53 +------
 sensorsafrica/api/models.py                   |  48 ++-----
 sensorsafrica/api/v2/router.py                |   2 +-
 sensorsafrica/api/v2/serializers.py           |  37 +++--
 sensorsafrica/api/v2/views.py                 | 129 ++++++++----------
 .../commands/calculate_data_statistics.py     |  95 -------------
 .../migrations/0005_auto_20200211_1224.py     |  30 ++++
 sensorsafrica/settings.py                     |   4 -
 sensorsafrica/tasks.py                        |   5 -
 sensorsafrica/tests/conftest.py               |  15 +-
 sensorsafrica/tests/test_city_view.py         |   2 +-
 .../tests/test_sensordatastats_view.py        |  33 +++--
 13 files changed, 151 insertions(+), 304 deletions(-)
 delete mode 100644 sensorsafrica/management/commands/calculate_data_statistics.py
 create mode 100644 sensorsafrica/migrations/0005_auto_20200211_1224.py

diff --git a/contrib/start.sh b/contrib/start.sh
index f3fa59e..e0468e1 100755
--- a/contrib/start.sh
+++ b/contrib/start.sh
@@ -13,6 +13,8 @@ touch /src/logs/gunicorn.log
 touch /src/logs/access.log
 tail -n 0 -f /src/logs/*.log &
 
+#purge all configured task queues
+celery -A sensorsafrica purge
 celery -A sensorsafrica beat -l info &> /src/logs/celery.log  &
 celery -A sensorsafrica worker --hostname=$DOKKU_APP_NAME -l info &> /src/logs/celery.log  &
 celery -A sensorsafrica flower --basic_auth=$SENSORSAFRICA_FLOWER_ADMIN_USERNAME:$SENSORSAFRICA_FLOWER_ADMIN_PASSWORD &> /src/logs/celery.log  &
diff --git a/sensorsafrica/admin.py b/sensorsafrica/admin.py
index 366d3dd..60d51e6 100644
--- a/sensorsafrica/admin.py
+++ b/sensorsafrica/admin.py
@@ -2,7 +2,7 @@
 from django.utils.html import format_html
 from django.conf.urls import include, url
 from django.template.response import TemplateResponse
-from .api.models import LastActiveNodes, SensorDataStat, City
+from .api.models import LastActiveNodes, City
 from django.db.models import Q
 
 from feinstaub.sensors.admin import (
@@ -77,57 +77,6 @@ def delete_model(self, request, obj):
     def save_related(self, request, form, formsets, change):
         pass
 
-
-@admin.register(SensorDataStat)
-class SensorDataStatAdmin(admin.ModelAdmin):
-    readonly_fields = [
-        "node",
-        "sensor",
-        "location",
-        "city_slug",
-        "value_type",
-        "average",
-        "maximum",
-        "minimum",
-        "timestamp",
-    ]
-    search_fields = ["city_slug", "value_type"]
-    list_display = [
-        "node",
-        "sensor",
-        "location",
-        "city_slug",
-        "value_type",
-        "average",
-        "maximum",
-        "minimum",
-        "timestamp",
-        "created",
-        "modified",
-    ]
-    list_filter = ["timestamp", "node", "sensor", "location"]
-
-    def get_actions(self, request):
-        actions = super(SensorDataStatAdmin, self).get_actions(request)
-        del actions["delete_selected"]
-        return actions
-
-    def has_add_permission(self, request):
-        return False
-
-    def has_delete_permission(self, request, obj=None):
-        return False
-
-    def save_model(self, request, obj, form, change):
-        pass
-
-    def delete_model(self, request, obj):
-        pass
-
-    def save_related(self, request, form, formsets, change):
-        pass
-
-
 @admin.register(City)
 class CityAdmin(admin.ModelAdmin):
     search_fields = ["slug", "name", "country"]
diff --git a/sensorsafrica/api/models.py b/sensorsafrica/api/models.py
index 2722452..f0cb35e 100644
--- a/sensorsafrica/api/models.py
+++ b/sensorsafrica/api/models.py
@@ -5,12 +5,18 @@
 
 
 class City(TimeStampedModel):
-    slug = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    name = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    country = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    location = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    latitude = models.DecimalField(max_digits=14, decimal_places=11, null=True, blank=True)
-    longitude = models.DecimalField(max_digits=14, decimal_places=11, null=True, blank=True)
+    slug = models.CharField(
+        max_length=255, db_index=True, null=False, blank=False)
+    name = models.CharField(
+        max_length=255, db_index=True, null=False, blank=False)
+    country = models.CharField(
+        max_length=255, db_index=True, null=False, blank=False)
+    location = models.CharField(
+        max_length=255, db_index=True, null=False, blank=False)
+    latitude = models.DecimalField(
+        max_digits=14, decimal_places=11, null=True, blank=True)
+    longitude = models.DecimalField(
+        max_digits=14, decimal_places=11, null=True, blank=True)
 
     class Meta:
         verbose_name_plural = "Cities"
@@ -20,36 +26,6 @@ def save(self, *args, **kwargs):
         return super(City, self).save(*args, **kwargs)
 
 
-class SensorDataStat(TimeStampedModel):
-    node = models.ForeignKey(Node)
-    sensor = models.ForeignKey(Sensor)
-    location = models.ForeignKey(SensorLocation)
-
-    city_slug = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-    value_type = models.CharField(max_length=255, db_index=True, null=False, blank=False)
-
-    average = models.FloatField(null=False, blank=False)
-    maximum = models.FloatField(null=False, blank=False)
-    minimum = models.FloatField(null=False, blank=False)
-
-    # Number of data points averaged
-    sample_size = models.IntegerField(null=False, blank=False)
-    # Last datetime of calculated stats
-    last_datetime = models.DateTimeField()
-
-    timestamp = models.DateTimeField()
-
-    def __str__(self):
-        return "%s %s %s avg=%s min=%s max=%s" % (
-            self.timestamp,
-            self.city_slug,
-            self.value_type,
-            self.average,
-            self.minimum,
-            self.maximum,
-        )
-
-
 class LastActiveNodes(TimeStampedModel):
     node = models.ForeignKey(Node)
     location = models.ForeignKey(SensorLocation)
diff --git a/sensorsafrica/api/v2/router.py b/sensorsafrica/api/v2/router.py
index 69f17e7..4de5dcd 100644
--- a/sensorsafrica/api/v2/router.py
+++ b/sensorsafrica/api/v2/router.py
@@ -5,7 +5,7 @@
 
 data_router = routers.DefaultRouter()
 
-data_router.register(r"", SensorDataStatView)
+data_router.register(r"", SensorDataStatView, basename="sensor_data_stat_view")
 
 city_router = routers.DefaultRouter()
 
diff --git a/sensorsafrica/api/v2/serializers.py b/sensorsafrica/api/v2/serializers.py
index 735a2ab..6af5d50 100644
--- a/sensorsafrica/api/v2/serializers.py
+++ b/sensorsafrica/api/v2/serializers.py
@@ -1,14 +1,35 @@
 from rest_framework import serializers
 
 
-class SensorDataStatSerializer(serializers.Serializer):
-    average = serializers.FloatField()
-    minimum = serializers.FloatField()
-    maximum = serializers.FloatField()
-    value_type = serializers.CharField(max_length=200)
-    start_datetime = serializers.DateTimeField()
-    end_datetime = serializers.DateTimeField()
-    city_slug = serializers.CharField(max_length=200)
+class RawSensorDataStatSerializer(serializers.Serializer):
+    average = serializers.SerializerMethodField()
+    minimum = serializers.SerializerMethodField()
+    maximum = serializers.SerializerMethodField()
+    value_type = serializers.SerializerMethodField()
+    start_datetime = serializers.SerializerMethodField()
+    end_datetime = serializers.SerializerMethodField()
+    city_slug = serializers.SerializerMethodField()
+
+    def get_city_slug(self, obj):
+        return obj[0]
+
+    def get_start_datetime(self, obj):
+        return obj[1]
+
+    def get_end_datetime(self, obj):
+        return obj[2]
+
+    def get_average(self, obj):
+        return obj[3]
+
+    def get_minimum(self, obj):
+        return obj[4]
+
+    def get_maximum(self, obj):
+        return obj[5]
+
+    def get_value_type(self, obj):
+        return obj[6]
 
 
 class CitySerializer(serializers.Serializer):
diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py
index 73b6c18..5274088 100644
--- a/sensorsafrica/api/v2/views.py
+++ b/sensorsafrica/api/v2/views.py
@@ -8,11 +8,13 @@
 from django.utils import timezone
 from dateutil.relativedelta import relativedelta
 from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q
-from django.db.models.functions import Cast, TruncDate
+from django.db.models.functions import Cast, TruncDay, TruncHour, TruncMinute, TruncMonth
 from rest_framework import mixins, pagination, viewsets
 
-from ..models import SensorDataStat, LastActiveNodes, City, Node
-from .serializers import SensorDataStatSerializer, CitySerializer
+from django.db import connection
+
+from ..models import LastActiveNodes, City, Node
+from .serializers import RawSensorDataStatSerializer, CitySerializer
 
 from feinstaub.sensors.views import StandardResultsSetPagination
 
@@ -76,7 +78,8 @@ def get_paginated_response(self, data_stats):
                 results[city_slug][value_type] = [] if from_date else {}
 
             values = results[city_slug][value_type]
-            include_result = getattr(values, "append" if from_date else "update")
+            include_result = getattr(
+                values, "append" if from_date else "update")
             include_result(
                 {
                     "average": data_stat["average"],
@@ -98,8 +101,7 @@ def get_paginated_response(self, data_stats):
 
 
 class SensorDataStatView(mixins.ListModelMixin, viewsets.GenericViewSet):
-    queryset = SensorDataStat.objects.none()
-    serializer_class = SensorDataStatSerializer
+    serializer_class = RawSensorDataStatSerializer
     pagination_class = CustomPagination
 
     @method_decorator(cache_page(3600))
@@ -112,60 +114,31 @@ def get_queryset(self):
         city_slugs = self.request.query_params.get("city", None)
         from_date = self.request.query_params.get("from", None)
         to_date = self.request.query_params.get("to", None)
+        avg = self.request.query_params.get("avg", 'day')
 
         if to_date and not from_date:
-            raise ValidationError({"from": "Must be provide along with to query"})
+            raise ValidationError(
+                {"from": "Must be provide along with to query"})
         if from_date:
-            validate_date(from_date, {"from": "Must be a date in the format Y-m-d."})
+            validate_date(
+                from_date, {"from": "Must be a date in the format Y-m-d."})
         if to_date:
-            validate_date(to_date, {"to": "Must be a date in the format Y-m-d."})
+            validate_date(
+                to_date, {"to": "Must be a date in the format Y-m-d."})
 
-        value_type_to_filter = self.request.query_params.get("value_type", None)
+        value_type_to_filter = self.request.query_params.get(
+            "value_type", None)
 
         filter_value_types = value_types[sensor_type]
         if value_type_to_filter:
-            filter_value_types = set(value_type_to_filter.upper().split(",")) & set(
+            filter_value_types = ",".join(set(value_type_to_filter.upper().split(",")) & set(
                 [x.upper() for x in value_types[sensor_type]]
-            )
+            ))
 
         if not from_date and not to_date:
-            return self._retrieve_past_24hrs(city_slugs, filter_value_types)
-
-        return self._retrieve_range(from_date, to_date, city_slugs, filter_value_types)
-
-    @staticmethod
-    def _retrieve_past_24hrs(city_slugs, filter_value_types):
-        to_date = timezone.now().replace(minute=0, second=0, microsecond=0)
-        from_date = to_date - datetime.timedelta(hours=24)
-
-        queryset = SensorDataStat.objects.filter(
-            value_type__in=filter_value_types,
-            timestamp__gte=from_date,
-            timestamp__lte=to_date,
-        )
-
-        if city_slugs:
-            queryset = queryset.filter(city_slug__in=city_slugs.split(","))
-
-        return (
-            queryset.order_by()
-            .values("value_type", "city_slug")
-            .annotate(
-                start_datetime=Min("timestamp"),
-                end_datetime=Max("timestamp"),
-                average=ExpressionWrapper(
-                    Sum(F("average") * F("sample_size")) / Sum("sample_size"),
-                    output_field=FloatField(),
-                ),
-                minimum=Min("minimum"),
-                maximum=Max("maximum"),
-            )
-            .order_by("city_slug")
-        )
-
-    @staticmethod
-    def _retrieve_range(from_date, to_date, city_slugs, filter_value_types):
-        if not to_date:
+            to_date = timezone.now().replace(minute=0, second=0, microsecond=0)
+            from_date = to_date - datetime.timedelta(hours=24)
+        elif not to_date:
             from_date = beginning_of_day(from_date)
             # Get data from_date until the end
             # of day yesterday which is the beginning of today
@@ -174,31 +147,36 @@ def _retrieve_range(from_date, to_date, city_slugs, filter_value_types):
             from_date = beginning_of_day(from_date)
             to_date = end_of_day(to_date)
 
-        queryset = SensorDataStat.objects.filter(
-            value_type__in=filter_value_types,
-            timestamp__gte=from_date,
-            timestamp__lt=to_date,
-        )
-
-        if city_slugs:
-            queryset = queryset.filter(city_slug__in=city_slugs.split(","))
-
-        return (
-            queryset.annotate(date=TruncDate("timestamp"))
-            .values("date", "value_type")
-            .annotate(
-                city_slug=F("city_slug"),
-                start_datetime=Min("timestamp"),
-                end_datetime=Max("timestamp"),
-                average=ExpressionWrapper(
-                    Sum(F("average") * F("sample_size")) / Sum("sample_size"),
-                    output_field=FloatField(),
-                ),
-                minimum=Min("minimum"),
-                maximum=Max("maximum"),
-            )
-            .order_by("-date")
-        )
+        with connection.cursor() as cursor:
+            cursor.execute(
+                """
+                    SELECT
+                        sl.city as city_slug,
+                        min(sd."timestamp") as start_datetime,
+                        max(sd."timestamp") as end_datetime,
+                        sum(CAST("value" as float)) / COUNT(*) AS average,
+                        min(CAST("value" as float)) as minimum,
+                        max(CAST("value" as float)) as maximum,
+                        v.value_type
+                    FROM
+                        sensors_sensordatavalue v
+                        INNER JOIN sensors_sensordata sd ON sd.id = sensordata_id
+                        INNER JOIN sensors_sensorlocation sl ON sl.id = location_id
+                    WHERE
+                        v.value_type IN (%s)
+                        """
+                +
+                ("AND sl.city IN (%s)" if city_slugs else "")
+                +
+                """
+                                        AND sd."timestamp" >= TIMESTAMP %s
+                                        AND sd."timestamp" <= TIMESTAMP %s
+                    GROUP BY
+                        DATE_TRUNC(%s, sd."timestamp"),
+                        v.value_type,
+                        sl.city
+                """, [filter_value_types, city_slugs, from_date, to_date, avg] if city_slugs else [filter_value_types, from_date, to_date, avg])
+            return cursor.fetchall()
 
 
 class CityView(mixins.ListModelMixin, viewsets.GenericViewSet):
@@ -225,7 +203,8 @@ def list(self, request):
             moved_to = None
             # Get data stats from 5mins before last_data_received_at
             if last_data_received_at:
-                last_5_mins = last_data_received_at - datetime.timedelta(minutes=5)
+                last_5_mins = last_data_received_at - \
+                    datetime.timedelta(minutes=5)
                 stats = (
                     SensorDataValue.objects.filter(
                         Q(sensordata__sensor__node=last_active.node.id),
diff --git a/sensorsafrica/management/commands/calculate_data_statistics.py b/sensorsafrica/management/commands/calculate_data_statistics.py
deleted file mode 100644
index e49f9cf..0000000
--- a/sensorsafrica/management/commands/calculate_data_statistics.py
+++ /dev/null
@@ -1,95 +0,0 @@
-from django.core.management import BaseCommand
-from django.db.models import Avg, Count, FloatField, Max, Min, Q
-from django.db.models.functions import Cast, TruncHour
-from django.utils.text import slugify
-from feinstaub.sensors.models import Node, Sensor, SensorDataValue, SensorLocation
-from ...api.models import SensorDataStat
-
-
-def map_stat(stat, city):
-    return SensorDataStat(
-        city_slug=slugify(city),
-        timestamp=stat["timestamp"],
-        value_type=stat["value_type"],
-        location=SensorLocation(pk=stat["sensordata__location"]),
-        sensor=Sensor(pk=stat["sensordata__sensor"]),
-        node=Node(pk=stat["sensordata__sensor__node"]),
-        average=stat["average"],
-        minimum=stat["minimum"],
-        maximum=stat["maximum"],
-        sample_size=stat["sample_size"],
-        last_datetime=stat["last_datetime"],
-    )
-
-
-class Command(BaseCommand):
-    help = "Calculate and store data statistics"
-
-    def handle(self, *args, **options):
-
-        cities = list(
-            set(
-                SensorLocation.objects.all()
-                .values_list("city", flat=True)
-                .order_by("city")
-            )
-        )
-
-        for city in cities:
-            if not city:
-                continue
-
-            last_date_time = (
-                SensorDataStat.objects.filter(city_slug=slugify(city))
-                .values_list("last_datetime", flat=True)
-                .order_by("-last_datetime")[:1]
-            )
-
-            if last_date_time:
-                queryset = SensorDataValue.objects.filter(
-                    Q(sensordata__location__city__iexact=city),
-                    # Get dates greater than last stat calculation
-                    Q(created__gt=last_date_time),
-                    # Ignore timestamp values
-                    ~Q(value_type="timestamp"),
-                    # Match only valid float text
-                    Q(value__regex=r"^\-?\d+(\.?\d+)?$"),
-                )
-            else:
-                queryset = SensorDataValue.objects.filter(
-                    Q(sensordata__location__city__iexact=city),
-                    # Ignore timestamp values
-                    ~Q(value_type="timestamp"),
-                    # Match only valid float text
-                    Q(value__regex=r"^\-?\d+(\.?\d+)?$"),
-                )
-
-            stats = list(
-                queryset.annotate(timestamp=TruncHour("created"))
-                .values(
-                    "timestamp",
-                    "value_type",
-                    "sensordata__sensor",
-                    "sensordata__location",
-                    "sensordata__sensor__node",
-                )
-                .order_by()
-                .annotate(
-                    last_datetime=Max("created"),
-                    average=Avg(Cast("value", FloatField())),
-                    minimum=Min(Cast("value", FloatField())),
-                    maximum=Max(Cast("value", FloatField())),
-                    sample_size=Count("created", FloatField()),
-                )
-                .filter(
-                    ~Q(average=float("NaN")),
-                    ~Q(minimum=float("NaN")),
-                    ~Q(maximum=float("NaN")),
-                )
-                .order_by("-timestamp")
-            )
-
-            if len(stats):
-                SensorDataStat.objects.bulk_create(
-                    list(map(lambda stat: map_stat(stat, city), stats))
-                )
diff --git a/sensorsafrica/migrations/0005_auto_20200211_1224.py b/sensorsafrica/migrations/0005_auto_20200211_1224.py
new file mode 100644
index 0000000..f092023
--- /dev/null
+++ b/sensorsafrica/migrations/0005_auto_20200211_1224.py
@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.27 on 2020-02-11 12:24
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sensorsafrica', '0004_auto_20190509_1145'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='sensordatastat',
+            name='location',
+        ),
+        migrations.RemoveField(
+            model_name='sensordatastat',
+            name='node',
+        ),
+        migrations.RemoveField(
+            model_name='sensordatastat',
+            name='sensor',
+        ),
+        migrations.DeleteModel(
+            name='SensorDataStat',
+        ),
+    ]
diff --git a/sensorsafrica/settings.py b/sensorsafrica/settings.py
index b27ae0a..99298ce 100644
--- a/sensorsafrica/settings.py
+++ b/sensorsafrica/settings.py
@@ -152,10 +152,6 @@
 CELERY_IGNORE_RESULT = True
 
 CELERY_BEAT_SCHEDULE = {
-    "statistics-task": {
-        "task": "sensorsafrica.tasks.calculate_data_statistics",
-        "schedule": crontab(hour="*", minute=0)
-    },
     "archive-task": {
         "task": "sensorsafrica.tasks.archive_data",
         "schedule": crontab(hour="*", minute=0)
diff --git a/sensorsafrica/tasks.py b/sensorsafrica/tasks.py
index 7b9f5be..496b2a6 100644
--- a/sensorsafrica/tasks.py
+++ b/sensorsafrica/tasks.py
@@ -2,11 +2,6 @@
 from django.core.management import call_command
 
 
-@shared_task
-def calculate_data_statistics():
-    call_command("calculate_data_statistics")
-
-
 @shared_task
 def archive_data():
     call_command("upload_to_ckan")
diff --git a/sensorsafrica/tests/conftest.py b/sensorsafrica/tests/conftest.py
index d95fd03..afd0697 100644
--- a/sensorsafrica/tests/conftest.py
+++ b/sensorsafrica/tests/conftest.py
@@ -203,14 +203,7 @@ def datavalues(sensors, sensordata):
 
 
 @pytest.fixture
-def sensorsdatastats(datavalues):
-    from django.core.management import call_command
-
-    call_command("calculate_data_statistics")
-
-
-@pytest.fixture
-def additional_sensorsdatastats(sensors, locations, sensorsdatastats):
+def additional_sensorsdatastats(sensors, locations):
     sensordata = SensorData.objects.bulk_create([
         SensorData(sensor=sensors[0], location=locations[0]),
         SensorData(sensor=sensors[0], location=locations[0]),
@@ -224,13 +217,9 @@ def additional_sensorsdatastats(sensors, locations, sensorsdatastats):
         SensorDataValue(sensordata=sensordata[2], value="4", value_type="P2"),
     ])
 
-    from django.core.management import call_command
-
-    call_command("calculate_data_statistics")
-
 
 @pytest.fixture
-def last_active(sensors, locations, sensorsdatastats):
+def last_active(sensors, locations):
     timestamps = [
         timezone.now(),
         timezone.now() + datetime.timedelta(minutes=2),
diff --git a/sensorsafrica/tests/test_city_view.py b/sensorsafrica/tests/test_city_view.py
index 8f15470..7a87748 100644
--- a/sensorsafrica/tests/test_city_view.py
+++ b/sensorsafrica/tests/test_city_view.py
@@ -3,7 +3,7 @@
 
 @pytest.mark.django_db
 class TestCityView:
-    def test_getting_cities(self, client, sensorsdatastats):
+    def test_getting_cities(self, client):
         response = client.get("/v2/cities/", format="json")
         assert response.status_code == 200
 
diff --git a/sensorsafrica/tests/test_sensordatastats_view.py b/sensorsafrica/tests/test_sensordatastats_view.py
index 1712f8f..50fd7bd 100644
--- a/sensorsafrica/tests/test_sensordatastats_view.py
+++ b/sensorsafrica/tests/test_sensordatastats_view.py
@@ -6,8 +6,9 @@
 
 @pytest.mark.django_db
 class TestGettingData:
-    def test_getting_air_data_now(self, client, sensorsdatastats):
-        response = client.get("/v2/data/air/?city=dar-es-salaam", format="json")
+    def test_getting_air_data_now(self, client):
+        response = client.get(
+            "/v2/data/air/?city=dar-es-salaam", format="json")
         assert response.status_code == 200
 
         data = response.json()
@@ -31,7 +32,7 @@ def test_getting_air_data_now(self, client, sensorsdatastats):
         assert result["P2"]["maximum"] == 8.0
         assert result["P2"]["minimum"] == 0.0
 
-    def test_getting_air_data_now_all_cities(self, client, sensorsdatastats):
+    def test_getting_air_data_now_all_cities(self, client):
         response = client.get("/v2/data/air/", format="json")
         assert response.status_code == 200
 
@@ -48,7 +49,7 @@ def test_getting_air_data_now_all_cities(self, client, sensorsdatastats):
         assert "P2" in results[1]
         assert results[2]["city_slug"] == "nairobi"
 
-    def test_getting_air_data_now_filter_cities(self, client, sensorsdatastats):
+    def test_getting_air_data_now_filter_cities(self, client):
         response = client.get(
             "/v2/data/air/?city=dar-es-salaam,bagamoyo", format="json"
         )
@@ -66,7 +67,7 @@ def test_getting_air_data_now_filter_cities(self, client, sensorsdatastats):
         assert results[1]["city_slug"] == "dar-es-salaam"
         assert "P2" in results[1]
 
-    def test_getting_air_data_value_type(self, client, sensorsdatastats):
+    def test_getting_air_data_value_type(self, client):
         response = client.get(
             "/v2/data/air/?city=dar-es-salaam&value_type=P2", format="json"
         )
@@ -80,7 +81,7 @@ def test_getting_air_data_value_type(self, client, sensorsdatastats):
         assert "temperature" not in data["results"][0]
         assert "humidity" not in data["results"][0]
 
-    def test_getting_air_data_from_date(self, client, sensorsdatastats):
+    def test_getting_air_data_from_date(self, client):
         response = client.get(
             "/v2/data/air/?city=dar-es-salaam&from=%s"
             % (timezone.now() - datetime.timedelta(days=2)).date(),
@@ -101,10 +102,11 @@ def test_getting_air_data_from_date(self, client, sensorsdatastats):
         # Check today is not included
         assert most_recent_date.date() < datetime.datetime.today().date()
 
-    def test_getting_air_data_from_date_to_date(self, client, sensorsdatastats):
+    def test_getting_air_data_from_date_to_date(self, client):
         now = timezone.now()
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam&from=%s&to=%s" % (now.date(), now.date()),
+            "/v2/data/air/?city=dar-es-salaam&from=%s&to=%s" % (
+                now.date(), now.date()),
             format="json",
         )
         assert response.status_code == 200
@@ -115,21 +117,23 @@ def test_getting_air_data_from_date_to_date(self, client, sensorsdatastats):
         assert type(data["results"][0]["P1"]) == list
         assert type(data["results"][0]["P2"]) == list
 
-    def test_getting_air_data_with_invalid_request(self, client, sensorsdatastats):
+    def test_getting_air_data_with_invalid_request(self, client):
         response = client.get(
             "/v2/data/air/?city=dar-es-salaam&to=2019-02-08", format="json"
         )
         assert response.status_code == 400
-        assert response.json() == {"from": "Must be provide along with to query"}
+        assert response.json() == {
+            "from": "Must be provide along with to query"}
 
-    def test_getting_air_data_with_invalid_from_request(self, client, sensorsdatastats):
+    def test_getting_air_data_with_invalid_from_request(self, client):
         response = client.get(
             "/v2/data/air/?city=dar-es-salaam&from=2019-23-08", format="json"
         )
         assert response.status_code == 400
-        assert response.json() == {"from": "Must be a date in the format Y-m-d."}
+        assert response.json() == {
+            "from": "Must be a date in the format Y-m-d."}
 
-    def test_getting_air_data_with_invalid_to_request(self, client, sensorsdatastats):
+    def test_getting_air_data_with_invalid_to_request(self, client):
         response = client.get(
             "/v2/data/air/?city=dar-es-salaam&from=2019-02-08&to=08-02-2019",
             format="json",
@@ -140,7 +144,8 @@ def test_getting_air_data_with_invalid_to_request(self, client, sensorsdatastats
     def test_getting_air_data_now_with_additional_values(
         self, client, additional_sensorsdatastats
     ):
-        response = client.get("/v2/data/air/?city=dar-es-salaam", format="json")
+        response = client.get(
+            "/v2/data/air/?city=dar-es-salaam", format="json")
         assert response.status_code == 200
 
         data = response.json()

From c0d380121efeab6222ba07ebba4569225e68fd4d Mon Sep 17 00:00:00 2001
From: karimkawambwa <4308339+karimkawambwa@users.noreply.github.com>
Date: Wed, 12 Feb 2020 10:05:52 +0300
Subject: [PATCH 3/7] Cleaning up tests

---
 sensorsafrica/admin.py                        |  1 +
 sensorsafrica/api/v2/views.py                 | 56 ++++++++++++-------
 sensorsafrica/tests/conftest.py               |  2 +-
 .../tests/test_sensordatastats_view.py        | 51 +++++++++--------
 4 files changed, 62 insertions(+), 48 deletions(-)

diff --git a/sensorsafrica/admin.py b/sensorsafrica/admin.py
index 60d51e6..6e63c9b 100644
--- a/sensorsafrica/admin.py
+++ b/sensorsafrica/admin.py
@@ -77,6 +77,7 @@ def delete_model(self, request, obj):
     def save_related(self, request, form, formsets, change):
         pass
 
+
 @admin.register(City)
 class CityAdmin(admin.ModelAdmin):
     search_fields = ["slug", "name", "country"]
diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py
index 5274088..5c9e090 100644
--- a/sensorsafrica/api/v2/views.py
+++ b/sensorsafrica/api/v2/views.py
@@ -65,19 +65,19 @@ def get_paginated_response(self, data_stats):
 
         results = {}
         for data_stat in data_stats:
-            city_slug = data_stat["city_slug"]
+            city_name = data_stat["city_name"]
             value_type = data_stat["value_type"]
 
-            if city_slug not in results:
-                results[city_slug] = {
-                    "city_slug": city_slug,
+            if city_name not in results:
+                results[city_name] = {
+                    "city_name": city_name,
                     value_type: [] if from_date else {},
                 }
 
-            if value_type not in results[city_slug]:
-                results[city_slug][value_type] = [] if from_date else {}
+            if value_type not in results[city_name]:
+                results[city_name][value_type] = [] if from_date else {}
 
-            values = results[city_slug][value_type]
+            values = results[city_name][value_type]
             include_result = getattr(
                 values, "append" if from_date else "update")
             include_result(
@@ -111,7 +111,7 @@ def dispatch(self, request, *args, **kwargs):
     def get_queryset(self):
         sensor_type = self.kwargs["sensor_type"]
 
-        city_slugs = self.request.query_params.get("city", None)
+        city_names = self.request.query_params.get("city", None)
         from_date = self.request.query_params.get("from", None)
         to_date = self.request.query_params.get("to", None)
         avg = self.request.query_params.get("avg", 'day')
@@ -129,7 +129,7 @@ def get_queryset(self):
         value_type_to_filter = self.request.query_params.get(
             "value_type", None)
 
-        filter_value_types = value_types[sensor_type]
+        filter_value_types = ",".join(value_types[sensor_type])
         if value_type_to_filter:
             filter_value_types = ",".join(set(value_type_to_filter.upper().split(",")) & set(
                 [x.upper() for x in value_types[sensor_type]]
@@ -147,11 +147,18 @@ def get_queryset(self):
             from_date = beginning_of_day(from_date)
             to_date = end_of_day(to_date)
 
+        if city_names:
+            city_names = ",".join(map(lambda city: "'%s'" %
+                                      city, city_names.split(',')))
+
+        if filter_value_types:
+            filter_value_types = ",".join(map(
+                lambda filter_value_type: "'%s'" % filter_value_type, filter_value_types.split(',')))
+
         with connection.cursor() as cursor:
-            cursor.execute(
-                """
+            cursor.execute("""
                     SELECT
-                        sl.city as city_slug,
+                        sl.city as city_name,
                         min(sd."timestamp") as start_datetime,
                         max(sd."timestamp") as end_datetime,
                         sum(CAST("value" as float)) / COUNT(*) AS average,
@@ -163,19 +170,26 @@ def get_queryset(self):
                         INNER JOIN sensors_sensordata sd ON sd.id = sensordata_id
                         INNER JOIN sensors_sensorlocation sl ON sl.id = location_id
                     WHERE
-                        v.value_type IN (%s)
+                        v.value_type IN (%(filter_value_types)s)
+                        AND v.value ~ '^\\-?\\d+(\\.?\\d+)?$'
                         """
-                +
-                ("AND sl.city IN (%s)" if city_slugs else "")
-                +
-                """
-                                        AND sd."timestamp" >= TIMESTAMP %s
-                                        AND sd."timestamp" <= TIMESTAMP %s
+                           +
+                           ("AND sl.city IN (%(city_names)s)" if city_names else "")
+                           +
+                           """
+                                        AND sd."timestamp" >= TIMESTAMP %(from_date)s
+                                        AND sd."timestamp" <= TIMESTAMP %(to_date)s
                     GROUP BY
-                        DATE_TRUNC(%s, sd."timestamp"),
+                        DATE_TRUNC(%(trunc)s, sd."timestamp"),
                         v.value_type,
                         sl.city
-                """, [filter_value_types, city_slugs, from_date, to_date, avg] if city_slugs else [filter_value_types, from_date, to_date, avg])
+                """, {
+                               'filter_value_types': filter_value_types,
+                               'city_names': city_names,
+                               'from_date': from_date,
+                               'to_date': to_date,
+                               'trunc': avg
+                           })
             return cursor.fetchall()
 
 
diff --git a/sensorsafrica/tests/conftest.py b/sensorsafrica/tests/conftest.py
index afd0697..492b472 100644
--- a/sensorsafrica/tests/conftest.py
+++ b/sensorsafrica/tests/conftest.py
@@ -203,7 +203,7 @@ def datavalues(sensors, sensordata):
 
 
 @pytest.fixture
-def additional_sensorsdatastats(sensors, locations):
+def additional_sensorsdatastats(sensors, locations, datavalues):
     sensordata = SensorData.objects.bulk_create([
         SensorData(sensor=sensors[0], location=locations[0]),
         SensorData(sensor=sensors[0], location=locations[0]),
diff --git a/sensorsafrica/tests/test_sensordatastats_view.py b/sensorsafrica/tests/test_sensordatastats_view.py
index 50fd7bd..3349e6c 100644
--- a/sensorsafrica/tests/test_sensordatastats_view.py
+++ b/sensorsafrica/tests/test_sensordatastats_view.py
@@ -3,12 +3,11 @@
 import pytest
 from django.utils import timezone
 
-
 @pytest.mark.django_db
 class TestGettingData:
-    def test_getting_air_data_now(self, client):
+    def test_getting_air_data_now(self, client, datavalues):
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam", format="json")
+            "/v2/data/air/?city=Dar es Salaam", format="json")
         assert response.status_code == 200
 
         data = response.json()
@@ -32,7 +31,7 @@ def test_getting_air_data_now(self, client):
         assert result["P2"]["maximum"] == 8.0
         assert result["P2"]["minimum"] == 0.0
 
-    def test_getting_air_data_now_all_cities(self, client):
+    def test_getting_air_data_now_all_cities(self, client, datavalues):
         response = client.get("/v2/data/air/", format="json")
         assert response.status_code == 200
 
@@ -42,16 +41,16 @@ def test_getting_air_data_now_all_cities(self, client):
 
         results = data["results"]
 
-        assert results[0]["city_slug"] == "bagamoyo"
-        assert results[1]["city_slug"] == "dar-es-salaam"
+        assert results[0]["city_slug"] == "Bagamoyo"
+        assert results[1]["city_slug"] == "Dar es Salaam"
         assert "P1" in results[1]
-        assert results[1]["city_slug"] == "dar-es-salaam"
+        assert results[1]["city_slug"] == "Dar es Salaam"
         assert "P2" in results[1]
-        assert results[2]["city_slug"] == "nairobi"
+        assert results[2]["city_slug"] == "Nairobi"
 
-    def test_getting_air_data_now_filter_cities(self, client):
+    def test_getting_air_data_now_filter_cities(self, client, datavalues):
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam,bagamoyo", format="json"
+            "/v2/data/air/?city=Dar es Salaam,Bagamoyo", format="json"
         )
         assert response.status_code == 200
 
@@ -61,15 +60,15 @@ def test_getting_air_data_now_filter_cities(self, client):
 
         results = data["results"]
 
-        assert results[0]["city_slug"] == "bagamoyo"
-        assert results[1]["city_slug"] == "dar-es-salaam"
+        assert results[0]["city_slug"] == "Bagamoyo"
+        assert results[1]["city_slug"] == "Dar es Salaam"
         assert "P1" in results[1]
-        assert results[1]["city_slug"] == "dar-es-salaam"
+        assert results[1]["city_slug"] == "Dar es Salaam"
         assert "P2" in results[1]
 
-    def test_getting_air_data_value_type(self, client):
+    def test_getting_air_data_value_type(self, client, datavalues):
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam&value_type=P2", format="json"
+            "/v2/data/air/?city=Dar es Salaam&value_type=P2", format="json"
         )
         assert response.status_code == 200
 
@@ -81,9 +80,9 @@ def test_getting_air_data_value_type(self, client):
         assert "temperature" not in data["results"][0]
         assert "humidity" not in data["results"][0]
 
-    def test_getting_air_data_from_date(self, client):
+    def test_getting_air_data_from_date(self, client, datavalues):
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam&from=%s"
+            "/v2/data/air/?city=Dar es Salaam&from=%s"
             % (timezone.now() - datetime.timedelta(days=2)).date(),
             format="json",
         )
@@ -102,10 +101,10 @@ def test_getting_air_data_from_date(self, client):
         # Check today is not included
         assert most_recent_date.date() < datetime.datetime.today().date()
 
-    def test_getting_air_data_from_date_to_date(self, client):
+    def test_getting_air_data_from_date_to_date(self, client, datavalues):
         now = timezone.now()
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam&from=%s&to=%s" % (
+            "/v2/data/air/?city=Dar es Salaam&from=%s&to=%s" % (
                 now.date(), now.date()),
             format="json",
         )
@@ -117,25 +116,25 @@ def test_getting_air_data_from_date_to_date(self, client):
         assert type(data["results"][0]["P1"]) == list
         assert type(data["results"][0]["P2"]) == list
 
-    def test_getting_air_data_with_invalid_request(self, client):
+    def test_getting_air_data_with_invalid_request(self, client, datavalues):
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam&to=2019-02-08", format="json"
+            "/v2/data/air/?city=Dar es Salaam&to=2019-02-08", format="json"
         )
         assert response.status_code == 400
         assert response.json() == {
             "from": "Must be provide along with to query"}
 
-    def test_getting_air_data_with_invalid_from_request(self, client):
+    def test_getting_air_data_with_invalid_from_request(self, client, datavalues):
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam&from=2019-23-08", format="json"
+            "/v2/data/air/?city=Dar es Salaam&from=2019-23-08", format="json"
         )
         assert response.status_code == 400
         assert response.json() == {
             "from": "Must be a date in the format Y-m-d."}
 
-    def test_getting_air_data_with_invalid_to_request(self, client):
+    def test_getting_air_data_with_invalid_to_request(self, client, datavalues):
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam&from=2019-02-08&to=08-02-2019",
+            "/v2/data/air/?city=Dar es Salaam&from=2019-02-08&to=08-02-2019",
             format="json",
         )
         assert response.status_code == 400
@@ -145,7 +144,7 @@ def test_getting_air_data_now_with_additional_values(
         self, client, additional_sensorsdatastats
     ):
         response = client.get(
-            "/v2/data/air/?city=dar-es-salaam", format="json")
+            "/v2/data/air/?city=Dar es Salaam", format="json")
         assert response.status_code == 200
 
         data = response.json()

From 13339667210362e7ab8121d6c9b78276fe37966a Mon Sep 17 00:00:00 2001
From: karimkawambwa <4308339+karimkawambwa@users.noreply.github.com>
Date: Wed, 12 Feb 2020 12:38:53 +0300
Subject: [PATCH 4/7] Fix tests

---
 sensorsafrica/api/v2/serializers.py           |  4 +-
 sensorsafrica/api/v2/views.py                 | 45 +++++++++----------
 sensorsafrica/tests/conftest.py               | 38 ++++++++--------
 .../tests/test_sensordatastats_view.py        | 37 ++++++++-------
 4 files changed, 62 insertions(+), 62 deletions(-)

diff --git a/sensorsafrica/api/v2/serializers.py b/sensorsafrica/api/v2/serializers.py
index 6af5d50..d7bab50 100644
--- a/sensorsafrica/api/v2/serializers.py
+++ b/sensorsafrica/api/v2/serializers.py
@@ -8,9 +8,9 @@ class RawSensorDataStatSerializer(serializers.Serializer):
     value_type = serializers.SerializerMethodField()
     start_datetime = serializers.SerializerMethodField()
     end_datetime = serializers.SerializerMethodField()
-    city_slug = serializers.SerializerMethodField()
+    city_name = serializers.SerializerMethodField()
 
-    def get_city_slug(self, obj):
+    def get_city_name(self, obj):
         return obj[0]
 
     def get_start_datetime(self, obj):
diff --git a/sensorsafrica/api/v2/views.py b/sensorsafrica/api/v2/views.py
index 5c9e090..c5514f9 100644
--- a/sensorsafrica/api/v2/views.py
+++ b/sensorsafrica/api/v2/views.py
@@ -129,14 +129,14 @@ def get_queryset(self):
         value_type_to_filter = self.request.query_params.get(
             "value_type", None)
 
-        filter_value_types = ",".join(value_types[sensor_type])
+        filter_value_types = value_types[sensor_type]
         if value_type_to_filter:
-            filter_value_types = ",".join(set(value_type_to_filter.upper().split(",")) & set(
+            filter_value_types = set(value_type_to_filter.upper().split(",")) & set(
                 [x.upper() for x in value_types[sensor_type]]
-            ))
+            )
 
         if not from_date and not to_date:
-            to_date = timezone.now().replace(minute=0, second=0, microsecond=0)
+            to_date = timezone.now()
             from_date = to_date - datetime.timedelta(hours=24)
         elif not to_date:
             from_date = beginning_of_day(from_date)
@@ -147,13 +147,7 @@ def get_queryset(self):
             from_date = beginning_of_day(from_date)
             to_date = end_of_day(to_date)
 
-        if city_names:
-            city_names = ",".join(map(lambda city: "'%s'" %
-                                      city, city_names.split(',')))
-
-        if filter_value_types:
-            filter_value_types = ",".join(map(
-                lambda filter_value_type: "'%s'" % filter_value_type, filter_value_types.split(',')))
+        city_names = city_names.split(',') if city_names else []
 
         with connection.cursor() as cursor:
             cursor.execute("""
@@ -164,33 +158,38 @@ def get_queryset(self):
                         sum(CAST("value" as float)) / COUNT(*) AS average,
                         min(CAST("value" as float)) as minimum,
                         max(CAST("value" as float)) as maximum,
-                        v.value_type
+                        v.value_type,
+                        STRING_AGG("value" || ' ' || sd."timestamp", ',') as debug
                     FROM
                         sensors_sensordatavalue v
                         INNER JOIN sensors_sensordata sd ON sd.id = sensordata_id
                         INNER JOIN sensors_sensorlocation sl ON sl.id = location_id
                     WHERE
-                        v.value_type IN (%(filter_value_types)s)
+                        v.value_type IN %(filter_value_types)s
                         AND v.value ~ '^\\-?\\d+(\\.?\\d+)?$'
                         """
                            +
-                           ("AND sl.city IN (%(city_names)s)" if city_names else "")
+                           ("AND sl.city IN %(city_names)s" if len(city_names) > 0 else "")
                            +
                            """
-                                        AND sd."timestamp" >= TIMESTAMP %(from_date)s
-                                        AND sd."timestamp" <= TIMESTAMP %(to_date)s
+                        AND sd."timestamp" >= %(from_date)s
+                        AND sd."timestamp" <= %(to_date)s
                     GROUP BY
                         DATE_TRUNC(%(trunc)s, sd."timestamp"),
                         v.value_type,
                         sl.city
                 """, {
-                               'filter_value_types': filter_value_types,
-                               'city_names': city_names,
-                               'from_date': from_date,
-                               'to_date': to_date,
-                               'trunc': avg
-                           })
-            return cursor.fetchall()
+                                'filter_value_types': tuple(filter_value_types),
+                                'city_names': tuple(city_names),
+                                'from_date': from_date,
+                                'to_date': to_date,
+                                'trunc': avg
+                            })
+            res = cursor.fetchall()
+
+            print(res)
+
+            return res
 
 
 class CityView(mixins.ListModelMixin, viewsets.GenericViewSet):
diff --git a/sensorsafrica/tests/conftest.py b/sensorsafrica/tests/conftest.py
index 492b472..9af9b03 100644
--- a/sensorsafrica/tests/conftest.py
+++ b/sensorsafrica/tests/conftest.py
@@ -178,32 +178,34 @@ def datavalues(sensors, sensordata):
         data_values.append(SensorDataValue(
             sensordata=sensordata[109 + i], value="0.0", value_type="P2"))
 
-    values = SensorDataValue.objects.bulk_create(data_values)
+    return SensorDataValue.objects.bulk_create(data_values)
 
-    now = timezone.now()
 
+@pytest.fixture
+def modified_datavalues(datavalues):
+    now = timezone.now()
     # Set Dar es salaam a day ago's data
-    values[1].update_modified = False
-    values[1].created = now - datetime.timedelta(days=2)
-    values[1].save()
-    values[2].update_modified = False
-    values[2].created = now - datetime.timedelta(days=2)
-    values[2].save()
+    datavalues[1].sensordata.update_modified = False
+    datavalues[1].sensordata.timestamp = now - datetime.timedelta(days=2)
+    datavalues[1].sensordata.save()
+    datavalues[2].sensordata.update_modified = False
+    datavalues[2].sensordata.timestamp = now - datetime.timedelta(days=2)
+    datavalues[2].sensordata.save()
 
     # Set data received at different hours
-    values[3].update_modified = False
-    values[3].created = now - datetime.timedelta(hours=1)
-    values[3].save()
-    values[4].update_modified = False
-    values[4].created = now - datetime.timedelta(hours=2)
-    values[4].save()
-    values[5].update_modified = False
-    values[5].created = now - datetime.timedelta(hours=3)
-    values[5].save()
+    datavalues[3].sensordata.update_modified = False
+    datavalues[3].sensordata.timestamp = now - datetime.timedelta(hours=1)
+    datavalues[3].sensordata.save()
+    datavalues[4].sensordata.update_modified = False
+    datavalues[4].sensordata.timestamp = now - datetime.timedelta(hours=2)
+    datavalues[4].sensordata.save()
+    datavalues[5].sensordata.update_modified = False
+    datavalues[5].sensordata.timestamp = now - datetime.timedelta(hours=3)
+    datavalues[5].sensordata.save()
 
 
 @pytest.fixture
-def additional_sensorsdatastats(sensors, locations, datavalues):
+def additional_sensorsdatastats(sensors, locations, modified_datavalues):
     sensordata = SensorData.objects.bulk_create([
         SensorData(sensor=sensors[0], location=locations[0]),
         SensorData(sensor=sensors[0], location=locations[0]),
diff --git a/sensorsafrica/tests/test_sensordatastats_view.py b/sensorsafrica/tests/test_sensordatastats_view.py
index 3349e6c..8e1fdf6 100644
--- a/sensorsafrica/tests/test_sensordatastats_view.py
+++ b/sensorsafrica/tests/test_sensordatastats_view.py
@@ -2,10 +2,11 @@
 
 import pytest
 from django.utils import timezone
+import dateutil.parser
 
 @pytest.mark.django_db
 class TestGettingData:
-    def test_getting_air_data_now(self, client, datavalues):
+    def test_getting_air_data_now(self, client, modified_datavalues):
         response = client.get(
             "/v2/data/air/?city=Dar es Salaam", format="json")
         assert response.status_code == 200
@@ -31,7 +32,7 @@ def test_getting_air_data_now(self, client, datavalues):
         assert result["P2"]["maximum"] == 8.0
         assert result["P2"]["minimum"] == 0.0
 
-    def test_getting_air_data_now_all_cities(self, client, datavalues):
+    def test_getting_air_data_now_all_cities(self, client, modified_datavalues):
         response = client.get("/v2/data/air/", format="json")
         assert response.status_code == 200
 
@@ -41,14 +42,14 @@ def test_getting_air_data_now_all_cities(self, client, datavalues):
 
         results = data["results"]
 
-        assert results[0]["city_slug"] == "Bagamoyo"
-        assert results[1]["city_slug"] == "Dar es Salaam"
+        assert results[0]["city_name"] == "Bagamoyo"
+        assert results[1]["city_name"] == "Dar es Salaam"
         assert "P1" in results[1]
-        assert results[1]["city_slug"] == "Dar es Salaam"
+        assert results[1]["city_name"] == "Dar es Salaam"
         assert "P2" in results[1]
-        assert results[2]["city_slug"] == "Nairobi"
+        assert results[2]["city_name"] == "Nairobi"
 
-    def test_getting_air_data_now_filter_cities(self, client, datavalues):
+    def test_getting_air_data_now_filter_cities(self, client, modified_datavalues):
         response = client.get(
             "/v2/data/air/?city=Dar es Salaam,Bagamoyo", format="json"
         )
@@ -60,13 +61,13 @@ def test_getting_air_data_now_filter_cities(self, client, datavalues):
 
         results = data["results"]
 
-        assert results[0]["city_slug"] == "Bagamoyo"
-        assert results[1]["city_slug"] == "Dar es Salaam"
+        assert results[0]["city_name"] == "Bagamoyo"
+        assert results[1]["city_name"] == "Dar es Salaam"
         assert "P1" in results[1]
-        assert results[1]["city_slug"] == "Dar es Salaam"
+        assert results[1]["city_name"] == "Dar es Salaam"
         assert "P2" in results[1]
 
-    def test_getting_air_data_value_type(self, client, datavalues):
+    def test_getting_air_data_value_type(self, client, modified_datavalues):
         response = client.get(
             "/v2/data/air/?city=Dar es Salaam&value_type=P2", format="json"
         )
@@ -80,7 +81,7 @@ def test_getting_air_data_value_type(self, client, datavalues):
         assert "temperature" not in data["results"][0]
         assert "humidity" not in data["results"][0]
 
-    def test_getting_air_data_from_date(self, client, datavalues):
+    def test_getting_air_data_from_date(self, client, modified_datavalues):
         response = client.get(
             "/v2/data/air/?city=Dar es Salaam&from=%s"
             % (timezone.now() - datetime.timedelta(days=2)).date(),
@@ -94,14 +95,12 @@ def test_getting_air_data_from_date(self, client, datavalues):
 
         # Data is in descending order by date
         most_recent_value = data["results"][0]["P2"][0]
-        most_recent_date = datetime.datetime.strptime(
-            most_recent_value["end_datetime"], "%Y-%m-%dT%H:%M:%SZ"
-        )
+        most_recent_date = dateutil.parser.parse(most_recent_value["end_datetime"])
 
         # Check today is not included
         assert most_recent_date.date() < datetime.datetime.today().date()
 
-    def test_getting_air_data_from_date_to_date(self, client, datavalues):
+    def test_getting_air_data_from_date_to_date(self, client, modified_datavalues):
         now = timezone.now()
         response = client.get(
             "/v2/data/air/?city=Dar es Salaam&from=%s&to=%s" % (
@@ -116,7 +115,7 @@ def test_getting_air_data_from_date_to_date(self, client, datavalues):
         assert type(data["results"][0]["P1"]) == list
         assert type(data["results"][0]["P2"]) == list
 
-    def test_getting_air_data_with_invalid_request(self, client, datavalues):
+    def test_getting_air_data_with_invalid_request(self, client, modified_datavalues):
         response = client.get(
             "/v2/data/air/?city=Dar es Salaam&to=2019-02-08", format="json"
         )
@@ -124,7 +123,7 @@ def test_getting_air_data_with_invalid_request(self, client, datavalues):
         assert response.json() == {
             "from": "Must be provide along with to query"}
 
-    def test_getting_air_data_with_invalid_from_request(self, client, datavalues):
+    def test_getting_air_data_with_invalid_from_request(self, client, modified_datavalues):
         response = client.get(
             "/v2/data/air/?city=Dar es Salaam&from=2019-23-08", format="json"
         )
@@ -132,7 +131,7 @@ def test_getting_air_data_with_invalid_from_request(self, client, datavalues):
         assert response.json() == {
             "from": "Must be a date in the format Y-m-d."}
 
-    def test_getting_air_data_with_invalid_to_request(self, client, datavalues):
+    def test_getting_air_data_with_invalid_to_request(self, client, modified_datavalues):
         response = client.get(
             "/v2/data/air/?city=Dar es Salaam&from=2019-02-08&to=08-02-2019",
             format="json",

From e1cf770e99b1d9c7a00fa7a9f26fb5d0eaa695d7 Mon Sep 17 00:00:00 2001
From: karimkawambwa <4308339+karimkawambwa@users.noreply.github.com>
Date: Wed, 12 Feb 2020 12:39:05 +0300
Subject: [PATCH 5/7] Additional tests

---
 .../tests/test_sensordatastats_view.py        | 54 +++++++++++++++++++
 1 file changed, 54 insertions(+)

diff --git a/sensorsafrica/tests/test_sensordatastats_view.py b/sensorsafrica/tests/test_sensordatastats_view.py
index 8e1fdf6..9d46067 100644
--- a/sensorsafrica/tests/test_sensordatastats_view.py
+++ b/sensorsafrica/tests/test_sensordatastats_view.py
@@ -166,3 +166,57 @@ def test_getting_air_data_now_with_additional_values(
 
         assert result["P2"]["maximum"] == 8.0
         assert result["P2"]["minimum"] == 0.0
+
+    def test_geting_air_data_by_the_hour(self, client, modified_datavalues):
+        now = timezone.now()
+        response = client.get(
+            "/v2/data/air/?city=Dar es Salaam&avg=hour&from=%s&to=%s" % (
+                now.date(), now.date()),
+            format="json",
+        )
+        assert response.status_code == 200
+
+        data = response.json()
+
+        assert data["count"] == 1
+        assert type(data["results"][0]["P1"]) == list
+        assert type(data["results"][0]["P2"]) == list
+
+        assert len(data["results"][0]["P1"]) == 1
+        assert len(data["results"][0]["P2"]) == 4
+
+    def test_geting_air_data_by_the_month(self, client, modified_datavalues):
+        now = timezone.now()
+        response = client.get(
+            "/v2/data/air/?city=Dar es Salaam&avg=month&from=%s&to=%s" % (
+                now.date(), now.date()),
+            format="json",
+        )
+        assert response.status_code == 200
+
+        data = response.json()
+
+        assert data["count"] == 1
+        assert type(data["results"][0]["P1"]) == list
+        assert type(data["results"][0]["P2"]) == list
+
+        assert len(data["results"][0]["P1"]) == 1
+        assert len(data["results"][0]["P2"]) == 1
+
+    def test_geting_air_data_by_the_minute(self, client, modified_datavalues):
+        now = timezone.now()
+        response = client.get(
+            "/v2/data/air/?city=Dar es Salaam&avg=minute&from=%s&to=%s" % (
+                now.date(), now.date()),
+            format="json",
+        )
+        assert response.status_code == 200
+
+        data = response.json()
+
+        assert data["count"] == 1
+        assert type(data["results"][0]["P1"]) == list
+        assert type(data["results"][0]["P2"]) == list
+
+        assert len(data["results"][0]["P1"]) == 1
+        assert len(data["results"][0]["P2"]) == 4

From 22a92c58b7bc8e9a634f3a5ef7c649cebccaabb0 Mon Sep 17 00:00:00 2001
From: karimkawambwa <4308339+karimkawambwa@users.noreply.github.com>
Date: Wed, 12 Feb 2020 12:39:38 +0300
Subject: [PATCH 6/7] Lint

---
 sensorsafrica/tests/test_sensordatastats_view.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sensorsafrica/tests/test_sensordatastats_view.py b/sensorsafrica/tests/test_sensordatastats_view.py
index 9d46067..c63f9b1 100644
--- a/sensorsafrica/tests/test_sensordatastats_view.py
+++ b/sensorsafrica/tests/test_sensordatastats_view.py
@@ -4,6 +4,7 @@
 from django.utils import timezone
 import dateutil.parser
 
+
 @pytest.mark.django_db
 class TestGettingData:
     def test_getting_air_data_now(self, client, modified_datavalues):

From 433068833d2fa0a315b21cbc37861d530b305401 Mon Sep 17 00:00:00 2001
From: karimkawambwa <4308339+karimkawambwa@users.noreply.github.com>
Date: Wed, 12 Feb 2020 12:48:00 +0300
Subject: [PATCH 7/7] Name migrations

---
 .../{0005_auto_20200211_1224.py => 0005_remove_sensordatastat.py} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename sensorsafrica/migrations/{0005_auto_20200211_1224.py => 0005_remove_sensordatastat.py} (100%)

diff --git a/sensorsafrica/migrations/0005_auto_20200211_1224.py b/sensorsafrica/migrations/0005_remove_sensordatastat.py
similarity index 100%
rename from sensorsafrica/migrations/0005_auto_20200211_1224.py
rename to sensorsafrica/migrations/0005_remove_sensordatastat.py