Merge pull request #3719 from unicef/lmsm-perf-improvements

lmsm performance improvements

src/etools/__init__.py

@@ -1,2 +1,2 @@
-VERSION = __version__ = '11.5.1'
+VERSION = __version__ = '11.5.3'
 NAME = 'eTools'

src/etools/applications/last_mile/serializers.py

@@ -33,12 +33,11 @@ def get_country(self, obj):
         return connection.tenant.name
 
     def get_region(self, obj):
-        # TODO: this will not work on multi country tenants . Not sure we need it at all
-        return obj.parent.name if obj.parent else ''
+        return obj.parent.name if hasattr(obj, 'parent') else ''
 
     class Meta:
         model = models.PointOfInterest
-        exclude = ('partner_organizations', 'point')
+        exclude = ('partner_organizations', 'point', 'created', 'modified', 'parent', 'other', 'private')
 
 
 class PointOfInterestLightSerializer(serializers.ModelSerializer):
@@ -94,7 +93,7 @@ class MaterialItemsSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = models.Item
-        exclude = ('material',)
+        exclude = ('material', 'transfers_history', 'created', 'modified',)
 
 
 class MaterialDetailSerializer(serializers.ModelSerializer):
@@ -103,15 +102,15 @@ class MaterialDetailSerializer(serializers.ModelSerializer):
 
     class Meta:
         model = models.Material
-        fields = '__all__'
+        exclude = ["partner_materials"]
 
 
 class MaterialListSerializer(serializers.ModelSerializer):
     description = serializers.CharField(read_only=True)
 
     class Meta:
         model = models.Material
-        fields = "__all__"
+        exclude = ["partner_materials"]
 
 
 class ItemSerializer(serializers.ModelSerializer):

src/etools/applications/last_mile/views.py

@@ -35,17 +35,19 @@ class PointOfInterestTypeViewSet(ReadOnlyModelViewSet):
 
 
 class POIQuerysetMixin:
-    def get_poi_queryset(self):
+    def get_poi_queryset(self, exclude_partner_prefetch=False):
         partner = self.request.user.partner
         if partner:
-            return (models.PointOfInterest.objects
-                    .filter(Q(partner_organizations=partner) | Q(partner_organizations__isnull=True))
-                    .filter(is_active=True)
-                    .exclude(name="UNICEF Warehouse")  # exclude UNICEF Warehouse
-                    .select_related('parent')
-                    .select_related('poi_type')
-                    .prefetch_related('partner_organizations')
-                    .order_by('name', 'id'))
+            qs = (models.PointOfInterest.objects
+                  .filter(Q(partner_organizations=partner) | Q(partner_organizations__isnull=True))
+                  .filter(is_active=True)
+                  .exclude(name="UNICEF Warehouse")  # exclude UNICEF Warehouse
+                  .select_related('parent').defer('parent__point', 'parent__geom', 'point')
+                  .select_related('poi_type')
+                  .order_by('name', 'id'))
+            if not exclude_partner_prefetch:
+                qs.prefetch_related('partner_organizations')
+            return qs
         return models.PointOfInterest.objects.none()
 
 
@@ -59,7 +61,9 @@ class PointOfInterestViewSet(POIQuerysetMixin, ModelViewSet):
     search_fields = ('name', 'p_code', 'parent__name', 'parent__p_code')
 
     def get_queryset(self):
-        return self.get_poi_queryset()
+        return self.get_poi_queryset(exclude_partner_prefetch=True).only(
+            'parent__name', 'p_code', 'name', 'is_active', 'description', 'poi_type'
+        )
 
     @action(detail=True, methods=['post'], url_path='upload-waybill',
             serializer_class=serializers.WaybillTransferSerializer)
@@ -147,17 +151,32 @@ def get_queryset(self):
         if not partner:
             return self.queryset.none()
         items_qs = models.Item.objects\
-            .select_related('transfer', 'transfer__partner_organization', 'transfer__destination_point')\
+            .select_related('transfer',
+                            'transfer__partner_organization',
+                            'transfer__partner_organization__organization',
+                            'transfer__origin_point',
+                            'transfer__origin_point__parent',
+                            'transfer__checked_in_by',
+                            'transfer__destination_point__parent',
+                            'transfer__destination_point')\
             .filter(transfer__status=models.Transfer.COMPLETED,
                     transfer__destination_point=poi.pk,
                     transfer__partner_organization=partner)\
             .exclude(transfer__transfer_type=models.Transfer.WASTAGE)\
+            .defer('transfer__origin_point__point',
+                   'transfer__destination_point__point',
+                   'transfer__origin_point__parent__geom',
+                   'transfer__origin_point__parent__point',
+                   'transfer__origin_point__parent__parent',
+                   'transfer__destination_point__parent__geom',
+                   'transfer__destination_point__parent__point',
+                   'transfer__destination_point__parent__parent')
 
         qs = models.Material.objects\
             .filter(items__in=items_qs)\
             .prefetch_related(Prefetch('items', queryset=items_qs)) \
             .annotate(description=Subquery(models.PartnerMaterial.objects.filter(
-                partner_organization=partner, material=OuterRef('id')).values('description'), output_field=CharField()))\
+                partner_organization=partner, material=OuterRef('id')).values('description')[:1], output_field=CharField()))\
             .distinct()\
             .order_by('id', 'short_description')
 
@@ -192,11 +211,20 @@ def get_queryset(self):
         partner = self.request.user.partner
         if not partner:
             return models.Transfer.objects.none()
-        qs = super(TransferViewSet, self).get_queryset()\
-            .select_related('partner_organization',
-                            'destination_point', 'origin_point',
-                            'checked_in_by', 'checked_out_by', 'origin_transfer')\
-            .filter(partner_organization=partner)
+        qs = (super(TransferViewSet, self).get_queryset()
+              .select_related('destination_point', 'origin_point',
+                              'destination_point__parent', 'origin_point__parent',
+                              'checked_in_by', 'checked_out_by', 'origin_transfer',)
+              .filter(partner_organization=partner)
+              .defer("partner_organization",
+                     "destination_point__point",
+                     "destination_point__parent__parent",
+                     "destination_point__parent__geom",
+                     "destination_point__parent__point",
+                     "origin_point__point",
+                     "origin_point__parent__parent",
+                     "origin_point__parent__geom",
+                     "origin_point__parent__point"))
         return qs
 
     def detail_qs(self):

src/etools/applications/last_mile/views_ext.py

@@ -215,12 +215,16 @@ def import_items(transfer_items):
                 except models.Material.DoesNotExist:
                     logging.error(f"No Material found in etools with # {item_dict['material']}")
                     continue
+
                 try:
-                    models.Item.objects.get(
-                        transfer__unicef_release_order=unicef_ro, unicef_ro_item=item_dict['unicef_ro_item'])
+                    models.Item.objects.get(other__itemid=f"{unicef_ro}-{item_dict['unicef_ro_item']}")
                 except models.Item.DoesNotExist:
-                    item_dict['transfer'] = transfer
-                    items_to_create.append(models.Item(**item_dict))
+                    try:
+                        models.Item.objects.get(
+                            transfer__unicef_release_order=unicef_ro, unicef_ro_item=item_dict['unicef_ro_item'])
+                    except models.Item.DoesNotExist:
+                        item_dict['transfer'] = transfer
+                        items_to_create.append(models.Item(**item_dict))
 
         models.Item.objects.bulk_create(items_to_create)
 
@@ -244,6 +248,7 @@ def post(self, request):
                         item_dict[self.item_mapping[k]] = v if v else None
                     else:
                         item_dict[self.item_mapping[k]] = strip_tags(v)
+            item_dict['other']['itemid'] = f"{transfer_dict['unicef_release_order']}-{item_dict['unicef_ro_item']}"
 
             transfer_obj = self.get_transfer(transfer_dict)
             if not transfer_obj:

src/etools/applications/locations/models.py

@@ -7,7 +7,7 @@
 class LocationsManager(TreeManager):
 
     def get_queryset(self):
-        return super().get_queryset().defer("geom", "point").select_related('parent')
+        return super().get_queryset().defer("geom", "point", "parent__geom", "parent__point").select_related('parent')
 
     def active(self):
         return self.get_queryset().filter(is_active=True)