From 09b12dac5093ae21a895ce5cd86e185e083f60e9 Mon Sep 17 00:00:00 2001 From: Robert Avram Date: Wed, 10 Jul 2024 19:19:24 -0400 Subject: [PATCH 1/3] performance improvements --- .../applications/last_mile/serializers.py | 11 ++-- src/etools/applications/last_mile/views.py | 59 +++++++++++++------ .../applications/last_mile/views_ext.py | 13 ++-- src/etools/applications/locations/models.py | 2 +- 4 files changed, 57 insertions(+), 28 deletions(-) diff --git a/src/etools/applications/last_mile/serializers.py b/src/etools/applications/last_mile/serializers.py index b25aa30b1..ae5cc8f33 100644 --- a/src/etools/applications/last_mile/serializers.py +++ b/src/etools/applications/last_mile/serializers.py @@ -33,12 +33,11 @@ def get_country(self, obj): return connection.tenant.name def get_region(self, obj): - # TODO: this will not work on multi country tenants . Not sure we need it at all - return obj.parent.name if obj.parent else '' + return obj.parent.name if hasattr(obj, 'parent') else '' class Meta: model = models.PointOfInterest - exclude = ('partner_organizations', 'point') + exclude = ('partner_organizations', 'point', 'created', 'modified') class PointOfInterestLightSerializer(serializers.ModelSerializer): @@ -94,7 +93,7 @@ class MaterialItemsSerializer(serializers.ModelSerializer): class Meta: model = models.Item - exclude = ('material',) + exclude = ('material', 'transfers_history', 'created', 'modified',) class MaterialDetailSerializer(serializers.ModelSerializer): @@ -103,7 +102,7 @@ class MaterialDetailSerializer(serializers.ModelSerializer): class Meta: model = models.Material - fields = '__all__' + exclude = ["partner_materials"] class MaterialListSerializer(serializers.ModelSerializer): @@ -111,7 +110,7 @@ class MaterialListSerializer(serializers.ModelSerializer): class Meta: model = models.Material - fields = "__all__" + exclude = ["partner_materials"] class ItemSerializer(serializers.ModelSerializer): diff --git a/src/etools/applications/last_mile/views.py b/src/etools/applications/last_mile/views.py index 938506fd7..dcc336117 100644 --- a/src/etools/applications/last_mile/views.py +++ b/src/etools/applications/last_mile/views.py @@ -35,17 +35,19 @@ class PointOfInterestTypeViewSet(ReadOnlyModelViewSet): class POIQuerysetMixin: - def get_poi_queryset(self): + def get_poi_queryset(self, exclude_partner_prefetch=False): partner = self.request.user.partner if partner: - return (models.PointOfInterest.objects - .filter(Q(partner_organizations=partner) | Q(partner_organizations__isnull=True)) - .filter(is_active=True) - .exclude(name="UNICEF Warehouse") # exclude UNICEF Warehouse - .select_related('parent') - .select_related('poi_type') - .prefetch_related('partner_organizations') - .order_by('name', 'id')) + qs = (models.PointOfInterest.objects + .filter(Q(partner_organizations=partner) | Q(partner_organizations__isnull=True)) + .filter(is_active=True) + .exclude(name="UNICEF Warehouse") # exclude UNICEF Warehouse + .select_related('parent').defer('parent__point', 'parent__geom', 'point') + .select_related('poi_type') + .order_by('name', 'id')) + if not exclude_partner_prefetch: + qs.prefetch_related('partner_organizations') + return qs return models.PointOfInterest.objects.none() @@ -59,7 +61,7 @@ class PointOfInterestViewSet(POIQuerysetMixin, ModelViewSet): search_fields = ('name', 'p_code', 'parent__name', 'parent__p_code') def get_queryset(self): - return self.get_poi_queryset() + return self.get_poi_queryset(exclude_partner_prefetch=True).defer('partner_organizations') @action(detail=True, methods=['post'], url_path='upload-waybill', serializer_class=serializers.WaybillTransferSerializer) @@ -147,17 +149,32 @@ def get_queryset(self): if not partner: return self.queryset.none() items_qs = models.Item.objects\ - .select_related('transfer', 'transfer__partner_organization', 'transfer__destination_point')\ + .select_related('transfer', + 'transfer__partner_organization', + 'transfer__partner_organization__organization', + 'transfer__origin_point', + 'transfer__origin_point__parent', + 'transfer__checked_in_by', + 'transfer__destination_point__parent', + 'transfer__destination_point')\ .filter(transfer__status=models.Transfer.COMPLETED, transfer__destination_point=poi.pk, transfer__partner_organization=partner)\ .exclude(transfer__transfer_type=models.Transfer.WASTAGE)\ + .defer('transfer__origin_point__point', + 'transfer__destination_point__point', + 'transfer__origin_point__parent__geom', + 'transfer__origin_point__parent__point', + 'transfer__origin_point__parent__parent', + 'transfer__destination_point__parent__geom', + 'transfer__destination_point__parent__point', + 'transfer__destination_point__parent__parent') qs = models.Material.objects\ .filter(items__in=items_qs)\ .prefetch_related(Prefetch('items', queryset=items_qs)) \ .annotate(description=Subquery(models.PartnerMaterial.objects.filter( - partner_organization=partner, material=OuterRef('id')).values('description'), output_field=CharField()))\ + partner_organization=partner, material=OuterRef('id')).values('description')[:1], output_field=CharField()))\ .distinct()\ .order_by('id', 'short_description') @@ -192,11 +209,19 @@ def get_queryset(self): partner = self.request.user.partner if not partner: return models.Transfer.objects.none() - qs = super(TransferViewSet, self).get_queryset()\ - .select_related('partner_organization', - 'destination_point', 'origin_point', - 'checked_in_by', 'checked_out_by', 'origin_transfer')\ - .filter(partner_organization=partner) + qs = (super(TransferViewSet, self).get_queryset() + .select_related('destination_point', 'origin_point', + 'checked_in_by', 'checked_out_by', 'origin_transfer') + .filter(partner_organization=partner) + .defer("partner_organization", + "destination_point__point", + "destination_point__parent__parent", + "destination_point__parent__geom", + "destination_point__parent__point", + "origin_point__point", + "origin_point__parent__parent", + "origin_point__parent__geom", + "origin_point__parent__point")) return qs def detail_qs(self): diff --git a/src/etools/applications/last_mile/views_ext.py b/src/etools/applications/last_mile/views_ext.py index 3039549f9..a1bb2e64f 100644 --- a/src/etools/applications/last_mile/views_ext.py +++ b/src/etools/applications/last_mile/views_ext.py @@ -215,12 +215,16 @@ def import_items(transfer_items): except models.Material.DoesNotExist: logging.error(f"No Material found in etools with # {item_dict['material']}") continue + try: - models.Item.objects.get( - transfer__unicef_release_order=unicef_ro, unicef_ro_item=item_dict['unicef_ro_item']) + models.Item.objects.get(other__itemid=f"{unicef_ro}-{item_dict['unicef_ro_item']}") except models.Item.DoesNotExist: - item_dict['transfer'] = transfer - items_to_create.append(models.Item(**item_dict)) + try: + models.Item.objects.get( + transfer__unicef_release_order=unicef_ro, unicef_ro_item=item_dict['unicef_ro_item']) + except models.Item.DoesNotExist: + item_dict['transfer'] = transfer + items_to_create.append(models.Item(**item_dict)) models.Item.objects.bulk_create(items_to_create) @@ -244,6 +248,7 @@ def post(self, request): item_dict[self.item_mapping[k]] = v if v else None else: item_dict[self.item_mapping[k]] = strip_tags(v) + item_dict['other']['itemid'] = f"{transfer_dict['unicef_release_order']}-{item_dict['unicef_ro_item']}" transfer_obj = self.get_transfer(transfer_dict) if not transfer_obj: diff --git a/src/etools/applications/locations/models.py b/src/etools/applications/locations/models.py index eb190acd0..a0ee3f760 100644 --- a/src/etools/applications/locations/models.py +++ b/src/etools/applications/locations/models.py @@ -7,7 +7,7 @@ class LocationsManager(TreeManager): def get_queryset(self): - return super().get_queryset().defer("geom", "point").select_related('parent') + return super().get_queryset().defer("geom", "point", "parent__geom", "parent__point").select_related('parent') def active(self): return self.get_queryset().filter(is_active=True) From fd03f745d647cafd075df72a56fcbd27d54e8330 Mon Sep 17 00:00:00 2001 From: Robert Avram Date: Wed, 10 Jul 2024 20:51:20 -0400 Subject: [PATCH 2/3] more tweaks --- src/etools/applications/last_mile/serializers.py | 2 +- src/etools/applications/last_mile/views.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/etools/applications/last_mile/serializers.py b/src/etools/applications/last_mile/serializers.py index ae5cc8f33..156d8544d 100644 --- a/src/etools/applications/last_mile/serializers.py +++ b/src/etools/applications/last_mile/serializers.py @@ -37,7 +37,7 @@ def get_region(self, obj): class Meta: model = models.PointOfInterest - exclude = ('partner_organizations', 'point', 'created', 'modified') + exclude = ('partner_organizations', 'point', 'created', 'modified', 'parent', 'other', 'private') class PointOfInterestLightSerializer(serializers.ModelSerializer): diff --git a/src/etools/applications/last_mile/views.py b/src/etools/applications/last_mile/views.py index dcc336117..63ae353dc 100644 --- a/src/etools/applications/last_mile/views.py +++ b/src/etools/applications/last_mile/views.py @@ -61,7 +61,9 @@ class PointOfInterestViewSet(POIQuerysetMixin, ModelViewSet): search_fields = ('name', 'p_code', 'parent__name', 'parent__p_code') def get_queryset(self): - return self.get_poi_queryset(exclude_partner_prefetch=True).defer('partner_organizations') + return self.get_poi_queryset(exclude_partner_prefetch=True).only( + 'parent__name', 'p_code', 'name', 'is_active', 'description', 'poi_type' + ) @action(detail=True, methods=['post'], url_path='upload-waybill', serializer_class=serializers.WaybillTransferSerializer) @@ -211,7 +213,8 @@ def get_queryset(self): return models.Transfer.objects.none() qs = (super(TransferViewSet, self).get_queryset() .select_related('destination_point', 'origin_point', - 'checked_in_by', 'checked_out_by', 'origin_transfer') + 'destination_point__parent', 'origin_point__parent', + 'checked_in_by', 'checked_out_by', 'origin_transfer',) .filter(partner_organization=partner) .defer("partner_organization", "destination_point__point", From 02f063d55f20183beafec527a7ebce03ea02b78e Mon Sep 17 00:00:00 2001 From: Robert Avram Date: Wed, 10 Jul 2024 20:51:48 -0400 Subject: [PATCH 3/3] Update __init__.py --- src/etools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/etools/__init__.py b/src/etools/__init__.py index a62c22494..e702ccf40 100644 --- a/src/etools/__init__.py +++ b/src/etools/__init__.py @@ -1,2 +1,2 @@ -VERSION = __version__ = '11.5.1' +VERSION = __version__ = '11.5.3' NAME = 'eTools'