areas: remove geojson_4326, add geom_4326

Thus, add marshmallow schemas for serialization, notably geojson serialization.

requirements.in

@@ -1,6 +1,7 @@
 alembic
 flask>=2.1
 flask-sqlalchemy
+flask-marshmallow
 python-dotenv
 sqlalchemy<2
 utils-flask-sqlalchemy>=0.3.0

src/ref_geo/env.py

@@ -2,6 +2,7 @@
 from importlib import import_module
 
 from flask_sqlalchemy import SQLAlchemy
+from flask_marshmallow import Marshmallow
 
 
 db_path = environ.get("FLASK_SQLALCHEMY_DB")
@@ -14,4 +15,16 @@
     environ["FLASK_SQLALCHEMY_DB"] = f"{__name__}.db"
 
 
-__all__ = ["db"]
+ma_path = environ.get("FLASK_MARSHMALLOW")
+if ma_path and ma_path != f"{__name__}.ma":
+    ma_module_name, ma_object_name = ma_path.rsplit(".", 1)
+    ma_module = import_module(ma_module_name)
+    ma = getattr(ma_module, ma_object_name)
+else:
+    ma = Marshmallow()
+    environ["FLASK_MARSHMALLOW"] = f"{__name__}.ma"
+    ma.SQLAlchemySchema.OPTIONS_CLASS.session = db.session
+    ma.SQLAlchemyAutoSchema.OPTIONS_CLASS.session = db.session
+
+
+__all__ = ["db", "ma"]

src/ref_geo/migrations/utils.py

@@ -14,12 +14,26 @@
 schema = "ref_geo"
 
 
-"""
-Supprimer les zones d’un type donnée, e.g. 'DEP', 'COM', …
-"""
+def geom_4326_exists():
+    return (
+        op.get_bind()
+        .execute(
+            """
+            SELECT EXISTS (
+                SELECT 1
+                FROM information_schema.COLUMNS
+                WHERE table_schema = 'ref_geo' AND table_name='l_areas' AND column_name = 'geom_4326'
+            )
+            """
+        )
+        .scalar()
+    )
 
 
 def delete_area_with_type(area_type):
+    """
+    Supprimer les zones d’un type donnée, e.g. 'DEP', 'COM', …
+    """
     op.execute(
         f"""
         DELETE FROM {schema}.l_areas la
@@ -52,18 +66,33 @@ def create_temporary_grids_table(schema, temp_table_name):
 
 def insert_grids_and_drop_temporary_table(schema, temp_table_name, area_type):
     logger.info("Copy grids in l_areas…")
-    op.execute(
-        f"""
-        INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geojson_4326)
-        SELECT
-            {schema}.get_id_area_type('{area_type}') AS id_type,
-            cd_sig,
-            code,
-            ST_Transform(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
-            geojson
-        FROM {schema}.{temp_table_name}
-    """
-    )
+    if geom_4326_exists():
+        # We insert geom and geom_4326 to avoid double conversion like 2154 → 3312 → 4326
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geom_4326)
+            SELECT
+                {schema}.get_id_area_type('{area_type}') AS id_type,
+                cd_sig,
+                code,
+                ST_Transform(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                ST_SetSRID(ST_GeomFromGeoJSON(geojson), 4326)
+            FROM {schema}.{temp_table_name}
+        """
+        )
+    else:  # legacy column geojson_4326
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geojson_4326)
+            SELECT
+                {schema}.get_id_area_type('{area_type}') AS id_type,
+                cd_sig,
+                code,
+                ST_Transform(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                geojson
+            FROM {schema}.{temp_table_name}
+        """
+        )
     logger.info("Copy grids in li_grids…")
     op.execute(
         f"""

src/ref_geo/migrations/versions/0dfdbfbccd63_ref_geo_french_municipalities.py

@@ -10,6 +10,7 @@
 from ref_geo.migrations.utils import (
     schema,
     delete_area_with_type,
+    geom_4326_exists,
 )
 from utils_flask_sqla.migrations.utils import logger, open_remote_file
 
@@ -59,24 +60,44 @@ def upgrade():
         logger.info("Inserting municipalities data in temporary table…")
         cursor.copy_expert(f"COPY {schema}.{temp_table_name} FROM STDIN", geofile)
     logger.info("Copy municipalities in l_areas…")
-    op.execute(
-        f"""
-        INSERT INTO {schema}.l_areas (
-            id_type,
-            area_code,
-            area_name,
-            geom,
-            geojson_4326
+    if geom_4326_exists():
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (
+                id_type,
+                area_code,
+                area_name,
+                geom,
+                geom_4326
+            )
+            SELECT
+                {schema}.get_id_area_type('COM') AS id_type,
+                insee_com,
+                nom_com,
+                ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                ST_SetSRID(ST_GeomFromGeoJSON(geojson), 4326)
+            FROM {schema}.{temp_table_name}
+        """
+        )
+    else:
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (
+                id_type,
+                area_code,
+                area_name,
+                geom,
+                geojson_4326
+            )
+            SELECT
+                {schema}.get_id_area_type('COM') AS id_type,
+                insee_com,
+                nom_com,
+                ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                geojson
+            FROM {schema}.{temp_table_name}
+        """
         )
-        SELECT
-            {schema}.get_id_area_type('COM') AS id_type,
-            insee_com,
-            nom_com,
-            ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
-            geojson
-        FROM {schema}.{temp_table_name}
-    """
-    )
     logger.info("Copy municipalities in li_municipalities…")
     op.execute(
         f"""

src/ref_geo/migrations/versions/3fdaa1805575_ref_geo_french_departments.py

@@ -7,7 +7,7 @@
 from alembic import op
 from shutil import copyfileobj
 
-from ref_geo.migrations.utils import schema, delete_area_with_type
+from ref_geo.migrations.utils import schema, delete_area_with_type, geom_4326_exists
 from utils_flask_sqla.migrations.utils import logger, open_remote_file
 
 
@@ -50,18 +50,32 @@ def upgrade():
         logger.info("Inserting departments data in temporary table…")
         cursor.copy_expert(f"COPY {schema}.{temp_table_name} FROM STDIN", geofile)
     logger.info("Copy departments data in l_areas…")
-    op.execute(
-        f"""
-        INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geojson_4326)
-        SELECT
-            {schema}.get_id_area_type('DEP') AS id_type,
-            insee_dep,
-            nom_dep,
-            ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
-            geojson
-        FROM {schema}.{temp_table_name}
-    """
-    )
+    if geom_4326_exists():
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geom_4326)
+            SELECT
+                {schema}.get_id_area_type('DEP') AS id_type,
+                insee_dep,
+                nom_dep,
+                ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                ST_SetSRID(ST_GeomFromGeoJSON(geojson), 4326)
+            FROM {schema}.{temp_table_name}
+        """
+        )
+    else:
+        op.execute(
+            f"""
+            INSERT INTO {schema}.l_areas (id_type, area_code, area_name, geom, geojson_4326)
+            SELECT
+                {schema}.get_id_area_type('DEP') AS id_type,
+                insee_dep,
+                nom_dep,
+                ST_TRANSFORM(geom, Find_SRID('{schema}', 'l_areas', 'geom')),
+                geojson
+            FROM {schema}.{temp_table_name}
+        """
+        )
     logger.info("Re-indexing…")
     op.execute(f"REINDEX INDEX {schema}.index_l_areas_geom")
     logger.info("Dropping temporary departments table…")

src/ref_geo/migrations/versions/bc2fcc772b46_geom_4326.py

@@ -0,0 +1,163 @@
+"""Add column LAreas.geom_4326
+
+Revision ID: bc2fcc772b46
+Revises: 795f6ea8ec45
+Create Date: 2023-11-28 17:34:39.273235
+
+"""
+from alembic import op
+import sqlalchemy as sa
+from geoalchemy2 import Geometry
+
+
+# revision identifiers, used by Alembic.
+revision = "bc2fcc772b46"
+down_revision = "795f6ea8ec45"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("DROP TRIGGER tri_calculate_geojson ON ref_geo.l_areas")
+    op.execute("DROP FUNCTION ref_geo.fct_tri_calculate_geojson()")
+    op.drop_column(
+        schema="ref_geo",
+        table_name="l_areas",
+        column_name="geojson_4326",
+    )
+    op.add_column(
+        schema="ref_geo",
+        table_name="l_areas",
+        column=sa.Column("geom_4326", Geometry("MULTIPOLYGON", 4326)),
+    )
+    op.execute(
+        """
+        CREATE FUNCTION ref_geo.fct_tri_transform_geom()
+          RETURNS trigger AS
+          $BODY$
+            DECLARE
+              local_srid integer;
+              c integer;
+            BEGIN
+              IF (TG_OP = 'INSERT') THEN
+                -- Insert policy: we set geom from geom_4326 if geom is null and geom_4326 is not null, and reciprocally.
+                -- If both geom and geom_4326 have been set (or both are null), we do nothing.
+                IF (NEW.geom IS NULL AND NEW.geom_4326 IS NOT NULL) THEN
+                  NEW.geom = ST_Transform(NEW.geom_4326, local_srid);
+                  RAISE NOTICE '(I) Updated geom';
+                ELSEIF (NEW.geom IS NOT NULL AND NEW.geom_4326 IS NULL) THEN
+                  NEW.geom_4326 = ST_Transform(NEW.geom, 4326);
+                  RAISE NOTICE '(I) Updated geom_4326';
+                END IF;
+              ELSEIF (TG_OP = 'UPDATE') THEN
+                -- Update policy: we set geom from geom_4326 if geom_4326 have been updated to non null value,
+                -- unless geom have also been modified to non null value, and reciprocally.
+                -- We also set geom from geom_4326 if geom is modified to null, and geom_4326 is not null (modified or not),
+                -- in order to be consistent when updating one or two columns at the same time.
+                IF (
+                  NEW.geom_4326 IS NOT NULL
+                  AND
+                  (
+                    (OLD.geom IS NOT DISTINCT FROM NEW.geom AND OLD.geom_4326 IS DISTINCT FROM NEW.geom_4326)
+                    OR
+                    (NEW.geom IS NULL AND OLD.geom IS NOT NULL)
+                  )
+                ) THEN
+                  SELECT INTO local_srid Find_SRID('ref_geo', 'l_areas', 'geom');
+                  NEW.geom = ST_Transform(NEW.geom_4326, local_srid);
+                  RAISE NOTICE '(U) Updated geom';
+                ELSEIF (
+                  NEW.geom IS NOT NULL
+                  AND
+                  (
+                    (OLD.geom_4326 IS NOT DISTINCT FROM NEW.geom_4326 AND OLD.geom IS DISTINCT FROM NEW.geom)
+                    OR
+                    (NEW.geom_4326 IS NULL AND OLD.geom_4326 IS NOT NULL)
+                  )
+                ) THEN
+                  NEW.geom_4326 = ST_Transform(NEW.geom, 4326);
+                  RAISE NOTICE '(U) Updated geom_4326';
+                END IF;
+              END IF;
+              RETURN NEW;
+            END;
+          $BODY$
+          LANGUAGE plpgsql VOLATILE
+          COST 100;
+        """
+    )
+    # Set geom_4326 before creating trigger!
+    op.execute(
+        """
+        UPDATE
+            ref_geo.l_areas
+        SET
+            geom_4326 = ST_Transform(geom, 4326)
+        WHERE
+            geom IS NOT NULL;
+        """
+    )
+    op.execute(
+        """
+        CREATE TRIGGER tri_transform_geom_insert
+            BEFORE
+                INSERT ON ref_geo.l_areas
+            FOR EACH
+                ROW
+            EXECUTE FUNCTION
+                ref_geo.fct_tri_transform_geom();
+        CREATE TRIGGER tri_transform_geom_update
+            BEFORE
+                UPDATE ON ref_geo.l_areas
+            FOR EACH
+                ROW
+            EXECUTE FUNCTION
+                ref_geo.fct_tri_transform_geom();
+        """
+    )
+
+
+def downgrade():
+    op.add_column(
+        schema="ref_geo",
+        table_name="l_areas",
+        column=sa.Column("geojson_4326", sa.String),
+    )
+    op.execute(
+        """
+       UPDATE
+           ref_geo.l_areas
+       SET
+           geojson_4326 = ST_AsGeoJson(geom_4326)
+       """
+    )
+    op.execute("DROP TRIGGER tri_transform_geom_insert ON ref_geo.l_areas")
+    op.execute("DROP TRIGGER tri_transform_geom_update ON ref_geo.l_areas")
+    op.execute("DROP FUNCTION ref_geo.fct_tri_transform_geom()")
+    op.drop_column(
+        schema="ref_geo",
+        table_name="l_areas",
+        column_name="geom_4326",
+    )
+    op.execute(
+        """
+       CREATE OR REPLACE FUNCTION ref_geo.fct_tri_calculate_geojson()
+          RETURNS trigger AS
+         $BODY$
+           BEGIN
+             NEW.geojson_4326 = public.ST_asgeojson(public.st_transform(NEW.geom, 4326));
+             RETURN NEW;
+           END;
+         $BODY$
+         LANGUAGE plpgsql VOLATILE
+         COST 100;
+       """
+    )
+    op.execute(
+        """
+       CREATE TRIGGER tri_calculate_geojson
+           BEFORE INSERT OR UPDATE OF geom ON ref_geo.l_areas
+           FOR EACH ROW
+           EXECUTE PROCEDURE ref_geo.fct_tri_calculate_geojson();
+       """
+    )