From ac8a377315c4796a10b9379dcf2bef18f6ce39b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Mon, 23 Jun 2025 18:59:49 +0200 Subject: [PATCH] feat: support multiple versions of the postgis extension Build multiple versions of the postgis extension on different PostgreSQL versions. Add test for the extensions and their upgrade on PostgreSQL 15 and 17. --- flake.nix | 2 + nix/ext/postgis.nix | 244 +++++++++++++++++++++++++++----------- nix/ext/tests/postgis.nix | 157 ++++++++++++++++++++++++ nix/ext/versions.json | 16 +++ 4 files changed, 351 insertions(+), 68 deletions(-) create mode 100644 nix/ext/tests/postgis.nix create mode 100644 nix/ext/versions.json diff --git a/flake.nix b/flake.nix index 18cd9ba13..629b0fc8b 100644 --- a/flake.nix +++ b/flake.nix @@ -1384,6 +1384,8 @@ devShell = devShells.default; } // pkgs.lib.optionalAttrs (system == "aarch64-linux") { inherit (basePackages) postgresql_15_debug postgresql_15_src postgresql_orioledb-17_debug postgresql_orioledb-17_src postgresql_17_debug postgresql_17_src; + } // pkgs.lib.optionalAttrs (system == "x86_64-linux") { + postgis = import ./nix/ext/tests/postgis.nix { inherit self; inherit pkgs; }; }; # Apps is a list of names of things that can be executed with 'nix run'; diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index e5ccda443..ce390eebf 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -1,86 +1,194 @@ -{ fetchurl -, lib, stdenv -, perl -, libxml2 -, postgresql -, geos -, proj -, json_c -, pkg-config -, file -, protobufc -, libiconv -, pcre2 -, nixosTests -, callPackage +{ + fetchurl, + lib, + stdenv, + perl, + libxml2, + postgresql, + geos, + proj, + json_c, + pkg-config, + file, + protobufc, + libiconv, + pcre2, + nixosTests, + callPackage, + buildEnv, }: let sfcgal = callPackage ./sfcgal/sfcgal.nix { }; gdal = callPackage ./gdal.nix { inherit postgresql; }; -in -stdenv.mkDerivation rec { pname = "postgis"; - version = "3.3.7"; - outputs = [ "out" "doc" ]; + # Load version configuration from external file + allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname}; - src = fetchurl { - url = "https://download.osgeo.org/postgis/source/postgis-${version}.tar.gz"; - sha256 = "sha256-UHJKDd5JrcJT5Z4CTYsY/va+ToU0GUPG1eHhuXTkP84="; - }; + # Filter versions compatible with current PostgreSQL version + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; - buildInputs = [ libxml2 postgresql geos proj gdal json_c protobufc pcre2.dev sfcgal ] - ++ lib.optional stdenv.isDarwin libiconv; - nativeBuildInputs = [ perl pkg-config ]; - dontDisableStatic = true; + # Derived version information + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash) supportedVersions + ); + # List of C extensions to be included in the build + cExtensions = [ + "address_standardizer" + "postgis" + "postgis_raster" + "postgis_sfcgal" + "postgis_topology" + ]; - env.NIX_LDFLAGS = "-L${lib.getLib json_c}/lib"; + sqlExtensions = [ + "address_standardizer_data_us" + "postgis_tiger_geocoder" + ]; - preConfigure = '' - sed -i 's@/usr/bin/file@${file}/bin/file@' configure - configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --disable-extension-upgrades-install --with-sfcgal" + # Build function for individual versions + build = + version: hash: + stdenv.mkDerivation rec { + inherit pname version; - makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${pname}" - ''; + outputs = [ + "out" + "doc" + ]; + + src = fetchurl { + url = "https://download.osgeo.org/postgis/source/postgis-${version}.tar.gz"; + inherit hash; + }; + + buildInputs = [ + libxml2 + postgresql + geos + proj + gdal + json_c + protobufc + pcre2.dev + sfcgal + ] ++ lib.optional stdenv.isDarwin libiconv; + nativeBuildInputs = [ + perl + pkg-config + ]; + dontDisableStatic = true; + + env.NIX_LDFLAGS = "-L${lib.getLib json_c}/lib"; + + preConfigure = '' + sed -i 's@/usr/bin/file@${file}/bin/file@' configure + configureFlags="--datadir=$out/share/postgresql --datarootdir=$out/share/postgresql --bindir=$out/bin --docdir=$doc/share/doc/${pname} --with-gdalconfig=${gdal}/bin/gdal-config --with-jsondir=${json_c.dev} --with-sfcgal --with-library-minor-version" + + makeFlags="PERL=${perl}/bin/perl datadir=$out/share/postgresql pkglibdir=$out/lib bindir=$out/bin docdir=$doc/share/doc/${pname}" + ''; + + postConfigure = '' + sed -i "s|@mkdir -p \$(DESTDIR)\$(PGSQL_BINDIR)||g ; + s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g + " \ + "raster/loader/Makefile"; + sed -i "s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g + " \ + "raster/scripts/python/Makefile"; + mkdir -p $out/bin + ln -s ${postgresql}/bin/postgres $out/bin/postgres + ''; + + postInstall = '' + MIN_MAJ_VERSION=${lib.concatStringsSep "." (lib.take 2 (builtins.splitVersion version))} + rm $out/bin/postgres + + # move control files + for ext in ${lib.concatStringsSep " " (cExtensions ++ sqlExtensions)}; do + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/$ext'|" \ + $out/share/postgresql/extension/$ext.control > $out/share/postgresql/extension/$ext--$MIN_MAJ_VERSION.control + rm $out/share/postgresql/extension/$ext.control + ln -s $out/share/postgresql/extension/$ext--${version}.sql $out/share/postgresql/extension/$ext--$MIN_MAJ_VERSION.sql + done + + # Add function definition and usage to tiger geocoder files + for file in $out/share/postgresql/extension/postgis_tiger_geocoder*--${version}.sql; do + sed -i "/SELECT postgis_extension_AddToSearchPath('tiger');/a SELECT postgis_extension_AddToSearchPath('extensions');" "$file" + done + # Original topology patching + for file in $out/share/postgresql/extension/postgis_topology*--${version}.sql; do + sed -i "/SELECT topology.AddToSearchPath('topology');/i SELECT topology.AddToSearchPath('extensions');" "$file" + done + + # For the latest version, create default control file and symlink and copy SQL upgrade scripts + if [[ "${version}" == "${latestVersion}" ]]; then + for ext in ${lib.concatStringsSep " " (cExtensions ++ sqlExtensions)}; do + { + echo "default_version = '$MIN_MAJ_VERSION'" + cat $out/share/postgresql/extension/$ext--$MIN_MAJ_VERSION.control + } > $out/share/postgresql/extension/$ext.control + done + for prog in $out/bin/*; do # */ + ln -s $prog $prog-$MIN_MAJ_VERSION + done + else + # remove migration scripts for non-latest version + find $out/share/postgresql/extension -regex '.*--.*--.*\.sql' -delete + + for prog in $out/bin/*; do # */ + mv $prog $prog-$MIN_MAJ_VERSION + done + fi + + mkdir -p $doc/share/doc/postgis + mv doc/* $doc/share/doc/postgis/ + ''; + + passthru.tests.postgis = nixosTests.postgis; + + meta = with lib; { + description = "Geographic Objects for PostgreSQL"; + homepage = "https://postgis.net/"; + changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${version}/NEWS"; + license = licenses.gpl2; + inherit (postgresql.meta) platforms; + }; + }; +in +buildEnv { + name = pname; + paths = packages; + + pathsToLink = [ + "/lib" + "/share/postgresql/extension" + ]; + postBuild = '' + # Verify all expected library files are present + expectedFiles=${toString (numberOfVersions * builtins.length cExtensions)} + actualFiles=$(ls -A $out/lib/*${postgresql.dlSuffix} | wc -l) - postConfigure = '' - sed -i "s|@mkdir -p \$(DESTDIR)\$(PGSQL_BINDIR)||g ; - s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g - " \ - "raster/loader/Makefile"; - sed -i "s|\$(DESTDIR)\$(PGSQL_BINDIR)|$prefix/bin|g - " \ - "raster/scripts/python/Makefile"; - mkdir -p $out/bin - ln -s ${postgresql}/bin/postgres $out/bin/postgres + if [[ "$actualFiles" != "$expectedFiles" ]]; then + echo "Error: Expected $expectedFiles library files, found $actualFiles" + echo "Files found:" + ls -la $out/lib/${pname}*${postgresql.dlSuffix} || true + exit 1 + fi ''; -postInstall = '' - rm $out/bin/postgres - for prog in $out/bin/*; do # */ - ln -s $prog $prog-${version} - done - # Add function definition and usage to tiger geocoder files - for file in $out/share/postgresql/extension/postgis_tiger_geocoder*--${version}.sql; do - sed -i "/SELECT postgis_extension_AddToSearchPath('tiger');/a SELECT postgis_extension_AddToSearchPath('extensions');" "$file" - done - # Original topology patching - for file in $out/share/postgresql/extension/postgis_topology*--${version}.sql; do - sed -i "/SELECT topology.AddToSearchPath('topology');/i SELECT topology.AddToSearchPath('extensions');" "$file" - done - mkdir -p $doc/share/doc/postgis - mv doc/* $doc/share/doc/postgis/ -''; - - passthru.tests.postgis = nixosTests.postgis; - - meta = with lib; { - description = "Geographic Objects for PostgreSQL"; - homepage = "https://postgis.net/"; - changelog = "https://git.osgeo.org/gitea/postgis/postgis/raw/tag/${version}/NEWS"; - license = licenses.gpl2; - inherit (postgresql.meta) platforms; + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; } diff --git a/nix/ext/tests/postgis.nix b/nix/ext/tests/postgis.nix new file mode 100644 index 000000000..4c97f5fe2 --- /dev/null +++ b/nix/ext/tests/postgis.nix @@ -0,0 +1,157 @@ +{ self, pkgs }: +let + pname = "postgis"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + }; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + in + '' + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + + def run_sql(query): + return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def check_upgrade_path(pg_version): + with subtest("Check ${pname} upgrade path"): + firstVersion = versions[pg_version][0] + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS ${pname};'") + run_sql(f"""CREATE EXTENSION ${pname} WITH VERSION '{firstVersion}' CASCADE;""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == firstVersion, f"Expected ${pname} version {firstVersion}, but found {installed_version}" + for version in versions[pg_version][1:]: + run_sql(f"""ALTER EXTENSION ${pname} UPDATE TO '{version}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = '${pname}';""") + assert installed_version == version, f"Expected ${pname} version {version}, but found {installed_version}" + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + check_upgrade_path("15") + + with subtest("Check ${pname} latest extension version"): + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION ${pname};'") + server.succeed("sudo -u postgres psql -c 'CREATE EXTENSION ${pname} CASCADE;'") + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension where extname = '${pname}';""") + latestVersion = versions["15"][-1] + majMinVersion = ".".join(latestVersion.split('.')[:1]) + assert f"${pname},{majMinVersion}" in installed_extensions, f"Expected ${pname} version {latestVersion}, but found {installed_extensions}" + + with subtest("switch to postgresql 17"): + server.succeed( + "${pg17-configuration}/bin/switch-to-configuration test >&2" + ) + + with subtest("Check ${pname} latest extension version after upgrade"): + installed_extensions=run_sql(r"""SELECT extname, extversion FROM pg_extension;""") + latestVersion = versions["17"][-1] + majMinVersion = ".".join(latestVersion.split('.')[:1]) + assert f"${pname},{majMinVersion}" in installed_extensions + + check_upgrade_path("17") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json new file mode 100644 index 000000000..612ffb059 --- /dev/null +++ b/nix/ext/versions.json @@ -0,0 +1,16 @@ +{ + "postgis": { + "3.3.2": { + "postgresql": [ + "15" + ], + "hash": "sha256-miohnaAFoXMKOdGVmhx87GGbHvsAm2W+gP/CW60pkGg=" + }, + "3.3.7": { + "postgresql": [ + "17" + ], + "hash": "sha256-UHJKDd5JrcJT5Z4CTYsY/va+ToU0GUPG1eHhuXTkP84=" + } + } +}