From 215a0b761376da79b2a216a874248ce1ca586983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Fran=C3=A7ois=20Roche?= Date: Thu, 3 Jul 2025 00:33:58 +0200 Subject: [PATCH] feat: support multiple versions of the pg_repack extension Build multiple versions of the pg_repack extension on different PostgreSQL versions. Add test for the extensions and their upgrade on PostgreSQL 15 and 17. --- flake.nix | 2 + nix/ext/pg_repack.nix | 187 +++++++++++++++++++++++++----------- nix/ext/tests/lib.py | 109 +++++++++++++++++++++ nix/ext/tests/pg_repack.nix | 152 +++++++++++++++++++++++++++++ nix/ext/versions.json | 32 ++++++ 5 files changed, 426 insertions(+), 56 deletions(-) create mode 100644 nix/ext/tests/lib.py create mode 100644 nix/ext/tests/pg_repack.nix create mode 100644 nix/ext/versions.json diff --git a/flake.nix b/flake.nix index 18cd9ba13..245ea011b 100644 --- a/flake.nix +++ b/flake.nix @@ -1384,6 +1384,8 @@ devShell = devShells.default; } // pkgs.lib.optionalAttrs (system == "aarch64-linux") { inherit (basePackages) postgresql_15_debug postgresql_15_src postgresql_orioledb-17_debug postgresql_orioledb-17_src postgresql_17_debug postgresql_17_src; + } // pkgs.lib.optionalAttrs (system == "x86_64-linux") { + pg_repack = import ./nix/ext/tests/pg_repack.nix { inherit self; inherit pkgs; }; }; # Apps is a list of names of things that can be executed with 'nix run'; diff --git a/nix/ext/pg_repack.nix b/nix/ext/pg_repack.nix index 076e878f2..2386d0174 100644 --- a/nix/ext/pg_repack.nix +++ b/nix/ext/pg_repack.nix @@ -1,65 +1,140 @@ -{ lib -, stdenv -, fetchFromGitHub -, openssl -, postgresql -, postgresqlTestHook -, readline -, testers -, zlib +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + postgresqlTestHook, + testers, + buildEnv, }: - -stdenv.mkDerivation (finalAttrs: { +let pname = "pg_repack"; - version = "1.5.2"; - buildInputs = postgresql.buildInputs ++ [ postgresql ]; + # Load version configuration from external file + allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname}; - src = fetchFromGitHub { - owner = "reorg"; - repo = "pg_repack"; - rev = "ver_${finalAttrs.version}"; - hash = "sha256-wfjiLkx+S3zVrAynisX1GdazueVJ3EOwQEPcgUQt7eA="; - }; + # Filter versions compatible with current PostgreSQL version + supportedVersions = lib.filterAttrs ( + _: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql + ) allVersions; - installPhase = '' - install -D bin/pg_repack -t $out/bin/ - install -D lib/pg_repack${postgresql.dlSuffix} -t $out/lib/ - install -D lib/{pg_repack--${finalAttrs.version}.sql,pg_repack.control} -t $out/share/postgresql/extension - ''; + # Derived version information + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + packages = builtins.attrValues ( + lib.mapAttrs (name: value: build name value.hash) supportedVersions + ); + + # Build function for individual versions + build = + version: hash: + stdenv.mkDerivation (finalAttrs: { + inherit pname version; + #version = "1.5.2"; + + buildInputs = postgresql.buildInputs ++ [ postgresql ]; + + src = fetchFromGitHub { + owner = "reorg"; + repo = "pg_repack"; + rev = "ver_${finalAttrs.version}"; + inherit hash; + }; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension,bin} + + mv bin/${pname} $out/bin/${pname}-${version} + + # Install shared library with version suffix + mv lib/${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix} - passthru.tests = { - version = testers.testVersion { - package = finalAttrs.finalPackage; - }; - extension = stdenv.mkDerivation { - name = "plpgsql-check-test"; - dontUnpack = true; - doCheck = true; - buildInputs = [ postgresqlTestHook ]; - nativeCheckInputs = [ (postgresql.withPackages (ps: [ ps.pg_repack ])) ]; - postgresqlTestUserOptions = "LOGIN SUPERUSER"; - failureHook = "postgresqlStop"; - checkPhase = '' - runHook preCheck - psql -a -v ON_ERROR_STOP=1 -c "CREATE EXTENSION pg_repack;" - runHook postCheck + # Create version-specific control file + sed -e "/^default_version =/d" \ + -e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}-${version}'|" \ + lib/${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control + + # Copy SQL install script + cp lib/${pname}--${version}.sql $out/share/postgresql/extension + + # For the latest version, create default control file and symlink and copy SQL upgrade scripts + if [[ "${version}" == "${latestVersion}" ]]; then + { + echo "default_version = '${version}'" + cat $out/share/postgresql/extension/${pname}--${version}.control + } > $out/share/postgresql/extension/${pname}.control + ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix} + ln -sfn $out/bin/${pname}-${latestVersion} $out/bin/${pname} + fi + #install -D bin/pg_repack -t $out/bin/ + #install -D lib/pg_repack${postgresql.dlSuffix} -t $out/lib/ + #install -D lib/{pg_repack--${finalAttrs.version}.sql,pg_repack.control} -t $out/share/postgresql/extension ''; - installPhase = "touch $out"; - }; - }; - meta = with lib; { - description = "Reorganize tables in PostgreSQL databases with minimal locks"; - longDescription = '' - pg_repack is a PostgreSQL extension which lets you remove bloat from tables and indexes, and optionally restore - the physical order of clustered indexes. Unlike CLUSTER and VACUUM FULL it works online, without holding an - exclusive lock on the processed tables during processing. pg_repack is efficient to boot, - with performance comparable to using CLUSTER directly. - ''; - homepage = "https://github.com/reorg/pg_repack"; - license = licenses.bsd3; - inherit (postgresql.meta) platforms; - mainProgram = "pg_repack"; + passthru.tests = { + version = testers.testVersion { + package = finalAttrs.finalPackage; + }; + extension = stdenv.mkDerivation { + name = "plpgsql-check-test"; + dontUnpack = true; + doCheck = true; + buildInputs = [ postgresqlTestHook ]; + nativeCheckInputs = [ (postgresql.withPackages (ps: [ ps.pg_repack ])) ]; + postgresqlTestUserOptions = "LOGIN SUPERUSER"; + failureHook = "postgresqlStop"; + checkPhase = '' + runHook preCheck + psql -a -v ON_ERROR_STOP=1 -c "CREATE EXTENSION pg_repack;" + runHook postCheck + ''; + installPhase = "touch $out"; + }; + }; + + meta = with lib; { + description = "Reorganize tables in PostgreSQL databases with minimal locks"; + longDescription = '' + pg_repack is a PostgreSQL extension which lets you remove bloat from tables and indexes, and optionally restore + the physical order of clustered indexes. Unlike CLUSTER and VACUUM FULL it works online, without holding an + exclusive lock on the processed tables during processing. pg_repack is efficient to boot, + with performance comparable to using CLUSTER directly. + ''; + homepage = "https://github.com/reorg/pg_repack"; + license = licenses.bsd3; + inherit (postgresql.meta) platforms; + mainProgram = "pg_repack"; + }; + }); +in +buildEnv { + name = pname; + paths = packages; + + pathsToLink = [ + "/bin" + "/lib" + "/share/postgresql/extension" + ]; + + postBuild = '' + # Verify all expected library files are present + expectedFiles=${toString (numberOfVersions + 1)} + actualFiles=$(ls -l $out/lib/${pname}*${postgresql.dlSuffix} | wc -l) + + if [[ "$actualFiles" != "$expectedFiles" ]]; then + echo "Error: Expected $expectedFiles library files, found $actualFiles" + echo "Files found:" + ls -la $out/lib/*${postgresql.dlSuffix} || true + exit 1 + fi + ''; + + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = + "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; -}) +} diff --git a/nix/ext/tests/lib.py b/nix/ext/tests/lib.py new file mode 100644 index 000000000..16e43cbfb --- /dev/null +++ b/nix/ext/tests/lib.py @@ -0,0 +1,109 @@ +"""PostgreSQL extension testing framework for multi-version compatibility. + +This module provides a test framework for PostgreSQL extensions that need to be +tested across multiple PostgreSQL versions and extension versions. It handles +installation, upgrades, and version verification of PostgreSQL extensions. +""" + +from typing import Sequence, Mapping +from test_driver.machine import Machine + +Versions = Mapping[str, Sequence[str]] + +class PostgresExtensionTest(object): + def __init__(self, vm: Machine, extension_name: str, versions: Versions, support_upgrade: bool = True): + """Initialize the PostgreSQL extension test framework. + + Args: + vm: Test machine instance for executing commands + extension_name: Name of the PostgreSQL extension to test + versions: Mapping of PostgreSQL versions to available extension versions + support_upgrade: Whether the extension supports in-place upgrades + """ + self.vm = vm + self.extension_name = extension_name + self.versions = versions + self.support_upgrade = support_upgrade + + def run_sql(self, query: str) -> str: + return self.vm.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def drop_extension(self): + self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name};") + + def install_extension(self, version: str): + self.run_sql(f"""CREATE EXTENSION {self.extension_name} WITH VERSION '{version}' CASCADE;""") + # Verify version was installed correctly + self.assert_version_matches(version) + + def update_extension(self, version: str): + self.run_sql(f"""ALTER EXTENSION {self.extension_name} UPDATE TO '{version}';""") + # Verify version was installed correctly + self.assert_version_matches(version) + + def get_installed_version(self) -> str: + """Get the currently installed version of the extension. + + Returns: + Version string of the currently installed extension, + or empty string if extension is not installed + """ + return self.run_sql(f"""SELECT extversion FROM pg_extension WHERE extname = '{self.extension_name}';""") + + def assert_version_matches(self, expected_version: str): + """Check if the installed version matches the expected version. + + Args: + expected_version: Expected version string to verify against + + Raises: + AssertionError: If the installed version does not match the expected version + """ + installed_version = self.get_installed_version() + assert installed_version == expected_version, f"Expected version {expected_version}, but found {installed_version}" + + def check_upgrade_path(self, pg_version): + """Test the complete upgrade path for a PostgreSQL version. + + This method tests all available extension versions for a given PostgreSQL + version, either through in-place upgrades or reinstallation depending on + the support_upgrade setting. + + Args: + pg_version: PostgreSQL version to test (e.g., "14", "15") + + Raises: + ValueError: If no versions are available for the specified PostgreSQL version + AssertionError: If version installation or upgrade fails + """ + available_versions = self.versions.get(pg_version, []) + if not available_versions: + raise ValueError(f"No versions available for PostgreSQL version {pg_version}") + + # Install and verify first version + firstVersion = available_versions[0] + self.drop_extension() + self.install_extension(firstVersion) + + # Test remaining versions + for version in available_versions[1:]: + if self.support_upgrade: + self.update_extension(version) + else: + self.drop_extension() + self.install_extension(version) + + + def check_install_last_version(self, pg_version: str) -> str: + """Test if the install of the last version of the extension works for a given PostgreSQL version. + + Args: + pg_version: PostgreSQL version to check (e.g., "14", "15") + """ + available_versions = self.versions.get(pg_version, []) + if not available_versions: + raise ValueError(f"No versions available for PostgreSQL version {pg_version}") + last_version = available_versions[-1] + self.drop_extension() + self.install_extension(last_version) + return last_version diff --git a/nix/ext/tests/pg_repack.nix b/nix/ext/tests/pg_repack.nix new file mode 100644 index 000000000..0f57eef11 --- /dev/null +++ b/nix/ext/tests/pg_repack.nix @@ -0,0 +1,152 @@ +{ self, pkgs }: +let + pname = "pg_repack"; + inherit (pkgs) lib; + installedExtension = + postgresMajorVersion: self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/${pname}-all"; + versions = postgresqlMajorVersion: (installedExtension postgresqlMajorVersion).versions; + postgresqlWithExtension = + postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-${pname}"; + paths = [ + postgresql + postgresql.lib + (installedExtension majorVersion) + ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ + "/" + "/bin" + "/lib" + ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in + pkg; +in +self.inputs.nixpkgs.lib.nixos.runTest { + name = pname; + hostPkgs = pkgs; + nodes.server = + { config, ... }: + { + virtualisation = { + forwardPorts = [ + { + from = "host"; + host.port = 13022; + guest.port = 22; + } + ]; + }; + services.openssh = { + enable = true; + }; + + services.postgresql = { + enable = true; + package = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + enableTCPIP = true; + initialScript = pkgs.writeText "init-postgres-with-password" '' + CREATE USER test WITH PASSWORD 'secret'; + ''; + authentication = '' + host test postgres samenet scram-sha-256 + ''; + }; + + networking.firewall.allowedTCPPorts = [ config.services.postgresql.settings.port ]; + + specialisation.postgresql17.configuration = { + services.postgresql = { + package = lib.mkForce (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17); + }; + + systemd.services.postgresql-migrate = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + StateDirectory = "postgresql"; + WorkingDirectory = "${builtins.dirOf config.services.postgresql.dataDir}"; + }; + script = + let + oldPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + newPostgresql = postgresqlWithExtension self.packages.${pkgs.system}.postgresql_17; + oldDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${oldPostgresql.psqlSchema}"; + newDataDir = "${builtins.dirOf config.services.postgresql.dataDir}/${newPostgresql.psqlSchema}"; + in + '' + if [[ ! -d ${newDataDir} ]]; then + install -d -m 0700 -o postgres -g postgres "${newDataDir}" + ${newPostgresql}/bin/initdb -D "${newDataDir}" + ${newPostgresql}/bin/pg_upgrade --old-datadir "${oldDataDir}" --new-datadir "${newDataDir}" \ + --old-bindir "${oldPostgresql}/bin" --new-bindir "${newPostgresql}/bin" + else + echo "${newDataDir} already exists" + fi + ''; + }; + + systemd.services.postgresql = { + after = [ "postgresql-migrate.service" ]; + requires = [ "postgresql-migrate.service" ]; + }; + }; + }; + testScript = + { nodes, ... }: + let + pg17-configuration = "${nodes.server.system.build.toplevel}/specialisation/postgresql17"; + in + '' + versions = { + "15": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "15"))}], + "17": [${lib.concatStringsSep ", " (map (s: ''"${s}"'') (versions "17"))}], + } + extension_name = "${pname}" + support_upgrade = False + pg17_configuration = "${pg17-configuration}" + + ${builtins.readFile ./lib.py} + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + test = PostgresExtensionTest(server, extension_name, versions, support_upgrade) + + with subtest("Check upgrade path with postgresql 15"): + test.check_upgrade_path("15") + + last_version = None + with subtest("Check the install of the last version of the extension"): + last_version = test.check_install_last_version("15") + + with subtest("switch to postgresql 17"): + server.succeed( + f"{pg17_configuration}/bin/switch-to-configuration test >&2" + ) + + with subtest("Check last version of the extension after upgrade"): + test.assert_version_matches(last_version) + + with subtest("Check upgrade path with postgresql 17"): + test.check_upgrade_path("17") + ''; +} diff --git a/nix/ext/versions.json b/nix/ext/versions.json new file mode 100644 index 000000000..e8f62e1c1 --- /dev/null +++ b/nix/ext/versions.json @@ -0,0 +1,32 @@ +{ + "pg_repack": { + "1.4.8": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-Et8aMRzG7ez0uy9wG6qsg57/kPPZdUhb+/gFxW86D08=" + }, + "1.5.0": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-do80phyMxwcRIkYyUt9z02z7byNQhK+pbSaCUmzG+4c=" + }, + "1.5.1": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-wJwy4qIt6/kgWqT6HbckUVqDayDkixqHpYiC1liLERw=" + }, + "1.5.2": { + "postgresql": [ + "15", + "17" + ], + "hash": "sha256-wfjiLkx+S3zVrAynisX1GdazueVJ3EOwQEPcgUQt7eA=" + } + } +}