Skip to content

feat: support multiple versions of the pg_repack extension #1688

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -1384,6 +1384,8 @@
devShell = devShells.default;
} // pkgs.lib.optionalAttrs (system == "aarch64-linux") {
inherit (basePackages) postgresql_15_debug postgresql_15_src postgresql_orioledb-17_debug postgresql_orioledb-17_src postgresql_17_debug postgresql_17_src;
} // pkgs.lib.optionalAttrs (system == "x86_64-linux") {
pg_repack = import ./nix/ext/tests/pg_repack.nix { inherit self; inherit pkgs; };
};

# Apps is a list of names of things that can be executed with 'nix run';
Expand Down
187 changes: 131 additions & 56 deletions nix/ext/pg_repack.nix
Original file line number Diff line number Diff line change
@@ -1,65 +1,140 @@
{ lib
, stdenv
, fetchFromGitHub
, openssl
, postgresql
, postgresqlTestHook
, readline
, testers
, zlib
{
lib,
stdenv,
fetchFromGitHub,
postgresql,
postgresqlTestHook,
testers,
buildEnv,
}:

stdenv.mkDerivation (finalAttrs: {
let
pname = "pg_repack";
version = "1.5.2";

buildInputs = postgresql.buildInputs ++ [ postgresql ];
# Load version configuration from external file
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname};

src = fetchFromGitHub {
owner = "reorg";
repo = "pg_repack";
rev = "ver_${finalAttrs.version}";
hash = "sha256-wfjiLkx+S3zVrAynisX1GdazueVJ3EOwQEPcgUQt7eA=";
};
# Filter versions compatible with current PostgreSQL version
supportedVersions = lib.filterAttrs (
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
) allVersions;

installPhase = ''
install -D bin/pg_repack -t $out/bin/
install -D lib/pg_repack${postgresql.dlSuffix} -t $out/lib/
install -D lib/{pg_repack--${finalAttrs.version}.sql,pg_repack.control} -t $out/share/postgresql/extension
'';
# Derived version information
versions = lib.naturalSort (lib.attrNames supportedVersions);
latestVersion = lib.last versions;
numberOfVersions = builtins.length versions;
packages = builtins.attrValues (
lib.mapAttrs (name: value: build name value.hash) supportedVersions
);

# Build function for individual versions
build =
version: hash:
stdenv.mkDerivation (finalAttrs: {
inherit pname version;
#version = "1.5.2";

buildInputs = postgresql.buildInputs ++ [ postgresql ];

src = fetchFromGitHub {
owner = "reorg";
repo = "pg_repack";
rev = "ver_${finalAttrs.version}";
inherit hash;
};

installPhase = ''
mkdir -p $out/{lib,share/postgresql/extension,bin}

mv bin/${pname} $out/bin/${pname}-${version}

# Install shared library with version suffix
mv lib/${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix}

passthru.tests = {
version = testers.testVersion {
package = finalAttrs.finalPackage;
};
extension = stdenv.mkDerivation {
name = "plpgsql-check-test";
dontUnpack = true;
doCheck = true;
buildInputs = [ postgresqlTestHook ];
nativeCheckInputs = [ (postgresql.withPackages (ps: [ ps.pg_repack ])) ];
postgresqlTestUserOptions = "LOGIN SUPERUSER";
failureHook = "postgresqlStop";
checkPhase = ''
runHook preCheck
psql -a -v ON_ERROR_STOP=1 -c "CREATE EXTENSION pg_repack;"
runHook postCheck
# Create version-specific control file
sed -e "/^default_version =/d" \
-e "s|^module_pathname = .*|module_pathname = '\$libdir/${pname}-${version}'|" \
lib/${pname}.control > $out/share/postgresql/extension/${pname}--${version}.control

# Copy SQL install script
cp lib/${pname}--${version}.sql $out/share/postgresql/extension

# For the latest version, create default control file and symlink and copy SQL upgrade scripts
if [[ "${version}" == "${latestVersion}" ]]; then
{
echo "default_version = '${version}'"
cat $out/share/postgresql/extension/${pname}--${version}.control
} > $out/share/postgresql/extension/${pname}.control
ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
ln -sfn $out/bin/${pname}-${latestVersion} $out/bin/${pname}
fi
#install -D bin/pg_repack -t $out/bin/
#install -D lib/pg_repack${postgresql.dlSuffix} -t $out/lib/
#install -D lib/{pg_repack--${finalAttrs.version}.sql,pg_repack.control} -t $out/share/postgresql/extension
'';
installPhase = "touch $out";
};
};

meta = with lib; {
description = "Reorganize tables in PostgreSQL databases with minimal locks";
longDescription = ''
pg_repack is a PostgreSQL extension which lets you remove bloat from tables and indexes, and optionally restore
the physical order of clustered indexes. Unlike CLUSTER and VACUUM FULL it works online, without holding an
exclusive lock on the processed tables during processing. pg_repack is efficient to boot,
with performance comparable to using CLUSTER directly.
'';
homepage = "https://github.com/reorg/pg_repack";
license = licenses.bsd3;
inherit (postgresql.meta) platforms;
mainProgram = "pg_repack";
passthru.tests = {
version = testers.testVersion {
package = finalAttrs.finalPackage;
};
extension = stdenv.mkDerivation {
name = "plpgsql-check-test";
dontUnpack = true;
doCheck = true;
buildInputs = [ postgresqlTestHook ];
nativeCheckInputs = [ (postgresql.withPackages (ps: [ ps.pg_repack ])) ];
postgresqlTestUserOptions = "LOGIN SUPERUSER";
failureHook = "postgresqlStop";
checkPhase = ''
runHook preCheck
psql -a -v ON_ERROR_STOP=1 -c "CREATE EXTENSION pg_repack;"
runHook postCheck
'';
installPhase = "touch $out";
};
};

meta = with lib; {
description = "Reorganize tables in PostgreSQL databases with minimal locks";
longDescription = ''
pg_repack is a PostgreSQL extension which lets you remove bloat from tables and indexes, and optionally restore
the physical order of clustered indexes. Unlike CLUSTER and VACUUM FULL it works online, without holding an
exclusive lock on the processed tables during processing. pg_repack is efficient to boot,
with performance comparable to using CLUSTER directly.
'';
homepage = "https://github.com/reorg/pg_repack";
license = licenses.bsd3;
inherit (postgresql.meta) platforms;
mainProgram = "pg_repack";
};
});
in
buildEnv {
name = pname;
paths = packages;

pathsToLink = [
"/bin"
"/lib"
"/share/postgresql/extension"
];

postBuild = ''
# Verify all expected library files are present
expectedFiles=${toString (numberOfVersions + 1)}
actualFiles=$(ls -l $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)

if [[ "$actualFiles" != "$expectedFiles" ]]; then
echo "Error: Expected $expectedFiles library files, found $actualFiles"
echo "Files found:"
ls -la $out/lib/*${postgresql.dlSuffix} || true
exit 1
fi
'';

passthru = {
inherit versions numberOfVersions;
pname = "${pname}-all";
version =
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
};
})
}
109 changes: 109 additions & 0 deletions nix/ext/tests/lib.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
"""PostgreSQL extension testing framework for multi-version compatibility.

This module provides a test framework for PostgreSQL extensions that need to be
tested across multiple PostgreSQL versions and extension versions. It handles
installation, upgrades, and version verification of PostgreSQL extensions.
"""

from typing import Sequence, Mapping
from test_driver.machine import Machine

Versions = Mapping[str, Sequence[str]]

class PostgresExtensionTest(object):
def __init__(self, vm: Machine, extension_name: str, versions: Versions, support_upgrade: bool = True):
"""Initialize the PostgreSQL extension test framework.

Args:
vm: Test machine instance for executing commands
extension_name: Name of the PostgreSQL extension to test
versions: Mapping of PostgreSQL versions to available extension versions
support_upgrade: Whether the extension supports in-place upgrades
"""
self.vm = vm
self.extension_name = extension_name
self.versions = versions
self.support_upgrade = support_upgrade

def run_sql(self, query: str) -> str:
return self.vm.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip()

def drop_extension(self):
self.run_sql(f"DROP EXTENSION IF EXISTS {self.extension_name};")

def install_extension(self, version: str):
self.run_sql(f"""CREATE EXTENSION {self.extension_name} WITH VERSION '{version}' CASCADE;""")
# Verify version was installed correctly
self.assert_version_matches(version)

def update_extension(self, version: str):
self.run_sql(f"""ALTER EXTENSION {self.extension_name} UPDATE TO '{version}';""")
# Verify version was installed correctly
self.assert_version_matches(version)

def get_installed_version(self) -> str:
"""Get the currently installed version of the extension.

Returns:
Version string of the currently installed extension,
or empty string if extension is not installed
"""
return self.run_sql(f"""SELECT extversion FROM pg_extension WHERE extname = '{self.extension_name}';""")

def assert_version_matches(self, expected_version: str):
"""Check if the installed version matches the expected version.

Args:
expected_version: Expected version string to verify against

Raises:
AssertionError: If the installed version does not match the expected version
"""
installed_version = self.get_installed_version()
assert installed_version == expected_version, f"Expected version {expected_version}, but found {installed_version}"

def check_upgrade_path(self, pg_version):
"""Test the complete upgrade path for a PostgreSQL version.

This method tests all available extension versions for a given PostgreSQL
version, either through in-place upgrades or reinstallation depending on
the support_upgrade setting.

Args:
pg_version: PostgreSQL version to test (e.g., "14", "15")

Raises:
ValueError: If no versions are available for the specified PostgreSQL version
AssertionError: If version installation or upgrade fails
"""
available_versions = self.versions.get(pg_version, [])
if not available_versions:
raise ValueError(f"No versions available for PostgreSQL version {pg_version}")

# Install and verify first version
firstVersion = available_versions[0]
self.drop_extension()
self.install_extension(firstVersion)

# Test remaining versions
for version in available_versions[1:]:
if self.support_upgrade:
self.update_extension(version)
else:
self.drop_extension()
self.install_extension(version)


def check_install_last_version(self, pg_version: str) -> str:
"""Test if the install of the last version of the extension works for a given PostgreSQL version.

Args:
pg_version: PostgreSQL version to check (e.g., "14", "15")
"""
available_versions = self.versions.get(pg_version, [])
if not available_versions:
raise ValueError(f"No versions available for PostgreSQL version {pg_version}")
last_version = available_versions[-1]
self.drop_extension()
self.install_extension(last_version)
return last_version
Loading
Loading