Skip to content

feat: support multiple versions of the pgroonga extension #1677

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@
paramiko
]);
sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { };
supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { };
supabase-groonga = pkgs.callPackage ./nix/ext/pgroonga/supabase-groonga-14.0.5.nix { };
mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { };
inherit (pkgs.callPackage ./nix/wal-g.nix { }) wal-g-2 wal-g-3;
# Our list of PostgreSQL extensions which come from upstream Nixpkgs.
Expand Down Expand Up @@ -132,7 +132,7 @@
./nix/ext/rum.nix
./nix/ext/timescaledb.nix
./nix/ext/timescaledb-2.9.1.nix
./nix/ext/pgroonga.nix
./nix/ext/pgroonga
./nix/ext/index_advisor.nix
./nix/ext/wal2json.nix
./nix/ext/pgmq.nix
Expand Down Expand Up @@ -344,7 +344,7 @@
PG_IDENT = "${paths.pgIdentConfigFile}";
LOCALES = "${localeArchive}";
EXTENSION_CUSTOM_SCRIPTS_DIR = "${paths.postgresqlExtensionCustomScriptsPath}";
MECAB_LIB = "${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so";
MECAB_LIB = "${supabase-groonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so";
GROONGA_DIR = "${supabase-groonga}";
MIGRATIONS_DIR = "${paths.migrationsDir}";
POSTGRESQL_SCHEMA_SQL = "${paths.postgresqlSchemaSql}";
Expand Down Expand Up @@ -1384,6 +1384,8 @@
devShell = devShells.default;
} // pkgs.lib.optionalAttrs (system == "aarch64-linux") {
inherit (basePackages) postgresql_15_debug postgresql_15_src postgresql_orioledb-17_debug postgresql_orioledb-17_src postgresql_17_debug postgresql_17_src;
} // pkgs.lib.optionalAttrs (system == "x86_64-linux") {
pgroonga = import ./nix/ext/tests/pgroonga.nix { inherit self; inherit pkgs; };
};

# Apps is a list of names of things that can be executed with 'nix run';
Expand Down
237 changes: 170 additions & 67 deletions nix/ext/pgroonga.nix
Original file line number Diff line number Diff line change
@@ -1,78 +1,181 @@
{ lib, stdenv, fetchurl, pkg-config, postgresql, msgpack-c, callPackage, mecab, makeWrapper, xxHash }:
{
lib,
stdenv,
fetchurl,
pkg-config,
postgresql,
msgpack-c,
callPackage,
mecab,
makeWrapper,
xxHash,
buildEnv,
}:
let
supabase-groonga = callPackage ../supabase-groonga.nix { };
in
stdenv.mkDerivation rec {
supabase-groonga = callPackage ./supabase-groonga { };
pname = "pgroonga";
version = "3.2.5";
src = fetchurl {
url = "https://packages.groonga.org/source/${pname}/${pname}-${version}.tar.gz";
sha256 = "sha256-GM9EOQty72hdE4Ecq8jpDudhZLiH3pP9ODLxs8DXcSY=";
};
nativeBuildInputs = [ pkg-config makeWrapper ];

buildInputs = [ postgresql msgpack-c supabase-groonga mecab ] ++ lib.optionals stdenv.isDarwin [
xxHash
];

propagatedBuildInputs = [ supabase-groonga ];
configureFlags = [
"--with-mecab=${mecab}"
"--enable-mecab"
"--with-groonga=${supabase-groonga}"
"--with-groonga-plugin-dir=${supabase-groonga}/lib/groonga/plugins"
];
# Load version configuration from external file
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).${pname};

makeFlags = [
"HAVE_MSGPACK=1"
"MSGPACK_PACKAGE_NAME=msgpack-c"
"HAVE_MECAB=1"
# Filter versions compatible with current PostgreSQL version
supportedVersions = lib.filterAttrs (
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
) allVersions;

# Derived version information
versions = lib.naturalSort (lib.attrNames supportedVersions);
latestVersion = lib.last versions;
numberOfVersions = builtins.length versions;
packages = builtins.attrValues (
lib.mapAttrs (name: value: build name value.hash) supportedVersions
);

# List of C extensions to be included in the build
cExtensions = [
"pgroonga"
"pgroonga_database"
];

NIX_CFLAGS_COMPILE = lib.optionalString stdenv.isDarwin (builtins.concatStringsSep " " [
"-Wno-error=incompatible-function-pointer-types"
"-Wno-error=format"
"-Wno-format"
"-I${supabase-groonga}/include/groonga"
"-I${xxHash}/include"
"-DPGRN_VERSION=\"${version}\""
]);

preConfigure = ''
export GROONGA_LIBS="-L${supabase-groonga}/lib -lgroonga"
export GROONGA_CFLAGS="-I${supabase-groonga}/include"
export MECAB_CONFIG="${mecab}/bin/mecab-config"
${lib.optionalString stdenv.isDarwin ''
export CPPFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\""
export CFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\""
export PG_CPPFLAGS="-Wno-error=incompatible-function-pointer-types -Wno-error=format"
''}
'';
# Build function for individual versions
build =
version: hash:
stdenv.mkDerivation rec {
inherit pname version;

src = fetchurl {
url = "https://packages.groonga.org/source/${pname}/${pname}-${version}.tar.gz";
inherit hash;
};
nativeBuildInputs = [
pkg-config
makeWrapper
];

buildInputs =
[
postgresql
msgpack-c
supabase-groonga
mecab
]
++ lib.optionals stdenv.isDarwin [
xxHash
];

propagatedBuildInputs = [ supabase-groonga ];
configureFlags = [
"--with-mecab=${mecab}"
"--enable-mecab"
"--with-groonga=${supabase-groonga}"
"--with-groonga-plugin-dir=${supabase-groonga}/lib/groonga/plugins"
];

makeFlags = [
"HAVE_MSGPACK=1"
"MSGPACK_PACKAGE_NAME=msgpack-c"
"HAVE_MECAB=1"
];

NIX_CFLAGS_COMPILE = lib.optionalString stdenv.isDarwin (
builtins.concatStringsSep " " [
"-Wno-error=incompatible-function-pointer-types"
"-Wno-error=format"
"-Wno-format"
"-I${supabase-groonga}/include/groonga"
"-I${xxHash}/include"
"-DPGRN_VERSION=\"${version}\""
]
);

preConfigure = ''
export GROONGA_LIBS="-L${supabase-groonga}/lib -lgroonga"
export GROONGA_CFLAGS="-I${supabase-groonga}/include"
export MECAB_CONFIG="${mecab}/bin/mecab-config"
${lib.optionalString stdenv.isDarwin ''
export CPPFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\""
export CFLAGS="-I${supabase-groonga}/include/groonga -I${xxHash}/include -DPGRN_VERSION=\"${version}\""
export PG_CPPFLAGS="-Wno-error=incompatible-function-pointer-types -Wno-error=format"
''}
'';

installPhase = ''
runHook preInstall

mkdir -p $out/{lib,share/postgresql/extension}

for ext in ${lib.concatStringsSep " " cExtensions}; do
# Install shared library with version suffix
mv $ext${postgresql.dlSuffix} $out/lib/$ext-${version}${postgresql.dlSuffix}

# Create version-specific control file
sed -e "/^default_version =/d" \
-e "s|^module_pathname = .*|module_pathname = '\$libdir/$ext'|" \
$ext.control > $out/share/postgresql/extension/$ext--${version}.control

# Copy SQL file to install the specific version
cp data/$ext--${version}.sql $out/share/postgresql/extension

# Create versioned control file with modified module path
sed -e "/^default_version =/d" \
-e "s|^module_pathname = .*|module_pathname = '\$libdir/$ext'|" \
$ext.control > $out/share/postgresql/extension/$ext--${version}.control

# For the latest version, create default control file and symlink and copy SQL upgrade scripts
if [[ "${version}" == "${latestVersion}" ]]; then
{
echo "default_version = '${version}'"
cat $out/share/postgresql/extension/$ext--${version}.control
} > $out/share/postgresql/extension/$ext.control
ln -sfn $ext-${version}${postgresql.dlSuffix} $out/lib/$ext${postgresql.dlSuffix}
cp data/$ext--*--*.sql $out/share/postgresql/extension
fi
done

echo "Debug: Groonga plugins directory contents:"
ls -l ${supabase-groonga}/lib/groonga/plugins/tokenizers/
'';

meta = with lib; {
description = "A PostgreSQL extension to use Groonga as the index";
longDescription = ''
PGroonga is a PostgreSQL extension to use Groonga as the index.
PostgreSQL supports full text search against languages that use only alphabet and digit.
It means that PostgreSQL doesn't support full text search against Japanese, Chinese and so on.
You can use super fast full text search feature against all languages by installing PGroonga into your PostgreSQL.
'';
homepage = "https://pgroonga.github.io/";
changelog = "https://github.com/pgroonga/pgroonga/releases/tag/${version}";
license = licenses.postgresql;
inherit (postgresql.meta) platforms;
};
};
in
buildEnv {
name = pname;
paths = packages;

pathsToLink = [
"/lib"
"/share/postgresql/extension"
];
postBuild = ''
# Verify all expected library files are present
expectedFiles=${toString ((numberOfVersions + 1) * (builtins.length cExtensions))}
actualFiles=$(ls -l $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)

installPhase = ''
mkdir -p $out/lib $out/share/postgresql/extension $out/bin
install -D pgroonga${postgresql.dlSuffix} -t $out/lib/
install -D pgroonga.control -t $out/share/postgresql/extension
install -D data/pgroonga-*.sql -t $out/share/postgresql/extension
install -D pgroonga_database${postgresql.dlSuffix} -t $out/lib/
install -D pgroonga_database.control -t $out/share/postgresql/extension
install -D data/pgroonga_database-*.sql -t $out/share/postgresql/extension

echo "Debug: Groonga plugins directory contents:"
ls -l ${supabase-groonga}/lib/groonga/plugins/tokenizers/
if [[ "$actualFiles" != "$expectedFiles" ]]; then
echo "Error: Expected $expectedFiles library files, found $actualFiles"
echo "Files found:"
ls -la $out/lib/*${postgresql.dlSuffix} || true
exit 1
fi
'';

meta = with lib; {
description = "A PostgreSQL extension to use Groonga as the index";
longDescription = ''
PGroonga is a PostgreSQL extension to use Groonga as the index.
PostgreSQL supports full text search against languages that use only alphabet and digit.
It means that PostgreSQL doesn't support full text search against Japanese, Chinese and so on.
You can use super fast full text search feature against all languages by installing PGroonga into your PostgreSQL.
'';
homepage = "https://pgroonga.github.io/";
changelog = "https://github.com/pgroonga/pgroonga/releases/tag/${version}";
license = licenses.postgresql;
platforms = postgresql.meta.platforms;
passthru = {
inherit versions numberOfVersions;
pname = "${pname}-all";
version =
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
};
}
Loading
Loading