diff --git a/.gitignore b/.gitignore index d32cc8f60..9f7b68818 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ result* db/schema.sql common-nix.vars.pkr.hcl + +.envrc diff --git a/flake.nix b/flake.nix index 18cd9ba13..49953c9e0 100644 --- a/flake.nix +++ b/flake.nix @@ -112,7 +112,7 @@ /*"postgis"*/ ]; - #FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue + # FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue # with building timescaledb with the orioledb patched version of postgresql orioledbPsqlExtensions = [ /* pljava */ @@ -131,7 +131,6 @@ ourExtensions = [ ./nix/ext/rum.nix ./nix/ext/timescaledb.nix - ./nix/ext/timescaledb-2.9.1.nix ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix @@ -162,16 +161,13 @@ ./nix/ext/plv8.nix ]; - #Where we import and build the orioledb extension, we add on our custom extensions - # plus the orioledb option - #we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions + # Where we import and build the orioledb extension, we add on our + # custom extensions plus the orioledb option. We're not using + # timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase + # extensions orioleFilteredExtensions = builtins.filter - ( - x: - x != ./nix/ext/timescaledb.nix && - x != ./nix/ext/timescaledb-2.9.1.nix && - x != ./nix/ext/plv8.nix - ) ourExtensions; + (x: x != ./nix/ext/timescaledb.nix && x != ./nix/ext/plv8.nix) + ourExtensions; orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; dbExtensions17 = orioleFilteredExtensions; @@ -1377,6 +1373,7 @@ # The list of exported 'checks' that are run with every run of 'nix # flake check'. This is run in the CI system, as well. checks = { + timescaledb = import ./nix/tests/timescaledb.nix { inherit self; inherit pkgs; }; psql_15 = makeCheckHarness basePackages.psql_15.bin; psql_17 = makeCheckHarness basePackages.psql_17.bin; psql_orioledb-17 = makeCheckHarness basePackages.psql_orioledb-17.bin; diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix deleted file mode 100644 index 92d5d73fe..000000000 --- a/nix/ext/timescaledb-2.9.1.nix +++ /dev/null @@ -1,50 +0,0 @@ -{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: - -stdenv.mkDerivation rec { - pname = "timescaledb-apache"; - version = "2.9.1"; - - nativeBuildInputs = [ cmake ]; - buildInputs = [ postgresql openssl libkrb5 ]; - - src = fetchFromGitHub { - owner = "timescale"; - repo = "timescaledb"; - rev = version; - hash = "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="; - }; - - cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] - ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; - - # Fix the install phase which tries to install into the pgsql extension dir, - # and cannot be manually overridden. This is rather fragile but works OK. - postPatch = '' - for x in CMakeLists.txt sql/CMakeLists.txt; do - substituteInPlace "$x" \ - --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" - done - - for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do - substituteInPlace "$x" \ - --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" - done - ''; - - - # timescaledb-2.9.1.so already exists in the lib directory - # we have no need for the timescaledb.so or control file - postInstall = '' - rm $out/lib/timescaledb.so - rm $out/share/postgresql/extension/timescaledb.control - ''; - - meta = with lib; { - description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; - homepage = "https://www.timescale.com/"; - changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; - platforms = postgresql.meta.platforms; - license = licenses.asl20; - broken = versionOlder postgresql.version "13"; - }; -} diff --git a/nix/ext/timescaledb.nix b/nix/ext/timescaledb.nix index 1d7360762..204ce5451 100644 --- a/nix/ext/timescaledb.nix +++ b/nix/ext/timescaledb.nix @@ -1,42 +1,97 @@ -{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: +{ pkgs, lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: -stdenv.mkDerivation rec { - pname = "timescaledb-apache"; - version = "2.16.1"; +let + pname = "timescaledb"; + build = version: hash: revision: + stdenv.mkDerivation rec { + inherit pname version; - nativeBuildInputs = [ cmake ]; - buildInputs = [ postgresql openssl libkrb5 ]; + nativeBuildInputs = [ cmake ]; + buildInputs = [ postgresql openssl libkrb5 ]; - src = fetchFromGitHub { - owner = "timescale"; - repo = "timescaledb"; - rev = version; - hash = "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w="; - }; + src = fetchFromGitHub { + owner = "timescale"; + repo = "timescaledb"; + rev = version; + inherit hash; + }; - cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] - ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; - - # Fix the install phase which tries to install into the pgsql extension dir, - # and cannot be manually overridden. This is rather fragile but works OK. - postPatch = '' - for x in CMakeLists.txt sql/CMakeLists.txt; do - substituteInPlace "$x" \ - --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" - done - - for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do - substituteInPlace "$x" \ - --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" - done - ''; + cmakeFlags = [ + "-DSEND_TELEMETRY_DEFAULT=OFF" + "-DREGRESS_CHECKS=OFF" + "-DTAP_CHECKS=OFF" + "-DAPACHE_ONLY=1" + ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + + postPatch = '' + for x in CMakeLists.txt sql/CMakeLists.txt; do + if [ -f "$x" ]; then + substituteInPlace "$x" \ + --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" + fi + done + + for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do + if [ -f "$x" ]; then + substituteInPlace "$x" \ + --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" + fi + done + ''; - meta = with lib; { - description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; - homepage = "https://www.timescale.com/"; - changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; - platforms = postgresql.meta.platforms; - license = licenses.asl20; - broken = versionOlder postgresql.version "13"; + postInstall = '' + if [ -f $out/lib/timescaledb.so ]; then + mv $out/lib/timescaledb.so $out/lib/timescaledb-${version}.so + fi + if [ -f $out/share/postgresql/extension/timescaledb.control ]; then + mv $out/share/postgresql/extension/timescaledb.control $out/share/postgresql/extension/timescaledb--${version}.control + fi + ''; + + meta = with lib; { + description = + "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; + homepage = "https://www.timescale.com/"; + changelog = + "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; + license = licenses.postgresql; + inherit (postgresql.meta) platforms; + }; + }; + + allVersions = + (builtins.fromJSON (builtins.readFile ./versions.json)).timescaledb; + supportedVersions = lib.filterAttrs (_: value: + builtins.elem (lib.versions.major postgresql.version) value.postgresql) + allVersions; + versions = lib.naturalSort (lib.attrNames supportedVersions); + latestVersion = lib.last versions; + numberOfVersions = builtins.length versions; + packages = builtins.attrValues + (lib.mapAttrs (name: value: build name value.hash (value.revision or name)) + supportedVersions); +in pkgs.buildEnv { + name = pname; + paths = packages; + postBuild = '' + { + echo "default_version = '${latestVersion}'" + cat $out/share/postgresql/extension/${pname}--${latestVersion}.control + } > $out/share/postgresql/extension/${pname}.control + ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix} + + # checks + (set -x + test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${ + toString (numberOfVersions + 1) + }" + ) + ''; + pathsToLink = [ "/lib" "/share/postgresql/extension" ]; + passthru = { + inherit versions numberOfVersions; + pname = "${pname}-all"; + version = "multi-" + lib.concatStringsSep "-" + (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions); }; } diff --git a/nix/ext/versions.json b/nix/ext/versions.json new file mode 100644 index 000000000..742e2f4af --- /dev/null +++ b/nix/ext/versions.json @@ -0,0 +1,16 @@ +{ + "timescaledb": { + "2.9.1": { + "postgresql": [ + "15" + ], + "hash": "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8=" + }, + "2.16.1": { + "postgresql": [ + "15" + ], + "hash": "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w=" + } + } +} diff --git a/nix/tests/timescaledb.nix b/nix/tests/timescaledb.nix new file mode 100644 index 000000000..1aab8db36 --- /dev/null +++ b/nix/tests/timescaledb.nix @@ -0,0 +1,83 @@ +{ self, pkgs }: +let + inherit (pkgs) lib; + installedExtension = postgresMajorVersion: + self.packages.${pkgs.system}."psql_${postgresMajorVersion}/exts/timescaledb-all"; + versions = (installedExtension "15").versions; + firstVersion = lib.head versions; + latestVersion = lib.last versions; + postgresqlWithExtension = postgresql: + let + majorVersion = lib.versions.major postgresql.version; + pkg = pkgs.buildEnv { + name = "postgresql-${majorVersion}-timescaledb"; + paths = [ postgresql postgresql.lib (installedExtension majorVersion) ]; + passthru = { + inherit (postgresql) version psqlSchema; + lib = pkg; + withPackages = _: pkg; + }; + nativeBuildInputs = [ pkgs.makeWrapper ]; + pathsToLink = [ "/" "/bin" "/lib" ]; + postBuild = '' + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_ctl --set NIX_PGLIBDIR $out/lib + wrapProgram $out/bin/pg_upgrade --set NIX_PGLIBDIR $out/lib + ''; + }; + in pkg; +in self.inputs.nixpkgs.lib.nixos.runTest { + name = "timescaledb"; + hostPkgs = pkgs; + nodes.server = { config, ... }: { + virtualisation = { + forwardPorts = [{ + from = "host"; + host.port = 13022; + guest.port = 22; + }]; + }; + services.openssh = { enable = true; }; + users.users.root.openssh.authorizedKeys.keys = [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIArkmq6Th79Z4klW6Urgi4phN8yq769/l/10jlE00tU9" + ]; + + services.postgresql = { + enable = true; + package = + postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15; + settings = { shared_preload_libraries = "timescaledb"; }; + }; + + specialisation.postgresql15.configuration = { + services.postgresql = { + package = lib.mkForce + (postgresqlWithExtension self.packages.${pkgs.system}.postgresql_15); + }; + }; + }; + testScript = { nodes, ... }: '' + def run_sql(query): + return server.succeed(f"""sudo -u postgres psql -t -A -F\",\" -c \"{query}\" """).strip() + + def check_upgrade_path(): + with subtest("Check timescaledb upgrade path"): + server.succeed("sudo -u postgres psql -c 'DROP EXTENSION IF EXISTS timescaledb;'") + run_sql(r"""CREATE EXTENSION timescaledb WITH VERSION \"${firstVersion}\";""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';""") + assert installed_version == "${firstVersion}", f"Expected timescaledb version ${firstVersion}, but found {installed_version}" + for version in [${ + lib.concatStringsSep ", " (map (s: ''"${s}"'') versions) + }][1:]: + run_sql(f"""ALTER EXTENSION timescaledb UPDATE TO '{version}';""") + installed_version = run_sql(r"""SELECT extversion FROM pg_extension WHERE extname = 'timescaledb';""") + assert installed_version == version, f"Expected timescaledb version {version}, but found {installed_version}" + + start_all() + + server.wait_for_unit("multi-user.target") + server.wait_for_unit("postgresql.service") + + check_upgrade_path() + ''; +}