chore: add script to update versions.json

.gitignore

@@ -25,3 +25,5 @@ result*
 
 db/schema.sql
 common-nix.vars.pkr.hcl
+
+.envrc

flake.nix

@@ -111,7 +111,7 @@
           /*"postgis"*/
         ];
 
-        #FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue
+        # FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue
         # with building timescaledb with the orioledb patched version of postgresql
         orioledbPsqlExtensions = [
           /* pljava */
@@ -130,7 +130,6 @@
         ourExtensions = [
           ./nix/ext/rum.nix
           ./nix/ext/timescaledb.nix
-          ./nix/ext/timescaledb-2.9.1.nix
           ./nix/ext/pgroonga.nix
           ./nix/ext/index_advisor.nix
           ./nix/ext/wal2json.nix
@@ -161,16 +160,13 @@
           ./nix/ext/plv8.nix
         ];
 
-        #Where we import and build the orioledb extension, we add on our custom extensions
-        # plus the orioledb option
-        #we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions
+        # Where we import and build the orioledb extension, we add on our
+        # custom extensions plus the orioledb option. We're not using
+        # timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase
+        # extensions
         orioleFilteredExtensions = builtins.filter
-          (
-            x:
-            x != ./nix/ext/timescaledb.nix &&
-            x != ./nix/ext/timescaledb-2.9.1.nix &&
-            x != ./nix/ext/plv8.nix
-        ) ourExtensions;
+          (x: x != ./nix/ext/timescaledb.nix && x != ./nix/ext/plv8.nix)
+          ourExtensions;
 
         orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ];
         dbExtensions17 = orioleFilteredExtensions;
@@ -1377,6 +1373,7 @@
         # The list of exported 'checks' that are run with every run of 'nix
         # flake check'. This is run in the CI system, as well.
         checks = {
+          timescaledb = import ./nix/tests/timescaledb.nix { inherit self; inherit pkgs; };
           psql_15 = makeCheckHarness basePackages.psql_15.bin;
           psql_17 = makeCheckHarness basePackages.psql_17.bin;
           psql_orioledb-17 = makeCheckHarness basePackages.psql_orioledb-17.bin;

nix/ext/timescaledb.nix

@@ -1,42 +1,98 @@
-{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }:
+{ pkgs, lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }:
 
-stdenv.mkDerivation rec {
-  pname = "timescaledb-apache";
-  version = "2.16.1";
+let
+  pname = "timescaledb";
+  build = version: hash: revision:
+    stdenv.mkDerivation rec {
+      inherit pname version;
 
-  nativeBuildInputs = [ cmake ];
-  buildInputs = [ postgresql openssl libkrb5 ];
+      nativeBuildInputs = [ cmake ];
+      buildInputs = [ postgresql openssl libkrb5 ];
 
-  src = fetchFromGitHub {
-    owner = "timescale";
-    repo = "timescaledb";
-    rev = version;
-    hash = "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w=";
-  };
+      src = fetchFromGitHub {
+        owner = "timescale";
+        repo = "timescaledb";
+        rev = version;
+        inherit hash;
+      };
 
-  cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ]
-    ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ];
-
-  # Fix the install phase which tries to install into the pgsql extension dir,
-  # and cannot be manually overridden. This is rather fragile but works OK.
-  postPatch = ''
-    for x in CMakeLists.txt sql/CMakeLists.txt; do
-      substituteInPlace "$x" \
-        --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\""
-    done
-
-    for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do
-      substituteInPlace "$x" \
-        --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\""
-    done
-  '';
+      cmakeFlags = [
+        "-DSEND_TELEMETRY_DEFAULT=OFF"
+        "-DREGRESS_CHECKS=OFF"
+        "-DTAP_CHECKS=OFF"
+        "-DAPACHE_ONLY=1"
+      ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ];
+
+      postPatch = ''
+        for x in CMakeLists.txt sql/CMakeLists.txt; do
+          if [ -f "$x" ]; then
+            substituteInPlace "$x" \
+              --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\""
+          fi
+        done
+
+        for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do
+          if [ -f "$x" ]; then
+            substituteInPlace "$x" \
+              --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\""
+          fi
+        done
+      '';
 
-  meta = with lib; {
-    description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space";
-    homepage = "https://www.timescale.com/";
-    changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md";
-    platforms = postgresql.meta.platforms;
-    license = licenses.asl20;
-    broken = versionOlder postgresql.version "13";
+      postInstall = ''
+        if [ -f $out/lib/timescaledb.so ]; then
+          mv $out/lib/timescaledb.so $out/lib/timescaledb-${version}.so
+        fi
+        if [ -f $out/share/postgresql/extension/timescaledb.control ]; then
+          mv $out/share/postgresql/extension/timescaledb.control $out/share/postgresql/extension/timescaledb--${version}.control
+        fi
+      '';
+
+      meta = with lib; {
+        description =
+          "Scales PostgreSQL for time-series data via automatic partitioning across time and space";
+        homepage = "https://www.timescale.com/";
+        changelog =
+          "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md";
+        license = licenses.postgresql;
+        inherit (postgresql.meta) platforms;
+      };
+    };
+
+  allVersions =
+    (builtins.fromJSON (builtins.readFile ./versions.json)).timescaledb;
+  supportedVersions = lib.filterAttrs (_: value:
+    (!value ? ignore || value.ignore != true) &&
+    builtins.elem (lib.versions.major postgresql.version) value.postgresql
+  ) allVersions;
+  versions = lib.naturalSort (lib.attrNames supportedVersions);
+  latestVersion = lib.last versions;
+  numberOfVersions = builtins.length versions;
+  packages = builtins.attrValues
+    (lib.mapAttrs (name: value: build name value.hash (value.revision or name))
+      supportedVersions);
+in pkgs.buildEnv {
+  name = pname;
+  paths = packages;
+  postBuild = ''
+    {
+      echo "default_version = '${latestVersion}'"
+      cat $out/share/postgresql/extension/${pname}--${latestVersion}.control
+    } > $out/share/postgresql/extension/${pname}.control
+    ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
+
+    # checks
+    (set -x
+       test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${
+         toString (numberOfVersions + 1)
+       }"
+    )
+  '';
+  pathsToLink = [ "/lib" "/share/postgresql/extension" ];
+  passthru = {
+    inherit versions numberOfVersions;
+    pname = "${pname}-all";
+    version = "multi-" + lib.concatStringsSep "-"
+      (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
   };
 }

nix/ext/update_versions_json.py

@@ -0,0 +1,90 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i python3 -p python3 git nix-prefetch-git python3Packages.packaging
+
+import subprocess
+import json
+import argparse
+from pathlib import Path
+from typing import Dict, List, Union
+from packaging.version import parse as parse_version, InvalidVersion
+
+Schema = Dict[str, Dict[str, Dict[str, Union[List[str], str, bool]]]]
+
+POSTGRES_VERSIONS: List[str] = ["15", "17"]
+VERSIONS_JSON_PATH = "versions.json"
+
+
+def run(cmd: List[str]) -> str:
+    result = subprocess.run(cmd, capture_output=True, text=True, check=True)
+    return result.stdout.strip()
+
+
+def get_tags(url: str) -> Dict[str, str]:
+    output = run(["git", "ls-remote", "--tags", url])
+    tags: Dict[str, str] = {}
+    for line in output.splitlines():
+        if "^{}" not in line:
+            parts = line.split("\t")
+            if len(parts) == 2:
+                commit_hash, ref = parts
+                if ref.startswith("refs/tags/"):
+                    tag = ref.removeprefix("refs/tags/")
+                    try:
+                        parse_version(tag)
+                    except InvalidVersion:
+                        continue
+                    tags[tag] = commit_hash
+    return tags
+
+
+def get_sri_hash(url: str, commit_hash: str) -> str:
+    output = run(["nix-prefetch-git", "--quiet", "--url", url, "--rev", commit_hash])
+    nix_hash = json.loads(output)["sha256"]
+    return "sha256-" + run(["nix", "hash", "to-base64", "--type", "sha256", nix_hash])
+
+
+def load() -> Schema:
+    if not Path(VERSIONS_JSON_PATH).exists():
+        return {}
+    with open(VERSIONS_JSON_PATH, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def build(name: str, url: str, data: Schema, ignore: bool = False) -> Schema:
+    tags = get_tags(url)
+    versions = data.get(name, {})
+    for tag, commit_hash in tags.items():
+        if tag in versions:
+            continue
+        if ignore:
+            versions[tag] = {"ignore": True}
+        else:
+            sri_hash = get_sri_hash(url, commit_hash)
+            versions[tag] = {"postgresql": POSTGRES_VERSIONS, "hash": sri_hash}
+    data[name] = versions
+    return data
+
+
+def save(data: Schema) -> None:
+    sorted_data = {}
+    for name, versions in data.items():
+        sorted_data[name] = dict(
+            sorted(versions.items(), key=lambda item: parse_version(item[0]))
+        )
+    with open(VERSIONS_JSON_PATH, "w", encoding="utf-8") as f:
+        json.dump(sorted_data, f, indent=2)
+        f.write("\n")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("extension_name")
+    parser.add_argument("git_repo_url")
+    parser.add_argument("--ignore", action="store_true")
+    args = parser.parse_args()
+
+    save(build(args.extension_name, args.git_repo_url, load(), ignore=args.ignore))
+
+
+if __name__ == "__main__":
+    main()