diff --git a/.envrc.recommended b/.envrc.recommended new file mode 100644 index 000000000..a7aaf82ac --- /dev/null +++ b/.envrc.recommended @@ -0,0 +1,3 @@ +watch_file nix/devShells.nix + +use flake diff --git a/.gitignore b/.gitignore index d32cc8f60..5372bfdeb 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,8 @@ __pycache__/ result* .env-local .history +.envrc +.direnv #IDE @@ -25,3 +27,6 @@ result* db/schema.sql common-nix.vars.pkr.hcl + +# pre-commit config is managed in nix +.pre-commit-config.yaml diff --git a/flake.lock b/flake.lock index bea404a1f..167905994 100644 --- a/flake.lock +++ b/flake.lock @@ -1,6 +1,40 @@ { "nodes": { + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1749398372, + "narHash": "sha256-tYBdgS56eXYaWVW3fsnPQ/nFlgWi/Z2Ymhyu21zVM98=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "9305fe4e5c2a6fcf5ba6a3ff155720fbe4076569", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "flake-parts_2": { "inputs": { "nixpkgs-lib": [ "nix-fast-build", @@ -57,6 +91,49 @@ "type": "github" } }, + "git-hooks": { + "inputs": { + "flake-compat": "flake-compat", + "gitignore": "gitignore", + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1750779888, + "narHash": "sha256-wibppH3g/E2lxU43ZQHC5yA/7kIKLGxVEnsnVK1BtRg=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "16ec914f6fb6f599ce988427d9d94efddf25fe6d", + "type": "github" + }, + "original": { + "owner": "cachix", + "repo": "git-hooks.nix", + "type": "github" + } + }, + "gitignore": { + "inputs": { + "nixpkgs": [ + "git-hooks", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1709087332, + "narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=", + "owner": "hercules-ci", + "repo": "gitignore.nix", + "rev": "637db329424fd7e46cf4185293b9cc8c88c95394", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "gitignore.nix", + "type": "github" + } + }, "nix-editor": { "inputs": { "nixpkgs": "nixpkgs", @@ -78,7 +155,7 @@ }, "nix-fast-build": { "inputs": { - "flake-parts": "flake-parts", + "flake-parts": "flake-parts_2", "nixpkgs": "nixpkgs_2", "treefmt-nix": "treefmt-nix" }, @@ -131,6 +208,21 @@ "type": "github" } }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1750555020, + "narHash": "sha256-/MjivcZIz8dyLOTFdJzS5Yazt2QCePQBh8uZooODaYw=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "6fb7349157ee1bffd053b1fdd454aa74ff7b4aee", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, "nixpkgs_2": { "locked": { "lastModified": 1749411262, @@ -196,12 +288,15 @@ }, "root": { "inputs": { + "flake-parts": "flake-parts", "flake-utils": "flake-utils", + "git-hooks": "git-hooks", "nix-editor": "nix-editor", "nix-fast-build": "nix-fast-build", "nix2container": "nix2container", "nixpkgs": "nixpkgs_4", - "rust-overlay": "rust-overlay" + "rust-overlay": "rust-overlay", + "treefmt-nix": "treefmt-nix_2" } }, "rust-overlay": { @@ -273,6 +368,26 @@ "type": "github" } }, + "treefmt-nix_2": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1750931469, + "narHash": "sha256-0IEdQB1nS+uViQw4k3VGUXntjkDp7aAlqcxdewb/hAc=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "ac8e6f32e11e9c7f153823abc3ab007f2a65d3e1", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" + } + }, "utils": { "locked": { "lastModified": 1667395993, diff --git a/flake.nix b/flake.nix index 18cd9ba13..3aa651b09 100644 --- a/flake.nix +++ b/flake.nix @@ -8,1471 +8,33 @@ nix-editor.url = "github:snowfallorg/nix-editor"; rust-overlay.url = "github:oxalica/rust-overlay"; nix-fast-build.url = "github:Mic92/nix-fast-build"; + flake-parts.url = "github:hercules-ci/flake-parts"; + treefmt-nix.url = "github:numtide/treefmt-nix"; + treefmt-nix.inputs.nixpkgs.follows = "nixpkgs"; + git-hooks.url = "github:cachix/git-hooks.nix"; + git-hooks.inputs.nixpkgs.follows = "nixpkgs"; }; - outputs = { self, nixpkgs, flake-utils, nix-editor, rust-overlay, nix2container, ... }@inputs: - let - gitRev = "vcs=${self.shortRev or "dirty"}+${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}"; - - ourSystems = with flake-utils.lib; [ + outputs = + { flake-utils, ... }@inputs: + inputs.flake-parts.lib.mkFlake { inherit inputs; } (_: { + systems = with flake-utils.lib; [ system.x86_64-linux system.aarch64-linux system.aarch64-darwin ]; - in - flake-utils.lib.eachSystem ourSystems (system: - let - pgsqlDefaultPort = "5435"; - pgsqlDefaultHost = "localhost"; - pgsqlSuperuser = "supabase_admin"; - - pkgs = import nixpkgs { - config = { - allowUnfree = true; - permittedInsecurePackages = [ - "v8-9.7.106.18" - ]; - }; - inherit system; - overlays = [ - # NOTE: add any needed overlays here. in theory we could - # pull them from the overlays/ directory automatically, but we don't - # want to have an arbitrary order, since it might matter. being - # explicit is better. - (final: prev: { - xmrig = throw "The xmrig package has been explicitly disabled in this flake."; - }) - (import rust-overlay) - (final: prev: { - cargo-pgrx = final.callPackage ./nix/cargo-pgrx/default.nix { - inherit (final) lib; - inherit (final) darwin; - inherit (final) fetchCrate; - inherit (final) openssl; - inherit (final) pkg-config; - inherit (final) makeRustPlatform; - inherit (final) stdenv; - inherit (final) rust-bin; - }; - - buildPgrxExtension = final.callPackage ./nix/cargo-pgrx/buildPgrxExtension.nix { - inherit (final) cargo-pgrx; - inherit (final) lib; - inherit (final) Security; - inherit (final) pkg-config; - inherit (final) makeRustPlatform; - inherit (final) stdenv; - inherit (final) writeShellScriptBin; - }; - - buildPgrxExtension_0_11_3 = prev.buildPgrxExtension.override { - cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_11_3; - }; - - buildPgrxExtension_0_12_6 = prev.buildPgrxExtension.override { - cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_12_6; - }; - - buildPgrxExtension_0_12_9 = prev.buildPgrxExtension.override { - cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_12_9; - }; - - buildPgrxExtension_0_14_3 = prev.buildPgrxExtension.override { - cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_14_3; - }; - - }) - (final: prev: { - postgresql = final.callPackage ./nix/postgresql/default.nix { - inherit (final) lib stdenv fetchurl makeWrapper callPackage buildEnv newScope; - }; - }) - ]; - }; - # Define pythonEnv here - pythonEnv = pkgs.python3.withPackages (ps: with ps; [ - boto3 - docker - pytest - pytest-testinfra - requests - ec2instanceconnectcli - paramiko - ]); - sfcgal = pkgs.callPackage ./nix/ext/sfcgal/sfcgal.nix { }; - supabase-groonga = pkgs.callPackage ./nix/supabase-groonga.nix { }; - mecab-naist-jdic = pkgs.callPackage ./nix/ext/mecab-naist-jdic/default.nix { }; - inherit (pkgs.callPackage ./nix/wal-g.nix { }) wal-g-2 wal-g-3; - # Our list of PostgreSQL extensions which come from upstream Nixpkgs. - # These are maintained upstream and can easily be used here just by - # listing their name. Anytime the version of nixpkgs is upgraded, these - # may also bring in new versions of the extensions. - psqlExtensions = [ - /* pljava */ - /*"postgis"*/ - ]; - - #FIXME for now, timescaledb is not included in the orioledb version of supabase extensions, as there is an issue - # with building timescaledb with the orioledb patched version of postgresql - orioledbPsqlExtensions = [ - /* pljava */ - /*"timescaledb"*/ - ]; - - # Custom extensions that exist in our repository. These aren't upstream - # either because nobody has done the work, maintaining them here is - # easier and more expedient, or because they may not be suitable, or are - # too niche/one-off. - # - # Ideally, most of these should have copies upstream for third party - # use, but even if they did, keeping our own copies means that we can - # rollout new versions of these critical things easier without having to - # go through the upstream release engineering process. - ourExtensions = [ - ./nix/ext/rum.nix - ./nix/ext/timescaledb.nix - ./nix/ext/timescaledb-2.9.1.nix - ./nix/ext/pgroonga.nix - ./nix/ext/index_advisor.nix - ./nix/ext/wal2json.nix - ./nix/ext/pgmq.nix - ./nix/ext/pg_repack.nix - ./nix/ext/pg-safeupdate.nix - ./nix/ext/plpgsql-check.nix - ./nix/ext/pgjwt.nix - ./nix/ext/pgaudit.nix - ./nix/ext/postgis.nix - ./nix/ext/pgrouting.nix - ./nix/ext/pgtap.nix - ./nix/ext/pg_cron.nix - ./nix/ext/pgsql-http.nix - ./nix/ext/pg_plan_filter.nix - ./nix/ext/pg_net.nix - ./nix/ext/pg_hashids.nix - ./nix/ext/pgsodium.nix - ./nix/ext/pg_graphql.nix - ./nix/ext/pg_stat_monitor.nix - ./nix/ext/pg_jsonschema.nix - ./nix/ext/pgvector.nix - ./nix/ext/vault.nix - ./nix/ext/hypopg.nix - ./nix/ext/pg_tle.nix - ./nix/ext/wrappers/default.nix - ./nix/ext/supautils.nix - ./nix/ext/plv8.nix - ]; - - #Where we import and build the orioledb extension, we add on our custom extensions - # plus the orioledb option - #we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions - orioleFilteredExtensions = builtins.filter - ( - x: - x != ./nix/ext/timescaledb.nix && - x != ./nix/ext/timescaledb-2.9.1.nix && - x != ./nix/ext/plv8.nix - ) ourExtensions; - - orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; - dbExtensions17 = orioleFilteredExtensions; - getPostgresqlPackage = version: - pkgs.postgresql."postgresql_${version}"; - # Create a 'receipt' file for a given postgresql package. This is a way - # of adding a bit of metadata to the package, which can be used by other - # tools to inspect what the contents of the install are: the PSQL - # version, the installed extensions, et cetera. - # - # This takes two arguments: - # - pgbin: the postgresql package we are building on top of - # not a list of packages, but an attrset containing extension names - # mapped to versions. - # - ourExts: the list of extensions from upstream nixpkgs. This is not - # a list of packages, but an attrset containing extension names - # mapped to versions. - # - # The output is a package containing the receipt.json file, which can be - # merged with the PostgreSQL installation using 'symlinkJoin'. - makeReceipt = pgbin: ourExts: pkgs.writeTextFile { - name = "receipt"; - destination = "/receipt.json"; - text = builtins.toJSON { - revision = gitRev; - psql-version = pgbin.version; - nixpkgs = { - revision = nixpkgs.rev; - }; - extensions = ourExts; - - # NOTE this field can be used to do cache busting (e.g. - # force a rebuild of the psql packages) but also to helpfully inform - # tools what version of the schema is being used, for forwards and - # backwards compatibility - receipt-version = "1"; - }; - }; - - makeOurPostgresPkgs = version: - let - postgresql = getPostgresqlPackage version; - extensionsToUse = - if (builtins.elem version [ "orioledb-17" ]) - then orioledbExtensions - else if (builtins.elem version [ "17" ]) - then dbExtensions17 - else ourExtensions; - in - map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse; - - # Create an attrset that contains all the extensions included in a server. - makeOurPostgresPkgsSet = version: - (builtins.listToAttrs (map - (drv: - { name = drv.pname; value = drv; } - ) - (makeOurPostgresPkgs version))) - // { recurseForDerivations = true; }; - - - # Create a binary distribution of PostgreSQL, given a version. - # - # NOTE: The version here does NOT refer to the exact PostgreSQL version; - # it refers to the *major number only*, which is used to select the - # correct version of the package from nixpkgs. This is because we want - # to be able to do so in an open ended way. As an example, the version - # "15" passed in will use the nixpkgs package "postgresql_15" as the - # basis for building extensions, etc. - makePostgresBin = version: - let - postgresql = getPostgresqlPackage version; - ourExts = map (ext: { name = ext.pname; version = ext.version; }) (makeOurPostgresPkgs version); - - pgbin = postgresql.withPackages (ps: - makeOurPostgresPkgs version - ); - in - pkgs.symlinkJoin { - inherit (pgbin) name version; - paths = [ pgbin (makeReceipt pgbin ourExts) ]; - }; - - # Create an attribute set, containing all the relevant packages for a - # PostgreSQL install, wrapped up with a bow on top. There are three - # packages: - # - # - bin: the postgresql package itself, with all the extensions - # installed, and a receipt.json file containing metadata about the - # install. - # - exts: an attrset containing all the extensions, mapped to their - # package names. - makePostgres = version: rec { - bin = makePostgresBin version; - exts = makeOurPostgresPkgsSet version; - recurseForDerivations = true; - }; - - makePostgresDevSetup = { pkgs, name, extraSubstitutions ? { } }: - let - paths = { - migrationsDir = builtins.path { - name = "migrations"; - path = ./migrations/db; - }; - postgresqlSchemaSql = builtins.path { - name = "postgresql-schema"; - path = ./nix/tools/postgresql_schema.sql; - }; - pgbouncerAuthSchemaSql = builtins.path { - name = "pgbouncer-auth-schema"; - path = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; - }; - statExtensionSql = builtins.path { - name = "stat-extension"; - path = ./ansible/files/stat_extension.sql; - }; - pgconfigFile = builtins.path { - name = "postgresql.conf"; - path = ./ansible/files/postgresql_config/postgresql.conf.j2; - }; - supautilsConfigFile = builtins.path { - name = "supautils.conf"; - path = ./ansible/files/postgresql_config/supautils.conf.j2; - }; - loggingConfigFile = builtins.path { - name = "logging.conf"; - path = ./ansible/files/postgresql_config/postgresql-csvlog.conf; - }; - readReplicaConfigFile = builtins.path { - name = "readreplica.conf"; - path = ./ansible/files/postgresql_config/custom_read_replica.conf.j2; - }; - pgHbaConfigFile = builtins.path { - name = "pg_hba.conf"; - path = ./ansible/files/postgresql_config/pg_hba.conf.j2; - }; - pgIdentConfigFile = builtins.path { - name = "pg_ident.conf"; - path = ./ansible/files/postgresql_config/pg_ident.conf.j2; - }; - postgresqlExtensionCustomScriptsPath = builtins.path { - name = "extension-custom-scripts"; - path = ./ansible/files/postgresql_extension_custom_scripts; - }; - getkeyScript = builtins.path { - name = "pgsodium_getkey.sh"; - path = ./nix/tests/util/pgsodium_getkey.sh; - }; - }; - - localeArchive = - if pkgs.stdenv.isDarwin - then "${pkgs.darwin.locale}/share/locale" - else "${pkgs.glibcLocales}/lib/locale/locale-archive"; - - substitutions = { - SHELL_PATH = "${pkgs.bash}/bin/bash"; - PGSQL_DEFAULT_PORT = "${pgsqlDefaultPort}"; - PGSQL_SUPERUSER = "${pgsqlSuperuser}"; - PSQL15_BINDIR = "${basePackages.psql_15.bin}"; - PSQL17_BINDIR = "${basePackages.psql_17.bin}"; - PSQL_CONF_FILE = "${paths.pgconfigFile}"; - PSQLORIOLEDB17_BINDIR = "${basePackages.psql_orioledb-17.bin}"; - PGSODIUM_GETKEY = "${paths.getkeyScript}"; - READREPL_CONF_FILE = "${paths.readReplicaConfigFile}"; - LOGGING_CONF_FILE = "${paths.loggingConfigFile}"; - SUPAUTILS_CONF_FILE = "${paths.supautilsConfigFile}"; - PG_HBA = "${paths.pgHbaConfigFile}"; - PG_IDENT = "${paths.pgIdentConfigFile}"; - LOCALES = "${localeArchive}"; - EXTENSION_CUSTOM_SCRIPTS_DIR = "${paths.postgresqlExtensionCustomScriptsPath}"; - MECAB_LIB = "${basePackages.psql_15.exts.pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so"; - GROONGA_DIR = "${supabase-groonga}"; - MIGRATIONS_DIR = "${paths.migrationsDir}"; - POSTGRESQL_SCHEMA_SQL = "${paths.postgresqlSchemaSql}"; - PGBOUNCER_AUTH_SCHEMA_SQL = "${paths.pgbouncerAuthSchemaSql}"; - STAT_EXTENSION_SQL = "${paths.statExtensionSql}"; - CURRENT_SYSTEM = "${system}"; - } // extraSubstitutions; # Merge in any extra substitutions - in - pkgs.runCommand name - { - inherit (paths) migrationsDir postgresqlSchemaSql pgbouncerAuthSchemaSql statExtensionSql; - } '' - mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts - - # Copy config files with error handling - cp ${paths.supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; } - cp ${paths.pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; } - cp ${paths.loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; } - cp ${paths.readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; } - cp ${paths.pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; } - cp ${paths.pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; } - cp -r ${paths.postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; } - - echo "Copy operation completed" - chmod 644 $out/etc/postgresql-custom/supautils.conf - chmod 644 $out/etc/postgresql/postgresql.conf - chmod 644 $out/etc/postgresql-custom/logging.conf - chmod 644 $out/etc/postgresql/pg_hba.conf - - substitute ${./nix/tools/run-server.sh.in} $out/bin/start-postgres-server \ - ${builtins.concatStringsSep " " (builtins.attrValues (builtins.mapAttrs - (name: value: "--subst-var-by '${name}' '${value}'") - substitutions - ))} - chmod +x $out/bin/start-postgres-server - ''; - - # The base set of packages that we export from this Nix Flake, that can - # be used with 'nix build'. Don't use the names listed below; check the - # name in 'nix flake show' in order to make sure exactly what name you - # want. - basePackages = - let - # Function to get the PostgreSQL version from the attribute name - getVersion = name: - let - match = builtins.match "psql_([0-9]+)" name; - in - if match == null then null else builtins.head match; - - # Define the available PostgreSQL versions - postgresVersions = { - psql_15 = makePostgres "15"; - psql_17 = makePostgres "17"; - psql_orioledb-17 = makePostgres "orioledb-17"; - }; - - # Find the active PostgreSQL version - activeVersion = getVersion (builtins.head (builtins.attrNames postgresVersions)); - - # Function to create the pg_regress package - makePgRegress = version: - let - postgresqlPackage = pkgs."postgresql_${version}"; - in - pkgs.callPackage ./nix/ext/pg_regress.nix { - postgresql = postgresqlPackage; - }; - postgresql_15 = getPostgresqlPackage "15"; - postgresql_17 = getPostgresqlPackage "17"; - postgresql_orioledb-17 = getPostgresqlPackage "orioledb-17"; - in - postgresVersions // { - supabase-groonga = supabase-groonga; - cargo-pgrx_0_11_3 = pkgs.cargo-pgrx.cargo-pgrx_0_11_3; - cargo-pgrx_0_12_6 = pkgs.cargo-pgrx.cargo-pgrx_0_12_6; - cargo-pgrx_0_12_9 = pkgs.cargo-pgrx.cargo-pgrx_0_12_9; - cargo-pgrx_0_14_3 = pkgs.cargo-pgrx.cargo-pgrx_0_14_3; - # PostgreSQL versions. - psql_15 = postgresVersions.psql_15; - psql_17 = postgresVersions.psql_17; - psql_orioledb-17 = postgresVersions.psql_orioledb-17; - wal-g-2 = wal-g-2; - wal-g-3 = wal-g-3; - sfcgal = sfcgal; - pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15 postgresql_17 postgresql_orioledb-17; - postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; - postgresql_17_debug = if pkgs.stdenv.isLinux then postgresql_17.debug else null; - postgresql_orioledb-17_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-17.debug else null; - postgresql_15_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-15-src"; - version = postgresql_15.version; - - src = postgresql_15.src; - - nativeBuildInputs = [ pkgs.bzip2 ]; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -r . $out - ''; - - meta = with pkgs.lib; { - description = "PostgreSQL 15 source files"; - homepage = "https://www.postgresql.org/"; - license = licenses.postgresql; - platforms = platforms.all; - }; - }; - postgresql_17_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-17-src"; - version = postgresql_17.version; - src = postgresql_17.src; - - nativeBuildInputs = [ pkgs.bzip2 ]; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -r . $out - ''; - meta = with pkgs.lib; { - description = "PostgreSQL 17 source files"; - homepage = "https://www.postgresql.org/"; - license = licenses.postgresql; - platforms = platforms.all; - }; - }; - postgresql_orioledb-17_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-17-src"; - version = postgresql_orioledb-17.version; - - src = postgresql_orioledb-17.src; - - nativeBuildInputs = [ pkgs.bzip2 ]; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -r . $out - ''; - - meta = with pkgs.lib; { - description = "PostgreSQL 15 source files"; - homepage = "https://www.postgresql.org/"; - license = licenses.postgresql; - platforms = platforms.all; - }; - }; - mecab_naist_jdic = mecab-naist-jdic; - supabase_groonga = supabase-groonga; - pg_regress = makePgRegress activeVersion; - # Start a version of the server. - start-server = makePostgresDevSetup { - inherit pkgs; - name = "start-postgres-server"; - }; - - # Start a version of the client and runs migrations script on server. - start-client = - let - migrationsDir = ./migrations/db; - postgresqlSchemaSql = ./nix/tools/postgresql_schema.sql; - pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; - statExtensionSql = ./ansible/files/stat_extension.sql; - in - pkgs.runCommand "start-postgres-client" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \ - --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ - --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ - --subst-var-by 'PSQL17_BINDIR' '${basePackages.psql_17.bin}' \ - --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \ - --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ - --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ - --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ - --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' - chmod +x $out/bin/start-postgres-client - ''; - - # Migrate between two data directories. - migrate-tool = - let - configFile = ./nix/tests/postgresql.conf.in; - getkeyScript = ./nix/tests/util/pgsodium_getkey.sh; - primingScript = ./nix/tests/prime.sql; - migrationData = ./nix/tests/migrations/data.sql; - in - pkgs.runCommand "migrate-postgres" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/migrate-tool.sh.in} $out/bin/migrate-postgres \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ - --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ - --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ - --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \ - --subst-var-by 'MIGRATION_DATA' '${migrationData}' - - chmod +x $out/bin/migrate-postgres - ''; - - start-replica = pkgs.runCommand "start-postgres-replica" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/run-replica.sh.in} $out/bin/start-postgres-replica \ - --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' - chmod +x $out/bin/start-postgres-replica - ''; - pg-restore = - pkgs.runCommand "run-pg-restore" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/run-restore.sh.in} $out/bin/pg-restore \ - --subst-var-by PSQL15_BINDIR '${basePackages.psql_15.bin}' - chmod +x $out/bin/pg-restore - ''; - sync-exts-versions = pkgs.runCommand "sync-exts-versions" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \ - --subst-var-by 'YQ' '${pkgs.yq}/bin/yq' \ - --subst-var-by 'JQ' '${pkgs.jq}/bin/jq' \ - --subst-var-by 'NIX_EDITOR' '${nix-editor.packages.${system}.nix-editor}/bin/nix-editor' \ - --subst-var-by 'NIXPREFETCHURL' '${pkgs.nixVersions.nix_2_20}/bin/nix-prefetch-url' \ - --subst-var-by 'NIX' '${pkgs.nixVersions.nix_2_20}/bin/nix' - chmod +x $out/bin/sync-exts-versions - ''; - - local-infra-bootstrap = pkgs.runCommand "local-infra-bootstrap" { } '' - mkdir -p $out/bin - substitute ${./nix/tools/local-infra-bootstrap.sh.in} $out/bin/local-infra-bootstrap - chmod +x $out/bin/local-infra-bootstrap - ''; - dbmate-tool = - let - migrationsDir = ./migrations/db; - ansibleVars = ./ansible/vars.yml; - pgbouncerAuthSchemaSql = ./ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; - statExtensionSql = ./ansible/files/stat_extension.sql; - in - pkgs.runCommand "dbmate-tool" - { - buildInputs = with pkgs; [ - overmind - dbmate - nix - jq - yq - ]; - nativeBuildInputs = with pkgs; [ - makeWrapper - ]; - } '' - mkdir -p $out/bin $out/migrations - cp -r ${migrationsDir}/* $out - substitute ${./nix/tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \ - --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ - --subst-var-by 'MIGRATIONS_DIR' $out \ - --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ - --subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \ - --subst-var-by 'CURRENT_SYSTEM' '${system}' \ - --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ - --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' - chmod +x $out/bin/dbmate-tool - wrapProgram $out/bin/dbmate-tool \ - --prefix PATH : ${pkgs.lib.makeBinPath [ pkgs.overmind pkgs.dbmate pkgs.nix pkgs.jq pkgs.yq ]} - ''; - show-commands = pkgs.runCommand "show-commands" - { - nativeBuildInputs = [ pkgs.makeWrapper ]; - buildInputs = [ pkgs.nushell ]; - } '' - mkdir -p $out/bin - cat > $out/bin/show-commands << 'EOF' - #!${pkgs.nushell}/bin/nu - let json_output = (nix flake show --json --quiet --all-systems | from json) - let apps = ($json_output | get apps.${system}) - $apps | transpose name info | select name | each { |it| echo $"Run this app with: nix run .#($it.name)" } - EOF - chmod +x $out/bin/show-commands - wrapProgram $out/bin/show-commands \ - --prefix PATH : ${pkgs.nushell}/bin - ''; - update-readme = pkgs.runCommand "update-readme" - { - nativeBuildInputs = [ pkgs.makeWrapper ]; - buildInputs = [ pkgs.nushell ]; - } '' - mkdir -p $out/bin - cp ${./nix/tools/update_readme.nu} $out/bin/update-readme - chmod +x $out/bin/update-readme - wrapProgram $out/bin/update-readme \ - --prefix PATH : ${pkgs.nushell}/bin - ''; - # Script to run the AMI build and tests locally - build-test-ami = pkgs.runCommand "build-test-ami" - { - buildInputs = with pkgs; [ - packer - awscli2 - yq - jq - openssl - git - coreutils - aws-vault - ]; - } '' - mkdir -p $out/bin - cat > $out/bin/build-test-ami << 'EOL' - #!/usr/bin/env bash - set -euo pipefail - - show_help() { - cat << EOF - Usage: build-test-ami [--help] - - Build AMI images for PostgreSQL testing. - - This script will: - 1. Check for required tools and AWS authentication - 2. Build two AMI stages using Packer - 3. Clean up any temporary instances - 4. Output the final AMI name for use with run-testinfra - - Arguments: - postgres-version PostgreSQL major version to build (required) - - Options: - --help Show this help message and exit - - Requirements: - - AWS Vault profile must be set in AWS_VAULT environment variable - - Packer, AWS CLI, yq, jq, and OpenSSL must be installed - - Must be run from a git repository - - Example: - aws-vault exec -- nix run .#build-test-ami 15 - EOF - } - - # Handle help flag - if [[ "$#" -gt 0 && "$1" == "--help" ]]; then - show_help - exit 0 - fi - - export PATH="${pkgs.lib.makeBinPath (with pkgs; [ - packer - awscli2 - yq - jq - openssl - git - coreutils - aws-vault - ])}:$PATH" - - # Check for required tools - for cmd in packer aws-vault yq jq openssl; do - if ! command -v $cmd &> /dev/null; then - echo "Error: $cmd is required but not found" - exit 1 - fi - done - - # Check AWS Vault profile - if [ -z "''${AWS_VAULT:-}" ]; then - echo "Error: AWS_VAULT environment variable must be set with the profile name" - echo "Usage: aws-vault exec -- nix run .#build-test-ami " - exit 1 - fi - - # Set values - REGION="ap-southeast-1" - POSTGRES_VERSION="$1" - RANDOM_STRING=$(openssl rand -hex 8) - GIT_SHA=$(git rev-parse HEAD) - RUN_ID=$(date +%s) - - # Generate common-nix.vars.pkr.hcl - PG_VERSION=$(yq -r ".postgres_release[\"postgres$POSTGRES_VERSION\"]" ansible/vars.yml) - echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl - - # Build AMI Stage 1 - packer init amazon-arm64-nix.pkr.hcl - packer build \ - -var "git-head-version=$GIT_SHA" \ - -var "packer-execution-id=$RUN_ID" \ - -var-file="development-arm.vars.pkr.hcl" \ - -var-file="common-nix.vars.pkr.hcl" \ - -var "ansible_arguments=" \ - -var "postgres-version=$RANDOM_STRING" \ - -var "region=$REGION" \ - -var 'ami_regions=["'"$REGION"'"]' \ - -var "force-deregister=true" \ - -var "ansible_arguments=-e postgresql_major=$POSTGRES_VERSION" \ - amazon-arm64-nix.pkr.hcl - - # Build AMI Stage 2 - packer init stage2-nix-psql.pkr.hcl - packer build \ - -var "git-head-version=$GIT_SHA" \ - -var "packer-execution-id=$RUN_ID" \ - -var "postgres_major_version=$POSTGRES_VERSION" \ - -var-file="development-arm.vars.pkr.hcl" \ - -var-file="common-nix.vars.pkr.hcl" \ - -var "postgres-version=$RANDOM_STRING" \ - -var "region=$REGION" \ - -var 'ami_regions=["'"$REGION"'"]' \ - -var "force-deregister=true" \ - -var "git_sha=$GIT_SHA" \ - stage2-nix-psql.pkr.hcl - - # Cleanup instances from AMI builds - cleanup_instances() { - echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." - aws ec2 --region $REGION describe-instances \ - --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ - --query "Reservations[].Instances[].InstanceId" \ - --output text | xargs -r aws ec2 terminate-instances \ - --region $REGION --instance-ids || true - } - - # Set up traps for various signals to ensure cleanup - trap cleanup_instances EXIT HUP INT QUIT TERM - - # Create and activate virtual environment - VENV_DIR=$(mktemp -d) - trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM - python3 -m venv "$VENV_DIR" - source "$VENV_DIR/bin/activate" - - # Install required Python packages - echo "Installing required Python packages..." - pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests - - # Run the tests with aws-vault - echo "Running tests for AMI: $RANDOM_STRING using AWS Vault profile: $AWS_VAULT_PROFILE" - aws-vault exec $AWS_VAULT_PROFILE -- pytest -vv -s testinfra/test_ami_nix.py - - # Deactivate virtual environment (cleanup is handled by trap) - deactivate - EOL - chmod +x $out/bin/build-test-ami - ''; - - run-testinfra = pkgs.runCommand "run-testinfra" - { - buildInputs = with pkgs; [ - aws-vault - python3 - python3Packages.pip - coreutils - ]; - } '' - mkdir -p $out/bin - cat > $out/bin/run-testinfra << 'EOL' - #!/usr/bin/env bash - set -euo pipefail - - show_help() { - cat << EOF - Usage: run-testinfra --ami-name NAME [--aws-vault-profile PROFILE] - - Run the testinfra tests locally against a specific AMI. - - This script will: - 1. Check if aws-vault is installed and configured - 2. Set up the required environment variables - 3. Create and activate a virtual environment - 4. Install required Python packages from pip - 5. Run the tests with aws-vault credentials - 6. Clean up the virtual environment - - Required flags: - --ami-name NAME The name of the AMI to test - - Optional flags: - --aws-vault-profile PROFILE AWS Vault profile to use (default: staging) - --help Show this help message and exit - Requirements: - - aws-vault installed and configured - - Python 3 with pip - - Must be run from the repository root - - Examples: - run-testinfra --ami-name supabase-postgres-abc123 - run-testinfra --ami-name supabase-postgres-abc123 --aws-vault-profile production - EOF - } - - # Default values - AWS_VAULT_PROFILE="staging" - AMI_NAME="" - - # Parse arguments - while [[ $# -gt 0 ]]; do - case $1 in - --aws-vault-profile) - AWS_VAULT_PROFILE="$2" - shift 2 - ;; - --ami-name) - AMI_NAME="$2" - shift 2 - ;; - --help) - show_help - exit 0 - ;; - *) - echo "Error: Unexpected argument: $1" - show_help - exit 1 - ;; - esac - done - - # Check for required tools - if ! command -v aws-vault &> /dev/null; then - echo "Error: aws-vault is required but not found" - exit 1 - fi - - # Check for AMI name argument - if [ -z "$AMI_NAME" ]; then - echo "Error: --ami-name is required" - show_help - exit 1 - fi - - # Set environment variables - export AWS_REGION="ap-southeast-1" - export AWS_DEFAULT_REGION="ap-southeast-1" - export AMI_NAME="$AMI_NAME" # Export AMI_NAME for pytest - export RUN_ID="local-$(date +%s)" # Generate a unique RUN_ID - - # Function to terminate EC2 instances - terminate_instances() { - echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." - aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 --region ap-southeast-1 describe-instances \ - --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ - --query "Reservations[].Instances[].InstanceId" \ - --output text | xargs -r aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 terminate-instances \ - --region ap-southeast-1 --instance-ids || true - } - - # Set up traps for various signals to ensure cleanup - trap terminate_instances EXIT HUP INT QUIT TERM - - # Create and activate virtual environment - VENV_DIR=$(mktemp -d) - trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM - python3 -m venv "$VENV_DIR" - source "$VENV_DIR/bin/activate" - - # Install required Python packages - echo "Installing required Python packages..." - pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests - - # Function to run tests and ensure cleanup - run_tests() { - local exit_code=0 - echo "Running tests for AMI: $AMI_NAME using AWS Vault profile: $AWS_VAULT_PROFILE" - aws-vault exec "$AWS_VAULT_PROFILE" -- pytest -vv -s testinfra/test_ami_nix.py || exit_code=$? - return $exit_code - } - - # Run tests and capture exit code - run_tests - test_exit_code=$? - - # Deactivate virtual environment - deactivate - - # Explicitly call cleanup - terminate_instances - - # Exit with the test exit code - exit $test_exit_code - EOL - chmod +x $out/bin/run-testinfra - ''; - - cleanup-ami = pkgs.runCommand "cleanup-ami" - { - buildInputs = with pkgs; [ - awscli2 - aws-vault - ]; - } '' - mkdir -p $out/bin - cat > $out/bin/cleanup-ami << 'EOL' - #!/usr/bin/env bash - set -euo pipefail - - export PATH="${pkgs.lib.makeBinPath (with pkgs; [ - awscli2 - aws-vault - ])}:$PATH" - - # Check for required tools - for cmd in aws-vault; do - if ! command -v $cmd &> /dev/null; then - echo "Error: $cmd is required but not found" - exit 1 - fi - done - - # Check AWS Vault profile - if [ -z "''${AWS_VAULT:-}" ]; then - echo "Error: AWS_VAULT environment variable must be set with the profile name" - echo "Usage: aws-vault exec -- nix run .#cleanup-ami " - exit 1 - fi - - # Check for AMI name argument - if [ -z "''${1:-}" ]; then - echo "Error: AMI name must be provided" - echo "Usage: aws-vault exec -- nix run .#cleanup-ami " - exit 1 - fi - - AMI_NAME="$1" - REGION="ap-southeast-1" - - # Deregister AMIs - for AMI_PATTERN in "supabase-postgres-ci-ami-test-stage-1" "$AMI_NAME"; do - aws ec2 describe-images --region $REGION --owners self \ - --filters "Name=name,Values=$AMI_PATTERN" \ - --query 'Images[*].ImageId' --output text | while read -r ami_id; do - echo "Deregistering AMI: $ami_id" - aws ec2 deregister-image --region $REGION --image-id "$ami_id" || true - done - done - EOL - chmod +x $out/bin/cleanup-ami - ''; - - trigger-nix-build = pkgs.runCommand "trigger-nix-build" - { - buildInputs = with pkgs; [ - gh - git - coreutils - ]; - } '' - mkdir -p $out/bin - cat > $out/bin/trigger-nix-build << 'EOL' - #!/usr/bin/env bash - set -euo pipefail - - show_help() { - cat << EOF - Usage: trigger-nix-build [--help] - - Trigger the nix-build workflow for the current branch and watch its progress. - - This script will: - 1. Check if you're authenticated with GitHub - 2. Get the current branch and commit - 3. Verify you're on a standard branch (develop or release/*) or prompt for confirmation - 4. Trigger the nix-build workflow - 5. Watch the workflow progress until completion - - Options: - --help Show this help message and exit - - Requirements: - - GitHub CLI (gh) installed and authenticated - - Git installed - - Must be run from a git repository - - Example: - trigger-nix-build - EOF - } - - # Handle help flag - if [[ "$#" -gt 0 && "$1" == "--help" ]]; then - show_help - exit 0 - fi - - export PATH="${pkgs.lib.makeBinPath (with pkgs; [ - gh - git - coreutils - ])}:$PATH" - - # Check for required tools - for cmd in gh git; do - if ! command -v $cmd &> /dev/null; then - echo "Error: $cmd is required but not found" - exit 1 - fi - done - - # Check if user is authenticated with GitHub - if ! gh auth status &>/dev/null; then - echo "Error: Not authenticated with GitHub. Please run 'gh auth login' first." - exit 1 - fi - - # Get current branch and commit - BRANCH=$(git rev-parse --abbrev-ref HEAD) - COMMIT=$(git rev-parse HEAD) - - # Check if we're on a standard branch - if [[ "$BRANCH" != "develop" && ! "$BRANCH" =~ ^release/ ]]; then - echo "Warning: Running workflow from non-standard branch: $BRANCH" - echo "This is supported for testing purposes." - read -p "Continue? [y/N] " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - echo "Aborted." - exit 1 - fi - fi - - # Trigger the workflow - echo "Triggering nix-build workflow for branch $BRANCH (commit: $COMMIT)" - gh workflow run nix-build.yml --ref "$BRANCH" - - # Wait for workflow to start and get the run ID - echo "Waiting for workflow to start..." - sleep 5 - - # Get the latest run ID for this workflow - RUN_ID=$(gh run list --workflow=nix-build.yml --limit 1 --json databaseId --jq '.[0].databaseId') - - if [ -z "$RUN_ID" ]; then - echo "Error: Could not find workflow run ID" - exit 1 - fi - - echo "Watching workflow run $RUN_ID..." - echo "The script will automatically exit when the workflow completes." - echo "Press Ctrl+C to stop watching (workflow will continue running)" - echo "----------------------------------------" - - # Try to watch the run, but handle network errors gracefully - while true; do - if gh run watch "$RUN_ID" --exit-status; then - break - else - echo "Network error while watching workflow. Retrying in 5 seconds..." - echo "You can also check the status manually with: gh run view $RUN_ID" - sleep 5 - fi - done - EOL - chmod +x $out/bin/trigger-nix-build - ''; - }; - - - # Create a testing harness for a PostgreSQL package. This is used for - # 'nix flake check', and works with any PostgreSQL package you hand it. - - makeCheckHarness = pgpkg: - let - sqlTests = ./nix/tests/smoke; - pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - pg_regress = basePackages.pg_regress; - getkey-script = pkgs.stdenv.mkDerivation { - name = "pgsodium-getkey"; - buildCommand = '' - mkdir -p $out/bin - cat > $out/bin/pgsodium-getkey << 'EOF' - #!${pkgs.bash}/bin/bash - set -euo pipefail - - TMPDIR_BASE=$(mktemp -d) - - KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}" - KEY_FILE="$KEY_DIR/pgsodium.key" - - if ! mkdir -p "$KEY_DIR" 2>/dev/null; then - echo "Error: Could not create key directory $KEY_DIR" >&2 - exit 1 - fi - chmod 1777 "$KEY_DIR" - - if [[ ! -f "$KEY_FILE" ]]; then - if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then - if ! (openssl rand -hex 32 > "$KEY_FILE"); then - echo "00000000000000000000000000000000" > "$KEY_FILE" - echo "Warning: Using fallback key" >&2 - fi - fi - chmod 644 "$KEY_FILE" - fi - - if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then - cat "$KEY_FILE" - else - echo "Error: Cannot read key file $KEY_FILE" >&2 - exit 1 - fi - EOF - chmod +x $out/bin/pgsodium-getkey - ''; - }; - - # Use the shared setup but with a test-specific name - start-postgres-server-bin = makePostgresDevSetup { - inherit pkgs; - name = "start-postgres-server-test"; - extraSubstitutions = { - PGSODIUM_GETKEY = "${getkey-script}/bin/pgsodium-getkey"; - PGSQL_DEFAULT_PORT = pgPort; - }; - }; - - getVersionArg = pkg: - let - name = pkg.version; - in - if builtins.match "15.*" name != null then "15" - else if builtins.match "17.*" name != null then "17" - else if builtins.match "orioledb-17.*" name != null then "orioledb-17" - else throw "Unsupported PostgreSQL version: ${name}"; - - # Helper function to filter SQL files based on version - filterTestFiles = version: dir: - let - files = builtins.readDir dir; - isValidFile = name: - let - isVersionSpecific = builtins.match "z_.*" name != null; - matchesVersion = - if isVersionSpecific - then - if version == "orioledb-17" - then builtins.match "z_orioledb-17_.*" name != null - else if version == "17" - then builtins.match "z_17_.*" name != null - else builtins.match "z_15_.*" name != null - else true; - in - pkgs.lib.hasSuffix ".sql" name && matchesVersion; - in - pkgs.lib.filterAttrs (name: _: isValidFile name) files; - - # Get the major version for filtering - majorVersion = - let - version = builtins.trace "pgpkg.version is: ${pgpkg.version}" pgpkg.version; - _ = builtins.trace "Entering majorVersion logic"; - isOrioledbMatch = builtins.match "^17_[0-9]+$" version != null; - isSeventeenMatch = builtins.match "^17[.][0-9]+$" version != null; - result = - if isOrioledbMatch - then "orioledb-17" - else if isSeventeenMatch - then "17" - else "15"; - in - builtins.trace "Major version result: ${result}" result; # Trace the result # For "15.8" - - # Filter SQL test files - filteredSqlTests = filterTestFiles majorVersion ./nix/tests/sql; - - pgPort = if (majorVersion == "17") then - "5535" - else if (majorVersion == "15") then - "5536" - else "5537"; - - # Convert filtered tests to a sorted list of basenames (without extension) - testList = pkgs.lib.mapAttrsToList - (name: _: - builtins.substring 0 (pkgs.lib.stringLength name - 4) name - ) - filteredSqlTests; - sortedTestList = builtins.sort (a: b: a < b) testList; - - in - pkgs.runCommand "postgres-${pgpkg.version}-check-harness" - { - nativeBuildInputs = with pkgs; [ - coreutils - bash - perl - pgpkg - pg_prove - pg_regress - procps - start-postgres-server-bin - which - getkey-script - supabase-groonga - ]; - } '' - set -e - - #First we need to create a generic pg cluster for pgtap tests and run those - export GRN_PLUGINS_DIR=${supabase-groonga}/lib/groonga/plugins - PGTAP_CLUSTER=$(mktemp -d) - initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" - substitute ${./nix/tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ - --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" - echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf - echo "port = ${pgPort}" >> "$PGTAP_CLUSTER"/postgresql.conf - echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf - echo "Checking shared_preload_libraries setting:" - grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf - # Remove timescaledb if running orioledb-17 check - echo "I AM ${pgpkg.version}====================================================" - if [[ "${pgpkg.version}" == *"17"* ]]; then - perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf" - fi - #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole - echo "PGTAP_CLUSTER directory contents:" - ls -la "$PGTAP_CLUSTER" - - # Check if postgresql.conf exists - if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then - echo "postgresql.conf is missing!" - exit 1 - fi - - # PostgreSQL startup - if [[ "$(uname)" == "Darwin" ]]; then - pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p ${pgPort} -d 5" start 2>&1 - else - mkdir -p "$PGTAP_CLUSTER/sockets" - pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p ${pgPort} -d 5" start 2>&1 - fi || { - echo "pg_ctl failed to start PostgreSQL" - echo "Contents of postgresql.log:" - cat "$PGTAP_CLUSTER"/postgresql.log - exit 1 - } - for i in {1..60}; do - if pg_isready -h ${pgsqlDefaultHost} -p ${pgPort}; then - echo "PostgreSQL is ready" - break - fi - sleep 1 - if [ $i -eq 60 ]; then - echo "PostgreSQL is not ready after 60 seconds" - echo "PostgreSQL status:" - pg_ctl -D "$PGTAP_CLUSTER" status - echo "PostgreSQL log content:" - cat "$PGTAP_CLUSTER"/postgresql.log - exit 1 - fi - done - createdb -p ${pgPort} -h ${pgsqlDefaultHost} --username=supabase_admin testing - if ! psql -p ${pgPort} -h ${pgsqlDefaultHost} --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xf ${./nix/tests/prime.sql}; then - echo "Error executing SQL file. PostgreSQL log content:" - cat "$PGTAP_CLUSTER"/postgresql.log - pg_ctl -D "$PGTAP_CLUSTER" stop - exit 1 - fi - SORTED_DIR=$(mktemp -d) - for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do - psql -p ${pgPort} -h ${pgsqlDefaultHost} --username=supabase_admin -d testing -f "${./nix/tests/sql}/$t.sql" || true - done - rm -rf "$SORTED_DIR" - pg_ctl -D "$PGTAP_CLUSTER" stop - rm -rf $PGTAP_CLUSTER - - # End of pgtap tests - # from here on out we are running pg_regress tests, we use a different cluster for this - # which is start by the start-postgres-server-bin script - # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations - - unset GRN_PLUGINS_DIR - ${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize - - for i in {1..60}; do - if pg_isready -h ${pgsqlDefaultHost} -p ${pgPort} -U supabase_admin -q; then - echo "PostgreSQL is ready" - break - fi - sleep 1 - if [ $i -eq 60 ]; then - echo "PostgreSQL failed to start" - exit 1 - fi - done - - if ! psql -p ${pgPort} -h ${pgsqlDefaultHost} --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xf ${./nix/tests/prime.sql}; then - echo "Error executing SQL file" - exit 1 - fi - - mkdir -p $out/regression_output - if ! pg_regress \ - --use-existing \ - --dbname=postgres \ - --inputdir=${./nix/tests} \ - --outputdir=$out/regression_output \ - --host=${pgsqlDefaultHost} \ - --port=${pgPort} \ - --user=supabase_admin \ - ${builtins.concatStringsSep " " sortedTestList}; then - echo "pg_regress tests failed" - cat $out/regression_output/regression.diffs - exit 1 - fi - - echo "Running migrations tests" - pg_prove -p ${pgPort} -U supabase_admin -h ${pgsqlDefaultHost} -d postgres -v ${./migrations/tests}/test.sql - - # Copy logs to output - for logfile in $(find /tmp -name postgresql.log -type f); do - cp "$logfile" $out/postgresql.log - done - exit 0 - ''; - in - rec { - # The list of all packages that can be built with 'nix build'. The list - # of names that can be used can be shown with 'nix flake show' - packages = flake-utils.lib.flattenTree basePackages // { - # Any extra packages we might want to include in our package - # set can go here. - inherit (pkgs); - }; - - # The list of exported 'checks' that are run with every run of 'nix - # flake check'. This is run in the CI system, as well. - checks = { - psql_15 = makeCheckHarness basePackages.psql_15.bin; - psql_17 = makeCheckHarness basePackages.psql_17.bin; - psql_orioledb-17 = makeCheckHarness basePackages.psql_orioledb-17.bin; - inherit (basePackages) wal-g-2 wal-g-3 dbmate-tool pg_regress; - devShell = devShells.default; - } // pkgs.lib.optionalAttrs (system == "aarch64-linux") { - inherit (basePackages) postgresql_15_debug postgresql_15_src postgresql_orioledb-17_debug postgresql_orioledb-17_src postgresql_17_debug postgresql_17_src; - }; - - # Apps is a list of names of things that can be executed with 'nix run'; - # these are distinct from the things that can be built with 'nix build', - # so they need to be listed here too. - apps = - let - mkApp = attrName: binName: { - type = "app"; - program = "${basePackages."${attrName}"}/bin/${binName}"; - }; - in - { - start-server = mkApp "start-server" "start-postgres-server"; - start-client = mkApp "start-client" "start-postgres-client"; - start-replica = mkApp "start-replica" "start-postgres-replica"; - # migrate-postgres = mkApp "migrate-tool" "migrate-postgres"; - # sync-exts-versions = mkApp "sync-exts-versions" "sync-exts-versions"; - pg-restore = mkApp "pg-restore" "pg-restore"; - local-infra-bootstrap = mkApp "local-infra-bootstrap" "local-infra-bootstrap"; - dbmate-tool = mkApp "dbmate-tool" "dbmate-tool"; - update-readme = mkApp "update-readme" "update-readme"; - show-commands = mkApp "show-commands" "show-commands"; - build-test-ami = mkApp "build-test-ami" "build-test-ami"; - run-testinfra = mkApp "run-testinfra" "run-testinfra"; - cleanup-ami = mkApp "cleanup-ami" "cleanup-ami"; - trigger-nix-build = mkApp "trigger-nix-build" "trigger-nix-build"; - }; - - # 'devShells.default' lists the set of packages that are included in the - # ambient $PATH environment when you run 'nix develop'. This is useful - # for development and puts many convenient devtools instantly within - # reach. - - devShells = - let - mkCargoPgrxDevShell = { pgrxVersion, rustVersion }: pkgs.mkShell { - packages = with pkgs; [ - basePackages."cargo-pgrx_${pgrxVersion}" - (rust-bin.stable.${rustVersion}.default.override { - extensions = [ "rust-src" ]; - }) - ]; - shellHook = '' - export HISTFILE=.history - ''; - }; - nix-fast-build = inputs.nix-fast-build.packages.${system}.default; - in - { - default = pkgs.mkShell { - packages = with pkgs; [ - coreutils - just - nix-update - #pg_prove - shellcheck - ansible - ansible-lint - (packer.overrideAttrs (oldAttrs: { - version = "1.7.8"; - })) - - basePackages.start-server - basePackages.start-client - basePackages.start-replica - basePackages.migrate-tool - basePackages.sync-exts-versions - basePackages.build-test-ami - basePackages.run-testinfra - basePackages.cleanup-ami - dbmate - nushell - pythonEnv - nix-fast-build - ]; - shellHook = '' - export HISTFILE=.history - ''; - }; - cargo-pgrx_0_11_3 = mkCargoPgrxDevShell { - pgrxVersion = "0_11_3"; - rustVersion = "1.80.0"; - }; - cargo-pgrx_0_12_6 = mkCargoPgrxDevShell { - pgrxVersion = "0_12_6"; - rustVersion = "1.80.0"; - }; - }; - } - ); + imports = [ + nix/apps.nix + nix/checks.nix + nix/config.nix + nix/devShells.nix + nix/ext + nix/fmt.nix + nix/hooks.nix + nix/nixpkgs.nix + nix/packages + nix/overlays + ]; + }); } diff --git a/nix/apps.nix b/nix/apps.nix new file mode 100644 index 000000000..75eaae49e --- /dev/null +++ b/nix/apps.nix @@ -0,0 +1,32 @@ +{ ... }: +{ + perSystem = + { self', ... }: + let + mkApp = attrName: binName: { + type = "app"; + program = "${self'.packages."${attrName}"}/bin/${binName}"; + }; + in + { + # Apps is a list of names of things that can be executed with 'nix run'; + # these are distinct from the things that can be built with 'nix build', + # so they need to be listed here too. + apps = { + start-server = mkApp "start-server" "start-postgres-server"; + start-client = mkApp "start-client" "start-postgres-client"; + start-replica = mkApp "start-replica" "start-postgres-replica"; + # migrate-postgres = mkApp "migrate-tool" "migrate-postgres"; + # sync-exts-versions = mkApp "sync-exts-versions" "sync-exts-versions"; + pg-restore = mkApp "pg-restore" "pg-restore"; + local-infra-bootstrap = mkApp "local-infra-bootstrap" "local-infra-bootstrap"; + dbmate-tool = mkApp "dbmate-tool" "dbmate-tool"; + update-readme = mkApp "update-readme" "update-readme"; + show-commands = mkApp "show-commands" "show-commands"; + build-test-ami = mkApp "build-test-ami" "build-test-ami"; + run-testinfra = mkApp "run-testinfra" "run-testinfra"; + cleanup-ami = mkApp "cleanup-ami" "cleanup-ami"; + trigger-nix-build = mkApp "trigger-nix-build" "trigger-nix-build"; + }; + }; +} diff --git a/nix/cargo-pgrx/buildPgrxExtension.nix b/nix/cargo-pgrx/buildPgrxExtension.nix index 89293ab62..0b2b14fb9 100644 --- a/nix/cargo-pgrx/buildPgrxExtension.nix +++ b/nix/cargo-pgrx/buildPgrxExtension.nix @@ -27,13 +27,14 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -{ lib -, cargo-pgrx -, pkg-config -, rustPlatform -, stdenv -, Security -, writeShellScriptBin +{ + lib, + cargo-pgrx, + pkg-config, + rustPlatform, + stdenv, + Security, + writeShellScriptBin, }: # The idea behind: Use it mostly like rustPlatform.buildRustPackage and so @@ -47,26 +48,31 @@ # unnecessary and heavy dependency. If you set this to true, you also # have to add `rustfmt` to `nativeBuildInputs`. -{ buildAndTestSubdir ? null -, buildType ? "release" -, buildFeatures ? [ ] -, cargoBuildFlags ? [ ] -, postgresql -# cargo-pgrx calls rustfmt on generated bindings, this is not strictly necessary, so we avoid the -# dependency here. Set to false and provide rustfmt in nativeBuildInputs, if you need it, e.g. -# if you include the generated code in the output via postInstall. -, useFakeRustfmt ? true -, usePgTestCheckFeature ? true -, ... -} @ args: +{ + buildAndTestSubdir ? null, + buildType ? "release", + buildFeatures ? [ ], + cargoBuildFlags ? [ ], + postgresql, + # cargo-pgrx calls rustfmt on generated bindings, this is not strictly necessary, so we avoid the + # dependency here. Set to false and provide rustfmt in nativeBuildInputs, if you need it, e.g. + # if you include the generated code in the output via postInstall. + useFakeRustfmt ? true, + usePgTestCheckFeature ? true, + ... +}@args: let - rustfmtInNativeBuildInputs = lib.lists.any (dep: lib.getName dep == "rustfmt") (args.nativeBuildInputs or []); + rustfmtInNativeBuildInputs = lib.lists.any (dep: lib.getName dep == "rustfmt") ( + args.nativeBuildInputs or [ ] + ); in -assert lib.asserts.assertMsg ((args.installPhase or "") == "") - "buildPgrxExtensions overwrites the installPhase, so providing one does nothing"; -assert lib.asserts.assertMsg ((args.buildPhase or "") == "") - "buildPgrxExtensions overwrites the buildPhase, so providing one does nothing"; +assert lib.asserts.assertMsg ( + (args.installPhase or "") == "" +) "buildPgrxExtensions overwrites the installPhase, so providing one does nothing"; +assert lib.asserts.assertMsg ( + (args.buildPhase or "") == "" +) "buildPgrxExtensions overwrites the buildPhase, so providing one does nothing"; assert lib.asserts.assertMsg (useFakeRustfmt -> !rustfmtInNativeBuildInputs) "The parameter useFakeRustfmt is set to true, but rustfmt is included in nativeBuildInputs. Either set useFakeRustfmt to false or remove rustfmt from nativeBuildInputs."; assert lib.asserts.assertMsg (!useFakeRustfmt -> rustfmtInNativeBuildInputs) @@ -75,7 +81,7 @@ assert lib.asserts.assertMsg (!useFakeRustfmt -> rustfmtInNativeBuildInputs) let fakeRustfmt = writeShellScriptBin "rustfmt" '' exit 0 - ''; + ''; maybeDebugFlag = lib.optionalString (buildType != "release") "--debug"; maybeEnterBuildAndTestSubdir = lib.optionalString (buildAndTestSubdir != null) '' export CARGO_TARGET_DIR="$(pwd)/target" @@ -97,19 +103,26 @@ let pg_ctl stop ''; - argsForBuildRustPackage = builtins.removeAttrs args [ "postgresql" "useFakeRustfmt" "usePgTestCheckFeature" ]; + argsForBuildRustPackage = builtins.removeAttrs args [ + "postgresql" + "useFakeRustfmt" + "usePgTestCheckFeature" + ]; # so we don't accidentally `(rustPlatform.buildRustPackage argsForBuildRustPackage) // { ... }` because # we forgot parentheses finalArgs = argsForBuildRustPackage // { buildInputs = (args.buildInputs or [ ]) ++ lib.optionals stdenv.hostPlatform.isDarwin [ Security ]; - nativeBuildInputs = (args.nativeBuildInputs or [ ]) ++ [ - cargo-pgrx - postgresql - pkg-config - rustPlatform.bindgenHook - ] ++ lib.optionals useFakeRustfmt [ fakeRustfmt ]; + nativeBuildInputs = + (args.nativeBuildInputs or [ ]) + ++ [ + cargo-pgrx + postgresql + pkg-config + rustPlatform.bindgenHook + ] + ++ lib.optionals useFakeRustfmt [ fakeRustfmt ]; buildPhase = '' runHook preBuild @@ -155,7 +168,10 @@ let RUST_BACKTRACE = "full"; checkNoDefaultFeatures = true; - checkFeatures = (args.checkFeatures or [ ]) ++ (lib.optionals usePgTestCheckFeature [ "pg_test" ]) ++ [ "pg${pgrxPostgresMajor}" ]; + checkFeatures = + (args.checkFeatures or [ ]) + ++ (lib.optionals usePgTestCheckFeature [ "pg_test" ]) + ++ [ "pg${pgrxPostgresMajor}" ]; }; in rustPlatform.buildRustPackage finalArgs diff --git a/nix/cargo-pgrx/default.nix b/nix/cargo-pgrx/default.nix index c1e1a4dee..5b5cd7222 100644 --- a/nix/cargo-pgrx/default.nix +++ b/nix/cargo-pgrx/default.nix @@ -1,11 +1,12 @@ -{ lib -, darwin -, fetchCrate -, openssl -, pkg-config -, makeRustPlatform -, stdenv -, rust-bin +{ + lib, + darwin, + fetchCrate, + openssl, + pkg-config, + makeRustPlatform, + stdenv, + rust-bin, }: let rustVersion = "1.85.1"; @@ -14,9 +15,10 @@ let rustc = rust-bin.stable.${rustVersion}.default; }; generic = - { version - , hash - , cargoHash + { + version, + hash, + cargoHash, }: rustPlatform.buildRustPackage rec { # rust-overlay uses 'cargo-auditable' wrapper for 'cargo' command, but it @@ -26,19 +28,13 @@ let auditable = false; pname = "cargo-pgrx"; inherit version; - src = fetchCrate { - inherit version pname hash; - }; + src = fetchCrate { inherit version pname hash; }; inherit cargoHash; - nativeBuildInputs = lib.optionals stdenv.hostPlatform.isLinux [ - pkg-config - ]; - buildInputs = lib.optionals stdenv.hostPlatform.isLinux [ - openssl - ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ - darwin.apple_sdk.frameworks.Security - ]; - + nativeBuildInputs = lib.optionals stdenv.hostPlatform.isLinux [ pkg-config ]; + buildInputs = + lib.optionals stdenv.hostPlatform.isLinux [ openssl ] + ++ lib.optionals stdenv.hostPlatform.isDarwin [ darwin.apple_sdk.frameworks.Security ]; + OPENSSL_DIR = "${openssl.dev}"; OPENSSL_INCLUDE_DIR = "${openssl.dev}/include"; OPENSSL_LIB_DIR = "${openssl.out}/lib"; diff --git a/nix/checks.nix b/nix/checks.nix new file mode 100644 index 000000000..fc042ad12 --- /dev/null +++ b/nix/checks.nix @@ -0,0 +1,311 @@ +{ self, ... }: +{ + perSystem = + { + lib, + self', + system, + pkgs, + ... + }: + let + pkgs-lib = pkgs.callPackage ./packages/lib.nix { + psql_15 = self'.packages."psql_15/bin"; + psql_17 = self'.packages."psql_17/bin"; + psql_orioledb-17 = self'.packages."psql_orioledb-17/bin"; + pgroonga = self'.packages."psql_15/exts/pgroonga"; + inherit (self.supabase) defaults; + }; + in + { + checks = + let + # Create a testing harness for a PostgreSQL package. This is used for + # 'nix flake check', and works with any PostgreSQL package you hand it. + makeCheckHarness = + pgpkg: + let + pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; + pg_regress = self'.packages.pg_regress; + getkey-script = pkgs.stdenv.mkDerivation { + name = "pgsodium-getkey"; + buildCommand = '' + mkdir -p $out/bin + cat > $out/bin/pgsodium-getkey << 'EOF' + #!${pkgs.bash}/bin/bash + set -euo pipefail + + TMPDIR_BASE=$(mktemp -d) + + KEY_DIR="''${PGSODIUM_KEY_DIR:-$TMPDIR_BASE/pgsodium}" + KEY_FILE="$KEY_DIR/pgsodium.key" + + if ! mkdir -p "$KEY_DIR" 2>/dev/null; then + echo "Error: Could not create key directory $KEY_DIR" >&2 + exit 1 + fi + chmod 1777 "$KEY_DIR" + + if [[ ! -f "$KEY_FILE" ]]; then + if ! (dd if=/dev/urandom bs=32 count=1 2>/dev/null | od -A n -t x1 | tr -d ' \n' > "$KEY_FILE"); then + if ! (openssl rand -hex 32 > "$KEY_FILE"); then + echo "00000000000000000000000000000000" > "$KEY_FILE" + echo "Warning: Using fallback key" >&2 + fi + fi + chmod 644 "$KEY_FILE" + fi + + if [[ -f "$KEY_FILE" && -r "$KEY_FILE" ]]; then + cat "$KEY_FILE" + else + echo "Error: Cannot read key file $KEY_FILE" >&2 + exit 1 + fi + EOF + chmod +x $out/bin/pgsodium-getkey + ''; + }; + + # Use the shared setup but with a test-specific name + start-postgres-server-bin = pkgs-lib.makePostgresDevSetup { + inherit pkgs; + name = "start-postgres-server-test"; + extraSubstitutions = { + PGSODIUM_GETKEY = "${getkey-script}/bin/pgsodium-getkey"; + PGSQL_DEFAULT_PORT = pgPort; + }; + }; + + getVersionArg = + pkg: + let + name = pkg.version; + in + if builtins.match "15.*" name != null then + "15" + else if builtins.match "17.*" name != null then + "17" + else if builtins.match "orioledb-17.*" name != null then + "orioledb-17" + else + throw "Unsupported PostgreSQL version: ${name}"; + + # Helper function to filter SQL files based on version + filterTestFiles = + version: dir: + let + files = builtins.readDir dir; + isValidFile = + name: + let + isVersionSpecific = builtins.match "z_.*" name != null; + matchesVersion = + if isVersionSpecific then + if version == "orioledb-17" then + builtins.match "z_orioledb-17_.*" name != null + else if version == "17" then + builtins.match "z_17_.*" name != null + else + builtins.match "z_15_.*" name != null + else + true; + in + pkgs.lib.hasSuffix ".sql" name && matchesVersion; + in + pkgs.lib.filterAttrs (name: _: isValidFile name) files; + + # Get the major version for filtering + majorVersion = + let + version = builtins.trace "pgpkg.version is: ${pgpkg.version}" pgpkg.version; + isOrioledbMatch = builtins.match "^17_[0-9]+$" version != null; + isSeventeenMatch = builtins.match "^17[.][0-9]+$" version != null; + result = + if isOrioledbMatch then + "orioledb-17" + else if isSeventeenMatch then + "17" + else + "15"; + in + builtins.trace "Major version result: ${result}" result; # Trace the result # For "15.8" + + # Filter SQL test files + filteredSqlTests = filterTestFiles majorVersion ./tests/sql; + + pgPort = + if (majorVersion == "17") then + "5535" + else if (majorVersion == "15") then + "5536" + else + "5537"; + + # Convert filtered tests to a sorted list of basenames (without extension) + testList = pkgs.lib.mapAttrsToList ( + name: _: builtins.substring 0 (pkgs.lib.stringLength name - 4) name + ) filteredSqlTests; + sortedTestList = builtins.sort (a: b: a < b) testList; + in + pkgs.runCommand "postgres-${pgpkg.version}-check-harness" + { + nativeBuildInputs = with pkgs; [ + coreutils + bash + perl + pgpkg + pg_prove + pg_regress + procps + start-postgres-server-bin + which + getkey-script + supabase-groonga + ]; + } + '' + set -e + + #First we need to create a generic pg cluster for pgtap tests and run those + export GRN_PLUGINS_DIR=${pkgs.supabase-groonga}/lib/groonga/plugins + PGTAP_CLUSTER=$(mktemp -d) + initdb --locale=C --username=supabase_admin -D "$PGTAP_CLUSTER" + substitute ${./tests/postgresql.conf.in} "$PGTAP_CLUSTER"/postgresql.conf \ + --subst-var-by PGSODIUM_GETKEY_SCRIPT "${getkey-script}/bin/pgsodium-getkey" + echo "listen_addresses = '*'" >> "$PGTAP_CLUSTER"/postgresql.conf + echo "port = ${pgPort}" >> "$PGTAP_CLUSTER"/postgresql.conf + echo "host all all 127.0.0.1/32 trust" >> $PGTAP_CLUSTER/pg_hba.conf + echo "Checking shared_preload_libraries setting:" + grep -rn "shared_preload_libraries" "$PGTAP_CLUSTER"/postgresql.conf + # Remove timescaledb if running orioledb-17 check + echo "I AM ${pgpkg.version}====================================================" + if [[ "${pgpkg.version}" == *"17"* ]]; then + perl -pi -e 's/ timescaledb,//g' "$PGTAP_CLUSTER/postgresql.conf" + fi + #NOTE in the future we may also need to add the orioledb extension to the cluster when cluster is oriole + echo "PGTAP_CLUSTER directory contents:" + ls -la "$PGTAP_CLUSTER" + + # Check if postgresql.conf exists + if [ ! -f "$PGTAP_CLUSTER/postgresql.conf" ]; then + echo "postgresql.conf is missing!" + exit 1 + fi + + # PostgreSQL startup + if [[ "$(uname)" == "Darwin" ]]; then + pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k "$PGTAP_CLUSTER" -p ${pgPort} -d 5" start 2>&1 + else + mkdir -p "$PGTAP_CLUSTER/sockets" + pg_ctl -D "$PGTAP_CLUSTER" -l "$PGTAP_CLUSTER"/postgresql.log -o "-k $PGTAP_CLUSTER/sockets -p ${pgPort} -d 5" start 2>&1 + fi || { + echo "pg_ctl failed to start PostgreSQL" + echo "Contents of postgresql.log:" + cat "$PGTAP_CLUSTER"/postgresql.log + exit 1 + } + for i in {1..60}; do + if pg_isready -h ${self.supabase.defaults.host} -p ${pgPort}; then + echo "PostgreSQL is ready" + break + fi + sleep 1 + if [ $i -eq 60 ]; then + echo "PostgreSQL is not ready after 60 seconds" + echo "PostgreSQL status:" + pg_ctl -D "$PGTAP_CLUSTER" status + echo "PostgreSQL log content:" + cat "$PGTAP_CLUSTER"/postgresql.log + exit 1 + fi + done + createdb -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin testing + if ! psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -v ON_ERROR_STOP=1 -Xf ${./tests/prime.sql}; then + echo "Error executing SQL file. PostgreSQL log content:" + cat "$PGTAP_CLUSTER"/postgresql.log + pg_ctl -D "$PGTAP_CLUSTER" stop + exit 1 + fi + SORTED_DIR=$(mktemp -d) + for t in $(printf "%s\n" ${builtins.concatStringsSep " " sortedTestList}); do + psql -p ${pgPort} -h ${self.supabase.defaults.host} --username=supabase_admin -d testing -f "${./tests/sql}/$t.sql" || true + done + rm -rf "$SORTED_DIR" + pg_ctl -D "$PGTAP_CLUSTER" stop + rm -rf $PGTAP_CLUSTER + + # End of pgtap tests + # from here on out we are running pg_regress tests, we use a different cluster for this + # which is start by the start-postgres-server-bin script + # start-postgres-server-bin script closely matches our AMI setup, configurations and migrations + + unset GRN_PLUGINS_DIR + ${start-postgres-server-bin}/bin/start-postgres-server ${getVersionArg pgpkg} --daemonize + + for i in {1..60}; do + if pg_isready -h ${self.supabase.defaults.host} -p ${pgPort} -U supabase_admin -q; then + echo "PostgreSQL is ready" + break + fi + sleep 1 + if [ $i -eq 60 ]; then + echo "PostgreSQL failed to start" + exit 1 + fi + done + + if ! psql -p ${pgPort} -h ${self.supabase.defaults.host} --no-password --username=supabase_admin -d postgres -v ON_ERROR_STOP=1 -Xf ${./tests/prime.sql}; then + echo "Error executing SQL file" + exit 1 + fi + + mkdir -p $out/regression_output + if ! pg_regress \ + --use-existing \ + --dbname=postgres \ + --inputdir=${./tests} \ + --outputdir=$out/regression_output \ + --host=${self.supabase.defaults.host} \ + --port=${pgPort} \ + --user=supabase_admin \ + ${builtins.concatStringsSep " " sortedTestList}; then + echo "pg_regress tests failed" + cat $out/regression_output/regression.diffs + exit 1 + fi + + echo "Running migrations tests" + pg_prove -p ${pgPort} -U supabase_admin -h ${self.supabase.defaults.host} -d postgres -v ${../migrations/tests}/test.sql + + # Copy logs to output + for logfile in $(find /tmp -name postgresql.log -type f); do + cp "$logfile" $out/postgresql.log + done + exit 0 + ''; + in + { + psql_15 = makeCheckHarness self'.packages."psql_15/bin"; + psql_17 = makeCheckHarness self'.packages."psql_17/bin"; + psql_orioledb-17 = makeCheckHarness self'.packages."psql_orioledb-17/bin"; + inherit (self'.packages) + wal-g-2 + wal-g-3 + dbmate-tool + pg_regress + ; + devShell = self'.devShells.default; + } + // pkgs.lib.optionalAttrs (system == "aarch64-linux") { + inherit (self'.packages) + postgresql_15_debug + postgresql_15_src + postgresql_orioledb-17_debug + postgresql_orioledb-17_src + postgresql_17_debug + postgresql_17_src + ; + }; + }; +} diff --git a/nix/config.nix b/nix/config.nix new file mode 100644 index 000000000..b1ebe3fd7 --- /dev/null +++ b/nix/config.nix @@ -0,0 +1,62 @@ +{ lib, ... }: +let + postgresqlDefaults = lib.types.submodule { + options = { + port = lib.mkOption { + type = lib.types.str; + default = "5435"; + }; + host = lib.mkOption { + type = lib.types.str; + default = "localhost"; + }; + superuser = lib.mkOption { + type = lib.types.str; + default = "supabase_admin"; + }; + }; + }; + postgresqlVersion = lib.types.submodule { + options = { + version = lib.mkOption { type = lib.types.str; }; + hash = lib.mkOption { type = lib.types.str; }; + }; + }; + supabaseSubmodule = lib.types.submodule { + options = { + defaults = lib.mkOption { type = postgresqlDefaults; }; + supportedPostgresVersions = lib.mkOption { + type = lib.types.attrsOf (lib.types.attrsOf postgresqlVersion); + default = { }; + }; + }; + }; +in +{ + flake = { + options = { + supabase = lib.mkOption { type = supabaseSubmodule; }; + }; + config.supabase = { + defaults = { }; + supportedPostgresVersions = { + postgres = { + "15" = { + version = "15.8"; + hash = "sha256-RANRX5pp7rPv68mPMLjGlhIr/fiV6Ss7I/W452nty2o="; + }; + "17" = { + version = "17.4"; + hash = "sha256-xGBbc/6hGWNAZpn5Sblm5dFzp+4Myu+JON7AyoqZX+c="; + }; + }; + orioledb = { + "17" = { + version = "17_6"; + hash = "sha256-HbuTcXNanFOl9YfvlSzQJon8CfAhc8TFwo/y7jXy51w="; + }; + }; + }; + }; + }; +} diff --git a/nix/devShells.nix b/nix/devShells.nix new file mode 100644 index 000000000..7ff4d0720 --- /dev/null +++ b/nix/devShells.nix @@ -0,0 +1,81 @@ +{ ... }: +{ + perSystem = + { + pkgs, + self', + config, + ... + }: + let + # Define pythonEnv here + pythonEnv = pkgs.python3.withPackages ( + ps: with ps; [ + boto3 + docker + pytest + pytest-testinfra + requests + ec2instanceconnectcli + paramiko + ] + ); + mkCargoPgrxDevShell = + { pgrxVersion, rustVersion }: + pkgs.mkShell { + packages = with pkgs; [ + self'.packages."cargo-pgrx_${pgrxVersion}" + (rust-bin.stable.${rustVersion}.default.override { extensions = [ "rust-src" ]; }) + ]; + shellHook = '' + export HISTFILE=.history + ''; + }; + in + { + devShells = { + default = pkgs.mkShell { + packages = + with pkgs; + [ + coreutils + just + nix-update + #pg_prove + shellcheck + ansible + ansible-lint + (packer.overrideAttrs (_oldAttrs: { + version = "1.7.8"; + })) + + self'.packages.start-server + self'.packages.start-client + self'.packages.start-replica + self'.packages.migrate-tool + self'.packages.sync-exts-versions + self'.packages.build-test-ami + self'.packages.run-testinfra + self'.packages.cleanup-ami + dbmate + nushell + pythonEnv + config.treefmt.build.wrapper + ] + ++ self'.packages.docs.nativeBuildInputs; + shellHook = '' + export HISTFILE=.history + ${config.pre-commit.installationScript} + ''; + }; + cargo-pgrx_0_11_3 = mkCargoPgrxDevShell { + pgrxVersion = "0_11_3"; + rustVersion = "1.80.0"; + }; + cargo-pgrx_0_12_6 = mkCargoPgrxDevShell { + pgrxVersion = "0_12_6"; + rustVersion = "1.80.0"; + }; + }; + }; +} diff --git a/nix/docs/README.md b/nix/docs/README.md index 400632984..66d63ef4c 100644 --- a/nix/docs/README.md +++ b/nix/docs/README.md @@ -3,6 +3,36 @@ This directory contains most of the "runbooks" and documentation on how to use this repository. +## Getting Started + You probably want to start with the [starting guide](./start-here.md). Then, learn how to play with `postgres` in the [build guide](./build-postgres.md). -After that, you can probe around a bit. + +## Development + +- **[Nix tree structure](./nix-directory-structure.md)** - Overview of the Nix directory structure +- **[Development Workflow](./development-workflow.md)** - Complete development and testing workflow +- **[Build PostgreSQL](./build-postgres.md)** - Building PostgreSQL from source +- **[Receipt Files](./receipt-files.md)** - Understanding build receipts +- **[Start Client/Server](./start-client-server.md)** - Running PostgreSQL client and server +- **[Docker](./docker.md)** - Docker integration and usage +- **[Use direnv](./use-direnv.md)** - Development environment with direnv +- **[Pre-commit Hooks](./pre-commit-hooks.md)** - Automatic formatting and code checks before commits +- **[Nix Formatter](./nix-formatter.md)** - Code formatting with treefmt + +## Package Management + +- **[Adding New Packages](./adding-new-package.md)** - How to add new PostgreSQL extensions +- **[Update Extensions](./update-extension.md)** - How to update existing extensions +- **[New Major PostgreSQL](./new-major-postgres.md)** - Adding support for new PostgreSQL versions +- **[Nix Overlays](./nix-overlays.md)** - Understanding and using Nix overlays + +## Testing + +- **[Adding Tests](./adding-tests.md)** - How to add tests for extensions +- **[Migration Tests](./migration-tests.md)** - Testing database migrations +- **[Testing PG Upgrade Scripts](./testing-pg-upgrade-scripts.md)** - Testing PostgreSQL upgrades + +## Reference + +- **[References](./references.md)** - Useful links and resources diff --git a/nix/docs/adding-new-package.md b/nix/docs/adding-new-package.md index 6f7753953..83d546d21 100644 --- a/nix/docs/adding-new-package.md +++ b/nix/docs/adding-new-package.md @@ -48,7 +48,7 @@ This uses `stdenv.mkDerivation` which is a general nix builder for C and C++ pro Your build should produce all of the sql and control files needed for the install phase. -1. Once you have created this file, you can add it to `nix/ext/.nix` and edit `flake.nix` and add it to the `ourExtensions` list. +1. Once you have created this file, you can add it to `nix/ext/.nix` and edit `nix/packages/postgres.nix` and add it to the `ourExtensions` list. 2. `git add .` as nix uses git to track changes 3. In your package file, temporarily empty the `hash = "sha256<...>=";` to `hash = "";` and save and `git add .` 4. Run `nix build .#psql_15/exts/` to try to trigger a build, nix will print the calculated sha256 value that you can add back the the `hash` variable, save the file again, and re-run `nix build .#psql_15/exts/`. @@ -156,4 +156,4 @@ A few things about `buildPgrxExtension_x`: 5. You can also run `nix run .#start-server 15` and in a new terminal window run `nix run .#star-client-and-migrate 15` and try to `CREATE EXTENSION ` and work with it there 6. Check that your extension works with the `pg_upgrade` process (TODO documentation forthcoming) 7. Now you are ready to PR the extension -8. From here, the release process should typically take care of the rest. \ No newline at end of file +8. From here, the release process should typically take care of the rest. diff --git a/nix/docs/assets/images/logo.png b/nix/docs/assets/images/logo.png new file mode 100644 index 000000000..c5dd5e27a Binary files /dev/null and b/nix/docs/assets/images/logo.png differ diff --git a/nix/docs/documentation.md b/nix/docs/documentation.md new file mode 100644 index 000000000..034b1f460 --- /dev/null +++ b/nix/docs/documentation.md @@ -0,0 +1,19 @@ +# Documentation + +We are using markdown for documentation in the `nix/docs` directory, which is rendered by [mkdocs](https://www.mkdocs.org/). + +## Usage + +To generate the documentation locally, you can use the following command in a development shell: + +```bash +cd nix +mkdocs serve +``` + +This will start a local server at `http://localhost:8000` where you can view the documentation. +The documentation is automatically updated as you make changes to the markdown files. + +## Configuration + +Mkdocs configuration is done in the `nix/mkdocs.yml` file. Refer to the [Mkdocs documentation](https://www.mkdocs.org/user-guide/configuration/) for details on how to configure the site. diff --git a/nix/docs/new-major-postgres.md b/nix/docs/new-major-postgres.md index 1c5a2dffc..b934b38bf 100644 --- a/nix/docs/new-major-postgres.md +++ b/nix/docs/new-major-postgres.md @@ -1,9 +1,10 @@ PostgreSQL versions are managed in upstream nixpkgs. -See this example PR to add a new version of PostgreSQL; this version is for 16 -beta3, but any version is roughly the same. In short, you need to: +[See this example PR](https://github.com/NixOS/nixpkgs/pull/249030) to add a +new version of PostgreSQL; this version is for 16 beta3, but any version is +roughly the same. In short, you need to: -- Add a new version and hash +- Add a new version and hash to `nix/config.nix` - Possibly patch the source code for minor refactorings - In this example, an old patch had to be rewritten because a function was split into two different functions; the patch is functionally equivalent but @@ -12,16 +13,14 @@ beta3, but any version is roughly the same. In short, you need to: - Integrate inside the CI and get code review - Run `nix flake update` to get a new version, once it's ready -https://github.com/NixOS/nixpkgs/pull/249030 - ## Adding the major version to this repository It isn't well abstracted, unfortunately. In short: look for the strings `14` and -`15` under `flake.nix` and `nix/tools/`. More specifically: +`15` under the nix configuration files. More specifically: -- Add `psql_XX` to `basePackages` in `flake.nix` -- Ditto with `checks` in `flake.nix` -- Modify the tools under `tools/` to understand the new major version +- Add `psql_XX` to `basePackages` in `nix/packages/postgres.nix` +- Ditto with `checks` in `nix/checks.nix` +- Modify the tools under `nix/packages/` to understand the new major version - Make sure the CI is integrated under the GitHub Actions. The third step and fourth steps are the most annoying, really. The first two are diff --git a/nix/docs/nix-directory-structure.md b/nix/docs/nix-directory-structure.md new file mode 100644 index 000000000..24ca49943 --- /dev/null +++ b/nix/docs/nix-directory-structure.md @@ -0,0 +1,163 @@ +# Nix Directory Structure + +This document explains the Nix structure used in this repository. The project uses [flake-parts](https://flake.parts/) to split a `flake.nix` into specialized, maintainable modules. + +The root `flake.nix` serves only as an entry point that references specialized modules in the `nix/` directory: + +``` +├── flake.nix # Root flake file only referencing modules +nix/ + ├── apps.nix # Application definitions + ├── checks.nix # Build checks and tests + ├── config.nix # Global configuration + ├── devShells.nix # Development environment shell + ├── fmt.nix # Code formatting configuration + ├── hooks.nix # Git hooks and pre-commit + ├── ext/ # PostgreSQL extensions + ├── overlays/ # Nixpkgs overlays + ├── packages/ # Custom packages + └── postgresql/ # PostgreSQL packages +``` + +## Module Descriptions + +### Root Files + +#### `flake.nix` + +The main flake file that: + +- Declares inputs (nixpkgs, flake-parts, etc.) +- Sets up the systems to support (x86_64-linux, aarch64-linux, aarch64-darwin) +- Imports all module files using flake-parts + +#### `flake.lock` + +Lockfile containing exact versions of all flake inputs. + +### Core Configuration Modules + +#### `nix/config.nix` + +Global configuration and constants used throughout the flake: + +- PostgreSQL default ports and users +- System-wide package configurations +- Shared constants and variables + +#### `nix/nixpkgs.nix` + +Nixpkgs configuration: + +- System-specific package imports +- Overlay applications +- Package configuration (allow unfree packages, etc.) + +### Development Environment + +#### `nix/devShells.nix` + +Development shell configurations: + +- Default development environment +- Tool dependencies for development +- Environment variables and setup + +#### `nix/fmt.nix` + +Code formatting configuration using [treefmt](https://github.com/numtide/treefmt/): + +- nixfmt-rfc-style for Nix code formatting +- deadnix for removing unused nix code + +More details in [Code formatter](./nix-formatter.md). + +#### `nix/hooks.nix` + +Git hooks and pre-commit configuration: + +- Integration with [git-hooks.nix](https://github.com/cachix/git-hooks.nix) +- Automatic formatting on commit +- Code quality checks + +More details in [Pre-coommit hooks](./pre-commit-hooks.md). + +### Applications and Packages + +#### `nix/apps.nix` + +Application definitions accessible via `nix run`: + +- Development tools and scripts +- Build and deployment utilities +- Testing and validation tools + +#### `nix/packages/` + +Directory containing custom package definitions such as: + + - `default.nix` - Main package exports and basePackages + - `start-client.nix` - PostgreSQL client tools + - `start-replica.nix` - Replication tools + - `migrate-tool.nix` - Migration utilities + - `dbmate-tool.nix` - Database migration tool + +#### `nix/checks.nix` + +Build checks and validation: + +- Package build validation +- Integration tests +- Code quality checks +- Ensures all Postgres packages build correctly + +### PostgreSQL-Specific + +#### `nix/postgresql/` + +PostgreSQL package definitions: + +- `default.nix` - Main PostgreSQL package exports +- `generic.nix` - Generic PostgreSQL build functions +- `src.nix` - PostgreSQL source package generation +- `patches/` - PostgreSQL-specific patches + +#### `nix/ext/` + +PostgreSQL extensions: + +- `default.nix` - Extension registry and ourExtensions list +- Individual `.nix` files - Extension definitions like: + - `pgvector.nix` - Vector similarity search + - `pgsodium.nix` - Encryption extension + - `pg_graphql.nix` - GraphQL support + - `timescaledb.nix` - Time-series database + - And 30+ other extensions + +### System Customization + +#### `nix/overlays/` + +Nixpkgs overlays for package customization: + +- `default.nix` - Main overlay that imports all others +- `cargo-pgrx-0-11-3.nix` - PGRX toolchain overlay +- `psql_16-oriole.nix` - OrioleDB PostgreSQL variant + +#### `nix/cargo-pgrx/` + +Rust-based PostgreSQL extension building: + +- `default.nix` - cargo-pgrx package definitions +- `buildPgrxExtension.nix` - Builder for Rust extensions + +### Testing + +#### `nix/tests/` + +Test suites and expected outputs: + +- `sql/` - SQL test files +- `expected/` - Expected test outputs +- `migrations/` - Migration test data +- `smoke/` - Smoke tests for quick validation diff --git a/nix/docs/nix-formatter.md b/nix/docs/nix-formatter.md new file mode 100644 index 000000000..22047a4d4 --- /dev/null +++ b/nix/docs/nix-formatter.md @@ -0,0 +1,97 @@ +# Code Formatting + +This repository uses [treefmt](https://treefmt.com/) with [nixfmt](https://github.com/NixOS/nixfmt) and [deadnix](https://github.com/astro/deadnix) to maintain consistent formatting and clean code in Nix files. + +## Formatting Tools + +### nixfmt-rfc-style + +- **Purpose**: Formats Nix code according to [RFC 166](https://github.com/NixOS/rfcs/blob/master/rfcs/0166-nix-formatting.md) style guidelines +- **What it does**: + + - Standardizes indentation and spacing + - Formats function calls and attribute sets consistently + - Ensures consistent line breaks and alignment + +### deadnix + +- **Purpose**: Removes unused/dead code from Nix expressions +- **What it does**: + + - Identifies unused variables and bindings + - Removes unused function arguments + - Cleans up dead code paths + +## Usage + +### Command Line + +```bash +# Run treefmt and format all Nix files in the repository +nix fmt +``` + +### In Development Shell + +The formatter is available when you enter the development shell: + +```bash +# Enter development shell +nix develop + +# Format all nix files +treefmt + +# Format specific files +treefmt file1.nix file2.nix + +# Check formatting without making changes +treefmt --check + +# Format with verbose output +treefmt --verbose +``` + +### With direnv + +If you're using direnv, the formatter is automatically available: + +```bash +cd /path/to/project +treefmt +``` + +## Configuration + +The formatter configuration is defined with nix in `nix/fmt.nix` using [treefmt-nix](https://github.com/numtide/treefmt-nix). +See the [treefmt-nix project documentation](https://github.com/numtide/treefmt-nix?tab=readme-ov-file#supported-programs) +for the list of supported formatters and their configurations. + +## Integration with Pre-commit Hooks + +The formatter is automatically run via pre-commit hooks (see [pre-commit-hooks.md](./pre-commit-hooks.md)) to ensure all committed code is properly formatted. + +## Best Practices + +### 1. Run Formatter Before Committing + +```bash +nix fmt +git add . +git commit -m "your message" +``` + +### 2. Review Formatting Changes + +Sometimes we want to first review formatting changes to ensure they're sensible: + +```bash +# See what would be changed +treefmt --check --diff +``` + +## Editor Integration + +### VS Code + +Use the [Nix IDE extension](https://marketplace.visualstudio.com/items?itemName=jnoortheen.nix-ide) with treefmt support. diff --git a/nix/docs/nix-overlays.md b/nix/docs/nix-overlays.md index 90b6f221f..4ca7d246e 100644 --- a/nix/docs/nix-overlays.md +++ b/nix/docs/nix-overlays.md @@ -8,7 +8,7 @@ Overlays are a feature of Nixpkgs that allow you to: disabled-by-default feature. First, you need to define a file for the overlay under -[overlays/](../overlays/), and then import it in `flake.nix`. There is an +[overlays/](../overlays/), and then import it in `nix/overlays/default.nix`. There is an example pull request in [#14](https://github.com/supabase/nix-postgres/issues/14) for this; an overlay typically looks like this: diff --git a/nix/docs/pre-commit-hooks.md b/nix/docs/pre-commit-hooks.md new file mode 100644 index 000000000..820c3a001 --- /dev/null +++ b/nix/docs/pre-commit-hooks.md @@ -0,0 +1,69 @@ +# Pre-commit Hooks + +This repository uses [git-hooks.nix](https://github.com/cachix/git-hooks.nix) and [pre-commit](https://pre-commit.com) to automatically run checks before commits. + +## What it does + +The pre-commit hooks are configured to run `treefmt` which formats Nix files using: + +- **nixfmt** (RFC-style) for Nix code formatting +- **deadnix** for removing dead/unused Nix code + +## Setup + +### Automatic Setup (Recommended) + +If you're using the development shell (via `nix develop` or direnv), the pre-commit hooks are automatically installed and will run before each commit. + +## Usage + +### Automatic Formatting on Commit + +Once set up, the hooks will automatically run before each commit: + +```bash +git add . +git commit -m "your commit message" +# treefmt will run automatically and format files if needed +``` + +If formatting changes are made, the commit will be aborted and you'll need to review and stage the changes: + +```bash +# Review the formatting changes +git diff + +# Stage the formatted files +git add . + +# Commit again +git commit -m "your commit message" +``` + +### Manual Formatting + +You can also run the formatter manually at any time. See [nix-formatter.md](./nix-formatter.md) for details on using `treefmt`. + +### Bypassing Hooks (Not Recommended) + +If you need to bypass the pre-commit hooks (not recommended for normal development): + +```bash +git commit --no-verify -m "your commit message" +``` + +Note that this check will be enforced in CI, so it's best to always run the hooks locally. + +## Configuration + +The pre-commit hooks are configured in: + +- **`nix/hooks.nix`** - Main git-hooks configuration +- **`nix/fmt.nix`** - treefmt formatter configuration + +## Best Practices + +1. **Always run formatting before pushing** - Even if you bypass hooks locally, CI may reject improperly formatted code +2. **Review formatting changes** - Don't blindly accept all formatting changes; review them to ensure they make sense +3. **Keep formatting commits separate** - If you need to make large formatting changes, consider doing them in a separate commit +4. **Use the development shell** - The easiest way to ensure everything works is to use `nix develop` or direnv diff --git a/nix/docs/references.md b/nix/docs/references.md index fe5b791e2..940f51df0 100644 --- a/nix/docs/references.md +++ b/nix/docs/references.md @@ -1,31 +1,37 @@ Nix references and other useful tools: - **Zero to Nix**: Start here to get your feet wet with how Nix works, and how - to use Nixpkgs: https://zero-to-nix.com/ -- `nix-installer`: My recommended way to install Nix - - https://github.com/DeterminateSystems/nix-installer -- Nix manual https://nixos.org/manual/nix/stable/ - - Useful primarily for option and command references -- Flake schema reference https://nixos.wiki/wiki/Flakes - - Useful to know what `flake.nix` is referring to + to use Nixpkgs: + +- `nix-installer`: My recommended way to install Nix: + - + +- Nix manual : + - Useful primarily for option and command references + +- Flake schema reference + - Useful to know what `flake.nix` is referring to + - Example pull requests for this repo: - - Adding smoke tests for an extension: - https://github.com/supabase/nix-postgres/pull/2 - - Extension smoke tests, part 2: - https://github.com/supabase/nix-postgres/pull/3 - - Adding an extension and a smoke test at once: - https://github.com/supabase/nix-postgres/pull/4/files - - Updating an extension to trunk: - https://github.com/supabase/nix-postgres/pull/7 - - Updating an extension to the latest release: - https://github.com/supabase/nix-postgres/pull/9 + - Adding smoke tests for an extension: + + - Extension smoke tests, part 2: + + - Adding an extension and a smoke test at once: + + - Updating an extension to trunk: + + - Updating an extension to the latest release: + + - Contributing to [nixpkgs](https://github.com/nixos/nixpkgs) - - Adding a PGRX-powered extension: - https://github.com/NixOS/nixpkgs/pull/246803 - - Adding a normal extension: https://github.com/NixOS/nixpkgs/pull/249000 - - Adding new PostgreSQL versions: https://github.com/NixOS/nixpkgs/pull/249030 + - Adding a PGRX-powered extension: + + - Adding a normal extension: + - Adding new PostgreSQL versions: + - NixOS Discourse: https://discourse.nixos.org/ - - Useful for community feedback, guidance, and help -- `nix-update`: https://github.com/Mic92/nix-update - - Used in this repository to help update extensions -- pgTAP for testing: https://pgtap.org/documentation.html + - Useful for community feedback, guidance, and help +- `nix-update`: + - Used in this repository to help update extensions +- pgTAP for testing: diff --git a/nix/docs/site/.keep b/nix/docs/site/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/nix/ext/default.nix b/nix/ext/default.nix new file mode 100644 index 000000000..9cd3de8bc --- /dev/null +++ b/nix/ext/default.nix @@ -0,0 +1,11 @@ +{ ... }: +{ + perSystem = + { pkgs, ... }: + { + packages = { + sfcgal = pkgs.callPackage ./sfcgal/sfcgal.nix { }; + mecab_naist_jdic = pkgs.callPackage ./mecab-naist-jdic/default.nix { }; + }; + }; +} diff --git a/nix/ext/gdal.nix b/nix/ext/gdal.nix index 83924d9ff..739f7e432 100644 --- a/nix/ext/gdal.nix +++ b/nix/ext/gdal.nix @@ -1,22 +1,23 @@ -{ lib -, stdenv -, fetchFromGitHub -, cmake -, pkg-config -, curl -, expat -, libgeotiff -, geos -, json_c -, libxml2 -, postgresql -, proj -, sqlite -, libtiff -, zlib +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + pkg-config, + curl, + expat, + libgeotiff, + geos, + json_c, + libxml2, + postgresql, + proj, + sqlite, + libtiff, + zlib, }: -stdenv.mkDerivation rec { +stdenv.mkDerivation rec { pname = "gdal"; version = "3.8.5"; @@ -46,16 +47,15 @@ stdenv.mkDerivation rec { zlib ]; - cmakeFlags = [ - "-DGDAL_USE_INTERNAL_LIBS=OFF" - "-DGEOTIFF_INCLUDE_DIR=${lib.getDev libgeotiff}/include" - "-DGEOTIFF_LIBRARY_RELEASE=${lib.getLib libgeotiff}/lib/libgeotiff${stdenv.hostPlatform.extensions.sharedLibrary}" - "-DBUILD_PYTHON_BINDINGS=OFF" - ] ++ lib.optionals (!stdenv.isDarwin) [ - "-DCMAKE_SKIP_BUILD_RPATH=ON" - ] ++ lib.optionals stdenv.isDarwin [ - "-DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON" - ]; + cmakeFlags = + [ + "-DGDAL_USE_INTERNAL_LIBS=OFF" + "-DGEOTIFF_INCLUDE_DIR=${lib.getDev libgeotiff}/include" + "-DGEOTIFF_LIBRARY_RELEASE=${lib.getLib libgeotiff}/lib/libgeotiff${stdenv.hostPlatform.extensions.sharedLibrary}" + "-DBUILD_PYTHON_BINDINGS=OFF" + ] + ++ lib.optionals (!stdenv.isDarwin) [ "-DCMAKE_SKIP_BUILD_RPATH=ON" ] + ++ lib.optionals stdenv.isDarwin [ "-DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON" ]; enableParallelBuilding = true; @@ -63,7 +63,13 @@ stdenv.mkDerivation rec { description = "Translator library for raster geospatial data formats (PostGIS-focused build)"; homepage = "https://www.gdal.org/"; license = licenses.mit; - maintainers = with maintainers; teams.geospatial.members ++ [ marcweber dotlambda ]; + maintainers = + with maintainers; + teams.geospatial.members + ++ [ + marcweber + dotlambda + ]; platforms = platforms.unix; }; } diff --git a/nix/ext/hypopg.nix b/nix/ext/hypopg.nix index ffafe904e..1e38b11c6 100644 --- a/nix/ext/hypopg.nix +++ b/nix/ext/hypopg.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "hypopg"; diff --git a/nix/ext/index_advisor.nix b/nix/ext/index_advisor.nix index ae23354b9..19777f39e 100644 --- a/nix/ext/index_advisor.nix +++ b/nix/ext/index_advisor.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "index_advisor"; diff --git a/nix/ext/mecab-naist-jdic/default.nix b/nix/ext/mecab-naist-jdic/default.nix index 82f0ed37c..c3148a4e5 100644 --- a/nix/ext/mecab-naist-jdic/default.nix +++ b/nix/ext/mecab-naist-jdic/default.nix @@ -1,19 +1,22 @@ -{ lib, stdenv, fetchurl, mecab }: +{ + lib, + stdenv, + fetchurl, + mecab, +}: stdenv.mkDerivation rec { pname = "mecab-naist-jdic"; version = "0.6.3b-20111013"; - + src = fetchurl { url = "https://github.com/supabase/mecab-naist-jdic/raw/main/mecab-naist-jdic-${version}.tar.gz"; sha256 = "sha256-yzdwDcmne5U/K/OxW0nP7NZ4SFMKLPirywm1lMpWKMw="; }; - + buildInputs = [ mecab ]; - - configureFlags = [ - "--with-charset=utf8" - ]; + + configureFlags = [ "--with-charset=utf8" ]; buildPhase = '' runHook preBuild diff --git a/nix/ext/orioledb.nix b/nix/ext/orioledb.nix index 96e70e308..96976f83d 100644 --- a/nix/ext/orioledb.nix +++ b/nix/ext/orioledb.nix @@ -1,4 +1,13 @@ -{ lib, stdenv, fetchFromGitHub, curl, libkrb5, postgresql, python3, openssl }: +{ + lib, + stdenv, + fetchFromGitHub, + curl, + libkrb5, + postgresql, + python3, + openssl, +}: stdenv.mkDerivation rec { pname = "orioledb"; @@ -10,7 +19,13 @@ stdenv.mkDerivation rec { sha256 = "sha256-O4OTi8ickylVXE9FURm5R++A+l15Z22YLna7OVzVMjc="; }; version = "beta10"; - buildInputs = [ curl libkrb5 postgresql python3 openssl ]; + buildInputs = [ + curl + libkrb5 + postgresql + python3 + openssl + ]; buildPhase = "make USE_PGXS=1 ORIOLEDB_PATCHSET_VERSION=6"; installPhase = '' runHook preInstall diff --git a/nix/ext/pg-safeupdate.nix b/nix/ext/pg-safeupdate.nix index f5ad17a82..5ac32c352 100644 --- a/nix/ext/pg-safeupdate.nix +++ b/nix/ext/pg-safeupdate.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "pg-safeupdate"; @@ -7,9 +12,9 @@ stdenv.mkDerivation rec { buildInputs = [ postgresql ]; src = fetchFromGitHub { - owner = "eradman"; - repo = pname; - rev = version; + owner = "eradman"; + repo = pname; + rev = version; hash = "sha256-1cyvVEC9MQGMr7Tg6EUbsVBrMc8ahdFS3+CmDkmAq4Y="; }; @@ -19,10 +24,10 @@ stdenv.mkDerivation rec { meta = with lib; { description = "A simple extension to PostgreSQL that requires criteria for UPDATE and DELETE"; - homepage = "https://github.com/eradman/pg-safeupdate"; - changelog = "https://github.com/eradman/pg-safeupdate/raw/${src.rev}/NEWS"; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; - broken = versionOlder postgresql.version "14"; + homepage = "https://github.com/eradman/pg-safeupdate"; + changelog = "https://github.com/eradman/pg-safeupdate/raw/${src.rev}/NEWS"; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + broken = versionOlder postgresql.version "14"; }; } diff --git a/nix/ext/pg_backtrace.nix b/nix/ext/pg_backtrace.nix index 0a0b0ae4f..7b92f0637 100644 --- a/nix/ext/pg_backtrace.nix +++ b/nix/ext/pg_backtrace.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "pg_backtrace"; @@ -7,9 +12,9 @@ stdenv.mkDerivation rec { buildInputs = [ postgresql ]; src = fetchFromGitHub { - owner = "pashkinelfe"; - repo = pname; - rev = "d100bac815a7365e199263f5b3741baf71b14c70"; + owner = "pashkinelfe"; + repo = pname; + rev = "d100bac815a7365e199263f5b3741baf71b14c70"; hash = "sha256-IVCL4r4oj1Ams03D8y+XCFkckPFER/W9tQ68GkWQQMY="; }; @@ -25,8 +30,8 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Updated fork of pg_backtrace"; - homepage = "https://github.com/pashkinelfe/pg_backtrace"; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; + homepage = "https://github.com/pashkinelfe/pg_backtrace"; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; }; } diff --git a/nix/ext/pg_cron.nix b/nix/ext/pg_cron.nix index 792db7676..bcaaf180c 100644 --- a/nix/ext/pg_cron.nix +++ b/nix/ext/pg_cron.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "pg_cron"; @@ -7,9 +12,9 @@ stdenv.mkDerivation rec { buildInputs = [ postgresql ]; src = fetchFromGitHub { - owner = "citusdata"; - repo = pname; - rev = "v${version}"; + owner = "citusdata"; + repo = pname; + rev = "v${version}"; hash = "sha256-t1DpFkPiSfdoGG2NgNT7g1lkvSooZoRoUrix6cBID40="; }; @@ -23,9 +28,9 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Run Cron jobs through PostgreSQL"; - homepage = "https://github.com/citusdata/pg_cron"; - changelog = "https://github.com/citusdata/pg_cron/raw/v${version}/CHANGELOG.md"; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; + homepage = "https://github.com/citusdata/pg_cron"; + changelog = "https://github.com/citusdata/pg_cron/raw/v${version}/CHANGELOG.md"; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; }; } diff --git a/nix/ext/pg_graphql.nix b/nix/ext/pg_graphql.nix index ef406f8d4..ebe6fea7f 100644 --- a/nix/ext/pg_graphql.nix +++ b/nix/ext/pg_graphql.nix @@ -1,8 +1,15 @@ -{ lib, stdenv, fetchFromGitHub, postgresql, buildPgrxExtension_0_12_9, cargo, rust-bin }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + buildPgrxExtension_0_12_9, + rust-bin, +}: let - rustVersion = "1.81.0"; - cargo = rust-bin.stable.${rustVersion}.default; + rustVersion = "1.81.0"; + cargo = rust-bin.stable.${rustVersion}.default; in buildPgrxExtension_0_12_9 rec { pname = "pg_graphql"; @@ -18,25 +25,27 @@ buildPgrxExtension_0_12_9 rec { nativeBuildInputs = [ cargo ]; buildInputs = [ postgresql ]; - + CARGO = "${cargo}/bin/cargo"; - + cargoLock = { lockFile = "${src}/Cargo.lock"; }; # Setting RUSTFLAGS in env to ensure it's available for all phases env = lib.optionalAttrs stdenv.isDarwin { POSTGRES_LIB = "${postgresql}/lib"; - PGPORT = toString (5430 + - (if builtins.match ".*_.*" postgresql.version != null then 1 else 0) + # +1 for OrioleDB - ((builtins.fromJSON (builtins.substring 0 2 postgresql.version)) - 15) * 2); # +2 for each major version + PGPORT = toString ( + 5430 + + (if builtins.match ".*_.*" postgresql.version != null then 1 else 0) + # +1 for OrioleDB + + ((builtins.fromJSON (builtins.substring 0 2 postgresql.version)) - 15) * 2 + ); # +2 for each major version RUSTFLAGS = "-C link-arg=-undefined -C link-arg=dynamic_lookup"; - NIX_BUILD_CORES = "4"; # Limit parallel jobs + NIX_BUILD_CORES = "4"; # Limit parallel jobs CARGO_BUILD_JOBS = "4"; # Limit cargo parallelism }; CARGO_PROFILE_RELEASE_BUILD_OVERRIDE_DEBUG = true; - doCheck = false; meta = with lib; { diff --git a/nix/ext/pg_hashids.nix b/nix/ext/pg_hashids.nix index 828db60fb..20d39a623 100644 --- a/nix/ext/pg_hashids.nix +++ b/nix/ext/pg_hashids.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "pg_hashids"; diff --git a/nix/ext/pg_jsonschema.nix b/nix/ext/pg_jsonschema.nix index 654bb93f5..794757678 100644 --- a/nix/ext/pg_jsonschema.nix +++ b/nix/ext/pg_jsonschema.nix @@ -1,4 +1,11 @@ -{ lib, stdenv, fetchFromGitHub, postgresql, buildPgrxExtension_0_12_6, cargo, rust-bin }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + buildPgrxExtension_0_12_6, + rust-bin, +}: let rustVersion = "1.80.0"; cargo = rust-bin.stable.${rustVersion}.default; @@ -20,23 +27,34 @@ buildPgrxExtension_0_12_6 rec { # update the following array when the pg_jsonschema version is updated # required to ensure that extensions update scripts from previous versions are generated - previousVersions = ["0.3.1" "0.3.0" "0.2.0" "0.1.4" "0.1.4" "0.1.2" "0.1.1" "0.1.0"]; - CARGO="${cargo}/bin/cargo"; + previousVersions = [ + "0.3.1" + "0.3.0" + "0.2.0" + "0.1.4" + "0.1.4" + "0.1.2" + "0.1.1" + "0.1.0" + ]; + CARGO = "${cargo}/bin/cargo"; #darwin env needs PGPORT to be unique for build to not clash with other pgrx extensions env = lib.optionalAttrs stdenv.isDarwin { POSTGRES_LIB = "${postgresql}/lib"; RUSTFLAGS = "-C link-arg=-undefined -C link-arg=dynamic_lookup"; - PGPORT = toString (5441 + - (if builtins.match ".*_.*" postgresql.version != null then 1 else 0) + # +1 for OrioleDB - ((builtins.fromJSON (builtins.substring 0 2 postgresql.version)) - 15) * 2); # +2 for each major version - + PGPORT = toString ( + 5441 + + (if builtins.match ".*_.*" postgresql.version != null then 1 else 0) + # +1 for OrioleDB + + ((builtins.fromJSON (builtins.substring 0 2 postgresql.version)) - 15) * 2 + ); # +2 for each major version }; cargoLock = { lockFile = "${src}/Cargo.lock"; allowBuiltinFetchGit = false; }; - + # FIXME (aseipp): testsuite tries to write files into /nix/store; we'll have # to fix this a bit later. doCheck = false; @@ -50,7 +68,7 @@ buildPgrxExtension_0_12_6 rec { echo "Creating SQL files for previous versions..." current_version="${version}" sql_file="$out/share/postgresql/extension/pg_jsonschema--$current_version.sql" - + if [ -f "$sql_file" ]; then while read -r previous_version; do if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then @@ -65,7 +83,6 @@ buildPgrxExtension_0_12_6 rec { rm git_tags.txt ''; - meta = with lib; { description = "JSON Schema Validation for PostgreSQL"; homepage = "https://github.com/supabase/${pname}"; diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index 0f20163e5..02d673297 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -1,10 +1,19 @@ -{ lib, stdenv, fetchFromGitHub, curl, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + curl, + postgresql, +}: stdenv.mkDerivation rec { pname = "pg_net"; version = "0.14.0"; - buildInputs = [ curl postgresql ]; + buildInputs = [ + curl + postgresql + ]; src = fetchFromGitHub { owner = "supabase"; diff --git a/nix/ext/pg_partman.nix b/nix/ext/pg_partman.nix index 899716b44..08fc74cee 100644 --- a/nix/ext/pg_partman.nix +++ b/nix/ext/pg_partman.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "pg_partman"; @@ -7,9 +12,9 @@ stdenv.mkDerivation rec { buildInputs = [ postgresql ]; src = fetchFromGitHub { - owner = "pgpartman"; - repo = pname; - rev = "refs/tags/v${version}"; + owner = "pgpartman"; + repo = pname; + rev = "refs/tags/v${version}"; sha256 = "sha256-GrVOJ5ywZMyqyDroYDLdKkXDdIJSDGhDfveO/ZvrmYs="; }; @@ -24,10 +29,10 @@ stdenv.mkDerivation rec { meta = with lib; { description = "Partition management extension for PostgreSQL"; - homepage = "https://github.com/pgpartman/pg_partman"; - changelog = "https://github.com/pgpartman/pg_partman/blob/v${version}/CHANGELOG.md"; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; - broken = versionOlder postgresql.version "14"; + homepage = "https://github.com/pgpartman/pg_partman"; + changelog = "https://github.com/pgpartman/pg_partman/blob/v${version}/CHANGELOG.md"; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + broken = versionOlder postgresql.version "14"; }; } diff --git a/nix/ext/pg_plan_filter.nix b/nix/ext/pg_plan_filter.nix index 6281d0b67..1e11c1d3f 100644 --- a/nix/ext/pg_plan_filter.nix +++ b/nix/ext/pg_plan_filter.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "pg_plan_filter"; diff --git a/nix/ext/pg_regress.nix b/nix/ext/pg_regress.nix index 08f82aa79..6233219b2 100644 --- a/nix/ext/pg_regress.nix +++ b/nix/ext/pg_regress.nix @@ -1,6 +1,7 @@ -{ lib -, stdenv -, postgresql +{ + lib, + stdenv, + postgresql, }: stdenv.mkDerivation { diff --git a/nix/ext/pg_repack.nix b/nix/ext/pg_repack.nix index 076e878f2..1f21335df 100644 --- a/nix/ext/pg_repack.nix +++ b/nix/ext/pg_repack.nix @@ -1,12 +1,10 @@ -{ lib -, stdenv -, fetchFromGitHub -, openssl -, postgresql -, postgresqlTestHook -, readline -, testers -, zlib +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + postgresqlTestHook, + testers, }: stdenv.mkDerivation (finalAttrs: { @@ -29,9 +27,7 @@ stdenv.mkDerivation (finalAttrs: { ''; passthru.tests = { - version = testers.testVersion { - package = finalAttrs.finalPackage; - }; + version = testers.testVersion { package = finalAttrs.finalPackage; }; extension = stdenv.mkDerivation { name = "plpgsql-check-test"; dontUnpack = true; diff --git a/nix/ext/pg_stat_monitor.nix b/nix/ext/pg_stat_monitor.nix index 8469f4b98..742e1c0aa 100644 --- a/nix/ext/pg_stat_monitor.nix +++ b/nix/ext/pg_stat_monitor.nix @@ -1,20 +1,10 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: -let - # NOTE (aseipp): the 1.x series of pg_stat_monitor has some non-standard and - # weird build logic (Percona projects in general seem to have their own - # strange build harness) where it will try to pick the right .sql file to - # install into the extension dir based on the postgresql major version. for - # our purposes, we only need to support v13 and v14+, so just replicate this - # logic from the makefile and pick the right file here. - # - # this seems to all be cleaned up in version 2.0 of the extension, so ideally - # we could upgrade to it later on and nuke this. - # DEPRECATED sqlFilename = if lib.versionOlder postgresql.version "14" - # then "pg_stat_monitor--1.0.13.sql.in" - # else "pg_stat_monitor--1.0.14.sql.in"; - -in stdenv.mkDerivation rec { pname = "pg_stat_monitor"; version = "2.1.0"; @@ -32,7 +22,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - + cp *${postgresql.dlSuffix} $out/lib cp *.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension diff --git a/nix/ext/pg_tle.nix b/nix/ext/pg_tle.nix index a0e9385b2..df83d4307 100644 --- a/nix/ext/pg_tle.nix +++ b/nix/ext/pg_tle.nix @@ -1,11 +1,23 @@ -{ lib, stdenv, fetchFromGitHub, postgresql, flex, openssl, libkrb5 }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + flex, + openssl, + libkrb5, +}: stdenv.mkDerivation rec { pname = "pg_tle"; version = "1.4.0"; nativeBuildInputs = [ flex ]; - buildInputs = [ openssl postgresql libkrb5 ]; + buildInputs = [ + openssl + postgresql + libkrb5 + ]; src = fetchFromGitHub { owner = "aws"; @@ -14,10 +26,8 @@ stdenv.mkDerivation rec { hash = "sha256-crxj5R9jblIv0h8lpqddAoYe2UqgUlnvbOajKTzVces="; }; - makeFlags = [ "FLEX=flex" ]; - installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} diff --git a/nix/ext/pgaudit.nix b/nix/ext/pgaudit.nix index 01ab66f15..3535447a7 100644 --- a/nix/ext/pgaudit.nix +++ b/nix/ext/pgaudit.nix @@ -1,20 +1,30 @@ -{ lib, stdenv, fetchFromGitHub, libkrb5, openssl, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + libkrb5, + openssl, + postgresql, +}: #adapted from https://github.com/NixOS/nixpkgs/blob/master/pkgs/servers/sql/postgresql/ext/pgaudit.nix let - source = { - "17" = { - version = "17.0"; - hash = "sha256-3ksq09wiudQPuBQI3dhEQi8IkXKLVIsPFgBnwLiicro="; - }; - "16" = { - version = "16.0"; - hash = "sha256-8+tGOl1U5y9Zgu+9O5UDDE4bec4B0JC/BQ6GLhHzQzc="; - }; - "15" = { - version = "1.7.0"; - hash = "sha256-8pShPr4HJaJQPjW1iPJIpj3CutTx8Tgr+rOqoXtgCcw="; - }; - }.${lib.versions.major postgresql.version} or (throw "Source for pgaudit is not available for ${postgresql.version}"); + source = + { + "17" = { + version = "17.0"; + hash = "sha256-3ksq09wiudQPuBQI3dhEQi8IkXKLVIsPFgBnwLiicro="; + }; + "16" = { + version = "16.0"; + hash = "sha256-8+tGOl1U5y9Zgu+9O5UDDE4bec4B0JC/BQ6GLhHzQzc="; + }; + "15" = { + version = "1.7.0"; + hash = "sha256-8pShPr4HJaJQPjW1iPJIpj3CutTx8Tgr+rOqoXtgCcw="; + }; + } + .${lib.versions.major postgresql.version} + or (throw "Source for pgaudit is not available for ${postgresql.version}"); in stdenv.mkDerivation { pname = "pgaudit"; @@ -27,7 +37,11 @@ stdenv.mkDerivation { hash = source.hash; }; - buildInputs = [ libkrb5 openssl postgresql ]; + buildInputs = [ + libkrb5 + openssl + postgresql + ]; makeFlags = [ "USE_PGXS=1" ]; diff --git a/nix/ext/pgjwt.nix b/nix/ext/pgjwt.nix index 4ecd76339..1a2f895b7 100644 --- a/nix/ext/pgjwt.nix +++ b/nix/ext/pgjwt.nix @@ -1,13 +1,19 @@ -{ lib, stdenv, fetchFromGitHub, postgresql, unstableGitUpdater }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + unstableGitUpdater, +}: stdenv.mkDerivation rec { pname = "pgjwt"; version = "9742dab1b2f297ad3811120db7b21451bca2d3c9"; src = fetchFromGitHub { - owner = "michelp"; - repo = "pgjwt"; - rev = "${version}"; + owner = "michelp"; + repo = "pgjwt"; + rev = "${version}"; hash = "sha256-Hw3R9bMGDmh+dMzjmqZSy/rT4mX8cPU969OJiARFg10="; }; diff --git a/nix/ext/pgmq.nix b/nix/ext/pgmq.nix index 97a3c27e3..e718c1d02 100644 --- a/nix/ext/pgmq.nix +++ b/nix/ext/pgmq.nix @@ -1,13 +1,18 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "pgmq"; version = "1.4.4"; buildInputs = [ postgresql ]; src = fetchFromGitHub { - owner = "tembo-io"; - repo = pname; - rev = "v${version}"; + owner = "tembo-io"; + repo = pname; + rev = "v${version}"; hash = "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw="; }; @@ -25,9 +30,9 @@ stdenv.mkDerivation rec { meta = with lib; { description = "A lightweight message queue. Like AWS SQS and RSMQ but on Postgres."; - homepage = "https://github.com/tembo-io/pgmq"; + homepage = "https://github.com/tembo-io/pgmq"; maintainers = with maintainers; [ olirice ]; - platforms = postgresql.meta.platforms; - license = licenses.postgresql; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; }; } diff --git a/nix/ext/pgroonga.nix b/nix/ext/pgroonga.nix index de0dedc93..d57efc8fb 100644 --- a/nix/ext/pgroonga.nix +++ b/nix/ext/pgroonga.nix @@ -1,7 +1,15 @@ -{ lib, stdenv, fetchurl, pkg-config, postgresql, msgpack-c, callPackage, mecab, makeWrapper, xxHash }: -let - supabase-groonga = callPackage ../supabase-groonga.nix { }; -in +{ + lib, + stdenv, + fetchurl, + pkg-config, + postgresql, + msgpack-c, + mecab, + makeWrapper, + xxHash, + supabase-groonga, +}: stdenv.mkDerivation rec { pname = "pgroonga"; version = "3.2.5"; @@ -9,12 +17,19 @@ stdenv.mkDerivation rec { url = "https://packages.groonga.org/source/${pname}/${pname}-${version}.tar.gz"; sha256 = "sha256-GM9EOQty72hdE4Ecq8jpDudhZLiH3pP9ODLxs8DXcSY="; }; - nativeBuildInputs = [ pkg-config makeWrapper ]; - - buildInputs = [ postgresql msgpack-c supabase-groonga mecab ] ++ lib.optionals stdenv.isDarwin [ - xxHash + + nativeBuildInputs = [ + pkg-config + makeWrapper ]; + buildInputs = [ + postgresql + msgpack-c + supabase-groonga + mecab + ] ++ lib.optionals stdenv.isDarwin [ xxHash ]; + propagatedBuildInputs = [ supabase-groonga ]; configureFlags = [ "--with-mecab=${mecab}" @@ -23,20 +38,22 @@ stdenv.mkDerivation rec { "--with-groonga-plugin-dir=${supabase-groonga}/lib/groonga/plugins" ]; - makeFlags = [ + makeFlags = [ "HAVE_MSGPACK=1" "MSGPACK_PACKAGE_NAME=msgpack-c" "HAVE_MECAB=1" ]; - NIX_CFLAGS_COMPILE = lib.optionalString stdenv.isDarwin (builtins.concatStringsSep " " [ - "-Wno-error=incompatible-function-pointer-types" - "-Wno-error=format" - "-Wno-format" - "-I${supabase-groonga}/include/groonga" - "-I${xxHash}/include" - "-DPGRN_VERSION=\"${version}\"" - ]); + NIX_CFLAGS_COMPILE = lib.optionalString stdenv.isDarwin ( + builtins.concatStringsSep " " [ + "-Wno-error=incompatible-function-pointer-types" + "-Wno-error=format" + "-Wno-format" + "-I${supabase-groonga}/include/groonga" + "-I${xxHash}/include" + "-DPGRN_VERSION=\"${version}\"" + ] + ); preConfigure = '' export GROONGA_LIBS="-L${supabase-groonga}/lib -lgroonga" diff --git a/nix/ext/pgrouting.nix b/nix/ext/pgrouting.nix index 3e898022f..e51b8bada 100644 --- a/nix/ext/pgrouting.nix +++ b/nix/ext/pgrouting.nix @@ -1,30 +1,45 @@ -{ lib, stdenv, fetchFromGitHub, postgresql, perl, cmake, boost }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + perl, + cmake, + boost, +}: stdenv.mkDerivation rec { pname = "pgrouting"; version = "3.4.1"; - nativeBuildInputs = [ cmake perl ]; - buildInputs = [ postgresql boost ]; + nativeBuildInputs = [ + cmake + perl + ]; + buildInputs = [ + postgresql + boost + ]; src = fetchFromGitHub { - owner = "pgRouting"; - repo = pname; - rev = "v${version}"; + owner = "pgRouting"; + repo = pname; + rev = "v${version}"; hash = "sha256-QC77AnPGpPQGEWi6JtJdiNsB2su5+aV2pKg5ImR2B0k="; }; #disable compile time warnings for incompatible pointer types only on macos and pg16 - NIX_CFLAGS_COMPILE = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") - "-Wno-error=int-conversion -Wno-error=incompatible-pointer-types"; - - cmakeFlags = [ - "-DPOSTGRESQL_VERSION=${postgresql.version}" - ] ++ lib.optionals (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") [ - "-DCMAKE_MACOSX_RPATH=ON" - "-DCMAKE_SHARED_MODULE_SUFFIX=.dylib" - "-DCMAKE_SHARED_LIBRARY_SUFFIX=.dylib" - ]; + NIX_CFLAGS_COMPILE = lib.optionalString ( + stdenv.isDarwin && lib.versionAtLeast postgresql.version "16" + ) "-Wno-error=int-conversion -Wno-error=incompatible-pointer-types"; + + cmakeFlags = + [ "-DPOSTGRESQL_VERSION=${postgresql.version}" ] + ++ lib.optionals (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") [ + "-DCMAKE_MACOSX_RPATH=ON" + "-DCMAKE_SHARED_MODULE_SUFFIX=.dylib" + "-DCMAKE_SHARED_LIBRARY_SUFFIX=.dylib" + ]; preConfigure = lib.optionalString (stdenv.isDarwin && lib.versionAtLeast postgresql.version "16") '' export DLSUFFIX=.dylib @@ -51,9 +66,9 @@ stdenv.mkDerivation rec { meta = with lib; { description = "A PostgreSQL/PostGIS extension that provides geospatial routing functionality"; - homepage = "https://pgrouting.org/"; - changelog = "https://github.com/pgRouting/pgrouting/releases/tag/v${version}"; - platforms = postgresql.meta.platforms; - license = licenses.gpl2Plus; + homepage = "https://pgrouting.org/"; + changelog = "https://github.com/pgRouting/pgrouting/releases/tag/v${version}"; + platforms = postgresql.meta.platforms; + license = licenses.gpl2Plus; }; } diff --git a/nix/ext/pgsodium.nix b/nix/ext/pgsodium.nix index ec2979499..d843d434f 100644 --- a/nix/ext/pgsodium.nix +++ b/nix/ext/pgsodium.nix @@ -1,10 +1,19 @@ -{ lib, stdenv, fetchFromGitHub, libsodium, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + libsodium, + postgresql, +}: stdenv.mkDerivation rec { pname = "pgsodium"; version = "3.1.8"; - buildInputs = [ libsodium postgresql ]; + buildInputs = [ + libsodium + postgresql + ]; src = fetchFromGitHub { owner = "michelp"; diff --git a/nix/ext/pgsql-http.nix b/nix/ext/pgsql-http.nix index 8b74f191b..9f4bae789 100644 --- a/nix/ext/pgsql-http.nix +++ b/nix/ext/pgsql-http.nix @@ -1,10 +1,19 @@ -{ lib, stdenv, fetchFromGitHub, curl, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + curl, + postgresql, +}: stdenv.mkDerivation rec { pname = "pgsql-http"; version = "1.6.1"; - buildInputs = [ curl postgresql ]; + buildInputs = [ + curl + postgresql + ]; src = fetchFromGitHub { owner = "pramsey"; diff --git a/nix/ext/pgtap.nix b/nix/ext/pgtap.nix index 159fec433..47d211556 100644 --- a/nix/ext/pgtap.nix +++ b/nix/ext/pgtap.nix @@ -1,4 +1,12 @@ -{ lib, stdenv, fetchFromGitHub, postgresql, perl, perlPackages, which }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + perl, + perlPackages, + which, +}: stdenv.mkDerivation rec { pname = "pgtap"; @@ -11,7 +19,12 @@ stdenv.mkDerivation rec { hash = "sha256-lb0PRffwo6J5a6Hqw1ggvn0cW7gPZ02OEcLPi9ineI8="; }; - nativeBuildInputs = [ postgresql perl perlPackages.TAPParserSourceHandlerpgTAP which ]; + nativeBuildInputs = [ + postgresql + perl + perlPackages.TAPParserSourceHandlerpgTAP + which + ]; installPhase = '' install -D {sql/pgtap--${version}.sql,pgtap.control} -t $out/share/postgresql/extension diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index a7d58ec61..60a8efea3 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "pgvector"; diff --git a/nix/ext/pljava.nix b/nix/ext/pljava.nix index 51c5e1317..bac5a6646 100644 --- a/nix/ext/pljava.nix +++ b/nix/ext/pljava.nix @@ -1,23 +1,48 @@ -{ stdenv, lib, fetchFromGitHub, openssl, openjdk, maven, postgresql, libkrb5, makeWrapper, gcc, pkg-config, which }: +{ + stdenv, + lib, + fetchFromGitHub, + openssl, + openjdk, + maven, + postgresql, + libkrb5, + makeWrapper, + gcc, + pkg-config, + which, +}: maven.buildMavenPackage rec { pname = "pljava"; - version = "1.6.7"; + version = "1.6.7"; src = fetchFromGitHub { owner = "tada"; repo = "pljava"; - rev = "V1_6_7"; - sha256 = "sha256-M17adSLsw47KZ2BoUwxyWkXKRD8TcexDAy61Yfw4fNU="; - + rev = "V1_6_7"; + sha256 = "sha256-M17adSLsw47KZ2BoUwxyWkXKRD8TcexDAy61Yfw4fNU="; }; - mvnParameters = "clean install -Dmaven.test.skip -DskipTests -Dmaven.javadoc.skip=true"; + mvnParameters = "clean install -Dmaven.test.skip -DskipTests -Dmaven.javadoc.skip=true"; mvnHash = "sha256-lcxRduh/nKcPL6YQIVTsNH0L4ga0LgJpQKgX5IPkRzs="; - - nativeBuildInputs = [ makeWrapper maven openjdk postgresql openssl postgresql gcc libkrb5 pkg-config ]; - buildInputs = [ stdenv.cc.cc.lib which]; + + nativeBuildInputs = [ + makeWrapper + maven + openjdk + postgresql + openssl + postgresql + gcc + libkrb5 + pkg-config + ]; + buildInputs = [ + stdenv.cc.cc.lib + which + ]; buildPhase = '' export PATH=$(lib.makeBinPath [ postgresql ]):$PATH @@ -44,7 +69,7 @@ maven.buildMavenPackage rec { meta = with lib; { description = "PL/Java extension for PostgreSQL"; - homepage = https://github.com/tada/pljava; + homepage = "https://github.com/tada/pljava"; license = licenses.bsd3; }; } diff --git a/nix/ext/plpgsql-check.nix b/nix/ext/plpgsql-check.nix index 7be2aac2a..41312b1c7 100644 --- a/nix/ext/plpgsql-check.nix +++ b/nix/ext/plpgsql-check.nix @@ -1,4 +1,10 @@ -{ lib, stdenv, fetchFromGitHub, postgresql, postgresqlTestHook }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, + postgresqlTestHook, +}: stdenv.mkDerivation rec { pname = "plpgsql-check"; diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index b3fd03e07..9eaa40459 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -1,19 +1,17 @@ -{ stdenv -, lib -, fetchFromGitHub -, v8 -, perl -, postgresql -# For passthru test on various systems, and local development on macos -# not we are not currently using passthru tests but retaining for possible contrib -# to nixpkgs -, runCommand -, coreutils -, gnugrep -, clang -, xcbuild -, darwin -, patchelf +{ + stdenv, + lib, + fetchFromGitHub, + v8, + perl, + postgresql, + # For passthru test on various systems, and local development on macos + # not we are not currently using passthru tests but retaining for possible contrib + # to nixpkgs + clang, + xcbuild, + darwin, + patchelf, }: stdenv.mkDerivation (finalAttrs: { @@ -33,50 +31,58 @@ stdenv.mkDerivation (finalAttrs: { ./0001-build-Allow-using-V8-from-system.patch ]; - nativeBuildInputs = [ - perl - ] ++ lib.optionals stdenv.isDarwin [ - clang - xcbuild - ]; - - buildInputs = [ - v8 - postgresql - ] ++ lib.optionals stdenv.isDarwin [ - darwin.apple_sdk.frameworks.CoreFoundation - darwin.apple_sdk.frameworks.Kerberos - ]; + nativeBuildInputs = + [ perl ] + ++ lib.optionals stdenv.isDarwin [ + clang + xcbuild + ]; + + buildInputs = + [ + v8 + postgresql + ] + ++ lib.optionals stdenv.isDarwin [ + darwin.apple_sdk.frameworks.CoreFoundation + darwin.apple_sdk.frameworks.Kerberos + ]; buildFlags = [ "all" ]; - makeFlags = [ - # Nixpkgs build a v8 monolith instead of separate v8_libplatform. - "USE_SYSTEM_V8=1" - "V8_OUTDIR=${v8}/lib" - "PG_CONFIG=${postgresql}/bin/pg_config" - ] ++ lib.optionals stdenv.isDarwin [ - "CC=${clang}/bin/clang" - "CXX=${clang}/bin/clang++" - "SHLIB_LINK=-L${v8}/lib -lv8_monolith -Wl,-rpath,${v8}/lib" - ] ++ lib.optionals (!stdenv.isDarwin) [ - "SHLIB_LINK=-lv8" - ]; - - NIX_LDFLAGS = (lib.optionals stdenv.isDarwin [ - "-L${postgresql}/lib" - "-L${v8}/lib" - "-lv8_monolith" - "-lpq" - "-lpgcommon" - "-lpgport" - "-F${darwin.apple_sdk.frameworks.CoreFoundation}/Library/Frameworks" - "-framework" "CoreFoundation" - "-F${darwin.apple_sdk.frameworks.Kerberos}/Library/Frameworks" - "-framework" "Kerberos" - "-undefined" "dynamic_lookup" - "-flat_namespace" - ]); + makeFlags = + [ + # Nixpkgs build a v8 monolith instead of separate v8_libplatform. + "USE_SYSTEM_V8=1" + "V8_OUTDIR=${v8}/lib" + "PG_CONFIG=${postgresql}/bin/pg_config" + ] + ++ lib.optionals stdenv.isDarwin [ + "CC=${clang}/bin/clang" + "CXX=${clang}/bin/clang++" + "SHLIB_LINK=-L${v8}/lib -lv8_monolith -Wl,-rpath,${v8}/lib" + ] + ++ lib.optionals (!stdenv.isDarwin) [ "SHLIB_LINK=-lv8" ]; + + NIX_LDFLAGS = ( + lib.optionals stdenv.isDarwin [ + "-L${postgresql}/lib" + "-L${v8}/lib" + "-lv8_monolith" + "-lpq" + "-lpgcommon" + "-lpgport" + "-F${darwin.apple_sdk.frameworks.CoreFoundation}/Library/Frameworks" + "-framework" + "CoreFoundation" + "-F${darwin.apple_sdk.frameworks.Kerberos}/Library/Frameworks" + "-framework" + "Kerberos" + "-undefined" + "dynamic_lookup" + "-flat_namespace" + ] + ); installFlags = [ # PGXS only supports installing to postgresql prefix so we need to redirect this @@ -97,7 +103,7 @@ stdenv.mkDerivation (finalAttrs: { ''} ''; - postInstall = '' + postInstall = '' # Move the redirected to proper directory. # There appear to be no references to the install directories # so changing them does not cause issues. @@ -119,9 +125,11 @@ stdenv.mkDerivation (finalAttrs: { install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8.so ''} - ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so - ''} + ${ + lib.optionalString (!stdenv.isDarwin) '' + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so + '' + } else ${lib.optionalString stdenv.isDarwin '' install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} @@ -130,16 +138,23 @@ stdenv.mkDerivation (finalAttrs: { install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} ''} - ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} - ''} + ${ + lib.optionalString (!stdenv.isDarwin) '' + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}${postgresql.dlSuffix} + '' + } fi ''; meta = with lib; { description = "V8 Engine Javascript Procedural Language add-on for PostgreSQL"; homepage = "https://plv8.github.io/"; - platforms = [ "x86_64-linux" "aarch64-linux" "aarch64-darwin" "x86_64-darwin" ]; + platforms = [ + "x86_64-linux" + "aarch64-linux" + "aarch64-darwin" + "x86_64-darwin" + ]; license = licenses.postgresql; }; }) diff --git a/nix/ext/postgis.nix b/nix/ext/postgis.nix index e5ccda443..27a449210 100644 --- a/nix/ext/postgis.nix +++ b/nix/ext/postgis.nix @@ -1,18 +1,20 @@ -{ fetchurl -, lib, stdenv -, perl -, libxml2 -, postgresql -, geos -, proj -, json_c -, pkg-config -, file -, protobufc -, libiconv -, pcre2 -, nixosTests -, callPackage +{ + fetchurl, + lib, + stdenv, + perl, + libxml2, + postgresql, + geos, + proj, + json_c, + pkg-config, + file, + protobufc, + libiconv, + pcre2, + nixosTests, + callPackage, }: let @@ -23,19 +25,33 @@ stdenv.mkDerivation rec { pname = "postgis"; version = "3.3.7"; - outputs = [ "out" "doc" ]; + outputs = [ + "out" + "doc" + ]; src = fetchurl { url = "https://download.osgeo.org/postgis/source/postgis-${version}.tar.gz"; sha256 = "sha256-UHJKDd5JrcJT5Z4CTYsY/va+ToU0GUPG1eHhuXTkP84="; }; - buildInputs = [ libxml2 postgresql geos proj gdal json_c protobufc pcre2.dev sfcgal ] - ++ lib.optional stdenv.isDarwin libiconv; - nativeBuildInputs = [ perl pkg-config ]; + buildInputs = [ + libxml2 + postgresql + geos + proj + gdal + json_c + protobufc + pcre2.dev + sfcgal + ] ++ lib.optional stdenv.isDarwin libiconv; + nativeBuildInputs = [ + perl + pkg-config + ]; dontDisableStatic = true; - env.NIX_LDFLAGS = "-L${lib.getLib json_c}/lib"; preConfigure = '' @@ -57,22 +73,22 @@ stdenv.mkDerivation rec { ln -s ${postgresql}/bin/postgres $out/bin/postgres ''; -postInstall = '' - rm $out/bin/postgres - for prog in $out/bin/*; do # */ - ln -s $prog $prog-${version} - done - # Add function definition and usage to tiger geocoder files - for file in $out/share/postgresql/extension/postgis_tiger_geocoder*--${version}.sql; do - sed -i "/SELECT postgis_extension_AddToSearchPath('tiger');/a SELECT postgis_extension_AddToSearchPath('extensions');" "$file" - done - # Original topology patching - for file in $out/share/postgresql/extension/postgis_topology*--${version}.sql; do - sed -i "/SELECT topology.AddToSearchPath('topology');/i SELECT topology.AddToSearchPath('extensions');" "$file" - done - mkdir -p $doc/share/doc/postgis - mv doc/* $doc/share/doc/postgis/ -''; + postInstall = '' + rm $out/bin/postgres + for prog in $out/bin/*; do # */ + ln -s $prog $prog-${version} + done + # Add function definition and usage to tiger geocoder files + for file in $out/share/postgresql/extension/postgis_tiger_geocoder*--${version}.sql; do + sed -i "/SELECT postgis_extension_AddToSearchPath('tiger');/a SELECT postgis_extension_AddToSearchPath('extensions');" "$file" + done + # Original topology patching + for file in $out/share/postgresql/extension/postgis_topology*--${version}.sql; do + sed -i "/SELECT topology.AddToSearchPath('topology');/i SELECT topology.AddToSearchPath('extensions');" "$file" + done + mkdir -p $doc/share/doc/postgis + mv doc/* $doc/share/doc/postgis/ + ''; passthru.tests.postgis = nixosTests.postgis; diff --git a/nix/ext/rum.nix b/nix/ext/rum.nix index 574ef3d3a..9dc5d9525 100644 --- a/nix/ext/rum.nix +++ b/nix/ext/rum.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "rum"; diff --git a/nix/ext/sfcgal/sfcgal.nix b/nix/ext/sfcgal/sfcgal.nix index b1d5842f8..da962cbf8 100644 --- a/nix/ext/sfcgal/sfcgal.nix +++ b/nix/ext/sfcgal/sfcgal.nix @@ -1,4 +1,14 @@ -{ lib, stdenv, fetchFromGitLab, cgal, cmake, pkg-config, gmp, mpfr, boost }: +{ + lib, + stdenv, + fetchFromGitLab, + cgal, + cmake, + pkg-config, + gmp, + mpfr, + boost, +}: stdenv.mkDerivation rec { pname = "sfcgal"; @@ -11,10 +21,19 @@ stdenv.mkDerivation rec { hash = "sha256-nKSqiFyMkZAYptIeShb1zFg9lYSny3kcGJfxdeTFqxw="; }; - nativeBuildInputs = [ cmake pkg-config cgal gmp mpfr boost ]; - - cmakeFlags = [ "-DCGAL_DIR=${cgal}" "-DCMAKE_PREFIX_PATH=${cgal}" ]; + nativeBuildInputs = [ + cmake + pkg-config + cgal + gmp + mpfr + boost + ]; + cmakeFlags = [ + "-DCGAL_DIR=${cgal}" + "-DCMAKE_PREFIX_PATH=${cgal}" + ]; postPatch = '' substituteInPlace sfcgal.pc.in \ @@ -24,7 +43,10 @@ stdenv.mkDerivation rec { meta = with lib; { description = "A wrapper around CGAL that intents to implement 2D and 3D operations on OGC standards models"; homepage = "https://sfcgal.gitlab.io/SFCGAL/"; - license = with licenses; [ gpl3Plus lgpl3Plus]; + license = with licenses; [ + gpl3Plus + lgpl3Plus + ]; platforms = platforms.all; }; } diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index bb01bac55..75c5c029b 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "supautils"; diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix index 92d5d73fe..0df743671 100644 --- a/nix/ext/timescaledb-2.9.1.nix +++ b/nix/ext/timescaledb-2.9.1.nix @@ -1,11 +1,23 @@ -{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + postgresql, + openssl, + libkrb5, +}: stdenv.mkDerivation rec { pname = "timescaledb-apache"; version = "2.9.1"; nativeBuildInputs = [ cmake ]; - buildInputs = [ postgresql openssl libkrb5 ]; + buildInputs = [ + postgresql + openssl + libkrb5 + ]; src = fetchFromGitHub { owner = "timescale"; @@ -14,8 +26,12 @@ stdenv.mkDerivation rec { hash = "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="; }; - cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] - ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + cmakeFlags = [ + "-DSEND_TELEMETRY_DEFAULT=OFF" + "-DREGRESS_CHECKS=OFF" + "-DTAP_CHECKS=OFF" + "-DAPACHE_ONLY=1" + ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; # Fix the install phase which tries to install into the pgsql extension dir, # and cannot be manually overridden. This is rather fragile but works OK. @@ -31,7 +47,6 @@ stdenv.mkDerivation rec { done ''; - # timescaledb-2.9.1.so already exists in the lib directory # we have no need for the timescaledb.so or control file postInstall = '' diff --git a/nix/ext/timescaledb.nix b/nix/ext/timescaledb.nix index 1d7360762..6f5681546 100644 --- a/nix/ext/timescaledb.nix +++ b/nix/ext/timescaledb.nix @@ -1,11 +1,23 @@ -{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: +{ + lib, + stdenv, + fetchFromGitHub, + cmake, + postgresql, + openssl, + libkrb5, +}: stdenv.mkDerivation rec { pname = "timescaledb-apache"; version = "2.16.1"; nativeBuildInputs = [ cmake ]; - buildInputs = [ postgresql openssl libkrb5 ]; + buildInputs = [ + postgresql + openssl + libkrb5 + ]; src = fetchFromGitHub { owner = "timescale"; @@ -14,8 +26,12 @@ stdenv.mkDerivation rec { hash = "sha256-sLxWdBmih9mgiO51zLLxn9uwJVYc5JVHJjSWoADoJ+w="; }; - cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] - ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + cmakeFlags = [ + "-DSEND_TELEMETRY_DEFAULT=OFF" + "-DREGRESS_CHECKS=OFF" + "-DTAP_CHECKS=OFF" + "-DAPACHE_ONLY=1" + ] ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; # Fix the install phase which tries to install into the pgsql extension dir, # and cannot be manually overridden. This is rather fragile but works OK. diff --git a/nix/ext/vault.nix b/nix/ext/vault.nix index fcc1a9129..275d6967c 100644 --- a/nix/ext/vault.nix +++ b/nix/ext/vault.nix @@ -1,10 +1,19 @@ -{ lib, stdenv, fetchFromGitHub, libsodium, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + libsodium, + postgresql, +}: stdenv.mkDerivation rec { pname = "vault"; version = "0.3.1"; - buildInputs = [ libsodium postgresql ]; + buildInputs = [ + libsodium + postgresql + ]; src = fetchFromGitHub { owner = "supabase"; diff --git a/nix/ext/wal2json.nix b/nix/ext/wal2json.nix index b1f532dcf..deba7f1c6 100644 --- a/nix/ext/wal2json.nix +++ b/nix/ext/wal2json.nix @@ -1,4 +1,9 @@ -{ lib, stdenv, fetchFromGitHub, postgresql }: +{ + lib, + stdenv, + fetchFromGitHub, + postgresql, +}: stdenv.mkDerivation rec { pname = "wal2json"; @@ -7,7 +12,7 @@ stdenv.mkDerivation rec { src = fetchFromGitHub { owner = "eulerto"; repo = "wal2json"; - rev = "wal2json_${builtins.replaceStrings ["."] ["_"] version}"; + rev = "wal2json_${builtins.replaceStrings [ "." ] [ "_" ] version}"; hash = "sha256-+QoACPCKiFfuT2lJfSUmgfzC5MXf75KpSoc2PzPxKyM="; }; diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 2bd424942..fe8074189 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -1,15 +1,14 @@ -{ lib -, stdenv -, fetchFromGitHub -, openssl -, pkg-config -, postgresql -, buildPgrxExtension_0_12_9 -, cargo -, darwin -, jq -, rust-bin -, git +{ + lib, + stdenv, + fetchFromGitHub, + openssl, + pkg-config, + postgresql, + buildPgrxExtension_0_12_9, + darwin, + rust-bin, + git, }: let rustVersion = "1.84.0"; @@ -20,7 +19,35 @@ buildPgrxExtension_0_12_9 rec { version = "0.5.0"; # update the following array when the wrappers version is updated # required to ensure that extensions update scripts from previous versions are generated - previousVersions = ["0.4.6" "0.4.5" "0.4.4" "0.4.3" "0.4.2" "0.4.1" "0.4.0" "0.3.1" "0.3.0" "0.2.0" "0.1.19" "0.1.18" "0.1.17" "0.1.16" "0.1.15" "0.1.14" "0.1.12" "0.1.11" "0.1.10" "0.1.9" "0.1.8" "0.1.7" "0.1.6" "0.1.5" "0.1.4" "0.1.1" "0.1.0"]; + previousVersions = [ + "0.4.6" + "0.4.5" + "0.4.4" + "0.4.3" + "0.4.2" + "0.4.1" + "0.4.0" + "0.3.1" + "0.3.0" + "0.2.0" + "0.1.19" + "0.1.18" + "0.1.17" + "0.1.16" + "0.1.15" + "0.1.14" + "0.1.12" + "0.1.11" + "0.1.10" + "0.1.9" + "0.1.8" + "0.1.7" + "0.1.6" + "0.1.5" + "0.1.4" + "0.1.1" + "0.1.0" + ]; inherit postgresql; src = fetchFromGitHub { owner = "supabase"; @@ -28,13 +55,22 @@ buildPgrxExtension_0_12_9 rec { rev = "v${version}"; hash = "sha256-FbRTUcpEHBa5DI6dutvBeahYM0RZVAXIzIAZWIaxvn0"; }; - - nativeBuildInputs = [ pkg-config cargo git ]; - buildInputs = [ openssl postgresql ] ++ lib.optionals (stdenv.isDarwin) [ - darwin.apple_sdk.frameworks.CoreFoundation - darwin.apple_sdk.frameworks.Security - darwin.apple_sdk.frameworks.SystemConfiguration + + nativeBuildInputs = [ + pkg-config + cargo + git ]; + buildInputs = + [ + openssl + postgresql + ] + ++ lib.optionals (stdenv.isDarwin) [ + darwin.apple_sdk.frameworks.CoreFoundation + darwin.apple_sdk.frameworks.Security + darwin.apple_sdk.frameworks.SystemConfiguration + ]; NIX_LDFLAGS = "-L${postgresql}/lib -lpq"; @@ -50,16 +86,19 @@ buildPgrxExtension_0_12_9 rec { # - PostgreSQL 15.8 → 5435 + 0 + (15-15)*2 = 5435 # - PostgreSQL 17_0 (OrioleDB) → 5435 + 1 + (17-15)*2 = 5440 # - PostgreSQL 17.4 → 5435 + 0 + (17-15)*2 = 5439 - PGPORT = toString (5534 + - (if builtins.match ".*_.*" postgresql.version != null then 1 else 0) + # +1 for OrioleDB - ((builtins.fromJSON (builtins.substring 0 2 postgresql.version)) - 15) * 2); # +2 for each major version + PGPORT = toString ( + 5534 + + (if builtins.match ".*_.*" postgresql.version != null then 1 else 0) + # +1 for OrioleDB + + ((builtins.fromJSON (builtins.substring 0 2 postgresql.version)) - 15) * 2 + ); # +2 for each major version }; OPENSSL_NO_VENDOR = 1; #need to set this to 2 to avoid cpu starvation CARGO_BUILD_JOBS = "2"; - CARGO="${cargo}/bin/cargo"; - + CARGO = "${cargo}/bin/cargo"; + #CARGO_NET_GIT_FETCH_WITH_CLI = "true"; cargoLock = { lockFile = "${src}/Cargo.lock"; @@ -69,9 +108,9 @@ buildPgrxExtension_0_12_9 rec { }; }; - preConfigure = '' + preConfigure = '' cd wrappers - + # update the clickhouse-rs dependency # append the branch name to the git URL to help cargo locate the commit # while maintaining the rev for reproducibility @@ -93,17 +132,17 @@ buildPgrxExtension_0_12_9 rec { } { print } ' Cargo.toml - + # Verify the file is still valid TOML, break build with this error # if it is not if ! cargo verify-project 2>/dev/null; then echo "Failed to maintain valid TOML syntax" exit 1 fi - + cd .. ''; - + buildAndTestSubdir = "wrappers"; buildFeatures = [ "helloworld_fdw" @@ -116,53 +155,53 @@ buildPgrxExtension_0_12_9 rec { echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt ''; - postInstall = '' - echo "Modifying main SQL file to use unversioned library name..." - current_version="${version}" - main_sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" - if [ -f "$main_sql_file" ]; then - sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$main_sql_file" - echo "Modified $main_sql_file" - else - echo "Warning: $main_sql_file not found" - fi - echo "Creating and modifying SQL files for previous versions..." - - if [ -f "$main_sql_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" - echo "Creating $new_file" - cp "$main_sql_file" "$new_file" - sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file" - echo "Modified $new_file" - fi - done < git_tags.txt - else - echo "Warning: $main_sql_file not found" - fi - mv $out/lib/wrappers-${version}${postgresql.dlSuffix} $out/lib/wrappers${postgresql.dlSuffix} - ln -s $out/lib/wrappers${postgresql.dlSuffix} $out/lib/wrappers-${version}${postgresql.dlSuffix} + postInstall = '' + echo "Modifying main SQL file to use unversioned library name..." + current_version="${version}" + main_sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" + if [ -f "$main_sql_file" ]; then + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$main_sql_file" + echo "Modified $main_sql_file" + else + echo "Warning: $main_sql_file not found" + fi + echo "Creating and modifying SQL files for previous versions..." + + if [ -f "$main_sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$main_sql_file" "$new_file" + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file" + echo "Modified $new_file" + fi + done < git_tags.txt + else + echo "Warning: $main_sql_file not found" + fi + mv $out/lib/wrappers-${version}${postgresql.dlSuffix} $out/lib/wrappers${postgresql.dlSuffix} + ln -s $out/lib/wrappers${postgresql.dlSuffix} $out/lib/wrappers-${version}${postgresql.dlSuffix} - echo "Creating wrappers.so symlinks to support pg_upgrade..." - if [ -f "$out/lib/wrappers.so" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/lib/wrappers-$previous_version.so" - echo "Creating $new_file" - ln -s "$out/lib/wrappers.so" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $out/lib/wrappers.so not found" - fi + echo "Creating wrappers.so symlinks to support pg_upgrade..." + if [ -f "$out/lib/wrappers.so" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/lib/wrappers-$previous_version.so" + echo "Creating $new_file" + ln -s "$out/lib/wrappers.so" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $out/lib/wrappers.so not found" + fi - rm git_tags.txt - echo "Contents of updated wrappers.control:" - cat "$out/share/postgresql/extension/wrappers.control" - echo "List of generated SQL files:" - ls -l $out/share/postgresql/extension/wrappers--*.sql - ''; + rm git_tags.txt + echo "Contents of updated wrappers.control:" + cat "$out/share/postgresql/extension/wrappers.control" + echo "List of generated SQL files:" + ls -l $out/share/postgresql/extension/wrappers--*.sql + ''; meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; diff --git a/nix/fmt.nix b/nix/fmt.nix new file mode 100644 index 000000000..760cb5930 --- /dev/null +++ b/nix/fmt.nix @@ -0,0 +1,12 @@ +{ inputs, ... }: +{ + imports = [ inputs.treefmt-nix.flakeModule ]; + perSystem = + { pkgs, ... }: + { + treefmt.programs.deadnix.enable = true; + + treefmt.programs.nixfmt.enable = true; + treefmt.programs.nixfmt.package = pkgs.nixfmt-rfc-style; + }; +} diff --git a/nix/hooks.nix b/nix/hooks.nix new file mode 100644 index 000000000..896c262ba --- /dev/null +++ b/nix/hooks.nix @@ -0,0 +1,19 @@ +{ inputs, ... }: +{ + imports = [ inputs.git-hooks.flakeModule ]; + perSystem = + { config, ... }: + { + pre-commit = { + check.enable = true; + settings = { + hooks = { + treefmt = { + enable = true; + package = config.treefmt.build.wrapper; + }; + }; + }; + }; + }; +} diff --git a/nix/mkdocs.yml b/nix/mkdocs.yml new file mode 100644 index 000000000..f472b51c4 --- /dev/null +++ b/nix/mkdocs.yml @@ -0,0 +1,132 @@ +# Project info +site_name: Postgres by Supabase +site_description: >- + Provide some of the most common Postgres extensions with a one-click install. + +# Repository +repo_name: supabase/postgres +repo_url: https://github.com/supabase/postgres + +nav: + - Home: README.md + - Development: + - Getting Started: start-here.md + - Nix tree structure: nix-directory-structure.md + - Workflow: development-workflow.md + - Build PostgreSQL: build-postgres.md + - Start Client/Server: start-client-server.md + - Docker: docker.md + - Use direnv: use-direnv.md + - Code Formatter: nix-formatter.md + - Pre-commit Hooks: pre-commit-hooks.md + - Documentation: documentation.md + - Package Management: + - Adding New Packages: adding-new-package.md + - Update Extensions: update-extension.md + - New Major PostgreSQL: new-major-postgres.md + - Nix Overlays: nix-overlays.md + - Testing: + - Adding Tests: adding-tests.md + - Migration Tests: migration-tests.md + - Testing PG Upgrade Scripts: testing-pg-upgrade-scripts.md + - References: references.md + +validation: + omitted_files: warn + absolute_links: warn + unrecognized_links: warn + +# Configuration + +docs_dir: docs +site_dir: out + +theme: + name: material + + logo: assets/images/logo.png + favicon: assets/images/logo.png + + features: + - content.code.annotate + - content.code.copy + - navigation.footer + - navigation.indexes + - navigation.path + - navigation.sections + - navigation.tabs + - navigation.tracking + - search.highlight + - search.share + - search.suggest + + font: + text: Inter + mono: Noto Sans Mono + palette: + # Palette toggle for automatic mode + - media: "(prefers-color-scheme)" + toggle: + icon: material/brightness-auto + name: Switch to light mode + + # Palette toggle for light mode + - media: "(prefers-color-scheme: light)" + scheme: default + toggle: + icon: material/brightness-7 + name: Switch to dark mode + + # Palette toggle for dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/brightness-4 + name: Switch to system preference + +extra: + version: + provider: mike + social: + - icon: fontawesome/brands/github + link: https://github.com/supabase/postgres + +extra_css: + - stylesheets/extra.css + +markdown_extensions: + - tables + - admonition + - attr_list + - footnotes + - md_in_html + - def_list + - meta + - pymdownx.emoji: + emoji_index: !!python/name:material.extensions.emoji.twemoji + emoji_generator: !!python/name:material.extensions.emoji.to_svg + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.superfences + - pymdownx.tabbed: + alternate_style: true + - pymdownx.details + - pymdownx.highlight: + use_pygments: true + linenums: true + anchor_linenums: true + - pymdownx.inlinehilite + - pymdownx.snippets: + base_path: ../. + - pymdownx.keys + - pymdownx.superfences: + # make exceptions to highlighting of code: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:mermaid2.fence_mermaid_custom + +plugins: + - mike + - search + - mermaid2 diff --git a/nix/nixpkgs.nix b/nix/nixpkgs.nix new file mode 100644 index 000000000..005dd80aa --- /dev/null +++ b/nix/nixpkgs.nix @@ -0,0 +1,16 @@ +{ self, inputs, ... }: +{ + perSystem = + { system, ... }: + { + _module.args.pkgs = import inputs.nixpkgs { + inherit system; + config.allowUnfree = true; + permittedInsecurePackages = [ "v8-9.7.106.18" ]; + overlays = [ + (import inputs.rust-overlay) + self.overlays.default + ]; + }; + }; +} diff --git a/nix/overlays/default.nix b/nix/overlays/default.nix new file mode 100644 index 000000000..f6eda4243 --- /dev/null +++ b/nix/overlays/default.nix @@ -0,0 +1,55 @@ +{ self, ... }: +{ + flake.overlays.default = final: prev: { + # NOTE: add any needed overlays here. in theory we could + # pull them from the overlays/ directory automatically, but we don't + # want to have an arbitrary order, since it might matter. being + # explicit is better. + + inherit (self.packages.${final.system}) + postgresql_15 + postgresql_17 + postgresql_orioledb-17 + supabase-groonga + ; + + xmrig = throw "The xmrig package has been explicitly disabled in this flake."; + + cargo-pgrx = final.callPackage ../cargo-pgrx/default.nix { + inherit (final) lib; + inherit (final) darwin; + inherit (final) fetchCrate; + inherit (final) openssl; + inherit (final) pkg-config; + inherit (final) makeRustPlatform; + inherit (final) stdenv; + inherit (final) rust-bin; + }; + + buildPgrxExtension = final.callPackage ../cargo-pgrx/buildPgrxExtension.nix { + inherit (final) cargo-pgrx; + inherit (final) lib; + inherit (final) Security; + inherit (final) pkg-config; + inherit (final) makeRustPlatform; + inherit (final) stdenv; + inherit (final) writeShellScriptBin; + }; + + buildPgrxExtension_0_11_3 = prev.buildPgrxExtension.override { + cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_11_3; + }; + + buildPgrxExtension_0_12_6 = prev.buildPgrxExtension.override { + cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_12_6; + }; + + buildPgrxExtension_0_12_9 = prev.buildPgrxExtension.override { + cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_12_9; + }; + + buildPgrxExtension_0_14_3 = prev.buildPgrxExtension.override { + cargo-pgrx = final.cargo-pgrx.cargo-pgrx_0_14_3; + }; + }; +} diff --git a/nix/packages/build-test-ami.nix b/nix/packages/build-test-ami.nix new file mode 100644 index 000000000..115ea793d --- /dev/null +++ b/nix/packages/build-test-ami.nix @@ -0,0 +1,158 @@ +{ pkgs, runCommand }: +runCommand "build-test-ami" + { + buildInputs = with pkgs; [ + packer + awscli2 + yq + jq + openssl + git + coreutils + aws-vault + ]; + } + '' + mkdir -p $out/bin + cat > $out/bin/build-test-ami << 'EOL' + #!/usr/bin/env bash + set -euo pipefail + + show_help() { + cat << EOF + Usage: build-test-ami [--help] + + Build AMI images for PostgreSQL testing. + + This script will: + 1. Check for required tools and AWS authentication + 2. Build two AMI stages using Packer + 3. Clean up any temporary instances + 4. Output the final AMI name for use with run-testinfra + + Arguments: + postgres-version PostgreSQL major version to build (required) + + Options: + --help Show this help message and exit + + Requirements: + - AWS Vault profile must be set in AWS_VAULT environment variable + - Packer, AWS CLI, yq, jq, and OpenSSL must be installed + - Must be run from a git repository + + Example: + aws-vault exec -- nix run .#build-test-ami 15 + EOF + } + + # Handle help flag + if [[ "$#" -gt 0 && "$1" == "--help" ]]; then + show_help + exit 0 + fi + + export PATH="${ + pkgs.lib.makeBinPath ( + with pkgs; + [ + packer + awscli2 + yq + jq + openssl + git + coreutils + aws-vault + ] + ) + }:$PATH" + + # Check for required tools + for cmd in packer aws-vault yq jq openssl; do + if ! command -v $cmd &> /dev/null; then + echo "Error: $cmd is required but not found" + exit 1 + fi + done + + # Check AWS Vault profile + if [ -z "''${AWS_VAULT:-}" ]; then + echo "Error: AWS_VAULT environment variable must be set with the profile name" + echo "Usage: aws-vault exec -- nix run .#build-test-ami " + exit 1 + fi + + # Set values + REGION="ap-southeast-1" + POSTGRES_VERSION="$1" + RANDOM_STRING=$(openssl rand -hex 8) + GIT_SHA=$(git rev-parse HEAD) + RUN_ID=$(date +%s) + + # Generate common-nix.vars.pkr.hcl + PG_VERSION=$(yq -r ".postgres_release[\"postgres$POSTGRES_VERSION\"]" ansible/vars.yml) + echo "postgres-version = \"$PG_VERSION\"" > common-nix.vars.pkr.hcl + + # Build AMI Stage 1 + packer init amazon-arm64-nix.pkr.hcl + packer build \ + -var "git-head-version=$GIT_SHA" \ + -var "packer-execution-id=$RUN_ID" \ + -var-file="development-arm.vars.pkr.hcl" \ + -var-file="common-nix.vars.pkr.hcl" \ + -var "ansible_arguments=" \ + -var "postgres-version=$RANDOM_STRING" \ + -var "region=$REGION" \ + -var 'ami_regions=["'"$REGION"'"]' \ + -var "force-deregister=true" \ + -var "ansible_arguments=-e postgresql_major=$POSTGRES_VERSION" \ + amazon-arm64-nix.pkr.hcl + + # Build AMI Stage 2 + packer init stage2-nix-psql.pkr.hcl + packer build \ + -var "git-head-version=$GIT_SHA" \ + -var "packer-execution-id=$RUN_ID" \ + -var "postgres_major_version=$POSTGRES_VERSION" \ + -var-file="development-arm.vars.pkr.hcl" \ + -var-file="common-nix.vars.pkr.hcl" \ + -var "postgres-version=$RANDOM_STRING" \ + -var "region=$REGION" \ + -var 'ami_regions=["'"$REGION"'"]' \ + -var "force-deregister=true" \ + -var "git_sha=$GIT_SHA" \ + stage2-nix-psql.pkr.hcl + + # Cleanup instances from AMI builds + cleanup_instances() { + echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." + aws ec2 --region $REGION describe-instances \ + --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ + --query "Reservations[].Instances[].InstanceId" \ + --output text | xargs -r aws ec2 terminate-instances \ + --region $REGION --instance-ids || true + } + + # Set up traps for various signals to ensure cleanup + trap cleanup_instances EXIT HUP INT QUIT TERM + + # Create and activate virtual environment + VENV_DIR=$(mktemp -d) + trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM + python3 -m venv "$VENV_DIR" + source "$VENV_DIR/bin/activate" + + # Install required Python packages + echo "Installing required Python packages..." + pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests + + # Run the tests with aws-vault + echo "Running tests for AMI: $RANDOM_STRING using AWS Vault profile: $AWS_VAULT_PROFILE" + aws-vault exec $AWS_VAULT_PROFILE -- pytest -vv -s testinfra/test_ami_nix.py + + # Deactivate virtual environment (cleanup is handled by trap) + deactivate + EOL + chmod +x $out/bin/build-test-ami + '' diff --git a/nix/packages/cleanup-ami.nix b/nix/packages/cleanup-ami.nix new file mode 100644 index 000000000..f1f9cde27 --- /dev/null +++ b/nix/packages/cleanup-ami.nix @@ -0,0 +1,61 @@ +{ pkgs, runCommand }: +runCommand "cleanup-ami" + { + buildInputs = with pkgs; [ + awscli2 + aws-vault + ]; + } + '' + mkdir -p $out/bin + cat > $out/bin/cleanup-ami << 'EOL' + #!/usr/bin/env bash + set -euo pipefail + + export PATH="${ + pkgs.lib.makeBinPath ( + with pkgs; + [ + awscli2 + aws-vault + ] + ) + }:$PATH" + + # Check for required tools + for cmd in aws-vault; do + if ! command -v $cmd &> /dev/null; then + echo "Error: $cmd is required but not found" + exit 1 + fi + done + + # Check AWS Vault profile + if [ -z "''${AWS_VAULT:-}" ]; then + echo "Error: AWS_VAULT environment variable must be set with the profile name" + echo "Usage: aws-vault exec -- nix run .#cleanup-ami " + exit 1 + fi + + # Check for AMI name argument + if [ -z "''${1:-}" ]; then + echo "Error: AMI name must be provided" + echo "Usage: aws-vault exec -- nix run .#cleanup-ami " + exit 1 + fi + + AMI_NAME="$1" + REGION="ap-southeast-1" + + # Deregister AMIs + for AMI_PATTERN in "supabase-postgres-ci-ami-test-stage-1" "$AMI_NAME"; do + aws ec2 describe-images --region $REGION --owners self \ + --filters "Name=name,Values=$AMI_PATTERN" \ + --query 'Images[*].ImageId' --output text | while read -r ami_id; do + echo "Deregistering AMI: $ami_id" + aws ec2 deregister-image --region $REGION --image-id "$ami_id" || true + done + done + EOL + chmod +x $out/bin/cleanup-ami + '' diff --git a/nix/packages/dbmate-tool.nix b/nix/packages/dbmate-tool.nix new file mode 100644 index 000000000..7ce2c5031 --- /dev/null +++ b/nix/packages/dbmate-tool.nix @@ -0,0 +1,45 @@ +{ + pkgs, + system, + defaults, +}: +let + migrationsDir = ../../migrations/db; + ansibleVars = ../../ansible/vars.yml; + pgbouncerAuthSchemaSql = ../../ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; + statExtensionSql = ../../ansible/files/stat_extension.sql; +in +pkgs.runCommand "dbmate-tool" + { + buildInputs = with pkgs; [ + overmind + dbmate + nix + jq + yq + ]; + nativeBuildInputs = with pkgs; [ makeWrapper ]; + } + '' + mkdir -p $out/bin $out/migrations + cp -r ${migrationsDir}/* $out + substitute ${../tools/dbmate-tool.sh.in} $out/bin/dbmate-tool \ + --subst-var-by 'PGSQL_DEFAULT_PORT' '${defaults.port}' \ + --subst-var-by 'MIGRATIONS_DIR' $out \ + --subst-var-by 'PGSQL_SUPERUSER' '${defaults.superuser}' \ + --subst-var-by 'ANSIBLE_VARS' ${ansibleVars} \ + --subst-var-by 'CURRENT_SYSTEM' '${system}' \ + --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ + --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' + chmod +x $out/bin/dbmate-tool + wrapProgram $out/bin/dbmate-tool \ + --prefix PATH : ${ + pkgs.lib.makeBinPath [ + pkgs.overmind + pkgs.dbmate + pkgs.nix + pkgs.jq + pkgs.yq + ] + } + '' diff --git a/nix/packages/default.nix b/nix/packages/default.nix new file mode 100644 index 000000000..cd6d6a1d0 --- /dev/null +++ b/nix/packages/default.nix @@ -0,0 +1,78 @@ +{ self, ... }: +{ + imports = [ ./postgres.nix ]; + perSystem = + { + inputs', + lib, + pkgs, + self', + ... + }: + let + activeVersion = "15"; + # Function to create the pg_regress package + makePgRegress = + version: + let + postgresqlPackage = self'.packages."postgresql_${version}"; + in + pkgs.callPackage ../ext/pg_regress.nix { postgresql = postgresqlPackage; }; + pgsqlSuperuser = "supabase_admin"; + pkgs-lib = pkgs.callPackage ./lib.nix { + psql_15 = self'.packages."psql_15/bin"; + psql_17 = self'.packages."psql_17/bin"; + psql_orioledb-17 = self'.packages."psql_orioledb-17/bin"; + pgroonga = self'.packages."psql_15/exts/pgroonga"; + inherit (self.supabase) defaults; + }; + in + { + packages = ( + { + build-test-ami = pkgs.callPackage ./build-test-ami.nix { }; + cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { }; + dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; }; + docs = pkgs.callPackage ./docs.nix { }; + supabase-groonga = pkgs.callPackage ./groonga { }; + local-infra-bootstrap = pkgs.callPackage ./local-infra-bootstrap.nix { }; + migrate-tool = pkgs.callPackage ./migrate-tool.nix { psql_15 = self'.packages."psql_15/bin"; }; + pg-restore = pkgs.callPackage ./pg-restore.nix { psql_15 = self'.packages."psql_15/bin"; }; + pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; + pg_regress = makePgRegress activeVersion; + run-testinfra = pkgs.callPackage ./run-testinfra.nix { }; + show-commands = pkgs.callPackage ./show-commands.nix { }; + start-client = pkgs.callPackage ./start-client.nix { + psql_15 = self'.packages."psql_15/bin"; + psql_17 = self'.packages."psql_17/bin"; + psql_orioledb-17 = self'.packages."psql_orioledb-17/bin"; + inherit (self.supabase) defaults; + }; + start-replica = pkgs.callPackage ./start-replica.nix { + psql_15 = self'.packages."psql_15/bin"; + inherit pgsqlSuperuser; + }; + start-server = pkgs-lib.makePostgresDevSetup { + inherit pkgs; + name = "start-postgres-server"; + }; + sync-exts-versions = pkgs.callPackage ./sync-exts-versions.nix { inherit (inputs') nix-editor; }; + trigger-nix-build = pkgs.callPackage ./trigger-nix-build.nix { }; + update-readme = pkgs.callPackage ./update-readme.nix { }; + inherit (pkgs.callPackage ./wal-g.nix { }) wal-g-2 wal-g-3; + inherit (pkgs.cargo-pgrx) + cargo-pgrx_0_11_3 + cargo-pgrx_0_12_6 + cargo-pgrx_0_12_9 + cargo-pgrx_0_14_3 + ; + } + // lib.filterAttrs (n: _v: n != "override" && n != "overrideAttrs" && n != "overrideDerivation") ( + pkgs.callPackage ../postgresql/default.nix { + inherit self'; + inherit (self.supabase) supportedPostgresVersions; + } + ) + ); + }; +} diff --git a/nix/packages/docs.nix b/nix/packages/docs.nix new file mode 100644 index 000000000..822661368 --- /dev/null +++ b/nix/packages/docs.nix @@ -0,0 +1,22 @@ +{ stdenvNoCC, python3Packages, ... }: +stdenvNoCC.mkDerivation { + name = "docs"; + + src = ../.; + + nativeBuildInputs = with python3Packages; [ + mike + mkdocs + mkdocs-material + mkdocs-linkcheck + mkdocs-mermaid2-plugin + ]; + + buildPhase = '' + mkdocs build + ''; + + installPhase = '' + mv out $out + ''; +} diff --git a/nix/supabase-groonga.nix b/nix/packages/groonga/default.nix similarity index 65% rename from nix/supabase-groonga.nix rename to nix/packages/groonga/default.nix index 5035f369f..3fcd4b691 100644 --- a/nix/supabase-groonga.nix +++ b/nix/packages/groonga/default.nix @@ -1,22 +1,59 @@ -{ lib, stdenv, cmake, fetchurl, kytea, msgpack-c, mecab, pkg-config, rapidjson -, testers, xxHash, zstd, postgresqlPackages, makeWrapper, suggestSupport ? false -, zeromq, libevent, openssl, lz4Support ? false, lz4, zlibSupport ? true, zlib -, writeShellScriptBin, callPackage }: -let mecab-naist-jdic = callPackage ./ext/mecab-naist-jdic { }; -in stdenv.mkDerivation (finalAttrs: { +{ + lib, + stdenv, + cmake, + fetchurl, + kytea, + msgpack-c, + mecab, + pkg-config, + rapidjson, + xxHash, + zstd, + makeWrapper, + suggestSupport ? false, + zeromq, + libevent, + lz4Support ? false, + lz4, + zlibSupport ? true, + zlib, + callPackage, +}: +let + mecab-naist-jdic = callPackage ../../ext/mecab-naist-jdic { }; +in +stdenv.mkDerivation (finalAttrs: { pname = "supabase-groonga"; version = "14.0.5"; src = fetchurl { - url = - "https://packages.groonga.org/source/groonga/groonga-${finalAttrs.version}.tar.gz"; + url = "https://packages.groonga.org/source/groonga/groonga-${finalAttrs.version}.tar.gz"; hash = "sha256-y4UGnv8kK0z+br8wXpPf57NMXkdEJHcLCuTvYiubnIc="; }; - patches = - [ ./fix-cmake-install-path.patch ./do-not-use-vendored-libraries.patch ]; - nativeBuildInputs = [ cmake pkg-config makeWrapper ]; - buildInputs = [ rapidjson xxHash zstd mecab kytea msgpack-c ] - ++ lib.optionals lz4Support [ lz4 ] ++ lib.optional zlibSupport [ zlib ] - ++ lib.optionals suggestSupport [ zeromq libevent ]; + patches = [ + ./fix-cmake-install-path.patch + ./do-not-use-vendored-libraries.patch + ]; + nativeBuildInputs = [ + cmake + pkg-config + makeWrapper + ]; + buildInputs = + [ + rapidjson + xxHash + zstd + mecab + kytea + msgpack-c + ] + ++ lib.optionals lz4Support [ lz4 ] + ++ lib.optional zlibSupport [ zlib ] + ++ lib.optionals suggestSupport [ + zeromq + libevent + ]; cmakeFlags = [ "-DWITH_MECAB=ON" "-DMECAB_DICDIR=${mecab-naist-jdic}/lib/mecab/dic/naist-jdic" @@ -58,8 +95,7 @@ in stdenv.mkDerivation (finalAttrs: { --set GRN_PLUGINS_DIR $out/lib/groonga/plugins ''; - env.NIX_CFLAGS_COMPILE = - lib.optionalString zlibSupport "-I${zlib.dev}/include"; + env.NIX_CFLAGS_COMPILE = lib.optionalString zlibSupport "-I${zlib.dev}/include"; meta = with lib; { homepage = "https://groonga.org/"; diff --git a/nix/do-not-use-vendored-libraries.patch b/nix/packages/groonga/do-not-use-vendored-libraries.patch similarity index 100% rename from nix/do-not-use-vendored-libraries.patch rename to nix/packages/groonga/do-not-use-vendored-libraries.patch diff --git a/nix/fix-cmake-install-path.patch b/nix/packages/groonga/fix-cmake-install-path.patch similarity index 100% rename from nix/fix-cmake-install-path.patch rename to nix/packages/groonga/fix-cmake-install-path.patch diff --git a/nix/packages/lib.nix b/nix/packages/lib.nix new file mode 100644 index 000000000..971909162 --- /dev/null +++ b/nix/packages/lib.nix @@ -0,0 +1,137 @@ +{ + psql_17, + psql_15, + psql_orioledb-17, + defaults, + supabase-groonga, + system, + pgroonga, +}: +{ + makePostgresDevSetup = + { + pkgs, + name, + extraSubstitutions ? { }, + }: + let + paths = { + migrationsDir = builtins.path { + name = "migrations"; + path = ../../migrations/db; + }; + postgresqlSchemaSql = builtins.path { + name = "postgresql-schema"; + path = ../tools/postgresql_schema.sql; + }; + pgbouncerAuthSchemaSql = builtins.path { + name = "pgbouncer-auth-schema"; + path = ../../ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; + }; + statExtensionSql = builtins.path { + name = "stat-extension"; + path = ../../ansible/files/stat_extension.sql; + }; + pgconfigFile = builtins.path { + name = "postgresql.conf"; + path = ../../ansible/files/postgresql_config/postgresql.conf.j2; + }; + supautilsConfigFile = builtins.path { + name = "supautils.conf"; + path = ../../ansible/files/postgresql_config/supautils.conf.j2; + }; + loggingConfigFile = builtins.path { + name = "logging.conf"; + path = ../../ansible/files/postgresql_config/postgresql-csvlog.conf; + }; + readReplicaConfigFile = builtins.path { + name = "readreplica.conf"; + path = ../../ansible/files/postgresql_config/custom_read_replica.conf.j2; + }; + pgHbaConfigFile = builtins.path { + name = "pg_hba.conf"; + path = ../../ansible/files/postgresql_config/pg_hba.conf.j2; + }; + pgIdentConfigFile = builtins.path { + name = "pg_ident.conf"; + path = ../../ansible/files/postgresql_config/pg_ident.conf.j2; + }; + postgresqlExtensionCustomScriptsPath = builtins.path { + name = "extension-custom-scripts"; + path = ../../ansible/files/postgresql_extension_custom_scripts; + }; + getkeyScript = builtins.path { + name = "pgsodium_getkey.sh"; + path = ../tests/util/pgsodium_getkey.sh; + }; + }; + + localeArchive = + if pkgs.stdenv.isDarwin then + "${pkgs.darwin.locale}/share/locale" + else + "${pkgs.glibcLocales}/lib/locale/locale-archive"; + + substitutions = { + SHELL_PATH = "${pkgs.bash}/bin/bash"; + PGSQL_DEFAULT_PORT = "${defaults.port}"; + PGSQL_SUPERUSER = "${defaults.superuser}"; + PSQL15_BINDIR = "${psql_15}"; + PSQL17_BINDIR = "${psql_17}"; + PSQL_CONF_FILE = "${paths.pgconfigFile}"; + PSQLORIOLEDB17_BINDIR = "${psql_orioledb-17}"; + PGSODIUM_GETKEY = "${paths.getkeyScript}"; + READREPL_CONF_FILE = "${paths.readReplicaConfigFile}"; + LOGGING_CONF_FILE = "${paths.loggingConfigFile}"; + SUPAUTILS_CONF_FILE = "${paths.supautilsConfigFile}"; + PG_HBA = "${paths.pgHbaConfigFile}"; + PG_IDENT = "${paths.pgIdentConfigFile}"; + LOCALES = "${localeArchive}"; + EXTENSION_CUSTOM_SCRIPTS_DIR = "${paths.postgresqlExtensionCustomScriptsPath}"; + MECAB_LIB = "${pgroonga}/lib/groonga/plugins/tokenizers/tokenizer_mecab.so"; + GROONGA_DIR = "${supabase-groonga}"; + MIGRATIONS_DIR = "${paths.migrationsDir}"; + POSTGRESQL_SCHEMA_SQL = "${paths.postgresqlSchemaSql}"; + PGBOUNCER_AUTH_SCHEMA_SQL = "${paths.pgbouncerAuthSchemaSql}"; + STAT_EXTENSION_SQL = "${paths.statExtensionSql}"; + CURRENT_SYSTEM = "${system}"; + } // extraSubstitutions; # Merge in any extra substitutions + in + pkgs.runCommand name + { + inherit (paths) + migrationsDir + postgresqlSchemaSql + pgbouncerAuthSchemaSql + statExtensionSql + ; + } + '' + mkdir -p $out/bin $out/etc/postgresql-custom $out/etc/postgresql $out/extension-custom-scripts + + # Copy config files with error handling + cp ${paths.supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; } + cp ${paths.pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; } + cp ${paths.loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; } + cp ${paths.readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; } + cp ${paths.pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; } + cp ${paths.pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; } + cp -r ${paths.postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; } + + echo "Copy operation completed" + chmod 644 $out/etc/postgresql-custom/supautils.conf + chmod 644 $out/etc/postgresql/postgresql.conf + chmod 644 $out/etc/postgresql-custom/logging.conf + chmod 644 $out/etc/postgresql/pg_hba.conf + + substitute ${../tools/run-server.sh.in} $out/bin/start-postgres-server \ + ${ + builtins.concatStringsSep " " ( + builtins.attrValues ( + builtins.mapAttrs (name: value: "--subst-var-by '${name}' '${value}'") substitutions + ) + ) + } + chmod +x $out/bin/start-postgres-server + ''; +} diff --git a/nix/packages/local-infra-bootstrap.nix b/nix/packages/local-infra-bootstrap.nix new file mode 100644 index 000000000..82e15de37 --- /dev/null +++ b/nix/packages/local-infra-bootstrap.nix @@ -0,0 +1,6 @@ +{ runCommand }: +runCommand "local-infra-bootstrap" { } '' + mkdir -p $out/bin + substitute ${./local-infra-bootstrap.sh.in} $out/bin/local-infra-bootstrap + chmod +x $out/bin/local-infra-bootstrap +'' diff --git a/nix/tools/local-infra-bootstrap.sh.in b/nix/packages/local-infra-bootstrap.sh.in similarity index 100% rename from nix/tools/local-infra-bootstrap.sh.in rename to nix/packages/local-infra-bootstrap.sh.in diff --git a/nix/packages/migrate-tool.nix b/nix/packages/migrate-tool.nix new file mode 100644 index 000000000..57f218a63 --- /dev/null +++ b/nix/packages/migrate-tool.nix @@ -0,0 +1,18 @@ +{ runCommand, psql_15 }: +let + configFile = ../tests/postgresql.conf.in; + getkeyScript = ../tests/util/pgsodium_getkey.sh; + primingScript = ../tests/prime.sql; + migrationData = ../tests/migrations/data.sql; +in +runCommand "migrate-postgres" { } '' + mkdir -p $out/bin + substitute ${../tools/migrate-tool.sh.in} $out/bin/migrate-postgres \ + --subst-var-by 'PSQL15_BINDIR' '${psql_15}' \ + --subst-var-by 'PSQL_CONF_FILE' '${configFile}' \ + --subst-var-by 'PGSODIUM_GETKEY' '${getkeyScript}' \ + --subst-var-by 'PRIMING_SCRIPT' '${primingScript}' \ + --subst-var-by 'MIGRATION_DATA' '${migrationData}' + + chmod +x $out/bin/migrate-postgres +'' diff --git a/nix/packages/pg-restore.nix b/nix/packages/pg-restore.nix new file mode 100644 index 000000000..aa01e7a7e --- /dev/null +++ b/nix/packages/pg-restore.nix @@ -0,0 +1,7 @@ +{ runCommand, psql_15 }: +runCommand "run-pg-restore" { } '' + mkdir -p $out/bin + substitute ${./run-restore.sh.in} $out/bin/pg-restore \ + --subst-var-by PSQL15_BINDIR '${psql_15}' + chmod +x $out/bin/pg-restore +'' diff --git a/nix/packages/postgres.nix b/nix/packages/postgres.nix new file mode 100644 index 000000000..bf8710b4d --- /dev/null +++ b/nix/packages/postgres.nix @@ -0,0 +1,176 @@ +{ self, inputs, ... }: +{ + perSystem = + { pkgs, ... }: + let + gitRev = "vcs=${self.shortRev or "dirty"}+${ + builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101") + }"; + + # Custom extensions that exist in our repository. These aren't upstream + # either because nobody has done the work, maintaining them here is + # easier and more expedient, or because they may not be suitable, or are + # too niche/one-off. + # + # Ideally, most of these should have copies upstream for third party + # use, but even if they did, keeping our own copies means that we can + # rollout new versions of these critical things easier without having to + # go through the upstream release engineering process. + ourExtensions = [ + ../ext/rum.nix + ../ext/timescaledb.nix + ../ext/timescaledb-2.9.1.nix + ../ext/pgroonga.nix + ../ext/index_advisor.nix + ../ext/wal2json.nix + ../ext/pgmq.nix + ../ext/pg_repack.nix + ../ext/pg-safeupdate.nix + ../ext/plpgsql-check.nix + ../ext/pgjwt.nix + ../ext/pgaudit.nix + ../ext/postgis.nix + ../ext/pgrouting.nix + ../ext/pgtap.nix + ../ext/pg_cron.nix + ../ext/pgsql-http.nix + ../ext/pg_plan_filter.nix + ../ext/pg_net.nix + ../ext/pg_hashids.nix + ../ext/pgsodium.nix + ../ext/pg_graphql.nix + ../ext/pg_stat_monitor.nix + ../ext/pg_jsonschema.nix + ../ext/pgvector.nix + ../ext/vault.nix + ../ext/hypopg.nix + ../ext/pg_tle.nix + ../ext/wrappers/default.nix + ../ext/supautils.nix + ../ext/plv8.nix + ]; + + #Where we import and build the orioledb extension, we add on our custom extensions + # plus the orioledb option + #we're not using timescaledb or plv8 in the orioledb-17 version or pg 17 of supabase extensions + orioleFilteredExtensions = builtins.filter ( + x: x != ../ext/timescaledb.nix && x != ../ext/timescaledb-2.9.1.nix && x != ../ext/plv8.nix + ) ourExtensions; + + orioledbExtensions = orioleFilteredExtensions ++ [ ../ext/orioledb.nix ]; + dbExtensions17 = orioleFilteredExtensions; + getPostgresqlPackage = version: pkgs."postgresql_${version}"; + # Create a 'receipt' file for a given postgresql package. This is a way + # of adding a bit of metadata to the package, which can be used by other + # tools to inspect what the contents of the install are: the PSQL + # version, the installed extensions, et cetera. + # + # This takes two arguments: + # - pgbin: the postgresql package we are building on top of + # not a list of packages, but an attrset containing extension names + # mapped to versions. + # - ourExts: the list of extensions from upstream nixpkgs. This is not + # a list of packages, but an attrset containing extension names + # mapped to versions. + # + # The output is a package containing the receipt.json file, which can be + # merged with the PostgreSQL installation using 'symlinkJoin'. + makeReceipt = + pgbin: ourExts: + pkgs.writeTextFile { + name = "receipt"; + destination = "/receipt.json"; + text = builtins.toJSON { + revision = gitRev; + psql-version = pgbin.version; + nixpkgs = { + revision = inputs.nixpkgs.rev; + }; + extensions = ourExts; + + # NOTE this field can be used to do cache busting (e.g. + # force a rebuild of the psql packages) but also to helpfully inform + # tools what version of the schema is being used, for forwards and + # backwards compatibility + receipt-version = "1"; + }; + }; + + makeOurPostgresPkgs = + version: + let + postgresql = getPostgresqlPackage version; + extensionsToUse = + if (builtins.elem version [ "orioledb-17" ]) then + orioledbExtensions + else if (builtins.elem version [ "17" ]) then + dbExtensions17 + else + ourExtensions; + in + map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse; + + # Create an attrset that contains all the extensions included in a server. + makeOurPostgresPkgsSet = + version: + (builtins.listToAttrs ( + map (drv: { + name = drv.pname; + value = drv; + }) (makeOurPostgresPkgs version) + )) + // { + recurseForDerivations = true; + }; + + # Create a binary distribution of PostgreSQL, given a version. + # + # NOTE: The version here does NOT refer to the exact PostgreSQL version; + # it refers to the *major number only*, which is used to select the + # correct version of the package from nixpkgs. This is because we want + # to be able to do so in an open ended way. As an example, the version + # "15" passed in will use the nixpkgs package "postgresql_15" as the + # basis for building extensions, etc. + makePostgresBin = + version: + let + postgresql = getPostgresqlPackage version; + ourExts = map (ext: { + name = ext.pname; + version = ext.version; + }) (makeOurPostgresPkgs version); + + pgbin = postgresql.withPackages (_ps: makeOurPostgresPkgs version); + in + pkgs.symlinkJoin { + inherit (pgbin) name version; + paths = [ + pgbin + (makeReceipt pgbin ourExts) + ]; + }; + + # Create an attribute set, containing all the relevant packages for a + # PostgreSQL install, wrapped up with a bow on top. There are three + # packages: + # + # - bin: the postgresql package itself, with all the extensions + # installed, and a receipt.json file containing metadata about the + # install. + # - exts: an attrset containing all the extensions, mapped to their + # package names. + makePostgres = version: { + bin = makePostgresBin version; + exts = makeOurPostgresPkgsSet version; + recurseForDerivations = true; + }; + basePackages = { + psql_15 = makePostgres "15"; + psql_17 = makePostgres "17"; + psql_orioledb-17 = makePostgres "orioledb-17"; + }; + in + { + packages = inputs.flake-utils.lib.flattenTree basePackages; + }; +} diff --git a/nix/tools/run-restore.sh.in b/nix/packages/run-restore.sh.in similarity index 100% rename from nix/tools/run-restore.sh.in rename to nix/packages/run-restore.sh.in diff --git a/nix/packages/run-testinfra.nix b/nix/packages/run-testinfra.nix new file mode 100644 index 000000000..92f9d0a4a --- /dev/null +++ b/nix/packages/run-testinfra.nix @@ -0,0 +1,146 @@ +{ + runCommand, + aws-vault, + python3, + python3Packages, + coreutils, +}: +runCommand "run-testinfra" + { + buildInputs = [ + aws-vault + python3 + python3Packages.pip + coreutils + ]; + } + '' + mkdir -p $out/bin + cat > $out/bin/run-testinfra << 'EOL' + #!/usr/bin/env bash + set -euo pipefail + + show_help() { + cat << EOF + Usage: run-testinfra --ami-name NAME [--aws-vault-profile PROFILE] + + Run the testinfra tests locally against a specific AMI. + + This script will: + 1. Check if aws-vault is installed and configured + 2. Set up the required environment variables + 3. Create and activate a virtual environment + 4. Install required Python packages from pip + 5. Run the tests with aws-vault credentials + 6. Clean up the virtual environment + + Required flags: + --ami-name NAME The name of the AMI to test + + Optional flags: + --aws-vault-profile PROFILE AWS Vault profile to use (default: staging) + --help Show this help message and exit + + Requirements: + - aws-vault installed and configured + - Python 3 with pip + - Must be run from the repository root + + Examples: + run-testinfra --ami-name supabase-postgres-abc123 + run-testinfra --ami-name supabase-postgres-abc123 --aws-vault-profile production + EOF + } + + # Default values + AWS_VAULT_PROFILE="staging" + AMI_NAME="" + + # Parse arguments + while [[ $# -gt 0 ]]; do + case $1 in + --aws-vault-profile) + AWS_VAULT_PROFILE="$2" + shift 2 + ;; + --ami-name) + AMI_NAME="$2" + shift 2 + ;; + --help) + show_help + exit 0 + ;; + *) + echo "Error: Unexpected argument: $1" + show_help + exit 1 + ;; + esac + done + + # Check for required tools + if ! command -v aws-vault &> /dev/null; then + echo "Error: aws-vault is required but not found" + exit 1 + fi + + # Check for AMI name argument + if [ -z "$AMI_NAME" ]; then + echo "Error: --ami-name is required" + show_help + exit 1 + fi + + # Set environment variables + export AWS_REGION="ap-southeast-1" + export AWS_DEFAULT_REGION="ap-southeast-1" + export AMI_NAME="$AMI_NAME" # Export AMI_NAME for pytest + export RUN_ID="local-$(date +%s)" # Generate a unique RUN_ID + + # Function to terminate EC2 instances + terminate_instances() { + echo "Terminating EC2 instances with tag testinfra-run-id=$RUN_ID..." + aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 --region ap-southeast-1 describe-instances \ + --filters "Name=tag:testinfra-run-id,Values=$RUN_ID" \ + --query "Reservations[].Instances[].InstanceId" \ + --output text | xargs -r aws-vault exec $AWS_VAULT_PROFILE -- aws ec2 terminate-instances \ + --region ap-southeast-1 --instance-ids || true + } + + # Set up traps for various signals to ensure cleanup + trap terminate_instances EXIT HUP INT QUIT TERM + + # Create and activate virtual environment + VENV_DIR=$(mktemp -d) + trap 'rm -rf "$VENV_DIR"' EXIT HUP INT QUIT TERM + python3 -m venv "$VENV_DIR" + source "$VENV_DIR/bin/activate" + + # Install required Python packages + echo "Installing required Python packages..." + pip install boto3 boto3-stubs[essential] docker ec2instanceconnectcli pytest paramiko requests + + # Function to run tests and ensure cleanup + run_tests() { + local exit_code=0 + echo "Running tests for AMI: $AMI_NAME using AWS Vault profile: $AWS_VAULT_PROFILE" + aws-vault exec "$AWS_VAULT_PROFILE" -- pytest -vv -s testinfra/test_ami_nix.py || exit_code=$? + return $exit_code + } + + # Run tests and capture exit code + run_tests + test_exit_code=$? + + # Deactivate virtual environment + deactivate + + # Explicitly call cleanup + terminate_instances + + # Exit with the test exit code + exit $test_exit_code + EOL + chmod +x $out/bin/run-testinfra + '' diff --git a/nix/packages/show-commands.nix b/nix/packages/show-commands.nix new file mode 100644 index 000000000..b0733761e --- /dev/null +++ b/nix/packages/show-commands.nix @@ -0,0 +1,23 @@ +{ + runCommand, + makeWrapper, + nushell, + system ? builtins.currentSystem, +}: +runCommand "show-commands" + { + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ nushell ]; + } + '' + mkdir -p $out/bin + cat > $out/bin/show-commands << 'EOF' + #!${nushell}/bin/nu + let json_output = (nix flake show --json --quiet --all-systems | from json) + let apps = ($json_output | get apps.${system}) + $apps | transpose name info | select name | each { |it| echo $"Run this app with: nix run .#($it.name)" } + EOF + chmod +x $out/bin/show-commands + wrapProgram $out/bin/show-commands \ + --prefix PATH : ${nushell}/bin + '' diff --git a/nix/packages/start-client.nix b/nix/packages/start-client.nix new file mode 100644 index 000000000..84ab06209 --- /dev/null +++ b/nix/packages/start-client.nix @@ -0,0 +1,27 @@ +{ + runCommand, + psql_15, + psql_17, + psql_orioledb-17, + defaults, +}: +let + migrationsDir = ../../migrations/db; + postgresqlSchemaSql = ../tools/postgresql_schema.sql; + pgbouncerAuthSchemaSql = ../../ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql; + statExtensionSql = ../../ansible/files/stat_extension.sql; +in +runCommand "start-postgres-client" { } '' + mkdir -p $out/bin + substitute ${../tools/run-client.sh.in} $out/bin/start-postgres-client \ + --subst-var-by 'PGSQL_DEFAULT_PORT' '${defaults.port}' \ + --subst-var-by 'PGSQL_SUPERUSER' '${defaults.superuser}' \ + --subst-var-by 'PSQL15_BINDIR' '${psql_15}' \ + --subst-var-by 'PSQL17_BINDIR' '${psql_17}' \ + --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${psql_orioledb-17}' \ + --subst-var-by 'MIGRATIONS_DIR' '${migrationsDir}' \ + --subst-var-by 'POSTGRESQL_SCHEMA_SQL' '${postgresqlSchemaSql}' \ + --subst-var-by 'PGBOUNCER_AUTH_SCHEMA_SQL' '${pgbouncerAuthSchemaSql}' \ + --subst-var-by 'STAT_EXTENSION_SQL' '${statExtensionSql}' + chmod +x $out/bin/start-postgres-client +'' diff --git a/nix/packages/start-replica.nix b/nix/packages/start-replica.nix new file mode 100644 index 000000000..129ccd9cc --- /dev/null +++ b/nix/packages/start-replica.nix @@ -0,0 +1,12 @@ +{ + runCommand, + pgsqlSuperuser, + psql_15, +}: +runCommand "start-postgres-replica" { } '' + mkdir -p $out/bin + substitute ${./start-replica.sh.in} $out/bin/start-postgres-replica \ + --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ + --subst-var-by 'PSQL15_BINDIR' '${psql_15}' + chmod +x $out/bin/start-postgres-replica +'' diff --git a/nix/tools/run-replica.sh.in b/nix/packages/start-replica.sh.in similarity index 100% rename from nix/tools/run-replica.sh.in rename to nix/packages/start-replica.sh.in diff --git a/nix/packages/sync-exts-versions.nix b/nix/packages/sync-exts-versions.nix new file mode 100644 index 000000000..d76e66553 --- /dev/null +++ b/nix/packages/sync-exts-versions.nix @@ -0,0 +1,17 @@ +{ + runCommand, + jq, + yq, + nix-editor, + nixVersions, +}: +runCommand "sync-exts-versions" { } '' + mkdir -p $out/bin + substitute ${../tools/sync-exts-versions.sh.in} $out/bin/sync-exts-versions \ + --subst-var-by 'YQ' '${yq}/bin/yq' \ + --subst-var-by 'JQ' '${jq}/bin/jq' \ + --subst-var-by 'NIX_EDITOR' '${nix-editor.packages.nix-editor}/bin/nix-editor' \ + --subst-var-by 'NIXPREFETCHURL' '${nixVersions.nix_2_20}/bin/nix-prefetch-url' \ + --subst-var-by 'NIX' '${nixVersions.nix_2_20}/bin/nix' + chmod +x $out/bin/sync-exts-versions +'' diff --git a/nix/packages/trigger-nix-build.nix b/nix/packages/trigger-nix-build.nix new file mode 100644 index 000000000..3c006c028 --- /dev/null +++ b/nix/packages/trigger-nix-build.nix @@ -0,0 +1,125 @@ +{ + runCommand, + gh, + git, + coreutils, + lib, +}: +runCommand "trigger-nix-build" + { + buildInputs = [ + gh + git + coreutils + ]; + } + '' + mkdir -p $out/bin + cat > $out/bin/trigger-nix-build << 'EOL' + #!/usr/bin/env bash + set -euo pipefail + + show_help() { + cat << EOF + Usage: trigger-nix-build [--help] + + Trigger the nix-build workflow for the current branch and watch its progress. + + This script will: + 1. Check if you're authenticated with GitHub + 2. Get the current branch and commit + 3. Verify you're on a standard branch (develop or release/*) or prompt for confirmation + 4. Trigger the nix-build workflow + 5. Watch the workflow progress until completion + + Options: + --help Show this help message and exit + + Requirements: + - GitHub CLI (gh) installed and authenticated + - Git installed + - Must be run from a git repository + + Example: + trigger-nix-build + EOF + } + + # Handle help flag + if [[ "$#" -gt 0 && "$1" == "--help" ]]; then + show_help + exit 0 + fi + + export PATH="${ + lib.makeBinPath ([ + gh + git + coreutils + ]) + }:$PATH" + + # Check for required tools + for cmd in gh git; do + if ! command -v $cmd &> /dev/null; then + echo "Error: $cmd is required but not found" + exit 1 + fi + done + + # Check if user is authenticated with GitHub + if ! gh auth status &>/dev/null; then + echo "Error: Not authenticated with GitHub. Please run 'gh auth login' first." + exit 1 + fi + + # Get current branch and commit + BRANCH=$(git rev-parse --abbrev-ref HEAD) + COMMIT=$(git rev-parse HEAD) + + # Check if we're on a standard branch + if [[ "$BRANCH" != "develop" && ! "$BRANCH" =~ ^release/ ]]; then + echo "Warning: Running workflow from non-standard branch: $BRANCH" + echo "This is supported for testing purposes." + read -p "Continue? [y/N] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + echo "Aborted." + exit 1 + fi + fi + + # Trigger the workflow + echo "Triggering nix-build workflow for branch $BRANCH (commit: $COMMIT)" + gh workflow run nix-build.yml --ref "$BRANCH" + + # Wait for workflow to start and get the run ID + echo "Waiting for workflow to start..." + sleep 5 + + # Get the latest run ID for this workflow + RUN_ID=$(gh run list --workflow=nix-build.yml --limit 1 --json databaseId --jq '.[0].databaseId') + + if [ -z "$RUN_ID" ]; then + echo "Error: Could not find workflow run ID" + exit 1 + fi + + echo "Watching workflow run $RUN_ID..." + echo "The script will automatically exit when the workflow completes." + echo "Press Ctrl+C to stop watching (workflow will continue running)" + echo "----------------------------------------" + + # Try to watch the run, but handle network errors gracefully + while true; do + if gh run watch "$RUN_ID" --exit-status; then + break + else + echo "Network error while watching workflow. Retrying in 5 seconds..." + echo "You can also check the status manually with: gh run view $RUN_ID" + sleep 5 + fi + done + EOL + chmod +x $out/bin/trigger-nix-build + '' diff --git a/nix/packages/update-readme.nix b/nix/packages/update-readme.nix new file mode 100644 index 000000000..40fd1e6e5 --- /dev/null +++ b/nix/packages/update-readme.nix @@ -0,0 +1,17 @@ +{ + runCommand, + makeWrapper, + nushell, +}: +runCommand "update-readme" + { + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ nushell ]; + } + '' + mkdir -p $out/bin + cp ${../tools/update_readme.nu} $out/bin/update-readme + chmod +x $out/bin/update-readme + wrapProgram $out/bin/update-readme \ + --prefix PATH : ${nushell}/bin + '' diff --git a/nix/wal-g.nix b/nix/packages/wal-g.nix similarity index 90% rename from nix/wal-g.nix rename to nix/packages/wal-g.nix index 0e29fd272..37f4addb7 100644 --- a/nix/wal-g.nix +++ b/nix/packages/wal-g.nix @@ -1,14 +1,20 @@ -{ lib -, buildGoModule -, fetchFromGitHub -, brotli -, libsodium -, installShellFiles -, +{ + lib, + buildGoModule, + fetchFromGitHub, + brotli, + libsodium, + installShellFiles, }: let - walGCommon = { version, vendorHash, sha256, majorVersion }: + walGCommon = + { + version, + vendorHash, + sha256, + majorVersion, + }: buildGoModule rec { pname = "wal-g-${majorVersion}"; inherit version; @@ -45,14 +51,14 @@ let postInstall = '' mv $out/bin/pg $out/bin/wal-g-${majorVersion} - + # Create version-specific completions mkdir -p $out/share/bash-completion/completions $out/bin/wal-g-${majorVersion} completion bash > $out/share/bash-completion/completions/wal-g-${majorVersion} - + mkdir -p $out/share/zsh/site-functions $out/bin/wal-g-${majorVersion} completion zsh > $out/share/zsh/site-functions/_wal-g-${majorVersion} - + ''; meta = with lib; { diff --git a/nix/postgresql/15.nix b/nix/postgresql/15.nix deleted file mode 100644 index 63f492840..000000000 --- a/nix/postgresql/15.nix +++ /dev/null @@ -1,4 +0,0 @@ -import ./generic.nix { - version = "15.8"; - hash = "sha256-RANRX5pp7rPv68mPMLjGlhIr/fiV6Ss7I/W452nty2o="; -} diff --git a/nix/postgresql/17.nix b/nix/postgresql/17.nix deleted file mode 100644 index 32102fdb6..000000000 --- a/nix/postgresql/17.nix +++ /dev/null @@ -1,4 +0,0 @@ -import ./generic.nix { - version = "17.4"; - hash = "sha256-xGBbc/6hGWNAZpn5Sblm5dFzp+4Myu+JON7AyoqZX+c="; -} diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index f7b3c5583..272dc4f35 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -1,19 +1,85 @@ -self: +{ + self', + pkgs, + supportedPostgresVersions, +}: let - versions = { - postgresql_15 = ./15.nix; - postgresql_17 = ./17.nix; - postgresql_orioledb-17 = ./orioledb-17.nix; - }; - mkAttributes = jitSupport: - self.lib.mapAttrs' (version: path: + # Creates Postgres packages for a specific flavor (standard or orioledb) + mkPostgresqlPackages = + { + namePrefix, + jitSupport, + supportedVersions, + }: + pkgs.lib.mapAttrs' ( + version: config: let - attrName = if jitSupport then "${version}_jit" else version; + versionSuffix = if jitSupport then "${version}_jit" else version; in - self.lib.nameValuePair attrName (import path { - inherit jitSupport self; + pkgs.lib.nameValuePair "${namePrefix}${versionSuffix}" ( + pkgs.callPackage ./generic.nix { + inherit (config) version hash; + jitSupport = jitSupport; + self = pkgs; + } + ) + ) supportedVersions; + + # Define Postgres flavors with their configuration + postgresFlavors = [ + { + namePrefix = "postgresql_"; + versions = supportedPostgresVersions.postgres; + } + { + namePrefix = "postgresql_orioledb-"; + versions = supportedPostgresVersions.orioledb; + } + ]; + + # Generate packages for all flavors with both JIT enabled and disabled + mkAllPackages = + flavors: jitSupport: + pkgs.lib.foldl' ( + acc: flavor: + acc + // (mkPostgresqlPackages { + inherit (flavor) namePrefix; + inherit jitSupport; + supportedVersions = flavor.versions; }) - ) versions; + ) { } flavors; + + # Generate source packages dynamically from supported versions + mkSourcePackages = + flavors: + pkgs.lib.foldl' ( + acc: flavor: + acc + // (pkgs.lib.mapAttrs' ( + version: _: + pkgs.lib.nameValuePair "${flavor.namePrefix}${version}_src" ( + pkgs.callPackage ./src.nix { postgresql = self'.packages."${flavor.namePrefix}${version}"; } + ) + ) flavor.versions) + ) { } flavors; + + # Generate debug packages dynamically from supported versions (Linux only) + mkDebugPackages = + flavors: + pkgs.lib.foldl' ( + acc: flavor: + acc + // (pkgs.lib.mapAttrs' ( + version: _: + pkgs.lib.nameValuePair "${flavor.namePrefix}${version}_debug" ( + self'.packages."${flavor.namePrefix}${version}".debug + ) + ) flavor.versions) + ) { } flavors; in -# variations without and with JIT -(mkAttributes false) // (mkAttributes true) +# Combine all PostgreSQL packages: runtime packages + source packages + debug packages +(mkAllPackages postgresFlavors false) +// (mkAllPackages postgresFlavors true) +// (mkSourcePackages postgresFlavors) +// pkgs.lib.optionalAttrs (pkgs.stdenv.isLinux) (mkDebugPackages postgresFlavors) diff --git a/nix/postgresql/generic.nix b/nix/postgresql/generic.nix index cefd5a61b..d922895fa 100644 --- a/nix/postgresql/generic.nix +++ b/nix/postgresql/generic.nix @@ -1,320 +1,411 @@ let generic = - # adapted from the nixpkgs postgresql package - # dependencies - { stdenv, lib, fetchurl, fetchpatch, makeWrapper - , glibc, zlib, readline, openssl, icu, lz4, zstd, systemd, libossp_uuid - , pkg-config, libxml2, tzdata, libkrb5, substituteAll, darwin - , linux-pam + # adapted from the nixpkgs postgresql package + # dependencies + { + stdenv, + lib, + fetchurl, + makeWrapper, + glibc, + zlib, + readline, + openssl, + icu, + lz4, + zstd, + systemd, + libossp_uuid, + pkg-config, + libxml2, + tzdata, + libkrb5, + substituteAll, + darwin, + linux-pam, #orioledb specific - , perl, bison, flex, docbook_xsl, docbook_xml_dtd_45, docbook_xsl_ns, libxslt + perl, + bison, + flex, + docbook_xsl, + docbook_xml_dtd_45, + docbook_xsl_ns, + libxslt, # This is important to obtain a version of `libpq` that does not depend on systemd. - , systemdSupport ? lib.meta.availableOn stdenv.hostPlatform systemd && !stdenv.hostPlatform.isStatic - , enableSystemd ? null - , gssSupport ? with stdenv.hostPlatform; !isWindows && !isStatic + systemdSupport ? lib.meta.availableOn stdenv.hostPlatform systemd && !stdenv.hostPlatform.isStatic, + enableSystemd ? null, + gssSupport ? with stdenv.hostPlatform; !isWindows && !isStatic, # for postgresql.pkgs - , self, newScope, buildEnv + self, + newScope, + buildEnv, # source specification - , version, hash, muslPatches ? {} + version, + hash, + muslPatches ? { }, # for tests - , testers + testers, # JIT - , jitSupport - , nukeReferences, patchelf, llvmPackages + jitSupport, + nukeReferences, + patchelf, + llvmPackages, # PL/Python - , pythonSupport ? false - , python3 + pythonSupport ? false, + python3, # detection of crypt fails when using llvm stdenv, so we add it manually # for <13 (where it got removed: https://github.com/postgres/postgres/commit/c45643d618e35ec2fe91438df15abd4f3c0d85ca) - , libxcrypt - } @args: - let - atLeast = lib.versionAtLeast version; - olderThan = lib.versionOlder version; - lz4Enabled = atLeast "14"; - zstdEnabled = atLeast "15"; - - systemdSupport' = if enableSystemd == null then systemdSupport else (lib.warn "postgresql: argument enableSystemd is deprecated, please use systemdSupport instead." enableSystemd); - - pname = "postgresql"; - - stdenv' = if jitSupport then llvmPackages.stdenv else stdenv; - in stdenv'.mkDerivation (finalAttrs: { - inherit version; - pname = pname + lib.optionalString jitSupport "-jit"; - - src = if (builtins.match "[0-9][0-9]_.*" version != null) then - fetchurl { - url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz"; - inherit hash; - } - else - fetchurl { - url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; - inherit hash; - }; - - hardeningEnable = lib.optionals (!stdenv'.cc.isClang) [ "pie" ]; - - outputs = [ "out" "lib" ]; - setOutputFlags = false; # $out retains configureFlags :-/ - - buildInputs = [ - zlib - readline - openssl - (libxml2.override {python = python3;}) - icu - ] - ++ lib.optionals (olderThan "13") [ libxcrypt ] - ++ lib.optionals jitSupport [ llvmPackages.llvm ] - ++ lib.optionals lz4Enabled [ lz4 ] - ++ lib.optionals zstdEnabled [ zstd ] - ++ lib.optionals systemdSupport' [ systemd ] - ++ lib.optionals pythonSupport [ python3 ] - ++ lib.optionals gssSupport [ libkrb5 ] - ++ lib.optionals stdenv'.isLinux [ linux-pam ] - ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ] - ++ lib.optionals ((builtins.match "[0-9][0-9]_.*" version != null) || (lib.versionAtLeast version "17")) [ - perl bison flex docbook_xsl docbook_xml_dtd_45 docbook_xsl_ns libxslt + libxcrypt, + }@args: + let + atLeast = lib.versionAtLeast version; + olderThan = lib.versionOlder version; + lz4Enabled = atLeast "14"; + zstdEnabled = atLeast "15"; + + systemdSupport' = + if enableSystemd == null then + systemdSupport + else + (lib.warn "postgresql: argument enableSystemd is deprecated, please use systemdSupport instead." enableSystemd); + + pname = "postgresql"; + + stdenv' = if jitSupport then llvmPackages.stdenv else stdenv; + in + stdenv'.mkDerivation (finalAttrs: { + inherit version; + pname = pname + lib.optionalString jitSupport "-jit"; + + src = + if (builtins.match "[0-9][0-9]_.*" version != null) then + fetchurl { + url = "https://github.com/orioledb/postgres/archive/refs/tags/patches${version}.tar.gz"; + inherit hash; + } + else + fetchurl { + url = "mirror://postgresql/source/v${version}/${pname}-${version}.tar.bz2"; + inherit hash; + }; + + hardeningEnable = lib.optionals (!stdenv'.cc.isClang) [ "pie" ]; + + outputs = [ + "out" + "lib" ]; - - nativeBuildInputs = [ - makeWrapper - pkg-config - ] - ++ lib.optionals jitSupport [ llvmPackages.llvm.dev nukeReferences patchelf ]; - - enableParallelBuilding = true; - - separateDebugInfo = true; - - buildFlags = [ "world-bin" ]; - - # Makes cross-compiling work when xml2-config can't be executed on the host. - # Fixed upstream in https://github.com/postgres/postgres/commit/0bc8cebdb889368abdf224aeac8bc197fe4c9ae6 - env.NIX_CFLAGS_COMPILE = lib.optionalString (olderThan "13") "-I${libxml2.dev}/include/libxml2"; - - configureFlags = [ - "--with-openssl" - "--with-libxml" - "--with-icu" - "--sysconfdir=/etc" - "--libdir=$(lib)/lib" - "--with-system-tzdata=${tzdata}/share/zoneinfo" - "--enable-debug" - (lib.optionalString systemdSupport' "--with-systemd") - (if stdenv'.isDarwin then "--with-uuid=e2fs" else "--with-ossp-uuid") - ] ++ lib.optionals lz4Enabled [ "--with-lz4" ] - ++ lib.optionals zstdEnabled [ "--with-zstd" ] - ++ lib.optionals gssSupport [ "--with-gssapi" ] - ++ lib.optionals pythonSupport [ "--with-python" ] - ++ lib.optionals jitSupport [ "--with-llvm" ] - ++ lib.optionals stdenv'.isLinux [ "--with-pam" ]; - - patches = [ - (if atLeast "16" then ./patches/relative-to-symlinks-16+.patch else ./patches/relative-to-symlinks.patch) - ./patches/less-is-more.patch - ./patches/paths-for-split-outputs.patch - ./patches/specify_pkglibdir_at_runtime.patch - ./patches/paths-with-postgresql-suffix.patch - - (substituteAll { - src = ./patches/locale-binary-path.patch; - locale = "${if stdenv.isDarwin then darwin.adv_cmds else lib.getBin stdenv.cc.libc}/bin/locale"; - }) - ] ++ lib.optionals stdenv'.hostPlatform.isMusl ( - # Using fetchurl instead of fetchpatch on purpose: https://github.com/NixOS/nixpkgs/issues/240141 - map fetchurl (lib.attrValues muslPatches) - ) ++ lib.optionals stdenv'.isLinux [ - (if atLeast "13" then ./patches/socketdir-in-run-13+.patch else ./patches/socketdir-in-run.patch) - ]; - - installTargets = [ "install-world-bin" ]; - - postPatch = '' - # Hardcode the path to pgxs so pg_config returns the path in $out - substituteInPlace "src/common/config_info.c" --subst-var out - '' + lib.optionalString jitSupport '' - # Force lookup of jit stuff in $out instead of $lib - substituteInPlace src/backend/jit/jit.c --replace pkglib_path \"$out/lib\" - substituteInPlace src/backend/jit/llvm/llvmjit.c --replace pkglib_path \"$out/lib\" - substituteInPlace src/backend/jit/llvm/llvmjit_inline.cpp --replace pkglib_path \"$out/lib\" - ''; - - postInstall = - '' - moveToOutput "lib/pgxs" "$out" # looks strange, but not deleting it - moveToOutput "lib/libpgcommon*.a" "$out" - moveToOutput "lib/libpgport*.a" "$out" - moveToOutput "lib/libecpg*" "$out" - - # Prevent a retained dependency on gcc-wrapper. - substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv'.cc}/bin/ld ld - - if [ -z "''${dontDisableStatic:-}" ]; then - # Remove static libraries in case dynamic are available. - for i in $out/lib/*.a $lib/lib/*.a; do - name="$(basename "$i")" - ext="${stdenv'.hostPlatform.extensions.sharedLibrary}" - if [ -e "$lib/lib/''${name%.a}$ext" ] || [ -e "''${i%.a}$ext" ]; then - rm "$i" - fi - done - fi - '' + lib.optionalString jitSupport '' - # Move the bitcode and libllvmjit.so library out of $lib; otherwise, every client that - # depends on libpq.so will also have libLLVM.so in its closure too, bloating it - moveToOutput "lib/bitcode" "$out" - moveToOutput "lib/llvmjit*" "$out" - - # In the case of JIT support, prevent a retained dependency on clang-wrapper - substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv'.cc}/bin/clang clang - nuke-refs $out/lib/llvmjit_types.bc $(find $out/lib/bitcode -type f) - - # Stop out depending on the default output of llvm - substituteInPlace $out/lib/pgxs/src/Makefile.global \ - --replace ${llvmPackages.llvm.out}/bin "" \ - --replace '$(LLVM_BINPATH)/' "" - - # Stop out depending on the -dev output of llvm - substituteInPlace $out/lib/pgxs/src/Makefile.global \ - --replace ${llvmPackages.llvm.dev}/bin/llvm-config llvm-config \ - --replace -I${llvmPackages.llvm.dev}/include "" - - ${lib.optionalString (!stdenv'.isDarwin) '' - # Stop lib depending on the -dev output of llvm - rpath=$(patchelf --print-rpath $out/lib/llvmjit.so) - nuke-refs -e $out $out/lib/llvmjit.so - # Restore the correct rpath - patchelf $out/lib/llvmjit.so --set-rpath "$rpath" - ''} - ''; - - postFixup = lib.optionalString (!stdenv'.isDarwin && stdenv'.hostPlatform.libc == "glibc") - '' + setOutputFlags = false; # $out retains configureFlags :-/ + + buildInputs = + [ + zlib + readline + openssl + (libxml2.override { python = python3; }) + icu + ] + ++ lib.optionals (olderThan "13") [ libxcrypt ] + ++ lib.optionals jitSupport [ llvmPackages.llvm ] + ++ lib.optionals lz4Enabled [ lz4 ] + ++ lib.optionals zstdEnabled [ zstd ] + ++ lib.optionals systemdSupport' [ systemd ] + ++ lib.optionals pythonSupport [ python3 ] + ++ lib.optionals gssSupport [ libkrb5 ] + ++ lib.optionals stdenv'.isLinux [ linux-pam ] + ++ lib.optionals (!stdenv'.isDarwin) [ libossp_uuid ] + ++ + lib.optionals + ((builtins.match "[0-9][0-9]_.*" version != null) || (lib.versionAtLeast version "17")) + [ + perl + bison + flex + docbook_xsl + docbook_xml_dtd_45 + docbook_xsl_ns + libxslt + ]; + + nativeBuildInputs = + [ + makeWrapper + pkg-config + ] + ++ lib.optionals jitSupport [ + llvmPackages.llvm.dev + nukeReferences + patchelf + ]; + + enableParallelBuilding = true; + + separateDebugInfo = true; + + buildFlags = [ "world-bin" ]; + + # Makes cross-compiling work when xml2-config can't be executed on the host. + # Fixed upstream in https://github.com/postgres/postgres/commit/0bc8cebdb889368abdf224aeac8bc197fe4c9ae6 + env.NIX_CFLAGS_COMPILE = lib.optionalString (olderThan "13") "-I${libxml2.dev}/include/libxml2"; + + configureFlags = + [ + "--with-openssl" + "--with-libxml" + "--with-icu" + "--sysconfdir=/etc" + "--libdir=$(lib)/lib" + "--with-system-tzdata=${tzdata}/share/zoneinfo" + "--enable-debug" + (lib.optionalString systemdSupport' "--with-systemd") + (if stdenv'.isDarwin then "--with-uuid=e2fs" else "--with-ossp-uuid") + ] + ++ lib.optionals lz4Enabled [ "--with-lz4" ] + ++ lib.optionals zstdEnabled [ "--with-zstd" ] + ++ lib.optionals gssSupport [ "--with-gssapi" ] + ++ lib.optionals pythonSupport [ "--with-python" ] + ++ lib.optionals jitSupport [ "--with-llvm" ] + ++ lib.optionals stdenv'.isLinux [ "--with-pam" ]; + + patches = + [ + ( + if atLeast "16" then + ./patches/relative-to-symlinks-16+.patch + else + ./patches/relative-to-symlinks.patch + ) + ./patches/less-is-more.patch + ./patches/paths-for-split-outputs.patch + ./patches/specify_pkglibdir_at_runtime.patch + ./patches/paths-with-postgresql-suffix.patch + + (substituteAll { + src = ./patches/locale-binary-path.patch; + locale = "${if stdenv.isDarwin then darwin.adv_cmds else lib.getBin stdenv.cc.libc}/bin/locale"; + }) + ] + ++ lib.optionals stdenv'.hostPlatform.isMusl ( + # Using fetchurl instead of fetchpatch on purpose: https://github.com/NixOS/nixpkgs/issues/240141 + map fetchurl (lib.attrValues muslPatches) + ) + ++ lib.optionals stdenv'.isLinux [ + (if atLeast "13" then ./patches/socketdir-in-run-13+.patch else ./patches/socketdir-in-run.patch) + ]; + + installTargets = [ "install-world-bin" ]; + + postPatch = + '' + # Hardcode the path to pgxs so pg_config returns the path in $out + substituteInPlace "src/common/config_info.c" --subst-var out + '' + + lib.optionalString jitSupport '' + # Force lookup of jit stuff in $out instead of $lib + substituteInPlace src/backend/jit/jit.c --replace pkglib_path \"$out/lib\" + substituteInPlace src/backend/jit/llvm/llvmjit.c --replace pkglib_path \"$out/lib\" + substituteInPlace src/backend/jit/llvm/llvmjit_inline.cpp --replace pkglib_path \"$out/lib\" + ''; + + postInstall = + '' + moveToOutput "lib/pgxs" "$out" # looks strange, but not deleting it + moveToOutput "lib/libpgcommon*.a" "$out" + moveToOutput "lib/libpgport*.a" "$out" + moveToOutput "lib/libecpg*" "$out" + + # Prevent a retained dependency on gcc-wrapper. + substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv'.cc}/bin/ld ld + + if [ -z "''${dontDisableStatic:-}" ]; then + # Remove static libraries in case dynamic are available. + for i in $out/lib/*.a $lib/lib/*.a; do + name="$(basename "$i")" + ext="${stdenv'.hostPlatform.extensions.sharedLibrary}" + if [ -e "$lib/lib/''${name%.a}$ext" ] || [ -e "''${i%.a}$ext" ]; then + rm "$i" + fi + done + fi + '' + + lib.optionalString jitSupport '' + # Move the bitcode and libllvmjit.so library out of $lib; otherwise, every client that + # depends on libpq.so will also have libLLVM.so in its closure too, bloating it + moveToOutput "lib/bitcode" "$out" + moveToOutput "lib/llvmjit*" "$out" + + # In the case of JIT support, prevent a retained dependency on clang-wrapper + substituteInPlace "$out/lib/pgxs/src/Makefile.global" --replace ${stdenv'.cc}/bin/clang clang + nuke-refs $out/lib/llvmjit_types.bc $(find $out/lib/bitcode -type f) + + # Stop out depending on the default output of llvm + substituteInPlace $out/lib/pgxs/src/Makefile.global \ + --replace ${llvmPackages.llvm.out}/bin "" \ + --replace '$(LLVM_BINPATH)/' "" + + # Stop out depending on the -dev output of llvm + substituteInPlace $out/lib/pgxs/src/Makefile.global \ + --replace ${llvmPackages.llvm.dev}/bin/llvm-config llvm-config \ + --replace -I${llvmPackages.llvm.dev}/include "" + + ${lib.optionalString (!stdenv'.isDarwin) '' + # Stop lib depending on the -dev output of llvm + rpath=$(patchelf --print-rpath $out/lib/llvmjit.so) + nuke-refs -e $out $out/lib/llvmjit.so + # Restore the correct rpath + patchelf $out/lib/llvmjit.so --set-rpath "$rpath" + ''} + ''; + + postFixup = lib.optionalString (!stdenv'.isDarwin && stdenv'.hostPlatform.libc == "glibc") '' # initdb needs access to "locale" command from glibc. wrapProgram $out/bin/initdb --prefix PATH ":" ${glibc.bin}/bin ''; - doCheck = !stdenv'.isDarwin; - # autodetection doesn't seem to able to find this, but it's there. - checkTarget = "check"; - - disallowedReferences = [ stdenv'.cc ]; - - passthru = let - this = self.callPackage generic args; - jitToggle = this.override { - jitSupport = !jitSupport; - }; - in - { - psqlSchema = lib.versions.major version; - - withJIT = if jitSupport then this else jitToggle; - withoutJIT = if jitSupport then jitToggle else this; - - dlSuffix = if olderThan "16" then ".so" else stdenv.hostPlatform.extensions.sharedLibrary; - - pkgs = let - scope = { - inherit jitSupport; - inherit (llvmPackages) llvm; - postgresql = this; - stdenv = stdenv'; + doCheck = !stdenv'.isDarwin; + # autodetection doesn't seem to able to find this, but it's there. + checkTarget = "check"; + + disallowedReferences = [ stdenv'.cc ]; + + passthru = + let + this = self.callPackage generic args; + jitToggle = this.override { jitSupport = !jitSupport; }; + in + { + psqlSchema = lib.versions.major version; + + withJIT = if jitSupport then this else jitToggle; + withoutJIT = if jitSupport then jitToggle else this; + + dlSuffix = if olderThan "16" then ".so" else stdenv.hostPlatform.extensions.sharedLibrary; + + pkgs = + let + scope = { + inherit jitSupport; + inherit (llvmPackages) llvm; + postgresql = this; + stdenv = stdenv'; + }; + newSelf = self // scope; + newSuper = { + callPackage = newScope (scope // this.pkgs); + }; + in + import ./ext newSelf newSuper; + + withPackages = postgresqlWithPackages { + inherit makeWrapper buildEnv; + postgresql = this; + } this.pkgs; + + tests = + { + postgresql-wal-receiver = import ../../../../nixos/tests/postgresql-wal-receiver.nix { + inherit (stdenv) system; + pkgs = self; + package = this; + }; + pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; + } + // lib.optionalAttrs jitSupport { + postgresql-jit = import ../../../../nixos/tests/postgresql-jit.nix { + inherit (stdenv) system; + pkgs = self; + package = this; + }; + }; }; - newSelf = self // scope; - newSuper = { callPackage = newScope (scope // this.pkgs); }; - in import ./ext newSelf newSuper; - - withPackages = postgresqlWithPackages { - inherit makeWrapper buildEnv; - postgresql = this; - } - this.pkgs; - - tests = { - postgresql-wal-receiver = import ../../../../nixos/tests/postgresql-wal-receiver.nix { - inherit (stdenv) system; - pkgs = self; - package = this; - }; - pkg-config = testers.testMetaPkgConfig finalAttrs.finalPackage; - } // lib.optionalAttrs jitSupport { - postgresql-jit = import ../../../../nixos/tests/postgresql-jit.nix { - inherit (stdenv) system; - pkgs = self; - package = this; - }; - }; - }; - meta = with lib; { - homepage = "https://www.postgresql.org"; - description = "Powerful, open source object-relational database system"; - license = licenses.postgresql; - changelog = "https://www.postgresql.org/docs/release/${finalAttrs.version}/"; - maintainers = with maintainers; [ thoughtpolice danbst globin ivan ma27 wolfgangwalther ]; - pkgConfigModules = [ "libecpg" "libecpg_compat" "libpgtypes" "libpq" ]; - platforms = platforms.unix; - - # JIT support doesn't work with cross-compilation. It is attempted to build LLVM-bytecode - # (`%.bc` is the corresponding `make(1)`-rule) for each sub-directory in `backend/` for - # the JIT apparently, but with a $(CLANG) that can produce binaries for the build, not the - # host-platform. - # - # I managed to get a cross-build with JIT support working with - # `depsBuildBuild = [ llvmPackages.clang ] ++ buildInputs`, but considering that the - # resulting LLVM IR isn't platform-independent this doesn't give you much. - # In fact, I tried to test the result in a VM-test, but as soon as JIT was used to optimize - # a query, postgres would coredump with `Illegal instruction`. - broken = (jitSupport && stdenv.hostPlatform != stdenv.buildPlatform) - # Allmost all tests fail FATAL errors for v12 and v13 - || (jitSupport && stdenv.hostPlatform.isMusl && olderThan "14"); - }; - }); + meta = with lib; { + homepage = "https://www.postgresql.org"; + description = "Powerful, open source object-relational database system"; + license = licenses.postgresql; + changelog = "https://www.postgresql.org/docs/release/${finalAttrs.version}/"; + maintainers = with maintainers; [ + thoughtpolice + danbst + globin + ivan + ma27 + wolfgangwalther + ]; + pkgConfigModules = [ + "libecpg" + "libecpg_compat" + "libpgtypes" + "libpq" + ]; + platforms = platforms.unix; + + # JIT support doesn't work with cross-compilation. It is attempted to build LLVM-bytecode + # (`%.bc` is the corresponding `make(1)`-rule) for each sub-directory in `backend/` for + # the JIT apparently, but with a $(CLANG) that can produce binaries for the build, not the + # host-platform. + # + # I managed to get a cross-build with JIT support working with + # `depsBuildBuild = [ llvmPackages.clang ] ++ buildInputs`, but considering that the + # resulting LLVM IR isn't platform-independent this doesn't give you much. + # In fact, I tried to test the result in a VM-test, but as soon as JIT was used to optimize + # a query, postgres would coredump with `Illegal instruction`. + broken = + (jitSupport && stdenv.hostPlatform != stdenv.buildPlatform) + # Allmost all tests fail FATAL errors for v12 and v13 + || (jitSupport && stdenv.hostPlatform.isMusl && olderThan "14"); + }; + }); - postgresqlWithPackages = { postgresql, makeWrapper, buildEnv }: pkgs: f: buildEnv { - name = "postgresql-and-plugins-${postgresql.version}"; - paths = f pkgs ++ [ + postgresqlWithPackages = + { + postgresql, + makeWrapper, + buildEnv, + }: + pkgs: f: + buildEnv { + name = "postgresql-and-plugins-${postgresql.version}"; + paths = f pkgs ++ [ postgresql postgresql.lib #TODO RM postgresql.man # in case user installs this into environment - ]; - nativeBuildInputs = [ makeWrapper ]; - - - # We include /bin to ensure the $out/bin directory is created, which is - # needed because we'll be removing the files from that directory in postBuild - # below. See #22653 - pathsToLink = ["/" "/bin"]; - - # Note: the duplication of executables is about 4MB size. - # So a nicer solution was patching postgresql to allow setting the - # libdir explicitly. - postBuild = '' - mkdir -p $out/bin - rm $out/bin/{pg_config,postgres,pg_ctl} - cp --target-directory=$out/bin ${postgresql}/bin/{postgres,pg_config,pg_ctl} - wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib - ''; + ]; + nativeBuildInputs = [ makeWrapper ]; + + # We include /bin to ensure the $out/bin directory is created, which is + # needed because we'll be removing the files from that directory in postBuild + # below. See #22653 + pathsToLink = [ + "/" + "/bin" + ]; - passthru.version = postgresql.version; - passthru.psqlSchema = postgresql.psqlSchema; - }; + # Note: the duplication of executables is about 4MB size. + # So a nicer solution was patching postgresql to allow setting the + # libdir explicitly. + postBuild = '' + mkdir -p $out/bin + rm $out/bin/{pg_config,postgres,pg_ctl} + cp --target-directory=$out/bin ${postgresql}/bin/{postgres,pg_config,pg_ctl} + wrapProgram $out/bin/postgres --set NIX_PGLIBDIR $out/lib + ''; + passthru.version = postgresql.version; + passthru.psqlSchema = postgresql.psqlSchema; + }; in +generic # passed by .nix -versionArgs: -# passed by default.nix -{ self, ... } @defaultArgs: -self.callPackage generic (defaultArgs // versionArgs) +# versionArgs: +# # passed by default.nix +# { self, ... } @defaultArgs: +# self.callPackage generic (defaultArgs // versionArgs) diff --git a/nix/postgresql/orioledb-17.nix b/nix/postgresql/orioledb-17.nix deleted file mode 100644 index b8e5bc374..000000000 --- a/nix/postgresql/orioledb-17.nix +++ /dev/null @@ -1,4 +0,0 @@ -import ./generic.nix { - version = "17_6"; - hash = "sha256-HbuTcXNanFOl9YfvlSzQJon8CfAhc8TFwo/y7jXy51w="; -} diff --git a/nix/postgresql/src.nix b/nix/postgresql/src.nix new file mode 100644 index 000000000..65ba8b1cb --- /dev/null +++ b/nix/postgresql/src.nix @@ -0,0 +1,31 @@ +{ + stdenv, + postgresql, + lib, + bzip2, +}: +stdenv.mkDerivation { + pname = "postgresql-${postgresql.version}-src"; + version = postgresql.version; + + src = postgresql.src; + + nativeBuildInputs = [ bzip2 ]; + + phases = [ + "unpackPhase" + "installPhase" + ]; + + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + + meta = with lib; { + description = "PostgreSQL 15 source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + inherit (platforms) all; + }; +}