From 99ac88c06da59ffd3cec74b6fb16096ccd84cf98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 01:37:55 +0200 Subject: [PATCH 1/4] Disable debug info for Nix builds For example, this slims down the airflow-operator image from 615MiB to 147MiB. This improvement is a combination of the size of the debuginfo itself (~200MiB per binary), and not having to include build dependencies just because our debuginfo refers to them (primarily, this ended up pulling in GCC, which sits at ~260MiB just on its own). --- template/default.nix | 107 +++++++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 34 deletions(-) diff --git a/template/default.nix b/template/default.nix index 02a30009..7eaee191 100644 --- a/template/default.nix +++ b/template/default.nix @@ -36,45 +36,84 @@ # We're only using this for dev builds at the moment, # so don't pay for release optimization. release = false; - defaultCrateOverrides = pkgs.defaultCrateOverrides // { - prost-build = attrs: { - buildInputs = [ pkgs.protobuf ]; - }; - tonic-reflection = attrs: { - buildInputs = [ pkgs.rustfmt ]; - }; - csi-grpc = attrs: { - nativeBuildInputs = [ pkgs.protobuf ]; - }; - stackable-secret-operator = attrs: { - buildInputs = [ pkgs.protobuf pkgs.rustfmt ]; - }; - stackable-opa-user-info-fetcher = attrs: { - # TODO: why is this not pulled in via libgssapi-sys? - buildInputs = [ pkgs.krb5 ]; - }; - krb5-sys = attrs: { - nativeBuildInputs = [ pkgs.pkg-config ]; - buildInputs = [ pkgs.krb5 ]; - LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; - # Clang's resource directory is located at ${pkgs.clang.cc.lib}/lib/clang/. - # Starting with Clang 16, only the major version is used for the resource directory, - # whereas the full version was used in prior Clang versions (see - # https://github.com/llvm/llvm-project/commit/e1b88c8a09be25b86b13f98755a9bd744b4dbf14). - # The clang wrapper ${pkgs.clang} provides a symlink to the resource directory, which - # we use instead. - BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include"; - }; - libgssapi-sys = attrs: { - buildInputs = [ pkgs.krb5 ]; - LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; - BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include"; + + buildRustCrateForPkgs = pkgs: attrs: pkgs.buildRustCrate.override { + # Consider migrating to mold for faster linking, but in my (@nightkr's) + # quick testing so far it actually seems to perform slightly worse than + # the default one. + # stdenv = pkgs.stdenvAdapters.useMoldLinker pkgs.stdenv; + + defaultCrateOverrides = pkgs.defaultCrateOverrides // { + # Attributes applied here apply to a single crate + + prost-build = attrs: { + buildInputs = [ pkgs.protobuf ]; + }; + tonic-reflection = attrs: { + buildInputs = [ pkgs.rustfmt ]; + }; + csi-grpc = attrs: { + nativeBuildInputs = [ pkgs.protobuf ]; + }; + stackable-secret-operator = attrs: { + buildInputs = [ pkgs.protobuf pkgs.rustfmt ]; + }; + stackable-opa-user-info-fetcher = attrs: { + # TODO: why is this not pulled in via libgssapi-sys? + buildInputs = [ pkgs.krb5 ]; + }; + krb5-sys = attrs: { + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = [ pkgs.krb5 ]; + LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; + # Clang's resource directory is located at ${pkgs.clang.cc.lib}/lib/clang/. + # Starting with Clang 16, only the major version is used for the resource directory, + # whereas the full version was used in prior Clang versions (see + # https://github.com/llvm/llvm-project/commit/e1b88c8a09be25b86b13f98755a9bd744b4dbf14). + # The clang wrapper ${pkgs.clang} provides a symlink to the resource directory, which + # we use instead. + BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include"; + }; + libgssapi-sys = attrs: { + buildInputs = [ pkgs.krb5 ]; + LIBCLANG_PATH = "${pkgs.libclang.lib}/lib"; + BINDGEN_EXTRA_CLANG_ARGS = "-I${pkgs.glibc.dev}/include -I${pkgs.clang}/resource-root/include"; + }; }; - }; + } (attrs // { + # Attributes applied here apply to all built crates + # Note that these *take precedence over* per-crate overrides + + dontStrip = !strip; + + extraRustcOpts = [ + "-C debuginfo=${toString debuginfo}" + # Enabling optimization shrinks the binaries further, but also *vastly* + # increases the build time. + # "-C opt-level=3" + ] ++ attrs.extraRustcOpts; + + # Parallel codegen allows Rustc to use more cores. + # This should help speed up compiling "bottleneck" crates that Nix can't + # parallelize (like the operator binary itself). + codegenUnits = 32; + }); } , meta ? pkgsLocal.lib.importJSON ./nix/meta.json , dockerName ? "oci.stackable.tech/sandbox/${meta.operator.name}" , dockerTag ? null +# Controls the amount of debug information included in the built operator binaries, +# see https://doc.rust-lang.org/rustc/codegen-options/index.html#debuginfo +# For comparison, `cargo build --release` defaults to 0, and the debug profile +# (no `--release`) defaults to 2. +# see https://doc.rust-lang.org/cargo/reference/profiles.html#debug +# Set to 2 if you want to run a debugger, but note that it bloats the Docker +# images *significantly* (hundreds of megabytes). +, debuginfo ? 0 +# Strip operator binaries if we don't include debuginfo, because *something* +# still something still includes a reference to gcc (~230MiB), causing it to be +# added to the docker images. +, strip ? if debuginfo == 0 then true else false }: rec { inherit cargo sources pkgsLocal pkgsTarget meta; From 279b6f62b49d67ab105da9748c858e4aa9fc8734 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 01:58:54 +0200 Subject: [PATCH 2/4] Also remove the shell (by default) --- template/default.nix | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/template/default.nix b/template/default.nix index 7eaee191..38ac0df6 100644 --- a/template/default.nix +++ b/template/default.nix @@ -114,6 +114,9 @@ # still something still includes a reference to gcc (~230MiB), causing it to be # added to the docker images. , strip ? if debuginfo == 0 then true else false +# We normally don't include a shell in the (dev) operator images, but it can be +# enabled by enabling this flag. +, includeShell ? false }: rec { inherit cargo sources pkgsLocal pkgsTarget meta; @@ -135,14 +138,14 @@ rec { name = dockerName; tag = dockerTag; contents = [ - # Common debugging tools - pkgsTarget.bashInteractive - pkgsTarget.coreutils - pkgsTarget.util-linuxMinimal # Kerberos 5 must be installed globally to load plugins correctly pkgsTarget.krb5 # Make the whole cargo workspace available on $PATH build + ] ++ lib.optional includeShell [ + pkgsTarget.bashInteractive + pkgsTarget.coreutils + pkgsTarget.util-linuxMinimal ]; config = { Env = From 1b526693cb74018f0790dc35d8b83d50b2fdc661 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 01:59:38 +0200 Subject: [PATCH 3/4] Fix a remote build awareness warning --- template/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/default.nix b/template/default.nix index 38ac0df6..3a66c658 100644 --- a/template/default.nix +++ b/template/default.nix @@ -198,6 +198,6 @@ rec { # (see https://github.com/pre-commit/pre-commit-hooks?tab=readme-ov-file#trailing-whitespace). # So, remove the trailing newline already here to avoid that an # unnecessary change is shown in Git. - ${pkgs.gnused}/bin/sed -i '$d' Cargo.nix + ${pkgsLocal.gnused}/bin/sed -i '$d' Cargo.nix ''; } From 5b6ab127b71a12cceef8d5ca1736f3ac1bc3c110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Natalie=20Klestrup=20R=C3=B6ijezon?= Date: Wed, 11 Jun 2025 02:11:48 +0200 Subject: [PATCH 4/4] Re-enable shell for now --- template/default.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/template/default.nix b/template/default.nix index 3a66c658..071adcb2 100644 --- a/template/default.nix +++ b/template/default.nix @@ -116,7 +116,10 @@ , strip ? if debuginfo == 0 then true else false # We normally don't include a shell in the (dev) operator images, but it can be # enabled by enabling this flag. -, includeShell ? false +# TODO(@nightkr): Re-enabled for now, since some operators ship with bash init +# scripts (like secret-operator's CSI path migration job). Consider either +# removing them or integrating them into the main operator binary instead. +, includeShell ? true }: rec { inherit cargo sources pkgsLocal pkgsTarget meta;