From b54a938724e5551f6436f551cafec3d1324a6260 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 30 May 2022 11:11:35 +0200 Subject: Fix garage_version() now that GIT_VERSION is read in crate garage_rpc --- default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'default.nix') diff --git a/default.nix b/default.nix index 21a413b2..de996ac1 100644 --- a/default.nix +++ b/default.nix @@ -69,7 +69,7 @@ in let we ask the user (the CI often) to pass the value to Nix. */ (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage"; + name = "garage_rpc"; overrideAttrs = drv: /* [1] */ { hardeningDisable = [ "pie" ]; } // -- cgit v1.2.3 From a49d0ea19f000036ad39b7d6a1134bbd0d07ab1d Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Sat, 23 Jul 2022 12:17:41 +0200 Subject: Fix: compile aarch64+armv6 as static binaries --- default.nix | 56 +++++++++++++++++++++++++++++--------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) (limited to 'default.nix') diff --git a/default.nix b/default.nix index de996ac1..296c7592 100644 --- a/default.nix +++ b/default.nix @@ -56,45 +56,47 @@ in let */ overrides = pkgs.rustBuilder.overrides.all ++ [ /* - [1] We need to alter Nix hardening to be able to statically compile: PIE, + [1] We need to alter Nix hardening to make static binaries: PIE, Position Independent Executables seems to be supported only on amd64. Having - this flags set either make our executables crash or compile as dynamic on many platforms. - In the following section codegenOpts, we reactive it for the supported targets - (only amd64 curently) through the `-static-pie` flag. PIE is a feature used - by ASLR, which helps mitigate security issues. - Learn more about Nix Hardening: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh - - [2] We want to inject the git version while keeping the build deterministic. - As we do not want to consider the .git folder as part of the input source, - we ask the user (the CI often) to pass the value to Nix. + this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. + Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets + (only amd64 curently) through the `-static-pie` flag. + PIE is a feature used by ASLR, which helps mitigate security issues. + Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh */ + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage"; + overrideAttrs = drv: { hardeningDisable = [ "pie" ]; }; + }) + (pkgs.rustBuilder.rustLib.makeOverride { name = "garage_rpc"; + + /* + [2] We want to inject the git version while keeping the build deterministic. + As we do not want to consider the .git folder as part of the input source, + we ask the user (the CI often) to pass the value to Nix. + */ overrideAttrs = drv: - /* [1] */ { hardeningDisable = [ "pie" ]; } - // - /* [2] */ (if git_version != null then { + (if git_version != null then { preConfigure = '' ${drv.preConfigure or ""} export GIT_VERSION="${git_version}" ''; } else {}); - }) - /* - We ship some parts of the code disabled by default by putting them behind a flag. - It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). - But we want to ship these additional features when we release Garage. - In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds. - Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate. - */ - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_rpc"; - overrideArgs = old: - { - features = if release then [ "kubernetes-discovery" ] else []; - }; + /* + [3] We ship some parts of the code disabled by default by putting them behind a flag. + It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). + But we want to ship these additional features when we release Garage. + In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds. + Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate. + */ + overrideArgs = old: { + features = if release then [ "kubernetes-discovery" ] else []; + }; }) + ]; packageFun = import ./Cargo.nix; -- cgit v1.2.3 From 5fb858424793de8dc35dce8deaa8981a384e064f Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Mon, 25 Jul 2022 14:58:47 +0200 Subject: Refactor default.nix to follow Nix Flakes patterns --- default.nix | 161 +++++++++--------------------------------------------------- 1 file changed, 22 insertions(+), 139 deletions(-) (limited to 'default.nix') diff --git a/default.nix b/default.nix index 296c7592..5634f0af 100644 --- a/default.nix +++ b/default.nix @@ -1,149 +1,32 @@ { system ? builtins.currentSystem, - release ? false, - target ? "x86_64-unknown-linux-musl", - compileMode ? null, git_version ? null, }: with import ./nix/common.nix; -let - crossSystem = { config = target; }; -in let - log = v: builtins.trace v v; - - pkgs = import pkgsSrc { - inherit system crossSystem; - overlays = [ cargo2nixOverlay ]; - }; - - - /* - Rust and Nix triples are not the same. Cargo2nix has a dedicated library - to convert Nix triples to Rust ones. We need this conversion as we want to - set later options linked to our (rust) target in a generic way. Not only - the triple terminology is different, but also the "roles" are named differently. - Nix uses a build/host/target terminology where Nix's "host" maps to Cargo's "target". - */ - rustTarget = log (pkgs.rustBuilder.rustLib.rustTriple pkgs.stdenv.hostPlatform); - - /* - Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases. - We want our own Rust to avoid incompatibilities, like we had with musl 1.2.0. - rustc was built with musl < 1.2.0 and nix shipped musl >= 1.2.0 which lead to compilation breakage. - So we want a Rust release that is bound to our Nix repository to avoid these problems. - See here for more info: https://musl.libc.org/time64.html - Because Cargo2nix does not support the Rust environment shipped by NixOS, - we emulate the structure of the Rust object created by rustOverlay. - In practise, rustOverlay ships rustc+cargo in a single derivation while - NixOS ships them in separate ones. We reunite them with symlinkJoin. - */ - rustChannel = pkgs.symlinkJoin { - name ="rust-channel"; - paths = [ - pkgs.rustPlatform.rust.rustc - pkgs.rustPlatform.rust.cargo - ]; +let + compile = import ./nix/compile.nix; + build_debug_and_release = (target: { + debug = (compile { inherit target; release = false; }).workspace.garage { compileMode = "build"; }; + release = (compile { inherit target; release = true; }).workspace.garage { compileMode = "build"; }; + }); + +in { + pkgs = { + amd64 = build_debug_and_release "x86_64-unknown-linux-musl"; + i386 = build_debug_and_release "i686-unknown-linux-musl"; + arm64 = build_debug_and_release "aarch64-unknown-linux-musl"; + arm = build_debug_and_release "armv6l-unknown-linux-musleabihf"; }; - - /* - Cargo2nix provides many overrides by default, you can take inspiration from them: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix - - You can have a complete list of the available options by looking at the overriden object, mkcrate: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix - */ - overrides = pkgs.rustBuilder.overrides.all ++ [ - /* - [1] We need to alter Nix hardening to make static binaries: PIE, - Position Independent Executables seems to be supported only on amd64. Having - this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. - Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets - (only amd64 curently) through the `-static-pie` flag. - PIE is a feature used by ASLR, which helps mitigate security issues. - Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh - */ - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage"; - overrideAttrs = drv: { hardeningDisable = [ "pie" ]; }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_rpc"; - - /* - [2] We want to inject the git version while keeping the build deterministic. - As we do not want to consider the .git folder as part of the input source, - we ask the user (the CI often) to pass the value to Nix. - */ - overrideAttrs = drv: - (if git_version != null then { - preConfigure = '' - ${drv.preConfigure or ""} - export GIT_VERSION="${git_version}" - ''; - } else {}); - - /* - [3] We ship some parts of the code disabled by default by putting them behind a flag. - It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). - But we want to ship these additional features when we release Garage. - In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds. - Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate. - */ - overrideArgs = old: { - features = if release then [ "kubernetes-discovery" ] else []; + test = { + amd64 = let + pkgs = import pkgsSrc { }; + rustPkgs = compile { target = "x86_64-unknown-linux-musl"; }; + in + pkgs.symlinkJoin { + name ="garage-tests"; + paths = builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) (builtins.attrNames rustPkgs.workspace); }; - }) - - ]; - - packageFun = import ./Cargo.nix; - - /* - We compile fully static binaries with musl to simplify deployment on most systems. - When possible, we reactivate PIE hardening (see above). - - Also, if you set the RUSTFLAGS environment variable, the following parameters will - be ignored. - - For more information on static builds, please refer to Rust's RFC 1721. - https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage - */ - - codegenOpts = { - "armv6l-unknown-linux-musleabihf" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* compile as dynamic with static-pie */ - "aarch64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */ - "i686-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */ - "x86_64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static-pie" ]; }; - - /* - The following definition is not elegant as we use a low level function of Cargo2nix - that enables us to pass our custom rustChannel object. We need this low level definition - to pass Nix's Rust toolchains instead of Mozilla's one. - - target is mandatory but must be kept to null to allow cargo2nix to set it to the appropriate value - for each crate. - */ - rustPkgs = pkgs.rustBuilder.makePackageSet { - inherit packageFun rustChannel release codegenOpts; - packageOverrides = overrides; - target = null; - - buildRustPackages = pkgs.buildPackages.rustBuilder.makePackageSet { - inherit rustChannel packageFun codegenOpts; - packageOverrides = overrides; - target = null; - }; - }; - - -in - if compileMode == "test" - then pkgs.symlinkJoin { - name ="garage-tests"; - paths = builtins.map (key: rustPkgs.workspace.${key} { inherit compileMode; }) (builtins.attrNames rustPkgs.workspace); - } - else rustPkgs.workspace.garage { inherit compileMode; } +} -- cgit v1.2.3 From fcb04843f7619d767c3c865e4795fb3468653e28 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Mon, 25 Jul 2022 18:10:34 +0200 Subject: Run clippy in nix, leveraging nix caching ability --- default.nix | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'default.nix') diff --git a/default.nix b/default.nix index 5634f0af..4d7558c5 100644 --- a/default.nix +++ b/default.nix @@ -6,11 +6,16 @@ with import ./nix/common.nix; let + pkgs = import pkgsSrc { }; compile = import ./nix/compile.nix; build_debug_and_release = (target: { debug = (compile { inherit target; release = false; }).workspace.garage { compileMode = "build"; }; release = (compile { inherit target; release = true; }).workspace.garage { compileMode = "build"; }; }); + test = (rustPkgs: pkgs.symlinkJoin { + name ="garage-tests"; + paths = builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) (builtins.attrNames rustPkgs.workspace); + }); in { pkgs = { @@ -20,13 +25,9 @@ in { arm = build_debug_and_release "armv6l-unknown-linux-musleabihf"; }; test = { - amd64 = let - pkgs = import pkgsSrc { }; - rustPkgs = compile { target = "x86_64-unknown-linux-musl"; }; - in - pkgs.symlinkJoin { - name ="garage-tests"; - paths = builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) (builtins.attrNames rustPkgs.workspace); - }; + amd64 = test (compile { target = "x86_64-unknown-linux-musl"; }); + }; + clippy = { + amd64 = (compile { compiler = "clippy"; }).workspace.garage { compileMode = "build"; } ; }; } -- cgit v1.2.3 From 2c7bae935ac68acab831fe86e5330d3c9a84a953 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Wed, 10 Aug 2022 12:18:44 +0200 Subject: Configure structopt to report the right version By default, structopt reports the value provided by the env var CARGO_PKG_VERSION, feeded by Cargo when reading Cargo.toml. However for Garage we use a versioning based on git, so we often report a version that is behind the real version. In this commit, we create garage_util::version::garage() that reports the right version and configure all structopt subcommands to call this function instead of using the env var. --- default.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'default.nix') diff --git a/default.nix b/default.nix index 4d7558c5..7e44096c 100644 --- a/default.nix +++ b/default.nix @@ -9,8 +9,8 @@ let pkgs = import pkgsSrc { }; compile = import ./nix/compile.nix; build_debug_and_release = (target: { - debug = (compile { inherit target; release = false; }).workspace.garage { compileMode = "build"; }; - release = (compile { inherit target; release = true; }).workspace.garage { compileMode = "build"; }; + debug = (compile { inherit target git_version; release = false; }).workspace.garage { compileMode = "build"; }; + release = (compile { inherit target git_version; release = true; }).workspace.garage { compileMode = "build"; }; }); test = (rustPkgs: pkgs.symlinkJoin { name ="garage-tests"; @@ -25,9 +25,9 @@ in { arm = build_debug_and_release "armv6l-unknown-linux-musleabihf"; }; test = { - amd64 = test (compile { target = "x86_64-unknown-linux-musl"; }); + amd64 = test (compile { inherit git_version; target = "x86_64-unknown-linux-musl"; }); }; clippy = { - amd64 = (compile { compiler = "clippy"; }).workspace.garage { compileMode = "build"; } ; + amd64 = (compile { inherit git_version; compiler = "clippy"; }).workspace.garage { compileMode = "build"; } ; }; } -- cgit v1.2.3