diff --git a/default.nix b/default.nix index 1791b3c6..ecdf6436 100644 --- a/default.nix +++ b/default.nix @@ -1,7 +1,4 @@ -{ - system ? builtins.currentSystem, - git_version ? null, -}: +{ system ? builtins.currentSystem, git_version ? null, }: with import ./nix/common.nix; @@ -13,22 +10,21 @@ let debug = (compile { inherit system target git_version pkgsSrc cargo2nixOverlay; release = false; - }).workspace.garage { - compileMode = "build"; - }; + }).workspace.garage { compileMode = "build"; }; release = (compile { inherit system target git_version pkgsSrc cargo2nixOverlay; release = true; - }).workspace.garage { - compileMode = "build"; - }; + }).workspace.garage { compileMode = "build"; }; }); - test = (rustPkgs: pkgs.symlinkJoin { - name ="garage-tests"; - paths = builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) (builtins.attrNames rustPkgs.workspace); - }); + test = (rustPkgs: + pkgs.symlinkJoin { + name = "garage-tests"; + paths = + builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) + (builtins.attrNames rustPkgs.workspace); + }); in { pkgs = { @@ -55,8 +51,6 @@ in { inherit system git_version pkgsSrc cargo2nixOverlay; target = "x86_64-unknown-linux-musl"; compiler = "clippy"; - }).workspace.garage { - compileMode = "build"; - }; + }).workspace.garage { compileMode = "build"; }; }; } diff --git a/flake.nix b/flake.nix index c1d772bb..7eb9e33b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,9 @@ { - description = "Garage, an S3-compatible distributed object store for self-hosted deployments"; + description = + "Garage, an S3-compatible distributed object store for self-hosted deployments"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555"; + inputs.nixpkgs.url = + "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555"; inputs.cargo2nix = { # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36"; @@ -24,13 +26,11 @@ release = true; }).workspace.garage { compileMode = "build"; }; }; - devShell = ((compile { + devShell = (compile { inherit system git_version; pkgsSrc = nixpkgs; cargo2nixOverlay = cargo2nix.overlays.default; release = false; - }).workspaceShell { - packages = [ pkgs.rustfmt cargo2nix.packages.${system}.default ]; - }); + }).workspaceShell { packages = [ pkgs.rustfmt ]; }; }); } diff --git a/nix/build_index.nix b/nix/build_index.nix index 4625e3ae..7cc4f62c 100644 --- a/nix/build_index.nix +++ b/nix/build_index.nix @@ -1,10 +1,8 @@ -{ - path ? "/../aws-list.txt", -}: +{ path ? "/../aws-list.txt", }: with import ./common.nix; -let - pkgs = import pkgsSrc {}; +let + pkgs = import pkgsSrc { }; lib = pkgs.lib; /* Converts a key list and a value list to a set @@ -13,139 +11,182 @@ let listToSet [ "name" "version" ] [ "latex" "3.14" ] => { name = "latex"; version = "3.14"; } */ - listToSet = keys: values: - builtins.listToAttrs - (lib.zipListsWith - (a: b: { name = a; value = b; }) - keys - values); + listToSet = keys: values: + builtins.listToAttrs (lib.zipListsWith (a: b: { + name = a; + value = b; + }) keys values); /* Says if datetime a is more recent than datetime b - Example: - cmpDate { date = "2021-09-10"; time = "22:12:15"; } { date = "2021-02-03"; time = "23:54:12"; } - => true + Example: + cmpDate { date = "2021-09-10"; time = "22:12:15"; } { date = "2021-02-03"; time = "23:54:12"; } + => true */ - cmpDate = a: b: - let da = (builtins.head a.builds).date; - db = (builtins.head b.builds).date; - in - if da == db then (builtins.head a.builds).time > (builtins.head b.builds).time - else da > db; + cmpDate = a: b: + let + da = (builtins.head a.builds).date; + db = (builtins.head b.builds).date; + in if da == db then + (builtins.head a.builds).time > (builtins.head b.builds).time + else + da > db; - /* Pretty platforms */ + # Pretty platforms prettyPlatform = name: - if name == "aarch64-unknown-linux-musl" then "linux/arm64" - else if name == "armv6l-unknown-linux-musleabihf" then "linux/arm" - else if name == "x86_64-unknown-linux-musl" then "linux/amd64" - else if name == "i686-unknown-linux-musl" then "linux/386" - else name; + if name == "aarch64-unknown-linux-musl" then + "linux/arm64" + else if name == "armv6l-unknown-linux-musleabihf" then + "linux/arm" + else if name == "x86_64-unknown-linux-musl" then + "linux/amd64" + else if name == "i686-unknown-linux-musl" then + "linux/386" + else + name; - /* Parsing */ + # Parsing list = builtins.readFile (./. + path); entries = lib.splitString "\n" list; - elems = builtins.filter - (e: (builtins.length e) == 4) - (map - (x: builtins.filter (e: e != "") (lib.splitString " " x)) - entries); + elems = builtins.filter (e: (builtins.length e) == 4) + (map (x: builtins.filter (e: e != "") (lib.splitString " " x)) entries); - keys = ["date" "time" "size" "path"]; + keys = [ "date" "time" "size" "path" ]; parsed = map (entry: listToSet keys entry) elems; - subkeys = ["root" "version" "platform" "binary" ]; - builds = map (entry: entry // listToSet subkeys (lib.splitString "/" entry.path) // { url = "https://garagehq.deuxfleurs.fr/" + entry.path; }) parsed; + subkeys = [ "root" "version" "platform" "binary" ]; + builds = map (entry: + entry // listToSet subkeys (lib.splitString "/" entry.path) // { + url = "https://garagehq.deuxfleurs.fr/" + entry.path; + }) parsed; - /* Aggregation */ - builds_per_version = lib.foldl (acc: v: acc // { ${v.version} = if builtins.hasAttr v.version acc then acc.${v.version} ++ [ v ] else [ v ]; }) {} builds; + # Aggregation + builds_per_version = lib.foldl (acc: v: + acc // { + ${v.version} = if builtins.hasAttr v.version acc then + acc.${v.version} ++ [ v ] + else + [ v ]; + }) { } builds; versions = builtins.attrNames builds_per_version; - versions_release = builtins.filter (x: builtins.match "v[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?" x != null) versions; - versions_commit = builtins.filter (x: builtins.match "[0-9a-f]{40}" x != null) versions; - versions_extra = lib.subtractLists (versions_release ++ versions_commit) versions; + versions_release = builtins.filter + (x: builtins.match "v[0-9]+.[0-9]+.[0-9]+(.[0-9]+)?" x != null) versions; + versions_commit = + builtins.filter (x: builtins.match "[0-9a-f]{40}" x != null) versions; + versions_extra = + lib.subtractLists (versions_release ++ versions_commit) versions; sorted_builds = [ { name = "Release"; hide = false; type = "tag"; - description = "Release builds are the official builds, they are tailored for productions and are the most tested."; - builds = builtins.sort (a: b: a.version > b.version) (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_release); + description = + "Release builds are the official builds, they are tailored for productions and are the most tested."; + builds = builtins.sort (a: b: a.version > b.version) (map (x: { + version = x; + builds = builtins.getAttr x builds_per_version; + }) versions_release); } { name = "Extra"; hide = true; type = "tag"; - description = "Extra builds are built on demand to test a specific feature or a specific need."; - builds = builtins.sort cmpDate (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_extra); + description = + "Extra builds are built on demand to test a specific feature or a specific need."; + builds = builtins.sort cmpDate (map (x: { + version = x; + builds = builtins.getAttr x builds_per_version; + }) versions_extra); } { name = "Development"; hide = true; type = "commit"; - description = "Development builds are built periodically. Use them if you want to test a specific feature that is not yet released."; - builds = builtins.sort cmpDate (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_commit); + description = + "Development builds are built periodically. Use them if you want to test a specific feature that is not yet released."; + builds = builtins.sort cmpDate (map (x: { + version = x; + builds = builtins.getAttr x builds_per_version; + }) versions_commit); } ]; - json = pkgs.writeTextDir "share/_releases.json" (builtins.toJSON sorted_builds); + json = + pkgs.writeTextDir "share/_releases.json" (builtins.toJSON sorted_builds); html = pkgs.writeTextDir "share/_releases.html" '' - - - - - Garage releases - - - - ${ builtins.toString (lib.forEach sorted_builds (r: '' -
-

${r.name} builds

+ + + + + Garage releases + + + + ${ + builtins.toString (lib.forEach sorted_builds (r: '' +
+

${r.name} builds

-

${r.description}

+

${r.description}

- ${if r.hide then "
Show ${r.name} builds" else ""} - ${ builtins.toString (lib.forEach r.builds (x: '' -

${x.version} (${(builtins.head x.builds).date})

-

See this build on

-

Binaries: -

-

Sources: -

- '')) } - ${ if builtins.length r.builds == 0 then "There is no build for this category" else "" } - ${if r.hide then "
" else ""} -
- ''))} - - -''; -in - pkgs.symlinkJoin { - name = "releases"; - paths = [ json html ]; - } + ${ + if r.hide then + "
Show ${r.name} builds" + else + "" + } + ${ + builtins.toString (lib.forEach r.builds (x: '' +

${x.version} (${(builtins.head x.builds).date})

+

See this build on

+

Binaries: +

+

Sources: +

+ '')) + } + ${ + if builtins.length r.builds == 0 then + "There is no build for this category" + else + "" + } + ${if r.hide then "
" else ""} +
+ '')) + } + + + ''; +in pkgs.symlinkJoin { + name = "releases"; + paths = [ json html ]; +} diff --git a/nix/common.nix b/nix/common.nix index 90e3afaf..57f354dd 100644 --- a/nix/common.nix +++ b/nix/common.nix @@ -1,10 +1,9 @@ rec { - /* - * Fixed dependencies - */ + # * Fixed dependencies pkgsSrc = fetchTarball { # As of 2022-10-13 - url = "https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip"; + url = + "https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip"; sha256 = "1bz632psfbpmicyzjb8b4265y50shylccvfm6ry6mgnv5hvz324s"; }; cargo2nixSrc = fetchGit { @@ -14,9 +13,7 @@ rec { rev = "a7a61179b66054904ef6a195d8da736eaaa06c36"; }; - /* - * Shared objects - */ + # * Shared objects cargo2nix = import cargo2nixSrc; cargo2nixOverlay = cargo2nix.overlays.default; } diff --git a/nix/compile.nix b/nix/compile.nix index 3ea5035e..54e920db 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -1,227 +1,240 @@ -{ - system, - target ? null, - pkgsSrc, - cargo2nixOverlay, - compiler ? "rustc", - release ? false, - git_version ? null, - features ? null, -}: +{ system, target ? null, pkgsSrc, cargo2nixOverlay, compiler ? "rustc" +, release ? false, git_version ? null, features ? null, }: let log = v: builtins.trace v v; - pkgs = - if target != null then - import pkgsSrc { - inherit system; - crossSystem = { - config = target; - isStatic = true; - }; - overlays = [ cargo2nixOverlay ]; - } - else - import pkgsSrc { - inherit system; - overlays = [ cargo2nixOverlay ]; - }; - - /* - Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases. - This is fine for 64-bit platforms, but for 32-bit platforms, we need our own Rust - to avoid incompatibilities with time_t between different versions of musl - (>= 1.2.0 shipped by NixOS, < 1.2.0 with which rustc was built), which lead to compilation breakage. - So we want a Rust release that is bound to our Nix repository to avoid these problems. - See here for more info: https://musl.libc.org/time64.html - Because Cargo2nix does not support the Rust environment shipped by NixOS, - we emulate the structure of the Rust object created by rustOverlay. - In practise, rustOverlay ships rustc+cargo in a single derivation while - NixOS ships them in separate ones. We reunite them with symlinkJoin. - */ - toolchainOptions = - if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then { - rustVersion = "1.63.0"; - extraRustComponents = [ "clippy" ]; - } else { - rustToolchain = pkgs.symlinkJoin { - name = "rust-static-toolchain-${target}"; - paths = [ - pkgs.rustPlatform.rust.cargo - pkgs.rustPlatform.rust.rustc - # clippy not needed, it only runs on amd64 - ]; + pkgs = if target != null then + import pkgsSrc { + inherit system; + crossSystem = { + config = target; + isStatic = true; }; + overlays = [ cargo2nixOverlay ]; + } + else + import pkgsSrc { + inherit system; + overlays = [ cargo2nixOverlay ]; }; - - buildEnv = (drv: { - rustc = drv.setBuildEnv; - clippy = '' - ${drv.setBuildEnv or "" } - echo - echo --- BUILDING WITH CLIPPY --- - echo - - export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings" - export RUSTC="''${CLIPPY_DRIVER}" - ''; - }.${compiler}); - - /* - Cargo2nix provides many overrides by default, you can take inspiration from them: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix - - You can have a complete list of the available options by looking at the overriden object, mkcrate: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix + /* Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases. + This is fine for 64-bit platforms, but for 32-bit platforms, we need our own Rust + to avoid incompatibilities with time_t between different versions of musl + (>= 1.2.0 shipped by NixOS, < 1.2.0 with which rustc was built), which lead to compilation breakage. + So we want a Rust release that is bound to our Nix repository to avoid these problems. + See here for more info: https://musl.libc.org/time64.html + Because Cargo2nix does not support the Rust environment shipped by NixOS, + we emulate the structure of the Rust object created by rustOverlay. + In practise, rustOverlay ships rustc+cargo in a single derivation while + NixOS ships them in separate ones. We reunite them with symlinkJoin. */ - packageOverrides = pkgs: pkgs.rustBuilder.overrides.all ++ [ - /* - [1] We add some logic to compile our crates with clippy, it provides us many additional lints + toolchainOptions = if target == null || target == "x86_64-unknown-linux-musl" + || target == "aarch64-unknown-linux-musl" then { + rustVersion = "1.63.0"; + extraRustComponents = [ "clippy" ]; + } else { + rustToolchain = pkgs.symlinkJoin { + name = "rust-static-toolchain-${target}"; + paths = [ + pkgs.rustPlatform.rust.cargo + pkgs.rustPlatform.rust.rustc + # clippy not needed, it only runs on amd64 + ]; + }; + }; - [2] We need to alter Nix hardening to make static binaries: PIE, - Position Independent Executables seems to be supported only on amd64. Having - this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. - Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets - (only amd64 curently) through the `-static-pie` flag. - PIE is a feature used by ASLR, which helps mitigate security issues. - Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh + buildEnv = (drv: + { + rustc = drv.setBuildEnv; + clippy = '' + ${drv.setBuildEnv or ""} + echo + echo --- BUILDING WITH CLIPPY --- + echo - [3] We want to inject the git version while keeping the build deterministic. - As we do not want to consider the .git folder as part of the input source, - we ask the user (the CI often) to pass the value to Nix. + export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings" + export RUSTC="''${CLIPPY_DRIVER}" + ''; + }.${compiler}); - [4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library. - However the features to do so get activated for some reason (due to a bug in cargo2nix?), - so disable them manually here. - */ - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage"; - overrideAttrs = drv: - (if git_version != null then { - /* [3] */ preConfigure = '' - ${drv.preConfigure or ""} - export GIT_VERSION="${git_version}" - ''; - } else {}) - // - { - /* [1] */ setBuildEnv = (buildEnv drv); - /* [2] */ hardeningDisable = [ "pie" ]; - }; - }) + /* Cargo2nix provides many overrides by default, you can take inspiration from them: + https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_rpc"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_db"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_util"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_table"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_block"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_model"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_api"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_web"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "k2v-client"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "libsodium-sys"; - overrideArgs = old: { - features = [ ]; /* [4] */ - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "zstd-sys"; - overrideArgs = old: { - features = [ ]; /* [4] */ - }; - }) - ]; - - /* - We ship some parts of the code disabled by default by putting them behind a flag. - It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). - But we want to ship these additional features when we release Garage. - In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds. + You can have a complete list of the available options by looking at the overriden object, mkcrate: + https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix */ - rootFeatures = if features != null then features else - ([ - "garage/bundled-libs" - "garage/sled" - "garage/k2v" - ] ++ (if release then [ - "garage/consul-discovery" - "garage/kubernetes-discovery" - "garage/metrics" - "garage/telemetry-otlp" - "garage/lmdb" - "garage/sqlite" - ] else [])); + packageOverrides = pkgs: + pkgs.rustBuilder.overrides.all ++ [ + /* [1] We add some logic to compile our crates with clippy, it provides us many additional lints + [2] We need to alter Nix hardening to make static binaries: PIE, + Position Independent Executables seems to be supported only on amd64. Having + this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. + Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets + (only amd64 curently) through the `-static-pie` flag. + PIE is a feature used by ASLR, which helps mitigate security issues. + Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh + + [3] We want to inject the git version while keeping the build deterministic. + As we do not want to consider the .git folder as part of the input source, + we ask the user (the CI often) to pass the value to Nix. + + [4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library. + However the features to do so get activated for some reason (due to a bug in cargo2nix?), + so disable them manually here. + */ + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage"; + overrideAttrs = drv: + (if git_version != null then { + # [3] + preConfigure = '' + ${drv.preConfigure or ""} + export GIT_VERSION="${git_version}" + ''; + } else + { }) // { + # [1] + setBuildEnv = (buildEnv drv); + # [2] + hardeningDisable = [ "pie" ]; + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_rpc"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_db"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_util"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_table"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_block"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_model"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_api"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_web"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "k2v-client"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "libsodium-sys"; + overrideArgs = old: { + features = [ ]; # [4] + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "zstd-sys"; + overrideArgs = old: { + features = [ ]; # [4] + }; + }) + ]; + + /* We ship some parts of the code disabled by default by putting them behind a flag. + It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). + But we want to ship these additional features when we release Garage. + In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds. + */ + rootFeatures = if features != null then + features + else + ([ "garage/bundled-libs" "garage/sled" "garage/k2v" ] ++ (if release then [ + "garage/consul-discovery" + "garage/kubernetes-discovery" + "garage/metrics" + "garage/telemetry-otlp" + "garage/lmdb" + "garage/sqlite" + ] else + [ ])); packageFun = import ../Cargo.nix; - /* - We compile fully static binaries with musl to simplify deployment on most systems. - When possible, we reactivate PIE hardening (see above). + /* We compile fully static binaries with musl to simplify deployment on most systems. + When possible, we reactivate PIE hardening (see above). - Also, if you set the RUSTFLAGS environment variable, the following parameters will - be ignored. + Also, if you set the RUSTFLAGS environment variable, the following parameters will + be ignored. - For more information on static builds, please refer to Rust's RFC 1721. - https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage + For more information on static builds, please refer to Rust's RFC 1721. + https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage */ codegenOpts = { - "armv6l-unknown-linux-musleabihf" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* compile as dynamic with static-pie */ - "aarch64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */ - "i686-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */ - "x86_64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static-pie" ]; + "armv6l-unknown-linux-musleabihf" = [ + "target-feature=+crt-static" + "link-arg=-static" + ]; # compile as dynamic with static-pie + "aarch64-unknown-linux-musl" = [ + "target-feature=+crt-static" + "link-arg=-static" + ]; # segfault with static-pie + "i686-unknown-linux-musl" = [ + "target-feature=+crt-static" + "link-arg=-static" + ]; # segfault with static-pie + "x86_64-unknown-linux-musl" = + [ "target-feature=+crt-static" "link-arg=-static-pie" ]; }; - /* - NixOS and Rust/Cargo triples do not match for ARM, fix it here. - */ - rustTarget = if target == "armv6l-unknown-linux-musleabihf" - then "arm-unknown-linux-musleabihf" - else target; + # NixOS and Rust/Cargo triples do not match for ARM, fix it here. + rustTarget = if target == "armv6l-unknown-linux-musleabihf" then + "arm-unknown-linux-musleabihf" + else + target; -in - pkgs.rustBuilder.makePackageSet ({ - inherit release packageFun packageOverrides codegenOpts rootFeatures; - target = rustTarget; - } // toolchainOptions) +in pkgs.rustBuilder.makePackageSet ({ + inherit release packageFun packageOverrides codegenOpts rootFeatures; + target = rustTarget; +} // toolchainOptions) diff --git a/nix/kaniko.nix b/nix/kaniko.nix index 140328b8..64cadd14 100644 --- a/nix/kaniko.nix +++ b/nix/kaniko.nix @@ -15,7 +15,8 @@ pkgs.buildGoModule rec { checkPhase = "true"; meta = with pkgs.lib; { - description = "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster."; + description = + "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster."; homepage = "https://github.com/GoogleContainerTools/kaniko"; license = licenses.asl20; platforms = platforms.linux; diff --git a/nix/manifest-tool.nix b/nix/manifest-tool.nix index 182ccc0e..1090a6ef 100644 --- a/nix/manifest-tool.nix +++ b/nix/manifest-tool.nix @@ -15,7 +15,8 @@ pkgs.buildGoModule rec { checkPhase = "true"; meta = with pkgs.lib; { - description = "Command line tool to create and query container image manifest list/indexes"; + description = + "Command line tool to create and query container image manifest list/indexes"; homepage = "https://github.com/estesp/manifest-tool"; license = licenses.asl20; platforms = platforms.linux; diff --git a/nix/toolchain.nix b/nix/toolchain.nix index 079fcf13..532db74e 100644 --- a/nix/toolchain.nix +++ b/nix/toolchain.nix @@ -1,6 +1,4 @@ -{ - system ? builtins.currentSystem, -}: +{ system ? builtins.currentSystem, }: with import ./common.nix; @@ -11,27 +9,23 @@ let #"aarch64-unknown-linux-musl" "armv6l-unknown-linux-musleabihf" ]; - pkgsList = builtins.map (target: import pkgsSrc { - inherit system; - crossSystem = { - config = target; - isStatic = true; - }; - overlays = [ cargo2nixOverlay ]; - }) platforms; - pkgsHost = import pkgsSrc {}; + pkgsList = builtins.map (target: + import pkgsSrc { + inherit system; + crossSystem = { + config = target; + isStatic = true; + }; + overlays = [ cargo2nixOverlay ]; + }) platforms; + pkgsHost = import pkgsSrc { }; lib = pkgsHost.lib; kaniko = (import ./kaniko.nix) pkgsHost; winscp = (import ./winscp.nix) pkgsHost; manifestTool = (import ./manifest-tool.nix) pkgsHost; -in - lib.flatten (builtins.map (pkgs: [ - pkgs.rustPlatform.rust.rustc - pkgs.rustPlatform.rust.cargo - pkgs.buildPackages.stdenv.cc - ]) pkgsList) ++ [ - kaniko - winscp - manifestTool - ] +in lib.flatten (builtins.map (pkgs: [ + pkgs.rustPlatform.rust.rustc + pkgs.rustPlatform.rust.cargo + pkgs.buildPackages.stdenv.cc +]) pkgsList) ++ [ kaniko winscp manifestTool ] diff --git a/nix/winscp.nix b/nix/winscp.nix index 10d3cb28..113f4506 100644 --- a/nix/winscp.nix +++ b/nix/winscp.nix @@ -11,12 +11,12 @@ pkgs.stdenv.mkDerivation rec { }; buildPhase = '' - cat > winscp < winscp < /tmp/nix-signing-key.sec - nix copy \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR \ - $(nix-build --no-build-output --no-out-link nix/toolchain.nix)) - rm /tmp/nix-signing-key.sec + # --- Release shell --- + # A shell built to make releasing easier + release = pkgs.mkShell { + shellHook = '' + function refresh_toolchain { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + nix copy \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix-store -qR \ + $(nix-build --no-build-output --no-out-link nix/toolchain.nix)) + rm /tmp/nix-signing-key.sec + } + + function refresh_cache { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do + echo "Updating cache for ''${attr}" + derivation=$(nix-instantiate --attr ''${attr}) + nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix-store -qR ''${derivation%\!bin}) + done + rm /tmp/nix-signing-key.sec + } + + function refresh_flake_cache { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + for attr in packages.x86_64-linux.default devShell.x86_64-linux; do + echo "Updating cache for ''${attr}" + derivation=$(nix path-info --derivation ".#''${attr}") + nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix-store -qR ''${derivation}) + done + rm /tmp/nix-signing-key.sec + } + + function to_s3 { + aws \ + --endpoint-url https://garage.deuxfleurs.fr \ + --region garage \ + s3 cp \ + ./result-bin/bin/garage \ + s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage + } + + function to_docker { + executor \ + --force \ + --customPlatform="''${DOCKER_PLATFORM}" \ + --destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \ + --context dir://`pwd` \ + --verbosity=debug + } + + function multiarch_docker { + manifest-tool push from-spec <(cat < aws-list.txt + + nix-build nix/build_index.nix + + aws \ + --endpoint-url https://garage.deuxfleurs.fr \ + --region garage \ + s3 cp \ + result/share/_releases.json \ + s3://garagehq.deuxfleurs.fr/ + + aws \ + --endpoint-url https://garage.deuxfleurs.fr \ + --region garage \ + s3 cp \ + result/share/_releases.html \ + s3://garagehq.deuxfleurs.fr/ + } + ''; + nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ]; + }; } -function refresh_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do - echo "Updating cache for ''${attr}" - derivation=$(nix-instantiate --attr ''${attr}) - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation%\!bin}) - done - rm /tmp/nix-signing-key.sec -} - -function refresh_flake_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in packages.x86_64-linux.default; do - echo "Updating cache for ''${attr}" - derivation=$(nix path-info --derivation ".#''${attr}") - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation}) - done - rm /tmp/nix-signing-key.sec -} - -function to_s3 { - aws \ - --endpoint-url https://garage.deuxfleurs.fr \ - --region garage \ - s3 cp \ - ./result-bin/bin/garage \ - s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage -} - -function to_docker { - executor \ - --force \ - --customPlatform="''${DOCKER_PLATFORM}" \ - --destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \ - --context dir://`pwd` \ - --verbosity=debug -} - -function multiarch_docker { - manifest-tool push from-spec <(cat < aws-list.txt - - nix-build nix/build_index.nix - - aws \ - --endpoint-url https://garage.deuxfleurs.fr \ - --region garage \ - s3 cp \ - result/share/_releases.json \ - s3://garagehq.deuxfleurs.fr/ - - aws \ - --endpoint-url https://garage.deuxfleurs.fr \ - --region garage \ - s3 cp \ - result/share/_releases.html \ - s3://garagehq.deuxfleurs.fr/ -} - ''; - nativeBuildInputs = [ - pkgs.awscli2 - kaniko - manifest-tool - ]; - }; - } - -