From 9d8360573622475232cee463291596273c5df1d3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 12 Jan 2025 17:34:04 +0100 Subject: [PATCH 1/5] flake: update versions of nixpkgs and rust-overlay --- flake.lock | 16 ++++++++-------- flake.nix | 8 ++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/flake.lock b/flake.lock index bc9431a3..15cc70a5 100644 --- a/flake.lock +++ b/flake.lock @@ -58,17 +58,17 @@ }, "nixpkgs": { "locked": { - "lastModified": 1724681257, - "narHash": "sha256-EJRuc5Qp7yfXko5ZNeEMYAs4DzAvkCyALuJ/tGllhN4=", + "lastModified": 1736692550, + "narHash": "sha256-7tk8xH+g0sJkKLTJFOxphJxxOjMDFMWv24nXslaU2ro=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "0239aeb2f82ea27ccd6b61582b8f7fb8750eeada", + "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815", "type": "github" }, "original": { "owner": "NixOS", "repo": "nixpkgs", - "rev": "0239aeb2f82ea27ccd6b61582b8f7fb8750eeada", + "rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815", "type": "github" } }, @@ -91,17 +91,17 @@ ] }, "locked": { - "lastModified": 1724638882, - "narHash": "sha256-ap2jIQi/FuUHR6HCht6ASWhoz8EiB99XmI8Esot38VE=", + "lastModified": 1736649126, + "narHash": "sha256-XCw5sv/ePsroqiF3lJM6Y2X9EhPdHeE47gr3Q8b0UQw=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "19b70f147b9c67a759e35824b241f1ed92e46694", + "rev": "162ab0edc2936508470199b2e8e6c444a2535019", "type": "github" }, "original": { "owner": "oxalica", "repo": "rust-overlay", - "rev": "19b70f147b9c67a759e35824b241f1ed92e46694", + "rev": "162ab0edc2936508470199b2e8e6c444a2535019", "type": "github" } } diff --git a/flake.nix b/flake.nix index e690aef1..91340245 100644 --- a/flake.nix +++ b/flake.nix @@ -2,9 +2,9 @@ description = "Garage, an S3-compatible distributed object store for self-hosted deployments"; - # Nixpkgs 24.05 as of 2024-08-26 has rustc v1.77 + # Nixpkgs 24.11 as of 2025-01-12 has rustc v1.82 inputs.nixpkgs.url = - "github:NixOS/nixpkgs/0239aeb2f82ea27ccd6b61582b8f7fb8750eeada"; + "github:NixOS/nixpkgs/7c4869c47090dd7f9f1bdfb49a22aea026996815"; inputs.flake-compat.url = "github:nix-community/flake-compat"; @@ -17,9 +17,9 @@ # - rustc v1.66 # url = "github:cargo2nix/cargo2nix/8fb57a670f7993bfc24099c33eb9c5abb51f29a2"; - # Rust overlay as of 2024-08-26 + # Rust overlay as of 2025-01-12 inputs.rust-overlay.url = - "github:oxalica/rust-overlay/19b70f147b9c67a759e35824b241f1ed92e46694"; + "github:oxalica/rust-overlay/162ab0edc2936508470199b2e8e6c444a2535019"; inputs.nixpkgs.follows = "nixpkgs"; inputs.flake-compat.follows = "flake-compat"; From 295237476e2228cb58b417afe991cc2571a10bff Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 12 Jan 2025 17:36:25 +0100 Subject: [PATCH 2/5] fix formatting to comply with latest rustfmt --- src/block/layout.rs | 3 ++- src/model/helper/locked.rs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/block/layout.rs b/src/block/layout.rs index e78f3f08..00e3debb 100644 --- a/src/block/layout.rs +++ b/src/block/layout.rs @@ -279,7 +279,8 @@ impl DataLayout { u16::from_be_bytes([ hash.as_slice()[HASH_DRIVE_BYTES.0], hash.as_slice()[HASH_DRIVE_BYTES.1], - ]) as usize % DRIVE_NPART + ]) as usize + % DRIVE_NPART } fn block_dir_from(&self, hash: &Hash, dir: &PathBuf) -> PathBuf { diff --git a/src/model/helper/locked.rs b/src/model/helper/locked.rs index f8e06add..b541d548 100644 --- a/src/model/helper/locked.rs +++ b/src/model/helper/locked.rs @@ -279,7 +279,8 @@ impl<'a> LockedHelper<'a> { .local_aliases .get(alias_name) .cloned() - .flatten() != Some(bucket_id) + .flatten() + != Some(bucket_id) { return Err(GarageError::Message(format!( "Bucket {:?} does not have alias {} in namespace of key {}", From 0a15db6960add3e06a130d64b51d6ae55cd2d759 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 12 Jan 2025 17:37:36 +0100 Subject: [PATCH 3/5] nix build: update rustc to v1.78 --- nix/compile.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nix/compile.nix b/nix/compile.nix index c9cbebd5..61955d8a 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -20,7 +20,7 @@ let }; toolchainOptions = { - rustVersion = "1.77.0"; + rustVersion = "1.78.0"; extraRustComponents = [ "clippy" ]; }; From 83887a851911e18106ede698f491b0102d16b7a4 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 12 Jan 2025 17:40:34 +0100 Subject: [PATCH 4/5] nix build: remove clippy build env that doesn't work --- .woodpecker/debug.yaml | 6 +-- default.nix | 7 ---- nix/compile.nix | 95 ++++-------------------------------------- shell.nix | 2 +- 4 files changed, 11 insertions(+), 99 deletions(-) diff --git a/.woodpecker/debug.yaml b/.woodpecker/debug.yaml index 0076feda..06597a3b 100644 --- a/.woodpecker/debug.yaml +++ b/.woodpecker/debug.yaml @@ -16,7 +16,7 @@ steps: - name: build image: nixpkgs/nix:nixos-22.05 commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + - nix-build --no-build-output --attr pkgs.amd64.debug --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} - name: unit + func tests image: nixpkgs/nix:nixos-22.05 @@ -24,7 +24,7 @@ steps: GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + - nix-build --no-build-output --attr pkgs.amd64.debug --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} - nix-build --no-build-output --attr test.amd64 - ./result/bin/garage_db-* - ./result/bin/garage_api-* @@ -43,5 +43,5 @@ steps: - name: integration tests image: nixpkgs/nix:nixos-22.05 commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + - nix-build --no-build-output --attr pkgs.amd64.debug --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) diff --git a/default.nix b/default.nix index 57bc24a5..8ddc0af8 100644 --- a/default.nix +++ b/default.nix @@ -45,11 +45,4 @@ in { ]; }); }; - clippy = { - amd64 = (compile { - inherit system git_version pkgsSrc cargo2nixOverlay; - target = "x86_64-unknown-linux-musl"; - compiler = "clippy"; - }).workspace.garage { compileMode = "build"; }; - }; } diff --git a/nix/compile.nix b/nix/compile.nix index 61955d8a..2755f5be 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -1,4 +1,4 @@ -{ system, target ? null, pkgsSrc, cargo2nixOverlay, compiler ? "rustc" +{ system, target ? null, pkgsSrc, cargo2nixOverlay , release ? false, git_version ? null, features ? null, }: let @@ -24,20 +24,6 @@ let extraRustComponents = [ "clippy" ]; }; - buildEnv = (drv: - { - rustc = drv.setBuildEnv; - clippy = '' - ${drv.setBuildEnv or ""} - echo - echo --- BUILDING WITH CLIPPY --- - echo - - export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings" - export RUSTC="''${CLIPPY_DRIVER}" - ''; - }.${compiler}); - /* Cargo2nix provides many overrides by default, you can take inspiration from them: https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix @@ -46,9 +32,7 @@ let */ packageOverrides = pkgs: pkgs.rustBuilder.overrides.all ++ [ - /* [1] We add some logic to compile our crates with clippy, it provides us many additional lints - - [2] We need to alter Nix hardening to make static binaries: PIE, + /* [1] We need to alter Nix hardening to make static binaries: PIE, Position Independent Executables seems to be supported only on amd64. Having this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets @@ -56,11 +40,11 @@ let PIE is a feature used by ASLR, which helps mitigate security issues. Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh - [3] We want to inject the git version while keeping the build deterministic. + [2] We want to inject the git version while keeping the build deterministic. As we do not want to consider the .git folder as part of the input source, we ask the user (the CI often) to pass the value to Nix. - [4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library. + [3] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library. However the features to do so get activated for some reason (due to a bug in cargo2nix?), so disable them manually here. */ @@ -68,7 +52,7 @@ let name = "garage"; overrideAttrs = drv: (if git_version != null then { - # [3] + # [2] preConfigure = '' ${drv.preConfigure or ""} export GIT_VERSION="${git_version}" @@ -76,86 +60,21 @@ let } else { }) // { # [1] - setBuildEnv = (buildEnv drv); - # [2] hardeningDisable = [ "pie" ]; }; }) - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_rpc"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_db"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_util"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_table"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_block"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_model"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_api"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_web"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "k2v-client"; - overrideAttrs = drv: { # [1] - setBuildEnv = (buildEnv drv); - }; - }) - (pkgs.rustBuilder.rustLib.makeOverride { name = "libsodium-sys"; overrideArgs = old: { - features = [ ]; # [4] + features = [ ]; # [3] }; }) (pkgs.rustBuilder.rustLib.makeOverride { name = "zstd-sys"; overrideArgs = old: { - features = [ ]; # [4] + features = [ ]; # [3] }; }) ]; diff --git a/shell.nix b/shell.nix index 7236fdcb..cc06f0fb 100644 --- a/shell.nix +++ b/shell.nix @@ -115,7 +115,7 @@ in shellHook = '' function refresh_cache { pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do + for attr in pkgs.amd64.debug test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do echo "Updating cache for ''${attr}" nix copy -j8 \ --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ From 8ddb0dd485cf8290b7e3462f5614ca6d858ea8c3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 12 Jan 2025 18:16:23 +0100 Subject: [PATCH 5/5] nix build: switch to upstream cargo2nix (branch release-0.11.0) --- Cargo.nix | 27 ++++++++++++++------------- flake.lock | 12 ++++++------ flake.nix | 5 ++++- 3 files changed, 24 insertions(+), 20 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 419b1f67..fcc7eaea 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -25,6 +25,7 @@ args@{ target ? null, codegenOpts ? null, profileOpts ? null, + cargoUnstableFlags ? null, rustcLinkFlags ? null, rustcBuildFlags ? null, mkRustCrate, @@ -51,7 +52,7 @@ else let rootFeatures' = expandFeatures rootFeatures; overridableMkRustCrate = f: let - drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts rustcLinkFlags rustcBuildFlags; } // (f profileName))); + drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts cargoUnstableFlags rustcLinkFlags rustcBuildFlags; } // (f profileName))); in { compileMode ? null, profileName ? decideProfile compileMode release }: let drv = drvs.${profileName}; in if compileMode == null then drv else drv.override { inherit compileMode; }; in @@ -1756,7 +1757,7 @@ in name = "format_table"; version = "0.1.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/format-table"); + src = fetchCrateLocal workspaceSrc; }); "registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" = overridableMkRustCrate (profileName: rec { @@ -1914,7 +1915,7 @@ in name = "garage"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/garage"); + src = fetchCrateLocal workspaceSrc; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default") "bundled-libs") (lib.optional (rootFeatures' ? "garage/consul-discovery") "consul-discovery") @@ -1992,7 +1993,7 @@ in name = "garage_api"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/api"); + src = fetchCrateLocal workspaceSrc; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v") (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics") "metrics") @@ -2056,7 +2057,7 @@ in name = "garage_block"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/block"); + src = fetchCrateLocal workspaceSrc; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "system-libs") ]; @@ -2089,7 +2090,7 @@ in name = "garage_db"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/db"); + src = fetchCrateLocal workspaceSrc; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled-libs") (lib.optional (rootFeatures' ? "garage_db/default") "default") @@ -2118,7 +2119,7 @@ in name = "garage_model"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/model"); + src = fetchCrateLocal workspaceSrc; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "garage_model/default") "default") (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v") @@ -2157,7 +2158,7 @@ in name = "garage_net"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/net"); + src = fetchCrateLocal workspaceSrc; features = builtins.concatLists [ [ "default" ] (lib.optional (rootFeatures' ? "garage_net/opentelemetry" || rootFeatures' ? "garage_net/telemetry") "opentelemetry") @@ -2194,7 +2195,7 @@ in name = "garage_rpc"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/rpc"); + src = fetchCrateLocal workspaceSrc; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery") "consul-discovery") (lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive") "err-derive") @@ -2243,7 +2244,7 @@ in name = "garage_table"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/table"); + src = fetchCrateLocal workspaceSrc; dependencies = { arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out; async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.77" { profileName = "__noProfile"; }).out; @@ -2268,7 +2269,7 @@ in name = "garage_util"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/util"); + src = fetchCrateLocal workspaceSrc; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v") ]; @@ -2312,7 +2313,7 @@ in name = "garage_web"; version = "1.0.1"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/web"); + src = fetchCrateLocal workspaceSrc; dependencies = { err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out; @@ -3148,7 +3149,7 @@ in name = "k2v-client"; version = "0.0.4"; registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/k2v-client"); + src = fetchCrateLocal workspaceSrc; features = builtins.concatLists [ (lib.optional (rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "clap") (lib.optional (rootFeatures' ? "k2v-client/cli") "cli") diff --git a/flake.lock b/flake.lock index 15cc70a5..2c7be871 100644 --- a/flake.lock +++ b/flake.lock @@ -12,17 +12,17 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1666087781, - "narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=", - "owner": "Alexis211", + "lastModified": 1705129117, + "narHash": "sha256-LgdDHibvimzYhxBK3kxCk2gAL7k4Hyigl5KI0X9cijA=", + "owner": "cargo2nix", "repo": "cargo2nix", - "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36", + "rev": "ae19a9e1f8f0880c088ea155ab66cee1fa001f59", "type": "github" }, "original": { - "owner": "Alexis211", + "owner": "cargo2nix", "repo": "cargo2nix", - "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36", + "rev": "ae19a9e1f8f0880c088ea155ab66cee1fa001f59", "type": "github" } }, diff --git a/flake.nix b/flake.nix index 91340245..b1cb7dbb 100644 --- a/flake.nix +++ b/flake.nix @@ -10,13 +10,16 @@ inputs.cargo2nix = { # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection - url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36"; + #url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36"; # As of 2023-04-25: # - my two patches were merged into unstable (one for clippy and one to "fix" feature detection) # - rustc v1.66 # url = "github:cargo2nix/cargo2nix/8fb57a670f7993bfc24099c33eb9c5abb51f29a2"; + # Mainline cargo2nix as of of 2025-01-12 (branch release-0.11.0) + url = "github:cargo2nix/cargo2nix/ae19a9e1f8f0880c088ea155ab66cee1fa001f59"; + # Rust overlay as of 2025-01-12 inputs.rust-overlay.url = "github:oxalica/rust-overlay/162ab0edc2936508470199b2e8e6c444a2535019";