Merge pull request 'compile with crane' (#950) from nix-crane into main

Reviewed-on: Deuxfleurs/garage#950
This commit is contained in:
Alex 2025-02-03 15:54:54 +00:00
commit b9df2d1ad1
16 changed files with 281 additions and 7317 deletions

View file

@ -1,3 +0,0 @@
[target.x86_64-unknown-linux-gnu]
linker = "clang"
rustflags = ["-C", "link-arg=-fuse-ld=mold"]

View file

@ -16,34 +16,21 @@ steps:
- name: build
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --no-build-output --attr pkgs.amd64.debug --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- nix-build -j4 --attr flakePackages.dev
- name: unit + func tests
- name: unit + func tests (lmdb)
image: nixpkgs/nix:nixos-22.05
environment:
GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage
GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration
commands:
- nix-build -j4 --no-build-output --attr test.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- ./result/bin/garage_db-*
- ./result/bin/garage_api_common-*
- ./result/bin/garage_api_s3-*
- ./result/bin/garage_api_k2v-*
- ./result/bin/garage_api_admin-*
- ./result/bin/garage_model-*
- ./result/bin/garage_rpc-*
- ./result/bin/garage_table-*
- ./result/bin/garage_util-*
- ./result/bin/garage_web-*
- ./result/bin/garage-*
- GARAGE_TEST_INTEGRATION_DB_ENGINE=lmdb ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
- nix-shell --attr ci --run "killall -9 garage" || true
- GARAGE_TEST_INTEGRATION_DB_ENGINE=sqlite ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
- rm result
- rm -rv tmp-garage-integration
- nix-build -j4 --attr flakePackages.tests-lmdb
- name: unit + func tests (sqlite)
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --attr flakePackages.tests-sqlite
- name: integration tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build -j4 --no-build-output --attr pkgs.amd64.debug --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- nix-build -j4 --attr flakePackages.dev
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
depends_on: [ build ]

View file

@ -18,12 +18,12 @@ steps:
- name: build
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- nix-build --attr releasePackages.${ARCH} --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
- name: check is static binary
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- nix-shell --attr ci --run "./script/not-dynamic.sh result/bin/garage"
- name: integration tests
image: nixpkgs/nix:nixos-22.05

7046
Cargo.nix

File diff suppressed because it is too large Load diff

View file

@ -3,5 +3,5 @@ FROM scratch
ENV RUST_BACKTRACE=1
ENV RUST_LOG=garage=info
COPY result-bin/bin/garage /
COPY result/bin/garage /
CMD [ "/garage", "server"]

View file

@ -1,13 +1,10 @@
.PHONY: doc all release shell run1 run2 run3
.PHONY: doc all run1 run2 run3
all:
clear; cargo build
release:
nix-build --attr pkgs.amd64.release --no-build-output
shell:
nix-shell
clear
cargo build \
--config 'target.x86_64-unknown-linux-gnu.linker="clang"' \
--config 'target.x86_64-unknown-linux-gnu.rustflags=["-C", "link-arg=-fuse-ld=mold"]' \
# ----

View file

@ -3,46 +3,22 @@
with import ./nix/common.nix;
let
pkgs = import pkgsSrc { };
pkgs = import nixpkgs { };
compile = import ./nix/compile.nix;
build_debug_and_release = (target: {
debug = (compile {
inherit system target git_version pkgsSrc cargo2nixOverlay;
release = false;
}).workspace.garage { compileMode = "build"; };
release = (compile {
inherit system target git_version pkgsSrc cargo2nixOverlay;
release = true;
}).workspace.garage { compileMode = "build"; };
});
test = (rustPkgs:
pkgs.symlinkJoin {
name = "garage-tests";
paths =
builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; })
(builtins.attrNames rustPkgs.workspace);
});
build_release = target: (compile {
inherit target system git_version nixpkgs;
crane = flake.inputs.crane;
rust-overlay = flake.inputs.rust-overlay;
release = true;
}).garage;
in {
pkgs = {
amd64 = build_debug_and_release "x86_64-unknown-linux-musl";
i386 = build_debug_and_release "i686-unknown-linux-musl";
arm64 = build_debug_and_release "aarch64-unknown-linux-musl";
arm = build_debug_and_release "armv6l-unknown-linux-musleabihf";
};
test = {
amd64 = test (compile {
inherit system git_version pkgsSrc cargo2nixOverlay;
target = "x86_64-unknown-linux-musl";
features = [
"garage/bundled-libs"
"garage/k2v"
"garage/lmdb"
"garage/sqlite"
];
});
releasePackages = {
amd64 = build_release "x86_64-unknown-linux-musl";
i386 = build_release "i686-unknown-linux-musl";
arm64 = build_release "aarch64-unknown-linux-musl";
arm = build_release "armv6l-unknown-linux-musleabihf";
};
flakePackages = flake.packages.${system};
}

62
flake.lock generated
View file

@ -1,28 +1,17 @@
{
"nodes": {
"cargo2nix": {
"inputs": {
"flake-compat": [
"flake-compat"
],
"flake-utils": "flake-utils",
"nixpkgs": [
"nixpkgs"
],
"rust-overlay": "rust-overlay"
},
"crane": {
"locked": {
"lastModified": 1705129117,
"narHash": "sha256-LgdDHibvimzYhxBK3kxCk2gAL7k4Hyigl5KI0X9cijA=",
"owner": "cargo2nix",
"repo": "cargo2nix",
"rev": "ae19a9e1f8f0880c088ea155ab66cee1fa001f59",
"lastModified": 1737689766,
"narHash": "sha256-ivVXYaYlShxYoKfSo5+y5930qMKKJ8CLcAoIBPQfJ6s=",
"owner": "ipetkov",
"repo": "crane",
"rev": "6fe74265bbb6d016d663b1091f015e2976c4a527",
"type": "github"
},
"original": {
"owner": "cargo2nix",
"repo": "cargo2nix",
"rev": "ae19a9e1f8f0880c088ea155ab66cee1fa001f59",
"owner": "ipetkov",
"repo": "crane",
"type": "github"
}
},
@ -42,12 +31,15 @@
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1659877975,
"narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
@ -74,19 +66,16 @@
},
"root": {
"inputs": {
"cargo2nix": "cargo2nix",
"crane": "crane",
"flake-compat": "flake-compat",
"flake-utils": [
"cargo2nix",
"flake-utils"
],
"nixpkgs": "nixpkgs"
"flake-utils": "flake-utils",
"nixpkgs": "nixpkgs",
"rust-overlay": "rust-overlay"
}
},
"rust-overlay": {
"inputs": {
"nixpkgs": [
"cargo2nix",
"nixpkgs"
]
},
@ -104,6 +93,21 @@
"rev": "162ab0edc2936508470199b2e8e6c444a2535019",
"type": "github"
}
},
"systems": {
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"type": "github"
}
}
},
"root": "root",

118
flake.nix
View file

@ -6,88 +6,80 @@
inputs.nixpkgs.url =
"github:NixOS/nixpkgs/7c4869c47090dd7f9f1bdfb49a22aea026996815";
# Rust overlay as of 2025-01-12
inputs.rust-overlay.url =
"github:oxalica/rust-overlay/162ab0edc2936508470199b2e8e6c444a2535019";
inputs.rust-overlay.inputs.nixpkgs.follows = "nixpkgs";
inputs.crane.url = "github:ipetkov/crane";
inputs.flake-compat.url = "github:nix-community/flake-compat";
inputs.flake-utils.url = "github:numtide/flake-utils";
inputs.cargo2nix = {
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
#url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
# As of 2023-04-25:
# - my two patches were merged into unstable (one for clippy and one to "fix" feature detection)
# - rustc v1.66
# url = "github:cargo2nix/cargo2nix/8fb57a670f7993bfc24099c33eb9c5abb51f29a2";
# Mainline cargo2nix as of of 2025-01-12 (branch release-0.11.0)
url = "github:cargo2nix/cargo2nix/ae19a9e1f8f0880c088ea155ab66cee1fa001f59";
# Rust overlay as of 2025-01-12
inputs.rust-overlay.url =
"github:oxalica/rust-overlay/162ab0edc2936508470199b2e8e6c444a2535019";
inputs.nixpkgs.follows = "nixpkgs";
inputs.flake-compat.follows = "flake-compat";
};
inputs.flake-utils.follows = "cargo2nix/flake-utils";
outputs = { self, nixpkgs, cargo2nix, flake-utils, ... }:
outputs = { self, nixpkgs, flake-utils, crane, rust-overlay, ... }:
let
git_version = self.lastModifiedDate;
compile = import ./nix/compile.nix;
in
flake-utils.lib.eachDefaultSystem (system:
let
pkgs = nixpkgs.legacyPackages.${system};
packageFor = target: release: (compile {
inherit system target nixpkgs crane rust-overlay release;
}).garage;
testWith = extraTestEnv: (compile {
inherit system nixpkgs crane rust-overlay extraTestEnv;
release = false;
}).garage-test;
in
{
packages =
let
packageFor = target: (compile {
inherit system git_version target;
pkgsSrc = nixpkgs;
cargo2nixOverlay = cargo2nix.overlays.default;
release = true;
}).workspace.garage { compileMode = "build"; };
in
{
# default = native release build
default = packageFor null;
# other = cross-compiled, statically-linked builds
amd64 = packageFor "x86_64-unknown-linux-musl";
i386 = packageFor "i686-unknown-linux-musl";
arm64 = packageFor "aarch64-unknown-linux-musl";
arm = packageFor "armv6l-unknown-linux-musl";
packages = {
# default = native release build
default = packageFor null true;
# <arch> = cross-compiled, statically-linked release builds
amd64 = packageFor "x86_64-unknown-linux-musl" true;
i386 = packageFor "i686-unknown-linux-musl" true;
arm64 = packageFor "aarch64-unknown-linux-musl" true;
arm = packageFor "armv6l-unknown-linux-musl" true;
# dev = native dev build
dev = packageFor null false;
# test = cargo test
tests = testWith {};
tests-lmdb = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "lmdb";
};
tests-sqlite = testWith {
GARAGE_TEST_INTEGRATION_DB_ENGINE = "sqlite";
};
};
# ---- developpment shell, for making native builds only ----
devShells =
let
shellWithPackages = (packages: (compile {
inherit system git_version;
pkgsSrc = nixpkgs;
cargo2nixOverlay = cargo2nix.overlays.default;
}).workspaceShell { inherit packages; });
targets = compile {
inherit system nixpkgs crane rust-overlay;
};
in
{
default = shellWithPackages
(with pkgs; [
rustfmt
clang
mold
]);
default = targets.devShell;
# import the full shell using `nix develop .#full`
full = shellWithPackages (with pkgs; [
rustfmt
rust-analyzer
clang
mold
# ---- extra packages for dev tasks ----
cargo-audit
cargo-outdated
cargo-machete
nixpkgs-fmt
]);
full = pkgs.mkShell {
buildInputs = with pkgs; [
targets.toolchain
protobuf
clang
mold
# ---- extra packages for dev tasks ----
rust-analyzer
cargo-audit
cargo-outdated
cargo-machete
nixpkgs-fmt
];
};
};
});
}

View file

@ -2,7 +2,7 @@
with import ./common.nix;
let
pkgs = import pkgsSrc { };
pkgs = import nixpkgs { };
lib = pkgs.lib;
/* Converts a key list and a value list to a set

View file

@ -10,9 +10,9 @@ let
flake = (import flake-compat { system = builtins.currentSystem; src = ../.; });
in
rec {
pkgsSrc = flake.defaultNix.inputs.nixpkgs;
cargo2nix = flake.defaultNix.inputs.cargo2nix;
cargo2nixOverlay = cargo2nix.overlays.default;
devShells = builtins.getAttr builtins.currentSystem flake.defaultNix.devShells;
{
flake = flake.defaultNix;
nixpkgs = flake.defaultNix.inputs.nixpkgs;
devShells = flake.defaultNix.devShells.${builtins.currentSystem};
}

View file

@ -1,83 +1,64 @@
{ system, target ? null, pkgsSrc, cargo2nixOverlay
, release ? false, git_version ? null, features ? null, }:
{
/* build inputs */
nixpkgs,
crane,
rust-overlay,
/* parameters */
system,
git_version ? null,
target ? null,
release ? false,
features ? null,
extraTestEnv ? {}
}:
let
log = v: builtins.trace v v;
# NixOS and Rust/Cargo triples do not match for ARM, fix it here.
rustTarget = if target == "armv6l-unknown-linux-musleabihf" then
"arm-unknown-linux-musleabihf"
else
target;
rustTargetEnvMap = {
"x86_64-unknown-linux-musl" = "X86_64_UNKNOWN_LINUX_MUSL";
"aarch64-unknown-linux-musl" = "AARCH64_UNKNOWN_LINUX_MUSL";
"i686-unknown-linux-musl" = "I686_UNKNOWN_LINUX_MUSL";
"arm-unknown-linux-musleabihf" = "ARM_UNKNOWN_LINUX_MUSLEABIHF";
};
pkgsNative = import nixpkgs {
inherit system;
overlays = [ (import rust-overlay) ];
};
pkgs = if target != null then
import pkgsSrc {
import nixpkgs {
inherit system;
crossSystem = {
config = target;
isStatic = true;
};
overlays = [ cargo2nixOverlay ];
overlays = [ (import rust-overlay) ];
}
else
import pkgsSrc {
inherit system;
overlays = [ cargo2nixOverlay ];
};
pkgsNative;
toolchainOptions = {
rustVersion = "1.78.0";
extraRustComponents = [ "clippy" ];
};
inherit (pkgs) lib stdenv;
/* Cargo2nix provides many overrides by default, you can take inspiration from them:
https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix
You can have a complete list of the available options by looking at the overriden object, mkcrate:
https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix
*/
packageOverrides = pkgs:
pkgs.rustBuilder.overrides.all ++ [
/* [1] We need to alter Nix hardening to make static binaries: PIE,
Position Independent Executables seems to be supported only on amd64. Having
this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms.
Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets
(only amd64 curently) through the `-static-pie` flag.
PIE is a feature used by ASLR, which helps mitigate security issues.
Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh
[2] We want to inject the git version while keeping the build deterministic.
As we do not want to consider the .git folder as part of the input source,
we ask the user (the CI often) to pass the value to Nix.
[3] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library.
However the features to do so get activated for some reason (due to a bug in cargo2nix?),
so disable them manually here.
*/
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage";
overrideAttrs = drv:
(if git_version != null then {
# [2]
preConfigure = ''
${drv.preConfigure or ""}
export GIT_VERSION="${git_version}"
'';
} else
{ }) // {
# [1]
hardeningDisable = [ "pie" ];
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "libsodium-sys";
overrideArgs = old: {
features = [ ]; # [3]
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "zstd-sys";
overrideArgs = old: {
features = [ ]; # [3]
};
})
toolchainFn = (p: p.rust-bin.stable."1.78.0".default.override {
targets = lib.optionals (target != null) [ rustTarget ];
extensions = [
"rust-src"
"rustfmt"
];
});
craneLib = (crane.mkLib pkgs).overrideToolchain toolchainFn;
src = craneLib.cleanCargoSource ../.;
/* We ship some parts of the code disabled by default by putting them behind a flag.
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
@ -87,16 +68,15 @@ let
rootFeatures = if features != null then
features
else
([ "garage/bundled-libs" "garage/lmdb" "garage/sqlite" "garage/k2v" ] ++ (if release then [
"garage/consul-discovery"
"garage/kubernetes-discovery"
"garage/metrics"
"garage/telemetry-otlp"
"garage/syslog"
] else
[ ]));
([ "bundled-libs" "lmdb" "sqlite" "k2v" ] ++ (lib.optionals release [
"consul-discovery"
"kubernetes-discovery"
"metrics"
"telemetry-otlp"
"syslog"
]));
packageFun = import ../Cargo.nix;
featuresStr = lib.concatStringsSep "," rootFeatures;
/* We compile fully static binaries with musl to simplify deployment on most systems.
When possible, we reactivate PIE hardening (see above).
@ -107,12 +87,9 @@ let
For more information on static builds, please refer to Rust's RFC 1721.
https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage
*/
codegenOpts = {
"armv6l-unknown-linux-musleabihf" = [
"target-feature=+crt-static"
"link-arg=-static"
]; # compile as dynamic with static-pie
codegenOptsMap = {
"x86_64-unknown-linux-musl" =
[ "target-feature=+crt-static" "link-arg=-static-pie" ];
"aarch64-unknown-linux-musl" = [
"target-feature=+crt-static"
"link-arg=-static"
@ -121,18 +98,95 @@ let
"target-feature=+crt-static"
"link-arg=-static"
]; # segfault with static-pie
"x86_64-unknown-linux-musl" =
[ "target-feature=+crt-static" "link-arg=-static-pie" ];
"armv6l-unknown-linux-musleabihf" = [
"target-feature=+crt-static"
"link-arg=-static"
]; # compile as dynamic with static-pie
};
# NixOS and Rust/Cargo triples do not match for ARM, fix it here.
rustTarget = if target == "armv6l-unknown-linux-musleabihf" then
"arm-unknown-linux-musleabihf"
else
target;
codegenOpts = if target != null then codegenOptsMap.${target} else [
"link-arg=-fuse-ld=mold"
];
in pkgs.rustBuilder.makePackageSet ({
inherit release packageFun packageOverrides codegenOpts rootFeatures;
target = rustTarget;
workspaceSrc = pkgs.lib.cleanSource ../.;
} // toolchainOptions)
commonArgs =
{
inherit src;
pname = "garage";
version = "dev";
strictDeps = true;
cargoExtraArgs = "--locked --features ${featuresStr}";
cargoTestExtraArgs = "--workspace";
nativeBuildInputs = [
pkgsNative.protobuf
pkgs.stdenv.cc
] ++ lib.optionals (target == null) [
pkgs.clang
pkgs.mold
];
CARGO_PROFILE = if release then "release" else "dev";
CARGO_BUILD_RUSTFLAGS =
lib.concatStringsSep
" "
(builtins.map (flag: "-C ${flag}") codegenOpts);
}
//
(if rustTarget != null then {
CARGO_BUILD_TARGET = rustTarget;
"CARGO_TARGET_${rustTargetEnvMap.${rustTarget}}_LINKER" = "${stdenv.cc.targetPrefix}cc";
HOST_CC = "${stdenv.cc.nativePrefix}cc";
TARGET_CC = "${stdenv.cc.targetPrefix}cc";
} else {
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER = "clang";
});
in rec {
toolchain = toolchainFn pkgs;
devShell = pkgs.mkShell {
buildInputs = [
toolchain
] ++ (with pkgs; [
protobuf
clang
mold
]);
};
# ---- building garage ----
garage-deps = craneLib.buildDepsOnly commonArgs;
garage = craneLib.buildPackage (commonArgs // {
cargoArtifacts = garage-deps;
doCheck = false;
} //
(if git_version != null then {
version = git_version;
GIT_VERSION = git_version;
} else {}));
# ---- testing garage ----
garage-test-bin = craneLib.cargoBuild (commonArgs // {
cargoArtifacts = garage-deps;
pname = "garage-tests";
CARGO_PROFILE = "test";
cargoExtraArgs = "${commonArgs.cargoExtraArgs} --tests --workspace";
doCheck = false;
});
garage-test = craneLib.cargoTest (commonArgs // {
cargoArtifacts = garage-test-bin;
nativeBuildInputs = commonArgs.nativeBuildInputs ++ [
pkgs.cacert
];
} // extraTestEnv);
}

View file

@ -7,7 +7,12 @@ if [ "$#" -ne 1 ]; then
exit 2
fi
if file $1 | grep 'dynamically linked' 2>&1; then
if [ ! -x "$1" ]; then
echo "[fail] $1 does not exist or is not an executable"
exit 1
fi
if file "$1" | grep 'dynamically linked' 2>&1; then
echo "[fail] $1 is dynamic"
exit 1
fi

View file

@ -3,7 +3,7 @@
with import ./nix/common.nix;
let
pkgs = import pkgsSrc {
pkgs = import nixpkgs {
inherit system;
};
winscp = (import ./nix/winscp.nix) pkgs;
@ -39,7 +39,7 @@ in
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 cp \
./result-bin/bin/garage \
./result/bin/garage \
s3://garagehq.deuxfleurs.fr/_releases/''${CI_COMMIT_TAG:-$CI_COMMIT_SHA}/''${TARGET}/garage
}

View file

@ -279,8 +279,7 @@ impl DataLayout {
u16::from_be_bytes([
hash.as_slice()[HASH_DRIVE_BYTES.0],
hash.as_slice()[HASH_DRIVE_BYTES.1],
]) as usize
% DRIVE_NPART
]) as usize % DRIVE_NPART
}
fn block_dir_from(&self, hash: &Hash, dir: &PathBuf) -> PathBuf {

View file

@ -279,8 +279,7 @@ impl<'a> LockedHelper<'a> {
.local_aliases
.get(alias_name)
.cloned()
.flatten()
!= Some(bucket_id)
.flatten() != Some(bucket_id)
{
return Err(GarageError::Message(format!(
"Bucket {:?} does not have alias {} in namespace of key {}",