Deuxfleurs/garage
41
56
Fork 73
Code Issues 98 Pull requests 14 Releases 30 Wiki Activity

Apply nixfmt to all .nix files; fix devshell and add it to cache
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse source
This commit is contained in:
Alex 2023-01-26 12:20:12 +01:00
parent 1311742fe0
commit f251b4721f
10 changed files with 551 additions and 521 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
default.nix
View file

@ -1,7 +1,4 @@
{
system ? builtins.currentSystem,
git_version ? null,
}:
{ system ? builtins.currentSystem, git_version ? null, }:
with import ./nix/common.nix;
@ -13,22 +10,21 @@ let
debug = (compile {
inherit system target git_version pkgsSrc cargo2nixOverlay;
release = false;
}).workspace.garage {
compileMode = "build";
};
}).workspace.garage { compileMode = "build"; };
release = (compile {
inherit system target git_version pkgsSrc cargo2nixOverlay;
release = true;
}).workspace.garage {
compileMode = "build";
};
}).workspace.garage { compileMode = "build"; };
});
test = (rustPkgs: pkgs.symlinkJoin {
name ="garage-tests";
paths = builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) (builtins.attrNames rustPkgs.workspace);
});
test = (rustPkgs:
pkgs.symlinkJoin {
name = "garage-tests";
paths =
builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; })
(builtins.attrNames rustPkgs.workspace);
});
in {
pkgs = {
@ -55,8 +51,6 @@ in {
inherit system git_version pkgsSrc cargo2nixOverlay;
target = "x86_64-unknown-linux-musl";
compiler = "clippy";
}).workspace.garage {
compileMode = "build";
};
}).workspace.garage { compileMode = "build"; };
};
}

12
flake.nix
View file

@ -1,7 +1,9 @@
{
description = "Garage, an S3-compatible distributed object store for self-hosted deployments";
description =
"Garage, an S3-compatible distributed object store for self-hosted deployments";
inputs.nixpkgs.url = "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555";
inputs.nixpkgs.url =
"github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555";
inputs.cargo2nix = {
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
@ -24,13 +26,11 @@
release = true;
}).workspace.garage { compileMode = "build"; };
};
devShell = ((compile {
devShell = (compile {
inherit system git_version;
pkgsSrc = nixpkgs;
cargo2nixOverlay = cargo2nix.overlays.default;
release = false;
}).workspaceShell {
packages = [ pkgs.rustfmt cargo2nix.packages.${system}.default ];
});
}).workspaceShell { packages = [ pkgs.rustfmt ]; };
});
}

247
nix/build_index.nix
View file

@ -1,10 +1,8 @@
{
path ? "/../aws-list.txt",
}:
{ path ? "/../aws-list.txt", }:
with import ./common.nix;
let
pkgs = import pkgsSrc {};
let
pkgs = import pkgsSrc { };
lib = pkgs.lib;
/* Converts a key list and a value list to a set
@ -13,139 +11,182 @@ let
listToSet [ "name" "version" ] [ "latex" "3.14" ]
=> { name = "latex"; version = "3.14"; }
*/
listToSet = keys: values:
builtins.listToAttrs
(lib.zipListsWith
(a: b: { name = a; value = b; })
keys
values);
listToSet = keys: values:
builtins.listToAttrs (lib.zipListsWith (a: b: {
name = a;
value = b;
}) keys values);
/* Says if datetime a is more recent than datetime b
Example:
cmpDate { date = "2021-09-10"; time = "22:12:15"; } { date = "2021-02-03"; time = "23:54:12"; }
=> true
Example:
cmpDate { date = "2021-09-10"; time = "22:12:15"; } { date = "2021-02-03"; time = "23:54:12"; }
=> true
*/
cmpDate = a: b:
let da = (builtins.head a.builds).date;
db = (builtins.head b.builds).date;
in
if da == db then (builtins.head a.builds).time > (builtins.head b.builds).time
else da > db;
cmpDate = a: b:
let
da = (builtins.head a.builds).date;
db = (builtins.head b.builds).date;
in if da == db then
(builtins.head a.builds).time > (builtins.head b.builds).time
else
da > db;
/* Pretty platforms */
# Pretty platforms
prettyPlatform = name:
if name == "aarch64-unknown-linux-musl" then "linux/arm64"
else if name == "armv6l-unknown-linux-musleabihf" then "linux/arm"
else if name == "x86_64-unknown-linux-musl" then "linux/amd64"
else if name == "i686-unknown-linux-musl" then "linux/386"
else name;
if name == "aarch64-unknown-linux-musl" then
"linux/arm64"
else if name == "armv6l-unknown-linux-musleabihf" then
"linux/arm"
else if name == "x86_64-unknown-linux-musl" then
"linux/amd64"
else if name == "i686-unknown-linux-musl" then
"linux/386"
else
name;
/* Parsing */
# Parsing
list = builtins.readFile (./. + path);
entries = lib.splitString "\n" list;
elems = builtins.filter
(e: (builtins.length e) == 4)
(map
(x: builtins.filter (e: e != "") (lib.splitString " " x))
entries);
elems = builtins.filter (e: (builtins.length e) == 4)
(map (x: builtins.filter (e: e != "") (lib.splitString " " x)) entries);
keys = ["date" "time" "size" "path"];
keys = [ "date" "time" "size" "path" ];
parsed = map (entry: listToSet keys entry) elems;
subkeys = ["root" "version" "platform" "binary" ];
builds = map (entry: entry // listToSet subkeys (lib.splitString "/" entry.path) // { url = "https://garagehq.deuxfleurs.fr/" + entry.path; }) parsed;
subkeys = [ "root" "version" "platform" "binary" ];
builds = map (entry:
entry // listToSet subkeys (lib.splitString "/" entry.path) // {
url = "https://garagehq.deuxfleurs.fr/" + entry.path;
}) parsed;
/* Aggregation */
builds_per_version = lib.foldl (acc: v: acc // { ${v.version} = if builtins.hasAttr v.version acc then acc.${v.version} ++ [ v ] else [ v ]; }) {} builds;
# Aggregation
builds_per_version = lib.foldl (acc: v:
acc // {
${v.version} = if builtins.hasAttr v.version acc then
acc.${v.version} ++ [ v ]
else
[ v ];
}) { } builds;
versions = builtins.attrNames builds_per_version;
versions_release = builtins.filter (x: builtins.match "v[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?" x != null) versions;
versions_commit = builtins.filter (x: builtins.match "[0-9a-f]{40}" x != null) versions;
versions_extra = lib.subtractLists (versions_release ++ versions_commit) versions;
versions_release = builtins.filter
(x: builtins.match "v[0-9]+.[0-9]+.[0-9]+(.[0-9]+)?" x != null) versions;
versions_commit =
builtins.filter (x: builtins.match "[0-9a-f]{40}" x != null) versions;
versions_extra =
lib.subtractLists (versions_release ++ versions_commit) versions;
sorted_builds = [
{
name = "Release";
hide = false;
type = "tag";
description = "Release builds are the official builds, they are tailored for productions and are the most tested.";
builds = builtins.sort (a: b: a.version > b.version) (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_release);
description =
"Release builds are the official builds, they are tailored for productions and are the most tested.";
builds = builtins.sort (a: b: a.version > b.version) (map (x: {
version = x;
builds = builtins.getAttr x builds_per_version;
}) versions_release);
}
{
name = "Extra";
hide = true;
type = "tag";
description = "Extra builds are built on demand to test a specific feature or a specific need.";
builds = builtins.sort cmpDate (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_extra);
description =
"Extra builds are built on demand to test a specific feature or a specific need.";
builds = builtins.sort cmpDate (map (x: {
version = x;
builds = builtins.getAttr x builds_per_version;
}) versions_extra);
}
{
name = "Development";
hide = true;
type = "commit";
description = "Development builds are built periodically. Use them if you want to test a specific feature that is not yet released.";
builds = builtins.sort cmpDate (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_commit);
description =
"Development builds are built periodically. Use them if you want to test a specific feature that is not yet released.";
builds = builtins.sort cmpDate (map (x: {
version = x;
builds = builtins.getAttr x builds_per_version;
}) versions_commit);
}
];
json = pkgs.writeTextDir "share/_releases.json" (builtins.toJSON sorted_builds);
json =
pkgs.writeTextDir "share/_releases.json" (builtins.toJSON sorted_builds);
html = pkgs.writeTextDir "share/_releases.html" ''
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>Garage releases</title>
<style>
html, body { margin:0; padding: 0 }
body { font-family: 'Helvetica', Sans; }
section { margin: 1rem; }
ul { padding:0; margin: 0.2rem }
li {
border-radius: 0.2rem;
display: inline;
border: 2px #0b5d83 solid;
padding: 0.5rem;
line-height: 3rem;
color: #0b5d83;
}
li:hover { background-color: #0b5d83; color: #fff; }
li a, li a:hover { color: inherit; text-decoration: none }
</style>
</head>
<body>
${ builtins.toString (lib.forEach sorted_builds (r: ''
<section>
<h2>${r.name} builds</h2>
<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<title>Garage releases</title>
<style>
html, body { margin:0; padding: 0 }
body { font-family: 'Helvetica', Sans; }
section { margin: 1rem; }
ul { padding:0; margin: 0.2rem }
li {
border-radius: 0.2rem;
display: inline;
border: 2px #0b5d83 solid;
padding: 0.5rem;
line-height: 3rem;
color: #0b5d83;
}
li:hover { background-color: #0b5d83; color: #fff; }
li a, li a:hover { color: inherit; text-decoration: none }
</style>
</head>
<body>
${
builtins.toString (lib.forEach sorted_builds (r: ''
<section>
<h2>${r.name} builds</h2>
<p>${r.description}</p>
<p>${r.description}</p>
${if r.hide then "<details><summary>Show ${r.name} builds</summary>" else ""}
${ builtins.toString (lib.forEach r.builds (x: ''
<h3> ${x.version} (${(builtins.head x.builds).date}) </h3>
<p>See this build on</p>
<p> Binaries:
<ul>
${ builtins.toString (lib.forEach x.builds (b: ''
<li><a href="/${b.path}">${prettyPlatform b.platform}</a></li>
''))}
</ul></p>
<p> Sources:
<ul>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/src/${r.type}/${x.version}">gitea</a></li>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.zip">.zip</a></li>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.tar.gz">.tar.gz</a></li>
</ul></p>
'')) }
${ if builtins.length r.builds == 0 then "<em>There is no build for this category</em>" else "" }
${if r.hide then "</details>" else ""}
</section>
''))}
</body>
</html>
'';
in
pkgs.symlinkJoin {
name = "releases";
paths = [ json html ];
}
${
if r.hide then
"<details><summary>Show ${r.name} builds</summary>"
else
""
}
${
builtins.toString (lib.forEach r.builds (x: ''
<h3> ${x.version} (${(builtins.head x.builds).date}) </h3>
<p>See this build on</p>
<p> Binaries:
<ul>
${builtins.toString (lib.forEach x.builds (b: ''
<li><a href="/${b.path}">${
prettyPlatform b.platform
}</a></li>
''))}
</ul></p>
<p> Sources:
<ul>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/src/${r.type}/${x.version}">gitea</a></li>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.zip">.zip</a></li>
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.tar.gz">.tar.gz</a></li>
</ul></p>
''))
}
${
if builtins.length r.builds == 0 then
"<em>There is no build for this category</em>"
else
""
}
${if r.hide then "</details>" else ""}
</section>
''))
}
</body>
</html>
'';
in pkgs.symlinkJoin {
name = "releases";
paths = [ json html ];
}

11
nix/common.nix
View file

@ -1,10 +1,9 @@
rec {
/*
* Fixed dependencies
*/
# * Fixed dependencies
pkgsSrc = fetchTarball {
# As of 2022-10-13
url = "https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip";
url =
"https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip";
sha256 = "1bz632psfbpmicyzjb8b4265y50shylccvfm6ry6mgnv5hvz324s";
};
cargo2nixSrc = fetchGit {
@ -14,9 +13,7 @@ rec {
rev = "a7a61179b66054904ef6a195d8da736eaaa06c36";
};
/*
* Shared objects
*/
# * Shared objects
cargo2nix = import cargo2nixSrc;
cargo2nixOverlay = cargo2nix.overlays.default;
}

417
nix/compile.nix
View file

@ -1,227 +1,240 @@
{
system,
target ? null,
pkgsSrc,
cargo2nixOverlay,
compiler ? "rustc",
release ? false,
git_version ? null,
features ? null,
}:
{ system, target ? null, pkgsSrc, cargo2nixOverlay, compiler ? "rustc"
, release ? false, git_version ? null, features ? null, }:
let
log = v: builtins.trace v v;
pkgs =
if target != null then
import pkgsSrc {
inherit system;
crossSystem = {
config = target;
isStatic = true;
};
overlays = [ cargo2nixOverlay ];
}
else
import pkgsSrc {
inherit system;
overlays = [ cargo2nixOverlay ];
};
/*
Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
This is fine for 64-bit platforms, but for 32-bit platforms, we need our own Rust
to avoid incompatibilities with time_t between different versions of musl
(>= 1.2.0 shipped by NixOS, < 1.2.0 with which rustc was built), which lead to compilation breakage.
So we want a Rust release that is bound to our Nix repository to avoid these problems.
See here for more info: https://musl.libc.org/time64.html
Because Cargo2nix does not support the Rust environment shipped by NixOS,
we emulate the structure of the Rust object created by rustOverlay.
In practise, rustOverlay ships rustc+cargo in a single derivation while
NixOS ships them in separate ones. We reunite them with symlinkJoin.
*/
toolchainOptions =
if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
rustVersion = "1.63.0";
extraRustComponents = [ "clippy" ];
} else {
rustToolchain = pkgs.symlinkJoin {
name = "rust-static-toolchain-${target}";
paths = [
pkgs.rustPlatform.rust.cargo
pkgs.rustPlatform.rust.rustc
# clippy not needed, it only runs on amd64
];
pkgs = if target != null then
import pkgsSrc {
inherit system;
crossSystem = {
config = target;
isStatic = true;
};
overlays = [ cargo2nixOverlay ];
}
else
import pkgsSrc {
inherit system;
overlays = [ cargo2nixOverlay ];
};
buildEnv = (drv: {
rustc = drv.setBuildEnv;
clippy = ''
${drv.setBuildEnv or "" }
echo
echo --- BUILDING WITH CLIPPY ---
echo
export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings"
export RUSTC="''${CLIPPY_DRIVER}"
'';
}.${compiler});
/*
Cargo2nix provides many overrides by default, you can take inspiration from them:
https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix
You can have a complete list of the available options by looking at the overriden object, mkcrate:
https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix
/* Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
This is fine for 64-bit platforms, but for 32-bit platforms, we need our own Rust
to avoid incompatibilities with time_t between different versions of musl
(>= 1.2.0 shipped by NixOS, < 1.2.0 with which rustc was built), which lead to compilation breakage.
So we want a Rust release that is bound to our Nix repository to avoid these problems.
See here for more info: https://musl.libc.org/time64.html
Because Cargo2nix does not support the Rust environment shipped by NixOS,
we emulate the structure of the Rust object created by rustOverlay.
In practise, rustOverlay ships rustc+cargo in a single derivation while
NixOS ships them in separate ones. We reunite them with symlinkJoin.
*/
packageOverrides = pkgs: pkgs.rustBuilder.overrides.all ++ [
/*
[1] We add some logic to compile our crates with clippy, it provides us many additional lints
toolchainOptions = if target == null || target == "x86_64-unknown-linux-musl"
|| target == "aarch64-unknown-linux-musl" then {
rustVersion = "1.63.0";
extraRustComponents = [ "clippy" ];
} else {
rustToolchain = pkgs.symlinkJoin {
name = "rust-static-toolchain-${target}";
paths = [
pkgs.rustPlatform.rust.cargo
pkgs.rustPlatform.rust.rustc
# clippy not needed, it only runs on amd64
];
};
};
[2] We need to alter Nix hardening to make static binaries: PIE,
Position Independent Executables seems to be supported only on amd64. Having
this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms.
Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets
(only amd64 curently) through the `-static-pie` flag.
PIE is a feature used by ASLR, which helps mitigate security issues.
Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh
buildEnv = (drv:
{
rustc = drv.setBuildEnv;
clippy = ''
${drv.setBuildEnv or ""}
echo
echo --- BUILDING WITH CLIPPY ---
echo
[3] We want to inject the git version while keeping the build deterministic.
As we do not want to consider the .git folder as part of the input source,
we ask the user (the CI often) to pass the value to Nix.
export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings"
export RUSTC="''${CLIPPY_DRIVER}"
'';
}.${compiler});
[4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library.
However the features to do so get activated for some reason (due to a bug in cargo2nix?),
so disable them manually here.
*/
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage";
overrideAttrs = drv:
(if git_version != null then {
/* [3] */ preConfigure = ''
${drv.preConfigure or ""}
export GIT_VERSION="${git_version}"
'';
} else {})
//
{
/* [1] */ setBuildEnv = (buildEnv drv);
/* [2] */ hardeningDisable = [ "pie" ];
};
})
/* Cargo2nix provides many overrides by default, you can take inspiration from them:
https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_rpc";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_db";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_util";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_table";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_block";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_model";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_api";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_web";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "k2v-client";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "libsodium-sys";
overrideArgs = old: {
features = [ ]; /* [4] */
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "zstd-sys";
overrideArgs = old: {
features = [ ]; /* [4] */
};
})
];
/*
We ship some parts of the code disabled by default by putting them behind a flag.
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
But we want to ship these additional features when we release Garage.
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
You can have a complete list of the available options by looking at the overriden object, mkcrate:
https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix
*/
rootFeatures = if features != null then features else
([
"garage/bundled-libs"
"garage/sled"
"garage/k2v"
] ++ (if release then [
"garage/consul-discovery"
"garage/kubernetes-discovery"
"garage/metrics"
"garage/telemetry-otlp"
"garage/lmdb"
"garage/sqlite"
] else []));
packageOverrides = pkgs:
pkgs.rustBuilder.overrides.all ++ [
/* [1] We add some logic to compile our crates with clippy, it provides us many additional lints
[2] We need to alter Nix hardening to make static binaries: PIE,
Position Independent Executables seems to be supported only on amd64. Having
this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms.
Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets
(only amd64 curently) through the `-static-pie` flag.
PIE is a feature used by ASLR, which helps mitigate security issues.
Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh
[3] We want to inject the git version while keeping the build deterministic.
As we do not want to consider the .git folder as part of the input source,
we ask the user (the CI often) to pass the value to Nix.
[4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library.
However the features to do so get activated for some reason (due to a bug in cargo2nix?),
so disable them manually here.
*/
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage";
overrideAttrs = drv:
(if git_version != null then {
# [3]
preConfigure = ''
${drv.preConfigure or ""}
export GIT_VERSION="${git_version}"
'';
} else
{ }) // {
# [1]
setBuildEnv = (buildEnv drv);
# [2]
hardeningDisable = [ "pie" ];
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_rpc";
overrideAttrs = drv: { # [1]
setBuildEnv = (buildEnv drv);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_db";
overrideAttrs = drv: { # [1]
setBuildEnv = (buildEnv drv);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_util";
overrideAttrs = drv: { # [1]
setBuildEnv = (buildEnv drv);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_table";
overrideAttrs = drv: { # [1]
setBuildEnv = (buildEnv drv);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_block";
overrideAttrs = drv: { # [1]
setBuildEnv = (buildEnv drv);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_model";
overrideAttrs = drv: { # [1]
setBuildEnv = (buildEnv drv);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_api";
overrideAttrs = drv: { # [1]
setBuildEnv = (buildEnv drv);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_web";
overrideAttrs = drv: { # [1]
setBuildEnv = (buildEnv drv);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "k2v-client";
overrideAttrs = drv: { # [1]
setBuildEnv = (buildEnv drv);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "libsodium-sys";
overrideArgs = old: {
features = [ ]; # [4]
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "zstd-sys";
overrideArgs = old: {
features = [ ]; # [4]
};
})
];
/* We ship some parts of the code disabled by default by putting them behind a flag.
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
But we want to ship these additional features when we release Garage.
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
*/
rootFeatures = if features != null then
features
else
([ "garage/bundled-libs" "garage/sled" "garage/k2v" ] ++ (if release then [
"garage/consul-discovery"
"garage/kubernetes-discovery"
"garage/metrics"
"garage/telemetry-otlp"
"garage/lmdb"
"garage/sqlite"
] else
[ ]));
packageFun = import ../Cargo.nix;
/*
We compile fully static binaries with musl to simplify deployment on most systems.
When possible, we reactivate PIE hardening (see above).
/* We compile fully static binaries with musl to simplify deployment on most systems.
When possible, we reactivate PIE hardening (see above).
Also, if you set the RUSTFLAGS environment variable, the following parameters will
be ignored.
Also, if you set the RUSTFLAGS environment variable, the following parameters will
be ignored.
For more information on static builds, please refer to Rust's RFC 1721.
https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage
For more information on static builds, please refer to Rust's RFC 1721.
https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage
*/
codegenOpts = {
"armv6l-unknown-linux-musleabihf" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* compile as dynamic with static-pie */
"aarch64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */
"i686-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */
"x86_64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static-pie" ];
"armv6l-unknown-linux-musleabihf" = [
"target-feature=+crt-static"
"link-arg=-static"
]; # compile as dynamic with static-pie
"aarch64-unknown-linux-musl" = [
"target-feature=+crt-static"
"link-arg=-static"
]; # segfault with static-pie
"i686-unknown-linux-musl" = [
"target-feature=+crt-static"
"link-arg=-static"
]; # segfault with static-pie
"x86_64-unknown-linux-musl" =
[ "target-feature=+crt-static" "link-arg=-static-pie" ];
};
/*
NixOS and Rust/Cargo triples do not match for ARM, fix it here.
*/
rustTarget = if target == "armv6l-unknown-linux-musleabihf"
then "arm-unknown-linux-musleabihf"
else target;
# NixOS and Rust/Cargo triples do not match for ARM, fix it here.
rustTarget = if target == "armv6l-unknown-linux-musleabihf" then
"arm-unknown-linux-musleabihf"
else
target;
in
pkgs.rustBuilder.makePackageSet ({
inherit release packageFun packageOverrides codegenOpts rootFeatures;
target = rustTarget;
} // toolchainOptions)
in pkgs.rustBuilder.makePackageSet ({
inherit release packageFun packageOverrides codegenOpts rootFeatures;
target = rustTarget;
} // toolchainOptions)

3
nix/kaniko.nix
View file

@ -15,7 +15,8 @@ pkgs.buildGoModule rec {
checkPhase = "true";
meta = with pkgs.lib; {
description = "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster.";
description =
"kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster.";
homepage = "https://github.com/GoogleContainerTools/kaniko";
license = licenses.asl20;
platforms = platforms.linux;

3
nix/manifest-tool.nix
View file

@ -15,7 +15,8 @@ pkgs.buildGoModule rec {
checkPhase = "true";
meta = with pkgs.lib; {
description = "Command line tool to create and query container image manifest list/indexes";
description =
"Command line tool to create and query container image manifest list/indexes";
homepage = "https://github.com/estesp/manifest-tool";
license = licenses.asl20;
platforms = platforms.linux;

38
nix/toolchain.nix
View file

@ -1,6 +1,4 @@
{
system ? builtins.currentSystem,
}:
{ system ? builtins.currentSystem, }:
with import ./common.nix;
@ -11,27 +9,23 @@ let
#"aarch64-unknown-linux-musl"
"armv6l-unknown-linux-musleabihf"
];
pkgsList = builtins.map (target: import pkgsSrc {
inherit system;
crossSystem = {
config = target;
isStatic = true;
};
overlays = [ cargo2nixOverlay ];
}) platforms;
pkgsHost = import pkgsSrc {};
pkgsList = builtins.map (target:
import pkgsSrc {
inherit system;
crossSystem = {
config = target;
isStatic = true;
};
overlays = [ cargo2nixOverlay ];
}) platforms;
pkgsHost = import pkgsSrc { };
lib = pkgsHost.lib;
kaniko = (import ./kaniko.nix) pkgsHost;
winscp = (import ./winscp.nix) pkgsHost;
manifestTool = (import ./manifest-tool.nix) pkgsHost;
in
lib.flatten (builtins.map (pkgs: [
pkgs.rustPlatform.rust.rustc
pkgs.rustPlatform.rust.cargo
pkgs.buildPackages.stdenv.cc
]) pkgsList) ++ [
kaniko
winscp
manifestTool
]
in lib.flatten (builtins.map (pkgs: [
pkgs.rustPlatform.rust.rustc
pkgs.rustPlatform.rust.cargo
pkgs.buildPackages.stdenv.cc
]) pkgsList) ++ [ kaniko winscp manifestTool ]

10
nix/winscp.nix
View file

@ -11,12 +11,12 @@ pkgs.stdenv.mkDerivation rec {
};
buildPhase = ''
cat > winscp <<EOF
#!${pkgs.bash}/bin/bash
cat > winscp <<EOF
#!${pkgs.bash}/bin/bash
WINEDEBUG=-all
${pkgs.winePackages.minimal}/bin/wine $out/opt/WinSCP.com
EOF
WINEDEBUG=-all
${pkgs.winePackages.minimal}/bin/wine $out/opt/WinSCP.com
EOF
'';
installPhase = ''

303
shell.nix
View file

@ -1,6 +1,4 @@
{
system ? builtins.currentSystem,
}:
{ system ? builtins.currentSystem, }:
with import ./nix/common.nix;
@ -13,161 +11,152 @@ let
manifest-tool = (import ./nix/manifest-tool.nix) pkgs;
winscp = (import ./nix/winscp.nix) pkgs;
in
{
/* --- Rust Shell ---
* Use it to compile Garage
*/
rust = pkgs.mkShell {
nativeBuildInputs = [
#pkgs.rustPlatform.rust.rustc
pkgs.rustPlatform.rust.cargo
#pkgs.clippy
pkgs.rustfmt
#pkgs.perl
#pkgs.protobuf
#pkgs.pkg-config
#pkgs.openssl
pkgs.file
#cargo2nix.packages.x86_64-linux.cargo2nix
];
};
in {
# --- Rust Shell ---
# Use it to compile Garage
rust = pkgs.mkShell {
nativeBuildInputs = [
#pkgs.rustPlatform.rust.rustc
pkgs.rustPlatform.rust.cargo
#pkgs.clippy
pkgs.rustfmt
#pkgs.perl
#pkgs.protobuf
#pkgs.pkg-config
#pkgs.openssl
pkgs.file
#cargo2nix.packages.x86_64-linux.cargo2nix
];
};
/* --- Integration shell ---
* Use it to test Garage with common S3 clients
*/
integration = pkgs.mkShell {
nativeBuildInputs = [
winscp
pkgs.s3cmd
pkgs.awscli2
pkgs.minio-client
pkgs.rclone
pkgs.socat
pkgs.psmisc
pkgs.which
pkgs.openssl
pkgs.curl
pkgs.jq
];
};
# --- Integration shell ---
# Use it to test Garage with common S3 clients
integration = pkgs.mkShell {
nativeBuildInputs = [
winscp
pkgs.s3cmd
pkgs.awscli2
pkgs.minio-client
pkgs.rclone
pkgs.socat
pkgs.psmisc
pkgs.which
pkgs.openssl
pkgs.curl
pkgs.jq
];
};
/* --- Release shell ---
* A shell built to make releasing easier
*/
release = pkgs.mkShell {
shellHook = ''
function refresh_toolchain {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
nix copy \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR \
$(nix-build --no-build-output --no-out-link nix/toolchain.nix))
rm /tmp/nix-signing-key.sec
# --- Release shell ---
# A shell built to make releasing easier
release = pkgs.mkShell {
shellHook = ''
function refresh_toolchain {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
nix copy \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR \
$(nix-build --no-build-output --no-out-link nix/toolchain.nix))
rm /tmp/nix-signing-key.sec
}
function refresh_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
echo "Updating cache for ''${attr}"
derivation=$(nix-instantiate --attr ''${attr})
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR ''${derivation%\!bin})
done
rm /tmp/nix-signing-key.sec
}
function refresh_flake_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in packages.x86_64-linux.default devShell.x86_64-linux; do
echo "Updating cache for ''${attr}"
derivation=$(nix path-info --derivation ".#''${attr}")
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR ''${derivation})
done
rm /tmp/nix-signing-key.sec
}
function to_s3 {
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 cp \
./result-bin/bin/garage \
s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage
}
function to_docker {
executor \
--force \
--customPlatform="''${DOCKER_PLATFORM}" \
--destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \
--context dir://`pwd` \
--verbosity=debug
}
function multiarch_docker {
manifest-tool push from-spec <(cat <<EOF
image: dxflrs/garage:''${CONTAINER_TAG}
manifests:
-
image: dxflrs/arm64_garage:''${CONTAINER_TAG}
platform:
architecture: arm64
os: linux
-
image: dxflrs/amd64_garage:''${CONTAINER_TAG}
platform:
architecture: amd64
os: linux
-
image: dxflrs/386_garage:''${CONTAINER_TAG}
platform:
architecture: 386
os: linux
-
image: dxflrs/arm_garage:''${CONTAINER_TAG}
platform:
architecture: arm
os: linux
EOF
)
}
function refresh_index {
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 ls \
--recursive \
s3://garagehq.deuxfleurs.fr/_releases/ \
> aws-list.txt
nix-build nix/build_index.nix
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 cp \
result/share/_releases.json \
s3://garagehq.deuxfleurs.fr/
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 cp \
result/share/_releases.html \
s3://garagehq.deuxfleurs.fr/
}
'';
nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ];
};
}
function refresh_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
echo "Updating cache for ''${attr}"
derivation=$(nix-instantiate --attr ''${attr})
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR ''${derivation%\!bin})
done
rm /tmp/nix-signing-key.sec
}
function refresh_flake_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in packages.x86_64-linux.default; do
echo "Updating cache for ''${attr}"
derivation=$(nix path-info --derivation ".#''${attr}")
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR ''${derivation})
done
rm /tmp/nix-signing-key.sec
}
function to_s3 {
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 cp \
./result-bin/bin/garage \
s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage
}
function to_docker {
executor \
--force \
--customPlatform="''${DOCKER_PLATFORM}" \
--destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \
--context dir://`pwd` \
--verbosity=debug
}
function multiarch_docker {
manifest-tool push from-spec <(cat <<EOF
image: dxflrs/garage:''${CONTAINER_TAG}
manifests:
-
image: dxflrs/arm64_garage:''${CONTAINER_TAG}
platform:
architecture: arm64
os: linux
-
image: dxflrs/amd64_garage:''${CONTAINER_TAG}
platform:
architecture: amd64
os: linux
-
image: dxflrs/386_garage:''${CONTAINER_TAG}
platform:
architecture: 386
os: linux
-
image: dxflrs/arm_garage:''${CONTAINER_TAG}
platform:
architecture: arm
os: linux
EOF
)
}
function refresh_index {
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 ls \
--recursive \
s3://garagehq.deuxfleurs.fr/_releases/ \
> aws-list.txt
nix-build nix/build_index.nix
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 cp \
result/share/_releases.json \
s3://garagehq.deuxfleurs.fr/
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 cp \
result/share/_releases.html \
s3://garagehq.deuxfleurs.fr/
}
'';
nativeBuildInputs = [
pkgs.awscli2
kaniko
manifest-tool
];
};
}