From f251b4721f2c016e3a90d913cb766a57a53a3d30 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 26 Jan 2023 12:20:12 +0100 Subject: [PATCH 1/3] Apply nixfmt to all .nix files; fix devshell and add it to cache --- default.nix | 28 ++- flake.nix | 12 +- nix/build_index.nix | 247 ++++++++++++++----------- nix/common.nix | 11 +- nix/compile.nix | 417 ++++++++++++++++++++++-------------------- nix/kaniko.nix | 3 +- nix/manifest-tool.nix | 3 +- nix/toolchain.nix | 38 ++-- nix/winscp.nix | 10 +- shell.nix | 303 +++++++++++++++--------------- 10 files changed, 551 insertions(+), 521 deletions(-) diff --git a/default.nix b/default.nix index 1791b3c6..ecdf6436 100644 --- a/default.nix +++ b/default.nix @@ -1,7 +1,4 @@ -{ - system ? builtins.currentSystem, - git_version ? null, -}: +{ system ? builtins.currentSystem, git_version ? null, }: with import ./nix/common.nix; @@ -13,22 +10,21 @@ let debug = (compile { inherit system target git_version pkgsSrc cargo2nixOverlay; release = false; - }).workspace.garage { - compileMode = "build"; - }; + }).workspace.garage { compileMode = "build"; }; release = (compile { inherit system target git_version pkgsSrc cargo2nixOverlay; release = true; - }).workspace.garage { - compileMode = "build"; - }; + }).workspace.garage { compileMode = "build"; }; }); - test = (rustPkgs: pkgs.symlinkJoin { - name ="garage-tests"; - paths = builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) (builtins.attrNames rustPkgs.workspace); - }); + test = (rustPkgs: + pkgs.symlinkJoin { + name = "garage-tests"; + paths = + builtins.map (key: rustPkgs.workspace.${key} { compileMode = "test"; }) + (builtins.attrNames rustPkgs.workspace); + }); in { pkgs = { @@ -55,8 +51,6 @@ in { inherit system git_version pkgsSrc cargo2nixOverlay; target = "x86_64-unknown-linux-musl"; compiler = "clippy"; - }).workspace.garage { - compileMode = "build"; - }; + }).workspace.garage { compileMode = "build"; }; }; } diff --git a/flake.nix b/flake.nix index c1d772bb..7eb9e33b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,7 +1,9 @@ { - description = "Garage, an S3-compatible distributed object store for self-hosted deployments"; + description = + "Garage, an S3-compatible distributed object store for self-hosted deployments"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555"; + inputs.nixpkgs.url = + "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555"; inputs.cargo2nix = { # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36"; @@ -24,13 +26,11 @@ release = true; }).workspace.garage { compileMode = "build"; }; }; - devShell = ((compile { + devShell = (compile { inherit system git_version; pkgsSrc = nixpkgs; cargo2nixOverlay = cargo2nix.overlays.default; release = false; - }).workspaceShell { - packages = [ pkgs.rustfmt cargo2nix.packages.${system}.default ]; - }); + }).workspaceShell { packages = [ pkgs.rustfmt ]; }; }); } diff --git a/nix/build_index.nix b/nix/build_index.nix index 4625e3ae..7cc4f62c 100644 --- a/nix/build_index.nix +++ b/nix/build_index.nix @@ -1,10 +1,8 @@ -{ - path ? "/../aws-list.txt", -}: +{ path ? "/../aws-list.txt", }: with import ./common.nix; -let - pkgs = import pkgsSrc {}; +let + pkgs = import pkgsSrc { }; lib = pkgs.lib; /* Converts a key list and a value list to a set @@ -13,139 +11,182 @@ let listToSet [ "name" "version" ] [ "latex" "3.14" ] => { name = "latex"; version = "3.14"; } */ - listToSet = keys: values: - builtins.listToAttrs - (lib.zipListsWith - (a: b: { name = a; value = b; }) - keys - values); + listToSet = keys: values: + builtins.listToAttrs (lib.zipListsWith (a: b: { + name = a; + value = b; + }) keys values); /* Says if datetime a is more recent than datetime b - Example: - cmpDate { date = "2021-09-10"; time = "22:12:15"; } { date = "2021-02-03"; time = "23:54:12"; } - => true + Example: + cmpDate { date = "2021-09-10"; time = "22:12:15"; } { date = "2021-02-03"; time = "23:54:12"; } + => true */ - cmpDate = a: b: - let da = (builtins.head a.builds).date; - db = (builtins.head b.builds).date; - in - if da == db then (builtins.head a.builds).time > (builtins.head b.builds).time - else da > db; + cmpDate = a: b: + let + da = (builtins.head a.builds).date; + db = (builtins.head b.builds).date; + in if da == db then + (builtins.head a.builds).time > (builtins.head b.builds).time + else + da > db; - /* Pretty platforms */ + # Pretty platforms prettyPlatform = name: - if name == "aarch64-unknown-linux-musl" then "linux/arm64" - else if name == "armv6l-unknown-linux-musleabihf" then "linux/arm" - else if name == "x86_64-unknown-linux-musl" then "linux/amd64" - else if name == "i686-unknown-linux-musl" then "linux/386" - else name; + if name == "aarch64-unknown-linux-musl" then + "linux/arm64" + else if name == "armv6l-unknown-linux-musleabihf" then + "linux/arm" + else if name == "x86_64-unknown-linux-musl" then + "linux/amd64" + else if name == "i686-unknown-linux-musl" then + "linux/386" + else + name; - /* Parsing */ + # Parsing list = builtins.readFile (./. + path); entries = lib.splitString "\n" list; - elems = builtins.filter - (e: (builtins.length e) == 4) - (map - (x: builtins.filter (e: e != "") (lib.splitString " " x)) - entries); + elems = builtins.filter (e: (builtins.length e) == 4) + (map (x: builtins.filter (e: e != "") (lib.splitString " " x)) entries); - keys = ["date" "time" "size" "path"]; + keys = [ "date" "time" "size" "path" ]; parsed = map (entry: listToSet keys entry) elems; - subkeys = ["root" "version" "platform" "binary" ]; - builds = map (entry: entry // listToSet subkeys (lib.splitString "/" entry.path) // { url = "https://garagehq.deuxfleurs.fr/" + entry.path; }) parsed; + subkeys = [ "root" "version" "platform" "binary" ]; + builds = map (entry: + entry // listToSet subkeys (lib.splitString "/" entry.path) // { + url = "https://garagehq.deuxfleurs.fr/" + entry.path; + }) parsed; - /* Aggregation */ - builds_per_version = lib.foldl (acc: v: acc // { ${v.version} = if builtins.hasAttr v.version acc then acc.${v.version} ++ [ v ] else [ v ]; }) {} builds; + # Aggregation + builds_per_version = lib.foldl (acc: v: + acc // { + ${v.version} = if builtins.hasAttr v.version acc then + acc.${v.version} ++ [ v ] + else + [ v ]; + }) { } builds; versions = builtins.attrNames builds_per_version; - versions_release = builtins.filter (x: builtins.match "v[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?" x != null) versions; - versions_commit = builtins.filter (x: builtins.match "[0-9a-f]{40}" x != null) versions; - versions_extra = lib.subtractLists (versions_release ++ versions_commit) versions; + versions_release = builtins.filter + (x: builtins.match "v[0-9]+.[0-9]+.[0-9]+(.[0-9]+)?" x != null) versions; + versions_commit = + builtins.filter (x: builtins.match "[0-9a-f]{40}" x != null) versions; + versions_extra = + lib.subtractLists (versions_release ++ versions_commit) versions; sorted_builds = [ { name = "Release"; hide = false; type = "tag"; - description = "Release builds are the official builds, they are tailored for productions and are the most tested."; - builds = builtins.sort (a: b: a.version > b.version) (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_release); + description = + "Release builds are the official builds, they are tailored for productions and are the most tested."; + builds = builtins.sort (a: b: a.version > b.version) (map (x: { + version = x; + builds = builtins.getAttr x builds_per_version; + }) versions_release); } { name = "Extra"; hide = true; type = "tag"; - description = "Extra builds are built on demand to test a specific feature or a specific need."; - builds = builtins.sort cmpDate (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_extra); + description = + "Extra builds are built on demand to test a specific feature or a specific need."; + builds = builtins.sort cmpDate (map (x: { + version = x; + builds = builtins.getAttr x builds_per_version; + }) versions_extra); } { name = "Development"; hide = true; type = "commit"; - description = "Development builds are built periodically. Use them if you want to test a specific feature that is not yet released."; - builds = builtins.sort cmpDate (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_commit); + description = + "Development builds are built periodically. Use them if you want to test a specific feature that is not yet released."; + builds = builtins.sort cmpDate (map (x: { + version = x; + builds = builtins.getAttr x builds_per_version; + }) versions_commit); } ]; - json = pkgs.writeTextDir "share/_releases.json" (builtins.toJSON sorted_builds); + json = + pkgs.writeTextDir "share/_releases.json" (builtins.toJSON sorted_builds); html = pkgs.writeTextDir "share/_releases.html" '' - - - - - Garage releases - - - - ${ builtins.toString (lib.forEach sorted_builds (r: '' -
-

${r.name} builds

+ + + + + Garage releases + + + + ${ + builtins.toString (lib.forEach sorted_builds (r: '' +
+

${r.name} builds

-

${r.description}

+

${r.description}

- ${if r.hide then "
Show ${r.name} builds" else ""} - ${ builtins.toString (lib.forEach r.builds (x: '' -

${x.version} (${(builtins.head x.builds).date})

-

See this build on

-

Binaries: -

-

Sources: -

- '')) } - ${ if builtins.length r.builds == 0 then "There is no build for this category" else "" } - ${if r.hide then "
" else ""} -
- ''))} - - -''; -in - pkgs.symlinkJoin { - name = "releases"; - paths = [ json html ]; - } + ${ + if r.hide then + "
Show ${r.name} builds" + else + "" + } + ${ + builtins.toString (lib.forEach r.builds (x: '' +

${x.version} (${(builtins.head x.builds).date})

+

See this build on

+

Binaries: +

+

Sources: +

+ '')) + } + ${ + if builtins.length r.builds == 0 then + "There is no build for this category" + else + "" + } + ${if r.hide then "
" else ""} +
+ '')) + } + + + ''; +in pkgs.symlinkJoin { + name = "releases"; + paths = [ json html ]; +} diff --git a/nix/common.nix b/nix/common.nix index 90e3afaf..57f354dd 100644 --- a/nix/common.nix +++ b/nix/common.nix @@ -1,10 +1,9 @@ rec { - /* - * Fixed dependencies - */ + # * Fixed dependencies pkgsSrc = fetchTarball { # As of 2022-10-13 - url = "https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip"; + url = + "https://github.com/NixOS/nixpkgs/archive/a3073c49bc0163fea6a121c276f526837672b555.zip"; sha256 = "1bz632psfbpmicyzjb8b4265y50shylccvfm6ry6mgnv5hvz324s"; }; cargo2nixSrc = fetchGit { @@ -14,9 +13,7 @@ rec { rev = "a7a61179b66054904ef6a195d8da736eaaa06c36"; }; - /* - * Shared objects - */ + # * Shared objects cargo2nix = import cargo2nixSrc; cargo2nixOverlay = cargo2nix.overlays.default; } diff --git a/nix/compile.nix b/nix/compile.nix index 3ea5035e..54e920db 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -1,227 +1,240 @@ -{ - system, - target ? null, - pkgsSrc, - cargo2nixOverlay, - compiler ? "rustc", - release ? false, - git_version ? null, - features ? null, -}: +{ system, target ? null, pkgsSrc, cargo2nixOverlay, compiler ? "rustc" +, release ? false, git_version ? null, features ? null, }: let log = v: builtins.trace v v; - pkgs = - if target != null then - import pkgsSrc { - inherit system; - crossSystem = { - config = target; - isStatic = true; - }; - overlays = [ cargo2nixOverlay ]; - } - else - import pkgsSrc { - inherit system; - overlays = [ cargo2nixOverlay ]; - }; - - /* - Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases. - This is fine for 64-bit platforms, but for 32-bit platforms, we need our own Rust - to avoid incompatibilities with time_t between different versions of musl - (>= 1.2.0 shipped by NixOS, < 1.2.0 with which rustc was built), which lead to compilation breakage. - So we want a Rust release that is bound to our Nix repository to avoid these problems. - See here for more info: https://musl.libc.org/time64.html - Because Cargo2nix does not support the Rust environment shipped by NixOS, - we emulate the structure of the Rust object created by rustOverlay. - In practise, rustOverlay ships rustc+cargo in a single derivation while - NixOS ships them in separate ones. We reunite them with symlinkJoin. - */ - toolchainOptions = - if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then { - rustVersion = "1.63.0"; - extraRustComponents = [ "clippy" ]; - } else { - rustToolchain = pkgs.symlinkJoin { - name = "rust-static-toolchain-${target}"; - paths = [ - pkgs.rustPlatform.rust.cargo - pkgs.rustPlatform.rust.rustc - # clippy not needed, it only runs on amd64 - ]; + pkgs = if target != null then + import pkgsSrc { + inherit system; + crossSystem = { + config = target; + isStatic = true; }; + overlays = [ cargo2nixOverlay ]; + } + else + import pkgsSrc { + inherit system; + overlays = [ cargo2nixOverlay ]; }; - - buildEnv = (drv: { - rustc = drv.setBuildEnv; - clippy = '' - ${drv.setBuildEnv or "" } - echo - echo --- BUILDING WITH CLIPPY --- - echo - - export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings" - export RUSTC="''${CLIPPY_DRIVER}" - ''; - }.${compiler}); - - /* - Cargo2nix provides many overrides by default, you can take inspiration from them: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix - - You can have a complete list of the available options by looking at the overriden object, mkcrate: - https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix + /* Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases. + This is fine for 64-bit platforms, but for 32-bit platforms, we need our own Rust + to avoid incompatibilities with time_t between different versions of musl + (>= 1.2.0 shipped by NixOS, < 1.2.0 with which rustc was built), which lead to compilation breakage. + So we want a Rust release that is bound to our Nix repository to avoid these problems. + See here for more info: https://musl.libc.org/time64.html + Because Cargo2nix does not support the Rust environment shipped by NixOS, + we emulate the structure of the Rust object created by rustOverlay. + In practise, rustOverlay ships rustc+cargo in a single derivation while + NixOS ships them in separate ones. We reunite them with symlinkJoin. */ - packageOverrides = pkgs: pkgs.rustBuilder.overrides.all ++ [ - /* - [1] We add some logic to compile our crates with clippy, it provides us many additional lints + toolchainOptions = if target == null || target == "x86_64-unknown-linux-musl" + || target == "aarch64-unknown-linux-musl" then { + rustVersion = "1.63.0"; + extraRustComponents = [ "clippy" ]; + } else { + rustToolchain = pkgs.symlinkJoin { + name = "rust-static-toolchain-${target}"; + paths = [ + pkgs.rustPlatform.rust.cargo + pkgs.rustPlatform.rust.rustc + # clippy not needed, it only runs on amd64 + ]; + }; + }; - [2] We need to alter Nix hardening to make static binaries: PIE, - Position Independent Executables seems to be supported only on amd64. Having - this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. - Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets - (only amd64 curently) through the `-static-pie` flag. - PIE is a feature used by ASLR, which helps mitigate security issues. - Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh + buildEnv = (drv: + { + rustc = drv.setBuildEnv; + clippy = '' + ${drv.setBuildEnv or ""} + echo + echo --- BUILDING WITH CLIPPY --- + echo - [3] We want to inject the git version while keeping the build deterministic. - As we do not want to consider the .git folder as part of the input source, - we ask the user (the CI often) to pass the value to Nix. + export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings" + export RUSTC="''${CLIPPY_DRIVER}" + ''; + }.${compiler}); - [4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library. - However the features to do so get activated for some reason (due to a bug in cargo2nix?), - so disable them manually here. - */ - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage"; - overrideAttrs = drv: - (if git_version != null then { - /* [3] */ preConfigure = '' - ${drv.preConfigure or ""} - export GIT_VERSION="${git_version}" - ''; - } else {}) - // - { - /* [1] */ setBuildEnv = (buildEnv drv); - /* [2] */ hardeningDisable = [ "pie" ]; - }; - }) + /* Cargo2nix provides many overrides by default, you can take inspiration from them: + https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_rpc"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_db"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_util"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_table"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_block"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_model"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_api"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_web"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "k2v-client"; - overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "libsodium-sys"; - overrideArgs = old: { - features = [ ]; /* [4] */ - }; - }) - - (pkgs.rustBuilder.rustLib.makeOverride { - name = "zstd-sys"; - overrideArgs = old: { - features = [ ]; /* [4] */ - }; - }) - ]; - - /* - We ship some parts of the code disabled by default by putting them behind a flag. - It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). - But we want to ship these additional features when we release Garage. - In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds. + You can have a complete list of the available options by looking at the overriden object, mkcrate: + https://github.com/cargo2nix/cargo2nix/blob/master/overlay/mkcrate.nix */ - rootFeatures = if features != null then features else - ([ - "garage/bundled-libs" - "garage/sled" - "garage/k2v" - ] ++ (if release then [ - "garage/consul-discovery" - "garage/kubernetes-discovery" - "garage/metrics" - "garage/telemetry-otlp" - "garage/lmdb" - "garage/sqlite" - ] else [])); + packageOverrides = pkgs: + pkgs.rustBuilder.overrides.all ++ [ + /* [1] We add some logic to compile our crates with clippy, it provides us many additional lints + [2] We need to alter Nix hardening to make static binaries: PIE, + Position Independent Executables seems to be supported only on amd64. Having + this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. + Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets + (only amd64 curently) through the `-static-pie` flag. + PIE is a feature used by ASLR, which helps mitigate security issues. + Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh + + [3] We want to inject the git version while keeping the build deterministic. + As we do not want to consider the .git folder as part of the input source, + we ask the user (the CI often) to pass the value to Nix. + + [4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library. + However the features to do so get activated for some reason (due to a bug in cargo2nix?), + so disable them manually here. + */ + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage"; + overrideAttrs = drv: + (if git_version != null then { + # [3] + preConfigure = '' + ${drv.preConfigure or ""} + export GIT_VERSION="${git_version}" + ''; + } else + { }) // { + # [1] + setBuildEnv = (buildEnv drv); + # [2] + hardeningDisable = [ "pie" ]; + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_rpc"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_db"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_util"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_table"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_block"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_model"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_api"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage_web"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "k2v-client"; + overrideAttrs = drv: { # [1] + setBuildEnv = (buildEnv drv); + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "libsodium-sys"; + overrideArgs = old: { + features = [ ]; # [4] + }; + }) + + (pkgs.rustBuilder.rustLib.makeOverride { + name = "zstd-sys"; + overrideArgs = old: { + features = [ ]; # [4] + }; + }) + ]; + + /* We ship some parts of the code disabled by default by putting them behind a flag. + It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). + But we want to ship these additional features when we release Garage. + In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds. + */ + rootFeatures = if features != null then + features + else + ([ "garage/bundled-libs" "garage/sled" "garage/k2v" ] ++ (if release then [ + "garage/consul-discovery" + "garage/kubernetes-discovery" + "garage/metrics" + "garage/telemetry-otlp" + "garage/lmdb" + "garage/sqlite" + ] else + [ ])); packageFun = import ../Cargo.nix; - /* - We compile fully static binaries with musl to simplify deployment on most systems. - When possible, we reactivate PIE hardening (see above). + /* We compile fully static binaries with musl to simplify deployment on most systems. + When possible, we reactivate PIE hardening (see above). - Also, if you set the RUSTFLAGS environment variable, the following parameters will - be ignored. + Also, if you set the RUSTFLAGS environment variable, the following parameters will + be ignored. - For more information on static builds, please refer to Rust's RFC 1721. - https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage + For more information on static builds, please refer to Rust's RFC 1721. + https://rust-lang.github.io/rfcs/1721-crt-static.html#specifying-dynamicstatic-c-runtime-linkage */ codegenOpts = { - "armv6l-unknown-linux-musleabihf" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* compile as dynamic with static-pie */ - "aarch64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */ - "i686-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static" ]; /* segfault with static-pie */ - "x86_64-unknown-linux-musl" = [ "target-feature=+crt-static" "link-arg=-static-pie" ]; + "armv6l-unknown-linux-musleabihf" = [ + "target-feature=+crt-static" + "link-arg=-static" + ]; # compile as dynamic with static-pie + "aarch64-unknown-linux-musl" = [ + "target-feature=+crt-static" + "link-arg=-static" + ]; # segfault with static-pie + "i686-unknown-linux-musl" = [ + "target-feature=+crt-static" + "link-arg=-static" + ]; # segfault with static-pie + "x86_64-unknown-linux-musl" = + [ "target-feature=+crt-static" "link-arg=-static-pie" ]; }; - /* - NixOS and Rust/Cargo triples do not match for ARM, fix it here. - */ - rustTarget = if target == "armv6l-unknown-linux-musleabihf" - then "arm-unknown-linux-musleabihf" - else target; + # NixOS and Rust/Cargo triples do not match for ARM, fix it here. + rustTarget = if target == "armv6l-unknown-linux-musleabihf" then + "arm-unknown-linux-musleabihf" + else + target; -in - pkgs.rustBuilder.makePackageSet ({ - inherit release packageFun packageOverrides codegenOpts rootFeatures; - target = rustTarget; - } // toolchainOptions) +in pkgs.rustBuilder.makePackageSet ({ + inherit release packageFun packageOverrides codegenOpts rootFeatures; + target = rustTarget; +} // toolchainOptions) diff --git a/nix/kaniko.nix b/nix/kaniko.nix index 140328b8..64cadd14 100644 --- a/nix/kaniko.nix +++ b/nix/kaniko.nix @@ -15,7 +15,8 @@ pkgs.buildGoModule rec { checkPhase = "true"; meta = with pkgs.lib; { - description = "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster."; + description = + "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster."; homepage = "https://github.com/GoogleContainerTools/kaniko"; license = licenses.asl20; platforms = platforms.linux; diff --git a/nix/manifest-tool.nix b/nix/manifest-tool.nix index 182ccc0e..1090a6ef 100644 --- a/nix/manifest-tool.nix +++ b/nix/manifest-tool.nix @@ -15,7 +15,8 @@ pkgs.buildGoModule rec { checkPhase = "true"; meta = with pkgs.lib; { - description = "Command line tool to create and query container image manifest list/indexes"; + description = + "Command line tool to create and query container image manifest list/indexes"; homepage = "https://github.com/estesp/manifest-tool"; license = licenses.asl20; platforms = platforms.linux; diff --git a/nix/toolchain.nix b/nix/toolchain.nix index 079fcf13..532db74e 100644 --- a/nix/toolchain.nix +++ b/nix/toolchain.nix @@ -1,6 +1,4 @@ -{ - system ? builtins.currentSystem, -}: +{ system ? builtins.currentSystem, }: with import ./common.nix; @@ -11,27 +9,23 @@ let #"aarch64-unknown-linux-musl" "armv6l-unknown-linux-musleabihf" ]; - pkgsList = builtins.map (target: import pkgsSrc { - inherit system; - crossSystem = { - config = target; - isStatic = true; - }; - overlays = [ cargo2nixOverlay ]; - }) platforms; - pkgsHost = import pkgsSrc {}; + pkgsList = builtins.map (target: + import pkgsSrc { + inherit system; + crossSystem = { + config = target; + isStatic = true; + }; + overlays = [ cargo2nixOverlay ]; + }) platforms; + pkgsHost = import pkgsSrc { }; lib = pkgsHost.lib; kaniko = (import ./kaniko.nix) pkgsHost; winscp = (import ./winscp.nix) pkgsHost; manifestTool = (import ./manifest-tool.nix) pkgsHost; -in - lib.flatten (builtins.map (pkgs: [ - pkgs.rustPlatform.rust.rustc - pkgs.rustPlatform.rust.cargo - pkgs.buildPackages.stdenv.cc - ]) pkgsList) ++ [ - kaniko - winscp - manifestTool - ] +in lib.flatten (builtins.map (pkgs: [ + pkgs.rustPlatform.rust.rustc + pkgs.rustPlatform.rust.cargo + pkgs.buildPackages.stdenv.cc +]) pkgsList) ++ [ kaniko winscp manifestTool ] diff --git a/nix/winscp.nix b/nix/winscp.nix index 10d3cb28..113f4506 100644 --- a/nix/winscp.nix +++ b/nix/winscp.nix @@ -11,12 +11,12 @@ pkgs.stdenv.mkDerivation rec { }; buildPhase = '' - cat > winscp < winscp < /tmp/nix-signing-key.sec - nix copy \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR \ - $(nix-build --no-build-output --no-out-link nix/toolchain.nix)) - rm /tmp/nix-signing-key.sec + # --- Release shell --- + # A shell built to make releasing easier + release = pkgs.mkShell { + shellHook = '' + function refresh_toolchain { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + nix copy \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix-store -qR \ + $(nix-build --no-build-output --no-out-link nix/toolchain.nix)) + rm /tmp/nix-signing-key.sec + } + + function refresh_cache { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do + echo "Updating cache for ''${attr}" + derivation=$(nix-instantiate --attr ''${attr}) + nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix-store -qR ''${derivation%\!bin}) + done + rm /tmp/nix-signing-key.sec + } + + function refresh_flake_cache { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + for attr in packages.x86_64-linux.default devShell.x86_64-linux; do + echo "Updating cache for ''${attr}" + derivation=$(nix path-info --derivation ".#''${attr}") + nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix-store -qR ''${derivation}) + done + rm /tmp/nix-signing-key.sec + } + + function to_s3 { + aws \ + --endpoint-url https://garage.deuxfleurs.fr \ + --region garage \ + s3 cp \ + ./result-bin/bin/garage \ + s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage + } + + function to_docker { + executor \ + --force \ + --customPlatform="''${DOCKER_PLATFORM}" \ + --destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \ + --context dir://`pwd` \ + --verbosity=debug + } + + function multiarch_docker { + manifest-tool push from-spec <(cat < aws-list.txt + + nix-build nix/build_index.nix + + aws \ + --endpoint-url https://garage.deuxfleurs.fr \ + --region garage \ + s3 cp \ + result/share/_releases.json \ + s3://garagehq.deuxfleurs.fr/ + + aws \ + --endpoint-url https://garage.deuxfleurs.fr \ + --region garage \ + s3 cp \ + result/share/_releases.html \ + s3://garagehq.deuxfleurs.fr/ + } + ''; + nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ]; + }; } -function refresh_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do - echo "Updating cache for ''${attr}" - derivation=$(nix-instantiate --attr ''${attr}) - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation%\!bin}) - done - rm /tmp/nix-signing-key.sec -} - -function refresh_flake_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in packages.x86_64-linux.default; do - echo "Updating cache for ''${attr}" - derivation=$(nix path-info --derivation ".#''${attr}") - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation}) - done - rm /tmp/nix-signing-key.sec -} - -function to_s3 { - aws \ - --endpoint-url https://garage.deuxfleurs.fr \ - --region garage \ - s3 cp \ - ./result-bin/bin/garage \ - s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage -} - -function to_docker { - executor \ - --force \ - --customPlatform="''${DOCKER_PLATFORM}" \ - --destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \ - --context dir://`pwd` \ - --verbosity=debug -} - -function multiarch_docker { - manifest-tool push from-spec <(cat < aws-list.txt - - nix-build nix/build_index.nix - - aws \ - --endpoint-url https://garage.deuxfleurs.fr \ - --region garage \ - s3 cp \ - result/share/_releases.json \ - s3://garagehq.deuxfleurs.fr/ - - aws \ - --endpoint-url https://garage.deuxfleurs.fr \ - --region garage \ - s3 cp \ - result/share/_releases.html \ - s3://garagehq.deuxfleurs.fr/ -} - ''; - nativeBuildInputs = [ - pkgs.awscli2 - kaniko - manifest-tool - ]; - }; - } - - From efb6b6e868be1be0725125485810ce96e74ed124 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 26 Jan 2023 15:04:32 +0100 Subject: [PATCH 2/3] Disk space report Report available disk space on nodes and calculate cluster-wide available space in `garage stats` (fix #479) --- Cargo.lock | 15 +++++ Cargo.nix | 19 +++++- Makefile | 2 +- src/garage/admin.rs | 125 ++++++++++++++++++++++++++++++++------ src/garage/cli/cmd.rs | 15 ++++- src/garage/cli/structs.rs | 5 ++ src/rpc/Cargo.toml | 1 + src/rpc/system.rs | 73 +++++++++++++++++----- 8 files changed, 220 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5d38b92a..871d897f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1230,6 +1230,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "systemstat", "tokio", "tokio-stream", "tracing", @@ -3567,6 +3568,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "systemstat" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a24aec24a9312c83999a28e3ef9db7e2afd5c64bf47725b758cdc1cafd5b0bd2" +dependencies = [ + "bytesize", + "lazy_static", + "libc", + "nom", + "time 0.3.9", + "winapi", +] + [[package]] name = "tempfile" version = "3.3.0" diff --git a/Cargo.nix b/Cargo.nix index 9ee5b9fb..24c09e92 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -32,7 +32,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "8461dcfb984a8d042fecb5745d5da17912135dbf2a8ef7e6c3ae8e64c03d9744"; + nixifiedLockHash = "e59ef222aaaada125e2a5fccbd215b545740b2abd21ce381c42a7a6e3a7e672a"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -1753,6 +1753,7 @@ in serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out; serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out; serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.81" { inherit profileName; }).out; + systemstat = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".systemstat."0.2.3" { inherit profileName; }).out; tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out; tokio_stream = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }).out; tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }).out; @@ -4917,6 +4918,21 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".systemstat."0.2.3" = overridableMkRustCrate (profileName: rec { + name = "systemstat"; + version = "0.2.3"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "a24aec24a9312c83999a28e3ef9db7e2afd5c64bf47725b758cdc1cafd5b0bd2"; }; + dependencies = { + bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out; + lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; + libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out; + ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "nom" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.1" { inherit profileName; }).out; + time = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".time."0.3.9" { inherit profileName; }).out; + ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".tempfile."3.3.0" = overridableMkRustCrate (profileName: rec { name = "tempfile"; version = "3.3.0"; @@ -5890,6 +5906,7 @@ in [ "ntsecapi" ] [ "ntstatus" ] [ "objbase" ] + [ "pdh" ] [ "processenv" ] [ "processthreadsapi" ] [ "profileapi" ] diff --git a/Makefile b/Makefile index 23e10f78..55fa16dd 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ all: clear; cargo build release: - nix-build --arg release true + nix-build --attr pkgs.amd64.release --no-build-output shell: nix-shell diff --git a/src/garage/admin.rs b/src/garage/admin.rs index 305c5c65..4eabebca 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -15,6 +15,7 @@ use garage_util::time::*; use garage_table::replication::*; use garage_table::*; +use garage_rpc::ring::PARTITION_BITS; use garage_rpc::*; use garage_block::manager::BlockResyncErrorInfo; @@ -783,6 +784,7 @@ impl AdminRpcHandler { for node in ring.layout.node_ids().iter() { let mut opt = opt.clone(); opt.all_nodes = false; + opt.skip_global = true; writeln!(&mut ret, "\n======================").unwrap(); writeln!(&mut ret, "Stats for node {:?}:", node).unwrap(); @@ -799,6 +801,15 @@ impl AdminRpcHandler { Err(e) => writeln!(&mut ret, "Network error: {}", e).unwrap(), } } + + writeln!(&mut ret, "\n======================").unwrap(); + write!( + &mut ret, + "Cluster statistics:\n\n{}", + self.gather_cluster_stats() + ) + .unwrap(); + Ok(AdminRpc::Ok(ret)) } else { Ok(AdminRpc::Ok(self.gather_stats_local(opt)?)) @@ -819,22 +830,6 @@ impl AdminRpcHandler { writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap(); - // Gather ring statistics - let ring = self.garage.system.ring.borrow().clone(); - let mut ring_nodes = HashMap::new(); - for (_i, loc) in ring.partitions().iter() { - for n in ring.get_nodes(loc, ring.replication_factor).iter() { - if !ring_nodes.contains_key(n) { - ring_nodes.insert(*n, 0usize); - } - *ring_nodes.get_mut(n).unwrap() += 1; - } - } - writeln!(&mut ret, "\nRing nodes & partition count:").unwrap(); - for (n, c) in ring_nodes.iter() { - writeln!(&mut ret, " {:?} {}", n, c).unwrap(); - } - // Gather table statistics let mut table = vec![" Table\tItems\tMklItems\tMklTodo\tGcTodo".into()]; table.push(self.gather_table_stats(&self.garage.bucket_table, opt.detailed)?); @@ -881,12 +876,108 @@ impl AdminRpcHandler { .unwrap(); if !opt.detailed { - writeln!(&mut ret, "\nIf values are missing (marked as NC), consider adding the --detailed flag - this will be slow.").unwrap(); + writeln!(&mut ret, "\nIf values are missing above (marked as NC), consider adding the --detailed flag (this will be slow).").unwrap(); + } + + if !opt.skip_global { + write!(&mut ret, "\n{}", self.gather_cluster_stats()).unwrap(); } Ok(ret) } + fn gather_cluster_stats(&self) -> String { + let mut ret = String::new(); + + // Gather storage node and free space statistics + let layout = &self.garage.system.ring.borrow().layout; + let mut node_partition_count = HashMap::::new(); + for short_id in layout.ring_assignation_data.iter() { + let id = layout.node_id_vec[*short_id as usize]; + *node_partition_count.entry(id).or_default() += 1; + } + let node_info = self + .garage + .system + .get_known_nodes() + .into_iter() + .map(|n| (n.id, n)) + .collect::>(); + + let mut table = vec![" ID\tHostname\tZone\tCapacity\tPart.\tDataAvail\tMetaAvail".into()]; + for (id, parts) in node_partition_count.iter() { + let info = node_info.get(id); + let status = info.map(|x| &x.status); + let role = layout.roles.get(id).and_then(|x| x.0.as_ref()); + let hostname = status.map(|x| x.hostname.as_str()).unwrap_or("?"); + let zone = role.map(|x| x.zone.as_str()).unwrap_or("?"); + let capacity = role.map(|x| x.capacity_string()).unwrap_or("?".into()); + let avail_str = |x| match x { + Some((avail, total)) => { + let pct = (avail as f64) / (total as f64) * 100.; + let avail = bytesize::ByteSize::b(avail); + let total = bytesize::ByteSize::b(total); + format!("{}/{} ({:.1}%)", avail, total, pct) + } + None => "?".into(), + }; + let data_avail = avail_str(status.and_then(|x| x.data_disk_avail)); + let meta_avail = avail_str(status.and_then(|x| x.meta_disk_avail)); + table.push(format!( + " {:?}\t{}\t{}\t{}\t{}\t{}\t{}", + id, hostname, zone, capacity, parts, data_avail, meta_avail + )); + } + write!( + &mut ret, + "Storage nodes:\n{}", + format_table_to_string(table) + ) + .unwrap(); + + let meta_part_avail = node_partition_count + .iter() + .filter_map(|(id, parts)| { + node_info + .get(id) + .and_then(|x| x.status.meta_disk_avail) + .map(|c| c.0 / *parts) + }) + .collect::>(); + let data_part_avail = node_partition_count + .iter() + .filter_map(|(id, parts)| { + node_info + .get(id) + .and_then(|x| x.status.data_disk_avail) + .map(|c| c.0 / *parts) + }) + .collect::>(); + if !meta_part_avail.is_empty() && !data_part_avail.is_empty() { + let meta_avail = + bytesize::ByteSize(meta_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS)); + let data_avail = + bytesize::ByteSize(data_part_avail.iter().min().unwrap() * (1 << PARTITION_BITS)); + writeln!( + &mut ret, + "\nEstimated available storage space cluster-wide (might be lower in practice):" + ) + .unwrap(); + if meta_part_avail.len() < node_partition_count.len() + || data_part_avail.len() < node_partition_count.len() + { + writeln!(&mut ret, " data: < {}", data_avail).unwrap(); + writeln!(&mut ret, " metadata: < {}", meta_avail).unwrap(); + writeln!(&mut ret, "A precise estimate could not be given as information is missing for some storage nodes.").unwrap(); + } else { + writeln!(&mut ret, " data: {}", data_avail).unwrap(); + writeln!(&mut ret, " metadata: {}", meta_avail).unwrap(); + } + } + + ret + } + fn gather_table_stats( &self, t: &Arc>, diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs index 46e9113c..af7f1aa1 100644 --- a/src/garage/cli/cmd.rs +++ b/src/garage/cli/cmd.rs @@ -59,18 +59,29 @@ pub async fn cmd_status(rpc_cli: &Endpoint, rpc_host: NodeID) -> let layout = fetch_layout(rpc_cli, rpc_host).await?; println!("==== HEALTHY NODES ===="); - let mut healthy_nodes = vec!["ID\tHostname\tAddress\tTags\tZone\tCapacity".to_string()]; + let mut healthy_nodes = + vec!["ID\tHostname\tAddress\tTags\tZone\tCapacity\tDataAvail\tMetaAvail".to_string()]; for adv in status.iter().filter(|adv| adv.is_up) { match layout.roles.get(&adv.id) { Some(NodeRoleV(Some(cfg))) => { + let data_avail = match &adv.status.data_disk_avail { + _ if cfg.capacity.is_none() => "N/A".into(), + Some((avail, total)) => { + let pct = (*avail as f64) / (*total as f64) * 100.; + let avail = bytesize::ByteSize::b(*avail); + format!("{} ({:.1}%)", avail, pct) + } + None => "?".into(), + }; healthy_nodes.push(format!( - "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}", + "{id:?}\t{host}\t{addr}\t[{tags}]\t{zone}\t{capacity}\t{data_avail}", id = adv.id, host = adv.status.hostname, addr = adv.addr, tags = cfg.tags.join(","), zone = cfg.zone, capacity = cfg.capacity_string(), + data_avail = data_avail, )); } _ => { diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index 661a71f0..01ae92da 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -504,6 +504,11 @@ pub struct StatsOpt { /// Gather detailed statistics (this can be long) #[structopt(short = "d", long = "detailed")] pub detailed: bool, + + /// Don't show global cluster stats (internal use in RPC) + #[structopt(skip)] + #[serde(default)] + pub skip_global: bool, } #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)] diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index e9a0929a..2daa0176 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -23,6 +23,7 @@ hex = "0.4" tracing = "0.1.30" rand = "0.8" sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } +systemstat = "0.2.3" async-trait = "0.1.7" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 90f6a4c2..a9e91e19 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -118,18 +118,28 @@ pub struct System { /// Path to metadata directory pub metadata_dir: PathBuf, + /// Path to data directory + pub data_dir: PathBuf, } #[derive(Debug, Clone, Serialize, Deserialize)] pub struct NodeStatus { /// Hostname of the node pub hostname: String, + /// Replication factor configured on the node pub replication_factor: usize, /// Cluster layout version pub cluster_layout_version: u64, /// Hash of cluster layout staging data pub cluster_layout_staging_hash: Hash, + + /// Disk usage on partition containing metadata directory (tuple: `(avail, total)`) + #[serde(default)] + pub meta_disk_avail: Option<(u64, u64)>, + /// Disk usage on partition containing data directory (tuple: `(avail, total)`) + #[serde(default)] + pub data_disk_avail: Option<(u64, u64)>, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -271,14 +281,8 @@ impl System { } }; - let local_status = NodeStatus { - hostname: gethostname::gethostname() - .into_string() - .unwrap_or_else(|_| "".to_string()), - replication_factor, - cluster_layout_version: cluster_layout.version, - cluster_layout_staging_hash: cluster_layout.staging_hash, - }; + let mut local_status = NodeStatus::initial(replication_factor, &cluster_layout); + local_status.update_disk_usage(&config.metadata_dir, &config.data_dir); #[cfg(feature = "metrics")] let metrics = SystemMetrics::new(replication_factor); @@ -379,6 +383,7 @@ impl System { ring, update_ring: Mutex::new(update_ring), metadata_dir: config.metadata_dir.clone(), + data_dir: config.data_dir.clone(), }); sys.system_endpoint.set_handler(sys.clone()); Ok(sys) @@ -416,12 +421,7 @@ impl System { .get(&n.id.into()) .cloned() .map(|(_, st)| st) - .unwrap_or(NodeStatus { - hostname: "?".to_string(), - replication_factor: 0, - cluster_layout_version: 0, - cluster_layout_staging_hash: Hash::from([0u8; 32]), - }), + .unwrap_or(NodeStatus::unknown()), }) .collect::>(); known_nodes @@ -600,6 +600,9 @@ impl System { let ring = self.ring.borrow(); new_si.cluster_layout_version = ring.layout.version; new_si.cluster_layout_staging_hash = ring.layout.staging_hash; + + new_si.update_disk_usage(&self.metadata_dir, &self.data_dir); + self.local_status.swap(Arc::new(new_si)); } @@ -864,6 +867,48 @@ impl EndpointHandler for System { } } +impl NodeStatus { + fn initial(replication_factor: usize, layout: &ClusterLayout) -> Self { + NodeStatus { + hostname: gethostname::gethostname() + .into_string() + .unwrap_or_else(|_| "".to_string()), + replication_factor, + cluster_layout_version: layout.version, + cluster_layout_staging_hash: layout.staging_hash, + meta_disk_avail: None, + data_disk_avail: None, + } + } + + fn unknown() -> Self { + NodeStatus { + hostname: "?".to_string(), + replication_factor: 0, + cluster_layout_version: 0, + cluster_layout_staging_hash: Hash::from([0u8; 32]), + meta_disk_avail: None, + data_disk_avail: None, + } + } + + fn update_disk_usage(&mut self, meta_dir: &Path, data_dir: &Path) { + use systemstat::{Platform, System}; + let mounts = System::new().mounts().unwrap_or_default(); + + let mount_avail = |path: &Path| { + mounts + .iter() + .filter(|x| path.starts_with(&x.fs_mounted_on)) + .max_by_key(|x| x.fs_mounted_on.len()) + .map(|x| (x.avail.as_u64(), x.total.as_u64())) + }; + + self.meta_disk_avail = mount_avail(meta_dir); + self.data_disk_avail = mount_avail(data_dir); + } +} + fn get_default_ip() -> Option { pnet_datalink::interfaces() .iter() From c7d0ad0aa0e492b913c5dda8ff1a7ad5a579fb3a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 26 Jan 2023 15:30:36 +0100 Subject: [PATCH 3/3] Add local disk usage to exported prometheus metrics --- src/rpc/lib.rs | 6 +++--- src/rpc/system.rs | 37 ++++++++++++++++++++++++-------- src/rpc/system_metrics.rs | 44 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 12 deletions(-) diff --git a/src/rpc/lib.rs b/src/rpc/lib.rs index a8cc0030..5aec92c0 100644 --- a/src/rpc/lib.rs +++ b/src/rpc/lib.rs @@ -3,6 +3,9 @@ #[macro_use] extern crate tracing; +mod metrics; +mod system_metrics; + #[cfg(feature = "consul-discovery")] mod consul; #[cfg(feature = "kubernetes-discovery")] @@ -13,9 +16,6 @@ pub mod replication_mode; pub mod ring; pub mod system; -mod metrics; pub mod rpc_helper; pub use rpc_helper::*; - -pub mod system_metrics; diff --git a/src/rpc/system.rs b/src/rpc/system.rs index a9e91e19..e0ced8cc 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::io::{Read, Write}; use std::net::{IpAddr, SocketAddr}; use std::path::{Path, PathBuf}; +use std::sync::atomic::Ordering; use std::sync::{Arc, RwLock}; use std::time::{Duration, Instant}; @@ -38,7 +39,6 @@ use crate::replication_mode::*; use crate::ring::*; use crate::rpc_helper::*; -#[cfg(feature = "metrics")] use crate::system_metrics::*; const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60); @@ -106,7 +106,7 @@ pub struct System { consul_discovery: Option, #[cfg(feature = "kubernetes-discovery")] kubernetes_discovery: Option, - #[cfg(feature = "metrics")] + metrics: SystemMetrics, replication_mode: ReplicationMode, @@ -281,12 +281,11 @@ impl System { } }; - let mut local_status = NodeStatus::initial(replication_factor, &cluster_layout); - local_status.update_disk_usage(&config.metadata_dir, &config.data_dir); - - #[cfg(feature = "metrics")] let metrics = SystemMetrics::new(replication_factor); + let mut local_status = NodeStatus::initial(replication_factor, &cluster_layout); + local_status.update_disk_usage(&config.metadata_dir, &config.data_dir, &metrics); + let ring = Ring::new(cluster_layout, replication_factor); let (update_ring, ring) = watch::channel(Arc::new(ring)); @@ -377,7 +376,6 @@ impl System { consul_discovery, #[cfg(feature = "kubernetes-discovery")] kubernetes_discovery: config.kubernetes_discovery.clone(), - #[cfg(feature = "metrics")] metrics, ring, @@ -601,7 +599,7 @@ impl System { new_si.cluster_layout_version = ring.layout.version; new_si.cluster_layout_staging_hash = ring.layout.staging_hash; - new_si.update_disk_usage(&self.metadata_dir, &self.data_dir); + new_si.update_disk_usage(&self.metadata_dir, &self.data_dir, &self.metrics); self.local_status.swap(Arc::new(new_si)); } @@ -892,7 +890,7 @@ impl NodeStatus { } } - fn update_disk_usage(&mut self, meta_dir: &Path, data_dir: &Path) { + fn update_disk_usage(&mut self, meta_dir: &Path, data_dir: &Path, metrics: &SystemMetrics) { use systemstat::{Platform, System}; let mounts = System::new().mounts().unwrap_or_default(); @@ -906,6 +904,27 @@ impl NodeStatus { self.meta_disk_avail = mount_avail(meta_dir); self.data_disk_avail = mount_avail(data_dir); + + if let Some((avail, total)) = self.meta_disk_avail { + metrics + .values + .meta_disk_avail + .store(avail, Ordering::Relaxed); + metrics + .values + .meta_disk_total + .store(total, Ordering::Relaxed); + } + if let Some((avail, total)) = self.data_disk_avail { + metrics + .values + .data_disk_avail + .store(avail, Ordering::Relaxed); + metrics + .values + .data_disk_total + .store(total, Ordering::Relaxed); + } } } diff --git a/src/rpc/system_metrics.rs b/src/rpc/system_metrics.rs index d96b67e4..83f5fa97 100644 --- a/src/rpc/system_metrics.rs +++ b/src/rpc/system_metrics.rs @@ -1,14 +1,31 @@ +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + use opentelemetry::{global, metrics::*, KeyValue}; /// TableMetrics reference all counter used for metrics pub struct SystemMetrics { pub(crate) _garage_build_info: ValueObserver, pub(crate) _replication_factor: ValueObserver, + pub(crate) _disk_avail: ValueObserver, + pub(crate) _disk_total: ValueObserver, + pub(crate) values: Arc, +} + +#[derive(Default)] +pub struct SystemMetricsValues { + pub(crate) data_disk_total: AtomicU64, + pub(crate) data_disk_avail: AtomicU64, + pub(crate) meta_disk_total: AtomicU64, + pub(crate) meta_disk_avail: AtomicU64, } impl SystemMetrics { pub fn new(replication_factor: usize) -> Self { let meter = global::meter("garage_system"); + let values = Arc::new(SystemMetricsValues::default()); + let values1 = values.clone(); + let values2 = values.clone(); Self { _garage_build_info: meter .u64_value_observer("garage_build_info", move |observer| { @@ -28,6 +45,33 @@ impl SystemMetrics { }) .with_description("Garage replication factor setting") .init(), + _disk_avail: meter + .u64_value_observer("garage_local_disk_avail", move |observer| { + match values1.data_disk_avail.load(Ordering::Relaxed) { + 0 => (), + x => observer.observe(x, &[KeyValue::new("volume", "data")]), + }; + match values1.meta_disk_avail.load(Ordering::Relaxed) { + 0 => (), + x => observer.observe(x, &[KeyValue::new("volume", "metadata")]), + }; + }) + .with_description("Garage available disk space on each node") + .init(), + _disk_total: meter + .u64_value_observer("garage_local_disk_total", move |observer| { + match values2.data_disk_total.load(Ordering::Relaxed) { + 0 => (), + x => observer.observe(x, &[KeyValue::new("volume", "data")]), + }; + match values2.meta_disk_total.load(Ordering::Relaxed) { + 0 => (), + x => observer.observe(x, &[KeyValue::new("volume", "metadata")]), + }; + }) + .with_description("Garage total disk space on each node") + .init(), + values, } } }