Merge pull request 'Bump everything to v0.8.1' (#458) from up-v0.8.1 into main

Reviewed-on: #458

.gitattributes

@@ -1 +0,0 @@
-*.pdf filter=lfs diff=lfs merge=lfs -text

Cargo.lock

@@ -1048,7 +1048,7 @@ dependencies = [
 
 [[package]]
 name = "garage"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "assert-json-diff",
  "async-trait",
@@ -1096,7 +1096,7 @@ dependencies = [
 
 [[package]]
 name = "garage_api"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "async-trait",
  "base64",
@@ -1141,7 +1141,7 @@ dependencies = [
 
 [[package]]
 name = "garage_block"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "arc-swap",
  "async-compression",
@@ -1167,7 +1167,7 @@ dependencies = [
 
 [[package]]
 name = "garage_db"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "clap 3.1.18",
  "err-derive",
@@ -1182,7 +1182,7 @@ dependencies = [
 
 [[package]]
 name = "garage_model"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1210,7 +1210,7 @@ dependencies = [
 
 [[package]]
 name = "garage_rpc"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1241,7 +1241,7 @@ dependencies = [
 
 [[package]]
 name = "garage_table"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "async-trait",
  "bytes",
@@ -1263,7 +1263,7 @@ dependencies = [
 
 [[package]]
 name = "garage_util"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1294,7 +1294,7 @@ dependencies = [
 
 [[package]]
 name = "garage_web"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "err-derive",
  "futures",

Cargo.nix

@@ -32,7 +32,7 @@ args@{
   ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "90b29705f5037c7e1b33f4650841f1266f2e86fa03d5d0c87ad80be7619985c7";
+  nixifiedLockHash = "463114c4544bfa9b442a43afc6b39eb588f5720825c7a246ba9188c4bdb52944";
   workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
   currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
   lockHashIgnored = if ignoreLockHash
@@ -56,15 +56,15 @@ in
 {
   cargo2nixVersion = "0.11.0";
   workspace = {
-    garage_db = rustPackages.unknown.garage_db."0.8.0";
+    garage_db = rustPackages.unknown.garage_db."0.8.1";
-    garage_util = rustPackages.unknown.garage_util."0.8.0";
+    garage_util = rustPackages.unknown.garage_util."0.8.1";
-    garage_rpc = rustPackages.unknown.garage_rpc."0.8.0";
+    garage_rpc = rustPackages.unknown.garage_rpc."0.8.1";
-    garage_table = rustPackages.unknown.garage_table."0.8.0";
+    garage_table = rustPackages.unknown.garage_table."0.8.1";
-    garage_block = rustPackages.unknown.garage_block."0.8.0";
+    garage_block = rustPackages.unknown.garage_block."0.8.1";
-    garage_model = rustPackages.unknown.garage_model."0.8.0";
+    garage_model = rustPackages.unknown.garage_model."0.8.1";
-    garage_api = rustPackages.unknown.garage_api."0.8.0";
+    garage_api = rustPackages.unknown.garage_api."0.8.1";
-    garage_web = rustPackages.unknown.garage_web."0.8.0";
+    garage_web = rustPackages.unknown.garage_web."0.8.1";
-    garage = rustPackages.unknown.garage."0.8.0";
+    garage = rustPackages.unknown.garage."0.8.1";
     k2v-client = rustPackages.unknown.k2v-client."0.0.1";
   };
   "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec {
@@ -1494,9 +1494,9 @@ in
     };
   });
   
-  "unknown".garage."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/garage");
     features = builtins.concatLists [
@@ -1522,14 +1522,14 @@ in
       bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
-      garage_web = (rustPackages."unknown".garage_web."0.8.0" { inherit profileName; }).out;
+      garage_web = (rustPackages."unknown".garage_web."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
       netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
@@ -1563,9 +1563,9 @@ in
     };
   });
   
-  "unknown".garage_api."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_api."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_api";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/api");
     features = builtins.concatLists [
@@ -1584,11 +1584,11 @@ in
       form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
@@ -1617,9 +1617,9 @@ in
     };
   });
   
-  "unknown".garage_block."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_block."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_block";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/block");
     features = builtins.concatLists [
@@ -1632,10 +1632,10 @@ in
       bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
@@ -1649,9 +1649,9 @@ in
     };
   });
   
-  "unknown".garage_db."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_db."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_db";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/db");
     features = builtins.concatLists [
@@ -1681,9 +1681,9 @@ in
     };
   });
   
-  "unknown".garage_model."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_model."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_model";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/model");
     features = builtins.concatLists [
@@ -1701,11 +1701,11 @@ in
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -1719,9 +1719,9 @@ in
     };
   });
   
-  "unknown".garage_rpc."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_rpc."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_rpc";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/rpc");
     features = builtins.concatLists [
@@ -1741,7 +1741,7 @@ in
       ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out;
@@ -1763,9 +1763,9 @@ in
     };
   });
   
-  "unknown".garage_table."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_table."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_table";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/table");
     dependencies = {
@@ -1773,9 +1773,9 @@ in
       bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -1788,9 +1788,9 @@ in
     };
   });
   
-  "unknown".garage_util."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_util."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_util";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/util");
     features = builtins.concatLists [
@@ -1805,7 +1805,7 @@ in
       digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; }).out;
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
       git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
@@ -1825,18 +1825,18 @@ in
     };
   });
   
-  "unknown".garage_web."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_web."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_web";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/web");
     dependencies = {
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
       hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -2450,7 +2450,7 @@ in
     dependencies = {
       base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
       log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
       rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out;
@@ -5398,7 +5398,6 @@ in
       [ "attributes" ]
       [ "default" ]
       [ "log" ]
-      [ "log-always" ]
       [ "std" ]
       [ "tracing-attributes" ]
     ];

default.nix

@@ -11,14 +11,14 @@ let
 
   build_debug_and_release = (target: {
     debug = (compile {
-      inherit target git_version;
+      inherit system target git_version pkgsSrc cargo2nixOverlay;
       release = false;
     }).workspace.garage {
       compileMode = "build";
     };
 
     release = (compile {
-      inherit target git_version;
+      inherit system target git_version pkgsSrc cargo2nixOverlay;
       release = true;
     }).workspace.garage {
       compileMode = "build";
@@ -39,7 +39,7 @@ in {
   };
   test = {
     amd64 = test (compile {
-      inherit git_version;
+      inherit system git_version pkgsSrc cargo2nixOverlay;
       target = "x86_64-unknown-linux-musl";
       features = [
         "garage/bundled-libs"
@@ -52,7 +52,7 @@ in {
   };
   clippy = {
     amd64 = (compile {
-      inherit git_version;
+      inherit system git_version pkgsSrc cargo2nixOverlay;
       target = "x86_64-unknown-linux-musl";
       compiler = "clippy";
     }).workspace.garage {

doc/book/build/python.md

@@ -5,16 +5,59 @@ weight = 20
 
 ## S3
 
+### Using Minio SDK
+
+First install the SDK:
+
+```bash
+pip3 install minio
+```
+
+Then instantiate a client object using garage root domain, api key and secret:
+
+```python
+import minio
+
+client = minio.Minio(
+  "your.domain.tld",
+  "GKyourapikey",
+  "abcd[...]1234",
+  # Force the region, this is specific to garage
+  region="region",
+)
+```
+
+Then use all the standard S3 endpoints as implemented by the Minio SDK:
+
+```
+# List buckets
+print(client.list_buckets())
+
+# Put an object containing 'content' to /path in bucket named 'bucket':
+content = b"content"
+client.put_object(
+  "bucket",
+  "path",
+  io.BytesIO(content),
+  len(content),
+)
+
+# Read the object back and check contents
+data = client.get_object("bucket", "path").read()
+assert data == content
+```
+
+For further documentation, see the Minio SDK
+[Reference](https://docs.min.io/docs/python-client-api-reference.html)
+
+### Using Amazon boto3
+
 *Coming soon*
 
-Some refs:
+See the official documentation:
-  - Minio SDK
+  - [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
-    - [Reference](https://docs.min.io/docs/python-client-api-reference.html)
+  - [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
-
+  - [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
-  - Amazon boto3
-    - [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
-    - [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
-    - [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
 
 ## K2V
 

doc/book/connect/apps/index.md

@@ -8,7 +8,7 @@ In this section, we cover the following web applications:
 | Name | Status | Note |
 |------|--------|------|
 | [Nextcloud](#nextcloud)     | ✅       |  Both Primary Storage and External Storage are supported    |
-| [Peertube](#peertube)     | ✅       | Must be configured with the website endpoint     |
+| [Peertube](#peertube)     | ✅       | Supported with the website endpoint, proxifying private videos unsupported     |
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
 | [Pixelfed](#pixelfed)     | ❓       |  Not yet tested    |
@@ -128,6 +128,10 @@ In other words, Peertube is only responsible of the "control plane" and offload
 In return, this system is a bit harder to configure.
 We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
 
+Starting from version 5.0, Peertube also supports improving the security for private videos by not exposing them directly
+but relying on a single control point in the Peertube instance. This is based on S3 per-object and prefix ACL, which are not currently supported
+in Garage, so this feature is unsupported. While this technically impedes security for private videos, it is not a blocking issue and could be
+a reasonable trade-off for some instances.
 
 ### Create resources in Garage
 
@@ -195,6 +199,11 @@ object_storage:
 
   max_upload_part: 2GB
 
+  proxy:
+    # You may enable this feature, yet it will not provide any security benefit, so
+    # you should rather benefit from Garage public endpoint for all videos
+    proxify_private_files: false
+
   streaming_playlists:
     bucket_name: 'peertube-playlist'
 

doc/book/cookbook/real-world.md

@@ -109,7 +109,7 @@ especially you must consider the following folders/files:
   this folder will be your main data storage and must be on a large storage (e.g. large HDD)
 
 
-A valid `/etc/garage/garage.toml` for our cluster would look as follows:
+A valid `/etc/garage.toml` for our cluster would look as follows:
 
 ```toml
 metadata_dir = "/var/lib/garage/meta"

doc/book/design/goals.md

@@ -12,7 +12,7 @@ as pictures, video, images, documents, etc., in a redundant multi-node
 setting. S3 is versatile enough to also be used to publish a static
 website.
 
-Garage is an opinionated object storage solutoin, we focus on the following **desirable properties**:
+Garage is an opinionated object storage solution, we focus on the following **desirable properties**:
 
   - **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
   - **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).

doc/book/reference-manual/features.md

@@ -83,7 +83,7 @@ This feature is totally invisible to S3 clients and does not break compatibility
 ### Cluster administration API
 
 Garage provides a fully-fledged REST API to administer your cluster programatically.
-Functionnality included in the admin API include: setting up and monitoring
+Functionality included in the admin API include: setting up and monitoring
 cluster nodes, managing access credentials, and managing storage buckets and bucket aliases.
 A full reference of the administration API is available [here](@/documentation/reference-manual/admin-api.md).
 

doc/drafts/admin-api.md

@@ -0,0 +1,686 @@
++++
+title = "Administration API"
+weight = 60
++++
+
+The Garage administration API is accessible through a dedicated server whose
+listen address is specified in the `[admin]` section of the configuration
+file (see [configuration file
+reference](@/documentation/reference-manual/configuration.md))
+
+**WARNING.** At this point, there is no comittement to stability of the APIs described in this document.
+We will bump the version numbers prefixed to each API endpoint at each time the syntax
+or semantics change, meaning that code that relies on these endpoint will break
+when changes are introduced.
+
+The Garage administration API was introduced in version 0.7.2, this document
+does not apply to older versions of Garage.
+
+
+## Access control
+
+The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section:
+
+- `metrics_token`: the token for accessing the Metrics endpoint (if this token
+  is not set in the config file, the Metrics endpoint can be accessed without
+  access control);
+
+- `admin_token`: the token for accessing all of the other administration
+  endpoints (if this token is not set in the config file, access to these
+  endpoints is disabled entirely).
+
+These tokens are used as simple HTTP bearer tokens. In other words, to
+authenticate access to an admin API endpoint, add the following HTTP header
+to your request:
+
+```
+Authorization: Bearer <token>
+```
+
+## Administration API endpoints
+
+### Metrics-related endpoints
+
+#### Metrics `GET /metrics`
+
+Returns internal Garage metrics in Prometheus format.
+
+#### Health `GET /health`
+
+Used for simple health checks in a cluster setting with an orchestrator.
+Returns an HTTP status 200 if the node is ready to answer user's requests,
+and an HTTP status 503 (Service Unavailable) if there are some partitions
+for which a quorum of nodes is not available.
+A simple textual message is also returned in a body with content-type `text/plain`.
+See `/v0/health` for an API that also returns JSON output.
+
+### Cluster operations
+
+#### GetClusterStatus `GET /v0/status`
+
+Returns the cluster's current status in JSON, including:
+
+- ID of the node being queried and its version of the Garage daemon
+- Live nodes
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+Example response body:
+
+```json
+{
+  "node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f",
+  "garage_version": "git:v0.8.0",
+  "knownNodes": {
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+      "addr": "10.0.0.11:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 9,
+      "hostname": "node1"
+    },
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+      "addr": "10.0.0.12:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 1,
+      "hostname": "node2"
+    },
+    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+      "addr": "10.0.0.21:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 7,
+      "hostname": "node3"
+    },
+    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+      "addr": "10.0.0.22:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 1,
+      "hostname": "node4"
+    }
+  },
+  "layout": {
+    "version": 12,
+    "roles": {
+      "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+        "zone": "dc1",
+        "capacity": 4,
+        "tags": [
+          "node1"
+        ]
+      },
+      "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+        "zone": "dc1",
+        "capacity": 6,
+        "tags": [
+          "node2"
+        ]
+      },
+      "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+        "zone": "dc2",
+        "capacity": 10,
+        "tags": [
+          "node3"
+        ]
+      }
+    },
+    "stagedRoleChanges": {
+      "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+        "zone": "dc2",
+        "capacity": 5,
+        "tags": [
+          "node4"
+        ]
+      }
+    }
+  }
+}
+```
+
+#### GetClusterHealth `GET /v0/health`
+
+Returns the cluster's current health in JSON format, with the following variables:
+
+- `status`: one of `Healthy`, `Degraded` or `Unavailable`:
+  - Healthy: Garage node is connected to all storage nodes
+  - Degraded: Garage node is not connected to all storage nodes, but a quorum of write nodes is available for all partitions
+  - Unavailable: a quorum of write nodes is not available for some partitions
+- `known_nodes`: the number of nodes this Garage node has had a TCP connection to since the daemon started
+- `connected_nodes`: the nubmer of nodes this Garage node currently has an open connection to
+- `storage_nodes`: the number of storage nodes currently registered in the cluster layout
+- `storage_nodes_ok`: the number of storage nodes to which a connection is currently open
+- `partitions`: the total number of partitions of the data (currently always 256)
+- `partitions_quorum`: the number of partitions for which a quorum of write nodes is available
+- `partitions_all_ok`: the number of partitions for which we are connected to all storage nodes responsible of storing it
+
+Contrarily to `GET /health`, this endpoint always returns a 200 OK HTTP response code.
+
+Example response body:
+
+```json
+{
+    "status": "Degraded",
+    "known_nodes": 3,
+    "connected_nodes": 2,
+    "storage_nodes": 3,
+    "storage_nodes_ok": 2,
+    "partitions": 256,
+    "partitions_quorum": 256,
+    "partitions_all_ok": 0
+}
+```
+
+#### ConnectClusterNodes `POST /v0/connect`
+
+Instructs this Garage node to connect to other Garage nodes at specified addresses.
+
+Example request body:
+
+```json
+[
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901",
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901"
+]
+```
+
+The format of the string for a node to connect to is: `<node ID>@<ip address>:<port>`, same as in the `garage node connect` CLI call.
+
+Example response:
+
+```json
+[
+    {
+        "success": true,
+        "error": null
+    },
+    {
+        "success": false,
+        "error": "Handshake error"
+    }
+]
+```
+
+#### GetClusterLayout `GET /v0/layout`
+
+Returns the cluster's current layout in JSON, including:
+
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+(the info returned by this endpoint is a subset of the info returned by GetClusterStatus)
+
+Example response body:
+
+```json
+{
+  "version": 12,
+  "roles": {
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+      "zone": "dc1",
+      "capacity": 4,
+      "tags": [
+        "node1"
+      ]
+    },
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+      "zone": "dc1",
+      "capacity": 6,
+      "tags": [
+        "node2"
+      ]
+    },
+    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+      "zone": "dc2",
+      "capacity": 10,
+      "tags": [
+        "node3"
+      ]
+    }
+  },
+  "stagedRoleChanges": {
+    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+      "zone": "dc2",
+      "capacity": 5,
+      "tags": [
+        "node4"
+      ]
+    }
+  }
+}
+```
+
+#### UpdateClusterLayout `POST /v0/layout`
+
+Send modifications to the cluster layout. These modifications will
+be included in the staged role changes, visible in subsequent calls
+of `GetClusterLayout`. Once the set of staged changes is satisfactory,
+the user may call `ApplyClusterLayout` to apply the changed changes,
+or `Revert ClusterLayout` to clear all of the staged changes in
+the layout.
+
+Request body format:
+
+```json
+{
+  <node_id>: {
+    "capacity": <new_capacity>,
+    "zone": <new_zone>,
+    "tags": [
+      <new_tag>,
+      ...
+    ]
+  },
+  <node_id_to_remove>: null,
+  ...
+}
+```
+
+Contrary to the CLI that may update only a subset of the fields
+`capacity`, `zone` and `tags`, when calling this API all of these
+values must be specified.
+
+
+#### ApplyClusterLayout `POST /v0/layout/apply`
+
+Applies to the cluster the layout changes currently registered as
+staged layout changes.
+
+Request body format:
+
+```json
+{
+  "version": 13
+}
+```
+
+Similarly to the CLI, the body must include the version of the new layout
+that will be created, which MUST be 1 + the value of the currently
+existing layout in the cluster.
+
+#### RevertClusterLayout `POST /v0/layout/revert`
+
+Clears all of the staged layout changes.
+
+Request body format:
+
+```json
+{
+  "version": 13
+}
+```
+
+Reverting the staged changes is done by incrementing the version number
+and clearing the contents of the staged change list.
+Similarly to the CLI, the body must include the incremented
+version number, which MUST be 1 + the value of the currently
+existing layout in the cluster.
+
+
+### Access key operations
+
+#### ListKeys `GET /v0/key`
+
+Returns all API access keys in the cluster.
+
+Example response:
+
+```json
+[
+  {
+    "id": "GK31c2f218a2e44f485b94239e",
+    "name": "test"
+  },
+  {
+    "id": "GKe10061ac9c2921f09e4c5540",
+    "name": "test2"
+  }
+]
+```
+
+#### CreateKey `POST /v0/key`
+
+Creates a new API access key.
+
+Request body format:
+
+```json
+{
+    "name": "NameOfMyKey"
+}
+```
+
+#### ImportKey `POST /v0/key/import`
+
+Imports an existing API key.
+
+Request body format:
+
+```json
+{
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
+    "name": "NameOfMyKey"
+}
+```
+
+#### GetKeyInfo `GET /v0/key?id=<acces key id>`
+#### GetKeyInfo `GET /v0/key?search=<pattern>`
+
+Returns information about the requested API access key.
+
+If `id` is set, the key is looked up using its exact identifier (faster).
+If `search` is set, the key is looked up using its name or prefix
+of identifier (slower, all keys are enumerated to do this).
+
+Example response:
+
+```json
+{
+  "name": "test",
+  "accessKeyId": "GK31c2f218a2e44f485b94239e",
+  "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
+  "permissions": {
+    "createBucket": false
+  },
+  "buckets": [
+    {
+      "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
+      "globalAliases": [
+        "test2"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": false
+      }
+    },
+    {
+      "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
+      "globalAliases": [
+        "test3"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": false
+      }
+    },
+    {
+      "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+      "globalAliases": [],
+      "localAliases": [
+        "test"
+      ],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+      }
+    },
+    {
+      "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
+      "globalAliases": [
+        "alex"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+      }
+    }
+  ]
+}
+```
+
+#### DeleteKey `DELETE /v0/key?id=<acces key id>`
+
+Deletes an API access key.
+
+#### UpdateKey `POST /v0/key?id=<acces key id>`
+
+Updates information about the specified API access key.
+
+Request body format:
+
+```json
+{
+    "name": "NameOfMyKey",
+    "allow": {
+        "createBucket": true,
+    },
+    "deny": {}
+}
+```
+
+All fields (`name`, `allow` and `deny`) are optionnal.
+If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed.
+The possible flags in `allow` and `deny` are: `createBucket`.
+
+
+### Bucket operations
+
+#### ListBuckets `GET /v0/bucket`
+
+Returns all storage buckets in the cluster.
+
+Example response:
+
+```json
+[
+  {
+    "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
+    "globalAliases": [
+      "test2"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
+    "globalAliases": [
+      "alex"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
+    "globalAliases": [
+      "test3"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "globalAliases": [],
+    "localAliases": [
+      {
+        "accessKeyId": "GK31c2f218a2e44f485b94239e",
+        "alias": "test"
+      }
+    ]
+  }
+]
+```
+
+#### GetBucketInfo `GET /v0/bucket?id=<bucket id>`
+#### GetBucketInfo `GET /v0/bucket?globalAlias=<alias>`
+
+Returns information about the requested storage bucket.
+
+If `id` is set, the bucket is looked up using its exact identifier.
+If `globalAlias` is set, the bucket is looked up using its global alias.
+(both are fast)
+
+Example response:
+
+```json
+{
+    "id": "afa8f0a22b40b1247ccd0affb869b0af5cff980924a20e4b5e0720a44deb8d39",
+        "globalAliases": [],
+        "websiteAccess": false,
+        "websiteConfig": null,
+        "keys": [
+        {
+            "accessKeyId": "GK31c2f218a2e44f485b94239e",
+            "name": "Imported key",
+            "permissions": {
+                "read": true,
+                "write": true,
+                "owner": true
+            },
+            "bucketLocalAliases": [
+                "debug"
+            ]
+        }
+        ],
+        "objects": 14827,
+        "bytes": 13189855625,
+        "unfinshedUploads": 0,
+        "quotas": {
+            "maxSize": null,
+            "maxObjects": null
+        }
+}
+```
+
+#### CreateBucket `POST /v0/bucket`
+
+Creates a new storage bucket.
+
+Request body format:
+
+```json
+{
+    "globalAlias": "NameOfMyBucket"
+}
+```
+
+OR
+
+```json
+{
+    "localAlias": {
+        "accessKeyId": "GK31c2f218a2e44f485b94239e",
+        "alias": "NameOfMyBucket",
+        "allow": {
+            "read": true,
+            "write": true,
+            "owner": false
+        }
+    }
+}
+```
+
+OR
+
+```json
+{}
+```
+
+Creates a new bucket, either with a global alias, a local one,
+or no alias at all.
+
+Technically, you can also specify both `globalAlias` and `localAlias` and that would create
+two aliases, but I don't see why you would want to do that.
+
+#### DeleteBucket `DELETE /v0/bucket?id=<bucket id>`
+
+Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
+
+Warning: this will delete all aliases associated with the bucket!
+
+#### UpdateBucket `PUT /v0/bucket?id=<bucket id>`
+
+Updates configuration of the given bucket.
+
+Request body format:
+
+```json
+{
+    "websiteAccess": {
+        "enabled": true,
+        "indexDocument": "index.html",
+        "errorDocument": "404.html"
+    },
+    "quotas": {
+        "maxSize": 19029801,
+        "maxObjects": null,
+    }
+}
+```
+
+All fields (`websiteAccess` and `quotas`) are optionnal.
+If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
+
+In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.
+The field `errorDocument` is optional, if no error document is set a generic
+error message is displayed when errors happen. Conversely, if `enabled` is
+`false`, neither `indexDocument` nor `errorDocument` must be specified.
+
+In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null`
+to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
+to change only one of the two quotas.
+
+### Operations on permissions for keys on buckets
+
+#### BucketAllowKey `POST /v0/bucket/allow`
+
+Allows a key to do read/write/owner operations on a bucket.
+
+Request body format:
+
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+    },
+}
+```
+
+Flags in `permissions` which have the value `true` will be activated.
+Other flags will remain unchanged.
+
+#### BucketDenyKey `POST /v0/bucket/deny`
+
+Denies a key from doing read/write/owner operations on a bucket.
+
+Request body format:
+
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "permissions": {
+        "read": false,
+        "write": false,
+        "owner": true
+    },
+}
+```
+
+Flags in `permissions` which have the value `true` will be deactivated.
+Other flags will remain unchanged.
+
+
+### Operations on bucket aliases
+
+#### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+
+Empty body. Creates a global alias for a bucket.
+
+#### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+
+Removes a global alias for a bucket.
+
+#### LocalAliasBucket `PUT /v0/bucket/alias/local?id=<bucket id>&accessKeyId=<access key ID>&alias=<local alias>`
+
+Empty body. Creates a local alias for a bucket in the namespace of a specific access key.
+
+#### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=<bucket id>&accessKeyId<access key ID>&alias=<local alias>`
+
+Removes a local alias for a bucket in the namespace of a specific access key.
+

doc/talks/2022-11-19-Capitole-du-Libre/.gitignore

@@ -0,0 +1,10 @@
+*.aux
+*.bbl
+*.blg
+*.log
+*.nav
+*.out
+*.snm
+*.synctex.gz
+*.toc
+*.dvi

doc/talks/2022-11-19-Capitole-du-Libre/Makefile

@@ -0,0 +1,8 @@
+all:
+	pdflatex présentation.tex
+
+clean:
+	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi présentation.pdf
+
+clean_sauf_pdf:
+	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi

doc/talks/2022-11-19-Capitole-du-Libre/présentation.tex

@@ -0,0 +1,340 @@
+\documentclass[11pt, aspectratio=1610]{beamer}
+\usetheme{Warsaw}
+\usepackage[utf8]{inputenc}
+\usepackage[french]{babel}
+\usepackage{amsmath}
+\usepackage{amsfonts}
+\usepackage{amssymb}
+\usepackage{tikz}
+\usepackage{graphicx}
+\usepackage{xcolor}
+\usepackage{setspace}
+\usepackage{todonotes}
+\presetkeys{todonotes}{inline}{}
+\renewcommand{\baselinestretch}{1.25}
+
+\definecolor{orange_garage}{RGB}{255,147,41}
+\definecolor{gris_garage}{RGB}{78,78,78}
+
+\author[Association Deuxfleurs]{~\linebreak Vincent Giraud}
+\title[De l'auto-hébergement à l'entre-hébergement avec Garage]{De l'auto-hébergement à l'entre-hébergement :\\Garage, pour conserver ses données ensemble}
+%\setbeamercovered{transparent} 
+%\setbeamertemplate{navigation symbols}{} 
+\date{Capitole du Libre 2022\linebreak
+
+\scriptsize Samedi 19 novembre 2022\linebreak
+} 
+
+\setbeamercolor{palette primary}{fg=gris_garage,bg=orange_garage}
+\setbeamercolor{palette secondary}{fg=gris_garage,bg=gris_garage}
+\setbeamercolor{palette tiertary}{fg=white,bg=gris_garage}
+\setbeamercolor{palette quaternary}{fg=white,bg=gris_garage}
+\setbeamercolor{navigation symbols}{fg=black, bg=white}
+\setbeamercolor{navigation symbols dimmed}{fg=darkgray, bg=white}
+\setbeamercolor{itemize item}{fg=gris_garage}
+\setbeamertemplate{itemize item}[circle]
+
+\addtobeamertemplate{navigation symbols}{}{%
+    \usebeamerfont{footline}%
+    \usebeamercolor[fg]{footline}%
+    \hspace{1em}%
+    \insertframenumber/\inserttotalframenumber
+}
+
+\setbeamertemplate{headline}
+{%
+  \leavevmode%
+  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{section in head/foot}%
+    \hbox to .5\paperwidth{\hfil\insertsectionhead\hfil}
+  \end{beamercolorbox}%
+  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{subsection in head/foot}%
+    \hbox to .5\paperwidth{\hfil\insertsubsectionhead\hfil}
+  \end{beamercolorbox}%
+}
+\addtobeamertemplate{footnote}{}{\vspace{2ex}}
+
+\begin{document}
+\begin{frame}
+\titlepage
+\end{frame}
+
+\section{Introduction}
+\subsection{Présentation}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\begin{center}
+\includegraphics[width=3.5cm]{deuxfleurs-logo.png}\linebreak
+
+\texttt{https://deuxfleurs.fr}
+\end{center}
+\column{0.4 \linewidth}
+\begin{center}
+Deuxfleurs est une association militant en faveur d'un internet plus convivial, avec une organisation et des rapports de force repensés.\linebreak
+
+Nous faisons partie du CHATONS\footnote[frame]{Collectif des Hébergeurs Alternatifs, Transparents, Ouverts, Neutres et Solidaires} depuis avril 2022.
+
+\includegraphics[width=2cm]{logo_chatons.png}
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Héberger à la maison}
+\begin{frame}
+\begin{columns}
+\begin{column}{0.5 \linewidth}
+\begin{center}
+Pour échapper au contrôle et au giron des opérateurs de clouds, héberger ses données à la maison présente de nombreux avantages...
+\end{center}
+
+\vspace{0.5cm}
+
+\begin{itemize}[<+(1)->]
+\item On récupère la souveraineté sur ses données
+\item On gagne en vie privée
+\item On gagne en libertés
+\item On est responsabilisé face à ses besoins
+\end{itemize}
+\end{column}
+\vrule{}
+\begin{column}{0.5 \linewidth}
+\begin{center}
+\onslide<6->{... mais aussi bien des contraintes...}
+\end{center}
+
+\vspace{0.5cm}
+
+\begin{itemize}[<+(2)->]
+\item On repose sur une connexion internet pour particulier
+\item Un certain savoir-faire et moultes compétences sont requis
+\item Assurer la résilience de ses services est difficile
+\item Bien sauvegarder ses données, et ceci au-delà de son site géographique, n'est pas évident
+\end{itemize}
+\end{column}
+\end{columns}
+\end{frame}
+
+\subsection{Sauvegarder pour se parer à tout imprévu}
+\begin{frame}
+\begin{center}
+Sauvegarder pour se parer contre les pannes matérielles est une chose...
+
+Sauvegarder pour se parer contre les cambriolages et les incendies en est une autre !\linebreak
+
+\vspace{1cm}
+\onslide<2->{Répartir géographiquement ses données devient alors nécessaire.}
+\end{center}
+\end{frame}
+
+\section{Les solutions à explorer}
+\subsection{L'entre-hébergement}
+\begin{frame}
+\begin{center}
+On a vu récemment se développer au sein du CHATONS la notion d'entre-hébergement : en plus de renforcer l'intégrité des sauvegardes, on va améliorer la disponibilité pendant les coupures de liaison internet, de courant, ou pendant les déménagements d'administrateurs par exemple.\linebreak
+
+\vspace{1cm}
+\onslide<2->
+{
+Dans le cadre du collectif, il s'agit de partager ses volumes de données entre hébergeurs.\linebreak
+
+Pour assurer la confidentialité, on peut chiffrer les données au niveau applicatif.
+}
+\end{center}
+\end{frame}
+
+\subsection{S3 contre les systèmes de fichiers}
+\begin{frame}
+\begin{center}
+Dans le cadre de l'administration de services en ligne, les systèmes de fichiers recèlent certaines difficultés.\linebreak
+
+\vspace{1cm}
+Le standard S3 apporte des facilités; on réduit le stockage à un paradigme de clé-valeur basé essentiellement sur deux opérations seulement: lire ou écrire une clé.
+\end{center}
+\end{frame}
+
+\section{Garage}
+\subsection{Présentation}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\begin{center}
+Garage essaye de répondre à l'ensemble de ces besoins.\linebreak
+
+\vspace{0.5cm}
+Il s'agit d'un logiciel libre permettant de distribuer un service S3 sur diverses machines éloignées.
+\end{center}
+\column{0.5 \linewidth}
+\begin{center}
+\includegraphics[width=4cm]{garage-logo.png}\linebreak
+
+\texttt{https://garagehq.deuxfleurs.fr/}
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Gestion des zones}
+\begin{frame}
+\begin{center}
+Garage va prendre en compte les zones géographiques au moment de répliquer les données.\linebreak
+
+\vspace{1cm}
+\includegraphics[width=13.25cm]{zones.png}
+\end{center}
+\end{frame}
+
+\subsection{Comment ça marche ?}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\input{schéma europe}
+\column{0.5 \linewidth}
+\begin{center}
+Chaque objet est dupliqué sur plusieurs zones différentes.\linebreak
+
+\onslide<5->{Lorsqu'un nouvel hébergeur rejoint le réseau, la charge se voit équilibrée.}\linebreak
+
+\onslide<12->{Si une zone devient indisponible, les autres continuent d'assurer le service.}\linebreak
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Financement}
+\begin{frame}
+\begin{center}
+Dans le cadre du programme \textit{Horizon 2021} de l'Union Européenne, nous avons reçu une subvention de la part de l'initiative NGI Pointer\footnote[frame]{Next Generation Internet Program for Open Internet Renovation}.\linebreak
+
+\includegraphics[width=3cm]{drapeau_européen.png}\hspace{1cm}
+\includegraphics[width=3cm]{NGI.png}\linebreak
+
+Nous avons ainsi pu financer le développement de Garage pendant 1 an.
+\end{center}
+\end{frame}
+
+\subsection{Licence}
+\begin{frame}
+\begin{center}
+De par nos valeurs, nous avons attribué la licence AGPL version 3 à Garage, notamment afin qu'il reste parmi les biens communs.\linebreak
+
+\vspace{0.5cm}
+\includegraphics[width=5cm]{agpl-v3-logo.png}\linebreak
+\end{center}
+\end{frame}
+
+\subsection{Langage utilisé}
+\begin{frame}
+\begin{center}
+Nous avons décidé d'écrire Garage à l'aide du langage Rust, afin d'obtenir une compilation vers des binaires natifs et efficaces.\linebreak
+
+\includegraphics[width=3.5cm]{rust-logo.png}\linebreak
+
+Ce choix permet également de bénéficier des avantages reconnus de Rust en termes de sécurité.
+\end{center}
+\end{frame}
+
+\subsection{Matériel requis}
+\begin{frame}
+\begin{center}
+Garage peut ainsi être performant sur des machines limitées. Les prérequis sont minimes : n'importe quelle machine avec un processeur qui a moins d'une décennie, 1~gigaoctet de mémoire vive, et 16~gigaoctets de stockage suffit.\linebreak
+
+\vspace{1cm}
+
+Cet aspect est déterminant : il permet en effet d'héberger sur du matériel acheté d'occasion, pour réduire l'impact écologique de nos infrastructures.
+\end{center}
+\end{frame}
+
+\subsection{Performances}
+\begin{frame}
+\begin{center}
+\includegraphics[width=13.25cm]{rpc-amplification.png}
+\end{center}
+\end{frame}
+
+\begin{frame}
+\begin{center}
+\includegraphics[width=11cm]{rpc-complexity.png}
+\end{center}
+\end{frame}
+
+\subsection{Services}
+\begin{frame}
+\begin{center}
+Puisqu'il suit le standard S3, beaucoup de services populaires sont par conséquence compatibles avec Garage :\linebreak
+
+\begin{columns}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{nextcloud-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{peertube-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{matrix-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{mastodon-logo.png}
+\end{center}
+\end{columns}
+~\linebreak
+
+Et comme souvent avec S3, on peut assimiler un bucket à un site, et utiliser le serveur pour héberger des sites web statiques.
+\end{center}
+\end{frame}
+
+\section{Intégration chez Deuxfleurs}
+\subsection{Matériel}
+\begin{frame}
+\begin{center}
+\includegraphics[width=13cm]{neptune.jpg}\linebreak
+
+En pratique, nos serveurs ne sont effectivement que des machines achetées d'occasion (très souvent des anciens ordinateurs destinés à la bureautique en entreprise).
+\end{center}
+\end{frame}
+
+\subsection{Environnement logiciel}
+\begin{frame}
+\begin{center}
+Pour faciliter la reproduction d'un environnement connu, NixOS est installé sur nos machines.\linebreak
+
+\vspace{1cm}
+Pour s’accommoder des réseaux qu'on trouve derrière des routeurs pour particuliers, on s'aide de notre logiciel Diplonat\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/diplonat}}.
+\end{center}
+\end{frame}
+
+\section{Au-delà...}
+\subsection{... de Deuxfleurs}
+\begin{frame}
+\begin{center}
+\includegraphics[width=10cm]{tedomum.png}
+\end{center}
+\end{frame}
+
+\subsection{... de Garage}
+\begin{frame}
+\begin{center}
+Nous avons récemment lancé le développement d'Aérogramme\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/aerogramme}}.\linebreak
+
+\vspace{1cm}
+Il s'agit d'un serveur de stockage de courriels chiffrés.\linebreak
+
+\vspace{1cm}
+Il est conçu pour pouvoir travailler avec Garage.
+\end{center}
+\end{frame}
+
+\section{Fin}
+\subsection{Contacts}
+\begin{frame}
+\begin{center}
+\begin{tikzpicture}
+\node (ronce) {\includegraphics[width=0.95\textwidth]{ronce.jpg}};
+\node[white] at (-5.1,3.6) {Intéressé(e) ?};
+\node[white, align=center] at (4.2,-2.6) {Contactez-nous !\\\texttt{coucou@deuxfleurs.fr}\\\texttt{\#forum:deuxfleurs.fr}};
+\end{tikzpicture}
+\end{center}
+\end{frame}
+\end{document}
+

doc/talks/2022-11-19-Capitole-du-Libre/schéma europe.tex

@@ -0,0 +1,52 @@
+\begin{tikzpicture}
+\node (carte) {\includegraphics[width=\textwidth]{carte-Europe.pdf}};
+
+% \personnage{position X}{position Y}{facteur d'échelle}
+\newcommand{\personnage}[4]
+{
+\fill[#4] ({#1-(0.4 * #3)},{#2-(0.9 * #3)}) .. controls ({#1-(0.4 * #3)},#2) and ({#1+(0.4 * #3)},#2) .. ({#1+(0.4 * #3)},{#2-(0.9 * #3)}) -- ({#1-(0.4 * #3)},{#2-(0.9 * #3)});
+\fill[#4] (#1,#2) circle ({0.25 * #3});
+}
+
+\onslide<1-11>{\personnage{-2.25}{-0.75}{0.75}{green}}
+\onslide<1-11>{\draw (-1.9,-1.6) rectangle ++(1,1.2);}
+\onslide<2-11>{\draw[fill=green] (-1.8,-1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4-5>{\draw[fill=red] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<7-11>{\draw[fill=yellow] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<9-11>{\draw[fill=red] (-1.8,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<3-11>{\draw[fill=blue] (-1.35,-1.525) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<8-11>{\draw[fill=blue] (-1.35,-1.15) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
+\onslide<11-11>{\draw[fill=yellow] (-1.35,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\personnage{1.65}{1.5}{0.75}{blue}
+\draw (0.3,0.7) rectangle ++(1,1.2);
+\onslide<2->{\draw[fill=green] (0.4,0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4->{\draw[fill=red] (0.4,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<10->{\draw[fill=green] (0.4,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+\onslide<3->{\draw[fill=blue] (0.85,0.775) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<9->{\draw[fill=red] (0.85,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<11->{\draw[fill=yellow] (0.85,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\personnage{1.85}{-2.3}{0.75}{red}
+\draw (0.5,-3.15) rectangle ++(1,1.2);
+\onslide<2->{\draw[fill=green] (0.6,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4-5>{\draw[fill=red] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<7->{\draw[fill=yellow] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<9->{\draw[fill=red] (0.6,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<3-5>{\draw[fill=blue] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<6->{\draw[fill=red] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<8->{\draw[fill=blue] (1.05,-2.7) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
+\onslide<10->{\draw[fill=green] (1.05,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+
+\onslide<5->{\personnage{1.05}{-0.15}{0.75}{yellow}}
+\onslide<5->{\draw (-0.35,-1) rectangle ++(1,1.2);}
+\onslide<6->{\draw[fill=blue] (-0.25,-0.925) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<7->{\draw[fill=yellow] (-0.25,-0.55) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<10->{\draw[fill=green] (-0.25,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+\onslide<6->{\draw[fill=red] (0.2,-0.925) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<8->{\draw[fill=blue] (0.2,-0.55) rectangle ++(0.35,0.3) node[pos=0.5,white] {\tiny 5};}
+\onslide<11->{\draw[fill=yellow] (0.2,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\onslide<12->{\draw[line width=0.25cm] (-2.15,-0.5) -- ++(1,-1);}
+\onslide<12->{\draw[line width=0.25cm] (-2.15,-1.5) -- ++(1,1);}
+\end{tikzpicture}

flake.lock

@@ -0,0 +1,108 @@
+{
+  "nodes": {
+    "cargo2nix": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1666087781,
+        "narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=",
+        "owner": "Alexis211",
+        "repo": "cargo2nix",
+        "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Alexis211",
+        "repo": "cargo2nix",
+        "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1650374568,
+        "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "b4a34015c698c7793d592d66adbab377907a2be8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1659877975,
+        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1665657542,
+        "narHash": "sha256-mojxNyzbvmp8NtVtxqiHGhRfjCALLfk9i/Uup68Y5q8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a3073c49bc0163fea6a121c276f526837672b555",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a3073c49bc0163fea6a121c276f526837672b555",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "cargo2nix": "cargo2nix",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "flake-utils": [
+          "cargo2nix",
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "cargo2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1664247556,
+        "narHash": "sha256-J4vazHU3609ekn7dr+3wfqPo5WGlZVAgV7jfux352L0=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "524db9c9ea7bc7743bb74cdd45b6d46ea3fcc2ab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,28 @@
+{
+  description = "Garage, an S3-compatible distributed object store for self-hosted deployments";
+
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555";
+  inputs.cargo2nix = {
+    # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
+    url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
+    inputs.nixpkgs.follows = "nixpkgs";
+  };
+
+  outputs = { self, nixpkgs, cargo2nix }: let
+    git_version = self.lastModifiedDate;
+    compile = import ./nix/compile.nix;
+    forAllSystems = nixpkgs.lib.genAttrs nixpkgs.lib.systems.flakeExposed;
+  in
+  {
+    packages = forAllSystems (system: {
+      default = (compile {
+        inherit system git_version;
+        pkgsSrc = nixpkgs;
+        cargo2nixOverlay = cargo2nix.overlays.default;
+        release = true;
+      }).workspace.garage {
+        compileMode = "build";
+      };
+    });
+  };
+}

nix/compile.nix

@@ -1,25 +1,32 @@
 {
-  system ? builtins.currentSystem,
+  system,
-  target,
+  target ? null,
+  pkgsSrc,
+  cargo2nixOverlay,
   compiler ? "rustc",
   release ? false,
   git_version ? null,
   features ? null,
 }:
 
-with import ./common.nix;
-
 let
   log = v: builtins.trace v v;
 
-  pkgs = import pkgsSrc {
+  pkgs =
-    inherit system;
+    if target != null then
-    crossSystem = {
+      import pkgsSrc {
-      config = target;
+        inherit system;
-      isStatic = true;
+        crossSystem = {
-    };
+          config = target;
-    overlays = [ cargo2nixOverlay ];
+          isStatic = true;
-  };
+        };
+        overlays = [ cargo2nixOverlay ];
+      }
+    else
+      import pkgsSrc {
+        inherit system;
+        overlays = [ cargo2nixOverlay ];
+      };
 
   /*
    Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
@@ -34,7 +41,7 @@ let
    NixOS ships them in separate ones. We reunite them with symlinkJoin.
   */
   toolchainOptions =
-    if target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
+    if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
       rustVersion = "1.63.0";
       extraRustComponents = [ "clippy" ];
     } else {

script/helm/garage/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

script/helm/garage/templates/ingress.yaml

@@ -18,6 +18,9 @@ metadata:
   name: {{ $fullName }}-s3-api
   labels:
     {{- include "garage.labels" . | nindent 4 }}
+    {{- with .Values.ingress.s3.api.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.ingress.s3.api.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
@@ -80,6 +83,9 @@ metadata:
   name: {{ $fullName }}-s3-web
   labels:
     {{- include "garage.labels" . | nindent 4 }}
+    {{- with .Values.ingress.s3.web.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.ingress.s3.web.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

script/helm/garage/values.yaml

@@ -85,14 +85,15 @@ service:
 ingress:
   s3:
     api:
-      enabled: true
+      enabled: false
       # Rely either on the className or the annotation below but not both
       # replace "nginx" by an Ingress controller
       # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
-      className: "nginx"
+      # className: "nginx"
-      annotations:
+      annotations: {}
         # kubernetes.io/ingress.class: "nginx"
         # kubernetes.io/tls-acme: "true"
+      labels: {}
       hosts:
         - host: "s3.garage.tld" # garage S3 API endpoint
           paths:
@@ -107,11 +108,15 @@ ingress:
       #    hosts:
       #      - kubernetes.docker.internal
     web:
-      enabled: true
+      enabled: false
-      className: "nginx"
+      # Rely either on the className or the annotation below but not both
+      # replace "nginx" by an Ingress controller
+      # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
+      # className: "nginx"
       annotations: {}
         # kubernetes.io/ingress.class: nginx
         # kubernetes.io/tls-acme: "true"
+      labels: {}
       hosts:
        - host: "*.web.garage.tld" # wildcard website access with bucket name prefix
          paths:

shell.nix

@@ -1,5 +1,5 @@
 {
-  system ? builtins.currentSystem,
+    system ? builtins.currentSystem,
 }:
 
 with import ./nix/common.nix;
@@ -71,13 +71,25 @@ function refresh_cache {
   for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
     echo "Updating cache for ''${attr}"
     derivation=$(nix-instantiate --attr ''${attr})
-    nix copy \
+    nix copy -j8 \
       --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
       $(nix-store -qR ''${derivation%\!bin})
   done
   rm /tmp/nix-signing-key.sec
 }
 
+function refresh_flake_cache {
+  pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+  for attr in packages.x86_64-linux.default; do
+    echo "Updating cache for ''${attr}"
+    derivation=$(nix path-info --derivation ".#''${attr}")
+    nix copy -j8 \
+      --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+      $(nix-store -qR ''${derivation})
+  done
+  rm /tmp/nix-signing-key.sec
+}
+
 function to_s3 {
   aws \
       --endpoint-url https://garage.deuxfleurs.fr \

src/api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,11 +14,11 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_model = { version = "0.8.0", path = "../model" }
+garage_model = { version = "0.8.1", path = "../model" }
-garage_table = { version = "0.8.0", path = "../table" }
+garage_table = { version = "0.8.1", path = "../table" }
-garage_block = { version = "0.8.0", path = "../block" }
+garage_block = { version = "0.8.1", path = "../block" }
-garage_util = { version = "0.8.0", path = "../util" }
+garage_util = { version = "0.8.1", path = "../util" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
 
 async-trait = "0.1.7"
 base64 = "0.13"

src/api/admin/api_server.rs

@@ -15,6 +15,7 @@ use opentelemetry_prometheus::PrometheusExporter;
 use prometheus::{Encoder, TextEncoder};
 
 use garage_model::garage::Garage;
+use garage_rpc::system::ClusterHealthStatus;
 use garage_util::error::Error as GarageError;
 
 use crate::generic_server::*;
@@ -76,6 +77,31 @@ impl AdminApiServer {
 			.body(Body::empty())?)
 	}
 
+	fn handle_health(&self) -> Result<Response<Body>, Error> {
+		let health = self.garage.system.health();
+
+		let (status, status_str) = match health.status {
+			ClusterHealthStatus::Healthy => (StatusCode::OK, "Garage is fully operational"),
+			ClusterHealthStatus::Degraded => (
+				StatusCode::OK,
+				"Garage is operational but some storage nodes are unavailable",
+			),
+			ClusterHealthStatus::Unavailable => (
+				StatusCode::SERVICE_UNAVAILABLE,
+				"Quorum is not available for some/all partitions, reads and writes will fail",
+			),
+		};
+		let status_str = format!(
+			"{}\nConsult the full health check API endpoint at /v0/health for more details\n",
+			status_str
+		);
+
+		Ok(Response::builder()
+			.status(status)
+			.header(http::header::CONTENT_TYPE, "text/plain")
+			.body(Body::from(status_str))?)
+	}
+
 	fn handle_metrics(&self) -> Result<Response<Body>, Error> {
 		#[cfg(feature = "metrics")]
 		{
@@ -124,6 +150,7 @@ impl ApiHandler for AdminApiServer {
 	) -> Result<Response<Body>, Error> {
 		let expected_auth_header =
 			match endpoint.authorization_type() {
+				Authorization::None => None,
 				Authorization::MetricsToken => self.metrics_token.as_ref(),
 				Authorization::AdminToken => match &self.admin_token {
 					None => return Err(Error::forbidden(
@@ -147,8 +174,10 @@ impl ApiHandler for AdminApiServer {
 
 		match endpoint {
 			Endpoint::Options => self.handle_options(&req),
+			Endpoint::Health => self.handle_health(),
 			Endpoint::Metrics => self.handle_metrics(),
 			Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await,
+			Endpoint::GetClusterHealth => handle_get_cluster_health(&self.garage).await,
 			Endpoint::ConnectClusterNodes => handle_connect_cluster_nodes(&self.garage, req).await,
 			// Layout
 			Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await,

src/api/admin/cluster.rs

@@ -43,6 +43,11 @@ pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<
 	Ok(json_ok_response(&res)?)
 }
 
+pub async fn handle_get_cluster_health(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
+	let health = garage.system.health();
+	Ok(json_ok_response(&health)?)
+}
+
 pub async fn handle_connect_cluster_nodes(
 	garage: &Arc<Garage>,
 	req: Request<Body>,

src/api/admin/router.rs

@@ -6,6 +6,7 @@ use crate::admin::error::*;
 use crate::router_macros::*;
 
 pub enum Authorization {
+	None,
 	MetricsToken,
 	AdminToken,
 }
@@ -16,8 +17,10 @@ router_match! {@func
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Endpoint {
 	Options,
+	Health,
 	Metrics,
 	GetClusterStatus,
+	GetClusterHealth,
 	ConnectClusterNodes,
 	// Layout
 	GetClusterLayout,
@@ -88,8 +91,10 @@ impl Endpoint {
 
 		let res = router_match!(@gen_path_parser (req.method(), path, query) [
 			OPTIONS _ => Options,
+			GET "/health" => Health,
 			GET "/metrics" => Metrics,
 			GET "/v0/status" => GetClusterStatus,
+			GET "/v0/health" => GetClusterHealth,
 			POST "/v0/connect" => ConnectClusterNodes,
 			// Layout endpoints
 			GET "/v0/layout" => GetClusterLayout,
@@ -130,6 +135,7 @@ impl Endpoint {
 	/// Get the kind of authorization which is required to perform the operation.
 	pub fn authorization_type(&self) -> Authorization {
 		match self {
+			Self::Health => Authorization::None,
 			Self::Metrics => Authorization::MetricsToken,
 			_ => Authorization::AdminToken,
 		}
@@ -137,9 +143,13 @@ impl Endpoint {
 }
 
 generateQueryParameters! {
-	"id" => id,
+	keywords: [],
-	"search" => search,
+	fields: [
-	"globalAlias" => global_alias,
+		"format" => format,
-	"alias" => alias,
+		"id" => id,
-	"accessKeyId" => access_key_id
+		"search" => search,
+		"globalAlias" => global_alias,
+		"alias" => alias,
+		"accessKeyId" => access_key_id
+	]
 }

src/api/k2v/router.rs

@@ -96,7 +96,7 @@ impl Endpoint {
 	fn from_get(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY if causality_token => PollItem (query::sort_key, query::causality_token, opt_parse::timeout),
 				EMPTY => ReadItem (query::sort_key),
@@ -111,7 +111,7 @@ impl Endpoint {
 	fn from_search(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 			],
 			no_key: [
@@ -125,7 +125,7 @@ impl Endpoint {
 	fn from_head(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
 			],
@@ -140,7 +140,7 @@ impl Endpoint {
 	fn from_post(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 			],
 			no_key: [
@@ -155,7 +155,7 @@ impl Endpoint {
 	fn from_put(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY => InsertItem (query::sort_key),
 
@@ -169,7 +169,7 @@ impl Endpoint {
 	fn from_delete(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY => DeleteItem (query::sort_key),
 			],
@@ -232,21 +232,18 @@ impl Endpoint {
 
 // parameter name => struct field
 generateQueryParameters! {
-	"prefix" => prefix,
+	keywords: [
-	"start" => start,
+		"delete" => DELETE,
-	"causality_token" => causality_token,
+		"search" => SEARCH
-	"end" => end,
+	],
-	"limit" => limit,
+	fields: [
-	"reverse" => reverse,
+		"prefix" => prefix,
-	"sort_key" => sort_key,
+		"start" => start,
-	"timeout" => timeout
+		"causality_token" => causality_token,
-}
+		"end" => end,
-
+		"limit" => limit,
-mod keywords {
+		"reverse" => reverse,
-	//! This module contain all query parameters with no associated value
+		"sort_key" => sort_key,
-	//! used to differentiate endpoints.
+		"timeout" => timeout
-	pub const EMPTY: &str = "";
+	]
-
-	pub const DELETE: &str = "delete";
-	pub const SEARCH: &str = "search";
 }

src/api/router_macros.rs

@@ -4,10 +4,9 @@ macro_rules! router_match {
     (@match $enum:expr , [ $($endpoint:ident,)* ]) => {{
         // usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] }
         // returns true if the variant was one of the listed variants, false otherwise.
-        use Endpoint::*;
         match $enum {
             $(
-            $endpoint { .. } => true,
+            Endpoint::$endpoint { .. } => true,
             )*
             _ => false
         }
@@ -15,37 +14,35 @@ macro_rules! router_match {
     (@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{
         // usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] }
         // returns Some(field_value), or None if the variant was not one of the listed variants.
-        use Endpoint::*;
         match $enum {
             $(
-            $endpoint {$param, ..} => Some($param),
+            Endpoint::$endpoint {$param, ..} => Some($param),
             )*
             _ => None
         }
     }};
-	(@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
+    (@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
-	 [
+     [
-	 $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
+     $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
-	 ]) => {{
+     ]) => {{
-		{
+        {
-			use Endpoint::*;
+            match ($method, $reqpath) {
-			match ($method, $reqpath) {
+                $(
-				$(
+                    (&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api {
-					(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => $api {
+                        $($(
-						$($(
+                            $param: router_match!(@@parse_param $query, $conv, $param),
-							$param: router_match!(@@parse_param $query, $conv, $param),
+                        )*)?
-						)*)?
+                    },
-					},
+                )*
-				)*
+                (m, p) => {
-				(m, p) => {
+                    return Err(Error::bad_request(format!(
-					return Err(Error::bad_request(format!(
+                        "Unknown API endpoint: {} {}",
-						"Unknown API endpoint: {} {}",
+                        m, p
-						m, p
+                    )))
-					)))
+                }
-				}
+            }
-			}
+        }
-		}
+    }};
-	}};
     (@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr),
         key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*],
         no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{
@@ -60,11 +57,9 @@ macro_rules! router_match {
         //   ]
         // }
         // See in from_{method} for more detailed usage.
-        use Endpoint::*;
-        use keywords::*;
         match ($keyword, !$key.is_empty()){
             $(
-            ($kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok($api_k {
+            (Keyword::$kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok(Endpoint::$api_k {
                 $key,
                 $($(
                     $param_k: router_match!(@@parse_param $query, $conv_k, $param_k),
@@ -72,7 +67,7 @@ macro_rules! router_match {
             }),
             )*
             $(
-            ($kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok($api_nk {
+            (Keyword::$kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok(Endpoint::$api_nk {
                 $($(
                     $param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk),
                 )*)?
@@ -84,7 +79,7 @@ macro_rules! router_match {
 
     (@@parse_param $query:expr, query_opt, $param:ident) => {{
         // extract optional query parameter
-		$query.$param.take().map(|param| param.into_owned())
+        $query.$param.take().map(|param| param.into_owned())
     }};
     (@@parse_param $query:expr, query, $param:ident) => {{
         // extract mendatory query parameter
@@ -93,7 +88,7 @@ macro_rules! router_match {
     (@@parse_param $query:expr, opt_parse, $param:ident) => {{
         // extract and parse optional query parameter
         // missing parameter is file, however parse error is reported as an error
-		$query.$param
+        $query.$param
             .take()
             .map(|param| param.parse())
             .transpose()
@@ -144,14 +139,39 @@ macro_rules! router_match {
 /// This macro is used to generate part of the code in this module. It must be called only one, and
 /// is useless outside of this module.
 macro_rules! generateQueryParameters {
-    ( $($rest:expr => $name:ident),* ) => {
+    (
+        keywords: [ $($kw_param:expr => $kw_name: ident),* ],
+        fields: [ $($f_param:expr => $f_name:ident),* ]
+    ) => {
+        #[derive(Debug)]
+        #[allow(non_camel_case_types)]
+        enum Keyword {
+            EMPTY,
+            $( $kw_name, )*
+        }
+
+        impl std::fmt::Display for Keyword {
+            fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+                match self {
+                    Keyword::EMPTY => write!(f, "``"),
+                    $( Keyword::$kw_name => write!(f, "`{}`", $kw_param), )*
+                }
+            }
+        }
+
+        impl Default for Keyword {
+            fn default() -> Self {
+                Keyword::EMPTY
+            }
+        }
+
         /// Struct containing all query parameters used in endpoints. Think of it as an HashMap,
         /// but with keys statically known.
         #[derive(Debug, Default)]
         struct QueryParameters<'a> {
-            keyword: Option<Cow<'a, str>>,
+            keyword: Option<Keyword>,
             $(
-            $name: Option<Cow<'a, str>>,
+            $f_name: Option<Cow<'a, str>>,
             )*
         }
 
@@ -160,34 +180,29 @@ macro_rules! generateQueryParameters {
             fn from_query(query: &'a str) -> Result<Self, Error> {
                 let mut res: Self = Default::default();
                 for (k, v) in url::form_urlencoded::parse(query.as_bytes()) {
-                    let repeated = match k.as_ref() {
+                    match k.as_ref() {
                         $(
-                            $rest => if !v.is_empty() {
+                            $kw_param => if let Some(prev_kw) = res.keyword.replace(Keyword::$kw_name) {
-                                res.$name.replace(v).is_some()
+                                return Err(Error::bad_request(format!(
-                            } else {
+                                    "Multiple keywords: '{}' and '{}'", prev_kw, $kw_param
-                                false
+                                )));
+                            },
+                        )*
+                        $(
+                            $f_param => if !v.is_empty() {
+                                if res.$f_name.replace(v).is_some() {
+                                    return Err(Error::bad_request(format!(
+                                        "Query parameter repeated: '{}'", k
+                                    )));
+                                }
                             },
                         )*
                         _ => {
-                            if k.starts_with("response-") || k.starts_with("X-Amz-") {
+                            if !(k.starts_with("response-") || k.starts_with("X-Amz-")) {
-                                false
-                            } else if v.as_ref().is_empty() {
-                                if res.keyword.replace(k).is_some() {
-                                    return Err(Error::bad_request("Multiple keywords"));
-                                }
-                                continue;
-                            } else {
                                 debug!("Received an unknown query parameter: '{}'", k);
-                                false
                             }
                         }
                     };
-                    if repeated {
-                        return Err(Error::bad_request(format!(
-                            "Query parameter repeated: '{}'",
-                            k
-                        )));
-                    }
                 }
                 Ok(res)
             }
@@ -198,8 +213,8 @@ macro_rules! generateQueryParameters {
                 if self.keyword.is_some() {
                     Some("Keyword not used")
                 } $(
-                    else if self.$name.is_some() {
+                    else if self.$f_name.is_some() {
-                        Some(concat!("'", $rest, "'"))
+                        Some(concat!("'", $f_param, "'"))
                     }
                 )* else {
                     None

src/api/s3/bucket.rs

@@ -161,6 +161,15 @@ pub async fn handle_create_bucket(
 			return Err(CommonError::BucketAlreadyExists.into());
 		}
 	} else {
+		// Check user is allowed to create bucket
+		if !key_params.allow_create_bucket.get() {
+			return Err(CommonError::Forbidden(format!(
+				"Access key {} is not allowed to create buckets",
+				api_key.key_id
+			))
+			.into());
+		}
+
 		// Create the bucket!
 		if !is_valid_bucket_name(&bucket_name) {
 			return Err(Error::bad_request(format!(

src/api/s3/router.rs

@@ -355,7 +355,7 @@ impl Endpoint {
 	fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker),
 				EMPTY => GetObject (query_opt::version_id, opt_parse::part_number),
@@ -412,7 +412,7 @@ impl Endpoint {
 	fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
 			],
@@ -426,7 +426,7 @@ impl Endpoint {
 	fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY if upload_id  => CompleteMultipartUpload (query::upload_id),
 				RESTORE => RestoreObject (query_opt::version_id),
@@ -448,7 +448,7 @@ impl Endpoint {
 	) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, headers),
+			(query.keyword.take().unwrap_or_default(), key, query, headers),
 			key: [
 				EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id),
 				EMPTY header "x-amz-copy-source" => CopyObject,
@@ -490,7 +490,7 @@ impl Endpoint {
 	fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY if upload_id => AbortMultipartUpload (query::upload_id),
 				EMPTY => DeleteObject (query_opt::version_id),
@@ -624,63 +624,60 @@ impl Endpoint {
 
 // parameter name => struct field
 generateQueryParameters! {
-	"continuation-token" => continuation_token,
+	keywords: [
-	"delimiter" => delimiter,
+		"accelerate" => ACCELERATE,
-	"encoding-type" => encoding_type,
+		"acl" => ACL,
-	"fetch-owner" => fetch_owner,
+		"analytics" => ANALYTICS,
-	"id" => id,
+		"cors" => CORS,
-	"key-marker" => key_marker,
+		"delete" => DELETE,
-	"list-type" => list_type,
+		"encryption" => ENCRYPTION,
-	"marker" => marker,
+		"intelligent-tiering" => INTELLIGENT_TIERING,
-	"max-keys" => max_keys,
+		"inventory" => INVENTORY,
-	"max-parts" => max_parts,
+		"legal-hold" => LEGAL_HOLD,
-	"max-uploads" => max_uploads,
+		"lifecycle" => LIFECYCLE,
-	"partNumber" => part_number,
+		"location" => LOCATION,
-	"part-number-marker" => part_number_marker,
+		"logging" => LOGGING,
-	"prefix" => prefix,
+		"metrics" => METRICS,
-	"select-type" => select_type,
+		"notification" => NOTIFICATION,
-	"start-after" => start_after,
+		"object-lock" => OBJECT_LOCK,
-	"uploadId" => upload_id,
+		"ownershipControls" => OWNERSHIP_CONTROLS,
-	"upload-id-marker" => upload_id_marker,
+		"policy" => POLICY,
-	"versionId" => version_id,
+		"policyStatus" => POLICY_STATUS,
-	"version-id-marker" => version_id_marker
+		"publicAccessBlock" => PUBLIC_ACCESS_BLOCK,
-}
+		"replication" => REPLICATION,
-
+		"requestPayment" => REQUEST_PAYMENT,
-mod keywords {
+		"restore" => RESTORE,
-	//! This module contain all query parameters with no associated value S3 uses to differentiate
+		"retention" => RETENTION,
-	//! endpoints.
+		"select" => SELECT,
-	pub const EMPTY: &str = "";
+		"tagging" => TAGGING,
-
+		"torrent" => TORRENT,
-	pub const ACCELERATE: &str = "accelerate";
+		"uploads" => UPLOADS,
-	pub const ACL: &str = "acl";
+		"versioning" => VERSIONING,
-	pub const ANALYTICS: &str = "analytics";
+		"versions" => VERSIONS,
-	pub const CORS: &str = "cors";
+		"website" => WEBSITE
-	pub const DELETE: &str = "delete";
+	],
-	pub const ENCRYPTION: &str = "encryption";
+	fields: [
-	pub const INTELLIGENT_TIERING: &str = "intelligent-tiering";
+		"continuation-token" => continuation_token,
-	pub const INVENTORY: &str = "inventory";
+		"delimiter" => delimiter,
-	pub const LEGAL_HOLD: &str = "legal-hold";
+		"encoding-type" => encoding_type,
-	pub const LIFECYCLE: &str = "lifecycle";
+		"fetch-owner" => fetch_owner,
-	pub const LOCATION: &str = "location";
+		"id" => id,
-	pub const LOGGING: &str = "logging";
+		"key-marker" => key_marker,
-	pub const METRICS: &str = "metrics";
+		"list-type" => list_type,
-	pub const NOTIFICATION: &str = "notification";
+		"marker" => marker,
-	pub const OBJECT_LOCK: &str = "object-lock";
+		"max-keys" => max_keys,
-	pub const OWNERSHIP_CONTROLS: &str = "ownershipControls";
+		"max-parts" => max_parts,
-	pub const POLICY: &str = "policy";
+		"max-uploads" => max_uploads,
-	pub const POLICY_STATUS: &str = "policyStatus";
+		"partNumber" => part_number,
-	pub const PUBLIC_ACCESS_BLOCK: &str = "publicAccessBlock";
+		"part-number-marker" => part_number_marker,
-	pub const REPLICATION: &str = "replication";
+		"prefix" => prefix,
-	pub const REQUEST_PAYMENT: &str = "requestPayment";
+		"select-type" => select_type,
-	pub const RESTORE: &str = "restore";
+		"start-after" => start_after,
-	pub const RETENTION: &str = "retention";
+		"uploadId" => upload_id,
-	pub const SELECT: &str = "select";
+		"upload-id-marker" => upload_id_marker,
-	pub const TAGGING: &str = "tagging";
+		"versionId" => version_id,
-	pub const TORRENT: &str = "torrent";
+		"version-id-marker" => version_id_marker
-	pub const UPLOADS: &str = "uploads";
+	]
-	pub const VERSIONING: &str = "versioning";
-	pub const VERSIONS: &str = "versions";
-	pub const WEBSITE: &str = "website";
 }
 
 #[cfg(test)]

src/block/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,10 +14,10 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
+garage_db = { version = "0.8.1", path = "../db" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
-garage_util = { version = "0.8.0", path = "../util" }
+garage_util = { version = "0.8.1", path = "../util" }
-garage_table = { version = "0.8.0", path = "../table" }
+garage_table = { version = "0.8.1", path = "../table" }
 
 opentelemetry = "0.17"
 

src/block/manager.rs

@@ -90,6 +90,15 @@ pub struct BlockManager {
 	tx_scrub_command: mpsc::Sender<ScrubWorkerCommand>,
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct BlockResyncErrorInfo {
+	pub hash: Hash,
+	pub refcount: u64,
+	pub error_count: u64,
+	pub last_try: u64,
+	pub next_try: u64,
+}
+
 // This custom struct contains functions that must only be ran
 // when the lock is held. We ensure that it is the case by storing
 // it INSIDE a Mutex.
@@ -114,7 +123,8 @@ impl BlockManager {
 			.netapp
 			.endpoint("garage_block/manager.rs/Rpc".to_string());
 
-		let metrics = BlockManagerMetrics::new(resync.queue.clone(), resync.errors.clone());
+		let metrics =
+			BlockManagerMetrics::new(rc.rc.clone(), resync.queue.clone(), resync.errors.clone());
 
 		let (scrub_tx, scrub_rx) = mpsc::channel(1);
 
@@ -309,11 +319,41 @@ impl BlockManager {
 		Ok(self.rc.rc.len()?)
 	}
 
+	/// Get number of items in the refcount table
+	pub fn rc_fast_len(&self) -> Result<Option<usize>, Error> {
+		Ok(self.rc.rc.fast_len()?)
+	}
+
 	/// Send command to start/stop/manager scrub worker
 	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) {
 		let _ = self.tx_scrub_command.send(cmd).await;
 	}
 
+	/// Get the reference count of a block
+	pub fn get_block_rc(&self, hash: &Hash) -> Result<u64, Error> {
+		Ok(self.rc.get_block_rc(hash)?.as_u64())
+	}
+
+	/// List all resync errors
+	pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
+		let mut blocks = Vec::with_capacity(self.resync.errors.len());
+		for ent in self.resync.errors.iter()? {
+			let (hash, cnt) = ent?;
+			let cnt = ErrorCounter::decode(&cnt);
+			blocks.push(BlockResyncErrorInfo {
+				hash: Hash::try_from(&hash).unwrap(),
+				refcount: 0,
+				error_count: cnt.errors,
+				last_try: cnt.last_try,
+				next_try: cnt.next_try(),
+			});
+		}
+		for block in blocks.iter_mut() {
+			block.refcount = self.get_block_rc(&block.hash)?;
+		}
+		Ok(blocks)
+	}
+
 	//// ----- Managing the reference counter ----
 
 	/// Increment the number of time a block is used, putting it to resynchronization if it is

src/block/metrics.rs

@@ -1,9 +1,11 @@
 use opentelemetry::{global, metrics::*};
 
+use garage_db as db;
 use garage_db::counted_tree_hack::CountedTree;
 
 /// TableMetrics reference all counter used for metrics
 pub struct BlockManagerMetrics {
+	pub(crate) _rc_size: ValueObserver<u64>,
 	pub(crate) _resync_queue_len: ValueObserver<u64>,
 	pub(crate) _resync_errored_blocks: ValueObserver<u64>,
 
@@ -23,9 +25,17 @@ pub struct BlockManagerMetrics {
 }
 
 impl BlockManagerMetrics {
-	pub fn new(resync_queue: CountedTree, resync_errors: CountedTree) -> Self {
+	pub fn new(rc_tree: db::Tree, resync_queue: CountedTree, resync_errors: CountedTree) -> Self {
 		let meter = global::meter("garage_model/block");
 		Self {
+			_rc_size: meter
+				.u64_value_observer("block.rc_size", move |observer| {
+					if let Ok(Some(v)) = rc_tree.fast_len() {
+						observer.observe(v as u64, &[])
+					}
+				})
+				.with_description("Number of blocks known to the reference counter")
+				.init(),
 			_resync_queue_len: meter
 				.u64_value_observer("block.resync_queue_length", move |observer| {
 					observer.observe(resync_queue.len() as u64, &[])

src/block/rc.rs

@@ -169,4 +169,11 @@ impl RcEntry {
 	pub(crate) fn is_needed(&self) -> bool {
 		!self.is_deletable()
 	}
+
+	pub(crate) fn as_u64(&self) -> u64 {
+		match self {
+			RcEntry::Present { count } => *count,
+			_ => 0,
+		}
+	}
 }

src/block/repair.rs

@@ -53,7 +53,7 @@ impl Worker for RepairWorker {
 		"Block repair worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
 		match self.block_iter.as_ref() {
 			None => {
 				let idx_bytes = self
@@ -66,9 +66,20 @@ impl Worker for RepairWorker {
 				} else {
 					idx_bytes
 				};
-				Some(format!("Phase 1: {}", hex::encode(idx_bytes)))
+				WorkerStatus {
+					progress: Some("0.00%".into()),
+					freeform: vec![format!(
+						"Currently in phase 1, iterator position: {}",
+						hex::encode(idx_bytes)
+					)],
+					..Default::default()
+				}
 			}
-			Some(bi) => Some(format!("Phase 2: {:.2}% done", bi.progress() * 100.)),
+			Some(bi) => WorkerStatus {
+				progress: Some(format!("{:.2}%", bi.progress() * 100.)),
+				freeform: vec!["Currently in phase 2".into()],
+				..Default::default()
+			},
 		}
 	}
 
@@ -271,29 +282,28 @@ impl Worker for ScrubWorker {
 		"Block scrub worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
-		let s = match &self.work {
+		let mut s = WorkerStatus {
-			ScrubWorkerState::Running(bsi) => format!(
+			persistent_errors: Some(self.persisted.corruptions_detected),
-				"{:.2}% done (tranquility = {})",
+			tranquility: Some(self.persisted.tranquility),
-				bsi.progress() * 100.,
+			..Default::default()
-				self.persisted.tranquility
-			),
-			ScrubWorkerState::Paused(bsi, rt) => {
-				format!(
-					"Paused, {:.2}% done, resumes at {}",
-					bsi.progress() * 100.,
-					msec_to_rfc3339(*rt)
-				)
-			}
-			ScrubWorkerState::Finished => format!(
-				"Last completed scrub: {}",
-				msec_to_rfc3339(self.persisted.time_last_complete_scrub)
-			),
 		};
-		Some(format!(
+		match &self.work {
-			"{} ; corruptions detected: {}",
+			ScrubWorkerState::Running(bsi) => {
-			s, self.persisted.corruptions_detected
+				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
-		))
+			}
+			ScrubWorkerState::Paused(bsi, rt) => {
+				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
+				s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
+			}
+			ScrubWorkerState::Finished => {
+				s.freeform = vec![format!(
+					"Last scrub completed at {}",
+					msec_to_rfc3339(self.persisted.time_last_complete_scrub)
+				)];
+			}
+		}
+		s
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {

src/block/resync.rs

@@ -123,6 +123,24 @@ impl BlockResyncManager {
 		Ok(self.errors.len())
 	}
 
+	/// Clear the error counter for a block and put it in queue immediately
+	pub fn clear_backoff(&self, hash: &Hash) -> Result<(), Error> {
+		let now = now_msec();
+		if let Some(ec) = self.errors.get(hash)? {
+			let mut ec = ErrorCounter::decode(&ec);
+			if ec.errors > 0 {
+				ec.last_try = now - ec.delay_msec();
+				self.errors.insert(hash, ec.encode())?;
+				self.put_to_resync_at(hash, now)?;
+				return Ok(());
+			}
+		}
+		Err(Error::Message(format!(
+			"Block {:?} was not in an errored state",
+			hash
+		)))
+	}
+
 	// ---- Resync loop ----
 
 	// This part manages a queue of blocks that need to be
@@ -257,7 +275,7 @@ impl BlockResyncManager {
 
 				if let Err(e) = &res {
 					manager.metrics.resync_error_counter.add(1);
-					warn!("Error when resyncing {:?}: {}", hash, e);
+					error!("Error when resyncing {:?}: {}", hash, e);
 
 					let err_counter = match self.errors.get(hash.as_slice())? {
 						Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
@@ -477,27 +495,22 @@ impl Worker for ResyncWorker {
 		format!("Block resync worker #{}", self.index + 1)
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
 		let persisted = self.manager.resync.persisted.load();
 
 		if self.index >= persisted.n_workers {
-			return Some("(unused)".into());
+			return WorkerStatus {
+				freeform: vec!["This worker is currently disabled".into()],
+				..Default::default()
+			};
 		}
 
-		let mut ret = vec![];
+		WorkerStatus {
-		ret.push(format!("tranquility = {}", persisted.tranquility));
+			queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
-
+			tranquility: Some(persisted.tranquility),
-		let qlen = self.manager.resync.queue_len().unwrap_or(0);
+			persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
-		if qlen > 0 {
+			..Default::default()
-			ret.push(format!("{} blocks in queue", qlen));
 		}
-
-		let elen = self.manager.resync.errors_len().unwrap_or(0);
-		if elen > 0 {
-			ret.push(format!("{} blocks in error state", elen));
-		}
-
-		Some(ret.join(", "))
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
@@ -545,9 +558,9 @@ impl Worker for ResyncWorker {
 /// and the time of the last try.
 /// Used to implement exponential backoff.
 #[derive(Clone, Copy, Debug)]
-struct ErrorCounter {
+pub(crate) struct ErrorCounter {
-	errors: u64,
+	pub(crate) errors: u64,
-	last_try: u64,
+	pub(crate) last_try: u64,
 }
 
 impl ErrorCounter {
@@ -558,12 +571,13 @@ impl ErrorCounter {
 		}
 	}
 
-	fn decode(data: &[u8]) -> Self {
+	pub(crate) fn decode(data: &[u8]) -> Self {
 		Self {
 			errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
 			last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
 		}
 	}
+
 	fn encode(&self) -> Vec<u8> {
 		[
 			u64::to_be_bytes(self.errors),
@@ -583,7 +597,8 @@ impl ErrorCounter {
 		(RESYNC_RETRY_DELAY.as_millis() as u64)
 			<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
 	}
-	fn next_try(&self) -> u64 {
+
+	pub(crate) fn next_try(&self) -> u64 {
 		self.last_try + self.delay_msec()
 	}
 }

src/db/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"

src/db/lib.rs

@@ -181,6 +181,10 @@ impl Tree {
 	pub fn len(&self) -> Result<usize> {
 		self.0.len(self.1)
 	}
+	#[inline]
+	pub fn fast_len(&self) -> Result<Option<usize>> {
+		self.0.fast_len(self.1)
+	}
 
 	#[inline]
 	pub fn first(&self) -> Result<Option<(Value, Value)>> {
@@ -323,6 +327,9 @@ pub(crate) trait IDb: Send + Sync {
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
 	fn len(&self, tree: usize) -> Result<usize>;
+	fn fast_len(&self, _tree: usize) -> Result<Option<usize>> {
+		Ok(None)
+	}
 
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>>;
 	fn remove(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;

src/db/lmdb_adapter.rs

@@ -121,6 +121,10 @@ impl IDb for LmdbDb {
 		Ok(tree.len(&tx)?.try_into().unwrap())
 	}
 
+	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
+		Ok(Some(self.len(tree)?))
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		let tree = self.get_tree(tree)?;
 		let mut tx = self.db.write_txn()?;

src/db/sqlite_adapter.rs

@@ -144,6 +144,10 @@ impl IDb for SqliteDb {
 		}
 	}
 
+	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
+		Ok(Some(self.len(tree)?))
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		trace!("insert {}: lock db", tree);
 		let this = self.0.lock().unwrap();

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -21,14 +21,14 @@ path = "tests/lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
+garage_db = { version = "0.8.1", path = "../db" }
-garage_api = { version = "0.8.0", path = "../api" }
+garage_api = { version = "0.8.1", path = "../api" }
-garage_block = { version = "0.8.0", path = "../block" }
+garage_block = { version = "0.8.1", path = "../block" }
-garage_model = { version = "0.8.0", path = "../model" }
+garage_model = { version = "0.8.1", path = "../model" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
-garage_table = { version = "0.8.0", path = "../table" }
+garage_table = { version = "0.8.1", path = "../table" }
-garage_util = { version = "0.8.0", path = "../util" }
+garage_util = { version = "0.8.1", path = "../util" }
-garage_web = { version = "0.8.0", path = "../web" }
+garage_web = { version = "0.8.1", path = "../web" }
 
 backtrace = "0.3"
 bytes = "1.0"
@@ -36,7 +36,7 @@ bytesize = "1.1"
 timeago = "0.3"
 parse_duration = "2.1"
 hex = "0.4"
-tracing = { version = "0.1.30", features = ["log-always"] }
+tracing = { version = "0.1.30" }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 rand = "0.8"
 async-trait = "0.1.7"

src/garage/admin.rs

@@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize};
 use garage_util::crdt::*;
 use garage_util::data::*;
 use garage_util::error::Error as GarageError;
+use garage_util::formater::format_table_to_string;
 use garage_util::time::*;
 
 use garage_table::replication::*;
@@ -15,6 +16,7 @@ use garage_table::*;
 
 use garage_rpc::*;
 
+use garage_block::manager::BlockResyncErrorInfo;
 use garage_block::repair::ScrubWorkerCommand;
 
 use garage_model::bucket_alias_table::*;
@@ -24,6 +26,8 @@ use garage_model::helper::error::{Error, OkOrBadRequest};
 use garage_model::key_table::*;
 use garage_model::migrate::Migrate;
 use garage_model::permission::*;
+use garage_model::s3::object_table::*;
+use garage_model::s3::version_table::Version;
 
 use crate::cli::*;
 use crate::repair::online::launch_online_repair;
@@ -38,7 +42,8 @@ pub enum AdminRpc {
 	LaunchRepair(RepairOpt),
 	Migrate(MigrateOpt),
 	Stats(StatsOpt),
-	Worker(WorkerOpt),
+	Worker(WorkerOperation),
+	BlockOperation(BlockOperation),
 
 	// Replies
 	Ok(String),
@@ -54,6 +59,13 @@ pub enum AdminRpc {
 		HashMap<usize, garage_util::background::WorkerInfo>,
 		WorkerListOpt,
 	),
+	WorkerInfo(usize, garage_util::background::WorkerInfo),
+	BlockErrorList(Vec<BlockResyncErrorInfo>),
+	BlockInfo {
+		hash: Hash,
+		refcount: u64,
+		versions: Vec<Result<Version, Uuid>>,
+	},
 }
 
 impl Rpc for AdminRpc {
@@ -73,6 +85,8 @@ impl AdminRpcHandler {
 		admin
 	}
 
+	// ================ BUCKET COMMANDS ====================
+
 	async fn handle_bucket_cmd(&self, cmd: &BucketOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			BucketOperation::List => self.handle_list_buckets().await,
@@ -551,6 +565,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok(ret))
 	}
 
+	// ================ KEY COMMANDS ====================
+
 	async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			KeyOperation::List => self.handle_list_keys().await,
@@ -688,6 +704,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::KeyInfo(key, relevant_buckets))
 	}
 
+	// ================ MIGRATION COMMANDS ====================
+
 	async fn handle_migrate(self: &Arc<Self>, opt: MigrateOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -704,6 +722,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok("Migration successfull.".into()))
 	}
 
+	// ================ REPAIR COMMANDS ====================
+
 	async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -747,6 +767,8 @@ impl AdminRpcHandler {
 		}
 	}
 
+	// ================ STATS COMMANDS ====================
+
 	async fn handle_stats(&self, opt: StatsOpt) -> Result<AdminRpc, Error> {
 		if opt.all_nodes {
 			let mut ret = String::new();
@@ -763,11 +785,12 @@ impl AdminRpcHandler {
 				match self
 					.endpoint
 					.call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL)
-					.await?
+					.await
 				{
-					Ok(AdminRpc::Ok(s)) => writeln!(&mut ret, "{}", s).unwrap(),
+					Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(),
-					Ok(x) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
+					Ok(Ok(x)) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
-					Err(e) => writeln!(&mut ret, "Error: {}", e).unwrap(),
+					Ok(Err(e)) => writeln!(&mut ret, "Remote error: {}", e).unwrap(),
+					Err(e) => writeln!(&mut ret, "Network error: {}", e).unwrap(),
 				}
 			}
 			Ok(AdminRpc::Ok(ret))
@@ -787,6 +810,7 @@ impl AdminRpcHandler {
 				.unwrap_or_else(|| "(unknown)".into()),
 		)
 		.unwrap();
+
 		writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();
 
 		// Gather ring statistics
@@ -805,21 +829,38 @@ impl AdminRpcHandler {
 			writeln!(&mut ret, "  {:?} {}", n, c).unwrap();
 		}
 
-		self.gather_table_stats(&mut ret, &self.garage.bucket_table, &opt)?;
+		// Gather table statistics
-		self.gather_table_stats(&mut ret, &self.garage.key_table, &opt)?;
+		let mut table = vec!["  Table\tItems\tMklItems\tMklTodo\tGcTodo".into()];
-		self.gather_table_stats(&mut ret, &self.garage.object_table, &opt)?;
+		table.push(self.gather_table_stats(&self.garage.bucket_table, opt.detailed)?);
-		self.gather_table_stats(&mut ret, &self.garage.version_table, &opt)?;
+		table.push(self.gather_table_stats(&self.garage.key_table, opt.detailed)?);
-		self.gather_table_stats(&mut ret, &self.garage.block_ref_table, &opt)?;
+		table.push(self.gather_table_stats(&self.garage.object_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.version_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.block_ref_table, opt.detailed)?);
+		write!(
+			&mut ret,
+			"\nTable stats:\n{}",
+			format_table_to_string(table)
+		)
+		.unwrap();
 
+		// Gather block manager statistics
 		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
-		if opt.detailed {
+		let rc_len = if opt.detailed {
-			writeln!(
+			self.garage.block_manager.rc_len()?.to_string()
-				&mut ret,
+		} else {
-				"  number of RC entries (~= number of blocks): {}",
+			self.garage
-				self.garage.block_manager.rc_len()?
+				.block_manager
-			)
+				.rc_fast_len()?
-			.unwrap();
+				.map(|x| x.to_string())
-		}
+				.unwrap_or_else(|| "NC".into())
+		};
+
+		writeln!(
+			&mut ret,
+			"  number of RC entries (~= number of blocks): {}",
+			rc_len
+		)
+		.unwrap();
 		writeln!(
 			&mut ret,
 			"  resync queue length: {}",
@@ -833,67 +874,84 @@ impl AdminRpcHandler {
 		)
 		.unwrap();
 
+		if !opt.detailed {
+			writeln!(&mut ret, "\nIf values are missing (marked as NC), consider adding the --detailed flag - this will be slow.").unwrap();
+		}
+
 		Ok(ret)
 	}
 
 	fn gather_table_stats<F, R>(
 		&self,
-		to: &mut String,
 		t: &Arc<Table<F, R>>,
-		opt: &StatsOpt,
+		detailed: bool,
-	) -> Result<(), Error>
+	) -> Result<String, Error>
 	where
 		F: TableSchema + 'static,
 		R: TableReplication + 'static,
 	{
-		writeln!(to, "\nTable stats for {}", F::TABLE_NAME).unwrap();
+		let (data_len, mkl_len) = if detailed {
-		if opt.detailed {
+			(
-			writeln!(
+				t.data.store.len().map_err(GarageError::from)?.to_string(),
-				to,
+				t.merkle_updater.merkle_tree_len()?.to_string(),
-				"  number of items: {}",
-				t.data.store.len().map_err(GarageError::from)?
 			)
-			.unwrap();
+		} else {
-			writeln!(
+			(
-				to,
+				t.data
-				"  Merkle tree size: {}",
+					.store
-				t.merkle_updater.merkle_tree_len()?
+					.fast_len()
+					.map_err(GarageError::from)?
+					.map(|x| x.to_string())
+					.unwrap_or_else(|| "NC".into()),
+				t.merkle_updater
+					.merkle_tree_fast_len()?
+					.map(|x| x.to_string())
+					.unwrap_or_else(|| "NC".into()),
 			)
-			.unwrap();
+		};
-		}
-		writeln!(
-			to,
-			"  Merkle updater todo queue length: {}",
-			t.merkle_updater.todo_len()?
-		)
-		.unwrap();
-		writeln!(to, "  GC todo queue length: {}", t.data.gc_todo_len()?).unwrap();
 
-		Ok(())
+		Ok(format!(
+			"  {}\t{}\t{}\t{}\t{}",
+			F::TABLE_NAME,
+			data_len,
+			mkl_len,
+			t.merkle_updater.todo_len()?,
+			t.data.gc_todo_len()?
+		))
 	}
 
-	// ----
+	// ================ WORKER COMMANDS ====================
 
-	async fn handle_worker_cmd(&self, opt: WorkerOpt) -> Result<AdminRpc, Error> {
+	async fn handle_worker_cmd(&self, cmd: &WorkerOperation) -> Result<AdminRpc, Error> {
-		match opt.cmd {
+		match cmd {
-			WorkerCmd::List { opt } => {
+			WorkerOperation::List { opt } => {
 				let workers = self.garage.background.get_worker_info();
-				Ok(AdminRpc::WorkerList(workers, opt))
+				Ok(AdminRpc::WorkerList(workers, *opt))
 			}
-			WorkerCmd::Set { opt } => match opt {
+			WorkerOperation::Info { tid } => {
+				let info = self
+					.garage
+					.background
+					.get_worker_info()
+					.get(tid)
+					.ok_or_bad_request(format!("No worker with TID {}", tid))?
+					.clone();
+				Ok(AdminRpc::WorkerInfo(*tid, info))
+			}
+			WorkerOperation::Set { opt } => match opt {
 				WorkerSetCmd::ScrubTranquility { tranquility } => {
-					let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);
+					let scrub_command = ScrubWorkerCommand::SetTranquility(*tranquility);
 					self.garage
 						.block_manager
 						.send_scrub_command(scrub_command)
 						.await;
 					Ok(AdminRpc::Ok("Scrub tranquility updated".into()))
 				}
-				WorkerSetCmd::ResyncNWorkers { n_workers } => {
+				WorkerSetCmd::ResyncWorkerCount { worker_count } => {
 					self.garage
 						.block_manager
 						.resync
-						.set_n_workers(n_workers)
+						.set_n_workers(*worker_count)
 						.await?;
 					Ok(AdminRpc::Ok("Number of resync workers updated".into()))
 				}
@@ -901,13 +959,154 @@ impl AdminRpcHandler {
 					self.garage
 						.block_manager
 						.resync
-						.set_tranquility(tranquility)
+						.set_tranquility(*tranquility)
 						.await?;
 					Ok(AdminRpc::Ok("Resync tranquility updated".into()))
 				}
 			},
 		}
 	}
+
+	// ================ BLOCK COMMANDS ====================
+
+	async fn handle_block_cmd(&self, cmd: &BlockOperation) -> Result<AdminRpc, Error> {
+		match cmd {
+			BlockOperation::ListErrors => Ok(AdminRpc::BlockErrorList(
+				self.garage.block_manager.list_resync_errors()?,
+			)),
+			BlockOperation::Info { hash } => {
+				let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+				let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+				let refcount = self.garage.block_manager.get_block_rc(&hash)?;
+				let block_refs = self
+					.garage
+					.block_ref_table
+					.get_range(&hash, None, None, 10000, Default::default())
+					.await?;
+				let mut versions = vec![];
+				for br in block_refs {
+					if let Some(v) = self
+						.garage
+						.version_table
+						.get(&br.version, &EmptyKey)
+						.await?
+					{
+						versions.push(Ok(v));
+					} else {
+						versions.push(Err(br.version));
+					}
+				}
+				Ok(AdminRpc::BlockInfo {
+					hash,
+					refcount,
+					versions,
+				})
+			}
+			BlockOperation::RetryNow { all, blocks } => {
+				if *all {
+					if !blocks.is_empty() {
+						return Err(Error::BadRequest(
+							"--all was specified, cannot also specify blocks".into(),
+						));
+					}
+					let blocks = self.garage.block_manager.list_resync_errors()?;
+					for b in blocks.iter() {
+						self.garage.block_manager.resync.clear_backoff(&b.hash)?;
+					}
+					Ok(AdminRpc::Ok(format!(
+						"{} blocks returned in queue for a retry now (check logs to see results)",
+						blocks.len()
+					)))
+				} else {
+					for hash in blocks {
+						let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+						let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+						self.garage.block_manager.resync.clear_backoff(&hash)?;
+					}
+					Ok(AdminRpc::Ok(format!(
+						"{} blocks returned in queue for a retry now (check logs to see results)",
+						blocks.len()
+					)))
+				}
+			}
+			BlockOperation::Purge { yes, blocks } => {
+				if !yes {
+					return Err(Error::BadRequest(
+						"Pass the --yes flag to confirm block purge operation.".into(),
+					));
+				}
+
+				let mut obj_dels = 0;
+				let mut ver_dels = 0;
+
+				for hash in blocks {
+					let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+					let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+					let block_refs = self
+						.garage
+						.block_ref_table
+						.get_range(&hash, None, None, 10000, Default::default())
+						.await?;
+
+					for br in block_refs {
+						let version = match self
+							.garage
+							.version_table
+							.get(&br.version, &EmptyKey)
+							.await?
+						{
+							Some(v) => v,
+							None => continue,
+						};
+
+						if let Some(object) = self
+							.garage
+							.object_table
+							.get(&version.bucket_id, &version.key)
+							.await?
+						{
+							let ov = object.versions().iter().rev().find(|v| v.is_complete());
+							if let Some(ov) = ov {
+								if ov.uuid == br.version {
+									let del_uuid = gen_uuid();
+									let deleted_object = Object::new(
+										version.bucket_id,
+										version.key.clone(),
+										vec![ObjectVersion {
+											uuid: del_uuid,
+											timestamp: ov.timestamp + 1,
+											state: ObjectVersionState::Complete(
+												ObjectVersionData::DeleteMarker,
+											),
+										}],
+									);
+									self.garage.object_table.insert(&deleted_object).await?;
+									obj_dels += 1;
+								}
+							}
+						}
+
+						if !version.deleted.get() {
+							let deleted_version = Version::new(
+								version.uuid,
+								version.bucket_id,
+								version.key.clone(),
+								true,
+							);
+							self.garage.version_table.insert(&deleted_version).await?;
+							ver_dels += 1;
+						}
+					}
+				}
+				Ok(AdminRpc::Ok(format!(
+					"{} blocks were purged: {} object deletion markers added, {} versions marked deleted",
+					blocks.len(),
+					obj_dels,
+					ver_dels
+				)))
+			}
+		}
+	}
 }
 
 #[async_trait]
@@ -923,7 +1122,8 @@ impl EndpointHandler<AdminRpc> for AdminRpcHandler {
 			AdminRpc::Migrate(opt) => self.handle_migrate(opt.clone()).await,
 			AdminRpc::LaunchRepair(opt) => self.handle_launch_repair(opt.clone()).await,
 			AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await,
-			AdminRpc::Worker(opt) => self.handle_worker_cmd(opt.clone()).await,
+			AdminRpc::Worker(wo) => self.handle_worker_cmd(wo).await,
+			AdminRpc::BlockOperation(bo) => self.handle_block_cmd(bo).await,
 			m => Err(GarageError::unexpected_rpc_message(m).into()),
 		}
 	}

src/garage/cli/cmd.rs

@@ -41,6 +41,9 @@ pub async fn cli_command_dispatch(
 		}
 		Command::Stats(so) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Stats(so)).await,
 		Command::Worker(wo) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Worker(wo)).await,
+		Command::Block(bo) => {
+			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await
+		}
 		_ => unreachable!(),
 	}
 }
@@ -186,7 +189,20 @@ pub async fn cmd_admin(
 			print_key_info(&key, &rb);
 		}
 		AdminRpc::WorkerList(wi, wlo) => {
-			print_worker_info(wi, wlo);
+			print_worker_list(wi, wlo);
+		}
+		AdminRpc::WorkerInfo(tid, wi) => {
+			print_worker_info(tid, wi);
+		}
+		AdminRpc::BlockErrorList(el) => {
+			print_block_error_list(el);
+		}
+		AdminRpc::BlockInfo {
+			hash,
+			refcount,
+			versions,
+		} => {
+			print_block_info(hash, refcount, versions);
 		}
 		r => {
 			error!("Unexpected response: {:?}", r);

src/garage/cli/structs.rs

@@ -49,7 +49,11 @@ pub enum Command {
 
 	/// Manage background workers
 	#[structopt(name = "worker", version = garage_version())]
-	Worker(WorkerOpt),
+	Worker(WorkerOperation),
+
+	/// Low-level debug operations on data blocks
+	#[structopt(name = "block", version = garage_version())]
+	Block(BlockOperation),
 }
 
 #[derive(StructOpt, Debug)]
@@ -502,20 +506,17 @@ pub struct StatsOpt {
 	pub detailed: bool,
 }
 
-#[derive(Serialize, Deserialize, StructOpt, Debug, Clone)]
-pub struct WorkerOpt {
-	#[structopt(subcommand)]
-	pub cmd: WorkerCmd,
-}
-
 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
-pub enum WorkerCmd {
+pub enum WorkerOperation {
 	/// List all workers on Garage node
 	#[structopt(name = "list", version = garage_version())]
 	List {
 		#[structopt(flatten)]
 		opt: WorkerListOpt,
 	},
+	/// Get detailed information about a worker
+	#[structopt(name = "info", version = garage_version())]
+	Info { tid: usize },
 	/// Set worker parameter
 	#[structopt(name = "set", version = garage_version())]
 	Set {
@@ -540,9 +541,41 @@ pub enum WorkerSetCmd {
 	#[structopt(name = "scrub-tranquility", version = garage_version())]
 	ScrubTranquility { tranquility: u32 },
 	/// Set number of concurrent block resync workers
-	#[structopt(name = "resync-n-workers", version = garage_version())]
+	#[structopt(name = "resync-worker-count", version = garage_version())]
-	ResyncNWorkers { n_workers: usize },
+	ResyncWorkerCount { worker_count: usize },
 	/// Set tranquility of block resync operations
 	#[structopt(name = "resync-tranquility", version = garage_version())]
 	ResyncTranquility { tranquility: u32 },
 }
+
+#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
+pub enum BlockOperation {
+	/// List all blocks that currently have a resync error
+	#[structopt(name = "list-errors", version = garage_version())]
+	ListErrors,
+	/// Get detailed information about a single block
+	#[structopt(name = "info", version = garage_version())]
+	Info {
+		/// Hash of the block for which to retrieve information
+		hash: String,
+	},
+	/// Retry now the resync of one or many blocks
+	#[structopt(name = "retry-now", version = garage_version())]
+	RetryNow {
+		/// Retry all blocks that have a resync error
+		#[structopt(long = "all")]
+		all: bool,
+		/// Hashes of the block to retry to resync now
+		blocks: Vec<String>,
+	},
+	/// Delete all objects referencing a missing block
+	#[structopt(name = "purge", version = garage_version())]
+	Purge {
+		/// Mandatory to confirm this operation
+		#[structopt(long = "yes")]
+		yes: bool,
+		/// Hashes of the block to purge
+		#[structopt(required = true)]
+		blocks: Vec<String>,
+	},
+}

src/garage/cli/util.rs

@@ -3,14 +3,17 @@ use std::time::Duration;
 
 use garage_util::background::*;
 use garage_util::crdt::*;
-use garage_util::data::Uuid;
+use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::formater::format_table;
 use garage_util::time::*;
 
+use garage_block::manager::BlockResyncErrorInfo;
+
 use garage_model::bucket_table::*;
 use garage_model::key_table::*;
 use garage_model::s3::object_table::{BYTES, OBJECTS, UNFINISHED_UPLOADS};
+use garage_model::s3::version_table::Version;
 
 use crate::cli::structs::WorkerListOpt;
 
@@ -241,7 +244,7 @@ pub fn find_matching_node(
 	}
 }
 
-pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
+pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 	let mut wi = wi.into_iter().collect::<Vec<_>>();
 	wi.sort_by_key(|(tid, info)| {
 		(
@@ -254,7 +257,7 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 		)
 	});
 
-	let mut table = vec![];
+	let mut table = vec!["TID\tState\tName\tTranq\tDone\tQueue\tErrors\tConsec\tLast".to_string()];
 	for (tid, info) in wi.iter() {
 		if wlo.busy && !matches!(info.state, WorkerState::Busy | WorkerState::Throttled(_)) {
 			continue;
@@ -263,33 +266,147 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 			continue;
 		}
 
-		table.push(format!("{}\t{}\t{}", tid, info.state, info.name));
-		if let Some(i) = &info.info {
-			table.push(format!("\t\t  {}", i));
-		}
 		let tf = timeago::Formatter::new();
-		let (err_ago, err_msg) = info
+		let err_ago = info
 			.last_error
 			.as_ref()
-			.map(|(m, t)| {
+			.map(|(_, t)| tf.convert(Duration::from_millis(now_msec() - t)))
-				(
+			.unwrap_or_default();
-					tf.convert(Duration::from_millis(now_msec() - t)),
+		let (total_err, consec_err) = if info.errors > 0 {
-					m.as_str(),
+			(info.errors.to_string(), info.consecutive_errors.to_string())
-				)
+		} else {
-			})
+			("-".into(), "-".into())
-			.unwrap_or(("(?) ago".into(), "(?)"));
+		};
-		if info.consecutive_errors > 0 {
+
+		table.push(format!(
+			"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
+			tid,
+			info.state,
+			info.name,
+			info.status
+				.tranquility
+				.as_ref()
+				.map(ToString::to_string)
+				.unwrap_or_else(|| "-".into()),
+			info.status.progress.as_deref().unwrap_or("-"),
+			info.status
+				.queue_length
+				.as_ref()
+				.map(ToString::to_string)
+				.unwrap_or_else(|| "-".into()),
+			total_err,
+			consec_err,
+			err_ago,
+		));
+	}
+	format_table(table);
+}
+
+pub fn print_worker_info(tid: usize, info: WorkerInfo) {
+	let mut table = vec![];
+	table.push(format!("Task id:\t{}", tid));
+	table.push(format!("Worker name:\t{}", info.name));
+	match info.state {
+		WorkerState::Throttled(t) => {
 			table.push(format!(
-				"\t\t  {} consecutive errors ({} total), last {}",
+				"Worker state:\tBusy (throttled, paused for {:.3}s)",
-				info.consecutive_errors, info.errors, err_ago,
+				t
 			));
-			table.push(format!("\t\t  {}", err_msg));
+		}
-		} else if info.errors > 0 {
+		s => {
-			table.push(format!("\t\t  ({} errors, last {})", info.errors, err_ago,));
+			table.push(format!("Worker state:\t{}", s));
-			if wlo.errors {
+		}
-				table.push(format!("\t\t  {}", err_msg));
+	};
+	if let Some(tql) = info.status.tranquility {
+		table.push(format!("Tranquility:\t{}", tql));
+	}
+
+	table.push("".into());
+	table.push(format!("Total errors:\t{}", info.errors));
+	table.push(format!("Consecutive errs:\t{}", info.consecutive_errors));
+	if let Some((s, t)) = info.last_error {
+		table.push(format!("Last error:\t{}", s));
+		let tf = timeago::Formatter::new();
+		table.push(format!(
+			"Last error time:\t{}",
+			tf.convert(Duration::from_millis(now_msec() - t))
+		));
+	}
+
+	table.push("".into());
+	if let Some(p) = info.status.progress {
+		table.push(format!("Progress:\t{}", p));
+	}
+	if let Some(ql) = info.status.queue_length {
+		table.push(format!("Queue length:\t{}", ql));
+	}
+	if let Some(pe) = info.status.persistent_errors {
+		table.push(format!("Persistent errors:\t{}", pe));
+	}
+
+	for (i, s) in info.status.freeform.iter().enumerate() {
+		if i == 0 {
+			if table.last() != Some(&"".into()) {
+				table.push("".into());
 			}
+			table.push(format!("Message:\t{}", s));
+		} else {
+			table.push(format!("\t{}", s));
 		}
 	}
 	format_table(table);
 }
+
+pub fn print_block_error_list(el: Vec<BlockResyncErrorInfo>) {
+	let now = now_msec();
+	let tf = timeago::Formatter::new();
+	let mut tf2 = timeago::Formatter::new();
+	tf2.ago("");
+
+	let mut table = vec!["Hash\tRC\tErrors\tLast error\tNext try".into()];
+	for e in el {
+		table.push(format!(
+			"{}\t{}\t{}\t{}\tin {}",
+			hex::encode(e.hash.as_slice()),
+			e.refcount,
+			e.error_count,
+			tf.convert(Duration::from_millis(now - e.last_try)),
+			tf2.convert(Duration::from_millis(e.next_try - now))
+		));
+	}
+	format_table(table);
+}
+
+pub fn print_block_info(hash: Hash, refcount: u64, versions: Vec<Result<Version, Uuid>>) {
+	println!("Block hash: {}", hex::encode(hash.as_slice()));
+	println!("Refcount: {}", refcount);
+	println!();
+
+	let mut table = vec!["Version\tBucket\tKey\tDeleted".into()];
+	let mut nondeleted_count = 0;
+	for v in versions.iter() {
+		match v {
+			Ok(ver) => {
+				table.push(format!(
+					"{:?}\t{:?}\t{}\t{:?}",
+					ver.uuid,
+					ver.bucket_id,
+					ver.key,
+					ver.deleted.get()
+				));
+				if !ver.deleted.get() {
+					nondeleted_count += 1;
+				}
+			}
+			Err(vh) => {
+				table.push(format!("{:?}\t\t\tyes", vh));
+			}
+		}
+	}
+	format_table(table);
+
+	if refcount != nondeleted_count {
+		println!();
+		println!("Warning: refcount does not match number of non-deleted versions");
+	}
+}

src/garage/main.rs

@@ -127,9 +127,16 @@ async fn main() {
 		std::process::abort();
 	}));
 
+	// Parse arguments and dispatch command line
+	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
+
 	// Initialize logging as well as other libraries used in Garage
 	if std::env::var("RUST_LOG").is_err() {
-		std::env::set_var("RUST_LOG", "netapp=info,garage=info")
+		let default_log = match &opt.cmd {
+			Command::Server => "netapp=info,garage=info",
+			_ => "netapp=warn,garage=warn",
+		};
+		std::env::set_var("RUST_LOG", default_log)
 	}
 	tracing_subscriber::fmt()
 		.with_writer(std::io::stderr)
@@ -137,9 +144,6 @@ async fn main() {
 		.init();
 	sodiumoxide::init().expect("Unable to init sodiumoxide");
 
-	// Parse arguments and dispatch command line
-	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
-
 	let res = match opt.cmd {
 		Command::Server => server::run_server(opt.config_file).await,
 		Command::OfflineRepair(repair_opt) => {
@@ -182,9 +186,9 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk);
 
 	// Find and parse the address of the target host
-	let (id, addr) = if let Some(h) = opt.rpc_host {
+	let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {
 		let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?;
-		(id, addrs[0])
+		(id, addrs[0], false)
 	} else {
 		let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
 			.err_context(READ_KEY_ERROR)?;
@@ -195,24 +199,26 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?
 				.next()
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?;
-			(node_id, a)
+			(node_id, a, false)
 		} else {
 			let default_addr = SocketAddr::new(
 				"127.0.0.1".parse().unwrap(),
 				config.as_ref().unwrap().rpc_bind_addr.port(),
 			);
-			warn!(
+			(node_id, default_addr, true)
-				"Trying to contact Garage node at default address {}",
-				default_addr
-			);
-			warn!("If this doesn't work, consider adding rpc_public_addr in your config file or specifying the -h command line parameter.");
-			(node_id, default_addr)
 		}
 	};
 
 	// Connect to target host
-	netapp.clone().try_connect(addr, id).await
+	if let Err(e) = netapp.clone().try_connect(addr, id).await {
-		.err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+		if is_default_addr {
+			warn!(
+				"Tried to contact Garage node at default address {}, which didn't work. If that address is wrong, consider setting rpc_public_addr in your config file.",
+				addr
+			);
+		}
+		Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+	}
 
 	let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
 	let admin_rpc_endpoint = netapp.endpoint::<AdminRpc, ()>(ADMIN_RPC_PATH.into());

src/garage/repair/online.rs

@@ -85,8 +85,11 @@ impl Worker for RepairVersionsWorker {
 		"Version repair worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
-		Some(format!("{} items done", self.counter))
+		WorkerStatus {
+			progress: Some(self.counter.to_string()),
+			..Default::default()
+		}
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
@@ -163,8 +166,11 @@ impl Worker for RepairBlockrefsWorker {
 		"Block refs repair worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
-		Some(format!("{} items done", self.counter))
+		WorkerStatus {
+			progress: Some(self.counter.to_string()),
+			..Default::default()
+		}
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {

src/garage/tests/bucket.rs

@@ -1,4 +1,5 @@
 use crate::common;
+use crate::common::ext::CommandExt;
 use aws_sdk_s3::model::BucketLocationConstraint;
 use aws_sdk_s3::output::DeleteBucketOutput;
 
@@ -8,6 +9,27 @@ async fn test_bucket_all() {
 	let bucket_name = "hello";
 
 	{
+		// Check bucket cannot be created if not authorized
+		ctx.garage
+			.command()
+			.args(["key", "deny"])
+			.args(["--create-bucket", &ctx.garage.key.id])
+			.quiet()
+			.expect_success_output("Could not deny key to create buckets");
+
+		// Try create bucket, should fail
+		let r = ctx.client.create_bucket().bucket(bucket_name).send().await;
+		assert!(r.is_err());
+	}
+	{
+		// Now allow key to create bucket
+		ctx.garage
+			.command()
+			.args(["key", "allow"])
+			.args(["--create-bucket", &ctx.garage.key.id])
+			.quiet()
+			.expect_success_output("Could not deny key to create buckets");
+
 		// Create bucket
 		//@TODO check with an invalid bucket name + with an already existing bucket
 		let r = ctx

src/garage/tests/s3/streaming_signature.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 
 use crate::common;
+use crate::common::ext::CommandExt;
 use common::custom_requester::BodySignature;
 use hyper::Method;
 
@@ -105,6 +106,13 @@ async fn test_create_bucket_streaming() {
 	let ctx = common::context();
 	let bucket = "createbucket-streaming";
 
+	ctx.garage
+		.command()
+		.args(["key", "allow"])
+		.args(["--create-bucket", &ctx.garage.key.id])
+		.quiet()
+		.expect_success_output("Could not allow key to create buckets");
+
 	{
 		// create bucket
 		let _ = ctx

src/k2v-client/Cargo.toml

@@ -22,7 +22,7 @@ tokio = "1.17.0"
 
 # cli deps
 clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
-garage_util = { version = "0.8.0", path = "../util", optional = true }
+garage_util = { version = "0.8.1", path = "../util", optional = true }
 
 
 [features]

src/model/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,11 +14,11 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.0", default-features = false, path = "../db" }
+garage_db = { version = "0.8.1", default-features = false, path = "../db" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
-garage_table = { version = "0.8.0", path = "../table" }
+garage_table = { version = "0.8.1", path = "../table" }
-garage_block = { version = "0.8.0", path = "../block" }
+garage_block = { version = "0.8.1", path = "../block" }
-garage_util = { version = "0.8.0", path = "../util" }
+garage_util = { version = "0.8.1", path = "../util" }
 
 async-trait = "0.1.7"
 arc-swap = "1.0"

src/model/garage.rs

@@ -8,10 +8,10 @@ use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;
 
+use garage_rpc::replication_mode::ReplicationMode;
 use garage_rpc::system::System;
 
 use garage_block::manager::*;
-use garage_table::replication::ReplicationMode;
 use garage_table::replication::TableFullReplication;
 use garage_table::replication::TableShardedReplication;
 use garage_table::*;
@@ -34,6 +34,9 @@ pub struct Garage {
 	/// The parsed configuration Garage is running
 	pub config: Config,
 
+	/// The replication mode of this cluster
+	pub replication_mode: ReplicationMode,
+
 	/// The local database
 	pub db: db::Db,
 	/// A background job runner
@@ -164,12 +167,7 @@ impl Garage {
 			.expect("Invalid replication_mode in config file.");
 
 		info!("Initialize membership management system...");
-		let system = System::new(
+		let system = System::new(network_key, background.clone(), replication_mode, &config)?;
-			network_key,
-			background.clone(),
-			replication_mode.replication_factor(),
-			&config,
-		)?;
 
 		let data_rep_param = TableShardedReplication {
 			system: system.clone(),
@@ -258,6 +256,7 @@ impl Garage {
 		// -- done --
 		Ok(Arc::new(Self {
 			config,
+			replication_mode,
 			db,
 			background,
 			system,

src/model/index_counter.rs

@@ -404,14 +404,13 @@ impl<T: CountedItem> IndexPropagatorWorker<T> {
 #[async_trait]
 impl<T: CountedItem> Worker for IndexPropagatorWorker<T> {
 	fn name(&self) -> String {
-		format!("{} index counter propagator", T::COUNTER_TABLE_NAME)
+		format!("{} counter", T::COUNTER_TABLE_NAME)
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
-		if !self.buf.is_empty() {
+		WorkerStatus {
-			Some(format!("{} items in queue", self.buf.len()))
+			queue_length: Some(self.buf.len() as u64),
-		} else {
+			..Default::default()
-			None
 		}
 	}
 

src/rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,7 +14,7 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_util = { version = "0.8.0", path = "../util" }
+garage_util = { version = "0.8.1", path = "../util" }
 
 arc-swap = "1.0"
 bytes = "1.0"

src/rpc/layout.rs

@@ -254,9 +254,11 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		match self.initial_partition_assignation() {
 			Some(initial_partitions) => {
 				for (part, ipart) in partitions.iter_mut().zip(initial_partitions.iter()) {
-					for (id, info) in ipart.nodes.iter() {
+					for _ in 0..2 {
-						if part.nodes.len() < self.replication_factor {
+						for (id, info) in ipart.nodes.iter() {
-							part.add(None, n_zones, id, info.unwrap());
+							if part.nodes.len() < self.replication_factor {
+								part.add(None, n_zones, id, info.unwrap());
+							}
 						}
 					}
 					assert!(part.nodes.len() == self.replication_factor);

src/rpc/lib.rs

@@ -9,6 +9,7 @@ mod consul;
 mod kubernetes;
 
 pub mod layout;
+pub mod replication_mode;
 pub mod ring;
 pub mod system;
 

src/rpc/replication_mode.rs

@@ -1,3 +1,4 @@
+#[derive(Clone, Copy)]
 pub enum ReplicationMode {
 	None,
 	TwoWay,

src/rpc/system.rs

@@ -35,6 +35,7 @@ use crate::consul::ConsulDiscovery;
 #[cfg(feature = "kubernetes-discovery")]
 use crate::kubernetes::*;
 use crate::layout::*;
+use crate::replication_mode::*;
 use crate::ring::*;
 use crate::rpc_helper::*;
 
@@ -102,6 +103,7 @@ pub struct System {
 	#[cfg(feature = "kubernetes-discovery")]
 	kubernetes_discovery: Option<KubernetesDiscoveryConfig>,
 
+	replication_mode: ReplicationMode,
 	replication_factor: usize,
 
 	/// The ring
@@ -136,6 +138,37 @@ pub struct KnownNodeInfo {
 	pub status: NodeStatus,
 }
 
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct ClusterHealth {
+	/// The current health status of the cluster (see below)
+	pub status: ClusterHealthStatus,
+	/// Number of nodes already seen once in the cluster
+	pub known_nodes: usize,
+	/// Number of nodes currently connected
+	pub connected_nodes: usize,
+	/// Number of storage nodes declared in the current layout
+	pub storage_nodes: usize,
+	/// Number of storage nodes currently connected
+	pub storage_nodes_ok: usize,
+	/// Number of partitions in the layout
+	pub partitions: usize,
+	/// Number of partitions for which we have a quorum of connected nodes
+	pub partitions_quorum: usize,
+	/// Number of partitions for which all storage nodes are connected
+	pub partitions_all_ok: usize,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub enum ClusterHealthStatus {
+	/// All nodes are available
+	Healthy,
+	/// Some storage nodes are unavailable, but quorum is stil
+	/// achieved for all partitions
+	Degraded,
+	/// Quorum is not available for some partitions
+	Unavailable,
+}
+
 pub fn read_node_id(metadata_dir: &Path) -> Result<NodeID, Error> {
 	let mut pubkey_file = metadata_dir.to_path_buf();
 	pubkey_file.push("node_key.pub");
@@ -200,9 +233,11 @@ impl System {
 	pub fn new(
 		network_key: NetworkKey,
 		background: Arc<BackgroundRunner>,
-		replication_factor: usize,
+		replication_mode: ReplicationMode,
 		config: &Config,
 	) -> Result<Arc<Self>, Error> {
+		let replication_factor = replication_mode.replication_factor();
+
 		let node_key =
 			gen_node_key(&config.metadata_dir).expect("Unable to read or generate node ID");
 		info!(
@@ -324,6 +359,7 @@ impl System {
 				config.rpc_timeout_msec.map(Duration::from_millis),
 			),
 			system_endpoint,
+			replication_mode,
 			replication_factor,
 			rpc_listen_addr: config.rpc_bind_addr,
 			#[cfg(any(feature = "consul-discovery", feature = "kubernetes-discovery"))]
@@ -429,6 +465,67 @@ impl System {
 		}
 	}
 
+	pub fn health(&self) -> ClusterHealth {
+		let ring: Arc<_> = self.ring.borrow().clone();
+		let quorum = self.replication_mode.write_quorum();
+		let replication_factor = self.replication_factor;
+
+		let nodes = self
+			.get_known_nodes()
+			.into_iter()
+			.map(|n| (n.id, n))
+			.collect::<HashMap<Uuid, _>>();
+		let connected_nodes = nodes.iter().filter(|(_, n)| n.is_up).count();
+
+		let storage_nodes = ring
+			.layout
+			.roles
+			.items()
+			.iter()
+			.filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_some()))
+			.collect::<Vec<_>>();
+		let storage_nodes_ok = storage_nodes
+			.iter()
+			.filter(|(x, _, _)| nodes.get(x).map(|n| n.is_up).unwrap_or(false))
+			.count();
+
+		let partitions = ring.partitions();
+		let partitions_n_up = partitions
+			.iter()
+			.map(|(_, h)| {
+				let pn = ring.get_nodes(h, ring.replication_factor);
+				pn.iter()
+					.filter(|x| nodes.get(x).map(|n| n.is_up).unwrap_or(false))
+					.count()
+			})
+			.collect::<Vec<usize>>();
+		let partitions_all_ok = partitions_n_up
+			.iter()
+			.filter(|c| **c == replication_factor)
+			.count();
+		let partitions_quorum = partitions_n_up.iter().filter(|c| **c >= quorum).count();
+
+		let status =
+			if partitions_quorum == partitions.len() && storage_nodes_ok == storage_nodes.len() {
+				ClusterHealthStatus::Healthy
+			} else if partitions_quorum == partitions.len() {
+				ClusterHealthStatus::Degraded
+			} else {
+				ClusterHealthStatus::Unavailable
+			};
+
+		ClusterHealth {
+			status,
+			known_nodes: nodes.len(),
+			connected_nodes,
+			storage_nodes: storage_nodes.len(),
+			storage_nodes_ok,
+			partitions: partitions.len(),
+			partitions_quorum,
+			partitions_all_ok,
+		}
+	}
+
 	// ---- INTERNALS ----
 
 	#[cfg(feature = "consul-discovery")]

src/table/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_table"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,9 +14,9 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
+garage_db = { version = "0.8.1", path = "../db" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
-garage_util = { version = "0.8.0", path = "../util" }
+garage_util = { version = "0.8.1", path = "../util" }
 
 opentelemetry = "0.17"
 

src/table/data.rs

@@ -58,7 +58,13 @@ where
 			.expect("Unable to open DB tree");
 		let gc_todo = CountedTree::new(gc_todo).expect("Cannot count gc_todo_v2");
 
-		let metrics = TableMetrics::new(F::TABLE_NAME, merkle_todo.clone(), gc_todo.clone());
+		let metrics = TableMetrics::new(
+			F::TABLE_NAME,
+			store.clone(),
+			merkle_tree.clone(),
+			merkle_todo.clone(),
+			gc_todo.clone(),
+		);
 
 		Arc::new(Self {
 			system,

src/table/gc.rs

@@ -330,12 +330,10 @@ where
 		format!("{} GC", F::TABLE_NAME)
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
-		let l = self.gc.data.gc_todo_len().unwrap_or(0);
+		WorkerStatus {
-		if l > 0 {
+			queue_length: Some(self.gc.data.gc_todo_len().unwrap_or(0) as u64),
-			Some(format!("{} items in queue", l))
+			..Default::default()
-		} else {
-			None
 		}
 	}
 

src/table/merkle.rs

@@ -293,6 +293,10 @@ where
 		Ok(self.data.merkle_tree.len()?)
 	}
 
+	pub fn merkle_tree_fast_len(&self) -> Result<Option<usize>, Error> {
+		Ok(self.data.merkle_tree.fast_len()?)
+	}
+
 	pub fn todo_len(&self) -> Result<usize, Error> {
 		Ok(self.data.merkle_todo.len()?)
 	}
@@ -310,15 +314,13 @@ where
 	R: TableReplication + 'static,
 {
 	fn name(&self) -> String {
-		format!("{} Merkle tree updater", F::TABLE_NAME)
+		format!("{} Merkle", F::TABLE_NAME)
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
-		let l = self.0.todo_len().unwrap_or(0);
+		WorkerStatus {
-		if l > 0 {
+			queue_length: Some(self.0.todo_len().unwrap_or(0) as u64),
-			Some(format!("{} items in queue", l))
+			..Default::default()
-		} else {
-			None
 		}
 	}
 

src/table/metrics.rs

@@ -5,6 +5,8 @@ use garage_db::counted_tree_hack::CountedTree;
 
 /// TableMetrics reference all counter used for metrics
 pub struct TableMetrics {
+	pub(crate) _table_size: ValueObserver<u64>,
+	pub(crate) _merkle_tree_size: ValueObserver<u64>,
 	pub(crate) _merkle_todo_len: ValueObserver<u64>,
 	pub(crate) _gc_todo_len: ValueObserver<u64>,
 
@@ -20,9 +22,43 @@ pub struct TableMetrics {
 	pub(crate) sync_items_received: Counter<u64>,
 }
 impl TableMetrics {
-	pub fn new(table_name: &'static str, merkle_todo: db::Tree, gc_todo: CountedTree) -> Self {
+	pub fn new(
+		table_name: &'static str,
+		store: db::Tree,
+		merkle_tree: db::Tree,
+		merkle_todo: db::Tree,
+		gc_todo: CountedTree,
+	) -> Self {
 		let meter = global::meter(table_name);
 		TableMetrics {
+			_table_size: meter
+				.u64_value_observer(
+					"table.size",
+					move |observer| {
+						if let Ok(Some(v)) = store.fast_len() {
+							observer.observe(
+								v as u64,
+								&[KeyValue::new("table_name", table_name)],
+							);
+						}
+					},
+				)
+				.with_description("Number of items in table")
+				.init(),
+			_merkle_tree_size: meter
+				.u64_value_observer(
+					"table.merkle_tree_size",
+					move |observer| {
+						if let Ok(Some(v)) = merkle_tree.fast_len() {
+							observer.observe(
+								v as u64,
+								&[KeyValue::new("table_name", table_name)],
+							);
+						}
+					},
+				)
+				.with_description("Number of nodes in table's Merkle tree")
+				.init(),
 			_merkle_todo_len: meter
 				.u64_value_observer(
 					"table.merkle_updater_todo_queue_length",

src/table/replication/mod.rs

@@ -1,10 +1,8 @@
 mod parameters;
 
 mod fullcopy;
-mod mode;
 mod sharded;
 
 pub use fullcopy::TableFullReplication;
-pub use mode::ReplicationMode;
 pub use parameters::*;
 pub use sharded::TableShardedReplication;

src/table/sync.rs

@@ -570,12 +570,10 @@ impl<F: TableSchema + 'static, R: TableReplication + 'static> Worker for SyncWor
 		format!("{} sync", F::TABLE_NAME)
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
-		let l = self.todo.len();
+		WorkerStatus {
-		if l > 0 {
+			queue_length: Some(self.todo.len() as u64),
-			Some(format!("{} partitions remaining", l))
+			..Default::default()
-		} else {
-			None
 		}
 	}
 

src/table/util.rs

@@ -49,3 +49,9 @@ impl EnumerationOrder {
 		}
 	}
 }
+
+impl Default for EnumerationOrder {
+	fn default() -> Self {
+		EnumerationOrder::Forward
+	}
+}

src/util/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_util"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,7 +14,7 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
+garage_db = { version = "0.8.1", path = "../db" }
 
 arc-swap = "1.0"
 async-trait = "0.1"

src/util/background/mod.rs

@@ -29,13 +29,24 @@ pub struct BackgroundRunner {
 #[derive(Clone, Serialize, Deserialize, Debug)]
 pub struct WorkerInfo {
 	pub name: String,
-	pub info: Option<String>,
+	pub status: WorkerStatus,
 	pub state: WorkerState,
 	pub errors: usize,
 	pub consecutive_errors: usize,
 	pub last_error: Option<(String, u64)>,
 }
 
+/// WorkerStatus is a struct returned by the worker with a bunch of canonical
+/// fields to indicate their status to CLI users. All fields are optional.
+#[derive(Clone, Serialize, Deserialize, Debug, Default)]
+pub struct WorkerStatus {
+	pub tranquility: Option<u32>,
+	pub progress: Option<String>,
+	pub queue_length: Option<u64>,
+	pub persistent_errors: Option<u64>,
+	pub freeform: Vec<String>,
+}
+
 impl BackgroundRunner {
 	/// Create a new BackgroundRunner
 	pub fn new(

src/util/background/worker.rs

@@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
 use tokio::select;
 use tokio::sync::{mpsc, watch};
 
-use crate::background::WorkerInfo;
+use crate::background::{WorkerInfo, WorkerStatus};
 use crate::error::Error;
 use crate::time::now_msec;
 
@@ -26,7 +26,7 @@ impl std::fmt::Display for WorkerState {
 	fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
 		match self {
 			WorkerState::Busy => write!(f, "Busy"),
-			WorkerState::Throttled(t) => write!(f, "Thr:{:.3}", t),
+			WorkerState::Throttled(_) => write!(f, "Busy*"),
 			WorkerState::Idle => write!(f, "Idle"),
 			WorkerState::Done => write!(f, "Done"),
 		}
@@ -37,8 +37,8 @@ impl std::fmt::Display for WorkerState {
 pub trait Worker: Send {
 	fn name(&self) -> String;
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
-		None
+		Default::default()
 	}
 
 	/// Work: do a basic unit of work, if one is available (otherwise, should return
@@ -119,7 +119,7 @@ impl WorkerProcessor {
 						match wi.get_mut(&worker.task_id) {
 							Some(i) => {
 								i.state = worker.state;
-								i.info = worker.worker.info();
+								i.status = worker.worker.status();
 								i.errors = worker.errors;
 								i.consecutive_errors = worker.consecutive_errors;
 								if worker.last_error.is_some() {
@@ -130,7 +130,7 @@ impl WorkerProcessor {
 								wi.insert(worker.task_id, WorkerInfo {
 									name: worker.worker.name(),
 									state: worker.state,
-									info: worker.worker.info(),
+									status: worker.worker.status(),
 									errors: worker.errors,
 									consecutive_errors: worker.consecutive_errors,
 									last_error: worker.last_error.take(),

src/util/formater.rs

@@ -1,4 +1,4 @@
-pub fn format_table(data: Vec<String>) {
+pub fn format_table_to_string(data: Vec<String>) -> String {
 	let data = data
 		.iter()
 		.map(|s| s.split('\t').collect::<Vec<_>>())
@@ -24,5 +24,9 @@ pub fn format_table(data: Vec<String>) {
 		out.push('\n');
 	}
 
-	print!("{}", out);
+	out
+}
+
+pub fn format_table(data: Vec<String>) {
+	print!("{}", format_table_to_string(data));
 }

src/web/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_web"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,10 +14,10 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_api = { version = "0.8.0", path = "../api" }
+garage_api = { version = "0.8.1", path = "../api" }
-garage_model = { version = "0.8.0", path = "../model" }
+garage_model = { version = "0.8.1", path = "../model" }
-garage_util = { version = "0.8.0", path = "../util" }
+garage_util = { version = "0.8.1", path = "../util" }
-garage_table = { version = "0.8.0", path = "../table" }
+garage_table = { version = "0.8.1", path = "../table" }
 
 err-derive = "0.3"
 tracing = "0.1.30"