Merge pull request 'Implement rpc_secret_file' (#466) from felix.scheinost/garage:feature/implement-rpc-secret-file into main

Reviewed-on: Deuxfleurs/garage#466

.envrc

@@ -0,0 +1 @@
+use flake

.gitattributes

@@ -1 +0,0 @@
-*.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -3,3 +3,4 @@
 /pki
 **/*.rs.bk
 *.swp
+/.direnv

Cargo.lock

@@ -1048,7 +1048,7 @@ dependencies = [
 
 [[package]]
 name = "garage"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "assert-json-diff",
  "async-trait",
@@ -1080,7 +1080,6 @@ dependencies = [
  "parse_duration",
  "prometheus",
  "rand 0.8.5",
- "rmp-serde",
  "serde",
  "serde_bytes",
  "serde_json",
@@ -1096,7 +1095,7 @@ dependencies = [
 
 [[package]]
 name = "garage_api"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "async-trait",
  "base64",
@@ -1141,7 +1140,7 @@ dependencies = [
 
 [[package]]
 name = "garage_block"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "arc-swap",
  "async-compression",
@@ -1156,7 +1155,6 @@ dependencies = [
  "hex",
  "opentelemetry",
  "rand 0.8.5",
- "rmp-serde",
  "serde",
  "serde_bytes",
  "tokio",
@@ -1167,7 +1165,7 @@ dependencies = [
 
 [[package]]
 name = "garage_db"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "clap 3.1.18",
  "err-derive",
@@ -1182,7 +1180,7 @@ dependencies = [
 
 [[package]]
 name = "garage_model"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1200,7 +1198,6 @@ dependencies = [
  "netapp",
  "opentelemetry",
  "rand 0.8.5",
- "rmp-serde",
  "serde",
  "serde_bytes",
  "tokio",
@@ -1210,7 +1207,7 @@ dependencies = [
 
 [[package]]
 name = "garage_rpc"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1229,7 +1226,6 @@ dependencies = [
  "pnet_datalink",
  "rand 0.8.5",
  "reqwest",
- "rmp-serde",
  "schemars",
  "serde",
  "serde_bytes",
@@ -1241,8 +1237,9 @@ dependencies = [
 
 [[package]]
 name = "garage_table"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
+ "arc-swap",
  "async-trait",
  "bytes",
  "futures",
@@ -1254,7 +1251,6 @@ dependencies = [
  "hexdump",
  "opentelemetry",
  "rand 0.8.5",
- "rmp-serde",
  "serde",
  "serde_bytes",
  "tokio",
@@ -1263,7 +1259,7 @@ dependencies = [
 
 [[package]]
 name = "garage_util"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1276,9 +1272,11 @@ dependencies = [
  "garage_db",
  "git-version",
  "hex",
+ "hexdump",
  "http",
  "hyper",
  "lazy_static",
+ "mktemp",
  "netapp",
  "opentelemetry",
  "rand 0.8.5",
@@ -1294,7 +1292,7 @@ dependencies = [
 
 [[package]]
 name = "garage_web"
-version = "0.8.0"
+version = "0.8.1"
 dependencies = [
  "err-derive",
  "futures",

Cargo.nix

@@ -32,7 +32,7 @@ args@{
   ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "90b29705f5037c7e1b33f4650841f1266f2e86fa03d5d0c87ad80be7619985c7";
+  nixifiedLockHash = "8461dcfb984a8d042fecb5745d5da17912135dbf2a8ef7e6c3ae8e64c03d9744";
   workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
   currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
   lockHashIgnored = if ignoreLockHash
@@ -56,15 +56,15 @@ in
 {
   cargo2nixVersion = "0.11.0";
   workspace = {
-    garage_db = rustPackages.unknown.garage_db."0.8.0";
-    garage_util = rustPackages.unknown.garage_util."0.8.0";
-    garage_rpc = rustPackages.unknown.garage_rpc."0.8.0";
-    garage_table = rustPackages.unknown.garage_table."0.8.0";
-    garage_block = rustPackages.unknown.garage_block."0.8.0";
-    garage_model = rustPackages.unknown.garage_model."0.8.0";
-    garage_api = rustPackages.unknown.garage_api."0.8.0";
-    garage_web = rustPackages.unknown.garage_web."0.8.0";
-    garage = rustPackages.unknown.garage."0.8.0";
+    garage_db = rustPackages.unknown.garage_db."0.8.1";
+    garage_util = rustPackages.unknown.garage_util."0.8.1";
+    garage_rpc = rustPackages.unknown.garage_rpc."0.8.1";
+    garage_table = rustPackages.unknown.garage_table."0.8.1";
+    garage_block = rustPackages.unknown.garage_block."0.8.1";
+    garage_model = rustPackages.unknown.garage_model."0.8.1";
+    garage_api = rustPackages.unknown.garage_api."0.8.1";
+    garage_web = rustPackages.unknown.garage_web."0.8.1";
+    garage = rustPackages.unknown.garage."0.8.1";
     k2v-client = rustPackages.unknown.k2v-client."0.0.1";
   };
   "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec {
@@ -946,20 +946,20 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "alloc")
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "default")
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "lazy_static")
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "std")
+      [ "alloc" ]
+      [ "default" ]
+      [ "lazy_static" ]
+      [ "std" ]
     ];
     dependencies = {
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "crossbeam_utils" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.8" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "lazy_static" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "memoffset" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memoffset."0.6.5" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "scopeguard" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.1.0" { inherit profileName; }).out;
+      cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
+      crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.8" { inherit profileName; }).out;
+      lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
+      memoffset = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memoffset."0.6.5" { inherit profileName; }).out;
+      scopeguard = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.1.0" { inherit profileName; }).out;
     };
     buildDependencies = {
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "autocfg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
+      autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
     };
   });
   
@@ -995,7 +995,7 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "default")
+      [ "default" ]
       [ "lazy_static" ]
       [ "std" ]
     ];
@@ -1321,8 +1321,8 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"; };
     dependencies = {
-      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out;
-      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out;
+      ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out;
+      ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out;
     };
   });
   
@@ -1490,20 +1490,20 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"; };
     dependencies = {
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "byteorder" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out;
+      byteorder = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out;
     };
   });
   
-  "unknown".garage."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/garage");
     features = builtins.concatLists [
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default") "bundled-libs")
       (lib.optional (rootFeatures' ? "garage/consul-discovery") "consul-discovery")
       (lib.optional (rootFeatures' ? "garage/default") "default")
-      (lib.optional (rootFeatures' ? "garage/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v") "k2v")
       (lib.optional (rootFeatures' ? "garage/kubernetes-discovery") "kubernetes-discovery")
       (lib.optional (rootFeatures' ? "garage/lmdb") "lmdb")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics") "metrics")
@@ -1522,14 +1522,14 @@ in
       bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
-      garage_web = (rustPackages."unknown".garage_web."0.8.0" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      garage_web = (rustPackages."unknown".garage_web."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
       netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
@@ -1539,7 +1539,6 @@ in
       parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
       structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out;
@@ -1563,13 +1562,13 @@ in
     };
   });
   
-  "unknown".garage_api."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_api."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_api";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/api");
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics") "metrics")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus") "opentelemetry-prometheus")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus") "prometheus")
@@ -1584,11 +1583,11 @@ in
       form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
@@ -1617,9 +1616,9 @@ in
     };
   });
   
-  "unknown".garage_block."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_block."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_block";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/block");
     features = builtins.concatLists [
@@ -1632,14 +1631,13 @@ in
       bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
       tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
@@ -1649,20 +1647,21 @@ in
     };
   });
   
-  "unknown".garage_db."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_db."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_db";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/db");
     features = builtins.concatLists [
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled-libs")
       (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli") "clap")
       (lib.optional (rootFeatures' ? "garage_db/cli") "cli")
+      [ "default" ]
       (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "heed")
       (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb")
       (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "pretty_env_logger")
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "rusqlite")
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "sled")
+      [ "sled" ]
       (lib.optional (rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "sqlite")
     ];
     dependencies = {
@@ -1672,7 +1671,7 @@ in
       hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "pretty_env_logger" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.27.0" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "sled" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }).out;
+      sled = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }).out;
       tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }).out;
     };
     devDependencies = {
@@ -1680,15 +1679,16 @@ in
     };
   });
   
-  "unknown".garage_model."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_model."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_model";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/model");
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v")
+      [ "default" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v")
       (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb")
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_model/sled") "sled")
+      [ "sled" ]
       (lib.optional (rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_model/sqlite") "sqlite")
     ];
     dependencies = {
@@ -1699,16 +1699,15 @@ in
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
       tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
@@ -1717,9 +1716,9 @@ in
     };
   });
   
-  "unknown".garage_rpc."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_rpc."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_rpc";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/rpc");
     features = builtins.concatLists [
@@ -1739,7 +1738,7 @@ in
       ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out;
@@ -1750,7 +1749,6 @@ in
       pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.12" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.8" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
@@ -1761,24 +1759,24 @@ in
     };
   });
   
-  "unknown".garage_table."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_table."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_table";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/table");
     dependencies = {
+      arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }).out;
       async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }).out;
       bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
-      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
       tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
@@ -1786,13 +1784,13 @@ in
     };
   });
   
-  "unknown".garage_util."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_util."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_util";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/util");
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v")
     ];
     dependencies = {
       arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }).out;
@@ -1803,9 +1801,10 @@ in
       digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; }).out;
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
       git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
+      hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
       hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
       lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
@@ -1821,20 +1820,23 @@ in
       tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }).out;
       xxhash_rust = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.4" { inherit profileName; }).out;
     };
+    devDependencies = {
+      mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.4.1" { inherit profileName; }).out;
+    };
   });
   
-  "unknown".garage_web."0.8.0" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_web."0.8.1" = overridableMkRustCrate (profileName: rec {
     name = "garage_web";
-    version = "0.8.0";
+    version = "0.8.1";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/web");
     dependencies = {
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
       hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -2448,7 +2450,7 @@ in
     dependencies = {
       base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
       log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
       rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out;
@@ -2846,10 +2848,10 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "default")
+      [ "default" ]
     ];
     buildDependencies = {
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "autocfg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
+      autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
     };
   });
   
@@ -4732,18 +4734,18 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"; };
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "default")
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "no_metrics")
+      [ "default" ]
+      [ "no_metrics" ]
     ];
     dependencies = {
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "crc32fast" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "crossbeam_epoch" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.8" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "crossbeam_utils" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.8" { inherit profileName; }).out;
-      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") && (hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "windows") then "fs2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fs2."0.4.3" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "fxhash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fxhash."0.2.1" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }).out;
+      crc32fast = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out;
+      crossbeam_epoch = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.8" { inherit profileName; }).out;
+      crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.8" { inherit profileName; }).out;
+      ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "windows" then "fs2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fs2."0.4.3" { inherit profileName; }).out;
+      fxhash = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fxhash."0.2.1" { inherit profileName; }).out;
+      libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out;
+      log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
+      parking_lot = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }).out;
     };
   });
   
@@ -5396,7 +5398,6 @@ in
       [ "attributes" ]
       [ "default" ]
       [ "log" ]
-      [ "log-always" ]
       [ "std" ]
       [ "tracing-attributes" ]
     ];
@@ -5890,7 +5891,7 @@ in
       [ "ntstatus" ]
       [ "objbase" ]
       [ "processenv" ]
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "processthreadsapi")
+      [ "processthreadsapi" ]
       [ "profileapi" ]
       [ "schannel" ]
       [ "securitybaseapi" ]

default.nix

@@ -11,14 +11,14 @@ let
 
   build_debug_and_release = (target: {
     debug = (compile {
-      inherit target git_version;
+      inherit system target git_version pkgsSrc cargo2nixOverlay;
       release = false;
     }).workspace.garage {
       compileMode = "build";
     };
 
     release = (compile {
-      inherit target git_version;
+      inherit system target git_version pkgsSrc cargo2nixOverlay;
       release = true;
     }).workspace.garage {
       compileMode = "build";
@@ -39,7 +39,7 @@ in {
   };
   test = {
     amd64 = test (compile {
-      inherit git_version;
+      inherit system git_version pkgsSrc cargo2nixOverlay;
       target = "x86_64-unknown-linux-musl";
       features = [
         "garage/bundled-libs"
@@ -52,7 +52,7 @@ in {
   };
   clippy = {
     amd64 = (compile {
-      inherit git_version;
+      inherit system git_version pkgsSrc cargo2nixOverlay;
       target = "x86_64-unknown-linux-musl";
       compiler = "clippy";
     }).workspace.garage {

doc/book/build/python.md

@@ -5,13 +5,56 @@ weight = 20
 
 ## S3
 
+### Using Minio SDK
+
+First install the SDK:
+
+```bash
+pip3 install minio
+```
+
+Then instantiate a client object using garage root domain, api key and secret:
+
+```python
+import minio
+
+client = minio.Minio(
+  "your.domain.tld",
+  "GKyourapikey",
+  "abcd[...]1234",
+  # Force the region, this is specific to garage
+  region="region",
+)
+```
+
+Then use all the standard S3 endpoints as implemented by the Minio SDK:
+
+```
+# List buckets
+print(client.list_buckets())
+
+# Put an object containing 'content' to /path in bucket named 'bucket':
+content = b"content"
+client.put_object(
+  "bucket",
+  "path",
+  io.BytesIO(content),
+  len(content),
+)
+
+# Read the object back and check contents
+data = client.get_object("bucket", "path").read()
+assert data == content
+```
+
+For further documentation, see the Minio SDK
+[Reference](https://docs.min.io/docs/python-client-api-reference.html)
+
+### Using Amazon boto3
+
 *Coming soon*
 
-Some refs:
-  - Minio SDK
-    - [Reference](https://docs.min.io/docs/python-client-api-reference.html)
-
-  - Amazon boto3
+See the official documentation:
   - [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
   - [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
   - [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)

doc/book/connect/apps/index.md

@@ -8,7 +8,7 @@ In this section, we cover the following web applications:
 | Name | Status | Note |
 |------|--------|------|
 | [Nextcloud](#nextcloud)     | ✅       |  Both Primary Storage and External Storage are supported    |
-| [Peertube](#peertube)     | ✅       | Must be configured with the website endpoint     |
+| [Peertube](#peertube)     | ✅       | Supported with the website endpoint, proxifying private videos unsupported     |
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
 | [Pixelfed](#pixelfed)     | ❓       |  Not yet tested    |
@@ -128,6 +128,10 @@ In other words, Peertube is only responsible of the "control plane" and offload
 In return, this system is a bit harder to configure.
 We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
 
+Starting from version 5.0, Peertube also supports improving the security for private videos by not exposing them directly
+but relying on a single control point in the Peertube instance. This is based on S3 per-object and prefix ACL, which are not currently supported
+in Garage, so this feature is unsupported. While this technically impedes security for private videos, it is not a blocking issue and could be
+a reasonable trade-off for some instances.
 
 ### Create resources in Garage
 
@@ -195,6 +199,11 @@ object_storage:
 
   max_upload_part: 2GB
 
+  proxy:
+    # You may enable this feature, yet it will not provide any security benefit, so
+    # you should rather benefit from Garage public endpoint for all videos
+    proxify_private_files: false
+
   streaming_playlists:
     bucket_name: 'peertube-playlist'
 

doc/book/cookbook/real-world.md

@@ -109,7 +109,7 @@ especially you must consider the following folders/files:
   this folder will be your main data storage and must be on a large storage (e.g. large HDD)
 
 
-A valid `/etc/garage/garage.toml` for our cluster would look as follows:
+A valid `/etc/garage.toml` for our cluster would look as follows:
 
 ```toml
 metadata_dir = "/var/lib/garage/meta"

doc/book/design/goals.md

@@ -12,7 +12,7 @@ as pictures, video, images, documents, etc., in a redundant multi-node
 setting. S3 is versatile enough to also be used to publish a static
 website.
 
-Garage is an opinionated object storage solutoin, we focus on the following **desirable properties**:
+Garage is an opinionated object storage solution, we focus on the following **desirable properties**:
 
   - **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
   - **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).

doc/book/development/devenv.md

@@ -39,7 +39,7 @@ Now you can enter our nix-shell, all the required packages will be downloaded bu
 nix-shell
 ```
 
-You can use the traditionnal Rust development workflow:
+You can use the traditional Rust development workflow:
 
 ```bash
 cargo build  # compile the project

doc/book/reference-manual/configuration.md

@@ -96,7 +96,7 @@ Performance characteristics of the different DB engines are as follows:
 
 - Sled: the default database engine, which tends to produce
   large data files and also has performance issues, especially when the metadata folder
-  is on a traditionnal HDD and not on SSD.
+  is on a traditional HDD and not on SSD.
 - LMDB: the recommended alternative on 64-bit systems,
   much more space-efficiant and slightly faster. Note that the data format of LMDB is not portable
   between architectures, so for instance the Garage database of an x86-64
@@ -267,6 +267,10 @@ This key should be specified here in the form of a 32-byte hex-encoded
 random string. Such a string can be generated with a command
 such as `openssl rand -hex 32`.
 
+### `rpc_secret_file`
+
+Like `rpc_secret` above, just that this is the path to a file that Garage will try to read the secret from.
+
 ### `rpc_bind_addr`
 
 The address and port on which to bind for inter-cluster communcations

doc/book/reference-manual/features.md

@@ -83,7 +83,7 @@ This feature is totally invisible to S3 clients and does not break compatibility
 ### Cluster administration API
 
 Garage provides a fully-fledged REST API to administer your cluster programatically.
-Functionnality included in the admin API include: setting up and monitoring
+Functionality included in the admin API include: setting up and monitoring
 cluster nodes, managing access credentials, and managing storage buckets and bucket aliases.
 A full reference of the administration API is available [here](@/documentation/reference-manual/admin-api.md).
 

doc/drafts/admin-api.md

@@ -0,0 +1,686 @@
++++
+title = "Administration API"
+weight = 60
++++
+
+The Garage administration API is accessible through a dedicated server whose
+listen address is specified in the `[admin]` section of the configuration
+file (see [configuration file
+reference](@/documentation/reference-manual/configuration.md))
+
+**WARNING.** At this point, there is no comittement to stability of the APIs described in this document.
+We will bump the version numbers prefixed to each API endpoint at each time the syntax
+or semantics change, meaning that code that relies on these endpoint will break
+when changes are introduced.
+
+The Garage administration API was introduced in version 0.7.2, this document
+does not apply to older versions of Garage.
+
+
+## Access control
+
+The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section:
+
+- `metrics_token`: the token for accessing the Metrics endpoint (if this token
+  is not set in the config file, the Metrics endpoint can be accessed without
+  access control);
+
+- `admin_token`: the token for accessing all of the other administration
+  endpoints (if this token is not set in the config file, access to these
+  endpoints is disabled entirely).
+
+These tokens are used as simple HTTP bearer tokens. In other words, to
+authenticate access to an admin API endpoint, add the following HTTP header
+to your request:
+
+```
+Authorization: Bearer <token>
+```
+
+## Administration API endpoints
+
+### Metrics-related endpoints
+
+#### Metrics `GET /metrics`
+
+Returns internal Garage metrics in Prometheus format.
+
+#### Health `GET /health`
+
+Used for simple health checks in a cluster setting with an orchestrator.
+Returns an HTTP status 200 if the node is ready to answer user's requests,
+and an HTTP status 503 (Service Unavailable) if there are some partitions
+for which a quorum of nodes is not available.
+A simple textual message is also returned in a body with content-type `text/plain`.
+See `/v0/health` for an API that also returns JSON output.
+
+### Cluster operations
+
+#### GetClusterStatus `GET /v0/status`
+
+Returns the cluster's current status in JSON, including:
+
+- ID of the node being queried and its version of the Garage daemon
+- Live nodes
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+Example response body:
+
+```json
+{
+  "node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f",
+  "garage_version": "git:v0.8.0",
+  "knownNodes": {
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+      "addr": "10.0.0.11:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 9,
+      "hostname": "node1"
+    },
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+      "addr": "10.0.0.12:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 1,
+      "hostname": "node2"
+    },
+    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+      "addr": "10.0.0.21:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 7,
+      "hostname": "node3"
+    },
+    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+      "addr": "10.0.0.22:3901",
+      "is_up": true,
+      "last_seen_secs_ago": 1,
+      "hostname": "node4"
+    }
+  },
+  "layout": {
+    "version": 12,
+    "roles": {
+      "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+        "zone": "dc1",
+        "capacity": 4,
+        "tags": [
+          "node1"
+        ]
+      },
+      "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+        "zone": "dc1",
+        "capacity": 6,
+        "tags": [
+          "node2"
+        ]
+      },
+      "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+        "zone": "dc2",
+        "capacity": 10,
+        "tags": [
+          "node3"
+        ]
+      }
+    },
+    "stagedRoleChanges": {
+      "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+        "zone": "dc2",
+        "capacity": 5,
+        "tags": [
+          "node4"
+        ]
+      }
+    }
+  }
+}
+```
+
+#### GetClusterHealth `GET /v0/health`
+
+Returns the cluster's current health in JSON format, with the following variables:
+
+- `status`: one of `Healthy`, `Degraded` or `Unavailable`:
+  - Healthy: Garage node is connected to all storage nodes
+  - Degraded: Garage node is not connected to all storage nodes, but a quorum of write nodes is available for all partitions
+  - Unavailable: a quorum of write nodes is not available for some partitions
+- `known_nodes`: the number of nodes this Garage node has had a TCP connection to since the daemon started
+- `connected_nodes`: the nubmer of nodes this Garage node currently has an open connection to
+- `storage_nodes`: the number of storage nodes currently registered in the cluster layout
+- `storage_nodes_ok`: the number of storage nodes to which a connection is currently open
+- `partitions`: the total number of partitions of the data (currently always 256)
+- `partitions_quorum`: the number of partitions for which a quorum of write nodes is available
+- `partitions_all_ok`: the number of partitions for which we are connected to all storage nodes responsible of storing it
+
+Contrarily to `GET /health`, this endpoint always returns a 200 OK HTTP response code.
+
+Example response body:
+
+```json
+{
+    "status": "Degraded",
+    "known_nodes": 3,
+    "connected_nodes": 2,
+    "storage_nodes": 3,
+    "storage_nodes_ok": 2,
+    "partitions": 256,
+    "partitions_quorum": 256,
+    "partitions_all_ok": 0
+}
+```
+
+#### ConnectClusterNodes `POST /v0/connect`
+
+Instructs this Garage node to connect to other Garage nodes at specified addresses.
+
+Example request body:
+
+```json
+[
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901",
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901"
+]
+```
+
+The format of the string for a node to connect to is: `<node ID>@<ip address>:<port>`, same as in the `garage node connect` CLI call.
+
+Example response:
+
+```json
+[
+    {
+        "success": true,
+        "error": null
+    },
+    {
+        "success": false,
+        "error": "Handshake error"
+    }
+]
+```
+
+#### GetClusterLayout `GET /v0/layout`
+
+Returns the cluster's current layout in JSON, including:
+
+- Currently configured cluster layout
+- Staged changes to the cluster layout
+
+(the info returned by this endpoint is a subset of the info returned by GetClusterStatus)
+
+Example response body:
+
+```json
+{
+  "version": 12,
+  "roles": {
+    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
+      "zone": "dc1",
+      "capacity": 4,
+      "tags": [
+        "node1"
+      ]
+    },
+    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
+      "zone": "dc1",
+      "capacity": 6,
+      "tags": [
+        "node2"
+      ]
+    },
+    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
+      "zone": "dc2",
+      "capacity": 10,
+      "tags": [
+        "node3"
+      ]
+    }
+  },
+  "stagedRoleChanges": {
+    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
+      "zone": "dc2",
+      "capacity": 5,
+      "tags": [
+        "node4"
+      ]
+    }
+  }
+}
+```
+
+#### UpdateClusterLayout `POST /v0/layout`
+
+Send modifications to the cluster layout. These modifications will
+be included in the staged role changes, visible in subsequent calls
+of `GetClusterLayout`. Once the set of staged changes is satisfactory,
+the user may call `ApplyClusterLayout` to apply the changed changes,
+or `Revert ClusterLayout` to clear all of the staged changes in
+the layout.
+
+Request body format:
+
+```json
+{
+  <node_id>: {
+    "capacity": <new_capacity>,
+    "zone": <new_zone>,
+    "tags": [
+      <new_tag>,
+      ...
+    ]
+  },
+  <node_id_to_remove>: null,
+  ...
+}
+```
+
+Contrary to the CLI that may update only a subset of the fields
+`capacity`, `zone` and `tags`, when calling this API all of these
+values must be specified.
+
+
+#### ApplyClusterLayout `POST /v0/layout/apply`
+
+Applies to the cluster the layout changes currently registered as
+staged layout changes.
+
+Request body format:
+
+```json
+{
+  "version": 13
+}
+```
+
+Similarly to the CLI, the body must include the version of the new layout
+that will be created, which MUST be 1 + the value of the currently
+existing layout in the cluster.
+
+#### RevertClusterLayout `POST /v0/layout/revert`
+
+Clears all of the staged layout changes.
+
+Request body format:
+
+```json
+{
+  "version": 13
+}
+```
+
+Reverting the staged changes is done by incrementing the version number
+and clearing the contents of the staged change list.
+Similarly to the CLI, the body must include the incremented
+version number, which MUST be 1 + the value of the currently
+existing layout in the cluster.
+
+
+### Access key operations
+
+#### ListKeys `GET /v0/key`
+
+Returns all API access keys in the cluster.
+
+Example response:
+
+```json
+[
+  {
+    "id": "GK31c2f218a2e44f485b94239e",
+    "name": "test"
+  },
+  {
+    "id": "GKe10061ac9c2921f09e4c5540",
+    "name": "test2"
+  }
+]
+```
+
+#### CreateKey `POST /v0/key`
+
+Creates a new API access key.
+
+Request body format:
+
+```json
+{
+    "name": "NameOfMyKey"
+}
+```
+
+#### ImportKey `POST /v0/key/import`
+
+Imports an existing API key.
+
+Request body format:
+
+```json
+{
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
+    "name": "NameOfMyKey"
+}
+```
+
+#### GetKeyInfo `GET /v0/key?id=<acces key id>`
+#### GetKeyInfo `GET /v0/key?search=<pattern>`
+
+Returns information about the requested API access key.
+
+If `id` is set, the key is looked up using its exact identifier (faster).
+If `search` is set, the key is looked up using its name or prefix
+of identifier (slower, all keys are enumerated to do this).
+
+Example response:
+
+```json
+{
+  "name": "test",
+  "accessKeyId": "GK31c2f218a2e44f485b94239e",
+  "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
+  "permissions": {
+    "createBucket": false
+  },
+  "buckets": [
+    {
+      "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
+      "globalAliases": [
+        "test2"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": false
+      }
+    },
+    {
+      "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
+      "globalAliases": [
+        "test3"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": false
+      }
+    },
+    {
+      "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+      "globalAliases": [],
+      "localAliases": [
+        "test"
+      ],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+      }
+    },
+    {
+      "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
+      "globalAliases": [
+        "alex"
+      ],
+      "localAliases": [],
+      "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+      }
+    }
+  ]
+}
+```
+
+#### DeleteKey `DELETE /v0/key?id=<acces key id>`
+
+Deletes an API access key.
+
+#### UpdateKey `POST /v0/key?id=<acces key id>`
+
+Updates information about the specified API access key.
+
+Request body format:
+
+```json
+{
+    "name": "NameOfMyKey",
+    "allow": {
+        "createBucket": true,
+    },
+    "deny": {}
+}
+```
+
+All fields (`name`, `allow` and `deny`) are optionnal.
+If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed.
+The possible flags in `allow` and `deny` are: `createBucket`.
+
+
+### Bucket operations
+
+#### ListBuckets `GET /v0/bucket`
+
+Returns all storage buckets in the cluster.
+
+Example response:
+
+```json
+[
+  {
+    "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
+    "globalAliases": [
+      "test2"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
+    "globalAliases": [
+      "alex"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
+    "globalAliases": [
+      "test3"
+    ],
+    "localAliases": []
+  },
+  {
+    "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "globalAliases": [],
+    "localAliases": [
+      {
+        "accessKeyId": "GK31c2f218a2e44f485b94239e",
+        "alias": "test"
+      }
+    ]
+  }
+]
+```
+
+#### GetBucketInfo `GET /v0/bucket?id=<bucket id>`
+#### GetBucketInfo `GET /v0/bucket?globalAlias=<alias>`
+
+Returns information about the requested storage bucket.
+
+If `id` is set, the bucket is looked up using its exact identifier.
+If `globalAlias` is set, the bucket is looked up using its global alias.
+(both are fast)
+
+Example response:
+
+```json
+{
+    "id": "afa8f0a22b40b1247ccd0affb869b0af5cff980924a20e4b5e0720a44deb8d39",
+        "globalAliases": [],
+        "websiteAccess": false,
+        "websiteConfig": null,
+        "keys": [
+        {
+            "accessKeyId": "GK31c2f218a2e44f485b94239e",
+            "name": "Imported key",
+            "permissions": {
+                "read": true,
+                "write": true,
+                "owner": true
+            },
+            "bucketLocalAliases": [
+                "debug"
+            ]
+        }
+        ],
+        "objects": 14827,
+        "bytes": 13189855625,
+        "unfinshedUploads": 0,
+        "quotas": {
+            "maxSize": null,
+            "maxObjects": null
+        }
+}
+```
+
+#### CreateBucket `POST /v0/bucket`
+
+Creates a new storage bucket.
+
+Request body format:
+
+```json
+{
+    "globalAlias": "NameOfMyBucket"
+}
+```
+
+OR
+
+```json
+{
+    "localAlias": {
+        "accessKeyId": "GK31c2f218a2e44f485b94239e",
+        "alias": "NameOfMyBucket",
+        "allow": {
+            "read": true,
+            "write": true,
+            "owner": false
+        }
+    }
+}
+```
+
+OR
+
+```json
+{}
+```
+
+Creates a new bucket, either with a global alias, a local one,
+or no alias at all.
+
+Technically, you can also specify both `globalAlias` and `localAlias` and that would create
+two aliases, but I don't see why you would want to do that.
+
+#### DeleteBucket `DELETE /v0/bucket?id=<bucket id>`
+
+Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
+
+Warning: this will delete all aliases associated with the bucket!
+
+#### UpdateBucket `PUT /v0/bucket?id=<bucket id>`
+
+Updates configuration of the given bucket.
+
+Request body format:
+
+```json
+{
+    "websiteAccess": {
+        "enabled": true,
+        "indexDocument": "index.html",
+        "errorDocument": "404.html"
+    },
+    "quotas": {
+        "maxSize": 19029801,
+        "maxObjects": null,
+    }
+}
+```
+
+All fields (`websiteAccess` and `quotas`) are optionnal.
+If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
+
+In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.
+The field `errorDocument` is optional, if no error document is set a generic
+error message is displayed when errors happen. Conversely, if `enabled` is
+`false`, neither `indexDocument` nor `errorDocument` must be specified.
+
+In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null`
+to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
+to change only one of the two quotas.
+
+### Operations on permissions for keys on buckets
+
+#### BucketAllowKey `POST /v0/bucket/allow`
+
+Allows a key to do read/write/owner operations on a bucket.
+
+Request body format:
+
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "permissions": {
+        "read": true,
+        "write": true,
+        "owner": true
+    },
+}
+```
+
+Flags in `permissions` which have the value `true` will be activated.
+Other flags will remain unchanged.
+
+#### BucketDenyKey `POST /v0/bucket/deny`
+
+Denies a key from doing read/write/owner operations on a bucket.
+
+Request body format:
+
+```json
+{
+    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
+    "accessKeyId": "GK31c2f218a2e44f485b94239e",
+    "permissions": {
+        "read": false,
+        "write": false,
+        "owner": true
+    },
+}
+```
+
+Flags in `permissions` which have the value `true` will be deactivated.
+Other flags will remain unchanged.
+
+
+### Operations on bucket aliases
+
+#### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+
+Empty body. Creates a global alias for a bucket.
+
+#### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
+
+Removes a global alias for a bucket.
+
+#### LocalAliasBucket `PUT /v0/bucket/alias/local?id=<bucket id>&accessKeyId=<access key ID>&alias=<local alias>`
+
+Empty body. Creates a local alias for a bucket in the namespace of a specific access key.
+
+#### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=<bucket id>&accessKeyId<access key ID>&alias=<local alias>`
+
+Removes a local alias for a bucket in the namespace of a specific access key.
+

doc/talks/2022-11-19-Capitole-du-Libre/.gitignore

@@ -0,0 +1,10 @@
+*.aux
+*.bbl
+*.blg
+*.log
+*.nav
+*.out
+*.snm
+*.synctex.gz
+*.toc
+*.dvi

doc/talks/2022-11-19-Capitole-du-Libre/Makefile

@@ -0,0 +1,8 @@
+all:
+	pdflatex présentation.tex
+
+clean:
+	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi présentation.pdf
+
+clean_sauf_pdf:
+	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi

doc/talks/2022-11-19-Capitole-du-Libre/présentation.tex

@@ -0,0 +1,340 @@
+\documentclass[11pt, aspectratio=1610]{beamer}
+\usetheme{Warsaw}
+\usepackage[utf8]{inputenc}
+\usepackage[french]{babel}
+\usepackage{amsmath}
+\usepackage{amsfonts}
+\usepackage{amssymb}
+\usepackage{tikz}
+\usepackage{graphicx}
+\usepackage{xcolor}
+\usepackage{setspace}
+\usepackage{todonotes}
+\presetkeys{todonotes}{inline}{}
+\renewcommand{\baselinestretch}{1.25}
+
+\definecolor{orange_garage}{RGB}{255,147,41}
+\definecolor{gris_garage}{RGB}{78,78,78}
+
+\author[Association Deuxfleurs]{~\linebreak Vincent Giraud}
+\title[De l'auto-hébergement à l'entre-hébergement avec Garage]{De l'auto-hébergement à l'entre-hébergement :\\Garage, pour conserver ses données ensemble}
+%\setbeamercovered{transparent} 
+%\setbeamertemplate{navigation symbols}{} 
+\date{Capitole du Libre 2022\linebreak
+
+\scriptsize Samedi 19 novembre 2022\linebreak
+} 
+
+\setbeamercolor{palette primary}{fg=gris_garage,bg=orange_garage}
+\setbeamercolor{palette secondary}{fg=gris_garage,bg=gris_garage}
+\setbeamercolor{palette tiertary}{fg=white,bg=gris_garage}
+\setbeamercolor{palette quaternary}{fg=white,bg=gris_garage}
+\setbeamercolor{navigation symbols}{fg=black, bg=white}
+\setbeamercolor{navigation symbols dimmed}{fg=darkgray, bg=white}
+\setbeamercolor{itemize item}{fg=gris_garage}
+\setbeamertemplate{itemize item}[circle]
+
+\addtobeamertemplate{navigation symbols}{}{%
+    \usebeamerfont{footline}%
+    \usebeamercolor[fg]{footline}%
+    \hspace{1em}%
+    \insertframenumber/\inserttotalframenumber
+}
+
+\setbeamertemplate{headline}
+{%
+  \leavevmode%
+  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{section in head/foot}%
+    \hbox to .5\paperwidth{\hfil\insertsectionhead\hfil}
+  \end{beamercolorbox}%
+  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{subsection in head/foot}%
+    \hbox to .5\paperwidth{\hfil\insertsubsectionhead\hfil}
+  \end{beamercolorbox}%
+}
+\addtobeamertemplate{footnote}{}{\vspace{2ex}}
+
+\begin{document}
+\begin{frame}
+\titlepage
+\end{frame}
+
+\section{Introduction}
+\subsection{Présentation}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\begin{center}
+\includegraphics[width=3.5cm]{deuxfleurs-logo.png}\linebreak
+
+\texttt{https://deuxfleurs.fr}
+\end{center}
+\column{0.4 \linewidth}
+\begin{center}
+Deuxfleurs est une association militant en faveur d'un internet plus convivial, avec une organisation et des rapports de force repensés.\linebreak
+
+Nous faisons partie du CHATONS\footnote[frame]{Collectif des Hébergeurs Alternatifs, Transparents, Ouverts, Neutres et Solidaires} depuis avril 2022.
+
+\includegraphics[width=2cm]{logo_chatons.png}
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Héberger à la maison}
+\begin{frame}
+\begin{columns}
+\begin{column}{0.5 \linewidth}
+\begin{center}
+Pour échapper au contrôle et au giron des opérateurs de clouds, héberger ses données à la maison présente de nombreux avantages...
+\end{center}
+
+\vspace{0.5cm}
+
+\begin{itemize}[<+(1)->]
+\item On récupère la souveraineté sur ses données
+\item On gagne en vie privée
+\item On gagne en libertés
+\item On est responsabilisé face à ses besoins
+\end{itemize}
+\end{column}
+\vrule{}
+\begin{column}{0.5 \linewidth}
+\begin{center}
+\onslide<6->{... mais aussi bien des contraintes...}
+\end{center}
+
+\vspace{0.5cm}
+
+\begin{itemize}[<+(2)->]
+\item On repose sur une connexion internet pour particulier
+\item Un certain savoir-faire et moultes compétences sont requis
+\item Assurer la résilience de ses services est difficile
+\item Bien sauvegarder ses données, et ceci au-delà de son site géographique, n'est pas évident
+\end{itemize}
+\end{column}
+\end{columns}
+\end{frame}
+
+\subsection{Sauvegarder pour se parer à tout imprévu}
+\begin{frame}
+\begin{center}
+Sauvegarder pour se parer contre les pannes matérielles est une chose...
+
+Sauvegarder pour se parer contre les cambriolages et les incendies en est une autre !\linebreak
+
+\vspace{1cm}
+\onslide<2->{Répartir géographiquement ses données devient alors nécessaire.}
+\end{center}
+\end{frame}
+
+\section{Les solutions à explorer}
+\subsection{L'entre-hébergement}
+\begin{frame}
+\begin{center}
+On a vu récemment se développer au sein du CHATONS la notion d'entre-hébergement : en plus de renforcer l'intégrité des sauvegardes, on va améliorer la disponibilité pendant les coupures de liaison internet, de courant, ou pendant les déménagements d'administrateurs par exemple.\linebreak
+
+\vspace{1cm}
+\onslide<2->
+{
+Dans le cadre du collectif, il s'agit de partager ses volumes de données entre hébergeurs.\linebreak
+
+Pour assurer la confidentialité, on peut chiffrer les données au niveau applicatif.
+}
+\end{center}
+\end{frame}
+
+\subsection{S3 contre les systèmes de fichiers}
+\begin{frame}
+\begin{center}
+Dans le cadre de l'administration de services en ligne, les systèmes de fichiers recèlent certaines difficultés.\linebreak
+
+\vspace{1cm}
+Le standard S3 apporte des facilités; on réduit le stockage à un paradigme de clé-valeur basé essentiellement sur deux opérations seulement: lire ou écrire une clé.
+\end{center}
+\end{frame}
+
+\section{Garage}
+\subsection{Présentation}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\begin{center}
+Garage essaye de répondre à l'ensemble de ces besoins.\linebreak
+
+\vspace{0.5cm}
+Il s'agit d'un logiciel libre permettant de distribuer un service S3 sur diverses machines éloignées.
+\end{center}
+\column{0.5 \linewidth}
+\begin{center}
+\includegraphics[width=4cm]{garage-logo.png}\linebreak
+
+\texttt{https://garagehq.deuxfleurs.fr/}
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Gestion des zones}
+\begin{frame}
+\begin{center}
+Garage va prendre en compte les zones géographiques au moment de répliquer les données.\linebreak
+
+\vspace{1cm}
+\includegraphics[width=13.25cm]{zones.png}
+\end{center}
+\end{frame}
+
+\subsection{Comment ça marche ?}
+\begin{frame}
+\begin{columns}
+\column{0.5 \linewidth}
+\input{schéma europe}
+\column{0.5 \linewidth}
+\begin{center}
+Chaque objet est dupliqué sur plusieurs zones différentes.\linebreak
+
+\onslide<5->{Lorsqu'un nouvel hébergeur rejoint le réseau, la charge se voit équilibrée.}\linebreak
+
+\onslide<12->{Si une zone devient indisponible, les autres continuent d'assurer le service.}\linebreak
+\end{center}
+\end{columns}
+\end{frame}
+
+\subsection{Financement}
+\begin{frame}
+\begin{center}
+Dans le cadre du programme \textit{Horizon 2021} de l'Union Européenne, nous avons reçu une subvention de la part de l'initiative NGI Pointer\footnote[frame]{Next Generation Internet Program for Open Internet Renovation}.\linebreak
+
+\includegraphics[width=3cm]{drapeau_européen.png}\hspace{1cm}
+\includegraphics[width=3cm]{NGI.png}\linebreak
+
+Nous avons ainsi pu financer le développement de Garage pendant 1 an.
+\end{center}
+\end{frame}
+
+\subsection{Licence}
+\begin{frame}
+\begin{center}
+De par nos valeurs, nous avons attribué la licence AGPL version 3 à Garage, notamment afin qu'il reste parmi les biens communs.\linebreak
+
+\vspace{0.5cm}
+\includegraphics[width=5cm]{agpl-v3-logo.png}\linebreak
+\end{center}
+\end{frame}
+
+\subsection{Langage utilisé}
+\begin{frame}
+\begin{center}
+Nous avons décidé d'écrire Garage à l'aide du langage Rust, afin d'obtenir une compilation vers des binaires natifs et efficaces.\linebreak
+
+\includegraphics[width=3.5cm]{rust-logo.png}\linebreak
+
+Ce choix permet également de bénéficier des avantages reconnus de Rust en termes de sécurité.
+\end{center}
+\end{frame}
+
+\subsection{Matériel requis}
+\begin{frame}
+\begin{center}
+Garage peut ainsi être performant sur des machines limitées. Les prérequis sont minimes : n'importe quelle machine avec un processeur qui a moins d'une décennie, 1~gigaoctet de mémoire vive, et 16~gigaoctets de stockage suffit.\linebreak
+
+\vspace{1cm}
+
+Cet aspect est déterminant : il permet en effet d'héberger sur du matériel acheté d'occasion, pour réduire l'impact écologique de nos infrastructures.
+\end{center}
+\end{frame}
+
+\subsection{Performances}
+\begin{frame}
+\begin{center}
+\includegraphics[width=13.25cm]{rpc-amplification.png}
+\end{center}
+\end{frame}
+
+\begin{frame}
+\begin{center}
+\includegraphics[width=11cm]{rpc-complexity.png}
+\end{center}
+\end{frame}
+
+\subsection{Services}
+\begin{frame}
+\begin{center}
+Puisqu'il suit le standard S3, beaucoup de services populaires sont par conséquence compatibles avec Garage :\linebreak
+
+\begin{columns}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{nextcloud-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{peertube-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{matrix-logo.png}
+\end{center}
+\column{0.2 \linewidth}
+\begin{center}
+\includegraphics[width=2.5cm]{mastodon-logo.png}
+\end{center}
+\end{columns}
+~\linebreak
+
+Et comme souvent avec S3, on peut assimiler un bucket à un site, et utiliser le serveur pour héberger des sites web statiques.
+\end{center}
+\end{frame}
+
+\section{Intégration chez Deuxfleurs}
+\subsection{Matériel}
+\begin{frame}
+\begin{center}
+\includegraphics[width=13cm]{neptune.jpg}\linebreak
+
+En pratique, nos serveurs ne sont effectivement que des machines achetées d'occasion (très souvent des anciens ordinateurs destinés à la bureautique en entreprise).
+\end{center}
+\end{frame}
+
+\subsection{Environnement logiciel}
+\begin{frame}
+\begin{center}
+Pour faciliter la reproduction d'un environnement connu, NixOS est installé sur nos machines.\linebreak
+
+\vspace{1cm}
+Pour s’accommoder des réseaux qu'on trouve derrière des routeurs pour particuliers, on s'aide de notre logiciel Diplonat\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/diplonat}}.
+\end{center}
+\end{frame}
+
+\section{Au-delà...}
+\subsection{... de Deuxfleurs}
+\begin{frame}
+\begin{center}
+\includegraphics[width=10cm]{tedomum.png}
+\end{center}
+\end{frame}
+
+\subsection{... de Garage}
+\begin{frame}
+\begin{center}
+Nous avons récemment lancé le développement d'Aérogramme\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/aerogramme}}.\linebreak
+
+\vspace{1cm}
+Il s'agit d'un serveur de stockage de courriels chiffrés.\linebreak
+
+\vspace{1cm}
+Il est conçu pour pouvoir travailler avec Garage.
+\end{center}
+\end{frame}
+
+\section{Fin}
+\subsection{Contacts}
+\begin{frame}
+\begin{center}
+\begin{tikzpicture}
+\node (ronce) {\includegraphics[width=0.95\textwidth]{ronce.jpg}};
+\node[white] at (-5.1,3.6) {Intéressé(e) ?};
+\node[white, align=center] at (4.2,-2.6) {Contactez-nous !\\\texttt{coucou@deuxfleurs.fr}\\\texttt{\#forum:deuxfleurs.fr}};
+\end{tikzpicture}
+\end{center}
+\end{frame}
+\end{document}
+

doc/talks/2022-11-19-Capitole-du-Libre/schéma europe.tex

@@ -0,0 +1,52 @@
+\begin{tikzpicture}
+\node (carte) {\includegraphics[width=\textwidth]{carte-Europe.pdf}};
+
+% \personnage{position X}{position Y}{facteur d'échelle}
+\newcommand{\personnage}[4]
+{
+\fill[#4] ({#1-(0.4 * #3)},{#2-(0.9 * #3)}) .. controls ({#1-(0.4 * #3)},#2) and ({#1+(0.4 * #3)},#2) .. ({#1+(0.4 * #3)},{#2-(0.9 * #3)}) -- ({#1-(0.4 * #3)},{#2-(0.9 * #3)});
+\fill[#4] (#1,#2) circle ({0.25 * #3});
+}
+
+\onslide<1-11>{\personnage{-2.25}{-0.75}{0.75}{green}}
+\onslide<1-11>{\draw (-1.9,-1.6) rectangle ++(1,1.2);}
+\onslide<2-11>{\draw[fill=green] (-1.8,-1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4-5>{\draw[fill=red] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<7-11>{\draw[fill=yellow] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<9-11>{\draw[fill=red] (-1.8,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<3-11>{\draw[fill=blue] (-1.35,-1.525) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<8-11>{\draw[fill=blue] (-1.35,-1.15) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
+\onslide<11-11>{\draw[fill=yellow] (-1.35,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\personnage{1.65}{1.5}{0.75}{blue}
+\draw (0.3,0.7) rectangle ++(1,1.2);
+\onslide<2->{\draw[fill=green] (0.4,0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4->{\draw[fill=red] (0.4,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<10->{\draw[fill=green] (0.4,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+\onslide<3->{\draw[fill=blue] (0.85,0.775) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<9->{\draw[fill=red] (0.85,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<11->{\draw[fill=yellow] (0.85,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\personnage{1.85}{-2.3}{0.75}{red}
+\draw (0.5,-3.15) rectangle ++(1,1.2);
+\onslide<2->{\draw[fill=green] (0.6,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
+\onslide<4-5>{\draw[fill=red] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<7->{\draw[fill=yellow] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<9->{\draw[fill=red] (0.6,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
+\onslide<3-5>{\draw[fill=blue] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<6->{\draw[fill=red] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<8->{\draw[fill=blue] (1.05,-2.7) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
+\onslide<10->{\draw[fill=green] (1.05,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+
+\onslide<5->{\personnage{1.05}{-0.15}{0.75}{yellow}}
+\onslide<5->{\draw (-0.35,-1) rectangle ++(1,1.2);}
+\onslide<6->{\draw[fill=blue] (-0.25,-0.925) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
+\onslide<7->{\draw[fill=yellow] (-0.25,-0.55) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
+\onslide<10->{\draw[fill=green] (-0.25,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
+\onslide<6->{\draw[fill=red] (0.2,-0.925) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
+\onslide<8->{\draw[fill=blue] (0.2,-0.55) rectangle ++(0.35,0.3) node[pos=0.5,white] {\tiny 5};}
+\onslide<11->{\draw[fill=yellow] (0.2,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
+
+\onslide<12->{\draw[line width=0.25cm] (-2.15,-0.5) -- ++(1,-1);}
+\onslide<12->{\draw[line width=0.25cm] (-2.15,-1.5) -- ++(1,1);}
+\end{tikzpicture}

flake.lock

@@ -0,0 +1,124 @@
+{
+  "nodes": {
+    "cargo2nix": {
+      "inputs": {
+        "flake-compat": "flake-compat",
+        "flake-utils": "flake-utils",
+        "nixpkgs": [
+          "nixpkgs"
+        ],
+        "rust-overlay": "rust-overlay"
+      },
+      "locked": {
+        "lastModified": 1666087781,
+        "narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=",
+        "owner": "Alexis211",
+        "repo": "cargo2nix",
+        "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
+        "type": "github"
+      },
+      "original": {
+        "owner": "Alexis211",
+        "repo": "cargo2nix",
+        "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
+        "type": "github"
+      }
+    },
+    "flake-compat": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1650374568,
+        "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "b4a34015c698c7793d592d66adbab377907a2be8",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
+    "flake-utils": {
+      "locked": {
+        "lastModified": 1659877975,
+        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_2": {
+      "locked": {
+        "lastModified": 1667395993,
+        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "nixpkgs": {
+      "locked": {
+        "lastModified": 1665657542,
+        "narHash": "sha256-mojxNyzbvmp8NtVtxqiHGhRfjCALLfk9i/Uup68Y5q8=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a3073c49bc0163fea6a121c276f526837672b555",
+        "type": "github"
+      },
+      "original": {
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "a3073c49bc0163fea6a121c276f526837672b555",
+        "type": "github"
+      }
+    },
+    "root": {
+      "inputs": {
+        "cargo2nix": "cargo2nix",
+        "flake-utils": "flake-utils_2",
+        "nixpkgs": "nixpkgs"
+      }
+    },
+    "rust-overlay": {
+      "inputs": {
+        "flake-utils": [
+          "cargo2nix",
+          "flake-utils"
+        ],
+        "nixpkgs": [
+          "cargo2nix",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1664247556,
+        "narHash": "sha256-J4vazHU3609ekn7dr+3wfqPo5WGlZVAgV7jfux352L0=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "524db9c9ea7bc7743bb74cdd45b6d46ea3fcc2ab",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    }
+  },
+  "root": "root",
+  "version": 7
+}

flake.nix

@@ -0,0 +1,36 @@
+{
+  description = "Garage, an S3-compatible distributed object store for self-hosted deployments";
+
+  inputs.nixpkgs.url = "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555";
+  inputs.cargo2nix = {
+    # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
+    url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
+    inputs.nixpkgs.follows = "nixpkgs";
+  };
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+
+  outputs = { self, nixpkgs, cargo2nix, flake-utils }:
+    let
+      git_version = self.lastModifiedDate;
+      compile = import ./nix/compile.nix;
+    in flake-utils.lib.eachDefaultSystem (system:
+      let pkgs = nixpkgs.legacyPackages.${system};
+      in {
+        packages = {
+          default = (compile {
+            inherit system git_version;
+            pkgsSrc = nixpkgs;
+            cargo2nixOverlay = cargo2nix.overlays.default;
+            release = true;
+          }).workspace.garage { compileMode = "build"; };
+        };
+        devShell = ((compile {
+          inherit system git_version;
+          pkgsSrc = nixpkgs;
+          cargo2nixOverlay = cargo2nix.overlays.default;
+          release = false;
+        }).workspaceShell {
+          packages = [ pkgs.rustfmt cargo2nix.packages.${system}.default ];
+        });
+      });
+}

nix/compile.nix

@@ -1,24 +1,31 @@
 {
-  system ? builtins.currentSystem,
-  target,
+  system,
+  target ? null,
+  pkgsSrc,
+  cargo2nixOverlay,
   compiler ? "rustc",
   release ? false,
   git_version ? null,
   features ? null,
 }:
 
-with import ./common.nix;
-
 let
   log = v: builtins.trace v v;
 
-  pkgs = import pkgsSrc {
+  pkgs =
+    if target != null then
+      import pkgsSrc {
         inherit system;
         crossSystem = {
           config = target;
           isStatic = true;
         };
         overlays = [ cargo2nixOverlay ];
+      }
+    else
+      import pkgsSrc {
+        inherit system;
+        overlays = [ cargo2nixOverlay ];
       };
 
   /*
@@ -34,7 +41,7 @@ let
    NixOS ships them in separate ones. We reunite them with symlinkJoin.
   */
   toolchainOptions =
-    if target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
+    if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
       rustVersion = "1.63.0";
       extraRustComponents = [ "clippy" ];
     } else {

script/helm/garage/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.1.0
+version: 0.2.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

script/helm/garage/templates/ingress.yaml

@@ -18,6 +18,9 @@ metadata:
   name: {{ $fullName }}-s3-api
   labels:
     {{- include "garage.labels" . | nindent 4 }}
+    {{- with .Values.ingress.s3.api.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.ingress.s3.api.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
@@ -80,6 +83,9 @@ metadata:
   name: {{ $fullName }}-s3-web
   labels:
     {{- include "garage.labels" . | nindent 4 }}
+    {{- with .Values.ingress.s3.web.labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
   {{- with .Values.ingress.s3.web.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

script/helm/garage/values.yaml

@@ -85,14 +85,15 @@ service:
 ingress:
   s3:
     api:
-      enabled: true
+      enabled: false
       # Rely either on the className or the annotation below but not both
       # replace "nginx" by an Ingress controller
       # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
-      className: "nginx"
-      annotations:
+      # className: "nginx"
+      annotations: {}
         # kubernetes.io/ingress.class: "nginx"
         # kubernetes.io/tls-acme: "true"
+      labels: {}
       hosts:
         - host: "s3.garage.tld" # garage S3 API endpoint
           paths:
@@ -107,11 +108,15 @@ ingress:
       #    hosts:
       #      - kubernetes.docker.internal
     web:
-      enabled: true
-      className: "nginx"
+      enabled: false
+      # Rely either on the className or the annotation below but not both
+      # replace "nginx" by an Ingress controller
+      # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
+      # className: "nginx"
       annotations: {}
         # kubernetes.io/ingress.class: nginx
         # kubernetes.io/tls-acme: "true"
+      labels: {}
       hosts:
        - host: "*.web.garage.tld" # wildcard website access with bucket name prefix
          paths:

shell.nix

@@ -71,13 +71,25 @@ function refresh_cache {
   for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
     echo "Updating cache for ''${attr}"
     derivation=$(nix-instantiate --attr ''${attr})
-    nix copy \
+    nix copy -j8 \
       --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
       $(nix-store -qR ''${derivation%\!bin})
   done
   rm /tmp/nix-signing-key.sec
 }
 
+function refresh_flake_cache {
+  pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+  for attr in packages.x86_64-linux.default; do
+    echo "Updating cache for ''${attr}"
+    derivation=$(nix path-info --derivation ".#''${attr}")
+    nix copy -j8 \
+      --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+      $(nix-store -qR ''${derivation})
+  done
+  rm /tmp/nix-signing-key.sec
+}
+
 function to_s3 {
   aws \
       --endpoint-url https://garage.deuxfleurs.fr \

src/api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,11 +14,11 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_model = { version = "0.8.0", path = "../model" }
-garage_table = { version = "0.8.0", path = "../table" }
-garage_block = { version = "0.8.0", path = "../block" }
-garage_util = { version = "0.8.0", path = "../util" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
+garage_model = { version = "0.8.1", path = "../model" }
+garage_table = { version = "0.8.1", path = "../table" }
+garage_block = { version = "0.8.1", path = "../block" }
+garage_util = { version = "0.8.1", path = "../util" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
 
 async-trait = "0.1.7"
 base64 = "0.13"

src/api/admin/api_server.rs

@@ -15,6 +15,7 @@ use opentelemetry_prometheus::PrometheusExporter;
 use prometheus::{Encoder, TextEncoder};
 
 use garage_model::garage::Garage;
+use garage_rpc::system::ClusterHealthStatus;
 use garage_util::error::Error as GarageError;
 
 use crate::generic_server::*;
@@ -76,6 +77,31 @@ impl AdminApiServer {
 			.body(Body::empty())?)
 	}
 
+	fn handle_health(&self) -> Result<Response<Body>, Error> {
+		let health = self.garage.system.health();
+
+		let (status, status_str) = match health.status {
+			ClusterHealthStatus::Healthy => (StatusCode::OK, "Garage is fully operational"),
+			ClusterHealthStatus::Degraded => (
+				StatusCode::OK,
+				"Garage is operational but some storage nodes are unavailable",
+			),
+			ClusterHealthStatus::Unavailable => (
+				StatusCode::SERVICE_UNAVAILABLE,
+				"Quorum is not available for some/all partitions, reads and writes will fail",
+			),
+		};
+		let status_str = format!(
+			"{}\nConsult the full health check API endpoint at /v0/health for more details\n",
+			status_str
+		);
+
+		Ok(Response::builder()
+			.status(status)
+			.header(http::header::CONTENT_TYPE, "text/plain")
+			.body(Body::from(status_str))?)
+	}
+
 	fn handle_metrics(&self) -> Result<Response<Body>, Error> {
 		#[cfg(feature = "metrics")]
 		{
@@ -124,6 +150,7 @@ impl ApiHandler for AdminApiServer {
 	) -> Result<Response<Body>, Error> {
 		let expected_auth_header =
 			match endpoint.authorization_type() {
+				Authorization::None => None,
 				Authorization::MetricsToken => self.metrics_token.as_ref(),
 				Authorization::AdminToken => match &self.admin_token {
 					None => return Err(Error::forbidden(
@@ -147,8 +174,10 @@ impl ApiHandler for AdminApiServer {
 
 		match endpoint {
 			Endpoint::Options => self.handle_options(&req),
+			Endpoint::Health => self.handle_health(),
 			Endpoint::Metrics => self.handle_metrics(),
 			Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await,
+			Endpoint::GetClusterHealth => handle_get_cluster_health(&self.garage).await,
 			Endpoint::ConnectClusterNodes => handle_connect_cluster_nodes(&self.garage, req).await,
 			// Layout
 			Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await,

src/api/admin/cluster.rs

@@ -43,6 +43,11 @@ pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<
 	Ok(json_ok_response(&res)?)
 }
 
+pub async fn handle_get_cluster_health(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
+	let health = garage.system.health();
+	Ok(json_ok_response(&health)?)
+}
+
 pub async fn handle_connect_cluster_nodes(
 	garage: &Arc<Garage>,
 	req: Request<Body>,

src/api/admin/router.rs

@@ -6,6 +6,7 @@ use crate::admin::error::*;
 use crate::router_macros::*;
 
 pub enum Authorization {
+	None,
 	MetricsToken,
 	AdminToken,
 }
@@ -16,8 +17,10 @@ router_match! {@func
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Endpoint {
 	Options,
+	Health,
 	Metrics,
 	GetClusterStatus,
+	GetClusterHealth,
 	ConnectClusterNodes,
 	// Layout
 	GetClusterLayout,
@@ -88,8 +91,10 @@ impl Endpoint {
 
 		let res = router_match!(@gen_path_parser (req.method(), path, query) [
 			OPTIONS _ => Options,
+			GET "/health" => Health,
 			GET "/metrics" => Metrics,
 			GET "/v0/status" => GetClusterStatus,
+			GET "/v0/health" => GetClusterHealth,
 			POST "/v0/connect" => ConnectClusterNodes,
 			// Layout endpoints
 			GET "/v0/layout" => GetClusterLayout,
@@ -130,6 +135,7 @@ impl Endpoint {
 	/// Get the kind of authorization which is required to perform the operation.
 	pub fn authorization_type(&self) -> Authorization {
 		match self {
+			Self::Health => Authorization::None,
 			Self::Metrics => Authorization::MetricsToken,
 			_ => Authorization::AdminToken,
 		}
@@ -137,9 +143,13 @@ impl Endpoint {
 }
 
 generateQueryParameters! {
+	keywords: [],
+	fields: [
+		"format" => format,
 		"id" => id,
 		"search" => search,
 		"globalAlias" => global_alias,
 		"alias" => alias,
 		"accessKeyId" => access_key_id
+	]
 }

src/api/k2v/router.rs

@@ -96,7 +96,7 @@ impl Endpoint {
 	fn from_get(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY if causality_token => PollItem (query::sort_key, query::causality_token, opt_parse::timeout),
 				EMPTY => ReadItem (query::sort_key),
@@ -111,7 +111,7 @@ impl Endpoint {
 	fn from_search(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 			],
 			no_key: [
@@ -125,7 +125,7 @@ impl Endpoint {
 	fn from_head(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
 			],
@@ -140,7 +140,7 @@ impl Endpoint {
 	fn from_post(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 			],
 			no_key: [
@@ -155,7 +155,7 @@ impl Endpoint {
 	fn from_put(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY => InsertItem (query::sort_key),
 
@@ -169,7 +169,7 @@ impl Endpoint {
 	fn from_delete(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
 			key: [
 				EMPTY => DeleteItem (query::sort_key),
 			],
@@ -232,6 +232,11 @@ impl Endpoint {
 
 // parameter name => struct field
 generateQueryParameters! {
+	keywords: [
+		"delete" => DELETE,
+		"search" => SEARCH
+	],
+	fields: [
 		"prefix" => prefix,
 		"start" => start,
 		"causality_token" => causality_token,
@@ -240,13 +245,5 @@ generateQueryParameters! {
 		"reverse" => reverse,
 		"sort_key" => sort_key,
 		"timeout" => timeout
-}
-
-mod keywords {
-	//! This module contain all query parameters with no associated value
-	//! used to differentiate endpoints.
-	pub const EMPTY: &str = "";
-
-	pub const DELETE: &str = "delete";
-	pub const SEARCH: &str = "search";
+	]
 }

src/api/router_macros.rs

@@ -4,10 +4,9 @@ macro_rules! router_match {
     (@match $enum:expr , [ $($endpoint:ident,)* ]) => {{
         // usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] }
         // returns true if the variant was one of the listed variants, false otherwise.
-        use Endpoint::*;
         match $enum {
             $(
-            $endpoint { .. } => true,
+            Endpoint::$endpoint { .. } => true,
             )*
             _ => false
         }
@@ -15,10 +14,9 @@ macro_rules! router_match {
     (@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{
         // usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] }
         // returns Some(field_value), or None if the variant was not one of the listed variants.
-        use Endpoint::*;
         match $enum {
             $(
-            $endpoint {$param, ..} => Some($param),
+            Endpoint::$endpoint {$param, ..} => Some($param),
             )*
             _ => None
         }
@@ -28,10 +26,9 @@ macro_rules! router_match {
      $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
      ]) => {{
         {
-			use Endpoint::*;
             match ($method, $reqpath) {
                 $(
-					(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => $api {
+                    (&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api {
                         $($(
                             $param: router_match!(@@parse_param $query, $conv, $param),
                         )*)?
@@ -60,11 +57,9 @@ macro_rules! router_match {
         //   ]
         // }
         // See in from_{method} for more detailed usage.
-        use Endpoint::*;
-        use keywords::*;
         match ($keyword, !$key.is_empty()){
             $(
-            ($kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok($api_k {
+            (Keyword::$kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok(Endpoint::$api_k {
                 $key,
                 $($(
                     $param_k: router_match!(@@parse_param $query, $conv_k, $param_k),
@@ -72,7 +67,7 @@ macro_rules! router_match {
             }),
             )*
             $(
-            ($kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok($api_nk {
+            (Keyword::$kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok(Endpoint::$api_nk {
                 $($(
                     $param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk),
                 )*)?
@@ -144,14 +139,40 @@ macro_rules! router_match {
 /// This macro is used to generate part of the code in this module. It must be called only one, and
 /// is useless outside of this module.
 macro_rules! generateQueryParameters {
-    ( $($rest:expr => $name:ident),* ) => {
+    (
+        keywords: [ $($kw_param:expr => $kw_name: ident),* ],
+        fields: [ $($f_param:expr => $f_name:ident),* ]
+    ) => {
+        #[derive(Debug)]
+        #[allow(non_camel_case_types)]
+        #[allow(clippy::upper_case_acronyms)]
+        enum Keyword {
+            EMPTY,
+            $( $kw_name, )*
+        }
+
+        impl std::fmt::Display for Keyword {
+            fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+                match self {
+                    Keyword::EMPTY => write!(f, "``"),
+                    $( Keyword::$kw_name => write!(f, "`{}`", $kw_param), )*
+                }
+            }
+        }
+
+        impl Default for Keyword {
+            fn default() -> Self {
+                Keyword::EMPTY
+            }
+        }
+
         /// Struct containing all query parameters used in endpoints. Think of it as an HashMap,
         /// but with keys statically known.
         #[derive(Debug, Default)]
         struct QueryParameters<'a> {
-            keyword: Option<Cow<'a, str>>,
+            keyword: Option<Keyword>,
             $(
-            $name: Option<Cow<'a, str>>,
+            $f_name: Option<Cow<'a, str>>,
             )*
         }
 
@@ -160,34 +181,29 @@ macro_rules! generateQueryParameters {
             fn from_query(query: &'a str) -> Result<Self, Error> {
                 let mut res: Self = Default::default();
                 for (k, v) in url::form_urlencoded::parse(query.as_bytes()) {
-                    let repeated = match k.as_ref() {
+                    match k.as_ref() {
                         $(
-                            $rest => if !v.is_empty() {
-                                res.$name.replace(v).is_some()
-                            } else {
-                                false
+                            $kw_param => if let Some(prev_kw) = res.keyword.replace(Keyword::$kw_name) {
+                                return Err(Error::bad_request(format!(
+                                    "Multiple keywords: '{}' and '{}'", prev_kw, $kw_param
+                                )));
+                            },
+                        )*
+                        $(
+                            $f_param => if !v.is_empty() {
+                                if res.$f_name.replace(v).is_some() {
+                                    return Err(Error::bad_request(format!(
+                                        "Query parameter repeated: '{}'", k
+                                    )));
+                                }
                             },
                         )*
                         _ => {
-                            if k.starts_with("response-") || k.starts_with("X-Amz-") {
-                                false
-                            } else if v.as_ref().is_empty() {
-                                if res.keyword.replace(k).is_some() {
-                                    return Err(Error::bad_request("Multiple keywords"));
-                                }
-                                continue;
-                            } else {
+                            if !(k.starts_with("response-") || k.starts_with("X-Amz-")) {
                                 debug!("Received an unknown query parameter: '{}'", k);
-                                false
                             }
                         }
                     };
-                    if repeated {
-                        return Err(Error::bad_request(format!(
-                            "Query parameter repeated: '{}'",
-                            k
-                        )));
-                    }
                 }
                 Ok(res)
             }
@@ -198,8 +214,8 @@ macro_rules! generateQueryParameters {
                 if self.keyword.is_some() {
                     Some("Keyword not used")
                 } $(
-                    else if self.$name.is_some() {
-                        Some(concat!("'", $rest, "'"))
+                    else if self.$f_name.is_some() {
+                        Some(concat!("'", $f_param, "'"))
                     }
                 )* else {
                     None

src/api/s3/bucket.rs

@@ -161,6 +161,15 @@ pub async fn handle_create_bucket(
 			return Err(CommonError::BucketAlreadyExists.into());
 		}
 	} else {
+		// Check user is allowed to create bucket
+		if !key_params.allow_create_bucket.get() {
+			return Err(CommonError::Forbidden(format!(
+				"Access key {} is not allowed to create buckets",
+				api_key.key_id
+			))
+			.into());
+		}
+
 		// Create the bucket!
 		if !is_valid_bucket_name(&bucket_name) {
 			return Err(Error::bad_request(format!(

src/api/s3/put.rs

@@ -119,6 +119,17 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 		return Ok((version_uuid, data_md5sum_hex));
 	}
 
+	// The following consists in many steps that can each fail.
+	// Keep track that some cleanup will be needed if things fail
+	// before everything is finished (cleanup is done using the Drop trait).
+	let mut interrupted_cleanup = InterruptedCleanup(Some((
+		garage.clone(),
+		bucket.id,
+		key.into(),
+		version_uuid,
+		version_timestamp,
+	)));
+
 	// Write version identifier in object table so that we have a trace
 	// that we are uploading something
 	let mut object_version = ObjectVersion {
@@ -139,7 +150,6 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	// Transfer data and verify checksum
 	let first_block_hash = async_blake2sum(first_block.clone()).await;
 
-	let tx_result = (|| async {
 	let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks(
 		&garage,
 		&version,
@@ -159,24 +169,8 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 
 	check_quotas(&garage, bucket, key, total_size).await?;
 
-		Ok((total_size, data_md5sum))
-	})()
-	.await;
-
-	// If something went wrong, clean up
-	let (total_size, md5sum_arr) = match tx_result {
-		Ok(rv) => rv,
-		Err(e) => {
-			// Mark object as aborted, this will free the blocks further down
-			object_version.state = ObjectVersionState::Aborted;
-			let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]);
-			garage.object_table.insert(&object).await?;
-			return Err(e);
-		}
-	};
-
 	// Save final object state, marked as Complete
-	let md5sum_hex = hex::encode(md5sum_arr);
+	let md5sum_hex = hex::encode(data_md5sum);
 	object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
 		ObjectVersionMeta {
 			headers,
@@ -188,6 +182,10 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	let object = Object::new(bucket.id, key.into(), vec![object_version]);
 	garage.object_table.insert(&object).await?;
 
+	// We were not interrupted, everything went fine.
+	// We won't have to clean up on drop.
+	interrupted_cleanup.cancel();
+
 	Ok((version_uuid, md5sum_hex))
 }
 
@@ -426,6 +424,33 @@ pub fn put_response(version_uuid: Uuid, md5sum_hex: String) -> Response<Body> {
 		.unwrap()
 }
 
+struct InterruptedCleanup(Option<(Arc<Garage>, Uuid, String, Uuid, u64)>);
+
+impl InterruptedCleanup {
+	fn cancel(&mut self) {
+		drop(self.0.take());
+	}
+}
+impl Drop for InterruptedCleanup {
+	fn drop(&mut self) {
+		if let Some((garage, bucket_id, key, version_uuid, version_ts)) = self.0.take() {
+			tokio::spawn(async move {
+				let object_version = ObjectVersion {
+					uuid: version_uuid,
+					timestamp: version_ts,
+					state: ObjectVersionState::Aborted,
+				};
+				let object = Object::new(bucket_id, key, vec![object_version]);
+				if let Err(e) = garage.object_table.insert(&object).await {
+					warn!("Cannot cleanup after aborted PutObject: {}", e);
+				}
+			});
+		}
+	}
+}
+
+// ----
+
 pub async fn handle_create_multipart_upload(
 	garage: Arc<Garage>,
 	req: &Request<Body>,

src/api/s3/router.rs

@@ -355,7 +355,7 @@ impl Endpoint {
 	fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker),
 				EMPTY => GetObject (query_opt::version_id, opt_parse::part_number),
@@ -412,7 +412,7 @@ impl Endpoint {
 	fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
 			],
@@ -426,7 +426,7 @@ impl Endpoint {
 	fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY if upload_id  => CompleteMultipartUpload (query::upload_id),
 				RESTORE => RestoreObject (query_opt::version_id),
@@ -448,7 +448,7 @@ impl Endpoint {
 	) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, headers),
+			(query.keyword.take().unwrap_or_default(), key, query, headers),
 			key: [
 				EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id),
 				EMPTY header "x-amz-copy-source" => CopyObject,
@@ -490,7 +490,7 @@ impl Endpoint {
 	fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
+			(query.keyword.take().unwrap_or_default(), key, query, None),
 			key: [
 				EMPTY if upload_id => AbortMultipartUpload (query::upload_id),
 				EMPTY => DeleteObject (query_opt::version_id),
@@ -624,6 +624,39 @@ impl Endpoint {
 
 // parameter name => struct field
 generateQueryParameters! {
+	keywords: [
+		"accelerate" => ACCELERATE,
+		"acl" => ACL,
+		"analytics" => ANALYTICS,
+		"cors" => CORS,
+		"delete" => DELETE,
+		"encryption" => ENCRYPTION,
+		"intelligent-tiering" => INTELLIGENT_TIERING,
+		"inventory" => INVENTORY,
+		"legal-hold" => LEGAL_HOLD,
+		"lifecycle" => LIFECYCLE,
+		"location" => LOCATION,
+		"logging" => LOGGING,
+		"metrics" => METRICS,
+		"notification" => NOTIFICATION,
+		"object-lock" => OBJECT_LOCK,
+		"ownershipControls" => OWNERSHIP_CONTROLS,
+		"policy" => POLICY,
+		"policyStatus" => POLICY_STATUS,
+		"publicAccessBlock" => PUBLIC_ACCESS_BLOCK,
+		"replication" => REPLICATION,
+		"requestPayment" => REQUEST_PAYMENT,
+		"restore" => RESTORE,
+		"retention" => RETENTION,
+		"select" => SELECT,
+		"tagging" => TAGGING,
+		"torrent" => TORRENT,
+		"uploads" => UPLOADS,
+		"versioning" => VERSIONING,
+		"versions" => VERSIONS,
+		"website" => WEBSITE
+	],
+	fields: [
 		"continuation-token" => continuation_token,
 		"delimiter" => delimiter,
 		"encoding-type" => encoding_type,
@@ -644,43 +677,7 @@ generateQueryParameters! {
 		"upload-id-marker" => upload_id_marker,
 		"versionId" => version_id,
 		"version-id-marker" => version_id_marker
-}
-
-mod keywords {
-	//! This module contain all query parameters with no associated value S3 uses to differentiate
-	//! endpoints.
-	pub const EMPTY: &str = "";
-
-	pub const ACCELERATE: &str = "accelerate";
-	pub const ACL: &str = "acl";
-	pub const ANALYTICS: &str = "analytics";
-	pub const CORS: &str = "cors";
-	pub const DELETE: &str = "delete";
-	pub const ENCRYPTION: &str = "encryption";
-	pub const INTELLIGENT_TIERING: &str = "intelligent-tiering";
-	pub const INVENTORY: &str = "inventory";
-	pub const LEGAL_HOLD: &str = "legal-hold";
-	pub const LIFECYCLE: &str = "lifecycle";
-	pub const LOCATION: &str = "location";
-	pub const LOGGING: &str = "logging";
-	pub const METRICS: &str = "metrics";
-	pub const NOTIFICATION: &str = "notification";
-	pub const OBJECT_LOCK: &str = "object-lock";
-	pub const OWNERSHIP_CONTROLS: &str = "ownershipControls";
-	pub const POLICY: &str = "policy";
-	pub const POLICY_STATUS: &str = "policyStatus";
-	pub const PUBLIC_ACCESS_BLOCK: &str = "publicAccessBlock";
-	pub const REPLICATION: &str = "replication";
-	pub const REQUEST_PAYMENT: &str = "requestPayment";
-	pub const RESTORE: &str = "restore";
-	pub const RETENTION: &str = "retention";
-	pub const SELECT: &str = "select";
-	pub const TAGGING: &str = "tagging";
-	pub const TORRENT: &str = "torrent";
-	pub const UPLOADS: &str = "uploads";
-	pub const VERSIONING: &str = "versioning";
-	pub const VERSIONS: &str = "versions";
-	pub const WEBSITE: &str = "website";
+	]
 }
 
 #[cfg(test)]

src/block/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,10 +14,10 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_util = { version = "0.8.0", path = "../util" }
-garage_table = { version = "0.8.0", path = "../table" }
+garage_db = { version = "0.8.1", path = "../db" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_util = { version = "0.8.1", path = "../util" }
+garage_table = { version = "0.8.1", path = "../table" }
 
 opentelemetry = "0.17"
 
@@ -31,7 +31,6 @@ rand = "0.8"
 async-compression = { version = "0.3", features = ["tokio", "zstd"] }
 zstd = { version = "0.9", default-features = false }
 
-rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 

src/block/manager.rs

@@ -3,8 +3,10 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
 
+use arc_swap::ArcSwapOption;
 use async_trait::async_trait;
 use bytes::Bytes;
+use rand::prelude::*;
 use serde::{Deserialize, Serialize};
 
 use futures::Stream;
@@ -22,9 +24,12 @@ use garage_rpc::rpc_helper::netapp::stream::{stream_asyncread, ByteStream};
 
 use garage_db as db;
 
+use garage_util::background::{vars, BackgroundRunner};
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
+use garage_util::persister::PersisterShared;
+use garage_util::time::msec_to_rfc3339;
 
 use garage_rpc::rpc_helper::OrderTag;
 use garage_rpc::system::System;
@@ -87,7 +92,17 @@ pub struct BlockManager {
 
 	pub(crate) metrics: BlockManagerMetrics,
 
-	tx_scrub_command: mpsc::Sender<ScrubWorkerCommand>,
+	pub scrub_persister: PersisterShared<ScrubWorkerPersisted>,
+	tx_scrub_command: ArcSwapOption<mpsc::Sender<ScrubWorkerCommand>>,
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub struct BlockResyncErrorInfo {
+	pub hash: Hash,
+	pub refcount: u64,
+	pub error_count: u64,
+	pub last_try: u64,
+	pub next_try: u64,
 }
 
 // This custom struct contains functions that must only be ran
@@ -114,9 +129,14 @@ impl BlockManager {
 			.netapp
 			.endpoint("garage_block/manager.rs/Rpc".to_string());
 
-		let metrics = BlockManagerMetrics::new(resync.queue.clone(), resync.errors.clone());
+		let metrics = BlockManagerMetrics::new(
+			compression_level,
+			rc.rc.clone(),
+			resync.queue.clone(),
+			resync.errors.clone(),
+		);
 
-		let (scrub_tx, scrub_rx) = mpsc::channel(1);
+		let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info");
 
 		let block_manager = Arc::new(Self {
 			replication,
@@ -128,21 +148,46 @@ impl BlockManager {
 			system,
 			endpoint,
 			metrics,
-			tx_scrub_command: scrub_tx,
+			scrub_persister,
+			tx_scrub_command: ArcSwapOption::new(None),
 		});
 		block_manager.endpoint.set_handler(block_manager.clone());
 
+		block_manager
+	}
+
+	pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
 		// Spawn a bunch of resync workers
 		for index in 0..MAX_RESYNC_WORKERS {
-			let worker = ResyncWorker::new(index, block_manager.clone());
-			block_manager.system.background.spawn_worker(worker);
+			let worker = ResyncWorker::new(index, self.clone());
+			bg.spawn_worker(worker);
 		}
 
 		// Spawn scrub worker
-		let scrub_worker = ScrubWorker::new(block_manager.clone(), scrub_rx);
-		block_manager.system.background.spawn_worker(scrub_worker);
+		let (scrub_tx, scrub_rx) = mpsc::channel(1);
+		self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
+		bg.spawn_worker(ScrubWorker::new(
+			self.clone(),
+			scrub_rx,
+			self.scrub_persister.clone(),
+		));
+	}
 
-		block_manager
+	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
+		self.resync.register_bg_vars(vars);
+
+		vars.register_rw(
+			&self.scrub_persister,
+			"scrub-tranquility",
+			|p| p.get_with(|x| x.tranquility),
+			|p, tranquility| p.set_with(|x| x.tranquility = tranquility),
+		);
+		vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
+			p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
+		});
+		vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
+			p.get_with(|x| x.corruptions_detected)
+		});
 	}
 
 	/// Ask nodes that might have a (possibly compressed) block for it
@@ -309,9 +354,42 @@ impl BlockManager {
 		Ok(self.rc.rc.len()?)
 	}
 
+	/// Get number of items in the refcount table
+	pub fn rc_fast_len(&self) -> Result<Option<usize>, Error> {
+		Ok(self.rc.rc.fast_len()?)
+	}
+
 	/// Send command to start/stop/manager scrub worker
-	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) {
-		let _ = self.tx_scrub_command.send(cmd).await;
+	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) -> Result<(), Error> {
+		let tx = self.tx_scrub_command.load();
+		let tx = tx.as_ref().ok_or_message("scrub worker is not running")?;
+		tx.send(cmd).await.ok_or_message("send error")?;
+		Ok(())
+	}
+
+	/// Get the reference count of a block
+	pub fn get_block_rc(&self, hash: &Hash) -> Result<u64, Error> {
+		Ok(self.rc.get_block_rc(hash)?.as_u64())
+	}
+
+	/// List all resync errors
+	pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
+		let mut blocks = Vec::with_capacity(self.resync.errors.len());
+		for ent in self.resync.errors.iter()? {
+			let (hash, cnt) = ent?;
+			let cnt = ErrorCounter::decode(&cnt);
+			blocks.push(BlockResyncErrorInfo {
+				hash: Hash::try_from(&hash).unwrap(),
+				refcount: 0,
+				error_count: cnt.errors,
+				last_try: cnt.last_try,
+				next_try: cnt.next_try(),
+			});
+		}
+		for block in blocks.iter_mut() {
+			block.refcount = self.get_block_rc(&block.hash)?;
+		}
+		Ok(blocks)
 	}
 
 	//// ----- Managing the reference counter ----
@@ -603,14 +681,21 @@ impl BlockManagerLocked {
 			}
 		};
 
-		let mut path2 = path.clone();
-		path2.set_extension("tmp");
-		let mut f = fs::File::create(&path2).await?;
+		let mut path_tmp = path.clone();
+		let tmp_extension = format!("tmp{}", hex::encode(thread_rng().gen::<[u8; 4]>()));
+		path_tmp.set_extension(tmp_extension);
+
+		let mut delete_on_drop = DeleteOnDrop(Some(path_tmp.clone()));
+
+		let mut f = fs::File::create(&path_tmp).await?;
 		f.write_all(data).await?;
 		f.sync_all().await?;
 		drop(f);
 
-		fs::rename(path2, path).await?;
+		fs::rename(path_tmp, path).await?;
+
+		delete_on_drop.cancel();
+
 		if let Some(to_delete) = to_delete {
 			fs::remove_file(to_delete).await?;
 		}
@@ -676,3 +761,23 @@ async fn read_stream_to_end(mut stream: ByteStream) -> Result<Bytes, Error> {
 		.concat()
 		.into())
 }
+
+struct DeleteOnDrop(Option<PathBuf>);
+
+impl DeleteOnDrop {
+	fn cancel(&mut self) {
+		drop(self.0.take());
+	}
+}
+
+impl Drop for DeleteOnDrop {
+	fn drop(&mut self) {
+		if let Some(path) = self.0.take() {
+			tokio::spawn(async move {
+				if let Err(e) = fs::remove_file(&path).await {
+					debug!("DeleteOnDrop failed for {}: {}", path.display(), e);
+				}
+			});
+		}
+	}
+}

src/block/metrics.rs

@@ -1,9 +1,12 @@
 use opentelemetry::{global, metrics::*};
 
+use garage_db as db;
 use garage_db::counted_tree_hack::CountedTree;
 
 /// TableMetrics reference all counter used for metrics
 pub struct BlockManagerMetrics {
+	pub(crate) _compression_level: ValueObserver<u64>,
+	pub(crate) _rc_size: ValueObserver<u64>,
 	pub(crate) _resync_queue_len: ValueObserver<u64>,
 	pub(crate) _resync_errored_blocks: ValueObserver<u64>,
 
@@ -23,9 +26,31 @@ pub struct BlockManagerMetrics {
 }
 
 impl BlockManagerMetrics {
-	pub fn new(resync_queue: CountedTree, resync_errors: CountedTree) -> Self {
+	pub fn new(
+		compression_level: Option<i32>,
+		rc_tree: db::Tree,
+		resync_queue: CountedTree,
+		resync_errors: CountedTree,
+	) -> Self {
 		let meter = global::meter("garage_model/block");
 		Self {
+			_compression_level: meter
+				.u64_value_observer("block.compression_level", move |observer| {
+					match compression_level {
+						Some(v) => observer.observe(v as u64, &[]),
+						None => observer.observe(0 as u64, &[]),
+					}
+				})
+				.with_description("Garage compression level for node")
+				.init(),
+			_rc_size: meter
+				.u64_value_observer("block.rc_size", move |observer| {
+					if let Ok(Some(v)) = rc_tree.fast_len() {
+						observer.observe(v as u64, &[])
+					}
+				})
+				.with_description("Number of blocks known to the reference counter")
+				.init(),
 			_resync_queue_len: meter
 				.u64_value_observer("block.resync_queue_length", move |observer| {
 					observer.observe(resync_queue.len() as u64, &[])

src/block/rc.rs

@@ -169,4 +169,11 @@ impl RcEntry {
 	pub(crate) fn is_needed(&self) -> bool {
 		!self.is_deletable()
 	}
+
+	pub(crate) fn as_u64(&self) -> u64 {
+		match self {
+			RcEntry::Present { count } => *count,
+			_ => 0,
+		}
+	}
 }

src/block/repair.rs

@@ -13,7 +13,7 @@ use tokio::sync::watch;
 use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::*;
-use garage_util::persister::Persister;
+use garage_util::persister::PersisterShared;
 use garage_util::time::*;
 use garage_util::tranquilizer::Tranquilizer;
 
@@ -53,7 +53,7 @@ impl Worker for RepairWorker {
 		"Block repair worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
+	fn status(&self) -> WorkerStatus {
 		match self.block_iter.as_ref() {
 			None => {
 				let idx_bytes = self
@@ -66,9 +66,20 @@ impl Worker for RepairWorker {
 				} else {
 					idx_bytes
 				};
-				Some(format!("Phase 1: {}", hex::encode(idx_bytes)))
+				WorkerStatus {
+					progress: Some("0.00%".into()),
+					freeform: vec![format!(
+						"Currently in phase 1, iterator position: {}",
+						hex::encode(idx_bytes)
+					)],
+					..Default::default()
 				}
-			Some(bi) => Some(format!("Phase 2: {:.2}% done", bi.progress() * 100.)),
+			}
+			Some(bi) => WorkerStatus {
+				progress: Some(format!("{:.2}%", bi.progress() * 100.)),
+				freeform: vec!["Currently in phase 2".into()],
+				..Default::default()
+			},
 		}
 	}
 
@@ -137,7 +148,7 @@ impl Worker for RepairWorker {
 		}
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		unreachable!()
 	}
 }
@@ -157,15 +168,24 @@ pub struct ScrubWorker {
 	work: ScrubWorkerState,
 	tranquilizer: Tranquilizer,
 
-	persister: Persister<ScrubWorkerPersisted>,
-	persisted: ScrubWorkerPersisted,
+	persister: PersisterShared<ScrubWorkerPersisted>,
 }
 
 #[derive(Serialize, Deserialize)]
-struct ScrubWorkerPersisted {
-	tranquility: u32,
-	time_last_complete_scrub: u64,
-	corruptions_detected: u64,
+pub struct ScrubWorkerPersisted {
+	pub tranquility: u32,
+	pub(crate) time_last_complete_scrub: u64,
+	pub(crate) corruptions_detected: u64,
+}
+impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {}
+impl Default for ScrubWorkerPersisted {
+	fn default() -> Self {
+		ScrubWorkerPersisted {
+			time_last_complete_scrub: 0,
+			tranquility: INITIAL_SCRUB_TRANQUILITY,
+			corruptions_detected: 0,
+		}
+	}
 }
 
 enum ScrubWorkerState {
@@ -186,27 +206,20 @@ pub enum ScrubWorkerCommand {
 	Pause(Duration),
 	Resume,
 	Cancel,
-	SetTranquility(u32),
 }
 
 impl ScrubWorker {
-	pub fn new(manager: Arc<BlockManager>, rx_cmd: mpsc::Receiver<ScrubWorkerCommand>) -> Self {
-		let persister = Persister::new(&manager.system.metadata_dir, "scrub_info");
-		let persisted = match persister.load() {
-			Ok(v) => v,
-			Err(_) => ScrubWorkerPersisted {
-				time_last_complete_scrub: 0,
-				tranquility: INITIAL_SCRUB_TRANQUILITY,
-				corruptions_detected: 0,
-			},
-		};
+	pub(crate) fn new(
+		manager: Arc<BlockManager>,
+		rx_cmd: mpsc::Receiver<ScrubWorkerCommand>,
+		persister: PersisterShared<ScrubWorkerPersisted>,
+	) -> Self {
 		Self {
 			manager,
 			rx_cmd,
 			work: ScrubWorkerState::Finished,
 			tranquilizer: Tranquilizer::new(30),
 			persister,
-			persisted,
 		}
 	}
 
@@ -255,12 +268,6 @@ impl ScrubWorker {
 					}
 				}
 			}
-			ScrubWorkerCommand::SetTranquility(t) => {
-				self.persisted.tranquility = t;
-				if let Err(e) = self.persister.save_async(&self.persisted).await {
-					error!("Could not save new tranquilitiy value: {}", e);
-				}
-			}
 		}
 	}
 }
@@ -271,29 +278,37 @@ impl Worker for ScrubWorker {
 		"Block scrub worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
-		let s = match &self.work {
-			ScrubWorkerState::Running(bsi) => format!(
-				"{:.2}% done (tranquility = {})",
-				bsi.progress() * 100.,
-				self.persisted.tranquility
-			),
-			ScrubWorkerState::Paused(bsi, rt) => {
-				format!(
-					"Paused, {:.2}% done, resumes at {}",
-					bsi.progress() * 100.,
-					msec_to_rfc3339(*rt)
+	fn status(&self) -> WorkerStatus {
+		let (corruptions_detected, tranquility, time_last_complete_scrub) =
+			self.persister.get_with(|p| {
+				(
+					p.corruptions_detected,
+					p.tranquility,
+					p.time_last_complete_scrub,
 				)
-			}
-			ScrubWorkerState::Finished => format!(
-				"Last completed scrub: {}",
-				msec_to_rfc3339(self.persisted.time_last_complete_scrub)
-			),
+			});
+
+		let mut s = WorkerStatus {
+			persistent_errors: Some(corruptions_detected),
+			tranquility: Some(tranquility),
+			..Default::default()
 		};
-		Some(format!(
-			"{} ; corruptions detected: {}",
-			s, self.persisted.corruptions_detected
-		))
+		match &self.work {
+			ScrubWorkerState::Running(bsi) => {
+				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
+			}
+			ScrubWorkerState::Paused(bsi, rt) => {
+				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
+				s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
+			}
+			ScrubWorkerState::Finished => {
+				s.freeform = vec![format!(
+					"Last scrub completed at {}",
+					msec_to_rfc3339(time_last_complete_scrub)
+				)];
+			}
+		}
+		s
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
@@ -310,18 +325,17 @@ impl Worker for ScrubWorker {
 					match self.manager.read_block(&hash).await {
 						Err(Error::CorruptData(_)) => {
 							error!("Found corrupt data block during scrub: {:?}", hash);
-							self.persisted.corruptions_detected += 1;
-							self.persister.save_async(&self.persisted).await?;
+							self.persister.set_with(|p| p.corruptions_detected += 1)?;
 						}
 						Err(e) => return Err(e),
 						_ => (),
 					};
 					Ok(self
 						.tranquilizer
-						.tranquilize_worker(self.persisted.tranquility))
+						.tranquilize_worker(self.persister.get_with(|p| p.tranquility)))
 				} else {
-					self.persisted.time_last_complete_scrub = now_msec();
-					self.persister.save_async(&self.persisted).await?;
+					self.persister
+						.set_with(|p| p.time_last_complete_scrub = now_msec())?;
 					self.work = ScrubWorkerState::Finished;
 					self.tranquilizer.clear();
 					Ok(WorkerState::Idle)
@@ -331,12 +345,13 @@ impl Worker for ScrubWorker {
 		}
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		let (wait_until, command) = match &self.work {
 			ScrubWorkerState::Running(_) => return WorkerState::Busy,
 			ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
 			ScrubWorkerState::Finished => (
-				self.persisted.time_last_complete_scrub + SCRUB_INTERVAL.as_millis() as u64,
+				self.persister.get_with(|p| p.time_last_complete_scrub)
+					+ SCRUB_INTERVAL.as_millis() as u64,
 				ScrubWorkerCommand::Start,
 			),
 		};

src/block/resync.rs

@@ -3,7 +3,6 @@ use std::convert::TryInto;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use arc_swap::ArcSwap;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 
@@ -22,7 +21,7 @@ use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
-use garage_util::persister::Persister;
+use garage_util::persister::PersisterShared;
 use garage_util::time::*;
 use garage_util::tranquilizer::Tranquilizer;
 
@@ -49,13 +48,12 @@ const INITIAL_RESYNC_TRANQUILITY: u32 = 2;
 
 pub struct BlockResyncManager {
 	pub(crate) queue: CountedTree,
-	pub(crate) notify: Notify,
+	pub(crate) notify: Arc<Notify>,
 	pub(crate) errors: CountedTree,
 
 	busy_set: BusySet,
 
-	persister: Persister<ResyncPersistedConfig>,
-	persisted: ArcSwap<ResyncPersistedConfig>,
+	persister: PersisterShared<ResyncPersistedConfig>,
 }
 
 #[derive(Serialize, Deserialize, Clone, Copy)]
@@ -63,6 +61,15 @@ struct ResyncPersistedConfig {
 	n_workers: usize,
 	tranquility: u32,
 }
+impl garage_util::migrate::InitialFormat for ResyncPersistedConfig {}
+impl Default for ResyncPersistedConfig {
+	fn default() -> Self {
+		ResyncPersistedConfig {
+			n_workers: 1,
+			tranquility: INITIAL_RESYNC_TRANQUILITY,
+		}
+	}
+}
 
 enum ResyncIterResult {
 	BusyDidSomething,
@@ -90,22 +97,14 @@ impl BlockResyncManager {
 			.expect("Unable to open block_local_resync_errors tree");
 		let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors");
 
-		let persister = Persister::new(&system.metadata_dir, "resync_cfg");
-		let persisted = match persister.load() {
-			Ok(v) => v,
-			Err(_) => ResyncPersistedConfig {
-				n_workers: 1,
-				tranquility: INITIAL_RESYNC_TRANQUILITY,
-			},
-		};
+		let persister = PersisterShared::new(&system.metadata_dir, "resync_cfg");
 
 		Self {
 			queue,
-			notify: Notify::new(),
+			notify: Arc::new(Notify::new()),
 			errors,
 			busy_set: Arc::new(Mutex::new(HashSet::new())),
 			persister,
-			persisted: ArcSwap::new(Arc::new(persisted)),
 		}
 	}
 
@@ -123,6 +122,56 @@ impl BlockResyncManager {
 		Ok(self.errors.len())
 	}
 
+	/// Clear the error counter for a block and put it in queue immediately
+	pub fn clear_backoff(&self, hash: &Hash) -> Result<(), Error> {
+		let now = now_msec();
+		if let Some(ec) = self.errors.get(hash)? {
+			let mut ec = ErrorCounter::decode(&ec);
+			if ec.errors > 0 {
+				ec.last_try = now - ec.delay_msec();
+				self.errors.insert(hash, ec.encode())?;
+				self.put_to_resync_at(hash, now)?;
+				return Ok(());
+			}
+		}
+		Err(Error::Message(format!(
+			"Block {:?} was not in an errored state",
+			hash
+		)))
+	}
+
+	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
+		let notify = self.notify.clone();
+		vars.register_rw(
+			&self.persister,
+			"resync-worker-count",
+			|p| p.get_with(|x| x.n_workers),
+			move |p, n_workers| {
+				if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
+					return Err(Error::Message(format!(
+						"Invalid number of resync workers, must be between 1 and {}",
+						MAX_RESYNC_WORKERS
+					)));
+				}
+				p.set_with(|x| x.n_workers = n_workers)?;
+				notify.notify_waiters();
+				Ok(())
+			},
+		);
+
+		let notify = self.notify.clone();
+		vars.register_rw(
+			&self.persister,
+			"resync-tranquility",
+			|p| p.get_with(|x| x.tranquility),
+			move |p, tranquility| {
+				p.set_with(|x| x.tranquility = tranquility)?;
+				notify.notify_waiters();
+				Ok(())
+			},
+		);
+	}
+
 	// ---- Resync loop ----
 
 	// This part manages a queue of blocks that need to be
@@ -257,7 +306,7 @@ impl BlockResyncManager {
 
 				if let Err(e) = &res {
 					manager.metrics.resync_error_counter.add(1);
-					warn!("Error when resyncing {:?}: {}", hash, e);
+					error!("Error when resyncing {:?}: {}", hash, e);
 
 					let err_counter = match self.errors.get(hash.as_slice())? {
 						Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
@@ -417,33 +466,6 @@ impl BlockResyncManager {
 
 		Ok(())
 	}
-
-	async fn update_persisted(
-		&self,
-		update: impl Fn(&mut ResyncPersistedConfig),
-	) -> Result<(), Error> {
-		let mut cfg: ResyncPersistedConfig = *self.persisted.load().as_ref();
-		update(&mut cfg);
-		self.persister.save_async(&cfg).await?;
-		self.persisted.store(Arc::new(cfg));
-		self.notify.notify_waiters();
-		Ok(())
-	}
-
-	pub async fn set_n_workers(&self, n_workers: usize) -> Result<(), Error> {
-		if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
-			return Err(Error::Message(format!(
-				"Invalid number of resync workers, must be between 1 and {}",
-				MAX_RESYNC_WORKERS
-			)));
-		}
-		self.update_persisted(|cfg| cfg.n_workers = n_workers).await
-	}
-
-	pub async fn set_tranquility(&self, tranquility: u32) -> Result<(), Error> {
-		self.update_persisted(|cfg| cfg.tranquility = tranquility)
-			.await
-	}
 }
 
 impl Drop for BusyBlock {
@@ -458,15 +480,18 @@ pub(crate) struct ResyncWorker {
 	manager: Arc<BlockManager>,
 	tranquilizer: Tranquilizer,
 	next_delay: Duration,
+	persister: PersisterShared<ResyncPersistedConfig>,
 }
 
 impl ResyncWorker {
 	pub(crate) fn new(index: usize, manager: Arc<BlockManager>) -> Self {
+		let persister = manager.resync.persister.clone();
 		Self {
 			index,
 			manager,
 			tranquilizer: Tranquilizer::new(30),
 			next_delay: Duration::from_secs(10),
+			persister,
 		}
 	}
 }
@@ -477,39 +502,36 @@ impl Worker for ResyncWorker {
 		format!("Block resync worker #{}", self.index + 1)
 	}
 
-	fn info(&self) -> Option<String> {
-		let persisted = self.manager.resync.persisted.load();
+	fn status(&self) -> WorkerStatus {
+		let (n_workers, tranquility) = self.persister.get_with(|x| (x.n_workers, x.tranquility));
 
-		if self.index >= persisted.n_workers {
-			return Some("(unused)".into());
+		if self.index >= n_workers {
+			return WorkerStatus {
+				freeform: vec!["This worker is currently disabled".into()],
+				..Default::default()
+			};
 		}
 
-		let mut ret = vec![];
-		ret.push(format!("tranquility = {}", persisted.tranquility));
-
-		let qlen = self.manager.resync.queue_len().unwrap_or(0);
-		if qlen > 0 {
-			ret.push(format!("{} blocks in queue", qlen));
+		WorkerStatus {
+			queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
+			tranquility: Some(tranquility),
+			persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
+			..Default::default()
 		}
-
-		let elen = self.manager.resync.errors_len().unwrap_or(0);
-		if elen > 0 {
-			ret.push(format!("{} blocks in error state", elen));
-		}
-
-		Some(ret.join(", "))
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		if self.index >= self.manager.resync.persisted.load().n_workers {
+		let (n_workers, tranquility) = self.persister.get_with(|x| (x.n_workers, x.tranquility));
+
+		if self.index >= n_workers {
 			return Ok(WorkerState::Idle);
 		}
 
 		self.tranquilizer.reset();
 		match self.manager.resync.resync_iter(&self.manager).await {
-			Ok(ResyncIterResult::BusyDidSomething) => Ok(self
-				.tranquilizer
-				.tranquilize_worker(self.manager.resync.persisted.load().tranquility)),
+			Ok(ResyncIterResult::BusyDidSomething) => {
+				Ok(self.tranquilizer.tranquilize_worker(tranquility))
+			}
 			Ok(ResyncIterResult::BusyDidNothing) => Ok(WorkerState::Busy),
 			Ok(ResyncIterResult::IdleFor(delay)) => {
 				self.next_delay = delay;
@@ -527,8 +549,8 @@ impl Worker for ResyncWorker {
 		}
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
-		while self.index >= self.manager.resync.persisted.load().n_workers {
+	async fn wait_for_work(&mut self) -> WorkerState {
+		while self.index >= self.persister.get_with(|x| x.n_workers) {
 			self.manager.resync.notify.notified().await
 		}
 
@@ -545,9 +567,9 @@ impl Worker for ResyncWorker {
 /// and the time of the last try.
 /// Used to implement exponential backoff.
 #[derive(Clone, Copy, Debug)]
-struct ErrorCounter {
-	errors: u64,
-	last_try: u64,
+pub(crate) struct ErrorCounter {
+	pub(crate) errors: u64,
+	pub(crate) last_try: u64,
 }
 
 impl ErrorCounter {
@@ -558,12 +580,13 @@ impl ErrorCounter {
 		}
 	}
 
-	fn decode(data: &[u8]) -> Self {
+	pub(crate) fn decode(data: &[u8]) -> Self {
 		Self {
 			errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
 			last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
 		}
 	}
+
 	fn encode(&self) -> Vec<u8> {
 		[
 			u64::to_be_bytes(self.errors),
@@ -583,7 +606,8 @@ impl ErrorCounter {
 		(RESYNC_RETRY_DELAY.as_millis() as u64)
 			<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
 	}
-	fn next_try(&self) -> u64 {
+
+	pub(crate) fn next_try(&self) -> u64 {
 		self.last_try + self.delay_msec()
 	}
 }

src/db/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -33,6 +33,7 @@ pretty_env_logger = { version = "0.4", optional = true }
 mktemp = "0.4"
 
 [features]
+default = [ "sled" ]
 bundled-libs = [ "rusqlite/bundled" ]
 cli = ["clap", "pretty_env_logger"]
 lmdb = [ "heed" ]

src/db/lib.rs

@@ -181,6 +181,10 @@ impl Tree {
 	pub fn len(&self) -> Result<usize> {
 		self.0.len(self.1)
 	}
+	#[inline]
+	pub fn fast_len(&self) -> Result<Option<usize>> {
+		self.0.fast_len(self.1)
+	}
 
 	#[inline]
 	pub fn first(&self) -> Result<Option<(Value, Value)>> {
@@ -323,6 +327,9 @@ pub(crate) trait IDb: Send + Sync {
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
 	fn len(&self, tree: usize) -> Result<usize>;
+	fn fast_len(&self, _tree: usize) -> Result<Option<usize>> {
+		Ok(None)
+	}
 
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>>;
 	fn remove(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;

src/db/lmdb_adapter.rs

@@ -121,6 +121,10 @@ impl IDb for LmdbDb {
 		Ok(tree.len(&tx)?.try_into().unwrap())
 	}
 
+	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
+		Ok(Some(self.len(tree)?))
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		let tree = self.get_tree(tree)?;
 		let mut tx = self.db.write_txn()?;

src/db/sqlite_adapter.rs

@@ -144,6 +144,10 @@ impl IDb for SqliteDb {
 		}
 	}
 
+	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
+		Ok(Some(self.len(tree)?))
+	}
+
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		trace!("insert {}: lock db", tree);
 		let this = self.0.lock().unwrap();

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -21,14 +21,14 @@ path = "tests/lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
-garage_api = { version = "0.8.0", path = "../api" }
-garage_block = { version = "0.8.0", path = "../block" }
-garage_model = { version = "0.8.0", path = "../model" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_table = { version = "0.8.0", path = "../table" }
-garage_util = { version = "0.8.0", path = "../util" }
-garage_web = { version = "0.8.0", path = "../web" }
+garage_db = { version = "0.8.1", path = "../db" }
+garage_api = { version = "0.8.1", path = "../api" }
+garage_block = { version = "0.8.1", path = "../block" }
+garage_model = { version = "0.8.1", path = "../model" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_table = { version = "0.8.1", path = "../table" }
+garage_util = { version = "0.8.1", path = "../util" }
+garage_web = { version = "0.8.1", path = "../web" }
 
 backtrace = "0.3"
 bytes = "1.0"
@@ -36,13 +36,12 @@ bytesize = "1.1"
 timeago = "0.3"
 parse_duration = "2.1"
 hex = "0.4"
-tracing = { version = "0.1.30", features = ["log-always"] }
+tracing = { version = "0.1.30" }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 rand = "0.8"
 async-trait = "0.1.7"
 sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
 
-rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 structopt = { version = "0.3", default-features = false }
@@ -74,7 +73,7 @@ base64 = "0.13"
 
 
 [features]
-default = [ "bundled-libs", "metrics", "sled" ]
+default = [ "bundled-libs", "metrics", "sled", "k2v" ]
 
 k2v = [ "garage_util/k2v", "garage_api/k2v" ]
 

src/garage/admin.rs

@@ -5,9 +5,11 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 
+use garage_util::background::BackgroundRunner;
 use garage_util::crdt::*;
 use garage_util::data::*;
 use garage_util::error::Error as GarageError;
+use garage_util::formater::format_table_to_string;
 use garage_util::time::*;
 
 use garage_table::replication::*;
@@ -15,7 +17,7 @@ use garage_table::*;
 
 use garage_rpc::*;
 
-use garage_block::repair::ScrubWorkerCommand;
+use garage_block::manager::BlockResyncErrorInfo;
 
 use garage_model::bucket_alias_table::*;
 use garage_model::bucket_table::*;
@@ -24,6 +26,8 @@ use garage_model::helper::error::{Error, OkOrBadRequest};
 use garage_model::key_table::*;
 use garage_model::migrate::Migrate;
 use garage_model::permission::*;
+use garage_model::s3::object_table::*;
+use garage_model::s3::version_table::Version;
 
 use crate::cli::*;
 use crate::repair::online::launch_online_repair;
@@ -38,7 +42,8 @@ pub enum AdminRpc {
 	LaunchRepair(RepairOpt),
 	Migrate(MigrateOpt),
 	Stats(StatsOpt),
-	Worker(WorkerOpt),
+	Worker(WorkerOperation),
+	BlockOperation(BlockOperation),
 
 	// Replies
 	Ok(String),
@@ -54,6 +59,14 @@ pub enum AdminRpc {
 		HashMap<usize, garage_util::background::WorkerInfo>,
 		WorkerListOpt,
 	),
+	WorkerVars(Vec<(Uuid, String, String)>),
+	WorkerInfo(usize, garage_util::background::WorkerInfo),
+	BlockErrorList(Vec<BlockResyncErrorInfo>),
+	BlockInfo {
+		hash: Hash,
+		refcount: u64,
+		versions: Vec<Result<Version, Uuid>>,
+	},
 }
 
 impl Rpc for AdminRpc {
@@ -62,17 +75,24 @@ impl Rpc for AdminRpc {
 
 pub struct AdminRpcHandler {
 	garage: Arc<Garage>,
+	background: Arc<BackgroundRunner>,
 	endpoint: Arc<Endpoint<AdminRpc, Self>>,
 }
 
 impl AdminRpcHandler {
-	pub fn new(garage: Arc<Garage>) -> Arc<Self> {
+	pub fn new(garage: Arc<Garage>, background: Arc<BackgroundRunner>) -> Arc<Self> {
 		let endpoint = garage.system.netapp.endpoint(ADMIN_RPC_PATH.into());
-		let admin = Arc::new(Self { garage, endpoint });
+		let admin = Arc::new(Self {
+			garage,
+			background,
+			endpoint,
+		});
 		admin.endpoint.set_handler(admin.clone());
 		admin
 	}
 
+	// ================ BUCKET COMMANDS ====================
+
 	async fn handle_bucket_cmd(&self, cmd: &BucketOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			BucketOperation::List => self.handle_list_buckets().await,
@@ -551,6 +571,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok(ret))
 	}
 
+	// ================ KEY COMMANDS ====================
+
 	async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			KeyOperation::List => self.handle_list_keys().await,
@@ -688,6 +710,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::KeyInfo(key, relevant_buckets))
 	}
 
+	// ================ MIGRATION COMMANDS ====================
+
 	async fn handle_migrate(self: &Arc<Self>, opt: MigrateOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -704,6 +728,8 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok("Migration successfull.".into()))
 	}
 
+	// ================ REPAIR COMMANDS ====================
+
 	async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -739,7 +765,7 @@ impl AdminRpcHandler {
 				)))
 			}
 		} else {
-			launch_online_repair(self.garage.clone(), opt).await;
+			launch_online_repair(&self.garage, &self.background, opt).await?;
 			Ok(AdminRpc::Ok(format!(
 				"Repair launched on {:?}",
 				self.garage.system.id
@@ -747,6 +773,8 @@ impl AdminRpcHandler {
 		}
 	}
 
+	// ================ STATS COMMANDS ====================
+
 	async fn handle_stats(&self, opt: StatsOpt) -> Result<AdminRpc, Error> {
 		if opt.all_nodes {
 			let mut ret = String::new();
@@ -763,11 +791,12 @@ impl AdminRpcHandler {
 				match self
 					.endpoint
 					.call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL)
-					.await?
+					.await
 				{
-					Ok(AdminRpc::Ok(s)) => writeln!(&mut ret, "{}", s).unwrap(),
-					Ok(x) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
-					Err(e) => writeln!(&mut ret, "Error: {}", e).unwrap(),
+					Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(),
+					Ok(Ok(x)) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
+					Ok(Err(e)) => writeln!(&mut ret, "Remote error: {}", e).unwrap(),
+					Err(e) => writeln!(&mut ret, "Network error: {}", e).unwrap(),
 				}
 			}
 			Ok(AdminRpc::Ok(ret))
@@ -787,6 +816,7 @@ impl AdminRpcHandler {
 				.unwrap_or_else(|| "(unknown)".into()),
 		)
 		.unwrap();
+
 		writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();
 
 		// Gather ring statistics
@@ -805,21 +835,38 @@ impl AdminRpcHandler {
 			writeln!(&mut ret, "  {:?} {}", n, c).unwrap();
 		}
 
-		self.gather_table_stats(&mut ret, &self.garage.bucket_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.key_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.object_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.version_table, &opt)?;
-		self.gather_table_stats(&mut ret, &self.garage.block_ref_table, &opt)?;
+		// Gather table statistics
+		let mut table = vec!["  Table\tItems\tMklItems\tMklTodo\tGcTodo".into()];
+		table.push(self.gather_table_stats(&self.garage.bucket_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.key_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.object_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.version_table, opt.detailed)?);
+		table.push(self.gather_table_stats(&self.garage.block_ref_table, opt.detailed)?);
+		write!(
+			&mut ret,
+			"\nTable stats:\n{}",
+			format_table_to_string(table)
+		)
+		.unwrap();
 
+		// Gather block manager statistics
 		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
-		if opt.detailed {
+		let rc_len = if opt.detailed {
+			self.garage.block_manager.rc_len()?.to_string()
+		} else {
+			self.garage
+				.block_manager
+				.rc_fast_len()?
+				.map(|x| x.to_string())
+				.unwrap_or_else(|| "NC".into())
+		};
+
 		writeln!(
 			&mut ret,
 			"  number of RC entries (~= number of blocks): {}",
-				self.garage.block_manager.rc_len()?
+			rc_len
 		)
 		.unwrap();
-		}
 		writeln!(
 			&mut ret,
 			"  resync queue length: {}",
@@ -833,79 +880,305 @@ impl AdminRpcHandler {
 		)
 		.unwrap();
 
+		if !opt.detailed {
+			writeln!(&mut ret, "\nIf values are missing (marked as NC), consider adding the --detailed flag - this will be slow.").unwrap();
+		}
+
 		Ok(ret)
 	}
 
 	fn gather_table_stats<F, R>(
 		&self,
-		to: &mut String,
 		t: &Arc<Table<F, R>>,
-		opt: &StatsOpt,
-	) -> Result<(), Error>
+		detailed: bool,
+	) -> Result<String, Error>
 	where
 		F: TableSchema + 'static,
 		R: TableReplication + 'static,
 	{
-		writeln!(to, "\nTable stats for {}", F::TABLE_NAME).unwrap();
-		if opt.detailed {
-			writeln!(
-				to,
-				"  number of items: {}",
-				t.data.store.len().map_err(GarageError::from)?
+		let (data_len, mkl_len) = if detailed {
+			(
+				t.data.store.len().map_err(GarageError::from)?.to_string(),
+				t.merkle_updater.merkle_tree_len()?.to_string(),
 			)
-			.unwrap();
-			writeln!(
-				to,
-				"  Merkle tree size: {}",
-				t.merkle_updater.merkle_tree_len()?
+		} else {
+			(
+				t.data
+					.store
+					.fast_len()
+					.map_err(GarageError::from)?
+					.map(|x| x.to_string())
+					.unwrap_or_else(|| "NC".into()),
+				t.merkle_updater
+					.merkle_tree_fast_len()?
+					.map(|x| x.to_string())
+					.unwrap_or_else(|| "NC".into()),
 			)
-			.unwrap();
-		}
-		writeln!(
-			to,
-			"  Merkle updater todo queue length: {}",
-			t.merkle_updater.todo_len()?
-		)
-		.unwrap();
-		writeln!(to, "  GC todo queue length: {}", t.data.gc_todo_len()?).unwrap();
+		};
 
-		Ok(())
+		Ok(format!(
+			"  {}\t{}\t{}\t{}\t{}",
+			F::TABLE_NAME,
+			data_len,
+			mkl_len,
+			t.merkle_updater.todo_len()?,
+			t.data.gc_todo_len()?
+		))
 	}
 
-	// ----
+	// ================ WORKER COMMANDS ====================
 
-	async fn handle_worker_cmd(&self, opt: WorkerOpt) -> Result<AdminRpc, Error> {
-		match opt.cmd {
-			WorkerCmd::List { opt } => {
-				let workers = self.garage.background.get_worker_info();
-				Ok(AdminRpc::WorkerList(workers, opt))
+	async fn handle_worker_cmd(&self, cmd: &WorkerOperation) -> Result<AdminRpc, Error> {
+		match cmd {
+			WorkerOperation::List { opt } => {
+				let workers = self.background.get_worker_info();
+				Ok(AdminRpc::WorkerList(workers, *opt))
 			}
-			WorkerCmd::Set { opt } => match opt {
-				WorkerSetCmd::ScrubTranquility { tranquility } => {
-					let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);
-					self.garage
-						.block_manager
-						.send_scrub_command(scrub_command)
-						.await;
-					Ok(AdminRpc::Ok("Scrub tranquility updated".into()))
+			WorkerOperation::Info { tid } => {
+				let info = self
+					.background
+					.get_worker_info()
+					.get(tid)
+					.ok_or_bad_request(format!("No worker with TID {}", tid))?
+					.clone();
+				Ok(AdminRpc::WorkerInfo(*tid, info))
 			}
-				WorkerSetCmd::ResyncNWorkers { n_workers } => {
-					self.garage
-						.block_manager
-						.resync
-						.set_n_workers(n_workers)
+			WorkerOperation::Get {
+				all_nodes,
+				variable,
+			} => self.handle_get_var(*all_nodes, variable).await,
+			WorkerOperation::Set {
+				all_nodes,
+				variable,
+				value,
+			} => self.handle_set_var(*all_nodes, variable, value).await,
+		}
+	}
+
+	async fn handle_get_var(
+		&self,
+		all_nodes: bool,
+		variable: &Option<String>,
+	) -> Result<AdminRpc, Error> {
+		if all_nodes {
+			let mut ret = vec![];
+			let ring = self.garage.system.ring.borrow().clone();
+			for node in ring.layout.node_ids().iter() {
+				let node = (*node).into();
+				match self
+					.endpoint
+					.call(
+						&node,
+						AdminRpc::Worker(WorkerOperation::Get {
+							all_nodes: false,
+							variable: variable.clone(),
+						}),
+						PRIO_NORMAL,
+					)
+					.await??
+				{
+					AdminRpc::WorkerVars(v) => ret.extend(v),
+					m => return Err(GarageError::unexpected_rpc_message(m).into()),
+				}
+			}
+			Ok(AdminRpc::WorkerVars(ret))
+		} else {
+			#[allow(clippy::collapsible_else_if)]
+			if let Some(v) = variable {
+				Ok(AdminRpc::WorkerVars(vec![(
+					self.garage.system.id,
+					v.clone(),
+					self.garage.bg_vars.get(v)?,
+				)]))
+			} else {
+				let mut vars = self.garage.bg_vars.get_all();
+				vars.sort();
+				Ok(AdminRpc::WorkerVars(
+					vars.into_iter()
+						.map(|(k, v)| (self.garage.system.id, k.to_string(), v))
+						.collect(),
+				))
+			}
+		}
+	}
+
+	async fn handle_set_var(
+		&self,
+		all_nodes: bool,
+		variable: &str,
+		value: &str,
+	) -> Result<AdminRpc, Error> {
+		if all_nodes {
+			let mut ret = vec![];
+			let ring = self.garage.system.ring.borrow().clone();
+			for node in ring.layout.node_ids().iter() {
+				let node = (*node).into();
+				match self
+					.endpoint
+					.call(
+						&node,
+						AdminRpc::Worker(WorkerOperation::Set {
+							all_nodes: false,
+							variable: variable.to_string(),
+							value: value.to_string(),
+						}),
+						PRIO_NORMAL,
+					)
+					.await??
+				{
+					AdminRpc::WorkerVars(v) => ret.extend(v),
+					m => return Err(GarageError::unexpected_rpc_message(m).into()),
+				}
+			}
+			Ok(AdminRpc::WorkerVars(ret))
+		} else {
+			self.garage.bg_vars.set(variable, value)?;
+			Ok(AdminRpc::WorkerVars(vec![(
+				self.garage.system.id,
+				variable.to_string(),
+				value.to_string(),
+			)]))
+		}
+	}
+
+	// ================ BLOCK COMMANDS ====================
+
+	async fn handle_block_cmd(&self, cmd: &BlockOperation) -> Result<AdminRpc, Error> {
+		match cmd {
+			BlockOperation::ListErrors => Ok(AdminRpc::BlockErrorList(
+				self.garage.block_manager.list_resync_errors()?,
+			)),
+			BlockOperation::Info { hash } => {
+				let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+				let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+				let refcount = self.garage.block_manager.get_block_rc(&hash)?;
+				let block_refs = self
+					.garage
+					.block_ref_table
+					.get_range(&hash, None, None, 10000, Default::default())
 					.await?;
-					Ok(AdminRpc::Ok("Number of resync workers updated".into()))
+				let mut versions = vec![];
+				for br in block_refs {
+					if let Some(v) = self
+						.garage
+						.version_table
+						.get(&br.version, &EmptyKey)
+						.await?
+					{
+						versions.push(Ok(v));
+					} else {
+						versions.push(Err(br.version));
 					}
-				WorkerSetCmd::ResyncTranquility { tranquility } => {
-					self.garage
-						.block_manager
-						.resync
-						.set_tranquility(tranquility)
+				}
+				Ok(AdminRpc::BlockInfo {
+					hash,
+					refcount,
+					versions,
+				})
+			}
+			BlockOperation::RetryNow { all, blocks } => {
+				if *all {
+					if !blocks.is_empty() {
+						return Err(Error::BadRequest(
+							"--all was specified, cannot also specify blocks".into(),
+						));
+					}
+					let blocks = self.garage.block_manager.list_resync_errors()?;
+					for b in blocks.iter() {
+						self.garage.block_manager.resync.clear_backoff(&b.hash)?;
+					}
+					Ok(AdminRpc::Ok(format!(
+						"{} blocks returned in queue for a retry now (check logs to see results)",
+						blocks.len()
+					)))
+				} else {
+					for hash in blocks {
+						let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+						let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+						self.garage.block_manager.resync.clear_backoff(&hash)?;
+					}
+					Ok(AdminRpc::Ok(format!(
+						"{} blocks returned in queue for a retry now (check logs to see results)",
+						blocks.len()
+					)))
+				}
+			}
+			BlockOperation::Purge { yes, blocks } => {
+				if !yes {
+					return Err(Error::BadRequest(
+						"Pass the --yes flag to confirm block purge operation.".into(),
+					));
+				}
+
+				let mut obj_dels = 0;
+				let mut ver_dels = 0;
+
+				for hash in blocks {
+					let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
+					let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
+					let block_refs = self
+						.garage
+						.block_ref_table
+						.get_range(&hash, None, None, 10000, Default::default())
 						.await?;
-					Ok(AdminRpc::Ok("Resync tranquility updated".into()))
+
+					for br in block_refs {
+						let version = match self
+							.garage
+							.version_table
+							.get(&br.version, &EmptyKey)
+							.await?
+						{
+							Some(v) => v,
+							None => continue,
+						};
+
+						if let Some(object) = self
+							.garage
+							.object_table
+							.get(&version.bucket_id, &version.key)
+							.await?
+						{
+							let ov = object.versions().iter().rev().find(|v| v.is_complete());
+							if let Some(ov) = ov {
+								if ov.uuid == br.version {
+									let del_uuid = gen_uuid();
+									let deleted_object = Object::new(
+										version.bucket_id,
+										version.key.clone(),
+										vec![ObjectVersion {
+											uuid: del_uuid,
+											timestamp: ov.timestamp + 1,
+											state: ObjectVersionState::Complete(
+												ObjectVersionData::DeleteMarker,
+											),
+										}],
+									);
+									self.garage.object_table.insert(&deleted_object).await?;
+									obj_dels += 1;
+								}
+							}
+						}
+
+						if !version.deleted.get() {
+							let deleted_version = Version::new(
+								version.uuid,
+								version.bucket_id,
+								version.key.clone(),
+								true,
+							);
+							self.garage.version_table.insert(&deleted_version).await?;
+							ver_dels += 1;
+						}
+					}
+				}
+				Ok(AdminRpc::Ok(format!(
+					"{} blocks were purged: {} object deletion markers added, {} versions marked deleted",
+					blocks.len(),
+					obj_dels,
+					ver_dels
+				)))
 			}
-			},
 		}
 	}
 }
@@ -923,7 +1196,8 @@ impl EndpointHandler<AdminRpc> for AdminRpcHandler {
 			AdminRpc::Migrate(opt) => self.handle_migrate(opt.clone()).await,
 			AdminRpc::LaunchRepair(opt) => self.handle_launch_repair(opt.clone()).await,
 			AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await,
-			AdminRpc::Worker(opt) => self.handle_worker_cmd(opt.clone()).await,
+			AdminRpc::Worker(wo) => self.handle_worker_cmd(wo).await,
+			AdminRpc::BlockOperation(bo) => self.handle_block_cmd(bo).await,
 			m => Err(GarageError::unexpected_rpc_message(m).into()),
 		}
 	}

src/garage/cli/cmd.rs

@@ -41,6 +41,9 @@ pub async fn cli_command_dispatch(
 		}
 		Command::Stats(so) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Stats(so)).await,
 		Command::Worker(wo) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Worker(wo)).await,
+		Command::Block(bo) => {
+			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await
+		}
 		_ => unreachable!(),
 	}
 }
@@ -186,7 +189,23 @@ pub async fn cmd_admin(
 			print_key_info(&key, &rb);
 		}
 		AdminRpc::WorkerList(wi, wlo) => {
-			print_worker_info(wi, wlo);
+			print_worker_list(wi, wlo);
+		}
+		AdminRpc::WorkerVars(wv) => {
+			print_worker_vars(wv);
+		}
+		AdminRpc::WorkerInfo(tid, wi) => {
+			print_worker_info(tid, wi);
+		}
+		AdminRpc::BlockErrorList(el) => {
+			print_block_error_list(el);
+		}
+		AdminRpc::BlockInfo {
+			hash,
+			refcount,
+			versions,
+		} => {
+			print_block_info(hash, refcount, versions);
 		}
 		r => {
 			error!("Unexpected response: {:?}", r);

src/garage/cli/structs.rs

@@ -49,7 +49,11 @@ pub enum Command {
 
 	/// Manage background workers
 	#[structopt(name = "worker", version = garage_version())]
-	Worker(WorkerOpt),
+	Worker(WorkerOperation),
+
+	/// Low-level debug operations on data blocks
+	#[structopt(name = "block", version = garage_version())]
+	Block(BlockOperation),
 }
 
 #[derive(StructOpt, Debug)]
@@ -502,25 +506,36 @@ pub struct StatsOpt {
 	pub detailed: bool,
 }
 
-#[derive(Serialize, Deserialize, StructOpt, Debug, Clone)]
-pub struct WorkerOpt {
-	#[structopt(subcommand)]
-	pub cmd: WorkerCmd,
-}
-
 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
-pub enum WorkerCmd {
+pub enum WorkerOperation {
 	/// List all workers on Garage node
 	#[structopt(name = "list", version = garage_version())]
 	List {
 		#[structopt(flatten)]
 		opt: WorkerListOpt,
 	},
+	/// Get detailed information about a worker
+	#[structopt(name = "info", version = garage_version())]
+	Info { tid: usize },
+	/// Get worker parameter
+	#[structopt(name = "get", version = garage_version())]
+	Get {
+		/// Gather variable values from all nodes
+		#[structopt(short = "a", long = "all-nodes")]
+		all_nodes: bool,
+		/// Variable name to get, or none to get all variables
+		variable: Option<String>,
+	},
 	/// Set worker parameter
 	#[structopt(name = "set", version = garage_version())]
 	Set {
-		#[structopt(subcommand)]
-		opt: WorkerSetCmd,
+		/// Set variable values on all nodes
+		#[structopt(short = "a", long = "all-nodes")]
+		all_nodes: bool,
+		/// Variable node to set
+		variable: String,
+		/// Value to set the variable to
+		value: String,
 	},
 }
 
@@ -535,14 +550,33 @@ pub struct WorkerListOpt {
 }
 
 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
-pub enum WorkerSetCmd {
-	/// Set tranquility of scrub operations
-	#[structopt(name = "scrub-tranquility", version = garage_version())]
-	ScrubTranquility { tranquility: u32 },
-	/// Set number of concurrent block resync workers
-	#[structopt(name = "resync-n-workers", version = garage_version())]
-	ResyncNWorkers { n_workers: usize },
-	/// Set tranquility of block resync operations
-	#[structopt(name = "resync-tranquility", version = garage_version())]
-	ResyncTranquility { tranquility: u32 },
+pub enum BlockOperation {
+	/// List all blocks that currently have a resync error
+	#[structopt(name = "list-errors", version = garage_version())]
+	ListErrors,
+	/// Get detailed information about a single block
+	#[structopt(name = "info", version = garage_version())]
+	Info {
+		/// Hash of the block for which to retrieve information
+		hash: String,
+	},
+	/// Retry now the resync of one or many blocks
+	#[structopt(name = "retry-now", version = garage_version())]
+	RetryNow {
+		/// Retry all blocks that have a resync error
+		#[structopt(long = "all")]
+		all: bool,
+		/// Hashes of the block to retry to resync now
+		blocks: Vec<String>,
+	},
+	/// Delete all objects referencing a missing block
+	#[structopt(name = "purge", version = garage_version())]
+	Purge {
+		/// Mandatory to confirm this operation
+		#[structopt(long = "yes")]
+		yes: bool,
+		/// Hashes of the block to purge
+		#[structopt(required = true)]
+		blocks: Vec<String>,
+	},
 }

src/garage/cli/util.rs

@@ -3,14 +3,17 @@ use std::time::Duration;
 
 use garage_util::background::*;
 use garage_util::crdt::*;
-use garage_util::data::Uuid;
+use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::formater::format_table;
 use garage_util::time::*;
 
+use garage_block::manager::BlockResyncErrorInfo;
+
 use garage_model::bucket_table::*;
 use garage_model::key_table::*;
 use garage_model::s3::object_table::{BYTES, OBJECTS, UNFINISHED_UPLOADS};
+use garage_model::s3::version_table::Version;
 
 use crate::cli::structs::WorkerListOpt;
 
@@ -241,7 +244,7 @@ pub fn find_matching_node(
 	}
 }
 
-pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
+pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 	let mut wi = wi.into_iter().collect::<Vec<_>>();
 	wi.sort_by_key(|(tid, info)| {
 		(
@@ -254,7 +257,7 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 		)
 	});
 
-	let mut table = vec![];
+	let mut table = vec!["TID\tState\tName\tTranq\tDone\tQueue\tErrors\tConsec\tLast".to_string()];
 	for (tid, info) in wi.iter() {
 		if wlo.busy && !matches!(info.state, WorkerState::Busy | WorkerState::Throttled(_)) {
 			continue;
@@ -263,33 +266,155 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 			continue;
 		}
 
-		table.push(format!("{}\t{}\t{}", tid, info.state, info.name));
-		if let Some(i) = &info.info {
-			table.push(format!("\t\t  {}", i));
-		}
 		let tf = timeago::Formatter::new();
-		let (err_ago, err_msg) = info
+		let err_ago = info
 			.last_error
 			.as_ref()
-			.map(|(m, t)| {
-				(
-					tf.convert(Duration::from_millis(now_msec() - t)),
-					m.as_str(),
-				)
-			})
-			.unwrap_or(("(?) ago".into(), "(?)"));
-		if info.consecutive_errors > 0 {
+			.map(|(_, t)| tf.convert(Duration::from_millis(now_msec() - t)))
+			.unwrap_or_default();
+		let (total_err, consec_err) = if info.errors > 0 {
+			(info.errors.to_string(), info.consecutive_errors.to_string())
+		} else {
+			("-".into(), "-".into())
+		};
+
 		table.push(format!(
-				"\t\t  {} consecutive errors ({} total), last {}",
-				info.consecutive_errors, info.errors, err_ago,
+			"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
+			tid,
+			info.state,
+			info.name,
+			info.status
+				.tranquility
+				.as_ref()
+				.map(ToString::to_string)
+				.unwrap_or_else(|| "-".into()),
+			info.status.progress.as_deref().unwrap_or("-"),
+			info.status
+				.queue_length
+				.as_ref()
+				.map(ToString::to_string)
+				.unwrap_or_else(|| "-".into()),
+			total_err,
+			consec_err,
+			err_ago,
 		));
-			table.push(format!("\t\t  {}", err_msg));
-		} else if info.errors > 0 {
-			table.push(format!("\t\t  ({} errors, last {})", info.errors, err_ago,));
-			if wlo.errors {
-				table.push(format!("\t\t  {}", err_msg));
 	}
+	format_table(table);
+}
+
+pub fn print_worker_info(tid: usize, info: WorkerInfo) {
+	let mut table = vec![];
+	table.push(format!("Task id:\t{}", tid));
+	table.push(format!("Worker name:\t{}", info.name));
+	match info.state {
+		WorkerState::Throttled(t) => {
+			table.push(format!(
+				"Worker state:\tBusy (throttled, paused for {:.3}s)",
+				t
+			));
+		}
+		s => {
+			table.push(format!("Worker state:\t{}", s));
+		}
+	};
+	if let Some(tql) = info.status.tranquility {
+		table.push(format!("Tranquility:\t{}", tql));
+	}
+
+	table.push("".into());
+	table.push(format!("Total errors:\t{}", info.errors));
+	table.push(format!("Consecutive errs:\t{}", info.consecutive_errors));
+	if let Some((s, t)) = info.last_error {
+		table.push(format!("Last error:\t{}", s));
+		let tf = timeago::Formatter::new();
+		table.push(format!(
+			"Last error time:\t{}",
+			tf.convert(Duration::from_millis(now_msec() - t))
+		));
+	}
+
+	table.push("".into());
+	if let Some(p) = info.status.progress {
+		table.push(format!("Progress:\t{}", p));
+	}
+	if let Some(ql) = info.status.queue_length {
+		table.push(format!("Queue length:\t{}", ql));
+	}
+	if let Some(pe) = info.status.persistent_errors {
+		table.push(format!("Persistent errors:\t{}", pe));
+	}
+
+	for (i, s) in info.status.freeform.iter().enumerate() {
+		if i == 0 {
+			if table.last() != Some(&"".into()) {
+				table.push("".into());
+			}
+			table.push(format!("Message:\t{}", s));
+		} else {
+			table.push(format!("\t{}", s));
 		}
 	}
 	format_table(table);
 }
+
+pub fn print_worker_vars(wv: Vec<(Uuid, String, String)>) {
+	let table = wv
+		.into_iter()
+		.map(|(n, k, v)| format!("{:?}\t{}\t{}", n, k, v))
+		.collect::<Vec<_>>();
+	format_table(table);
+}
+
+pub fn print_block_error_list(el: Vec<BlockResyncErrorInfo>) {
+	let now = now_msec();
+	let tf = timeago::Formatter::new();
+	let mut tf2 = timeago::Formatter::new();
+	tf2.ago("");
+
+	let mut table = vec!["Hash\tRC\tErrors\tLast error\tNext try".into()];
+	for e in el {
+		table.push(format!(
+			"{}\t{}\t{}\t{}\tin {}",
+			hex::encode(e.hash.as_slice()),
+			e.refcount,
+			e.error_count,
+			tf.convert(Duration::from_millis(now - e.last_try)),
+			tf2.convert(Duration::from_millis(e.next_try - now))
+		));
+	}
+	format_table(table);
+}
+
+pub fn print_block_info(hash: Hash, refcount: u64, versions: Vec<Result<Version, Uuid>>) {
+	println!("Block hash: {}", hex::encode(hash.as_slice()));
+	println!("Refcount: {}", refcount);
+	println!();
+
+	let mut table = vec!["Version\tBucket\tKey\tDeleted".into()];
+	let mut nondeleted_count = 0;
+	for v in versions.iter() {
+		match v {
+			Ok(ver) => {
+				table.push(format!(
+					"{:?}\t{:?}\t{}\t{:?}",
+					ver.uuid,
+					ver.bucket_id,
+					ver.key,
+					ver.deleted.get()
+				));
+				if !ver.deleted.get() {
+					nondeleted_count += 1;
+				}
+			}
+			Err(vh) => {
+				table.push(format!("{:?}\t\t\tyes", vh));
+			}
+		}
+	}
+	format_table(table);
+
+	if refcount != nondeleted_count {
+		println!();
+		println!("Warning: refcount does not match number of non-deleted versions");
+	}
+}

src/garage/main.rs

@@ -127,9 +127,16 @@ async fn main() {
 		std::process::abort();
 	}));
 
+	// Parse arguments and dispatch command line
+	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
+
 	// Initialize logging as well as other libraries used in Garage
 	if std::env::var("RUST_LOG").is_err() {
-		std::env::set_var("RUST_LOG", "netapp=info,garage=info")
+		let default_log = match &opt.cmd {
+			Command::Server => "netapp=info,garage=info",
+			_ => "netapp=warn,garage=warn",
+		};
+		std::env::set_var("RUST_LOG", default_log)
 	}
 	tracing_subscriber::fmt()
 		.with_writer(std::io::stderr)
@@ -137,9 +144,6 @@ async fn main() {
 		.init();
 	sodiumoxide::init().expect("Unable to init sodiumoxide");
 
-	// Parse arguments and dispatch command line
-	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
-
 	let res = match opt.cmd {
 		Command::Server => server::run_server(opt.config_file).await,
 		Command::OfflineRepair(repair_opt) => {
@@ -169,7 +173,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let net_key_hex_str = opt
 		.rpc_secret
 		.as_ref()
-		.or_else(|| config.as_ref().map(|c| &c.rpc_secret))
+		.or_else(|| config.as_ref().and_then(|c| c.rpc_secret.as_ref()))
 		.ok_or("No RPC secret provided")?;
 	let network_key = NetworkKey::from_slice(
 		&hex::decode(net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..],
@@ -182,9 +186,9 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk);
 
 	// Find and parse the address of the target host
-	let (id, addr) = if let Some(h) = opt.rpc_host {
+	let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {
 		let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?;
-		(id, addrs[0])
+		(id, addrs[0], false)
 	} else {
 		let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
 			.err_context(READ_KEY_ERROR)?;
@@ -195,24 +199,26 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?
 				.next()
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?;
-			(node_id, a)
+			(node_id, a, false)
 		} else {
 			let default_addr = SocketAddr::new(
 				"127.0.0.1".parse().unwrap(),
 				config.as_ref().unwrap().rpc_bind_addr.port(),
 			);
-			warn!(
-				"Trying to contact Garage node at default address {}",
-				default_addr
-			);
-			warn!("If this doesn't work, consider adding rpc_public_addr in your config file or specifying the -h command line parameter.");
-			(node_id, default_addr)
+			(node_id, default_addr, true)
 		}
 	};
 
 	// Connect to target host
-	netapp.clone().try_connect(addr, id).await
-		.err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+	if let Err(e) = netapp.clone().try_connect(addr, id).await {
+		if is_default_addr {
+			warn!(
+				"Tried to contact Garage node at default address {}, which didn't work. If that address is wrong, consider setting rpc_public_addr in your config file.",
+				addr
+			);
+		}
+		Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
+	}
 
 	let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
 	let admin_rpc_endpoint = netapp.endpoint::<AdminRpc, ()>(ADMIN_RPC_PATH.into());

src/garage/repair/offline.rs

@@ -1,8 +1,5 @@
 use std::path::PathBuf;
 
-use tokio::sync::watch;
-
-use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;
 
@@ -20,12 +17,8 @@ pub async fn offline_repair(config_file: PathBuf, opt: OfflineRepairOpt) -> Resu
 	info!("Loading configuration...");
 	let config = read_config(config_file)?;
 
-	info!("Initializing background runner...");
-	let (done_tx, done_rx) = watch::channel(false);
-	let (background, await_background_done) = BackgroundRunner::new(16, done_rx);
-
 	info!("Initializing Garage main data store...");
-	let garage = Garage::new(config.clone(), background)?;
+	let garage = Garage::new(config)?;
 
 	info!("Launching repair operation...");
 	match opt.what {
@@ -43,13 +36,7 @@ pub async fn offline_repair(config_file: PathBuf, opt: OfflineRepairOpt) -> Resu
 		}
 	}
 
-	info!("Repair operation finished, shutting down Garage internals...");
-	done_tx.send(true).unwrap();
-	drop(garage);
-
-	await_background_done.await?;
-
-	info!("Cleaning up...");
+	info!("Repair operation finished, shutting down...");
 
 	Ok(())
 }

src/garage/repair/online.rs

@@ -12,36 +12,35 @@ use garage_model::s3::version_table::*;
 use garage_table::*;
 use garage_util::background::*;
 use garage_util::error::Error;
+use garage_util::migrate::Migrate;
 
 use crate::*;
 
-pub async fn launch_online_repair(garage: Arc<Garage>, opt: RepairOpt) {
+pub async fn launch_online_repair(
+	garage: &Arc<Garage>,
+	bg: &BackgroundRunner,
+	opt: RepairOpt,
+) -> Result<(), Error> {
 	match opt.what {
 		RepairWhat::Tables => {
 			info!("Launching a full sync of tables");
-			garage.bucket_table.syncer.add_full_sync();
-			garage.object_table.syncer.add_full_sync();
-			garage.version_table.syncer.add_full_sync();
-			garage.block_ref_table.syncer.add_full_sync();
-			garage.key_table.syncer.add_full_sync();
+			garage.bucket_table.syncer.add_full_sync()?;
+			garage.object_table.syncer.add_full_sync()?;
+			garage.version_table.syncer.add_full_sync()?;
+			garage.block_ref_table.syncer.add_full_sync()?;
+			garage.key_table.syncer.add_full_sync()?;
 		}
 		RepairWhat::Versions => {
 			info!("Repairing the versions table");
-			garage
-				.background
-				.spawn_worker(RepairVersionsWorker::new(garage.clone()));
+			bg.spawn_worker(RepairVersionsWorker::new(garage.clone()));
 		}
 		RepairWhat::BlockRefs => {
 			info!("Repairing the block refs table");
-			garage
-				.background
-				.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
+			bg.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
 		}
 		RepairWhat::Blocks => {
 			info!("Repairing the stored blocks");
-			garage
-				.background
-				.spawn_worker(garage_block::repair::RepairWorker::new(
+			bg.spawn_worker(garage_block::repair::RepairWorker::new(
 				garage.block_manager.clone(),
 			));
 		}
@@ -52,13 +51,18 @@ pub async fn launch_online_repair(garage: Arc<Garage>, opt: RepairOpt) {
 				ScrubCmd::Resume => ScrubWorkerCommand::Resume,
 				ScrubCmd::Cancel => ScrubWorkerCommand::Cancel,
 				ScrubCmd::SetTranquility { tranquility } => {
-					ScrubWorkerCommand::SetTranquility(tranquility)
+					garage
+						.block_manager
+						.scrub_persister
+						.set_with(|x| x.tranquility = tranquility)?;
+					return Ok(());
 				}
 			};
 			info!("Sending command to scrub worker: {:?}", cmd);
-			garage.block_manager.send_scrub_command(cmd).await;
+			garage.block_manager.send_scrub_command(cmd).await?;
 		}
 	}
+	Ok(())
 }
 
 // ----
@@ -85,25 +89,23 @@ impl Worker for RepairVersionsWorker {
 		"Version repair worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
-		Some(format!("{} items done", self.counter))
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			progress: Some(self.counter.to_string()),
+			..Default::default()
+		}
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let item_bytes = match self.garage.version_table.data.store.get_gt(&self.pos)? {
-			Some((k, v)) => {
-				self.pos = k;
-				v
-			}
+		let (item_bytes, next_pos) = match self.garage.version_table.data.store.get_gt(&self.pos)? {
+			Some((k, v)) => (v, k),
 			None => {
 				info!("repair_versions: finished, done {}", self.counter);
 				return Ok(WorkerState::Done);
 			}
 		};
 
-		self.counter += 1;
-
-		let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
+		let version = Version::decode(&item_bytes).ok_or_message("Cannot decode Version")?;
 		if !version.deleted.get() {
 			let object = self
 				.garage
@@ -131,10 +133,13 @@ impl Worker for RepairVersionsWorker {
 			}
 		}
 
+		self.counter += 1;
+		self.pos = next_pos;
+
 		Ok(WorkerState::Busy)
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		unreachable!()
 	}
 }
@@ -163,25 +168,24 @@ impl Worker for RepairBlockrefsWorker {
 		"Block refs repair worker".into()
 	}
 
-	fn info(&self) -> Option<String> {
-		Some(format!("{} items done", self.counter))
+	fn status(&self) -> WorkerStatus {
+		WorkerStatus {
+			progress: Some(self.counter.to_string()),
+			..Default::default()
+		}
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let item_bytes = match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
-			Some((k, v)) => {
-				self.pos = k;
-				v
-			}
+		let (item_bytes, next_pos) =
+			match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
+				Some((k, v)) => (v, k),
 				None => {
 					info!("repair_block_ref: finished, done {}", self.counter);
 					return Ok(WorkerState::Done);
 				}
 			};
 
-		self.counter += 1;
-
-		let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
+		let block_ref = BlockRef::decode(&item_bytes).ok_or_message("Cannot decode BlockRef")?;
 		if !block_ref.deleted.get() {
 			let version = self
 				.garage
@@ -206,10 +210,13 @@ impl Worker for RepairBlockrefsWorker {
 			}
 		}
 
+		self.counter += 1;
+		self.pos = next_pos;
+
 		Ok(WorkerState::Busy)
 	}
 
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+	async fn wait_for_work(&mut self) -> WorkerState {
 		unreachable!()
 	}
 }

src/garage/server.rs

@@ -35,12 +35,15 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	#[cfg(feature = "metrics")]
 	let metrics_exporter = opentelemetry_prometheus::exporter().init();
 
+	info!("Initializing Garage main data store...");
+	let garage = Garage::new(config.clone())?;
+
 	info!("Initializing background runner...");
 	let watch_cancel = watch_shutdown_signal();
-	let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
+	let (background, await_background_done) = BackgroundRunner::new(watch_cancel.clone());
 
-	info!("Initializing Garage main data store...");
-	let garage = Garage::new(config.clone(), background)?;
+	info!("Spawning Garage workers...");
+	garage.spawn_workers(&background);
 
 	if config.admin.trace_sink.is_some() {
 		info!("Initialize tracing...");
@@ -63,7 +66,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone()));
 
 	info!("Create admin RPC handler...");
-	AdminRpcHandler::new(garage.clone());
+	AdminRpcHandler::new(garage.clone(), background.clone());
 
 	// ---- Launch public-facing API servers ----
 

src/garage/tests/bucket.rs

@@ -1,4 +1,5 @@
 use crate::common;
+use crate::common::ext::CommandExt;
 use aws_sdk_s3::model::BucketLocationConstraint;
 use aws_sdk_s3::output::DeleteBucketOutput;
 
@@ -8,6 +9,27 @@ async fn test_bucket_all() {
 	let bucket_name = "hello";
 
 	{
+		// Check bucket cannot be created if not authorized
+		ctx.garage
+			.command()
+			.args(["key", "deny"])
+			.args(["--create-bucket", &ctx.garage.key.id])
+			.quiet()
+			.expect_success_output("Could not deny key to create buckets");
+
+		// Try create bucket, should fail
+		let r = ctx.client.create_bucket().bucket(bucket_name).send().await;
+		assert!(r.is_err());
+	}
+	{
+		// Now allow key to create bucket
+		ctx.garage
+			.command()
+			.args(["key", "allow"])
+			.args(["--create-bucket", &ctx.garage.key.id])
+			.quiet()
+			.expect_success_output("Could not deny key to create buckets");
+
 		// Create bucket
 		//@TODO check with an invalid bucket name + with an already existing bucket
 		let r = ctx

src/garage/tests/s3/streaming_signature.rs

@@ -1,6 +1,7 @@
 use std::collections::HashMap;
 
 use crate::common;
+use crate::common::ext::CommandExt;
 use common::custom_requester::BodySignature;
 use hyper::Method;
 
@@ -105,6 +106,13 @@ async fn test_create_bucket_streaming() {
 	let ctx = common::context();
 	let bucket = "createbucket-streaming";
 
+	ctx.garage
+		.command()
+		.args(["key", "allow"])
+		.args(["--create-bucket", &ctx.garage.key.id])
+		.quiet()
+		.expect_success_output("Could not allow key to create buckets");
+
 	{
 		// create bucket
 		let _ = ctx

src/k2v-client/Cargo.toml

@@ -22,7 +22,7 @@ tokio = "1.17.0"
 
 # cli deps
 clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
-garage_util = { version = "0.8.0", path = "../util", optional = true }
+garage_util = { version = "0.8.1", path = "../util", optional = true }
 
 
 [features]

src/model/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,11 +14,11 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_table = { version = "0.8.0", path = "../table" }
-garage_block = { version = "0.8.0", path = "../block" }
-garage_util = { version = "0.8.0", path = "../util" }
+garage_db = { version = "0.8.1", default-features = false, path = "../db" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_table = { version = "0.8.1", path = "../table" }
+garage_block = { version = "0.8.1", path = "../block" }
+garage_util = { version = "0.8.1", path = "../util" }
 
 async-trait = "0.1.7"
 arc-swap = "1.0"
@@ -30,7 +30,6 @@ tracing = "0.1.30"
 rand = "0.8"
 zstd = { version = "0.9", default-features = false }
 
-rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 
@@ -42,6 +41,7 @@ opentelemetry = "0.17"
 netapp = "0.5"
 
 [features]
+default = [ "sled" ]
 k2v = [ "garage_util/k2v" ]
 lmdb = [ "garage_db/lmdb" ]
 sled = [ "garage_db/sled" ]

src/model/bucket_alias_table.rs

@@ -1,18 +1,26 @@
-use serde::{Deserialize, Serialize};
-
 use garage_util::data::*;
 
 use garage_table::crdt::*;
 use garage_table::*;
 
-/// The bucket alias table holds the names given to buckets
-/// in the global namespace.
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct BucketAlias {
-	name: String,
+mod v08 {
+	use garage_util::crdt;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+
+	/// The bucket alias table holds the names given to buckets
+	/// in the global namespace.
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct BucketAlias {
+		pub(super) name: String,
 		pub state: crdt::Lww<Option<Uuid>>,
+	}
+
+	impl garage_util::migrate::InitialFormat for BucketAlias {}
 }
 
+pub use v08::*;
+
 impl BucketAlias {
 	pub fn new(name: String, ts: u64, bucket_id: Option<Uuid>) -> Option<Self> {
 		if !is_valid_bucket_name(&name) {

src/model/bucket_table.rs

@@ -1,5 +1,3 @@
-use serde::{Deserialize, Serialize};
-
 use garage_table::crdt::*;
 use garage_table::*;
 use garage_util::data::*;
@@ -7,22 +5,28 @@ use garage_util::time::*;
 
 use crate::permission::BucketKeyPerm;
 
-/// A bucket is a collection of objects
-///
-/// Its parameters are not directly accessible as:
-///  - It must be possible to merge paramaters, hence the use of a LWW CRDT.
-///  - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Bucket {
+mod v08 {
+	use crate::permission::BucketKeyPerm;
+	use garage_util::crdt;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+
+	/// A bucket is a collection of objects
+	///
+	/// Its parameters are not directly accessible as:
+	///  - It must be possible to merge paramaters, hence the use of a LWW CRDT.
+	///  - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Bucket {
 		/// ID of the bucket
 		pub id: Uuid,
 		/// State, and configuration if not deleted, of the bucket
 		pub state: crdt::Deletable<BucketParams>,
-}
+	}
 
-/// Configuration for a bucket
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct BucketParams {
+	/// Configuration for a bucket
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct BucketParams {
 		/// Bucket's creation date
 		pub creation_date: u64,
 		/// Map of key with access to the bucket, and what kind of access they give
@@ -47,32 +51,37 @@ pub struct BucketParams {
 		/// Bucket quotas
 		#[serde(default)]
 		pub quotas: crdt::Lww<BucketQuotas>,
-}
+	}
 
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct WebsiteConfig {
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct WebsiteConfig {
 		pub index_document: String,
 		pub error_document: Option<String>,
-}
+	}
 
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct CorsRule {
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct CorsRule {
 		pub id: Option<String>,
 		pub max_age_seconds: Option<u64>,
 		pub allow_origins: Vec<String>,
 		pub allow_methods: Vec<String>,
 		pub allow_headers: Vec<String>,
 		pub expose_headers: Vec<String>,
-}
+	}
 
-#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct BucketQuotas {
+	#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+	pub struct BucketQuotas {
 		/// Maximum size in bytes (bucket size = sum of sizes of objects in the bucket)
 		pub max_size: Option<u64>,
 		/// Maximum number of non-deleted objects in the bucket
 		pub max_objects: Option<u64>,
+	}
+
+	impl garage_util::migrate::InitialFormat for Bucket {}
 }
 
+pub use v08::*;
+
 impl AutoCrdt for BucketQuotas {
 	const WARN_IF_DIFFERENT: bool = true;
 }

src/model/garage.rs

@@ -8,10 +8,10 @@ use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;
 
+use garage_rpc::replication_mode::ReplicationMode;
 use garage_rpc::system::System;
 
 use garage_block::manager::*;
-use garage_table::replication::ReplicationMode;
 use garage_table::replication::TableFullReplication;
 use garage_table::replication::TableShardedReplication;
 use garage_table::*;
@@ -33,11 +33,14 @@ use crate::k2v::{item_table::*, poll::*, rpc::*};
 pub struct Garage {
 	/// The parsed configuration Garage is running
 	pub config: Config,
+	/// The set of background variables that can be viewed/modified at runtime
+	pub bg_vars: vars::BgVars,
+
+	/// The replication mode of this cluster
+	pub replication_mode: ReplicationMode,
 
 	/// The local database
 	pub db: db::Db,
-	/// A background job runner
-	pub background: Arc<BackgroundRunner>,
 	/// The membership manager
 	pub system: Arc<System>,
 	/// The block manager
@@ -75,7 +78,7 @@ pub struct GarageK2V {
 
 impl Garage {
 	/// Create and run garage
-	pub fn new(config: Config, background: Arc<BackgroundRunner>) -> Result<Arc<Self>, Error> {
+	pub fn new(config: Config) -> Result<Arc<Self>, Error> {
 		// Create meta dir and data dir if they don't exist already
 		std::fs::create_dir_all(&config.metadata_dir)
 			.ok_or_message("Unable to create Garage metadata directory")?;
@@ -156,7 +159,7 @@ impl Garage {
 		};
 
 		let network_key = NetworkKey::from_slice(
-			&hex::decode(&config.rpc_secret).expect("Invalid RPC secret key")[..],
+			&hex::decode(&config.rpc_secret.as_ref().unwrap()).expect("Invalid RPC secret key")[..],
 		)
 		.expect("Invalid RPC secret key");
 
@@ -164,12 +167,7 @@ impl Garage {
 			.expect("Invalid replication_mode in config file.");
 
 		info!("Initialize membership management system...");
-		let system = System::new(
-			network_key,
-			background.clone(),
-			replication_mode.replication_factor(),
-			&config,
-		)?;
+		let system = System::new(network_key, replication_mode, &config)?;
 
 		let data_rep_param = TableShardedReplication {
 			system: system.clone(),
@@ -227,7 +225,6 @@ impl Garage {
 		info!("Initialize version_table...");
 		let version_table = Table::new(
 			VersionTable {
-				background: background.clone(),
 				block_ref_table: block_ref_table.clone(),
 			},
 			meta_rep_param.clone(),
@@ -242,7 +239,6 @@ impl Garage {
 		#[allow(clippy::redundant_clone)]
 		let object_table = Table::new(
 			ObjectTable {
-				background: background.clone(),
 				version_table: version_table.clone(),
 				object_counter_table: object_counter_table.clone(),
 			},
@@ -255,11 +251,16 @@ impl Garage {
 		#[cfg(feature = "k2v")]
 		let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param);
 
+		// Initialize bg vars
+		let mut bg_vars = vars::BgVars::new();
+		block_manager.register_bg_vars(&mut bg_vars);
+
 		// -- done --
 		Ok(Arc::new(Self {
 			config,
+			bg_vars,
+			replication_mode,
 			db,
-			background,
 			system,
 			block_manager,
 			bucket_table,
@@ -274,6 +275,22 @@ impl Garage {
 		}))
 	}
 
+	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		self.block_manager.spawn_workers(bg);
+
+		self.bucket_table.spawn_workers(bg);
+		self.bucket_alias_table.spawn_workers(bg);
+		self.key_table.spawn_workers(bg);
+
+		self.object_table.spawn_workers(bg);
+		self.object_counter_table.spawn_workers(bg);
+		self.version_table.spawn_workers(bg);
+		self.block_ref_table.spawn_workers(bg);
+
+		#[cfg(feature = "k2v")]
+		self.k2v.spawn_workers(bg);
+	}
+
 	pub fn bucket_helper(&self) -> helper::bucket::BucketHelper {
 		helper::bucket::BucketHelper(self)
 	}
@@ -308,4 +325,9 @@ impl GarageK2V {
 			rpc,
 		}
 	}
+
+	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		self.item_table.spawn_workers(bg);
+		self.counter_table.spawn_workers(bg);
+	}
 }

src/model/index_counter.rs

@@ -1,19 +1,18 @@
 use core::ops::Bound;
-use std::collections::{hash_map, BTreeMap, HashMap};
+use std::collections::{BTreeMap, HashMap};
 use std::marker::PhantomData;
 use std::sync::Arc;
 
-use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
-use tokio::sync::{mpsc, watch};
 
 use garage_db as db;
 
 use garage_rpc::ring::Ring;
 use garage_rpc::system::System;
-use garage_util::background::*;
+use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::*;
+use garage_util::migrate::Migrate;
 use garage_util::time::*;
 
 use garage_table::crdt::*;
@@ -31,14 +30,44 @@ pub trait CountedItem: Clone + PartialEq + Send + Sync + 'static {
 	fn counts(&self) -> Vec<(&'static str, i64)>;
 }
 
-/// A counter entry in the global table
-#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
-pub struct CounterEntry<T: CountedItem> {
+mod v08 {
+	use super::CountedItem;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+	use std::collections::BTreeMap;
+
+	// ---- Global part (the table everyone queries) ----
+
+	/// A counter entry in the global table
+	#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
+	pub struct CounterEntry<T: CountedItem> {
 		pub pk: T::CP,
 		pub sk: T::CS,
 		pub values: BTreeMap<String, CounterValue>,
+	}
+
+	/// A counter entry in the global table
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct CounterValue {
+		pub node_values: BTreeMap<Uuid, (u64, i64)>,
+	}
+
+	impl<T: CountedItem> garage_util::migrate::InitialFormat for CounterEntry<T> {}
+
+	// ---- Local part (the counter we maintain transactionnaly on each node) ----
+
+	#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
+	pub(super) struct LocalCounterEntry<T: CountedItem> {
+		pub(super) pk: T::CP,
+		pub(super) sk: T::CS,
+		pub(super) values: BTreeMap<String, (u64, i64)>,
+	}
+
+	impl<T: CountedItem> garage_util::migrate::InitialFormat for LocalCounterEntry<T> {}
 }
 
+pub use v08::*;
+
 impl<T: CountedItem> Entry<T::CP, T::CS> for CounterEntry<T> {
 	fn partition_key(&self) -> &T::CP {
 		&self.pk
@@ -80,12 +109,6 @@ impl<T: CountedItem> CounterEntry<T> {
 	}
 }
 
-/// A counter entry in the global table
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct CounterValue {
-	pub node_values: BTreeMap<Uuid, (u64, i64)>,
-}
-
 impl<T: CountedItem> Crdt for CounterEntry<T> {
 	fn merge(&mut self, other: &Self) {
 		for (name, e2) in other.values.iter() {
@@ -142,7 +165,6 @@ impl<T: CountedItem> TableSchema for CounterTable<T> {
 pub struct IndexCounter<T: CountedItem> {
 	this_node: Uuid,
 	local_counter: db::Tree,
-	propagate_tx: mpsc::UnboundedSender<(T::CP, T::CS, LocalCounterEntry<T>)>,
 	pub table: Arc<Table<CounterTable<T>, TableShardedReplication>>,
 }
 
@@ -152,16 +174,11 @@ impl<T: CountedItem> IndexCounter<T> {
 		replication: TableShardedReplication,
 		db: &db::Db,
 	) -> Arc<Self> {
-		let background = system.background.clone();
-
-		let (propagate_tx, propagate_rx) = mpsc::unbounded_channel();
-
-		let this = Arc::new(Self {
+		Arc::new(Self {
 			this_node: system.id,
 			local_counter: db
 				.open_tree(format!("local_counter_v2:{}", T::COUNTER_TABLE_NAME))
 				.expect("Unable to open local counter tree"),
-			propagate_tx,
 			table: Table::new(
 				CounterTable {
 					_phantom_t: Default::default(),
@@ -170,16 +187,11 @@ impl<T: CountedItem> IndexCounter<T> {
 				system,
 				db,
 			),
-		});
+		})
+	}
 
-		background.spawn_worker(IndexPropagatorWorker {
-			index_counter: this.clone(),
-			propagate_rx,
-			buf: HashMap::new(),
-			errors: 0,
-		});
-
-		this
+	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		self.table.spawn_workers(bg);
 	}
 
 	pub fn count(
@@ -208,11 +220,9 @@ impl<T: CountedItem> IndexCounter<T> {
 		let tree_key = self.table.data.tree_key(pk, sk);
 
 		let mut entry = match tx.get(&self.local_counter, &tree_key[..])? {
-			Some(old_bytes) => {
-				rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(&old_bytes)
-					.map_err(Error::RmpDecode)
-					.map_err(db::TxError::Abort)?
-			}
+			Some(old_bytes) => LocalCounterEntry::<T>::decode(&old_bytes)
+				.ok_or_message("Cannot decode local counter entry")
+				.map_err(db::TxError::Abort)?,
 			None => LocalCounterEntry {
 				pk: pk.clone(),
 				sk: sk.clone(),
@@ -227,17 +237,14 @@ impl<T: CountedItem> IndexCounter<T> {
 			ent.1 += *inc;
 		}
 
-		let new_entry_bytes = rmp_to_vec_all_named(&entry)
+		let new_entry_bytes = entry
+			.encode()
 			.map_err(Error::RmpEncode)
 			.map_err(db::TxError::Abort)?;
 		tx.insert(&self.local_counter, &tree_key[..], new_entry_bytes)?;
 
-		if let Err(e) = self.propagate_tx.send((pk.clone(), sk.clone(), entry)) {
-			error!(
-				"Could not propagate updated counter values, failed to send to channel: {}",
-				e
-			);
-		}
+		let dist_entry = entry.into_counter_entry(self.this_node);
+		self.table.queue_insert(tx, &dist_entry)?;
 
 		Ok(())
 	}
@@ -250,23 +257,6 @@ impl<T: CountedItem> IndexCounter<T> {
 		TS: TableSchema<E = T>,
 		TR: TableReplication,
 	{
-		let save_counter_entry = |entry: CounterEntry<T>| -> Result<(), Error> {
-			let entry_k = self
-				.table
-				.data
-				.tree_key(entry.partition_key(), entry.sort_key());
-			self.table
-				.data
-				.update_entry_with(&entry_k, |ent| match ent {
-					Some(mut ent) => {
-						ent.merge(&entry);
-						ent
-					}
-					None => entry.clone(),
-				})?;
-			Ok(())
-		};
-
 		// 1. Set all old local counters to zero
 		let now = now_msec();
 		let mut next_start: Option<Vec<u8>> = None;
@@ -289,20 +279,22 @@ impl<T: CountedItem> IndexCounter<T> {
 
 			info!("zeroing old counters... ({})", hex::encode(&batch[0].0));
 			for (local_counter_k, local_counter) in batch {
-				let mut local_counter =
-					rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(&local_counter)?;
+				let mut local_counter = LocalCounterEntry::<T>::decode(&local_counter)
+					.ok_or_message("Cannot decode local counter entry")?;
 
 				for (_, tv) in local_counter.values.iter_mut() {
 					tv.0 = std::cmp::max(tv.0 + 1, now);
 					tv.1 = 0;
 				}
 
-				let local_counter_bytes = rmp_to_vec_all_named(&local_counter)?;
+				let local_counter_bytes = local_counter.encode()?;
 				self.local_counter
 					.insert(&local_counter_k, &local_counter_bytes)?;
 
 				let counter_entry = local_counter.into_counter_entry(self.this_node);
-				save_counter_entry(counter_entry)?;
+				self.local_counter
+					.db()
+					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;
 
 				next_start = Some(local_counter_k);
 			}
@@ -343,9 +335,8 @@ impl<T: CountedItem> IndexCounter<T> {
 				let local_counter_key = self.table.data.tree_key(pk, sk);
 				let mut local_counter = match self.local_counter.get(&local_counter_key)? {
 					Some(old_bytes) => {
-						let ent = rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(
-							&old_bytes,
-						)?;
+						let ent = LocalCounterEntry::<T>::decode(&old_bytes)
+							.ok_or_message("Cannot decode local counter entry")?;
 						assert!(ent.pk == *pk);
 						assert!(ent.sk == *sk);
 						ent
@@ -362,12 +353,14 @@ impl<T: CountedItem> IndexCounter<T> {
 					tv.1 += v;
 				}
 
-				let local_counter_bytes = rmp_to_vec_all_named(&local_counter)?;
+				let local_counter_bytes = local_counter.encode()?;
 				self.local_counter
 					.insert(&local_counter_key, local_counter_bytes)?;
 
 				let counter_entry = local_counter.into_counter_entry(self.this_node);
-				save_counter_entry(counter_entry)?;
+				self.local_counter
+					.db()
+					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;
 
 				next_start = Some(counted_entry_k);
 			}
@@ -378,104 +371,7 @@ impl<T: CountedItem> IndexCounter<T> {
 	}
 }
 
-struct IndexPropagatorWorker<T: CountedItem> {
-	index_counter: Arc<IndexCounter<T>>,
-	propagate_rx: mpsc::UnboundedReceiver<(T::CP, T::CS, LocalCounterEntry<T>)>,
-
-	buf: HashMap<Vec<u8>, CounterEntry<T>>,
-	errors: usize,
-}
-
-impl<T: CountedItem> IndexPropagatorWorker<T> {
-	fn add_ent(&mut self, pk: T::CP, sk: T::CS, counters: LocalCounterEntry<T>) {
-		let tree_key = self.index_counter.table.data.tree_key(&pk, &sk);
-		let dist_entry = counters.into_counter_entry(self.index_counter.this_node);
-		match self.buf.entry(tree_key) {
-			hash_map::Entry::Vacant(e) => {
-				e.insert(dist_entry);
-			}
-			hash_map::Entry::Occupied(mut e) => {
-				e.get_mut().merge(&dist_entry);
-			}
-		}
-	}
-}
-
-#[async_trait]
-impl<T: CountedItem> Worker for IndexPropagatorWorker<T> {
-	fn name(&self) -> String {
-		format!("{} index counter propagator", T::COUNTER_TABLE_NAME)
-	}
-
-	fn info(&self) -> Option<String> {
-		if !self.buf.is_empty() {
-			Some(format!("{} items in queue", self.buf.len()))
-		} else {
-			None
-		}
-	}
-
-	async fn work(&mut self, must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		// This loop batches updates to counters to be sent all at once.
-		// They are sent once the propagate_rx channel has been emptied (or is closed).
-		let closed = loop {
-			match self.propagate_rx.try_recv() {
-				Ok((pk, sk, counters)) => {
-					self.add_ent(pk, sk, counters);
-				}
-				Err(mpsc::error::TryRecvError::Empty) => break false,
-				Err(mpsc::error::TryRecvError::Disconnected) => break true,
-			}
-		};
-
-		if !self.buf.is_empty() {
-			let entries_k = self.buf.keys().take(100).cloned().collect::<Vec<_>>();
-			let entries = entries_k.iter().map(|k| self.buf.get(k).unwrap());
-			if let Err(e) = self.index_counter.table.insert_many(entries).await {
-				self.errors += 1;
-				if self.errors >= 2 && *must_exit.borrow() {
-					error!("({}) Could not propagate {} counter values: {}, these counters will not be updated correctly.", T::COUNTER_TABLE_NAME, self.buf.len(), e);
-					return Ok(WorkerState::Done);
-				}
-				// Propagate error up to worker manager, it will log it, increment a counter,
-				// and sleep for a certain delay (with exponential backoff), waiting for
-				// things to go back to normal
-				return Err(e);
-			} else {
-				for k in entries_k {
-					self.buf.remove(&k);
-				}
-				self.errors = 0;
-			}
-
-			return Ok(WorkerState::Busy);
-		} else if closed {
-			return Ok(WorkerState::Done);
-		} else {
-			return Ok(WorkerState::Idle);
-		}
-	}
-
-	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
-		match self.propagate_rx.recv().await {
-			Some((pk, sk, counters)) => {
-				self.add_ent(pk, sk, counters);
-				WorkerState::Busy
-			}
-			None => match self.buf.is_empty() {
-				false => WorkerState::Busy,
-				true => WorkerState::Done,
-			},
-		}
-	}
-}
-
-#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
-struct LocalCounterEntry<T: CountedItem> {
-	pk: T::CP,
-	sk: T::CS,
-	values: BTreeMap<String, (u64, i64)>,
-}
+// ----
 
 impl<T: CountedItem> LocalCounterEntry<T> {
 	fn into_counter_entry(self, this_node: Uuid) -> CounterEntry<T> {

src/model/k2v/item_table.rs

@@ -1,7 +1,8 @@
-use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 use std::sync::Arc;
 
+use serde::{Deserialize, Serialize};
+
 use garage_db as db;
 use garage_util::data::*;
 
@@ -17,32 +18,43 @@ pub const CONFLICTS: &str = "conflicts";
 pub const VALUES: &str = "values";
 pub const BYTES: &str = "bytes";
 
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct K2VItem {
+mod v08 {
+	use crate::k2v::causality::K2VNodeId;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+	use std::collections::BTreeMap;
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct K2VItem {
 		pub partition: K2VItemPartition,
 		pub sort_key: String,
 
-	items: BTreeMap<K2VNodeId, DvvsEntry>,
-}
+		pub(super) items: BTreeMap<K2VNodeId, DvvsEntry>,
+	}
 
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Hash)]
-pub struct K2VItemPartition {
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Hash)]
+	pub struct K2VItemPartition {
 		pub bucket_id: Uuid,
 		pub partition_key: String,
-}
+	}
 
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-struct DvvsEntry {
-	t_discard: u64,
-	values: Vec<(u64, DvvsValue)>,
-}
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct DvvsEntry {
+		pub(super) t_discard: u64,
+		pub(super) values: Vec<(u64, DvvsValue)>,
+	}
 
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub enum DvvsValue {
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub enum DvvsValue {
 		Value(#[serde(with = "serde_bytes")] Vec<u8>),
 		Deleted,
+	}
+
+	impl garage_util::migrate::InitialFormat for K2VItem {}
 }
 
+pub use v08::*;
+
 impl K2VItem {
 	/// Creates a new K2VItem when no previous entry existed in the db
 	pub fn new(bucket_id: Uuid, partition_key: String, sort_key: String) -> Self {

src/model/k2v/rpc.rs

@@ -273,14 +273,9 @@ impl K2VRpcHandler {
 	}
 
 	fn local_insert(&self, item: &InsertedItem) -> Result<Option<K2VItem>, Error> {
-		let tree_key = self
-			.item_table
-			.data
-			.tree_key(&item.partition, &item.sort_key);
-
 		self.item_table
 			.data
-			.update_entry_with(&tree_key[..], |ent| {
+			.update_entry_with(&item.partition, &item.sort_key, |ent| {
 				let mut ent = ent.unwrap_or_else(|| {
 					K2VItem::new(
 						item.partition.bucket_id,

src/model/key_table.rs

@@ -1,26 +1,72 @@
 use serde::{Deserialize, Serialize};
 
-use garage_table::crdt::*;
-use garage_table::*;
+use garage_util::crdt::{self, Crdt};
 use garage_util::data::*;
 
+use garage_table::{DeletedFilter, EmptyKey, Entry, TableSchema};
+
 use crate::permission::BucketKeyPerm;
 
-use crate::prev::v051::key_table as old;
+pub(crate) mod v05 {
+	use garage_util::crdt;
+	use serde::{Deserialize, Serialize};
 
-/// An api key
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Key {
+	/// An api key
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Key {
+		/// The id of the key (immutable), used as partition key
+		pub key_id: String,
+
+		/// The secret_key associated
+		pub secret_key: String,
+
+		/// Name for the key
+		pub name: crdt::Lww<String>,
+
+		/// Is the key deleted
+		pub deleted: crdt::Bool,
+
+		/// Buckets in which the key is authorized. Empty if `Key` is deleted
+		// CRDT interaction: deleted implies authorized_buckets is empty
+		pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
+	}
+
+	/// Permission given to a key in a bucket
+	#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct PermissionSet {
+		/// The key can be used to read the bucket
+		pub allow_read: bool,
+		/// The key can be used to write in the bucket
+		pub allow_write: bool,
+	}
+
+	impl crdt::AutoCrdt for PermissionSet {
+		const WARN_IF_DIFFERENT: bool = true;
+	}
+
+	impl garage_util::migrate::InitialFormat for Key {}
+}
+
+mod v08 {
+	use super::v05;
+	use crate::permission::BucketKeyPerm;
+	use garage_util::crdt;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+
+	/// An api key
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Key {
 		/// The id of the key (immutable), used as partition key
 		pub key_id: String,
 
 		/// Internal state of the key
 		pub state: crdt::Deletable<KeyParams>,
-}
+	}
 
-/// Configuration for a key
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct KeyParams {
+	/// Configuration for a key
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct KeyParams {
 		/// The secret_key associated (immutable)
 		pub secret_key: String,
 
@@ -38,8 +84,38 @@ pub struct KeyParams {
 		/// A key can have a local view of buckets names it is
 		/// the only one to see, this is the namespace for these aliases
 		pub local_aliases: crdt::LwwMap<String, Option<Uuid>>,
+	}
+
+	impl garage_util::migrate::Migrate for Key {
+		type Previous = v05::Key;
+
+		fn migrate(old_k: v05::Key) -> Key {
+			let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone());
+
+			let state = if old_k.deleted.get() {
+				crdt::Deletable::Deleted
+			} else {
+				// Authorized buckets is ignored here,
+				// migration is performed in specific migration code in
+				// garage/migrate.rs
+				crdt::Deletable::Present(KeyParams {
+					secret_key: old_k.secret_key,
+					name,
+					allow_create_bucket: crdt::Lww::new(false),
+					authorized_buckets: crdt::Map::new(),
+					local_aliases: crdt::LwwMap::new(),
+				})
+			};
+			Key {
+				key_id: old_k.key_id,
+				state,
+			}
+		}
+	}
 }
 
+pub use v08::*;
+
 impl KeyParams {
 	fn new(secret_key: &str, name: &str) -> Self {
 		KeyParams {
@@ -173,28 +249,4 @@ impl TableSchema for KeyTable {
 			}
 		}
 	}
-
-	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
-		let old_k = rmp_serde::decode::from_read_ref::<_, old::Key>(bytes).ok()?;
-		let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone());
-
-		let state = if old_k.deleted.get() {
-			crdt::Deletable::Deleted
-		} else {
-			// Authorized buckets is ignored here,
-			// migration is performed in specific migration code in
-			// garage/migrate.rs
-			crdt::Deletable::Present(KeyParams {
-				secret_key: old_k.secret_key,
-				name,
-				allow_create_bucket: crdt::Lww::new(false),
-				authorized_buckets: crdt::Map::new(),
-				local_aliases: crdt::LwwMap::new(),
-			})
-		};
-		Some(Key {
-			key_id: old_k.key_id,
-			state,
-		})
-	}
 }

src/model/migrate.rs

@@ -2,6 +2,7 @@ use std::sync::Arc;
 
 use garage_util::crdt::*;
 use garage_util::data::*;
+use garage_util::encode::nonversioned_decode;
 use garage_util::error::Error as GarageError;
 use garage_util::time::*;
 
@@ -28,8 +29,8 @@ impl Migrate {
 		let mut old_buckets = vec![];
 		for res in tree.iter().map_err(GarageError::from)? {
 			let (_k, v) = res.map_err(GarageError::from)?;
-			let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..])
-				.map_err(GarageError::from)?;
+			let bucket =
+				nonversioned_decode::<old_bucket::Bucket>(&v[..]).map_err(GarageError::from)?;
 			old_buckets.push(bucket);
 		}
 

src/model/prev/v051/bucket_table.rs

@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
 use garage_table::crdt::Crdt;
 use garage_table::*;
 
-use super::key_table::PermissionSet;
+use crate::key_table::v05::PermissionSet;
 
 /// A bucket is a collection of objects
 ///

src/model/prev/v051/key_table.rs

@@ -1,50 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use garage_table::crdt::*;
-use garage_table::*;
-
-/// An api key
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Key {
-	/// The id of the key (immutable), used as partition key
-	pub key_id: String,
-
-	/// The secret_key associated
-	pub secret_key: String,
-
-	/// Name for the key
-	pub name: crdt::Lww<String>,
-
-	/// Is the key deleted
-	pub deleted: crdt::Bool,
-
-	/// Buckets in which the key is authorized. Empty if `Key` is deleted
-	// CRDT interaction: deleted implies authorized_buckets is empty
-	pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
-}
-
-/// Permission given to a key in a bucket
-#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct PermissionSet {
-	/// The key can be used to read the bucket
-	pub allow_read: bool,
-	/// The key can be used to write in the bucket
-	pub allow_write: bool,
-}
-
-impl AutoCrdt for PermissionSet {
-	const WARN_IF_DIFFERENT: bool = true;
-}
-
-impl Crdt for Key {
-	fn merge(&mut self, other: &Self) {
-		self.name.merge(&other.name);
-		self.deleted.merge(&other.deleted);
-
-		if self.deleted.get() {
-			self.authorized_buckets.clear();
-		} else {
-			self.authorized_buckets.merge(&other.authorized_buckets);
-		}
-	}
-}

src/model/prev/v051/mod.rs

@@ -1,4 +1 @@
 pub(crate) mod bucket_table;
-pub(crate) mod key_table;
-pub(crate) mod object_table;
-pub(crate) mod version_table;

src/model/prev/v051/object_table.rs

@@ -1,149 +0,0 @@
-use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
-
-use garage_util::data::*;
-
-use garage_table::crdt::*;
-
-/// An object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Object {
-	/// The bucket in which the object is stored, used as partition key
-	pub bucket: String,
-
-	/// The key at which the object is stored in its bucket, used as sorting key
-	pub key: String,
-
-	/// The list of currenty stored versions of the object
-	versions: Vec<ObjectVersion>,
-}
-
-impl Object {
-	/// Get a list of currently stored versions of `Object`
-	pub fn versions(&self) -> &[ObjectVersion] {
-		&self.versions[..]
-	}
-}
-
-/// Informations about a version of an object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersion {
-	/// Id of the version
-	pub uuid: Uuid,
-	/// Timestamp of when the object was created
-	pub timestamp: u64,
-	/// State of the version
-	pub state: ObjectVersionState,
-}
-
-/// State of an object version
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub enum ObjectVersionState {
-	/// The version is being received
-	Uploading(ObjectVersionHeaders),
-	/// The version is fully received
-	Complete(ObjectVersionData),
-	/// The version uploaded containded errors or the upload was explicitly aborted
-	Aborted,
-}
-
-impl Crdt for ObjectVersionState {
-	fn merge(&mut self, other: &Self) {
-		use ObjectVersionState::*;
-		match other {
-			Aborted => {
-				*self = Aborted;
-			}
-			Complete(b) => match self {
-				Aborted => {}
-				Complete(a) => {
-					a.merge(b);
-				}
-				Uploading(_) => {
-					*self = Complete(b.clone());
-				}
-			},
-			Uploading(_) => {}
-		}
-	}
-}
-
-/// Data stored in object version
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub enum ObjectVersionData {
-	/// The object was deleted, this Version is a tombstone to mark it as such
-	DeleteMarker,
-	/// The object is short, it's stored inlined
-	Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
-	/// The object is not short, Hash of first block is stored here, next segments hashes are
-	/// stored in the version table
-	FirstBlock(ObjectVersionMeta, Hash),
-}
-
-impl AutoCrdt for ObjectVersionData {
-	const WARN_IF_DIFFERENT: bool = true;
-}
-
-/// Metadata about the object version
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersionMeta {
-	/// Headers to send to the client
-	pub headers: ObjectVersionHeaders,
-	/// Size of the object
-	pub size: u64,
-	/// etag of the object
-	pub etag: String,
-}
-
-/// Additional headers for an object
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersionHeaders {
-	/// Content type of the object
-	pub content_type: String,
-	/// Any other http headers to send
-	pub other: BTreeMap<String, String>,
-}
-
-impl ObjectVersion {
-	fn cmp_key(&self) -> (u64, Uuid) {
-		(self.timestamp, self.uuid)
-	}
-
-	/// Is the object version completely received
-	pub fn is_complete(&self) -> bool {
-		matches!(self.state, ObjectVersionState::Complete(_))
-	}
-}
-
-impl Crdt for Object {
-	fn merge(&mut self, other: &Self) {
-		// Merge versions from other into here
-		for other_v in other.versions.iter() {
-			match self
-				.versions
-				.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
-			{
-				Ok(i) => {
-					self.versions[i].state.merge(&other_v.state);
-				}
-				Err(i) => {
-					self.versions.insert(i, other_v.clone());
-				}
-			}
-		}
-
-		// Remove versions which are obsolete, i.e. those that come
-		// before the last version which .is_complete().
-		let last_complete = self
-			.versions
-			.iter()
-			.enumerate()
-			.rev()
-			.find(|(_, v)| v.is_complete())
-			.map(|(vi, _)| vi);
-
-		if let Some(last_vi) = last_complete {
-			self.versions = self.versions.drain(last_vi..).collect::<Vec<_>>();
-		}
-	}
-}

src/model/prev/v051/version_table.rs

@@ -1,79 +0,0 @@
-use serde::{Deserialize, Serialize};
-
-use garage_util::data::*;
-
-use garage_table::crdt::*;
-use garage_table::*;
-
-/// A version of an object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Version {
-	/// UUID of the version, used as partition key
-	pub uuid: Uuid,
-
-	// Actual data: the blocks for this version
-	// In the case of a multipart upload, also store the etags
-	// of individual parts and check them when doing CompleteMultipartUpload
-	/// Is this version deleted
-	pub deleted: crdt::Bool,
-	/// list of blocks of data composing the version
-	pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
-	/// Etag of each part in case of a multipart upload, empty otherwise
-	pub parts_etags: crdt::Map<u64, String>,
-
-	// Back link to bucket+key so that we can figure if
-	// this was deleted later on
-	/// Bucket in which the related object is stored
-	pub bucket: String,
-	/// Key in which the related object is stored
-	pub key: String,
-}
-
-#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
-pub struct VersionBlockKey {
-	/// Number of the part
-	pub part_number: u64,
-	/// Offset of this sub-segment in its part
-	pub offset: u64,
-}
-
-impl Ord for VersionBlockKey {
-	fn cmp(&self, other: &Self) -> std::cmp::Ordering {
-		self.part_number
-			.cmp(&other.part_number)
-			.then(self.offset.cmp(&other.offset))
-	}
-}
-
-impl PartialOrd for VersionBlockKey {
-	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
-		Some(self.cmp(other))
-	}
-}
-
-/// Informations about a single block
-#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
-pub struct VersionBlock {
-	/// Blake2 sum of the block
-	pub hash: Hash,
-	/// Size of the block
-	pub size: u64,
-}
-
-impl AutoCrdt for VersionBlock {
-	const WARN_IF_DIFFERENT: bool = true;
-}
-
-impl Crdt for Version {
-	fn merge(&mut self, other: &Self) {
-		self.deleted.merge(&other.deleted);
-
-		if self.deleted.get() {
-			self.blocks.clear();
-			self.parts_etags.clear();
-		} else {
-			self.blocks.merge(&other.blocks);
-			self.parts_etags.merge(&other.parts_etags);
-		}
-	}
-}

src/model/s3/block_ref_table.rs

@@ -1,4 +1,3 @@
-use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 
 use garage_db as db;
@@ -10,8 +9,13 @@ use garage_table::*;
 
 use garage_block::manager::*;
 
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct BlockRef {
+mod v08 {
+	use garage_util::crdt;
+	use garage_util::data::{Hash, Uuid};
+	use serde::{Deserialize, Serialize};
+
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct BlockRef {
 		/// Hash (blake2 sum) of the block, used as partition key
 		pub block: Hash,
 
@@ -21,8 +25,13 @@ pub struct BlockRef {
 		// Keep track of deleted status
 		/// Is the Version that contains this block deleted
 		pub deleted: crdt::Bool,
+	}
+
+	impl garage_util::migrate::InitialFormat for BlockRef {}
 }
 
+pub use v08::*;
+
 impl Entry<Hash, Uuid> for BlockRef {
 	fn partition_key(&self) -> &Hash {
 		&self.block

src/model/s3/object_table.rs

@@ -1,10 +1,8 @@
 use serde::{Deserialize, Serialize};
-use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use garage_db as db;
 
-use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 
 use garage_table::crdt::*;
@@ -14,15 +12,99 @@ use garage_table::*;
 use crate::index_counter::*;
 use crate::s3::version_table::*;
 
-use crate::prev::v051::object_table as old;
-
 pub const OBJECTS: &str = "objects";
 pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads";
 pub const BYTES: &str = "bytes";
 
-/// An object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Object {
+mod v05 {
+	use garage_util::data::{Hash, Uuid};
+	use serde::{Deserialize, Serialize};
+	use std::collections::BTreeMap;
+
+	/// An object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Object {
+		/// The bucket in which the object is stored, used as partition key
+		pub bucket: String,
+
+		/// The key at which the object is stored in its bucket, used as sorting key
+		pub key: String,
+
+		/// The list of currenty stored versions of the object
+		pub(super) versions: Vec<ObjectVersion>,
+	}
+
+	/// Informations about a version of an object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct ObjectVersion {
+		/// Id of the version
+		pub uuid: Uuid,
+		/// Timestamp of when the object was created
+		pub timestamp: u64,
+		/// State of the version
+		pub state: ObjectVersionState,
+	}
+
+	/// State of an object version
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub enum ObjectVersionState {
+		/// The version is being received
+		Uploading(ObjectVersionHeaders),
+		/// The version is fully received
+		Complete(ObjectVersionData),
+		/// The version uploaded containded errors or the upload was explicitly aborted
+		Aborted,
+	}
+
+	/// Data stored in object version
+	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+	pub enum ObjectVersionData {
+		/// The object was deleted, this Version is a tombstone to mark it as such
+		DeleteMarker,
+		/// The object is short, it's stored inlined
+		Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
+		/// The object is not short, Hash of first block is stored here, next segments hashes are
+		/// stored in the version table
+		FirstBlock(ObjectVersionMeta, Hash),
+	}
+
+	/// Metadata about the object version
+	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+	pub struct ObjectVersionMeta {
+		/// Headers to send to the client
+		pub headers: ObjectVersionHeaders,
+		/// Size of the object
+		pub size: u64,
+		/// etag of the object
+		pub etag: String,
+	}
+
+	/// Additional headers for an object
+	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+	pub struct ObjectVersionHeaders {
+		/// Content type of the object
+		pub content_type: String,
+		/// Any other http headers to send
+		pub other: BTreeMap<String, String>,
+	}
+
+	impl garage_util::migrate::InitialFormat for Object {}
+}
+
+mod v08 {
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+
+	use super::v05;
+
+	pub use v05::{
+		ObjectVersion, ObjectVersionData, ObjectVersionHeaders, ObjectVersionMeta,
+		ObjectVersionState,
+	};
+
+	/// An object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Object {
 		/// The bucket in which the object is stored, used as partition key
 		pub bucket_id: Uuid,
 
@@ -30,9 +112,26 @@ pub struct Object {
 		pub key: String,
 
 		/// The list of currenty stored versions of the object
-	versions: Vec<ObjectVersion>,
+		pub(super) versions: Vec<ObjectVersion>,
+	}
+
+	impl garage_util::migrate::Migrate for Object {
+		type Previous = v05::Object;
+
+		fn migrate(old: v05::Object) -> Object {
+			use garage_util::data::blake2sum;
+
+			Object {
+				bucket_id: blake2sum(old.bucket.as_bytes()),
+				key: old.key,
+				versions: old.versions,
+			}
+		}
+	}
 }
 
+pub use v08::*;
+
 impl Object {
 	/// Initialize an Object struct from parts
 	pub fn new(bucket_id: Uuid, key: String, versions: Vec<ObjectVersion>) -> Self {
@@ -69,28 +168,6 @@ impl Object {
 	}
 }
 
-/// Informations about a version of an object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersion {
-	/// Id of the version
-	pub uuid: Uuid,
-	/// Timestamp of when the object was created
-	pub timestamp: u64,
-	/// State of the version
-	pub state: ObjectVersionState,
-}
-
-/// State of an object version
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub enum ObjectVersionState {
-	/// The version is being received
-	Uploading(ObjectVersionHeaders),
-	/// The version is fully received
-	Complete(ObjectVersionData),
-	/// The version uploaded containded errors or the upload was explicitly aborted
-	Aborted,
-}
-
 impl Crdt for ObjectVersionState {
 	fn merge(&mut self, other: &Self) {
 		use ObjectVersionState::*;
@@ -112,42 +189,10 @@ impl Crdt for ObjectVersionState {
 	}
 }
 
-/// Data stored in object version
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub enum ObjectVersionData {
-	/// The object was deleted, this Version is a tombstone to mark it as such
-	DeleteMarker,
-	/// The object is short, it's stored inlined
-	Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
-	/// The object is not short, Hash of first block is stored here, next segments hashes are
-	/// stored in the version table
-	FirstBlock(ObjectVersionMeta, Hash),
-}
-
 impl AutoCrdt for ObjectVersionData {
 	const WARN_IF_DIFFERENT: bool = true;
 }
 
-/// Metadata about the object version
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersionMeta {
-	/// Headers to send to the client
-	pub headers: ObjectVersionHeaders,
-	/// Size of the object
-	pub size: u64,
-	/// etag of the object
-	pub etag: String,
-}
-
-/// Additional headers for an object
-#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-pub struct ObjectVersionHeaders {
-	/// Content type of the object
-	pub content_type: String,
-	/// Any other http headers to send
-	pub other: BTreeMap<String, String>,
-}
-
 impl ObjectVersion {
 	fn cmp_key(&self) -> (u64, Uuid) {
 		(self.timestamp, self.uuid)
@@ -221,7 +266,6 @@ impl Crdt for Object {
 }
 
 pub struct ObjectTable {
-	pub background: Arc<BackgroundRunner>,
 	pub version_table: Arc<Table<VersionTable, TableShardedReplication>>,
 	pub object_counter_table: Arc<IndexCounter<Object>>,
 }
@@ -255,12 +299,7 @@ impl TableSchema for ObjectTable {
 			);
 		}
 
-		// 2. Spawn threads that propagates deletions to version table
-		let version_table = self.version_table.clone();
-		let old = old.cloned();
-		let new = new.cloned();
-
-		self.background.spawn(async move {
+		// 2. Enqueue propagation deletions to version table
 		if let (Some(old_v), Some(new_v)) = (old, new) {
 			// Propagate deletion of old versions
 			for v in old_v.versions.iter() {
@@ -277,12 +316,17 @@ impl TableSchema for ObjectTable {
 				if newly_deleted {
 					let deleted_version =
 						Version::new(v.uuid, old_v.bucket_id, old_v.key.clone(), true);
-						version_table.insert(&deleted_version).await?;
+					let res = self.version_table.queue_insert(tx, &deleted_version);
+					if let Err(e) = db::unabort(res)? {
+						error!(
+							"Unable to enqueue version deletion propagation: {}. A repair will be needed.",
+							e
+						);
 					}
 				}
 			}
-			Ok(())
-		});
+		}
+
 		Ok(())
 	}
 
@@ -292,11 +336,6 @@ impl TableSchema for ObjectTable {
 			ObjectFilter::IsUploading => entry.versions.iter().any(|v| v.is_uploading()),
 		}
 	}
-
-	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
-		let old_obj = rmp_serde::decode::from_read_ref::<_, old::Object>(bytes).ok()?;
-		Some(migrate_object(old_obj))
-	}
 }
 
 impl CountedItem for Object {
@@ -341,64 +380,3 @@ impl CountedItem for Object {
 		]
 	}
 }
-
-// vvvvvvvv migration code, stupid stuff vvvvvvvvvvvv
-// (we just want to change bucket into bucket_id by hashing it)
-
-fn migrate_object(o: old::Object) -> Object {
-	let versions = o
-		.versions()
-		.iter()
-		.cloned()
-		.map(migrate_object_version)
-		.collect();
-	Object {
-		bucket_id: blake2sum(o.bucket.as_bytes()),
-		key: o.key,
-		versions,
-	}
-}
-
-fn migrate_object_version(v: old::ObjectVersion) -> ObjectVersion {
-	ObjectVersion {
-		uuid: Uuid::try_from(v.uuid.as_slice()).unwrap(),
-		timestamp: v.timestamp,
-		state: match v.state {
-			old::ObjectVersionState::Uploading(h) => {
-				ObjectVersionState::Uploading(migrate_object_version_headers(h))
-			}
-			old::ObjectVersionState::Complete(d) => {
-				ObjectVersionState::Complete(migrate_object_version_data(d))
-			}
-			old::ObjectVersionState::Aborted => ObjectVersionState::Aborted,
-		},
-	}
-}
-
-fn migrate_object_version_headers(h: old::ObjectVersionHeaders) -> ObjectVersionHeaders {
-	ObjectVersionHeaders {
-		content_type: h.content_type,
-		other: h.other,
-	}
-}
-
-fn migrate_object_version_data(d: old::ObjectVersionData) -> ObjectVersionData {
-	match d {
-		old::ObjectVersionData::DeleteMarker => ObjectVersionData::DeleteMarker,
-		old::ObjectVersionData::Inline(m, b) => {
-			ObjectVersionData::Inline(migrate_object_version_meta(m), b)
-		}
-		old::ObjectVersionData::FirstBlock(m, h) => ObjectVersionData::FirstBlock(
-			migrate_object_version_meta(m),
-			Hash::try_from(h.as_slice()).unwrap(),
-		),
-	}
-}
-
-fn migrate_object_version_meta(m: old::ObjectVersionMeta) -> ObjectVersionMeta {
-	ObjectVersionMeta {
-		headers: migrate_object_version_headers(m.headers),
-		size: m.size,
-		etag: m.etag,
-	}
-}

src/model/s3/version_table.rs

@@ -1,9 +1,7 @@
-use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 
 use garage_db as db;
 
-use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 
 use garage_table::crdt::*;
@@ -12,11 +10,65 @@ use garage_table::*;
 
 use crate::s3::block_ref_table::*;
 
-use crate::prev::v051::version_table as old;
+mod v05 {
+	use garage_util::crdt;
+	use garage_util::data::{Hash, Uuid};
+	use serde::{Deserialize, Serialize};
 
-/// A version of an object
-#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-pub struct Version {
+	/// A version of an object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Version {
+		/// UUID of the version, used as partition key
+		pub uuid: Uuid,
+
+		// Actual data: the blocks for this version
+		// In the case of a multipart upload, also store the etags
+		// of individual parts and check them when doing CompleteMultipartUpload
+		/// Is this version deleted
+		pub deleted: crdt::Bool,
+		/// list of blocks of data composing the version
+		pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
+		/// Etag of each part in case of a multipart upload, empty otherwise
+		pub parts_etags: crdt::Map<u64, String>,
+
+		// Back link to bucket+key so that we can figure if
+		// this was deleted later on
+		/// Bucket in which the related object is stored
+		pub bucket: String,
+		/// Key in which the related object is stored
+		pub key: String,
+	}
+
+	#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
+	pub struct VersionBlockKey {
+		/// Number of the part
+		pub part_number: u64,
+		/// Offset of this sub-segment in its part
+		pub offset: u64,
+	}
+
+	/// Informations about a single block
+	#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
+	pub struct VersionBlock {
+		/// Blake2 sum of the block
+		pub hash: Hash,
+		/// Size of the block
+		pub size: u64,
+	}
+
+	impl garage_util::migrate::InitialFormat for Version {}
+}
+
+mod v08 {
+	use garage_util::crdt;
+	use garage_util::data::Uuid;
+	use serde::{Deserialize, Serialize};
+
+	use super::v05;
+
+	/// A version of an object
+	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub struct Version {
 		/// UUID of the version, used as partition key
 		pub uuid: Uuid,
 
@@ -36,8 +88,30 @@ pub struct Version {
 		pub bucket_id: Uuid,
 		/// Key in which the related object is stored
 		pub key: String,
+	}
+
+	pub use v05::{VersionBlock, VersionBlockKey};
+
+	impl garage_util::migrate::Migrate for Version {
+		type Previous = v05::Version;
+
+		fn migrate(old: v05::Version) -> Version {
+			use garage_util::data::blake2sum;
+
+			Version {
+				uuid: old.uuid,
+				deleted: old.deleted,
+				blocks: old.blocks,
+				parts_etags: old.parts_etags,
+				bucket_id: blake2sum(old.bucket.as_bytes()),
+				key: old.key,
+			}
+		}
+	}
 }
 
+pub use v08::*;
+
 impl Version {
 	pub fn new(uuid: Uuid, bucket_id: Uuid, key: String, deleted: bool) -> Self {
 		Self {
@@ -65,14 +139,6 @@ impl Version {
 	}
 }
 
-#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
-pub struct VersionBlockKey {
-	/// Number of the part
-	pub part_number: u64,
-	/// Offset of this sub-segment in its part
-	pub offset: u64,
-}
-
 impl Ord for VersionBlockKey {
 	fn cmp(&self, other: &Self) -> std::cmp::Ordering {
 		self.part_number
@@ -87,15 +153,6 @@ impl PartialOrd for VersionBlockKey {
 	}
 }
 
-/// Informations about a single block
-#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
-pub struct VersionBlock {
-	/// Blake2 sum of the block
-	pub hash: Hash,
-	/// Size of the block
-	pub size: u64,
-}
-
 impl AutoCrdt for VersionBlock {
 	const WARN_IF_DIFFERENT: bool = true;
 }
@@ -127,7 +184,6 @@ impl Crdt for Version {
 }
 
 pub struct VersionTable {
-	pub background: Arc<BackgroundRunner>,
 	pub block_ref_table: Arc<Table<BlockRefTable, TableShardedReplication>>,
 }
 
@@ -141,33 +197,26 @@ impl TableSchema for VersionTable {
 
 	fn updated(
 		&self,
-		_tx: &mut db::Transaction,
+		tx: &mut db::Transaction,
 		old: Option<&Self::E>,
 		new: Option<&Self::E>,
 	) -> db::TxOpResult<()> {
-		let block_ref_table = self.block_ref_table.clone();
-		let old = old.cloned();
-		let new = new.cloned();
-
-		self.background.spawn(async move {
 		if let (Some(old_v), Some(new_v)) = (old, new) {
 			// Propagate deletion of version blocks
 			if new_v.deleted.get() && !old_v.deleted.get() {
-					let deleted_block_refs = old_v
-						.blocks
-						.items()
-						.iter()
-						.map(|(_k, vb)| BlockRef {
+				let deleted_block_refs = old_v.blocks.items().iter().map(|(_k, vb)| BlockRef {
 					block: vb.hash,
 					version: old_v.uuid,
 					deleted: true.into(),
-						})
-						.collect::<Vec<_>>();
-					block_ref_table.insert_many(&deleted_block_refs[..]).await?;
-				}
-			}
-			Ok(())
 				});
+				for block_ref in deleted_block_refs {
+					let res = self.block_ref_table.queue_insert(tx, &block_ref);
+					if let Err(e) = db::unabort(res)? {
+						error!("Unable to enqueue block ref deletion propagation: {}. A repair will be needed.", e);
+					}
+				}
+			}
+		}
 
 		Ok(())
 	}
@@ -175,42 +224,4 @@ impl TableSchema for VersionTable {
 	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
 		filter.apply(entry.deleted.get())
 	}
-
-	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
-		let old = rmp_serde::decode::from_read_ref::<_, old::Version>(bytes).ok()?;
-
-		let blocks = old
-			.blocks
-			.items()
-			.iter()
-			.map(|(k, v)| {
-				(
-					VersionBlockKey {
-						part_number: k.part_number,
-						offset: k.offset,
-					},
-					VersionBlock {
-						hash: Hash::try_from(v.hash.as_slice()).unwrap(),
-						size: v.size,
-					},
-				)
-			})
-			.collect::<crdt::Map<_, _>>();
-
-		let parts_etags = old
-			.parts_etags
-			.items()
-			.iter()
-			.map(|(k, v)| (*k, v.clone()))
-			.collect::<crdt::Map<_, _>>();
-
-		Some(Version {
-			uuid: Hash::try_from(old.uuid.as_slice()).unwrap(),
-			deleted: crdt::Bool::new(old.deleted.get()),
-			blocks,
-			parts_etags,
-			bucket_id: blake2sum(old.bucket.as_bytes()),
-			key: old.key,
-		})
-	}
 }

src/rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,7 +14,7 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_util = { version = "0.8.0", path = "../util" }
+garage_util = { version = "0.8.1", path = "../util" }
 
 arc-swap = "1.0"
 bytes = "1.0"
@@ -25,7 +25,6 @@ rand = "0.8"
 sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
 
 async-trait = "0.1.7"
-rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 serde_json = "1.0"

src/rpc/layout.rs

@@ -5,6 +5,7 @@ use serde::{Deserialize, Serialize};
 
 use garage_util::crdt::{AutoCrdt, Crdt, LwwMap};
 use garage_util::data::*;
+use garage_util::encode::nonversioned_encode;
 use garage_util::error::*;
 
 use crate::ring::*;
@@ -35,6 +36,8 @@ pub struct ClusterLayout {
 	pub staging_hash: Hash,
 }
 
+impl garage_util::migrate::InitialFormat for ClusterLayout {}
+
 #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
 pub struct NodeRoleV(pub Option<NodeRole>);
 
@@ -68,7 +71,7 @@ impl NodeRole {
 impl ClusterLayout {
 	pub fn new(replication_factor: usize) -> Self {
 		let empty_lwwmap = LwwMap::new();
-		let empty_lwwmap_hash = blake2sum(&rmp_to_vec_all_named(&empty_lwwmap).unwrap()[..]);
+		let empty_lwwmap_hash = blake2sum(&nonversioned_encode(&empty_lwwmap).unwrap()[..]);
 
 		ClusterLayout {
 			version: 0,
@@ -90,7 +93,7 @@ impl ClusterLayout {
 			Ordering::Equal => {
 				self.staging.merge(&other.staging);
 
-				let new_staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
+				let new_staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
 				let changed = new_staging_hash != self.staging_hash;
 
 				self.staging_hash = new_staging_hash;
@@ -125,7 +128,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		}
 
 		self.staging.clear();
-		self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
+		self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
 
 		self.version += 1;
 
@@ -149,7 +152,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		}
 
 		self.staging.clear();
-		self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
+		self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
 
 		self.version += 1;
 
@@ -178,7 +181,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 	/// returns true if consistent, false if error
 	pub fn check(&self) -> bool {
 		// Check that the hash of the staging data is correct
-		let staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
+		let staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
 		if staging_hash != self.staging_hash {
 			return false;
 		}
@@ -254,11 +257,13 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		match self.initial_partition_assignation() {
 			Some(initial_partitions) => {
 				for (part, ipart) in partitions.iter_mut().zip(initial_partitions.iter()) {
+					for _ in 0..2 {
 						for (id, info) in ipart.nodes.iter() {
 							if part.nodes.len() < self.replication_factor {
 								part.add(None, n_zones, id, info.unwrap());
 							}
 						}
+					}
 					assert!(part.nodes.len() == self.replication_factor);
 				}
 			}

src/rpc/lib.rs

@@ -9,6 +9,7 @@ mod consul;
 mod kubernetes;
 
 pub mod layout;
+pub mod replication_mode;
 pub mod ring;
 pub mod system;
 
@@ -16,3 +17,5 @@ mod metrics;
 pub mod rpc_helper;
 
 pub use rpc_helper::*;
+
+pub mod system_metrics;

src/rpc/replication_mode.rs

@@ -1,3 +1,4 @@
+#[derive(Clone, Copy)]
 pub enum ReplicationMode {
 	None,
 	TwoWay,

src/rpc/rpc_helper.rs

@@ -5,7 +5,6 @@ use std::time::Duration;
 use futures::future::join_all;
 use futures::stream::futures_unordered::FuturesUnordered;
 use futures::stream::StreamExt;
-use futures_util::future::FutureExt;
 use tokio::select;
 use tokio::sync::watch;
 
@@ -24,7 +23,6 @@ pub use netapp::message::{
 use netapp::peering::fullmesh::FullMeshPeeringStrategy;
 pub use netapp::{self, NetApp, NodeID};
 
-use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::Error;
 use garage_util::metrics::RecordDuration;
@@ -94,7 +92,6 @@ pub struct RpcHelper(Arc<RpcHelperInner>);
 struct RpcHelperInner {
 	our_node_id: Uuid,
 	fullmesh: Arc<FullMeshPeeringStrategy>,
-	background: Arc<BackgroundRunner>,
 	ring: watch::Receiver<Arc<Ring>>,
 	metrics: RpcMetrics,
 	rpc_timeout: Duration,
@@ -104,7 +101,6 @@ impl RpcHelper {
 	pub(crate) fn new(
 		our_node_id: Uuid,
 		fullmesh: Arc<FullMeshPeeringStrategy>,
-		background: Arc<BackgroundRunner>,
 		ring: watch::Receiver<Arc<Ring>>,
 		rpc_timeout: Option<Duration>,
 	) -> Self {
@@ -113,7 +109,6 @@ impl RpcHelper {
 		Self(Arc::new(RpcHelperInner {
 			our_node_id,
 			fullmesh,
-			background,
 			ring,
 			metrics,
 			rpc_timeout: rpc_timeout.unwrap_or(DEFAULT_TIMEOUT),
@@ -377,16 +372,13 @@ impl RpcHelper {
 
 			if !resp_stream.is_empty() {
 				// Continue remaining requests in background.
-				// Continue the remaining requests immediately using tokio::spawn
-				// but enqueue a task in the background runner
-				// to ensure that the process won't exit until the requests are done
-				// (if we had just enqueued the resp_stream.collect directly in the background runner,
-				// the requests might have been put on hold in the background runner's queue,
-				// in which case they might timeout or otherwise fail)
-				let wait_finished_fut = tokio::spawn(async move {
+				// Note: these requests can get interrupted on process shutdown,
+				// we must not count on them being executed for certain.
+				// For all background things that have to happen with certainty,
+				// they have to be put in a proper queue that is persisted to disk.
+				tokio::spawn(async move {
 					resp_stream.collect::<Vec<Result<_, _>>>().await;
 				});
-				self.0.background.spawn(wait_finished_fut.map(|_| Ok(())));
 			}
 		}
 

src/rpc/system.rs

@@ -21,7 +21,6 @@ use netapp::peering::fullmesh::FullMeshPeeringStrategy;
 use netapp::util::parse_and_resolve_peer_addr_async;
 use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
 
-use garage_util::background::BackgroundRunner;
 use garage_util::config::Config;
 #[cfg(feature = "kubernetes-discovery")]
 use garage_util::config::KubernetesDiscoveryConfig;
@@ -35,9 +34,13 @@ use crate::consul::ConsulDiscovery;
 #[cfg(feature = "kubernetes-discovery")]
 use crate::kubernetes::*;
 use crate::layout::*;
+use crate::replication_mode::*;
 use crate::ring::*;
 use crate::rpc_helper::*;
 
+#[cfg(feature = "metrics")]
+use crate::system_metrics::*;
+
 const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60);
 const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10);
 
@@ -49,8 +52,6 @@ pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650008; // garage 0x0008
 /// RPC endpoint used for calls related to membership
 pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc";
 
-pub const CONNECT_ERROR_MESSAGE: &str = "Error establishing RPC connection to remote node. This can happen if the remote node is not reachable on the network, but also if the two nodes are not configured with the same rpc_secret";
-
 /// RPC messages related to membership
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub enum SystemRpc {
@@ -75,13 +76,17 @@ impl Rpc for SystemRpc {
 	type Response = Result<SystemRpc, Error>;
 }
 
+#[derive(Serialize, Deserialize)]
+pub struct PeerList(Vec<(Uuid, SocketAddr)>);
+impl garage_util::migrate::InitialFormat for PeerList {}
+
 /// This node's membership manager
 pub struct System {
 	/// The id of this node
 	pub id: Uuid,
 
 	persist_cluster_layout: Persister<ClusterLayout>,
-	persist_peer_list: Persister<Vec<(Uuid, SocketAddr)>>,
+	persist_peer_list: Persister<PeerList>,
 
 	local_status: ArcSwap<NodeStatus>,
 	node_status: RwLock<HashMap<Uuid, (u64, NodeStatus)>>,
@@ -101,16 +106,16 @@ pub struct System {
 	consul_discovery: Option<ConsulDiscovery>,
 	#[cfg(feature = "kubernetes-discovery")]
 	kubernetes_discovery: Option<KubernetesDiscoveryConfig>,
+	#[cfg(feature = "metrics")]
+	metrics: SystemMetrics,
 
+	replication_mode: ReplicationMode,
 	replication_factor: usize,
 
 	/// The ring
 	pub ring: watch::Receiver<Arc<Ring>>,
 	update_ring: Mutex<watch::Sender<Arc<Ring>>>,
 
-	/// The job runner of this node
-	pub background: Arc<BackgroundRunner>,
-
 	/// Path to metadata directory
 	pub metadata_dir: PathBuf,
 }
@@ -136,6 +141,37 @@ pub struct KnownNodeInfo {
 	pub status: NodeStatus,
 }
 
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub struct ClusterHealth {
+	/// The current health status of the cluster (see below)
+	pub status: ClusterHealthStatus,
+	/// Number of nodes already seen once in the cluster
+	pub known_nodes: usize,
+	/// Number of nodes currently connected
+	pub connected_nodes: usize,
+	/// Number of storage nodes declared in the current layout
+	pub storage_nodes: usize,
+	/// Number of storage nodes currently connected
+	pub storage_nodes_ok: usize,
+	/// Number of partitions in the layout
+	pub partitions: usize,
+	/// Number of partitions for which we have a quorum of connected nodes
+	pub partitions_quorum: usize,
+	/// Number of partitions for which all storage nodes are connected
+	pub partitions_all_ok: usize,
+}
+
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+pub enum ClusterHealthStatus {
+	/// All nodes are available
+	Healthy,
+	/// Some storage nodes are unavailable, but quorum is stil
+	/// achieved for all partitions
+	Degraded,
+	/// Quorum is not available for some partitions
+	Unavailable,
+}
+
 pub fn read_node_id(metadata_dir: &Path) -> Result<NodeID, Error> {
 	let mut pubkey_file = metadata_dir.to_path_buf();
 	pubkey_file.push("node_key.pub");
@@ -199,10 +235,11 @@ impl System {
 	/// Create this node's membership manager
 	pub fn new(
 		network_key: NetworkKey,
-		background: Arc<BackgroundRunner>,
-		replication_factor: usize,
+		replication_mode: ReplicationMode,
 		config: &Config,
 	) -> Result<Arc<Self>, Error> {
+		let replication_factor = replication_mode.replication_factor();
+
 		let node_key =
 			gen_node_key(&config.metadata_dir).expect("Unable to read or generate node ID");
 		info!(
@@ -243,6 +280,9 @@ impl System {
 			cluster_layout_staging_hash: cluster_layout.staging_hash,
 		};
 
+		#[cfg(feature = "metrics")]
+		let metrics = SystemMetrics::new(replication_factor);
+
 		let ring = Ring::new(cluster_layout, replication_factor);
 		let (update_ring, ring) = watch::channel(Arc::new(ring));
 
@@ -319,11 +359,11 @@ impl System {
 			rpc: RpcHelper::new(
 				netapp.id.into(),
 				fullmesh,
-				background.clone(),
 				ring.clone(),
 				config.rpc_timeout_msec.map(Duration::from_millis),
 			),
 			system_endpoint,
+			replication_mode,
 			replication_factor,
 			rpc_listen_addr: config.rpc_bind_addr,
 			#[cfg(any(feature = "consul-discovery", feature = "kubernetes-discovery"))]
@@ -333,10 +373,11 @@ impl System {
 			consul_discovery,
 			#[cfg(feature = "kubernetes-discovery")]
 			kubernetes_discovery: config.kubernetes_discovery.clone(),
+			#[cfg(feature = "metrics")]
+			metrics,
 
 			ring,
 			update_ring: Mutex::new(update_ring),
-			background,
 			metadata_dir: config.metadata_dir.clone(),
 		});
 		sys.system_endpoint.set_handler(sys.clone());
@@ -408,17 +449,14 @@ impl System {
 				))
 			})?;
 		let mut errors = vec![];
-		for ip in addrs.iter() {
-			match self
-				.netapp
-				.clone()
-				.try_connect(*ip, pubkey)
-				.await
-				.err_context(CONNECT_ERROR_MESSAGE)
-			{
+		for addr in addrs.iter() {
+			match self.netapp.clone().try_connect(*addr, pubkey).await {
 				Ok(()) => return Ok(()),
 				Err(e) => {
-					errors.push((*ip, e));
+					errors.push((
+						*addr,
+						Error::Message(connect_error_message(*addr, pubkey, e)),
+					));
 				}
 			}
 		}
@@ -429,59 +467,125 @@ impl System {
 		}
 	}
 
+	pub fn health(&self) -> ClusterHealth {
+		let ring: Arc<_> = self.ring.borrow().clone();
+		let quorum = self.replication_mode.write_quorum();
+		let replication_factor = self.replication_factor;
+
+		let nodes = self
+			.get_known_nodes()
+			.into_iter()
+			.map(|n| (n.id, n))
+			.collect::<HashMap<Uuid, _>>();
+		let connected_nodes = nodes.iter().filter(|(_, n)| n.is_up).count();
+
+		let storage_nodes = ring
+			.layout
+			.roles
+			.items()
+			.iter()
+			.filter(|(_, _, v)| matches!(v, NodeRoleV(Some(r)) if r.capacity.is_some()))
+			.collect::<Vec<_>>();
+		let storage_nodes_ok = storage_nodes
+			.iter()
+			.filter(|(x, _, _)| nodes.get(x).map(|n| n.is_up).unwrap_or(false))
+			.count();
+
+		let partitions = ring.partitions();
+		let partitions_n_up = partitions
+			.iter()
+			.map(|(_, h)| {
+				let pn = ring.get_nodes(h, ring.replication_factor);
+				pn.iter()
+					.filter(|x| nodes.get(x).map(|n| n.is_up).unwrap_or(false))
+					.count()
+			})
+			.collect::<Vec<usize>>();
+		let partitions_all_ok = partitions_n_up
+			.iter()
+			.filter(|c| **c == replication_factor)
+			.count();
+		let partitions_quorum = partitions_n_up.iter().filter(|c| **c >= quorum).count();
+
+		let status =
+			if partitions_quorum == partitions.len() && storage_nodes_ok == storage_nodes.len() {
+				ClusterHealthStatus::Healthy
+			} else if partitions_quorum == partitions.len() {
+				ClusterHealthStatus::Degraded
+			} else {
+				ClusterHealthStatus::Unavailable
+			};
+
+		ClusterHealth {
+			status,
+			known_nodes: nodes.len(),
+			connected_nodes,
+			storage_nodes: storage_nodes.len(),
+			storage_nodes_ok,
+			partitions: partitions.len(),
+			partitions_quorum,
+			partitions_all_ok,
+		}
+	}
+
 	// ---- INTERNALS ----
 
 	#[cfg(feature = "consul-discovery")]
-	async fn advertise_to_consul(self: Arc<Self>) -> Result<(), Error> {
+	async fn advertise_to_consul(self: Arc<Self>) {
 		let c = match &self.consul_discovery {
 			Some(c) => c,
-			_ => return Ok(()),
+			_ => return,
 		};
 
 		let rpc_public_addr = match self.rpc_public_addr {
 			Some(addr) => addr,
 			None => {
 				warn!("Not advertising to Consul because rpc_public_addr is not defined in config file and could not be autodetected.");
-				return Ok(());
+				return;
 			}
 		};
 
-		c.publish_consul_service(
+		if let Err(e) = c
+			.publish_consul_service(
 				self.netapp.id,
 				&self.local_status.load_full().hostname,
 				rpc_public_addr,
 			)
 			.await
-		.err_context("Error while publishing Consul service")
+		{
+			error!("Error while publishing Consul service: {}", e);
+		}
 	}
 
 	#[cfg(feature = "kubernetes-discovery")]
-	async fn advertise_to_kubernetes(self: Arc<Self>) -> Result<(), Error> {
+	async fn advertise_to_kubernetes(self: Arc<Self>) {
 		let k = match &self.kubernetes_discovery {
 			Some(k) => k,
-			_ => return Ok(()),
+			_ => return,
 		};
 
 		let rpc_public_addr = match self.rpc_public_addr {
 			Some(addr) => addr,
 			None => {
 				warn!("Not advertising to Kubernetes because rpc_public_addr is not defined in config file and could not be autodetected.");
-				return Ok(());
+				return;
 			}
 		};
 
-		publish_kubernetes_node(
+		if let Err(e) = publish_kubernetes_node(
 			k,
 			self.netapp.id,
 			&self.local_status.load_full().hostname,
 			rpc_public_addr,
 		)
 		.await
-		.err_context("Error while publishing node to kubernetes")
+		{
+			error!("Error while publishing node to Kubernetes: {}", e);
+		}
 	}
 
 	/// Save network configuration to disc
-	async fn save_cluster_layout(self: Arc<Self>) -> Result<(), Error> {
+	async fn save_cluster_layout(&self) -> Result<(), Error> {
 		let ring: Arc<Ring> = self.ring.borrow().clone();
 		self.persist_cluster_layout
 			.save_async(&ring.layout)
@@ -533,11 +637,7 @@ impl System {
 		if info.cluster_layout_version > local_info.cluster_layout_version
 			|| info.cluster_layout_staging_hash != local_info.cluster_layout_staging_hash
 		{
-			let self2 = self.clone();
-			self.background.spawn_cancellable(async move {
-				self2.pull_cluster_layout(from).await;
-				Ok(())
-			});
+			tokio::spawn(self.clone().pull_cluster_layout(from));
 		}
 
 		self.node_status
@@ -579,18 +679,21 @@ impl System {
 			drop(update_ring);
 
 			let self2 = self.clone();
-			self.background.spawn_cancellable(async move {
-				self2
+			tokio::spawn(async move {
+				if let Err(e) = self2
 					.rpc
 					.broadcast(
 						&self2.system_endpoint,
 						SystemRpc::AdvertiseClusterLayout(layout),
 						RequestStrategy::with_priority(PRIO_HIGH),
 					)
-					.await?;
-				Ok(())
+					.await
+				{
+					warn!("Error while broadcasting new cluster layout: {}", e);
+				}
 			});
-			self.background.spawn(self.clone().save_cluster_layout());
+
+			self.save_cluster_layout().await?;
 		}
 
 		Ok(SystemRpc::Ok)
@@ -637,7 +740,7 @@ impl System {
 
 				// Add peer list from list stored on disk
 				if let Ok(peers) = self.persist_peer_list.load_async().await {
-					ping_list.extend(peers.iter().map(|(id, addr)| ((*id).into(), *addr)))
+					ping_list.extend(peers.0.iter().map(|(id, addr)| ((*id).into(), *addr)))
 				}
 
 				// Fetch peer list from Consul
@@ -676,12 +779,12 @@ impl System {
 				}
 
 				for (node_id, node_addr) in ping_list {
-					tokio::spawn(
-						self.netapp
-							.clone()
-							.try_connect(node_addr, node_id)
-							.map(|r| r.err_context(CONNECT_ERROR_MESSAGE)),
-					);
+					let self2 = self.clone();
+					tokio::spawn(async move {
+						if let Err(e) = self2.netapp.clone().try_connect(node_addr, node_id).await {
+							error!("{}", connect_error_message(node_addr, node_id, e));
+						}
+					});
 				}
 			}
 
@@ -690,11 +793,10 @@ impl System {
 			}
 
 			#[cfg(feature = "consul-discovery")]
-			self.background.spawn(self.clone().advertise_to_consul());
+			tokio::spawn(self.clone().advertise_to_consul());
 
 			#[cfg(feature = "kubernetes-discovery")]
-			self.background
-				.spawn(self.clone().advertise_to_kubernetes());
+			tokio::spawn(self.clone().advertise_to_kubernetes());
 
 			let restart_at = tokio::time::sleep(DISCOVERY_INTERVAL);
 			select! {
@@ -718,12 +820,16 @@ impl System {
 		// and append it to the list we are about to save,
 		// so that no peer ID gets lost in the process.
 		if let Ok(mut prev_peer_list) = self.persist_peer_list.load_async().await {
-			prev_peer_list.retain(|(id, _ip)| peer_list.iter().all(|(id2, _ip2)| id2 != id));
-			peer_list.extend(prev_peer_list);
+			prev_peer_list
+				.0
+				.retain(|(id, _ip)| peer_list.iter().all(|(id2, _ip2)| id2 != id));
+			peer_list.extend(prev_peer_list.0);
 		}
 
 		// Save new peer list to file
-		self.persist_peer_list.save_async(&peer_list).await
+		self.persist_peer_list
+			.save_async(&PeerList(peer_list))
+			.await
 	}
 
 	async fn pull_cluster_layout(self: Arc<Self>, peer: Uuid) {
@@ -784,3 +890,11 @@ async fn resolve_peers(peers: &[String]) -> Vec<(NodeID, SocketAddr)> {
 
 	ret
 }
+
+fn connect_error_message(
+	addr: SocketAddr,
+	pubkey: ed25519::PublicKey,
+	e: netapp::error::Error,
+) -> String {
+	format!("Error establishing RPC connection to remote node: {}@{}.\nThis can happen if the remote node is not reachable on the network, but also if the two nodes are not configured with the same rpc_secret.\n{}", hex::encode(pubkey), addr, e)
+}

src/rpc/system_metrics.rs

@@ -0,0 +1,33 @@
+use opentelemetry::{global, metrics::*, KeyValue};
+
+/// TableMetrics reference all counter used for metrics
+pub struct SystemMetrics {
+	pub(crate) _garage_build_info: ValueObserver<u64>,
+	pub(crate) _replication_factor: ValueObserver<u64>,
+}
+
+impl SystemMetrics {
+	pub fn new(replication_factor: usize) -> Self {
+		let meter = global::meter("garage_system");
+		Self {
+			_garage_build_info: meter
+				.u64_value_observer("garage_build_info", move |observer| {
+					observer.observe(
+						1,
+						&[KeyValue::new(
+							"version",
+							garage_util::version::garage_version(),
+						)],
+					)
+				})
+				.with_description("Garage build info")
+				.init(),
+			_replication_factor: meter
+				.u64_value_observer("garage_replication_factor", move |observer| {
+					observer.observe(replication_factor as u64, &[])
+				})
+				.with_description("Garage replication factor setting")
+				.init(),
+		}
+	}
+}

src/table/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_table"
-version = "0.8.0"
+version = "0.8.1"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,20 +14,20 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.0", path = "../db" }
-garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_util = { version = "0.8.0", path = "../util" }
+garage_db = { version = "0.8.1", path = "../db" }
+garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_util = { version = "0.8.1", path = "../util" }
 
 opentelemetry = "0.17"
 
 async-trait = "0.1.7"
+arc-swap = "1.0"
 bytes = "1.0"
 hex = "0.4"
 hexdump = "0.1"
 tracing = "0.1.30"
 rand = "0.8"
 
-rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"