Fix garage admin service

DaemonSet is a k8s resource that schedules one instance per node,
which is useful for some garage deployment use cases, including
managing garage nodes using k8s node labels

.envrc

@@ -1 +0,0 @@
-use flake

.gitattributes

@@ -0,0 +1 @@
+*.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -3,4 +3,3 @@
 /pki
 **/*.rs.bk
 *.swp
-/.direnv

Cargo.lock

@@ -1048,7 +1048,7 @@ dependencies = [
 
 [[package]]
 name = "garage"
-version = "0.8.1"
+version = "0.8.0"
 dependencies = [
  "assert-json-diff",
  "async-trait",
@@ -1080,6 +1080,7 @@ dependencies = [
  "parse_duration",
  "prometheus",
  "rand 0.8.5",
+ "rmp-serde",
  "serde",
  "serde_bytes",
  "serde_json",
@@ -1095,7 +1096,7 @@ dependencies = [
 
 [[package]]
 name = "garage_api"
-version = "0.8.1"
+version = "0.8.0"
 dependencies = [
  "async-trait",
  "base64",
@@ -1140,7 +1141,7 @@ dependencies = [
 
 [[package]]
 name = "garage_block"
-version = "0.8.1"
+version = "0.8.0"
 dependencies = [
  "arc-swap",
  "async-compression",
@@ -1155,6 +1156,7 @@ dependencies = [
  "hex",
  "opentelemetry",
  "rand 0.8.5",
+ "rmp-serde",
  "serde",
  "serde_bytes",
  "tokio",
@@ -1165,7 +1167,7 @@ dependencies = [
 
 [[package]]
 name = "garage_db"
-version = "0.8.1"
+version = "0.8.0"
 dependencies = [
  "clap 3.1.18",
  "err-derive",
@@ -1180,7 +1182,7 @@ dependencies = [
 
 [[package]]
 name = "garage_model"
-version = "0.8.1"
+version = "0.8.0"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1198,6 +1200,7 @@ dependencies = [
  "netapp",
  "opentelemetry",
  "rand 0.8.5",
+ "rmp-serde",
  "serde",
  "serde_bytes",
  "tokio",
@@ -1207,7 +1210,7 @@ dependencies = [
 
 [[package]]
 name = "garage_rpc"
-version = "0.8.1"
+version = "0.8.0"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1226,6 +1229,7 @@ dependencies = [
  "pnet_datalink",
  "rand 0.8.5",
  "reqwest",
+ "rmp-serde",
  "schemars",
  "serde",
  "serde_bytes",
@@ -1237,9 +1241,8 @@ dependencies = [
 
 [[package]]
 name = "garage_table"
-version = "0.8.1"
+version = "0.8.0"
 dependencies = [
- "arc-swap",
  "async-trait",
  "bytes",
  "futures",
@@ -1251,6 +1254,7 @@ dependencies = [
  "hexdump",
  "opentelemetry",
  "rand 0.8.5",
+ "rmp-serde",
  "serde",
  "serde_bytes",
  "tokio",
@@ -1259,7 +1263,7 @@ dependencies = [
 
 [[package]]
 name = "garage_util"
-version = "0.8.1"
+version = "0.8.0"
 dependencies = [
  "arc-swap",
  "async-trait",
@@ -1272,11 +1276,9 @@ dependencies = [
  "garage_db",
  "git-version",
  "hex",
- "hexdump",
  "http",
  "hyper",
  "lazy_static",
- "mktemp",
  "netapp",
  "opentelemetry",
  "rand 0.8.5",
@@ -1292,7 +1294,7 @@ dependencies = [
 
 [[package]]
 name = "garage_web"
-version = "0.8.1"
+version = "0.8.0"
 dependencies = [
  "err-derive",
  "futures",

Cargo.nix

@@ -32,7 +32,7 @@ args@{
   ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "8461dcfb984a8d042fecb5745d5da17912135dbf2a8ef7e6c3ae8e64c03d9744";
+  nixifiedLockHash = "90b29705f5037c7e1b33f4650841f1266f2e86fa03d5d0c87ad80be7619985c7";
   workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
   currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
   lockHashIgnored = if ignoreLockHash
@@ -56,15 +56,15 @@ in
 {
   cargo2nixVersion = "0.11.0";
   workspace = {
-    garage_db = rustPackages.unknown.garage_db."0.8.1";
+    garage_db = rustPackages.unknown.garage_db."0.8.0";
-    garage_util = rustPackages.unknown.garage_util."0.8.1";
+    garage_util = rustPackages.unknown.garage_util."0.8.0";
-    garage_rpc = rustPackages.unknown.garage_rpc."0.8.1";
+    garage_rpc = rustPackages.unknown.garage_rpc."0.8.0";
-    garage_table = rustPackages.unknown.garage_table."0.8.1";
+    garage_table = rustPackages.unknown.garage_table."0.8.0";
-    garage_block = rustPackages.unknown.garage_block."0.8.1";
+    garage_block = rustPackages.unknown.garage_block."0.8.0";
-    garage_model = rustPackages.unknown.garage_model."0.8.1";
+    garage_model = rustPackages.unknown.garage_model."0.8.0";
-    garage_api = rustPackages.unknown.garage_api."0.8.1";
+    garage_api = rustPackages.unknown.garage_api."0.8.0";
-    garage_web = rustPackages.unknown.garage_web."0.8.1";
+    garage_web = rustPackages.unknown.garage_web."0.8.0";
-    garage = rustPackages.unknown.garage."0.8.1";
+    garage = rustPackages.unknown.garage."0.8.0";
     k2v-client = rustPackages.unknown.k2v-client."0.0.1";
   };
   "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec {
@@ -946,20 +946,20 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"; };
     features = builtins.concatLists [
-      [ "alloc" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "alloc")
-      [ "default" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "default")
-      [ "lazy_static" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "lazy_static")
-      [ "std" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "std")
     ];
     dependencies = {
-      cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "cfg_if" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
-      crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.8" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "crossbeam_utils" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.8" { inherit profileName; }).out;
-      lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "lazy_static" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
-      memoffset = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memoffset."0.6.5" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "memoffset" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memoffset."0.6.5" { inherit profileName; }).out;
-      scopeguard = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.1.0" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "scopeguard" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".scopeguard."1.1.0" { inherit profileName; }).out;
     };
     buildDependencies = {
-      autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "autocfg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
     };
   });
   
@@ -995,7 +995,7 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "0bf124c720b7686e3c2663cf54062ab0f68a88af2fb6a030e87e30bf721fcb38"; };
     features = builtins.concatLists [
-      [ "default" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "default")
       [ "lazy_static" ]
       [ "std" ]
     ];
@@ -1321,8 +1321,8 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"; };
     dependencies = {
-      ${ if hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out;
+      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") && hostPlatform.isUnix then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out;
-      ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out;
+      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") && hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out;
     };
   });
   
@@ -1490,20 +1490,20 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"; };
     dependencies = {
-      byteorder = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "byteorder" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".byteorder."1.4.3" { inherit profileName; }).out;
     };
   });
   
-  "unknown".garage."0.8.1" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "garage";
-    version = "0.8.1";
+    version = "0.8.0";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/garage");
     features = builtins.concatLists [
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default") "bundled-libs")
       (lib.optional (rootFeatures' ? "garage/consul-discovery") "consul-discovery")
       (lib.optional (rootFeatures' ? "garage/default") "default")
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/k2v") "k2v")
       (lib.optional (rootFeatures' ? "garage/kubernetes-discovery") "kubernetes-discovery")
       (lib.optional (rootFeatures' ? "garage/lmdb") "lmdb")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics") "metrics")
@@ -1522,14 +1522,14 @@ in
       bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
-      garage_web = (rustPackages."unknown".garage_web."0.8.1" { inherit profileName; }).out;
+      garage_web = (rustPackages."unknown".garage_web."0.8.0" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
       netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
@@ -1539,6 +1539,7 @@ in
       parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
+      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
       structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out;
@@ -1562,13 +1563,13 @@ in
     };
   });
   
-  "unknown".garage_api."0.8.1" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_api."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "garage_api";
-    version = "0.8.1";
+    version = "0.8.0";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/api");
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics") "metrics")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/opentelemetry-prometheus") "opentelemetry-prometheus")
       (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics" || rootFeatures' ? "garage_api/prometheus") "prometheus")
@@ -1583,11 +1584,11 @@ in
       form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
@@ -1616,9 +1617,9 @@ in
     };
   });
   
-  "unknown".garage_block."0.8.1" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_block."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "garage_block";
-    version = "0.8.1";
+    version = "0.8.0";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/block");
     features = builtins.concatLists [
@@ -1631,13 +1632,14 @@ in
       bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
+      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
       tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
@@ -1647,21 +1649,20 @@ in
     };
   });
   
-  "unknown".garage_db."0.8.1" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_db."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "garage_db";
-    version = "0.8.1";
+    version = "0.8.0";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/db");
     features = builtins.concatLists [
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled-libs")
       (lib.optional (rootFeatures' ? "garage_db/clap" || rootFeatures' ? "garage_db/cli") "clap")
       (lib.optional (rootFeatures' ? "garage_db/cli") "cli")
-      [ "default" ]
       (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/heed" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "heed")
       (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_db/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb")
       (lib.optional (rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger") "pretty_env_logger")
       (lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "rusqlite")
-      [ "sled" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "sled")
       (lib.optional (rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite") "sqlite")
     ];
     dependencies = {
@@ -1671,7 +1672,7 @@ in
       hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage_db/cli" || rootFeatures' ? "garage_db/pretty_env_logger" then "pretty_env_logger" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_db/bundled-libs" || rootFeatures' ? "garage_db/rusqlite" || rootFeatures' ? "garage_db/sqlite" || rootFeatures' ? "garage_model/sqlite" then "rusqlite" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusqlite."0.27.0" { inherit profileName; }).out;
-      sled = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "sled" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }).out;
       tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }).out;
     };
     devDependencies = {
@@ -1679,16 +1680,15 @@ in
     };
   });
   
-  "unknown".garage_model."0.8.1" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_model."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "garage_model";
-    version = "0.8.1";
+    version = "0.8.0";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/model");
     features = builtins.concatLists [
-      [ "default" ]
+      (lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v")
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v")
       (lib.optional (rootFeatures' ? "garage/lmdb" || rootFeatures' ? "garage_model/lmdb") "lmdb")
-      [ "sled" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_model/sled") "sled")
       (lib.optional (rootFeatures' ? "garage/sqlite" || rootFeatures' ? "garage_model/sqlite") "sqlite")
     ];
     dependencies = {
@@ -1699,15 +1699,16 @@ in
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
+      garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
+      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
       tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
@@ -1716,9 +1717,9 @@ in
     };
   });
   
-  "unknown".garage_rpc."0.8.1" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_rpc."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "garage_rpc";
-    version = "0.8.1";
+    version = "0.8.0";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/rpc");
     features = builtins.concatLists [
@@ -1738,7 +1739,7 @@ in
       ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
       gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out;
@@ -1749,6 +1750,7 @@ in
       pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.12" { inherit profileName; }).out;
+      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.8" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
@@ -1759,24 +1761,24 @@ in
     };
   });
   
-  "unknown".garage_table."0.8.1" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_table."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "garage_table";
-    version = "0.8.1";
+    version = "0.8.0";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/table");
     dependencies = {
-      arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }).out;
       async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }).out;
       bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
-      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
+      garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
       hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
       rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
+      rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
       serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; }).out;
       serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }).out;
       tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }).out;
@@ -1784,13 +1786,13 @@ in
     };
   });
   
-  "unknown".garage_util."0.8.1" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_util."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "garage_util";
-    version = "0.8.1";
+    version = "0.8.0";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/util");
     features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v")
+      (lib.optional (rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v")
     ];
     dependencies = {
       arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }).out;
@@ -1801,10 +1803,9 @@ in
       digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; }).out;
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
-      garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
+      garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out;
       git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
       hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
-      hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
       hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
       lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
@@ -1820,23 +1821,20 @@ in
       tracing = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }).out;
       xxhash_rust = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.4" { inherit profileName; }).out;
     };
-    devDependencies = {
-      mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.4.1" { inherit profileName; }).out;
-    };
   });
   
-  "unknown".garage_web."0.8.1" = overridableMkRustCrate (profileName: rec {
+  "unknown".garage_web."0.8.0" = overridableMkRustCrate (profileName: rec {
     name = "garage_web";
-    version = "0.8.1";
+    version = "0.8.0";
     registry = "unknown";
     src = fetchCrateLocal (workspaceSrc + "/src/web");
     dependencies = {
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
-      garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
+      garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out;
-      garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
+      garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out;
-      garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
+      garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out;
-      garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
       hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
       opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@@ -2450,7 +2448,7 @@ in
     dependencies = {
       base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }).out;
       ${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }).out;
-      ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out;
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
       log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
       rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out;
@@ -2848,10 +2846,10 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"; };
     features = builtins.concatLists [
-      [ "default" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "default")
     ];
     buildDependencies = {
-      autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "autocfg" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
     };
   });
   
@@ -4734,18 +4732,18 @@ in
     registry = "registry+https://github.com/rust-lang/crates.io-index";
     src = fetchCratesIo { inherit name version; sha256 = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"; };
     features = builtins.concatLists [
-      [ "default" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "default")
-      [ "no_metrics" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "no_metrics")
     ];
     dependencies = {
-      crc32fast = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "crc32fast" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; }).out;
-      crossbeam_epoch = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.8" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "crossbeam_epoch" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.8" { inherit profileName; }).out;
-      crossbeam_utils = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.8" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "crossbeam_utils" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.8" { inherit profileName; }).out;
-      ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "windows" then "fs2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fs2."0.4.3" { inherit profileName; }).out;
+      ${ if (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") && (hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "darwin" || hostPlatform.parsed.kernel.name == "windows") then "fs2" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fs2."0.4.3" { inherit profileName; }).out;
-      fxhash = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fxhash."0.2.1" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "fxhash" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".fxhash."0.2.1" { inherit profileName; }).out;
-      libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out;
-      log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "log" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
-      parking_lot = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }).out;
+      ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled" then "parking_lot" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }).out;
     };
   });
   
@@ -5398,6 +5396,7 @@ in
       [ "attributes" ]
       [ "default" ]
       [ "log" ]
+      [ "log-always" ]
       [ "std" ]
       [ "tracing-attributes" ]
     ];
@@ -5891,7 +5890,7 @@ in
       [ "ntstatus" ]
       [ "objbase" ]
       [ "processenv" ]
-      [ "processthreadsapi" ]
+      (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sled" || rootFeatures' ? "garage_db/sled" || rootFeatures' ? "garage_model/sled") "processthreadsapi")
       [ "profileapi" ]
       [ "schannel" ]
       [ "securitybaseapi" ]

default.nix

@@ -11,14 +11,14 @@ let
 
   build_debug_and_release = (target: {
     debug = (compile {
-      inherit system target git_version pkgsSrc cargo2nixOverlay;
+      inherit target git_version;
       release = false;
     }).workspace.garage {
       compileMode = "build";
     };
 
     release = (compile {
-      inherit system target git_version pkgsSrc cargo2nixOverlay;
+      inherit target git_version;
       release = true;
     }).workspace.garage {
       compileMode = "build";
@@ -39,7 +39,7 @@ in {
   };
   test = {
     amd64 = test (compile {
-      inherit system git_version pkgsSrc cargo2nixOverlay;
+      inherit git_version;
       target = "x86_64-unknown-linux-musl";
       features = [
         "garage/bundled-libs"
@@ -52,7 +52,7 @@ in {
   };
   clippy = {
     amd64 = (compile {
-      inherit system git_version pkgsSrc cargo2nixOverlay;
+      inherit git_version;
       target = "x86_64-unknown-linux-musl";
       compiler = "clippy";
     }).workspace.garage {

doc/book/build/python.md

@@ -5,59 +5,16 @@ weight = 20
 
 ## S3
 
-### Using Minio SDK
-
-First install the SDK:
-
-```bash
-pip3 install minio
-```
-
-Then instantiate a client object using garage root domain, api key and secret:
-
-```python
-import minio
-
-client = minio.Minio(
-  "your.domain.tld",
-  "GKyourapikey",
-  "abcd[...]1234",
-  # Force the region, this is specific to garage
-  region="region",
-)
-```
-
-Then use all the standard S3 endpoints as implemented by the Minio SDK:
-
-```
-# List buckets
-print(client.list_buckets())
-
-# Put an object containing 'content' to /path in bucket named 'bucket':
-content = b"content"
-client.put_object(
-  "bucket",
-  "path",
-  io.BytesIO(content),
-  len(content),
-)
-
-# Read the object back and check contents
-data = client.get_object("bucket", "path").read()
-assert data == content
-```
-
-For further documentation, see the Minio SDK
-[Reference](https://docs.min.io/docs/python-client-api-reference.html)
-
-### Using Amazon boto3
-
 *Coming soon*
 
-See the official documentation:
+Some refs:
-  - [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
+  - Minio SDK
-  - [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
+    - [Reference](https://docs.min.io/docs/python-client-api-reference.html)
-  - [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
+
+  - Amazon boto3
+    - [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
+    - [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
+    - [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
 
 ## K2V
 

doc/book/connect/apps/index.md

@@ -8,7 +8,7 @@ In this section, we cover the following web applications:
 | Name | Status | Note |
 |------|--------|------|
 | [Nextcloud](#nextcloud)     | ✅       |  Both Primary Storage and External Storage are supported    |
-| [Peertube](#peertube)     | ✅       | Supported with the website endpoint, proxifying private videos unsupported     |
+| [Peertube](#peertube)     | ✅       | Must be configured with the website endpoint     |
 | [Mastodon](#mastodon)     | ✅       | Natively supported    |
 | [Matrix](#matrix)     | ✅       |  Tested with `synapse-s3-storage-provider`    |
 | [Pixelfed](#pixelfed)     | ❓       |  Not yet tested    |
@@ -128,10 +128,6 @@ In other words, Peertube is only responsible of the "control plane" and offload
 In return, this system is a bit harder to configure.
 We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
 
-Starting from version 5.0, Peertube also supports improving the security for private videos by not exposing them directly
-but relying on a single control point in the Peertube instance. This is based on S3 per-object and prefix ACL, which are not currently supported
-in Garage, so this feature is unsupported. While this technically impedes security for private videos, it is not a blocking issue and could be
-a reasonable trade-off for some instances.
 
 ### Create resources in Garage
 
@@ -199,11 +195,6 @@ object_storage:
 
   max_upload_part: 2GB
 
-  proxy:
-    # You may enable this feature, yet it will not provide any security benefit, so
-    # you should rather benefit from Garage public endpoint for all videos
-    proxify_private_files: false
-
   streaming_playlists:
     bucket_name: 'peertube-playlist'
 

doc/book/cookbook/kubernetes.md

@@ -47,8 +47,12 @@ garage:
   # Use only 2 replicas per object
   replicationMode: "2"
 
+  # Use recommended lmdb db engine
+  dbEngine: "lmdb"
+
 # Start 4 instances (StatefulSets) of garage
-replicaCount: 4
+deployment:
+  replicaCount: 4
 
 # Override default storage class and size
 persistence:

doc/book/cookbook/real-world.md

@@ -109,7 +109,7 @@ especially you must consider the following folders/files:
   this folder will be your main data storage and must be on a large storage (e.g. large HDD)
 
 
-A valid `/etc/garage.toml` for our cluster would look as follows:
+A valid `/etc/garage/garage.toml` for our cluster would look as follows:
 
 ```toml
 metadata_dir = "/var/lib/garage/meta"

doc/book/design/goals.md

@@ -12,7 +12,7 @@ as pictures, video, images, documents, etc., in a redundant multi-node
 setting. S3 is versatile enough to also be used to publish a static
 website.
 
-Garage is an opinionated object storage solution, we focus on the following **desirable properties**:
+Garage is an opinionated object storage solutoin, we focus on the following **desirable properties**:
 
   - **Internet enabled**: made for multi-sites (eg. datacenters, offices, households, etc.) interconnected through regular Internet connections.
   - **Self-contained & lightweight**: works everywhere and integrates well in existing environments to target [hyperconverged infrastructures](https://en.wikipedia.org/wiki/Hyper-converged_infrastructure).

doc/book/development/devenv.md

@@ -39,7 +39,7 @@ Now you can enter our nix-shell, all the required packages will be downloaded bu
 nix-shell
 ```
 
-You can use the traditional Rust development workflow:
+You can use the traditionnal Rust development workflow:
 
 ```bash
 cargo build  # compile the project

doc/book/reference-manual/configuration.md

@@ -96,7 +96,7 @@ Performance characteristics of the different DB engines are as follows:
 
 - Sled: the default database engine, which tends to produce
   large data files and also has performance issues, especially when the metadata folder
-  is on a traditional HDD and not on SSD.
+  is on a traditionnal HDD and not on SSD.
 - LMDB: the recommended alternative on 64-bit systems,
   much more space-efficiant and slightly faster. Note that the data format of LMDB is not portable
   between architectures, so for instance the Garage database of an x86-64
@@ -267,10 +267,6 @@ This key should be specified here in the form of a 32-byte hex-encoded
 random string. Such a string can be generated with a command
 such as `openssl rand -hex 32`.
 
-### `rpc_secret_file`
-
-Like `rpc_secret` above, just that this is the path to a file that Garage will try to read the secret from.
-
 ### `rpc_bind_addr`
 
 The address and port on which to bind for inter-cluster communcations

doc/book/reference-manual/features.md

@@ -83,7 +83,7 @@ This feature is totally invisible to S3 clients and does not break compatibility
 ### Cluster administration API
 
 Garage provides a fully-fledged REST API to administer your cluster programatically.
-Functionality included in the admin API include: setting up and monitoring
+Functionnality included in the admin API include: setting up and monitoring
 cluster nodes, managing access credentials, and managing storage buckets and bucket aliases.
 A full reference of the administration API is available [here](@/documentation/reference-manual/admin-api.md).
 

doc/drafts/admin-api.md

@@ -1,686 +0,0 @@
-+++
-title = "Administration API"
-weight = 60
-+++
-
-The Garage administration API is accessible through a dedicated server whose
-listen address is specified in the `[admin]` section of the configuration
-file (see [configuration file
-reference](@/documentation/reference-manual/configuration.md))
-
-**WARNING.** At this point, there is no comittement to stability of the APIs described in this document.
-We will bump the version numbers prefixed to each API endpoint at each time the syntax
-or semantics change, meaning that code that relies on these endpoint will break
-when changes are introduced.
-
-The Garage administration API was introduced in version 0.7.2, this document
-does not apply to older versions of Garage.
-
-
-## Access control
-
-The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section:
-
-- `metrics_token`: the token for accessing the Metrics endpoint (if this token
-  is not set in the config file, the Metrics endpoint can be accessed without
-  access control);
-
-- `admin_token`: the token for accessing all of the other administration
-  endpoints (if this token is not set in the config file, access to these
-  endpoints is disabled entirely).
-
-These tokens are used as simple HTTP bearer tokens. In other words, to
-authenticate access to an admin API endpoint, add the following HTTP header
-to your request:
-
-```
-Authorization: Bearer <token>
-```
-
-## Administration API endpoints
-
-### Metrics-related endpoints
-
-#### Metrics `GET /metrics`
-
-Returns internal Garage metrics in Prometheus format.
-
-#### Health `GET /health`
-
-Used for simple health checks in a cluster setting with an orchestrator.
-Returns an HTTP status 200 if the node is ready to answer user's requests,
-and an HTTP status 503 (Service Unavailable) if there are some partitions
-for which a quorum of nodes is not available.
-A simple textual message is also returned in a body with content-type `text/plain`.
-See `/v0/health` for an API that also returns JSON output.
-
-### Cluster operations
-
-#### GetClusterStatus `GET /v0/status`
-
-Returns the cluster's current status in JSON, including:
-
-- ID of the node being queried and its version of the Garage daemon
-- Live nodes
-- Currently configured cluster layout
-- Staged changes to the cluster layout
-
-Example response body:
-
-```json
-{
-  "node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f",
-  "garage_version": "git:v0.8.0",
-  "knownNodes": {
-    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
-      "addr": "10.0.0.11:3901",
-      "is_up": true,
-      "last_seen_secs_ago": 9,
-      "hostname": "node1"
-    },
-    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
-      "addr": "10.0.0.12:3901",
-      "is_up": true,
-      "last_seen_secs_ago": 1,
-      "hostname": "node2"
-    },
-    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
-      "addr": "10.0.0.21:3901",
-      "is_up": true,
-      "last_seen_secs_ago": 7,
-      "hostname": "node3"
-    },
-    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
-      "addr": "10.0.0.22:3901",
-      "is_up": true,
-      "last_seen_secs_ago": 1,
-      "hostname": "node4"
-    }
-  },
-  "layout": {
-    "version": 12,
-    "roles": {
-      "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
-        "zone": "dc1",
-        "capacity": 4,
-        "tags": [
-          "node1"
-        ]
-      },
-      "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
-        "zone": "dc1",
-        "capacity": 6,
-        "tags": [
-          "node2"
-        ]
-      },
-      "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
-        "zone": "dc2",
-        "capacity": 10,
-        "tags": [
-          "node3"
-        ]
-      }
-    },
-    "stagedRoleChanges": {
-      "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
-        "zone": "dc2",
-        "capacity": 5,
-        "tags": [
-          "node4"
-        ]
-      }
-    }
-  }
-}
-```
-
-#### GetClusterHealth `GET /v0/health`
-
-Returns the cluster's current health in JSON format, with the following variables:
-
-- `status`: one of `Healthy`, `Degraded` or `Unavailable`:
-  - Healthy: Garage node is connected to all storage nodes
-  - Degraded: Garage node is not connected to all storage nodes, but a quorum of write nodes is available for all partitions
-  - Unavailable: a quorum of write nodes is not available for some partitions
-- `known_nodes`: the number of nodes this Garage node has had a TCP connection to since the daemon started
-- `connected_nodes`: the nubmer of nodes this Garage node currently has an open connection to
-- `storage_nodes`: the number of storage nodes currently registered in the cluster layout
-- `storage_nodes_ok`: the number of storage nodes to which a connection is currently open
-- `partitions`: the total number of partitions of the data (currently always 256)
-- `partitions_quorum`: the number of partitions for which a quorum of write nodes is available
-- `partitions_all_ok`: the number of partitions for which we are connected to all storage nodes responsible of storing it
-
-Contrarily to `GET /health`, this endpoint always returns a 200 OK HTTP response code.
-
-Example response body:
-
-```json
-{
-    "status": "Degraded",
-    "known_nodes": 3,
-    "connected_nodes": 2,
-    "storage_nodes": 3,
-    "storage_nodes_ok": 2,
-    "partitions": 256,
-    "partitions_quorum": 256,
-    "partitions_all_ok": 0
-}
-```
-
-#### ConnectClusterNodes `POST /v0/connect`
-
-Instructs this Garage node to connect to other Garage nodes at specified addresses.
-
-Example request body:
-
-```json
-[
-    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901",
-    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901"
-]
-```
-
-The format of the string for a node to connect to is: `<node ID>@<ip address>:<port>`, same as in the `garage node connect` CLI call.
-
-Example response:
-
-```json
-[
-    {
-        "success": true,
-        "error": null
-    },
-    {
-        "success": false,
-        "error": "Handshake error"
-    }
-]
-```
-
-#### GetClusterLayout `GET /v0/layout`
-
-Returns the cluster's current layout in JSON, including:
-
-- Currently configured cluster layout
-- Staged changes to the cluster layout
-
-(the info returned by this endpoint is a subset of the info returned by GetClusterStatus)
-
-Example response body:
-
-```json
-{
-  "version": 12,
-  "roles": {
-    "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
-      "zone": "dc1",
-      "capacity": 4,
-      "tags": [
-        "node1"
-      ]
-    },
-    "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
-      "zone": "dc1",
-      "capacity": 6,
-      "tags": [
-        "node2"
-      ]
-    },
-    "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
-      "zone": "dc2",
-      "capacity": 10,
-      "tags": [
-        "node3"
-      ]
-    }
-  },
-  "stagedRoleChanges": {
-    "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
-      "zone": "dc2",
-      "capacity": 5,
-      "tags": [
-        "node4"
-      ]
-    }
-  }
-}
-```
-
-#### UpdateClusterLayout `POST /v0/layout`
-
-Send modifications to the cluster layout. These modifications will
-be included in the staged role changes, visible in subsequent calls
-of `GetClusterLayout`. Once the set of staged changes is satisfactory,
-the user may call `ApplyClusterLayout` to apply the changed changes,
-or `Revert ClusterLayout` to clear all of the staged changes in
-the layout.
-
-Request body format:
-
-```json
-{
-  <node_id>: {
-    "capacity": <new_capacity>,
-    "zone": <new_zone>,
-    "tags": [
-      <new_tag>,
-      ...
-    ]
-  },
-  <node_id_to_remove>: null,
-  ...
-}
-```
-
-Contrary to the CLI that may update only a subset of the fields
-`capacity`, `zone` and `tags`, when calling this API all of these
-values must be specified.
-
-
-#### ApplyClusterLayout `POST /v0/layout/apply`
-
-Applies to the cluster the layout changes currently registered as
-staged layout changes.
-
-Request body format:
-
-```json
-{
-  "version": 13
-}
-```
-
-Similarly to the CLI, the body must include the version of the new layout
-that will be created, which MUST be 1 + the value of the currently
-existing layout in the cluster.
-
-#### RevertClusterLayout `POST /v0/layout/revert`
-
-Clears all of the staged layout changes.
-
-Request body format:
-
-```json
-{
-  "version": 13
-}
-```
-
-Reverting the staged changes is done by incrementing the version number
-and clearing the contents of the staged change list.
-Similarly to the CLI, the body must include the incremented
-version number, which MUST be 1 + the value of the currently
-existing layout in the cluster.
-
-
-### Access key operations
-
-#### ListKeys `GET /v0/key`
-
-Returns all API access keys in the cluster.
-
-Example response:
-
-```json
-[
-  {
-    "id": "GK31c2f218a2e44f485b94239e",
-    "name": "test"
-  },
-  {
-    "id": "GKe10061ac9c2921f09e4c5540",
-    "name": "test2"
-  }
-]
-```
-
-#### CreateKey `POST /v0/key`
-
-Creates a new API access key.
-
-Request body format:
-
-```json
-{
-    "name": "NameOfMyKey"
-}
-```
-
-#### ImportKey `POST /v0/key/import`
-
-Imports an existing API key.
-
-Request body format:
-
-```json
-{
-    "accessKeyId": "GK31c2f218a2e44f485b94239e",
-    "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
-    "name": "NameOfMyKey"
-}
-```
-
-#### GetKeyInfo `GET /v0/key?id=<acces key id>`
-#### GetKeyInfo `GET /v0/key?search=<pattern>`
-
-Returns information about the requested API access key.
-
-If `id` is set, the key is looked up using its exact identifier (faster).
-If `search` is set, the key is looked up using its name or prefix
-of identifier (slower, all keys are enumerated to do this).
-
-Example response:
-
-```json
-{
-  "name": "test",
-  "accessKeyId": "GK31c2f218a2e44f485b94239e",
-  "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
-  "permissions": {
-    "createBucket": false
-  },
-  "buckets": [
-    {
-      "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
-      "globalAliases": [
-        "test2"
-      ],
-      "localAliases": [],
-      "permissions": {
-        "read": true,
-        "write": true,
-        "owner": false
-      }
-    },
-    {
-      "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
-      "globalAliases": [
-        "test3"
-      ],
-      "localAliases": [],
-      "permissions": {
-        "read": true,
-        "write": true,
-        "owner": false
-      }
-    },
-    {
-      "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
-      "globalAliases": [],
-      "localAliases": [
-        "test"
-      ],
-      "permissions": {
-        "read": true,
-        "write": true,
-        "owner": true
-      }
-    },
-    {
-      "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
-      "globalAliases": [
-        "alex"
-      ],
-      "localAliases": [],
-      "permissions": {
-        "read": true,
-        "write": true,
-        "owner": true
-      }
-    }
-  ]
-}
-```
-
-#### DeleteKey `DELETE /v0/key?id=<acces key id>`
-
-Deletes an API access key.
-
-#### UpdateKey `POST /v0/key?id=<acces key id>`
-
-Updates information about the specified API access key.
-
-Request body format:
-
-```json
-{
-    "name": "NameOfMyKey",
-    "allow": {
-        "createBucket": true,
-    },
-    "deny": {}
-}
-```
-
-All fields (`name`, `allow` and `deny`) are optionnal.
-If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed.
-The possible flags in `allow` and `deny` are: `createBucket`.
-
-
-### Bucket operations
-
-#### ListBuckets `GET /v0/bucket`
-
-Returns all storage buckets in the cluster.
-
-Example response:
-
-```json
-[
-  {
-    "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
-    "globalAliases": [
-      "test2"
-    ],
-    "localAliases": []
-  },
-  {
-    "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
-    "globalAliases": [
-      "alex"
-    ],
-    "localAliases": []
-  },
-  {
-    "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
-    "globalAliases": [
-      "test3"
-    ],
-    "localAliases": []
-  },
-  {
-    "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
-    "globalAliases": [],
-    "localAliases": [
-      {
-        "accessKeyId": "GK31c2f218a2e44f485b94239e",
-        "alias": "test"
-      }
-    ]
-  }
-]
-```
-
-#### GetBucketInfo `GET /v0/bucket?id=<bucket id>`
-#### GetBucketInfo `GET /v0/bucket?globalAlias=<alias>`
-
-Returns information about the requested storage bucket.
-
-If `id` is set, the bucket is looked up using its exact identifier.
-If `globalAlias` is set, the bucket is looked up using its global alias.
-(both are fast)
-
-Example response:
-
-```json
-{
-    "id": "afa8f0a22b40b1247ccd0affb869b0af5cff980924a20e4b5e0720a44deb8d39",
-        "globalAliases": [],
-        "websiteAccess": false,
-        "websiteConfig": null,
-        "keys": [
-        {
-            "accessKeyId": "GK31c2f218a2e44f485b94239e",
-            "name": "Imported key",
-            "permissions": {
-                "read": true,
-                "write": true,
-                "owner": true
-            },
-            "bucketLocalAliases": [
-                "debug"
-            ]
-        }
-        ],
-        "objects": 14827,
-        "bytes": 13189855625,
-        "unfinshedUploads": 0,
-        "quotas": {
-            "maxSize": null,
-            "maxObjects": null
-        }
-}
-```
-
-#### CreateBucket `POST /v0/bucket`
-
-Creates a new storage bucket.
-
-Request body format:
-
-```json
-{
-    "globalAlias": "NameOfMyBucket"
-}
-```
-
-OR
-
-```json
-{
-    "localAlias": {
-        "accessKeyId": "GK31c2f218a2e44f485b94239e",
-        "alias": "NameOfMyBucket",
-        "allow": {
-            "read": true,
-            "write": true,
-            "owner": false
-        }
-    }
-}
-```
-
-OR
-
-```json
-{}
-```
-
-Creates a new bucket, either with a global alias, a local one,
-or no alias at all.
-
-Technically, you can also specify both `globalAlias` and `localAlias` and that would create
-two aliases, but I don't see why you would want to do that.
-
-#### DeleteBucket `DELETE /v0/bucket?id=<bucket id>`
-
-Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
-
-Warning: this will delete all aliases associated with the bucket!
-
-#### UpdateBucket `PUT /v0/bucket?id=<bucket id>`
-
-Updates configuration of the given bucket.
-
-Request body format:
-
-```json
-{
-    "websiteAccess": {
-        "enabled": true,
-        "indexDocument": "index.html",
-        "errorDocument": "404.html"
-    },
-    "quotas": {
-        "maxSize": 19029801,
-        "maxObjects": null,
-    }
-}
-```
-
-All fields (`websiteAccess` and `quotas`) are optionnal.
-If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
-
-In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.
-The field `errorDocument` is optional, if no error document is set a generic
-error message is displayed when errors happen. Conversely, if `enabled` is
-`false`, neither `indexDocument` nor `errorDocument` must be specified.
-
-In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null`
-to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
-to change only one of the two quotas.
-
-### Operations on permissions for keys on buckets
-
-#### BucketAllowKey `POST /v0/bucket/allow`
-
-Allows a key to do read/write/owner operations on a bucket.
-
-Request body format:
-
-```json
-{
-    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
-    "accessKeyId": "GK31c2f218a2e44f485b94239e",
-    "permissions": {
-        "read": true,
-        "write": true,
-        "owner": true
-    },
-}
-```
-
-Flags in `permissions` which have the value `true` will be activated.
-Other flags will remain unchanged.
-
-#### BucketDenyKey `POST /v0/bucket/deny`
-
-Denies a key from doing read/write/owner operations on a bucket.
-
-Request body format:
-
-```json
-{
-    "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
-    "accessKeyId": "GK31c2f218a2e44f485b94239e",
-    "permissions": {
-        "read": false,
-        "write": false,
-        "owner": true
-    },
-}
-```
-
-Flags in `permissions` which have the value `true` will be deactivated.
-Other flags will remain unchanged.
-
-
-### Operations on bucket aliases
-
-#### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
-
-Empty body. Creates a global alias for a bucket.
-
-#### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
-
-Removes a global alias for a bucket.
-
-#### LocalAliasBucket `PUT /v0/bucket/alias/local?id=<bucket id>&accessKeyId=<access key ID>&alias=<local alias>`
-
-Empty body. Creates a local alias for a bucket in the namespace of a specific access key.
-
-#### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=<bucket id>&accessKeyId<access key ID>&alias=<local alias>`
-
-Removes a local alias for a bucket in the namespace of a specific access key.
-

doc/talks/2022-11-19-Capitole-du-Libre/.gitignore

@@ -1,10 +0,0 @@
-*.aux
-*.bbl
-*.blg
-*.log
-*.nav
-*.out
-*.snm
-*.synctex.gz
-*.toc
-*.dvi

doc/talks/2022-11-19-Capitole-du-Libre/Makefile

@@ -1,8 +0,0 @@
-all:
-	pdflatex présentation.tex
-
-clean:
-	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi présentation.pdf
-
-clean_sauf_pdf:
-	rm -f *.aux *.bbl *.blg *.log *.nav *.out *.snm *.synctex.gz *.toc *.dvi

doc/talks/2022-11-19-Capitole-du-Libre/présentation.tex

@@ -1,340 +0,0 @@
-\documentclass[11pt, aspectratio=1610]{beamer}
-\usetheme{Warsaw}
-\usepackage[utf8]{inputenc}
-\usepackage[french]{babel}
-\usepackage{amsmath}
-\usepackage{amsfonts}
-\usepackage{amssymb}
-\usepackage{tikz}
-\usepackage{graphicx}
-\usepackage{xcolor}
-\usepackage{setspace}
-\usepackage{todonotes}
-\presetkeys{todonotes}{inline}{}
-\renewcommand{\baselinestretch}{1.25}
-
-\definecolor{orange_garage}{RGB}{255,147,41}
-\definecolor{gris_garage}{RGB}{78,78,78}
-
-\author[Association Deuxfleurs]{~\linebreak Vincent Giraud}
-\title[De l'auto-hébergement à l'entre-hébergement avec Garage]{De l'auto-hébergement à l'entre-hébergement :\\Garage, pour conserver ses données ensemble}
-%\setbeamercovered{transparent} 
-%\setbeamertemplate{navigation symbols}{} 
-\date{Capitole du Libre 2022\linebreak
-
-\scriptsize Samedi 19 novembre 2022\linebreak
-} 
-
-\setbeamercolor{palette primary}{fg=gris_garage,bg=orange_garage}
-\setbeamercolor{palette secondary}{fg=gris_garage,bg=gris_garage}
-\setbeamercolor{palette tiertary}{fg=white,bg=gris_garage}
-\setbeamercolor{palette quaternary}{fg=white,bg=gris_garage}
-\setbeamercolor{navigation symbols}{fg=black, bg=white}
-\setbeamercolor{navigation symbols dimmed}{fg=darkgray, bg=white}
-\setbeamercolor{itemize item}{fg=gris_garage}
-\setbeamertemplate{itemize item}[circle]
-
-\addtobeamertemplate{navigation symbols}{}{%
-    \usebeamerfont{footline}%
-    \usebeamercolor[fg]{footline}%
-    \hspace{1em}%
-    \insertframenumber/\inserttotalframenumber
-}
-
-\setbeamertemplate{headline}
-{%
-  \leavevmode%
-  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{section in head/foot}%
-    \hbox to .5\paperwidth{\hfil\insertsectionhead\hfil}
-  \end{beamercolorbox}%
-  \begin{beamercolorbox}[wd=.5\paperwidth,ht=2.5ex,dp=1.125ex]{subsection in head/foot}%
-    \hbox to .5\paperwidth{\hfil\insertsubsectionhead\hfil}
-  \end{beamercolorbox}%
-}
-\addtobeamertemplate{footnote}{}{\vspace{2ex}}
-
-\begin{document}
-\begin{frame}
-\titlepage
-\end{frame}
-
-\section{Introduction}
-\subsection{Présentation}
-\begin{frame}
-\begin{columns}
-\column{0.5 \linewidth}
-\begin{center}
-\includegraphics[width=3.5cm]{deuxfleurs-logo.png}\linebreak
-
-\texttt{https://deuxfleurs.fr}
-\end{center}
-\column{0.4 \linewidth}
-\begin{center}
-Deuxfleurs est une association militant en faveur d'un internet plus convivial, avec une organisation et des rapports de force repensés.\linebreak
-
-Nous faisons partie du CHATONS\footnote[frame]{Collectif des Hébergeurs Alternatifs, Transparents, Ouverts, Neutres et Solidaires} depuis avril 2022.
-
-\includegraphics[width=2cm]{logo_chatons.png}
-\end{center}
-\end{columns}
-\end{frame}
-
-\subsection{Héberger à la maison}
-\begin{frame}
-\begin{columns}
-\begin{column}{0.5 \linewidth}
-\begin{center}
-Pour échapper au contrôle et au giron des opérateurs de clouds, héberger ses données à la maison présente de nombreux avantages...
-\end{center}
-
-\vspace{0.5cm}
-
-\begin{itemize}[<+(1)->]
-\item On récupère la souveraineté sur ses données
-\item On gagne en vie privée
-\item On gagne en libertés
-\item On est responsabilisé face à ses besoins
-\end{itemize}
-\end{column}
-\vrule{}
-\begin{column}{0.5 \linewidth}
-\begin{center}
-\onslide<6->{... mais aussi bien des contraintes...}
-\end{center}
-
-\vspace{0.5cm}
-
-\begin{itemize}[<+(2)->]
-\item On repose sur une connexion internet pour particulier
-\item Un certain savoir-faire et moultes compétences sont requis
-\item Assurer la résilience de ses services est difficile
-\item Bien sauvegarder ses données, et ceci au-delà de son site géographique, n'est pas évident
-\end{itemize}
-\end{column}
-\end{columns}
-\end{frame}
-
-\subsection{Sauvegarder pour se parer à tout imprévu}
-\begin{frame}
-\begin{center}
-Sauvegarder pour se parer contre les pannes matérielles est une chose...
-
-Sauvegarder pour se parer contre les cambriolages et les incendies en est une autre !\linebreak
-
-\vspace{1cm}
-\onslide<2->{Répartir géographiquement ses données devient alors nécessaire.}
-\end{center}
-\end{frame}
-
-\section{Les solutions à explorer}
-\subsection{L'entre-hébergement}
-\begin{frame}
-\begin{center}
-On a vu récemment se développer au sein du CHATONS la notion d'entre-hébergement : en plus de renforcer l'intégrité des sauvegardes, on va améliorer la disponibilité pendant les coupures de liaison internet, de courant, ou pendant les déménagements d'administrateurs par exemple.\linebreak
-
-\vspace{1cm}
-\onslide<2->
-{
-Dans le cadre du collectif, il s'agit de partager ses volumes de données entre hébergeurs.\linebreak
-
-Pour assurer la confidentialité, on peut chiffrer les données au niveau applicatif.
-}
-\end{center}
-\end{frame}
-
-\subsection{S3 contre les systèmes de fichiers}
-\begin{frame}
-\begin{center}
-Dans le cadre de l'administration de services en ligne, les systèmes de fichiers recèlent certaines difficultés.\linebreak
-
-\vspace{1cm}
-Le standard S3 apporte des facilités; on réduit le stockage à un paradigme de clé-valeur basé essentiellement sur deux opérations seulement: lire ou écrire une clé.
-\end{center}
-\end{frame}
-
-\section{Garage}
-\subsection{Présentation}
-\begin{frame}
-\begin{columns}
-\column{0.5 \linewidth}
-\begin{center}
-Garage essaye de répondre à l'ensemble de ces besoins.\linebreak
-
-\vspace{0.5cm}
-Il s'agit d'un logiciel libre permettant de distribuer un service S3 sur diverses machines éloignées.
-\end{center}
-\column{0.5 \linewidth}
-\begin{center}
-\includegraphics[width=4cm]{garage-logo.png}\linebreak
-
-\texttt{https://garagehq.deuxfleurs.fr/}
-\end{center}
-\end{columns}
-\end{frame}
-
-\subsection{Gestion des zones}
-\begin{frame}
-\begin{center}
-Garage va prendre en compte les zones géographiques au moment de répliquer les données.\linebreak
-
-\vspace{1cm}
-\includegraphics[width=13.25cm]{zones.png}
-\end{center}
-\end{frame}
-
-\subsection{Comment ça marche ?}
-\begin{frame}
-\begin{columns}
-\column{0.5 \linewidth}
-\input{schéma europe}
-\column{0.5 \linewidth}
-\begin{center}
-Chaque objet est dupliqué sur plusieurs zones différentes.\linebreak
-
-\onslide<5->{Lorsqu'un nouvel hébergeur rejoint le réseau, la charge se voit équilibrée.}\linebreak
-
-\onslide<12->{Si une zone devient indisponible, les autres continuent d'assurer le service.}\linebreak
-\end{center}
-\end{columns}
-\end{frame}
-
-\subsection{Financement}
-\begin{frame}
-\begin{center}
-Dans le cadre du programme \textit{Horizon 2021} de l'Union Européenne, nous avons reçu une subvention de la part de l'initiative NGI Pointer\footnote[frame]{Next Generation Internet Program for Open Internet Renovation}.\linebreak
-
-\includegraphics[width=3cm]{drapeau_européen.png}\hspace{1cm}
-\includegraphics[width=3cm]{NGI.png}\linebreak
-
-Nous avons ainsi pu financer le développement de Garage pendant 1 an.
-\end{center}
-\end{frame}
-
-\subsection{Licence}
-\begin{frame}
-\begin{center}
-De par nos valeurs, nous avons attribué la licence AGPL version 3 à Garage, notamment afin qu'il reste parmi les biens communs.\linebreak
-
-\vspace{0.5cm}
-\includegraphics[width=5cm]{agpl-v3-logo.png}\linebreak
-\end{center}
-\end{frame}
-
-\subsection{Langage utilisé}
-\begin{frame}
-\begin{center}
-Nous avons décidé d'écrire Garage à l'aide du langage Rust, afin d'obtenir une compilation vers des binaires natifs et efficaces.\linebreak
-
-\includegraphics[width=3.5cm]{rust-logo.png}\linebreak
-
-Ce choix permet également de bénéficier des avantages reconnus de Rust en termes de sécurité.
-\end{center}
-\end{frame}
-
-\subsection{Matériel requis}
-\begin{frame}
-\begin{center}
-Garage peut ainsi être performant sur des machines limitées. Les prérequis sont minimes : n'importe quelle machine avec un processeur qui a moins d'une décennie, 1~gigaoctet de mémoire vive, et 16~gigaoctets de stockage suffit.\linebreak
-
-\vspace{1cm}
-
-Cet aspect est déterminant : il permet en effet d'héberger sur du matériel acheté d'occasion, pour réduire l'impact écologique de nos infrastructures.
-\end{center}
-\end{frame}
-
-\subsection{Performances}
-\begin{frame}
-\begin{center}
-\includegraphics[width=13.25cm]{rpc-amplification.png}
-\end{center}
-\end{frame}
-
-\begin{frame}
-\begin{center}
-\includegraphics[width=11cm]{rpc-complexity.png}
-\end{center}
-\end{frame}
-
-\subsection{Services}
-\begin{frame}
-\begin{center}
-Puisqu'il suit le standard S3, beaucoup de services populaires sont par conséquence compatibles avec Garage :\linebreak
-
-\begin{columns}
-\column{0.2 \linewidth}
-\begin{center}
-\includegraphics[width=2.5cm]{nextcloud-logo.png}
-\end{center}
-\column{0.2 \linewidth}
-\begin{center}
-\includegraphics[width=2.5cm]{peertube-logo.png}
-\end{center}
-\column{0.2 \linewidth}
-\begin{center}
-\includegraphics[width=2.5cm]{matrix-logo.png}
-\end{center}
-\column{0.2 \linewidth}
-\begin{center}
-\includegraphics[width=2.5cm]{mastodon-logo.png}
-\end{center}
-\end{columns}
-~\linebreak
-
-Et comme souvent avec S3, on peut assimiler un bucket à un site, et utiliser le serveur pour héberger des sites web statiques.
-\end{center}
-\end{frame}
-
-\section{Intégration chez Deuxfleurs}
-\subsection{Matériel}
-\begin{frame}
-\begin{center}
-\includegraphics[width=13cm]{neptune.jpg}\linebreak
-
-En pratique, nos serveurs ne sont effectivement que des machines achetées d'occasion (très souvent des anciens ordinateurs destinés à la bureautique en entreprise).
-\end{center}
-\end{frame}
-
-\subsection{Environnement logiciel}
-\begin{frame}
-\begin{center}
-Pour faciliter la reproduction d'un environnement connu, NixOS est installé sur nos machines.\linebreak
-
-\vspace{1cm}
-Pour s’accommoder des réseaux qu'on trouve derrière des routeurs pour particuliers, on s'aide de notre logiciel Diplonat\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/diplonat}}.
-\end{center}
-\end{frame}
-
-\section{Au-delà...}
-\subsection{... de Deuxfleurs}
-\begin{frame}
-\begin{center}
-\includegraphics[width=10cm]{tedomum.png}
-\end{center}
-\end{frame}
-
-\subsection{... de Garage}
-\begin{frame}
-\begin{center}
-Nous avons récemment lancé le développement d'Aérogramme\footnote[frame]{\texttt{https://git.deuxfleurs.fr/Deuxfleurs/aerogramme}}.\linebreak
-
-\vspace{1cm}
-Il s'agit d'un serveur de stockage de courriels chiffrés.\linebreak
-
-\vspace{1cm}
-Il est conçu pour pouvoir travailler avec Garage.
-\end{center}
-\end{frame}
-
-\section{Fin}
-\subsection{Contacts}
-\begin{frame}
-\begin{center}
-\begin{tikzpicture}
-\node (ronce) {\includegraphics[width=0.95\textwidth]{ronce.jpg}};
-\node[white] at (-5.1,3.6) {Intéressé(e) ?};
-\node[white, align=center] at (4.2,-2.6) {Contactez-nous !\\\texttt{coucou@deuxfleurs.fr}\\\texttt{\#forum:deuxfleurs.fr}};
-\end{tikzpicture}
-\end{center}
-\end{frame}
-\end{document}
-

doc/talks/2022-11-19-Capitole-du-Libre/schéma europe.tex

@@ -1,52 +0,0 @@
-\begin{tikzpicture}
-\node (carte) {\includegraphics[width=\textwidth]{carte-Europe.pdf}};
-
-% \personnage{position X}{position Y}{facteur d'échelle}
-\newcommand{\personnage}[4]
-{
-\fill[#4] ({#1-(0.4 * #3)},{#2-(0.9 * #3)}) .. controls ({#1-(0.4 * #3)},#2) and ({#1+(0.4 * #3)},#2) .. ({#1+(0.4 * #3)},{#2-(0.9 * #3)}) -- ({#1-(0.4 * #3)},{#2-(0.9 * #3)});
-\fill[#4] (#1,#2) circle ({0.25 * #3});
-}
-
-\onslide<1-11>{\personnage{-2.25}{-0.75}{0.75}{green}}
-\onslide<1-11>{\draw (-1.9,-1.6) rectangle ++(1,1.2);}
-\onslide<2-11>{\draw[fill=green] (-1.8,-1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
-\onslide<4-5>{\draw[fill=red] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
-\onslide<7-11>{\draw[fill=yellow] (-1.8,-1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
-\onslide<9-11>{\draw[fill=red] (-1.8,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
-\onslide<3-11>{\draw[fill=blue] (-1.35,-1.525) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
-\onslide<8-11>{\draw[fill=blue] (-1.35,-1.15) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
-\onslide<11-11>{\draw[fill=yellow] (-1.35,-0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
-
-\personnage{1.65}{1.5}{0.75}{blue}
-\draw (0.3,0.7) rectangle ++(1,1.2);
-\onslide<2->{\draw[fill=green] (0.4,0.775) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
-\onslide<4->{\draw[fill=red] (0.4,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
-\onslide<10->{\draw[fill=green] (0.4,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
-\onslide<3->{\draw[fill=blue] (0.85,0.775) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
-\onslide<9->{\draw[fill=red] (0.85,1.15) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
-\onslide<11->{\draw[fill=yellow] (0.85,1.525) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
-
-\personnage{1.85}{-2.3}{0.75}{red}
-\draw (0.5,-3.15) rectangle ++(1,1.2);
-\onslide<2->{\draw[fill=green] (0.6,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 1};}
-\onslide<4-5>{\draw[fill=red] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
-\onslide<7->{\draw[fill=yellow] (0.6,-2.7) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
-\onslide<9->{\draw[fill=red] (0.6,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 6};}
-\onslide<3-5>{\draw[fill=blue] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
-\onslide<6->{\draw[fill=red] (1.05,-3.075) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
-\onslide<8->{\draw[fill=blue] (1.05,-2.7) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 5};}
-\onslide<10->{\draw[fill=green] (1.05,-2.325) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
-
-\onslide<5->{\personnage{1.05}{-0.15}{0.75}{yellow}}
-\onslide<5->{\draw (-0.35,-1) rectangle ++(1,1.2);}
-\onslide<6->{\draw[fill=blue] (-0.25,-0.925) rectangle ++(0.35,0.3) node[pos=0.5, white] {\tiny 2};}
-\onslide<7->{\draw[fill=yellow] (-0.25,-0.55) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 4};}
-\onslide<10->{\draw[fill=green] (-0.25,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 7};}
-\onslide<6->{\draw[fill=red] (0.2,-0.925) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 3};}
-\onslide<8->{\draw[fill=blue] (0.2,-0.55) rectangle ++(0.35,0.3) node[pos=0.5,white] {\tiny 5};}
-\onslide<11->{\draw[fill=yellow] (0.2,-0.175) rectangle ++(0.35,0.3) node[pos=0.5] {\tiny 8};}
-
-\onslide<12->{\draw[line width=0.25cm] (-2.15,-0.5) -- ++(1,-1);}
-\onslide<12->{\draw[line width=0.25cm] (-2.15,-1.5) -- ++(1,1);}
-\end{tikzpicture}

flake.lock

@@ -1,124 +0,0 @@
-{
-  "nodes": {
-    "cargo2nix": {
-      "inputs": {
-        "flake-compat": "flake-compat",
-        "flake-utils": "flake-utils",
-        "nixpkgs": [
-          "nixpkgs"
-        ],
-        "rust-overlay": "rust-overlay"
-      },
-      "locked": {
-        "lastModified": 1666087781,
-        "narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=",
-        "owner": "Alexis211",
-        "repo": "cargo2nix",
-        "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
-        "type": "github"
-      },
-      "original": {
-        "owner": "Alexis211",
-        "repo": "cargo2nix",
-        "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
-        "type": "github"
-      }
-    },
-    "flake-compat": {
-      "flake": false,
-      "locked": {
-        "lastModified": 1650374568,
-        "narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=",
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "rev": "b4a34015c698c7793d592d66adbab377907a2be8",
-        "type": "github"
-      },
-      "original": {
-        "owner": "edolstra",
-        "repo": "flake-compat",
-        "type": "github"
-      }
-    },
-    "flake-utils": {
-      "locked": {
-        "lastModified": 1659877975,
-        "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "flake-utils_2": {
-      "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
-    },
-    "nixpkgs": {
-      "locked": {
-        "lastModified": 1665657542,
-        "narHash": "sha256-mojxNyzbvmp8NtVtxqiHGhRfjCALLfk9i/Uup68Y5q8=",
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "a3073c49bc0163fea6a121c276f526837672b555",
-        "type": "github"
-      },
-      "original": {
-        "owner": "NixOS",
-        "repo": "nixpkgs",
-        "rev": "a3073c49bc0163fea6a121c276f526837672b555",
-        "type": "github"
-      }
-    },
-    "root": {
-      "inputs": {
-        "cargo2nix": "cargo2nix",
-        "flake-utils": "flake-utils_2",
-        "nixpkgs": "nixpkgs"
-      }
-    },
-    "rust-overlay": {
-      "inputs": {
-        "flake-utils": [
-          "cargo2nix",
-          "flake-utils"
-        ],
-        "nixpkgs": [
-          "cargo2nix",
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "lastModified": 1664247556,
-        "narHash": "sha256-J4vazHU3609ekn7dr+3wfqPo5WGlZVAgV7jfux352L0=",
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "rev": "524db9c9ea7bc7743bb74cdd45b6d46ea3fcc2ab",
-        "type": "github"
-      },
-      "original": {
-        "owner": "oxalica",
-        "repo": "rust-overlay",
-        "type": "github"
-      }
-    }
-  },
-  "root": "root",
-  "version": 7
-}

flake.nix

@@ -1,36 +0,0 @@
-{
-  description = "Garage, an S3-compatible distributed object store for self-hosted deployments";
-
-  inputs.nixpkgs.url = "github:NixOS/nixpkgs/a3073c49bc0163fea6a121c276f526837672b555";
-  inputs.cargo2nix = {
-    # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
-    url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
-    inputs.nixpkgs.follows = "nixpkgs";
-  };
-  inputs.flake-utils.url = "github:numtide/flake-utils";
-
-  outputs = { self, nixpkgs, cargo2nix, flake-utils }:
-    let
-      git_version = self.lastModifiedDate;
-      compile = import ./nix/compile.nix;
-    in flake-utils.lib.eachDefaultSystem (system:
-      let pkgs = nixpkgs.legacyPackages.${system};
-      in {
-        packages = {
-          default = (compile {
-            inherit system git_version;
-            pkgsSrc = nixpkgs;
-            cargo2nixOverlay = cargo2nix.overlays.default;
-            release = true;
-          }).workspace.garage { compileMode = "build"; };
-        };
-        devShell = ((compile {
-          inherit system git_version;
-          pkgsSrc = nixpkgs;
-          cargo2nixOverlay = cargo2nix.overlays.default;
-          release = false;
-        }).workspaceShell {
-          packages = [ pkgs.rustfmt cargo2nix.packages.${system}.default ];
-        });
-      });
-}

nix/compile.nix

@@ -1,32 +1,25 @@
 {
-  system,
+  system ? builtins.currentSystem,
-  target ? null,
+  target,
-  pkgsSrc,
-  cargo2nixOverlay,
   compiler ? "rustc",
   release ? false,
   git_version ? null,
   features ? null,
 }:
 
+with import ./common.nix;
+
 let
   log = v: builtins.trace v v;
 
-  pkgs =
+  pkgs = import pkgsSrc {
-    if target != null then
+    inherit system;
-      import pkgsSrc {
+    crossSystem = {
-        inherit system;
+      config = target;
-        crossSystem = {
+      isStatic = true;
-          config = target;
+    };
-          isStatic = true;
+    overlays = [ cargo2nixOverlay ];
-        };
+  };
-        overlays = [ cargo2nixOverlay ];
-      }
-    else
-      import pkgsSrc {
-        inherit system;
-        overlays = [ cargo2nixOverlay ];
-      };
 
   /*
    Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
@@ -41,7 +34,7 @@ let
    NixOS ships them in separate ones. We reunite them with symlinkJoin.
   */
   toolchainOptions =
-    if target == null || target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
+    if target == "x86_64-unknown-linux-musl" || target == "aarch64-unknown-linux-musl" then {
       rustVersion = "1.63.0";
       extraRustComponents = [ "clippy" ];
     } else {

script/helm/garage/Chart.yaml

@@ -15,7 +15,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.2.0
+version: 0.1.3
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

script/helm/garage/templates/configmap.yaml

@@ -7,6 +7,7 @@ data:
     metadata_dir = "{{ .Values.garage.metadataDir }}"
     data_dir = "{{ .Values.garage.dataDir }}"
 
+    db_engine = "{{ .Values.garage.dbEngine }}"
     replication_mode = "{{ .Values.garage.replicationMode }}"
 
     rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"
@@ -29,3 +30,6 @@ data:
     bind_addr = "[::]:3902"
     root_domain = "{{ .Values.garage.s3.web.rootDomain }}"
     index = "{{ .Values.garage.s3.web.index }}"
+
+    [admin]
+    api_bind_addr = "[::]:3903"

script/helm/garage/templates/ingress.yaml

@@ -18,9 +18,6 @@ metadata:
   name: {{ $fullName }}-s3-api
   labels:
     {{- include "garage.labels" . | nindent 4 }}
-    {{- with .Values.ingress.s3.api.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
   {{- with .Values.ingress.s3.api.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}
@@ -83,9 +80,6 @@ metadata:
   name: {{ $fullName }}-s3-web
   labels:
     {{- include "garage.labels" . | nindent 4 }}
-    {{- with .Values.ingress.s3.web.labels }}
-    {{- toYaml . | nindent 4 }}
-    {{- end }}
   {{- with .Values.ingress.s3.web.annotations }}
   annotations:
     {{- toYaml . | nindent 4 }}

script/helm/garage/templates/service.yaml

@@ -15,5 +15,9 @@ spec:
       targetPort: 3902
       protocol: TCP
       name: s3-web
+    - port: 3903
+      targetPort: 3903
+      protocol: TCP
+      name: admin
   selector:
     {{- include "garage.selectorLabels" . | nindent 4 }}

script/helm/garage/templates/workload.yaml

@@ -1,15 +1,17 @@
 apiVersion: apps/v1
-kind: StatefulSet
+kind: {{ .Values.deployment.kind }}
 metadata:
   name: {{ include "garage.fullname" . }}
   labels:
     {{- include "garage.labels" . | nindent 4 }}
 spec:
-  replicas: {{ .Values.replicaCount }}
   selector:
     matchLabels:
       {{- include "garage.selectorLabels" . | nindent 6 }}
+  {{- if eq .Values.deployment.kind "StatefulSet" }}
+  replicas: {{ .Values.deployment.replicaCount }}
   serviceName: {{ include "garage.fullname" . }}
+  {{- end }}
   template:
     metadata:
       {{- with .Values.podAnnotations }}
@@ -54,6 +56,8 @@ spec:
               name: s3-api
             - containerPort: 3902
               name: web-api
+            - containerPort: 3903
+              name: admin
           volumeMounts:
             - name: meta
               mountPath: /mnt/meta
@@ -79,6 +83,23 @@ spec:
             name: {{ include "garage.fullname" . }}-config
         - name: etc
           emptyDir: {}
+        {{- if .Values.persistence.enabled }}
+        {{- if eq .Values.deployment.kind "DaemonSet" }}
+        - name: meta
+          hostPath:
+            path: {{ .Values.persistence.meta.hostPath }}
+            type: DirectoryOrCreate
+        - name: data
+          hostPath:
+            path: {{ .Values.persistence.data.hostPath }}
+            type: DirectoryOrCreate
+        {{- end }}
+        {{- else }}
+        - name: meta
+          emptyDir: {}
+        - name: data
+          emptyDir: {}
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -91,7 +112,7 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-  {{- if .Values.persistence.enabled }}
+  {{- if and .Values.persistence.enabled (eq .Values.deployment.kind "StatefulSet") }}
   volumeClaimTemplates:
   - metadata:
       name: meta

script/helm/garage/values.yaml

@@ -29,12 +29,20 @@ persistence:
   meta:
     # storageClass: "fast-storage-class"
     size: 100Mi
+    # used only for daemon sets
+    hostPath: /var/lib/garage/meta
   data:
     # storageClass: "slow-storage-class"
     size: 100Mi
+    # used only for daemon sets
+    hostPath: /var/lib/garage/data
 
-# Number of StatefulSet replicas/garage nodes to start
+# Deployment configuration
-replicaCount: 3
+deployment:
+  # Switchable to DaemonSet
+  kind: StatefulSet
+  # Number of StatefulSet replicas/garage nodes to start
+  replicaCount: 3
 
 image:
   repository: dxflrs/amd64_garage
@@ -85,15 +93,14 @@ service:
 ingress:
   s3:
     api:
-      enabled: false
+      enabled: true
       # Rely either on the className or the annotation below but not both
       # replace "nginx" by an Ingress controller
       # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
-      # className: "nginx"
+      className: "nginx"
-      annotations: {}
+      annotations:
         # kubernetes.io/ingress.class: "nginx"
         # kubernetes.io/tls-acme: "true"
-      labels: {}
       hosts:
         - host: "s3.garage.tld" # garage S3 API endpoint
           paths:
@@ -108,15 +115,11 @@ ingress:
       #    hosts:
       #      - kubernetes.docker.internal
     web:
-      enabled: false
+      enabled: true
-      # Rely either on the className or the annotation below but not both
+      className: "nginx"
-      # replace "nginx" by an Ingress controller
-      # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
-      # className: "nginx"
       annotations: {}
         # kubernetes.io/ingress.class: nginx
         # kubernetes.io/tls-acme: "true"
-      labels: {}
       hosts:
        - host: "*.web.garage.tld" # wildcard website access with bucket name prefix
          paths:

shell.nix

@@ -1,5 +1,5 @@
 {
-    system ? builtins.currentSystem,
+  system ? builtins.currentSystem,
 }:
 
 with import ./nix/common.nix;
@@ -71,25 +71,13 @@ function refresh_cache {
   for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
     echo "Updating cache for ''${attr}"
     derivation=$(nix-instantiate --attr ''${attr})
-    nix copy -j8 \
+    nix copy \
       --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
       $(nix-store -qR ''${derivation%\!bin})
   done
   rm /tmp/nix-signing-key.sec
 }
 
-function refresh_flake_cache {
-  pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
-  for attr in packages.x86_64-linux.default; do
-    echo "Updating cache for ''${attr}"
-    derivation=$(nix path-info --derivation ".#''${attr}")
-    nix copy -j8 \
-      --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
-      $(nix-store -qR ''${derivation})
-  done
-  rm /tmp/nix-signing-key.sec
-}
-
 function to_s3 {
   aws \
       --endpoint-url https://garage.deuxfleurs.fr \

src/api/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_api"
-version = "0.8.1"
+version = "0.8.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,11 +14,11 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_model = { version = "0.8.1", path = "../model" }
+garage_model = { version = "0.8.0", path = "../model" }
-garage_table = { version = "0.8.1", path = "../table" }
+garage_table = { version = "0.8.0", path = "../table" }
-garage_block = { version = "0.8.1", path = "../block" }
+garage_block = { version = "0.8.0", path = "../block" }
-garage_util = { version = "0.8.1", path = "../util" }
+garage_util = { version = "0.8.0", path = "../util" }
-garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_rpc = { version = "0.8.0", path = "../rpc" }
 
 async-trait = "0.1.7"
 base64 = "0.13"

src/api/admin/api_server.rs

@@ -15,7 +15,6 @@ use opentelemetry_prometheus::PrometheusExporter;
 use prometheus::{Encoder, TextEncoder};
 
 use garage_model::garage::Garage;
-use garage_rpc::system::ClusterHealthStatus;
 use garage_util::error::Error as GarageError;
 
 use crate::generic_server::*;
@@ -77,31 +76,6 @@ impl AdminApiServer {
 			.body(Body::empty())?)
 	}
 
-	fn handle_health(&self) -> Result<Response<Body>, Error> {
-		let health = self.garage.system.health();
-
-		let (status, status_str) = match health.status {
-			ClusterHealthStatus::Healthy => (StatusCode::OK, "Garage is fully operational"),
-			ClusterHealthStatus::Degraded => (
-				StatusCode::OK,
-				"Garage is operational but some storage nodes are unavailable",
-			),
-			ClusterHealthStatus::Unavailable => (
-				StatusCode::SERVICE_UNAVAILABLE,
-				"Quorum is not available for some/all partitions, reads and writes will fail",
-			),
-		};
-		let status_str = format!(
-			"{}\nConsult the full health check API endpoint at /v0/health for more details\n",
-			status_str
-		);
-
-		Ok(Response::builder()
-			.status(status)
-			.header(http::header::CONTENT_TYPE, "text/plain")
-			.body(Body::from(status_str))?)
-	}
-
 	fn handle_metrics(&self) -> Result<Response<Body>, Error> {
 		#[cfg(feature = "metrics")]
 		{
@@ -150,7 +124,6 @@ impl ApiHandler for AdminApiServer {
 	) -> Result<Response<Body>, Error> {
 		let expected_auth_header =
 			match endpoint.authorization_type() {
-				Authorization::None => None,
 				Authorization::MetricsToken => self.metrics_token.as_ref(),
 				Authorization::AdminToken => match &self.admin_token {
 					None => return Err(Error::forbidden(
@@ -174,10 +147,8 @@ impl ApiHandler for AdminApiServer {
 
 		match endpoint {
 			Endpoint::Options => self.handle_options(&req),
-			Endpoint::Health => self.handle_health(),
 			Endpoint::Metrics => self.handle_metrics(),
 			Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await,
-			Endpoint::GetClusterHealth => handle_get_cluster_health(&self.garage).await,
 			Endpoint::ConnectClusterNodes => handle_connect_cluster_nodes(&self.garage, req).await,
 			// Layout
 			Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await,

src/api/admin/cluster.rs

@@ -43,11 +43,6 @@ pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<
 	Ok(json_ok_response(&res)?)
 }
 
-pub async fn handle_get_cluster_health(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
-	let health = garage.system.health();
-	Ok(json_ok_response(&health)?)
-}
-
 pub async fn handle_connect_cluster_nodes(
 	garage: &Arc<Garage>,
 	req: Request<Body>,

src/api/admin/router.rs

@@ -6,7 +6,6 @@ use crate::admin::error::*;
 use crate::router_macros::*;
 
 pub enum Authorization {
-	None,
 	MetricsToken,
 	AdminToken,
 }
@@ -17,10 +16,8 @@ router_match! {@func
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Endpoint {
 	Options,
-	Health,
 	Metrics,
 	GetClusterStatus,
-	GetClusterHealth,
 	ConnectClusterNodes,
 	// Layout
 	GetClusterLayout,
@@ -91,10 +88,8 @@ impl Endpoint {
 
 		let res = router_match!(@gen_path_parser (req.method(), path, query) [
 			OPTIONS _ => Options,
-			GET "/health" => Health,
 			GET "/metrics" => Metrics,
 			GET "/v0/status" => GetClusterStatus,
-			GET "/v0/health" => GetClusterHealth,
 			POST "/v0/connect" => ConnectClusterNodes,
 			// Layout endpoints
 			GET "/v0/layout" => GetClusterLayout,
@@ -135,7 +130,6 @@ impl Endpoint {
 	/// Get the kind of authorization which is required to perform the operation.
 	pub fn authorization_type(&self) -> Authorization {
 		match self {
-			Self::Health => Authorization::None,
 			Self::Metrics => Authorization::MetricsToken,
 			_ => Authorization::AdminToken,
 		}
@@ -143,13 +137,9 @@ impl Endpoint {
 }
 
 generateQueryParameters! {
-	keywords: [],
+	"id" => id,
-	fields: [
+	"search" => search,
-		"format" => format,
+	"globalAlias" => global_alias,
-		"id" => id,
+	"alias" => alias,
-		"search" => search,
+	"accessKeyId" => access_key_id
-		"globalAlias" => global_alias,
-		"alias" => alias,
-		"accessKeyId" => access_key_id
-	]
 }

src/api/k2v/router.rs

@@ -96,7 +96,7 @@ impl Endpoint {
 	fn from_get(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
 			key: [
 				EMPTY if causality_token => PollItem (query::sort_key, query::causality_token, opt_parse::timeout),
 				EMPTY => ReadItem (query::sort_key),
@@ -111,7 +111,7 @@ impl Endpoint {
 	fn from_search(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
 			key: [
 			],
 			no_key: [
@@ -125,7 +125,7 @@ impl Endpoint {
 	fn from_head(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
 			key: [
 				EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
 			],
@@ -140,7 +140,7 @@ impl Endpoint {
 	fn from_post(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
 			key: [
 			],
 			no_key: [
@@ -155,7 +155,7 @@ impl Endpoint {
 	fn from_put(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
 			key: [
 				EMPTY => InsertItem (query::sort_key),
 
@@ -169,7 +169,7 @@ impl Endpoint {
 	fn from_delete(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), partition_key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None),
 			key: [
 				EMPTY => DeleteItem (query::sort_key),
 			],
@@ -232,18 +232,21 @@ impl Endpoint {
 
 // parameter name => struct field
 generateQueryParameters! {
-	keywords: [
+	"prefix" => prefix,
-		"delete" => DELETE,
+	"start" => start,
-		"search" => SEARCH
+	"causality_token" => causality_token,
-	],
+	"end" => end,
-	fields: [
+	"limit" => limit,
-		"prefix" => prefix,
+	"reverse" => reverse,
-		"start" => start,
+	"sort_key" => sort_key,
-		"causality_token" => causality_token,
+	"timeout" => timeout
-		"end" => end,
+}
-		"limit" => limit,
+
-		"reverse" => reverse,
+mod keywords {
-		"sort_key" => sort_key,
+	//! This module contain all query parameters with no associated value
-		"timeout" => timeout
+	//! used to differentiate endpoints.
-	]
+	pub const EMPTY: &str = "";
+
+	pub const DELETE: &str = "delete";
+	pub const SEARCH: &str = "search";
 }

src/api/router_macros.rs

@@ -4,9 +4,10 @@ macro_rules! router_match {
     (@match $enum:expr , [ $($endpoint:ident,)* ]) => {{
         // usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] }
         // returns true if the variant was one of the listed variants, false otherwise.
+        use Endpoint::*;
         match $enum {
             $(
-            Endpoint::$endpoint { .. } => true,
+            $endpoint { .. } => true,
             )*
             _ => false
         }
@@ -14,35 +15,37 @@ macro_rules! router_match {
     (@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{
         // usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] }
         // returns Some(field_value), or None if the variant was not one of the listed variants.
+        use Endpoint::*;
         match $enum {
             $(
-            Endpoint::$endpoint {$param, ..} => Some($param),
+            $endpoint {$param, ..} => Some($param),
             )*
             _ => None
         }
     }};
-    (@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
+	(@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
-     [
+	 [
-     $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
+	 $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
-     ]) => {{
+	 ]) => {{
-        {
+		{
-            match ($method, $reqpath) {
+			use Endpoint::*;
-                $(
+			match ($method, $reqpath) {
-                    (&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api {
+				$(
-                        $($(
+					(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => $api {
-                            $param: router_match!(@@parse_param $query, $conv, $param),
+						$($(
-                        )*)?
+							$param: router_match!(@@parse_param $query, $conv, $param),
-                    },
+						)*)?
-                )*
+					},
-                (m, p) => {
+				)*
-                    return Err(Error::bad_request(format!(
+				(m, p) => {
-                        "Unknown API endpoint: {} {}",
+					return Err(Error::bad_request(format!(
-                        m, p
+						"Unknown API endpoint: {} {}",
-                    )))
+						m, p
-                }
+					)))
-            }
+				}
-        }
+			}
-    }};
+		}
+	}};
     (@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr),
         key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*],
         no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{
@@ -57,9 +60,11 @@ macro_rules! router_match {
         //   ]
         // }
         // See in from_{method} for more detailed usage.
+        use Endpoint::*;
+        use keywords::*;
         match ($keyword, !$key.is_empty()){
             $(
-            (Keyword::$kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok(Endpoint::$api_k {
+            ($kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok($api_k {
                 $key,
                 $($(
                     $param_k: router_match!(@@parse_param $query, $conv_k, $param_k),
@@ -67,7 +72,7 @@ macro_rules! router_match {
             }),
             )*
             $(
-            (Keyword::$kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok(Endpoint::$api_nk {
+            ($kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok($api_nk {
                 $($(
                     $param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk),
                 )*)?
@@ -79,7 +84,7 @@ macro_rules! router_match {
 
     (@@parse_param $query:expr, query_opt, $param:ident) => {{
         // extract optional query parameter
-        $query.$param.take().map(|param| param.into_owned())
+		$query.$param.take().map(|param| param.into_owned())
     }};
     (@@parse_param $query:expr, query, $param:ident) => {{
         // extract mendatory query parameter
@@ -88,7 +93,7 @@ macro_rules! router_match {
     (@@parse_param $query:expr, opt_parse, $param:ident) => {{
         // extract and parse optional query parameter
         // missing parameter is file, however parse error is reported as an error
-        $query.$param
+		$query.$param
             .take()
             .map(|param| param.parse())
             .transpose()
@@ -139,40 +144,14 @@ macro_rules! router_match {
 /// This macro is used to generate part of the code in this module. It must be called only one, and
 /// is useless outside of this module.
 macro_rules! generateQueryParameters {
-    (
+    ( $($rest:expr => $name:ident),* ) => {
-        keywords: [ $($kw_param:expr => $kw_name: ident),* ],
-        fields: [ $($f_param:expr => $f_name:ident),* ]
-    ) => {
-        #[derive(Debug)]
-        #[allow(non_camel_case_types)]
-        #[allow(clippy::upper_case_acronyms)]
-        enum Keyword {
-            EMPTY,
-            $( $kw_name, )*
-        }
-
-        impl std::fmt::Display for Keyword {
-            fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
-                match self {
-                    Keyword::EMPTY => write!(f, "``"),
-                    $( Keyword::$kw_name => write!(f, "`{}`", $kw_param), )*
-                }
-            }
-        }
-
-        impl Default for Keyword {
-            fn default() -> Self {
-                Keyword::EMPTY
-            }
-        }
-
         /// Struct containing all query parameters used in endpoints. Think of it as an HashMap,
         /// but with keys statically known.
         #[derive(Debug, Default)]
         struct QueryParameters<'a> {
-            keyword: Option<Keyword>,
+            keyword: Option<Cow<'a, str>>,
             $(
-            $f_name: Option<Cow<'a, str>>,
+            $name: Option<Cow<'a, str>>,
             )*
         }
 
@@ -181,29 +160,34 @@ macro_rules! generateQueryParameters {
             fn from_query(query: &'a str) -> Result<Self, Error> {
                 let mut res: Self = Default::default();
                 for (k, v) in url::form_urlencoded::parse(query.as_bytes()) {
-                    match k.as_ref() {
+                    let repeated = match k.as_ref() {
                         $(
-                            $kw_param => if let Some(prev_kw) = res.keyword.replace(Keyword::$kw_name) {
+                            $rest => if !v.is_empty() {
-                                return Err(Error::bad_request(format!(
+                                res.$name.replace(v).is_some()
-                                    "Multiple keywords: '{}' and '{}'", prev_kw, $kw_param
+                            } else {
-                                )));
+                                false
-                            },
-                        )*
-                        $(
-                            $f_param => if !v.is_empty() {
-                                if res.$f_name.replace(v).is_some() {
-                                    return Err(Error::bad_request(format!(
-                                        "Query parameter repeated: '{}'", k
-                                    )));
-                                }
                             },
                         )*
                         _ => {
-                            if !(k.starts_with("response-") || k.starts_with("X-Amz-")) {
+                            if k.starts_with("response-") || k.starts_with("X-Amz-") {
+                                false
+                            } else if v.as_ref().is_empty() {
+                                if res.keyword.replace(k).is_some() {
+                                    return Err(Error::bad_request("Multiple keywords"));
+                                }
+                                continue;
+                            } else {
                                 debug!("Received an unknown query parameter: '{}'", k);
+                                false
                             }
                         }
                     };
+                    if repeated {
+                        return Err(Error::bad_request(format!(
+                            "Query parameter repeated: '{}'",
+                            k
+                        )));
+                    }
                 }
                 Ok(res)
             }
@@ -214,8 +198,8 @@ macro_rules! generateQueryParameters {
                 if self.keyword.is_some() {
                     Some("Keyword not used")
                 } $(
-                    else if self.$f_name.is_some() {
+                    else if self.$name.is_some() {
-                        Some(concat!("'", $f_param, "'"))
+                        Some(concat!("'", $rest, "'"))
                     }
                 )* else {
                     None

src/api/s3/bucket.rs

@@ -161,15 +161,6 @@ pub async fn handle_create_bucket(
 			return Err(CommonError::BucketAlreadyExists.into());
 		}
 	} else {
-		// Check user is allowed to create bucket
-		if !key_params.allow_create_bucket.get() {
-			return Err(CommonError::Forbidden(format!(
-				"Access key {} is not allowed to create buckets",
-				api_key.key_id
-			))
-			.into());
-		}
-
 		// Create the bucket!
 		if !is_valid_bucket_name(&bucket_name) {
 			return Err(Error::bad_request(format!(

src/api/s3/put.rs

@@ -119,17 +119,6 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 		return Ok((version_uuid, data_md5sum_hex));
 	}
 
-	// The following consists in many steps that can each fail.
-	// Keep track that some cleanup will be needed if things fail
-	// before everything is finished (cleanup is done using the Drop trait).
-	let mut interrupted_cleanup = InterruptedCleanup(Some((
-		garage.clone(),
-		bucket.id,
-		key.into(),
-		version_uuid,
-		version_timestamp,
-	)));
-
 	// Write version identifier in object table so that we have a trace
 	// that we are uploading something
 	let mut object_version = ObjectVersion {
@@ -150,27 +139,44 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	// Transfer data and verify checksum
 	let first_block_hash = async_blake2sum(first_block.clone()).await;
 
-	let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks(
+	let tx_result = (|| async {
-		&garage,
+		let (total_size, data_md5sum, data_sha256sum) = read_and_put_blocks(
-		&version,
+			&garage,
-		1,
+			&version,
-		first_block,
+			1,
-		first_block_hash,
+			first_block,
-		&mut chunker,
+			first_block_hash,
-	)
+			&mut chunker,
-	.await?;
+		)
+		.await?;
 
-	ensure_checksum_matches(
+		ensure_checksum_matches(
-		data_md5sum.as_slice(),
+			data_md5sum.as_slice(),
-		data_sha256sum,
+			data_sha256sum,
-		content_md5.as_deref(),
+			content_md5.as_deref(),
-		content_sha256,
+			content_sha256,
-	)?;
+		)?;
 
-	check_quotas(&garage, bucket, key, total_size).await?;
+		check_quotas(&garage, bucket, key, total_size).await?;
+
+		Ok((total_size, data_md5sum))
+	})()
+	.await;
+
+	// If something went wrong, clean up
+	let (total_size, md5sum_arr) = match tx_result {
+		Ok(rv) => rv,
+		Err(e) => {
+			// Mark object as aborted, this will free the blocks further down
+			object_version.state = ObjectVersionState::Aborted;
+			let object = Object::new(bucket.id, key.into(), vec![object_version.clone()]);
+			garage.object_table.insert(&object).await?;
+			return Err(e);
+		}
+	};
 
 	// Save final object state, marked as Complete
-	let md5sum_hex = hex::encode(data_md5sum);
+	let md5sum_hex = hex::encode(md5sum_arr);
 	object_version.state = ObjectVersionState::Complete(ObjectVersionData::FirstBlock(
 		ObjectVersionMeta {
 			headers,
@@ -182,10 +188,6 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
 	let object = Object::new(bucket.id, key.into(), vec![object_version]);
 	garage.object_table.insert(&object).await?;
 
-	// We were not interrupted, everything went fine.
-	// We won't have to clean up on drop.
-	interrupted_cleanup.cancel();
-
 	Ok((version_uuid, md5sum_hex))
 }
 
@@ -424,33 +426,6 @@ pub fn put_response(version_uuid: Uuid, md5sum_hex: String) -> Response<Body> {
 		.unwrap()
 }
 
-struct InterruptedCleanup(Option<(Arc<Garage>, Uuid, String, Uuid, u64)>);
-
-impl InterruptedCleanup {
-	fn cancel(&mut self) {
-		drop(self.0.take());
-	}
-}
-impl Drop for InterruptedCleanup {
-	fn drop(&mut self) {
-		if let Some((garage, bucket_id, key, version_uuid, version_ts)) = self.0.take() {
-			tokio::spawn(async move {
-				let object_version = ObjectVersion {
-					uuid: version_uuid,
-					timestamp: version_ts,
-					state: ObjectVersionState::Aborted,
-				};
-				let object = Object::new(bucket_id, key, vec![object_version]);
-				if let Err(e) = garage.object_table.insert(&object).await {
-					warn!("Cannot cleanup after aborted PutObject: {}", e);
-				}
-			});
-		}
-	}
-}
-
-// ----
-
 pub async fn handle_create_multipart_upload(
 	garage: Arc<Garage>,
 	req: &Request<Body>,

src/api/s3/router.rs

@@ -355,7 +355,7 @@ impl Endpoint {
 	fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
 			key: [
 				EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker),
 				EMPTY => GetObject (query_opt::version_id, opt_parse::part_number),
@@ -412,7 +412,7 @@ impl Endpoint {
 	fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
 			key: [
 				EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
 			],
@@ -426,7 +426,7 @@ impl Endpoint {
 	fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
 			key: [
 				EMPTY if upload_id  => CompleteMultipartUpload (query::upload_id),
 				RESTORE => RestoreObject (query_opt::version_id),
@@ -448,7 +448,7 @@ impl Endpoint {
 	) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), key, query, headers),
+			(query.keyword.take().unwrap_or_default().as_ref(), key, query, headers),
 			key: [
 				EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id),
 				EMPTY header "x-amz-copy-source" => CopyObject,
@@ -490,7 +490,7 @@ impl Endpoint {
 	fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
 		router_match! {
 			@gen_parser
-			(query.keyword.take().unwrap_or_default(), key, query, None),
+			(query.keyword.take().unwrap_or_default().as_ref(), key, query, None),
 			key: [
 				EMPTY if upload_id => AbortMultipartUpload (query::upload_id),
 				EMPTY => DeleteObject (query_opt::version_id),
@@ -624,60 +624,63 @@ impl Endpoint {
 
 // parameter name => struct field
 generateQueryParameters! {
-	keywords: [
+	"continuation-token" => continuation_token,
-		"accelerate" => ACCELERATE,
+	"delimiter" => delimiter,
-		"acl" => ACL,
+	"encoding-type" => encoding_type,
-		"analytics" => ANALYTICS,
+	"fetch-owner" => fetch_owner,
-		"cors" => CORS,
+	"id" => id,
-		"delete" => DELETE,
+	"key-marker" => key_marker,
-		"encryption" => ENCRYPTION,
+	"list-type" => list_type,
-		"intelligent-tiering" => INTELLIGENT_TIERING,
+	"marker" => marker,
-		"inventory" => INVENTORY,
+	"max-keys" => max_keys,
-		"legal-hold" => LEGAL_HOLD,
+	"max-parts" => max_parts,
-		"lifecycle" => LIFECYCLE,
+	"max-uploads" => max_uploads,
-		"location" => LOCATION,
+	"partNumber" => part_number,
-		"logging" => LOGGING,
+	"part-number-marker" => part_number_marker,
-		"metrics" => METRICS,
+	"prefix" => prefix,
-		"notification" => NOTIFICATION,
+	"select-type" => select_type,
-		"object-lock" => OBJECT_LOCK,
+	"start-after" => start_after,
-		"ownershipControls" => OWNERSHIP_CONTROLS,
+	"uploadId" => upload_id,
-		"policy" => POLICY,
+	"upload-id-marker" => upload_id_marker,
-		"policyStatus" => POLICY_STATUS,
+	"versionId" => version_id,
-		"publicAccessBlock" => PUBLIC_ACCESS_BLOCK,
+	"version-id-marker" => version_id_marker
-		"replication" => REPLICATION,
+}
-		"requestPayment" => REQUEST_PAYMENT,
+
-		"restore" => RESTORE,
+mod keywords {
-		"retention" => RETENTION,
+	//! This module contain all query parameters with no associated value S3 uses to differentiate
-		"select" => SELECT,
+	//! endpoints.
-		"tagging" => TAGGING,
+	pub const EMPTY: &str = "";
-		"torrent" => TORRENT,
+
-		"uploads" => UPLOADS,
+	pub const ACCELERATE: &str = "accelerate";
-		"versioning" => VERSIONING,
+	pub const ACL: &str = "acl";
-		"versions" => VERSIONS,
+	pub const ANALYTICS: &str = "analytics";
-		"website" => WEBSITE
+	pub const CORS: &str = "cors";
-	],
+	pub const DELETE: &str = "delete";
-	fields: [
+	pub const ENCRYPTION: &str = "encryption";
-		"continuation-token" => continuation_token,
+	pub const INTELLIGENT_TIERING: &str = "intelligent-tiering";
-		"delimiter" => delimiter,
+	pub const INVENTORY: &str = "inventory";
-		"encoding-type" => encoding_type,
+	pub const LEGAL_HOLD: &str = "legal-hold";
-		"fetch-owner" => fetch_owner,
+	pub const LIFECYCLE: &str = "lifecycle";
-		"id" => id,
+	pub const LOCATION: &str = "location";
-		"key-marker" => key_marker,
+	pub const LOGGING: &str = "logging";
-		"list-type" => list_type,
+	pub const METRICS: &str = "metrics";
-		"marker" => marker,
+	pub const NOTIFICATION: &str = "notification";
-		"max-keys" => max_keys,
+	pub const OBJECT_LOCK: &str = "object-lock";
-		"max-parts" => max_parts,
+	pub const OWNERSHIP_CONTROLS: &str = "ownershipControls";
-		"max-uploads" => max_uploads,
+	pub const POLICY: &str = "policy";
-		"partNumber" => part_number,
+	pub const POLICY_STATUS: &str = "policyStatus";
-		"part-number-marker" => part_number_marker,
+	pub const PUBLIC_ACCESS_BLOCK: &str = "publicAccessBlock";
-		"prefix" => prefix,
+	pub const REPLICATION: &str = "replication";
-		"select-type" => select_type,
+	pub const REQUEST_PAYMENT: &str = "requestPayment";
-		"start-after" => start_after,
+	pub const RESTORE: &str = "restore";
-		"uploadId" => upload_id,
+	pub const RETENTION: &str = "retention";
-		"upload-id-marker" => upload_id_marker,
+	pub const SELECT: &str = "select";
-		"versionId" => version_id,
+	pub const TAGGING: &str = "tagging";
-		"version-id-marker" => version_id_marker
+	pub const TORRENT: &str = "torrent";
-	]
+	pub const UPLOADS: &str = "uploads";
+	pub const VERSIONING: &str = "versioning";
+	pub const VERSIONS: &str = "versions";
+	pub const WEBSITE: &str = "website";
 }
 
 #[cfg(test)]

src/block/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_block"
-version = "0.8.1"
+version = "0.8.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,10 +14,10 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.1", path = "../db" }
+garage_db = { version = "0.8.0", path = "../db" }
-garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_util = { version = "0.8.1", path = "../util" }
+garage_util = { version = "0.8.0", path = "../util" }
-garage_table = { version = "0.8.1", path = "../table" }
+garage_table = { version = "0.8.0", path = "../table" }
 
 opentelemetry = "0.17"
 
@@ -31,6 +31,7 @@ rand = "0.8"
 async-compression = { version = "0.3", features = ["tokio", "zstd"] }
 zstd = { version = "0.9", default-features = false }
 
+rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 

src/block/manager.rs

@@ -3,10 +3,8 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Duration;
 
-use arc_swap::ArcSwapOption;
 use async_trait::async_trait;
 use bytes::Bytes;
-use rand::prelude::*;
 use serde::{Deserialize, Serialize};
 
 use futures::Stream;
@@ -24,12 +22,9 @@ use garage_rpc::rpc_helper::netapp::stream::{stream_asyncread, ByteStream};
 
 use garage_db as db;
 
-use garage_util::background::{vars, BackgroundRunner};
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
-use garage_util::persister::PersisterShared;
-use garage_util::time::msec_to_rfc3339;
 
 use garage_rpc::rpc_helper::OrderTag;
 use garage_rpc::system::System;
@@ -92,17 +87,7 @@ pub struct BlockManager {
 
 	pub(crate) metrics: BlockManagerMetrics,
 
-	pub scrub_persister: PersisterShared<ScrubWorkerPersisted>,
+	tx_scrub_command: mpsc::Sender<ScrubWorkerCommand>,
-	tx_scrub_command: ArcSwapOption<mpsc::Sender<ScrubWorkerCommand>>,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct BlockResyncErrorInfo {
-	pub hash: Hash,
-	pub refcount: u64,
-	pub error_count: u64,
-	pub last_try: u64,
-	pub next_try: u64,
 }
 
 // This custom struct contains functions that must only be ran
@@ -129,14 +114,9 @@ impl BlockManager {
 			.netapp
 			.endpoint("garage_block/manager.rs/Rpc".to_string());
 
-		let metrics = BlockManagerMetrics::new(
+		let metrics = BlockManagerMetrics::new(resync.queue.clone(), resync.errors.clone());
-			compression_level,
-			rc.rc.clone(),
-			resync.queue.clone(),
-			resync.errors.clone(),
-		);
 
-		let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info");
+		let (scrub_tx, scrub_rx) = mpsc::channel(1);
 
 		let block_manager = Arc::new(Self {
 			replication,
@@ -148,46 +128,21 @@ impl BlockManager {
 			system,
 			endpoint,
 			metrics,
-			scrub_persister,
+			tx_scrub_command: scrub_tx,
-			tx_scrub_command: ArcSwapOption::new(None),
 		});
 		block_manager.endpoint.set_handler(block_manager.clone());
 
-		block_manager
-	}
-
-	pub fn spawn_workers(self: &Arc<Self>, bg: &BackgroundRunner) {
 		// Spawn a bunch of resync workers
 		for index in 0..MAX_RESYNC_WORKERS {
-			let worker = ResyncWorker::new(index, self.clone());
+			let worker = ResyncWorker::new(index, block_manager.clone());
-			bg.spawn_worker(worker);
+			block_manager.system.background.spawn_worker(worker);
 		}
 
 		// Spawn scrub worker
-		let (scrub_tx, scrub_rx) = mpsc::channel(1);
+		let scrub_worker = ScrubWorker::new(block_manager.clone(), scrub_rx);
-		self.tx_scrub_command.store(Some(Arc::new(scrub_tx)));
+		block_manager.system.background.spawn_worker(scrub_worker);
-		bg.spawn_worker(ScrubWorker::new(
-			self.clone(),
-			scrub_rx,
-			self.scrub_persister.clone(),
-		));
-	}
 
-	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
+		block_manager
-		self.resync.register_bg_vars(vars);
-
-		vars.register_rw(
-			&self.scrub_persister,
-			"scrub-tranquility",
-			|p| p.get_with(|x| x.tranquility),
-			|p, tranquility| p.set_with(|x| x.tranquility = tranquility),
-		);
-		vars.register_ro(&self.scrub_persister, "scrub-last-completed", |p| {
-			p.get_with(|x| msec_to_rfc3339(x.time_last_complete_scrub))
-		});
-		vars.register_ro(&self.scrub_persister, "scrub-corruptions_detected", |p| {
-			p.get_with(|x| x.corruptions_detected)
-		});
 	}
 
 	/// Ask nodes that might have a (possibly compressed) block for it
@@ -354,42 +309,9 @@ impl BlockManager {
 		Ok(self.rc.rc.len()?)
 	}
 
-	/// Get number of items in the refcount table
-	pub fn rc_fast_len(&self) -> Result<Option<usize>, Error> {
-		Ok(self.rc.rc.fast_len()?)
-	}
-
 	/// Send command to start/stop/manager scrub worker
-	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) -> Result<(), Error> {
+	pub async fn send_scrub_command(&self, cmd: ScrubWorkerCommand) {
-		let tx = self.tx_scrub_command.load();
+		let _ = self.tx_scrub_command.send(cmd).await;
-		let tx = tx.as_ref().ok_or_message("scrub worker is not running")?;
-		tx.send(cmd).await.ok_or_message("send error")?;
-		Ok(())
-	}
-
-	/// Get the reference count of a block
-	pub fn get_block_rc(&self, hash: &Hash) -> Result<u64, Error> {
-		Ok(self.rc.get_block_rc(hash)?.as_u64())
-	}
-
-	/// List all resync errors
-	pub fn list_resync_errors(&self) -> Result<Vec<BlockResyncErrorInfo>, Error> {
-		let mut blocks = Vec::with_capacity(self.resync.errors.len());
-		for ent in self.resync.errors.iter()? {
-			let (hash, cnt) = ent?;
-			let cnt = ErrorCounter::decode(&cnt);
-			blocks.push(BlockResyncErrorInfo {
-				hash: Hash::try_from(&hash).unwrap(),
-				refcount: 0,
-				error_count: cnt.errors,
-				last_try: cnt.last_try,
-				next_try: cnt.next_try(),
-			});
-		}
-		for block in blocks.iter_mut() {
-			block.refcount = self.get_block_rc(&block.hash)?;
-		}
-		Ok(blocks)
 	}
 
 	//// ----- Managing the reference counter ----
@@ -681,21 +603,14 @@ impl BlockManagerLocked {
 			}
 		};
 
-		let mut path_tmp = path.clone();
+		let mut path2 = path.clone();
-		let tmp_extension = format!("tmp{}", hex::encode(thread_rng().gen::<[u8; 4]>()));
+		path2.set_extension("tmp");
-		path_tmp.set_extension(tmp_extension);
+		let mut f = fs::File::create(&path2).await?;
-
-		let mut delete_on_drop = DeleteOnDrop(Some(path_tmp.clone()));
-
-		let mut f = fs::File::create(&path_tmp).await?;
 		f.write_all(data).await?;
 		f.sync_all().await?;
 		drop(f);
 
-		fs::rename(path_tmp, path).await?;
+		fs::rename(path2, path).await?;
-
-		delete_on_drop.cancel();
-
 		if let Some(to_delete) = to_delete {
 			fs::remove_file(to_delete).await?;
 		}
@@ -761,23 +676,3 @@ async fn read_stream_to_end(mut stream: ByteStream) -> Result<Bytes, Error> {
 		.concat()
 		.into())
 }
-
-struct DeleteOnDrop(Option<PathBuf>);
-
-impl DeleteOnDrop {
-	fn cancel(&mut self) {
-		drop(self.0.take());
-	}
-}
-
-impl Drop for DeleteOnDrop {
-	fn drop(&mut self) {
-		if let Some(path) = self.0.take() {
-			tokio::spawn(async move {
-				if let Err(e) = fs::remove_file(&path).await {
-					debug!("DeleteOnDrop failed for {}: {}", path.display(), e);
-				}
-			});
-		}
-	}
-}

src/block/metrics.rs

@@ -1,12 +1,9 @@
 use opentelemetry::{global, metrics::*};
 
-use garage_db as db;
 use garage_db::counted_tree_hack::CountedTree;
 
 /// TableMetrics reference all counter used for metrics
 pub struct BlockManagerMetrics {
-	pub(crate) _compression_level: ValueObserver<u64>,
-	pub(crate) _rc_size: ValueObserver<u64>,
 	pub(crate) _resync_queue_len: ValueObserver<u64>,
 	pub(crate) _resync_errored_blocks: ValueObserver<u64>,
 
@@ -26,31 +23,9 @@ pub struct BlockManagerMetrics {
 }
 
 impl BlockManagerMetrics {
-	pub fn new(
+	pub fn new(resync_queue: CountedTree, resync_errors: CountedTree) -> Self {
-		compression_level: Option<i32>,
-		rc_tree: db::Tree,
-		resync_queue: CountedTree,
-		resync_errors: CountedTree,
-	) -> Self {
 		let meter = global::meter("garage_model/block");
 		Self {
-			_compression_level: meter
-				.u64_value_observer("block.compression_level", move |observer| {
-					match compression_level {
-						Some(v) => observer.observe(v as u64, &[]),
-						None => observer.observe(0 as u64, &[]),
-					}
-				})
-				.with_description("Garage compression level for node")
-				.init(),
-			_rc_size: meter
-				.u64_value_observer("block.rc_size", move |observer| {
-					if let Ok(Some(v)) = rc_tree.fast_len() {
-						observer.observe(v as u64, &[])
-					}
-				})
-				.with_description("Number of blocks known to the reference counter")
-				.init(),
 			_resync_queue_len: meter
 				.u64_value_observer("block.resync_queue_length", move |observer| {
 					observer.observe(resync_queue.len() as u64, &[])

src/block/rc.rs

@@ -169,11 +169,4 @@ impl RcEntry {
 	pub(crate) fn is_needed(&self) -> bool {
 		!self.is_deletable()
 	}
-
-	pub(crate) fn as_u64(&self) -> u64 {
-		match self {
-			RcEntry::Present { count } => *count,
-			_ => 0,
-		}
-	}
 }

src/block/repair.rs

@@ -13,7 +13,7 @@ use tokio::sync::watch;
 use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::*;
-use garage_util::persister::PersisterShared;
+use garage_util::persister::Persister;
 use garage_util::time::*;
 use garage_util::tranquilizer::Tranquilizer;
 
@@ -53,7 +53,7 @@ impl Worker for RepairWorker {
 		"Block repair worker".into()
 	}
 
-	fn status(&self) -> WorkerStatus {
+	fn info(&self) -> Option<String> {
 		match self.block_iter.as_ref() {
 			None => {
 				let idx_bytes = self
@@ -66,20 +66,9 @@ impl Worker for RepairWorker {
 				} else {
 					idx_bytes
 				};
-				WorkerStatus {
+				Some(format!("Phase 1: {}", hex::encode(idx_bytes)))
-					progress: Some("0.00%".into()),
-					freeform: vec![format!(
-						"Currently in phase 1, iterator position: {}",
-						hex::encode(idx_bytes)
-					)],
-					..Default::default()
-				}
 			}
-			Some(bi) => WorkerStatus {
+			Some(bi) => Some(format!("Phase 2: {:.2}% done", bi.progress() * 100.)),
-				progress: Some(format!("{:.2}%", bi.progress() * 100.)),
-				freeform: vec!["Currently in phase 2".into()],
-				..Default::default()
-			},
 		}
 	}
 
@@ -148,7 +137,7 @@ impl Worker for RepairWorker {
 		}
 	}
 
-	async fn wait_for_work(&mut self) -> WorkerState {
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
 		unreachable!()
 	}
 }
@@ -168,24 +157,15 @@ pub struct ScrubWorker {
 	work: ScrubWorkerState,
 	tranquilizer: Tranquilizer,
 
-	persister: PersisterShared<ScrubWorkerPersisted>,
+	persister: Persister<ScrubWorkerPersisted>,
+	persisted: ScrubWorkerPersisted,
 }
 
 #[derive(Serialize, Deserialize)]
-pub struct ScrubWorkerPersisted {
+struct ScrubWorkerPersisted {
-	pub tranquility: u32,
+	tranquility: u32,
-	pub(crate) time_last_complete_scrub: u64,
+	time_last_complete_scrub: u64,
-	pub(crate) corruptions_detected: u64,
+	corruptions_detected: u64,
-}
-impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {}
-impl Default for ScrubWorkerPersisted {
-	fn default() -> Self {
-		ScrubWorkerPersisted {
-			time_last_complete_scrub: 0,
-			tranquility: INITIAL_SCRUB_TRANQUILITY,
-			corruptions_detected: 0,
-		}
-	}
 }
 
 enum ScrubWorkerState {
@@ -206,20 +186,27 @@ pub enum ScrubWorkerCommand {
 	Pause(Duration),
 	Resume,
 	Cancel,
+	SetTranquility(u32),
 }
 
 impl ScrubWorker {
-	pub(crate) fn new(
+	pub fn new(manager: Arc<BlockManager>, rx_cmd: mpsc::Receiver<ScrubWorkerCommand>) -> Self {
-		manager: Arc<BlockManager>,
+		let persister = Persister::new(&manager.system.metadata_dir, "scrub_info");
-		rx_cmd: mpsc::Receiver<ScrubWorkerCommand>,
+		let persisted = match persister.load() {
-		persister: PersisterShared<ScrubWorkerPersisted>,
+			Ok(v) => v,
-	) -> Self {
+			Err(_) => ScrubWorkerPersisted {
+				time_last_complete_scrub: 0,
+				tranquility: INITIAL_SCRUB_TRANQUILITY,
+				corruptions_detected: 0,
+			},
+		};
 		Self {
 			manager,
 			rx_cmd,
 			work: ScrubWorkerState::Finished,
 			tranquilizer: Tranquilizer::new(30),
 			persister,
+			persisted,
 		}
 	}
 
@@ -268,6 +255,12 @@ impl ScrubWorker {
 					}
 				}
 			}
+			ScrubWorkerCommand::SetTranquility(t) => {
+				self.persisted.tranquility = t;
+				if let Err(e) = self.persister.save_async(&self.persisted).await {
+					error!("Could not save new tranquilitiy value: {}", e);
+				}
+			}
 		}
 	}
 }
@@ -278,37 +271,29 @@ impl Worker for ScrubWorker {
 		"Block scrub worker".into()
 	}
 
-	fn status(&self) -> WorkerStatus {
+	fn info(&self) -> Option<String> {
-		let (corruptions_detected, tranquility, time_last_complete_scrub) =
+		let s = match &self.work {
-			self.persister.get_with(|p| {
+			ScrubWorkerState::Running(bsi) => format!(
-				(
+				"{:.2}% done (tranquility = {})",
-					p.corruptions_detected,
+				bsi.progress() * 100.,
-					p.tranquility,
+				self.persisted.tranquility
-					p.time_last_complete_scrub,
+			),
-				)
-			});
-
-		let mut s = WorkerStatus {
-			persistent_errors: Some(corruptions_detected),
-			tranquility: Some(tranquility),
-			..Default::default()
-		};
-		match &self.work {
-			ScrubWorkerState::Running(bsi) => {
-				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
-			}
 			ScrubWorkerState::Paused(bsi, rt) => {
-				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
+				format!(
-				s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
+					"Paused, {:.2}% done, resumes at {}",
+					bsi.progress() * 100.,
+					msec_to_rfc3339(*rt)
+				)
 			}
-			ScrubWorkerState::Finished => {
+			ScrubWorkerState::Finished => format!(
-				s.freeform = vec![format!(
+				"Last completed scrub: {}",
-					"Last scrub completed at {}",
+				msec_to_rfc3339(self.persisted.time_last_complete_scrub)
-					msec_to_rfc3339(time_last_complete_scrub)
+			),
-				)];
+		};
-			}
+		Some(format!(
-		}
+			"{} ; corruptions detected: {}",
-		s
+			s, self.persisted.corruptions_detected
+		))
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
@@ -325,17 +310,18 @@ impl Worker for ScrubWorker {
 					match self.manager.read_block(&hash).await {
 						Err(Error::CorruptData(_)) => {
 							error!("Found corrupt data block during scrub: {:?}", hash);
-							self.persister.set_with(|p| p.corruptions_detected += 1)?;
+							self.persisted.corruptions_detected += 1;
+							self.persister.save_async(&self.persisted).await?;
 						}
 						Err(e) => return Err(e),
 						_ => (),
 					};
 					Ok(self
 						.tranquilizer
-						.tranquilize_worker(self.persister.get_with(|p| p.tranquility)))
+						.tranquilize_worker(self.persisted.tranquility))
 				} else {
-					self.persister
+					self.persisted.time_last_complete_scrub = now_msec();
-						.set_with(|p| p.time_last_complete_scrub = now_msec())?;
+					self.persister.save_async(&self.persisted).await?;
 					self.work = ScrubWorkerState::Finished;
 					self.tranquilizer.clear();
 					Ok(WorkerState::Idle)
@@ -345,13 +331,12 @@ impl Worker for ScrubWorker {
 		}
 	}
 
-	async fn wait_for_work(&mut self) -> WorkerState {
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
 		let (wait_until, command) = match &self.work {
 			ScrubWorkerState::Running(_) => return WorkerState::Busy,
 			ScrubWorkerState::Paused(_, resume_time) => (*resume_time, ScrubWorkerCommand::Resume),
 			ScrubWorkerState::Finished => (
-				self.persister.get_with(|p| p.time_last_complete_scrub)
+				self.persisted.time_last_complete_scrub + SCRUB_INTERVAL.as_millis() as u64,
-					+ SCRUB_INTERVAL.as_millis() as u64,
 				ScrubWorkerCommand::Start,
 			),
 		};

src/block/resync.rs

@@ -3,6 +3,7 @@ use std::convert::TryInto;
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
+use arc_swap::ArcSwap;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 
@@ -21,7 +22,7 @@ use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::*;
 use garage_util::metrics::RecordDuration;
-use garage_util::persister::PersisterShared;
+use garage_util::persister::Persister;
 use garage_util::time::*;
 use garage_util::tranquilizer::Tranquilizer;
 
@@ -48,12 +49,13 @@ const INITIAL_RESYNC_TRANQUILITY: u32 = 2;
 
 pub struct BlockResyncManager {
 	pub(crate) queue: CountedTree,
-	pub(crate) notify: Arc<Notify>,
+	pub(crate) notify: Notify,
 	pub(crate) errors: CountedTree,
 
 	busy_set: BusySet,
 
-	persister: PersisterShared<ResyncPersistedConfig>,
+	persister: Persister<ResyncPersistedConfig>,
+	persisted: ArcSwap<ResyncPersistedConfig>,
 }
 
 #[derive(Serialize, Deserialize, Clone, Copy)]
@@ -61,15 +63,6 @@ struct ResyncPersistedConfig {
 	n_workers: usize,
 	tranquility: u32,
 }
-impl garage_util::migrate::InitialFormat for ResyncPersistedConfig {}
-impl Default for ResyncPersistedConfig {
-	fn default() -> Self {
-		ResyncPersistedConfig {
-			n_workers: 1,
-			tranquility: INITIAL_RESYNC_TRANQUILITY,
-		}
-	}
-}
 
 enum ResyncIterResult {
 	BusyDidSomething,
@@ -97,14 +90,22 @@ impl BlockResyncManager {
 			.expect("Unable to open block_local_resync_errors tree");
 		let errors = CountedTree::new(errors).expect("Could not count block_local_resync_errors");
 
-		let persister = PersisterShared::new(&system.metadata_dir, "resync_cfg");
+		let persister = Persister::new(&system.metadata_dir, "resync_cfg");
+		let persisted = match persister.load() {
+			Ok(v) => v,
+			Err(_) => ResyncPersistedConfig {
+				n_workers: 1,
+				tranquility: INITIAL_RESYNC_TRANQUILITY,
+			},
+		};
 
 		Self {
 			queue,
-			notify: Arc::new(Notify::new()),
+			notify: Notify::new(),
 			errors,
 			busy_set: Arc::new(Mutex::new(HashSet::new())),
 			persister,
+			persisted: ArcSwap::new(Arc::new(persisted)),
 		}
 	}
 
@@ -122,56 +123,6 @@ impl BlockResyncManager {
 		Ok(self.errors.len())
 	}
 
-	/// Clear the error counter for a block and put it in queue immediately
-	pub fn clear_backoff(&self, hash: &Hash) -> Result<(), Error> {
-		let now = now_msec();
-		if let Some(ec) = self.errors.get(hash)? {
-			let mut ec = ErrorCounter::decode(&ec);
-			if ec.errors > 0 {
-				ec.last_try = now - ec.delay_msec();
-				self.errors.insert(hash, ec.encode())?;
-				self.put_to_resync_at(hash, now)?;
-				return Ok(());
-			}
-		}
-		Err(Error::Message(format!(
-			"Block {:?} was not in an errored state",
-			hash
-		)))
-	}
-
-	pub fn register_bg_vars(&self, vars: &mut vars::BgVars) {
-		let notify = self.notify.clone();
-		vars.register_rw(
-			&self.persister,
-			"resync-worker-count",
-			|p| p.get_with(|x| x.n_workers),
-			move |p, n_workers| {
-				if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
-					return Err(Error::Message(format!(
-						"Invalid number of resync workers, must be between 1 and {}",
-						MAX_RESYNC_WORKERS
-					)));
-				}
-				p.set_with(|x| x.n_workers = n_workers)?;
-				notify.notify_waiters();
-				Ok(())
-			},
-		);
-
-		let notify = self.notify.clone();
-		vars.register_rw(
-			&self.persister,
-			"resync-tranquility",
-			|p| p.get_with(|x| x.tranquility),
-			move |p, tranquility| {
-				p.set_with(|x| x.tranquility = tranquility)?;
-				notify.notify_waiters();
-				Ok(())
-			},
-		);
-	}
-
 	// ---- Resync loop ----
 
 	// This part manages a queue of blocks that need to be
@@ -306,7 +257,7 @@ impl BlockResyncManager {
 
 				if let Err(e) = &res {
 					manager.metrics.resync_error_counter.add(1);
-					error!("Error when resyncing {:?}: {}", hash, e);
+					warn!("Error when resyncing {:?}: {}", hash, e);
 
 					let err_counter = match self.errors.get(hash.as_slice())? {
 						Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
@@ -466,6 +417,33 @@ impl BlockResyncManager {
 
 		Ok(())
 	}
+
+	async fn update_persisted(
+		&self,
+		update: impl Fn(&mut ResyncPersistedConfig),
+	) -> Result<(), Error> {
+		let mut cfg: ResyncPersistedConfig = *self.persisted.load().as_ref();
+		update(&mut cfg);
+		self.persister.save_async(&cfg).await?;
+		self.persisted.store(Arc::new(cfg));
+		self.notify.notify_waiters();
+		Ok(())
+	}
+
+	pub async fn set_n_workers(&self, n_workers: usize) -> Result<(), Error> {
+		if !(1..=MAX_RESYNC_WORKERS).contains(&n_workers) {
+			return Err(Error::Message(format!(
+				"Invalid number of resync workers, must be between 1 and {}",
+				MAX_RESYNC_WORKERS
+			)));
+		}
+		self.update_persisted(|cfg| cfg.n_workers = n_workers).await
+	}
+
+	pub async fn set_tranquility(&self, tranquility: u32) -> Result<(), Error> {
+		self.update_persisted(|cfg| cfg.tranquility = tranquility)
+			.await
+	}
 }
 
 impl Drop for BusyBlock {
@@ -480,18 +458,15 @@ pub(crate) struct ResyncWorker {
 	manager: Arc<BlockManager>,
 	tranquilizer: Tranquilizer,
 	next_delay: Duration,
-	persister: PersisterShared<ResyncPersistedConfig>,
 }
 
 impl ResyncWorker {
 	pub(crate) fn new(index: usize, manager: Arc<BlockManager>) -> Self {
-		let persister = manager.resync.persister.clone();
 		Self {
 			index,
 			manager,
 			tranquilizer: Tranquilizer::new(30),
 			next_delay: Duration::from_secs(10),
-			persister,
 		}
 	}
 }
@@ -502,36 +477,39 @@ impl Worker for ResyncWorker {
 		format!("Block resync worker #{}", self.index + 1)
 	}
 
-	fn status(&self) -> WorkerStatus {
+	fn info(&self) -> Option<String> {
-		let (n_workers, tranquility) = self.persister.get_with(|x| (x.n_workers, x.tranquility));
+		let persisted = self.manager.resync.persisted.load();
 
-		if self.index >= n_workers {
+		if self.index >= persisted.n_workers {
-			return WorkerStatus {
+			return Some("(unused)".into());
-				freeform: vec!["This worker is currently disabled".into()],
-				..Default::default()
-			};
 		}
 
-		WorkerStatus {
+		let mut ret = vec![];
-			queue_length: Some(self.manager.resync.queue_len().unwrap_or(0) as u64),
+		ret.push(format!("tranquility = {}", persisted.tranquility));
-			tranquility: Some(tranquility),
+
-			persistent_errors: Some(self.manager.resync.errors_len().unwrap_or(0) as u64),
+		let qlen = self.manager.resync.queue_len().unwrap_or(0);
-			..Default::default()
+		if qlen > 0 {
+			ret.push(format!("{} blocks in queue", qlen));
 		}
+
+		let elen = self.manager.resync.errors_len().unwrap_or(0);
+		if elen > 0 {
+			ret.push(format!("{} blocks in error state", elen));
+		}
+
+		Some(ret.join(", "))
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let (n_workers, tranquility) = self.persister.get_with(|x| (x.n_workers, x.tranquility));
+		if self.index >= self.manager.resync.persisted.load().n_workers {
-
-		if self.index >= n_workers {
 			return Ok(WorkerState::Idle);
 		}
 
 		self.tranquilizer.reset();
 		match self.manager.resync.resync_iter(&self.manager).await {
-			Ok(ResyncIterResult::BusyDidSomething) => {
+			Ok(ResyncIterResult::BusyDidSomething) => Ok(self
-				Ok(self.tranquilizer.tranquilize_worker(tranquility))
+				.tranquilizer
-			}
+				.tranquilize_worker(self.manager.resync.persisted.load().tranquility)),
 			Ok(ResyncIterResult::BusyDidNothing) => Ok(WorkerState::Busy),
 			Ok(ResyncIterResult::IdleFor(delay)) => {
 				self.next_delay = delay;
@@ -549,8 +527,8 @@ impl Worker for ResyncWorker {
 		}
 	}
 
-	async fn wait_for_work(&mut self) -> WorkerState {
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
-		while self.index >= self.persister.get_with(|x| x.n_workers) {
+		while self.index >= self.manager.resync.persisted.load().n_workers {
 			self.manager.resync.notify.notified().await
 		}
 
@@ -567,9 +545,9 @@ impl Worker for ResyncWorker {
 /// and the time of the last try.
 /// Used to implement exponential backoff.
 #[derive(Clone, Copy, Debug)]
-pub(crate) struct ErrorCounter {
+struct ErrorCounter {
-	pub(crate) errors: u64,
+	errors: u64,
-	pub(crate) last_try: u64,
+	last_try: u64,
 }
 
 impl ErrorCounter {
@@ -580,13 +558,12 @@ impl ErrorCounter {
 		}
 	}
 
-	pub(crate) fn decode(data: &[u8]) -> Self {
+	fn decode(data: &[u8]) -> Self {
 		Self {
 			errors: u64::from_be_bytes(data[0..8].try_into().unwrap()),
 			last_try: u64::from_be_bytes(data[8..16].try_into().unwrap()),
 		}
 	}
-
 	fn encode(&self) -> Vec<u8> {
 		[
 			u64::to_be_bytes(self.errors),
@@ -606,8 +583,7 @@ impl ErrorCounter {
 		(RESYNC_RETRY_DELAY.as_millis() as u64)
 			<< std::cmp::min(self.errors - 1, RESYNC_RETRY_DELAY_MAX_BACKOFF_POWER)
 	}
-
+	fn next_try(&self) -> u64 {
-	pub(crate) fn next_try(&self) -> u64 {
 		self.last_try + self.delay_msec()
 	}
 }

src/db/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_db"
-version = "0.8.1"
+version = "0.8.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -33,7 +33,6 @@ pretty_env_logger = { version = "0.4", optional = true }
 mktemp = "0.4"
 
 [features]
-default = [ "sled" ]
 bundled-libs = [ "rusqlite/bundled" ]
 cli = ["clap", "pretty_env_logger"]
 lmdb = [ "heed" ]

src/db/lib.rs

@@ -181,10 +181,6 @@ impl Tree {
 	pub fn len(&self) -> Result<usize> {
 		self.0.len(self.1)
 	}
-	#[inline]
-	pub fn fast_len(&self) -> Result<Option<usize>> {
-		self.0.fast_len(self.1)
-	}
 
 	#[inline]
 	pub fn first(&self) -> Result<Option<(Value, Value)>> {
@@ -327,9 +323,6 @@ pub(crate) trait IDb: Send + Sync {
 
 	fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;
 	fn len(&self, tree: usize) -> Result<usize>;
-	fn fast_len(&self, _tree: usize) -> Result<Option<usize>> {
-		Ok(None)
-	}
 
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>>;
 	fn remove(&self, tree: usize, key: &[u8]) -> Result<Option<Value>>;

src/db/lmdb_adapter.rs

@@ -121,10 +121,6 @@ impl IDb for LmdbDb {
 		Ok(tree.len(&tx)?.try_into().unwrap())
 	}
 
-	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
-		Ok(Some(self.len(tree)?))
-	}
-
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		let tree = self.get_tree(tree)?;
 		let mut tx = self.db.write_txn()?;

src/db/sqlite_adapter.rs

@@ -144,10 +144,6 @@ impl IDb for SqliteDb {
 		}
 	}
 
-	fn fast_len(&self, tree: usize) -> Result<Option<usize>> {
-		Ok(Some(self.len(tree)?))
-	}
-
 	fn insert(&self, tree: usize, key: &[u8], value: &[u8]) -> Result<Option<Value>> {
 		trace!("insert {}: lock db", tree);
 		let this = self.0.lock().unwrap();

src/garage/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage"
-version = "0.8.1"
+version = "0.8.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -21,14 +21,14 @@ path = "tests/lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.1", path = "../db" }
+garage_db = { version = "0.8.0", path = "../db" }
-garage_api = { version = "0.8.1", path = "../api" }
+garage_api = { version = "0.8.0", path = "../api" }
-garage_block = { version = "0.8.1", path = "../block" }
+garage_block = { version = "0.8.0", path = "../block" }
-garage_model = { version = "0.8.1", path = "../model" }
+garage_model = { version = "0.8.0", path = "../model" }
-garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_table = { version = "0.8.1", path = "../table" }
+garage_table = { version = "0.8.0", path = "../table" }
-garage_util = { version = "0.8.1", path = "../util" }
+garage_util = { version = "0.8.0", path = "../util" }
-garage_web = { version = "0.8.1", path = "../web" }
+garage_web = { version = "0.8.0", path = "../web" }
 
 backtrace = "0.3"
 bytes = "1.0"
@@ -36,12 +36,13 @@ bytesize = "1.1"
 timeago = "0.3"
 parse_duration = "2.1"
 hex = "0.4"
-tracing = { version = "0.1.30" }
+tracing = { version = "0.1.30", features = ["log-always"] }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 rand = "0.8"
 async-trait = "0.1.7"
 sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
 
+rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 structopt = { version = "0.3", default-features = false }
@@ -73,7 +74,7 @@ base64 = "0.13"
 
 
 [features]
-default = [ "bundled-libs", "metrics", "sled", "k2v" ]
+default = [ "bundled-libs", "metrics", "sled" ]
 
 k2v = [ "garage_util/k2v", "garage_api/k2v" ]
 

src/garage/admin.rs

@@ -5,11 +5,9 @@ use std::sync::Arc;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 
-use garage_util::background::BackgroundRunner;
 use garage_util::crdt::*;
 use garage_util::data::*;
 use garage_util::error::Error as GarageError;
-use garage_util::formater::format_table_to_string;
 use garage_util::time::*;
 
 use garage_table::replication::*;
@@ -17,7 +15,7 @@ use garage_table::*;
 
 use garage_rpc::*;
 
-use garage_block::manager::BlockResyncErrorInfo;
+use garage_block::repair::ScrubWorkerCommand;
 
 use garage_model::bucket_alias_table::*;
 use garage_model::bucket_table::*;
@@ -26,8 +24,6 @@ use garage_model::helper::error::{Error, OkOrBadRequest};
 use garage_model::key_table::*;
 use garage_model::migrate::Migrate;
 use garage_model::permission::*;
-use garage_model::s3::object_table::*;
-use garage_model::s3::version_table::Version;
 
 use crate::cli::*;
 use crate::repair::online::launch_online_repair;
@@ -42,8 +38,7 @@ pub enum AdminRpc {
 	LaunchRepair(RepairOpt),
 	Migrate(MigrateOpt),
 	Stats(StatsOpt),
-	Worker(WorkerOperation),
+	Worker(WorkerOpt),
-	BlockOperation(BlockOperation),
 
 	// Replies
 	Ok(String),
@@ -59,14 +54,6 @@ pub enum AdminRpc {
 		HashMap<usize, garage_util::background::WorkerInfo>,
 		WorkerListOpt,
 	),
-	WorkerVars(Vec<(Uuid, String, String)>),
-	WorkerInfo(usize, garage_util::background::WorkerInfo),
-	BlockErrorList(Vec<BlockResyncErrorInfo>),
-	BlockInfo {
-		hash: Hash,
-		refcount: u64,
-		versions: Vec<Result<Version, Uuid>>,
-	},
 }
 
 impl Rpc for AdminRpc {
@@ -75,24 +62,17 @@ impl Rpc for AdminRpc {
 
 pub struct AdminRpcHandler {
 	garage: Arc<Garage>,
-	background: Arc<BackgroundRunner>,
 	endpoint: Arc<Endpoint<AdminRpc, Self>>,
 }
 
 impl AdminRpcHandler {
-	pub fn new(garage: Arc<Garage>, background: Arc<BackgroundRunner>) -> Arc<Self> {
+	pub fn new(garage: Arc<Garage>) -> Arc<Self> {
 		let endpoint = garage.system.netapp.endpoint(ADMIN_RPC_PATH.into());
-		let admin = Arc::new(Self {
+		let admin = Arc::new(Self { garage, endpoint });
-			garage,
-			background,
-			endpoint,
-		});
 		admin.endpoint.set_handler(admin.clone());
 		admin
 	}
 
-	// ================ BUCKET COMMANDS ====================
-
 	async fn handle_bucket_cmd(&self, cmd: &BucketOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			BucketOperation::List => self.handle_list_buckets().await,
@@ -571,8 +551,6 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok(ret))
 	}
 
-	// ================ KEY COMMANDS ====================
-
 	async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result<AdminRpc, Error> {
 		match cmd {
 			KeyOperation::List => self.handle_list_keys().await,
@@ -710,8 +688,6 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::KeyInfo(key, relevant_buckets))
 	}
 
-	// ================ MIGRATION COMMANDS ====================
-
 	async fn handle_migrate(self: &Arc<Self>, opt: MigrateOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -728,8 +704,6 @@ impl AdminRpcHandler {
 		Ok(AdminRpc::Ok("Migration successfull.".into()))
 	}
 
-	// ================ REPAIR COMMANDS ====================
-
 	async fn handle_launch_repair(self: &Arc<Self>, opt: RepairOpt) -> Result<AdminRpc, Error> {
 		if !opt.yes {
 			return Err(Error::BadRequest(
@@ -765,7 +739,7 @@ impl AdminRpcHandler {
 				)))
 			}
 		} else {
-			launch_online_repair(&self.garage, &self.background, opt).await?;
+			launch_online_repair(self.garage.clone(), opt).await;
 			Ok(AdminRpc::Ok(format!(
 				"Repair launched on {:?}",
 				self.garage.system.id
@@ -773,8 +747,6 @@ impl AdminRpcHandler {
 		}
 	}
 
-	// ================ STATS COMMANDS ====================
-
 	async fn handle_stats(&self, opt: StatsOpt) -> Result<AdminRpc, Error> {
 		if opt.all_nodes {
 			let mut ret = String::new();
@@ -791,12 +763,11 @@ impl AdminRpcHandler {
 				match self
 					.endpoint
 					.call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL)
-					.await
+					.await?
 				{
-					Ok(Ok(AdminRpc::Ok(s))) => writeln!(&mut ret, "{}", s).unwrap(),
+					Ok(AdminRpc::Ok(s)) => writeln!(&mut ret, "{}", s).unwrap(),
-					Ok(Ok(x)) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
+					Ok(x) => writeln!(&mut ret, "Bad answer: {:?}", x).unwrap(),
-					Ok(Err(e)) => writeln!(&mut ret, "Remote error: {}", e).unwrap(),
+					Err(e) => writeln!(&mut ret, "Error: {}", e).unwrap(),
-					Err(e) => writeln!(&mut ret, "Network error: {}", e).unwrap(),
 				}
 			}
 			Ok(AdminRpc::Ok(ret))
@@ -816,7 +787,6 @@ impl AdminRpcHandler {
 				.unwrap_or_else(|| "(unknown)".into()),
 		)
 		.unwrap();
-
 		writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();
 
 		// Gather ring statistics
@@ -835,38 +805,21 @@ impl AdminRpcHandler {
 			writeln!(&mut ret, "  {:?} {}", n, c).unwrap();
 		}
 
-		// Gather table statistics
+		self.gather_table_stats(&mut ret, &self.garage.bucket_table, &opt)?;
-		let mut table = vec!["  Table\tItems\tMklItems\tMklTodo\tGcTodo".into()];
+		self.gather_table_stats(&mut ret, &self.garage.key_table, &opt)?;
-		table.push(self.gather_table_stats(&self.garage.bucket_table, opt.detailed)?);
+		self.gather_table_stats(&mut ret, &self.garage.object_table, &opt)?;
-		table.push(self.gather_table_stats(&self.garage.key_table, opt.detailed)?);
+		self.gather_table_stats(&mut ret, &self.garage.version_table, &opt)?;
-		table.push(self.gather_table_stats(&self.garage.object_table, opt.detailed)?);
+		self.gather_table_stats(&mut ret, &self.garage.block_ref_table, &opt)?;
-		table.push(self.gather_table_stats(&self.garage.version_table, opt.detailed)?);
-		table.push(self.gather_table_stats(&self.garage.block_ref_table, opt.detailed)?);
-		write!(
-			&mut ret,
-			"\nTable stats:\n{}",
-			format_table_to_string(table)
-		)
-		.unwrap();
 
-		// Gather block manager statistics
 		writeln!(&mut ret, "\nBlock manager stats:").unwrap();
-		let rc_len = if opt.detailed {
+		if opt.detailed {
-			self.garage.block_manager.rc_len()?.to_string()
+			writeln!(
-		} else {
+				&mut ret,
-			self.garage
+				"  number of RC entries (~= number of blocks): {}",
-				.block_manager
+				self.garage.block_manager.rc_len()?
-				.rc_fast_len()?
+			)
-				.map(|x| x.to_string())
+			.unwrap();
-				.unwrap_or_else(|| "NC".into())
+		}
-		};
-
-		writeln!(
-			&mut ret,
-			"  number of RC entries (~= number of blocks): {}",
-			rc_len
-		)
-		.unwrap();
 		writeln!(
 			&mut ret,
 			"  resync queue length: {}",
@@ -880,305 +833,79 @@ impl AdminRpcHandler {
 		)
 		.unwrap();
 
-		if !opt.detailed {
-			writeln!(&mut ret, "\nIf values are missing (marked as NC), consider adding the --detailed flag - this will be slow.").unwrap();
-		}
-
 		Ok(ret)
 	}
 
 	fn gather_table_stats<F, R>(
 		&self,
+		to: &mut String,
 		t: &Arc<Table<F, R>>,
-		detailed: bool,
+		opt: &StatsOpt,
-	) -> Result<String, Error>
+	) -> Result<(), Error>
 	where
 		F: TableSchema + 'static,
 		R: TableReplication + 'static,
 	{
-		let (data_len, mkl_len) = if detailed {
+		writeln!(to, "\nTable stats for {}", F::TABLE_NAME).unwrap();
-			(
+		if opt.detailed {
-				t.data.store.len().map_err(GarageError::from)?.to_string(),
+			writeln!(
-				t.merkle_updater.merkle_tree_len()?.to_string(),
+				to,
+				"  number of items: {}",
+				t.data.store.len().map_err(GarageError::from)?
 			)
-		} else {
+			.unwrap();
-			(
+			writeln!(
-				t.data
+				to,
-					.store
+				"  Merkle tree size: {}",
-					.fast_len()
+				t.merkle_updater.merkle_tree_len()?
-					.map_err(GarageError::from)?
-					.map(|x| x.to_string())
-					.unwrap_or_else(|| "NC".into()),
-				t.merkle_updater
-					.merkle_tree_fast_len()?
-					.map(|x| x.to_string())
-					.unwrap_or_else(|| "NC".into()),
 			)
-		};
+			.unwrap();
-
-		Ok(format!(
-			"  {}\t{}\t{}\t{}\t{}",
-			F::TABLE_NAME,
-			data_len,
-			mkl_len,
-			t.merkle_updater.todo_len()?,
-			t.data.gc_todo_len()?
-		))
-	}
-
-	// ================ WORKER COMMANDS ====================
-
-	async fn handle_worker_cmd(&self, cmd: &WorkerOperation) -> Result<AdminRpc, Error> {
-		match cmd {
-			WorkerOperation::List { opt } => {
-				let workers = self.background.get_worker_info();
-				Ok(AdminRpc::WorkerList(workers, *opt))
-			}
-			WorkerOperation::Info { tid } => {
-				let info = self
-					.background
-					.get_worker_info()
-					.get(tid)
-					.ok_or_bad_request(format!("No worker with TID {}", tid))?
-					.clone();
-				Ok(AdminRpc::WorkerInfo(*tid, info))
-			}
-			WorkerOperation::Get {
-				all_nodes,
-				variable,
-			} => self.handle_get_var(*all_nodes, variable).await,
-			WorkerOperation::Set {
-				all_nodes,
-				variable,
-				value,
-			} => self.handle_set_var(*all_nodes, variable, value).await,
 		}
+		writeln!(
+			to,
+			"  Merkle updater todo queue length: {}",
+			t.merkle_updater.todo_len()?
+		)
+		.unwrap();
+		writeln!(to, "  GC todo queue length: {}", t.data.gc_todo_len()?).unwrap();
+
+		Ok(())
 	}
 
-	async fn handle_get_var(
+	// ----
-		&self,
-		all_nodes: bool,
-		variable: &Option<String>,
-	) -> Result<AdminRpc, Error> {
-		if all_nodes {
-			let mut ret = vec![];
-			let ring = self.garage.system.ring.borrow().clone();
-			for node in ring.layout.node_ids().iter() {
-				let node = (*node).into();
-				match self
-					.endpoint
-					.call(
-						&node,
-						AdminRpc::Worker(WorkerOperation::Get {
-							all_nodes: false,
-							variable: variable.clone(),
-						}),
-						PRIO_NORMAL,
-					)
-					.await??
-				{
-					AdminRpc::WorkerVars(v) => ret.extend(v),
-					m => return Err(GarageError::unexpected_rpc_message(m).into()),
-				}
-			}
-			Ok(AdminRpc::WorkerVars(ret))
-		} else {
-			#[allow(clippy::collapsible_else_if)]
-			if let Some(v) = variable {
-				Ok(AdminRpc::WorkerVars(vec![(
-					self.garage.system.id,
-					v.clone(),
-					self.garage.bg_vars.get(v)?,
-				)]))
-			} else {
-				let mut vars = self.garage.bg_vars.get_all();
-				vars.sort();
-				Ok(AdminRpc::WorkerVars(
-					vars.into_iter()
-						.map(|(k, v)| (self.garage.system.id, k.to_string(), v))
-						.collect(),
-				))
-			}
-		}
-	}
 
-	async fn handle_set_var(
+	async fn handle_worker_cmd(&self, opt: WorkerOpt) -> Result<AdminRpc, Error> {
-		&self,
+		match opt.cmd {
-		all_nodes: bool,
+			WorkerCmd::List { opt } => {
-		variable: &str,
+				let workers = self.garage.background.get_worker_info();
-		value: &str,
+				Ok(AdminRpc::WorkerList(workers, opt))
-	) -> Result<AdminRpc, Error> {
-		if all_nodes {
-			let mut ret = vec![];
-			let ring = self.garage.system.ring.borrow().clone();
-			for node in ring.layout.node_ids().iter() {
-				let node = (*node).into();
-				match self
-					.endpoint
-					.call(
-						&node,
-						AdminRpc::Worker(WorkerOperation::Set {
-							all_nodes: false,
-							variable: variable.to_string(),
-							value: value.to_string(),
-						}),
-						PRIO_NORMAL,
-					)
-					.await??
-				{
-					AdminRpc::WorkerVars(v) => ret.extend(v),
-					m => return Err(GarageError::unexpected_rpc_message(m).into()),
-				}
 			}
-			Ok(AdminRpc::WorkerVars(ret))
+			WorkerCmd::Set { opt } => match opt {
-		} else {
+				WorkerSetCmd::ScrubTranquility { tranquility } => {
-			self.garage.bg_vars.set(variable, value)?;
+					let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);
-			Ok(AdminRpc::WorkerVars(vec![(
+					self.garage
-				self.garage.system.id,
+						.block_manager
-				variable.to_string(),
+						.send_scrub_command(scrub_command)
-				value.to_string(),
+						.await;
-			)]))
+					Ok(AdminRpc::Ok("Scrub tranquility updated".into()))
-		}
-	}
-
-	// ================ BLOCK COMMANDS ====================
-
-	async fn handle_block_cmd(&self, cmd: &BlockOperation) -> Result<AdminRpc, Error> {
-		match cmd {
-			BlockOperation::ListErrors => Ok(AdminRpc::BlockErrorList(
-				self.garage.block_manager.list_resync_errors()?,
-			)),
-			BlockOperation::Info { hash } => {
-				let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
-				let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
-				let refcount = self.garage.block_manager.get_block_rc(&hash)?;
-				let block_refs = self
-					.garage
-					.block_ref_table
-					.get_range(&hash, None, None, 10000, Default::default())
-					.await?;
-				let mut versions = vec![];
-				for br in block_refs {
-					if let Some(v) = self
-						.garage
-						.version_table
-						.get(&br.version, &EmptyKey)
-						.await?
-					{
-						versions.push(Ok(v));
-					} else {
-						versions.push(Err(br.version));
-					}
 				}
-				Ok(AdminRpc::BlockInfo {
+				WorkerSetCmd::ResyncNWorkers { n_workers } => {
-					hash,
+					self.garage
-					refcount,
+						.block_manager
-					versions,
+						.resync
-				})
+						.set_n_workers(n_workers)
-			}
-			BlockOperation::RetryNow { all, blocks } => {
-				if *all {
-					if !blocks.is_empty() {
-						return Err(Error::BadRequest(
-							"--all was specified, cannot also specify blocks".into(),
-						));
-					}
-					let blocks = self.garage.block_manager.list_resync_errors()?;
-					for b in blocks.iter() {
-						self.garage.block_manager.resync.clear_backoff(&b.hash)?;
-					}
-					Ok(AdminRpc::Ok(format!(
-						"{} blocks returned in queue for a retry now (check logs to see results)",
-						blocks.len()
-					)))
-				} else {
-					for hash in blocks {
-						let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
-						let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
-						self.garage.block_manager.resync.clear_backoff(&hash)?;
-					}
-					Ok(AdminRpc::Ok(format!(
-						"{} blocks returned in queue for a retry now (check logs to see results)",
-						blocks.len()
-					)))
-				}
-			}
-			BlockOperation::Purge { yes, blocks } => {
-				if !yes {
-					return Err(Error::BadRequest(
-						"Pass the --yes flag to confirm block purge operation.".into(),
-					));
-				}
-
-				let mut obj_dels = 0;
-				let mut ver_dels = 0;
-
-				for hash in blocks {
-					let hash = hex::decode(hash).ok_or_bad_request("invalid hash")?;
-					let hash = Hash::try_from(&hash).ok_or_bad_request("invalid hash")?;
-					let block_refs = self
-						.garage
-						.block_ref_table
-						.get_range(&hash, None, None, 10000, Default::default())
 						.await?;
-
+					Ok(AdminRpc::Ok("Number of resync workers updated".into()))
-					for br in block_refs {
-						let version = match self
-							.garage
-							.version_table
-							.get(&br.version, &EmptyKey)
-							.await?
-						{
-							Some(v) => v,
-							None => continue,
-						};
-
-						if let Some(object) = self
-							.garage
-							.object_table
-							.get(&version.bucket_id, &version.key)
-							.await?
-						{
-							let ov = object.versions().iter().rev().find(|v| v.is_complete());
-							if let Some(ov) = ov {
-								if ov.uuid == br.version {
-									let del_uuid = gen_uuid();
-									let deleted_object = Object::new(
-										version.bucket_id,
-										version.key.clone(),
-										vec![ObjectVersion {
-											uuid: del_uuid,
-											timestamp: ov.timestamp + 1,
-											state: ObjectVersionState::Complete(
-												ObjectVersionData::DeleteMarker,
-											),
-										}],
-									);
-									self.garage.object_table.insert(&deleted_object).await?;
-									obj_dels += 1;
-								}
-							}
-						}
-
-						if !version.deleted.get() {
-							let deleted_version = Version::new(
-								version.uuid,
-								version.bucket_id,
-								version.key.clone(),
-								true,
-							);
-							self.garage.version_table.insert(&deleted_version).await?;
-							ver_dels += 1;
-						}
-					}
 				}
-				Ok(AdminRpc::Ok(format!(
+				WorkerSetCmd::ResyncTranquility { tranquility } => {
-					"{} blocks were purged: {} object deletion markers added, {} versions marked deleted",
+					self.garage
-					blocks.len(),
+						.block_manager
-					obj_dels,
+						.resync
-					ver_dels
+						.set_tranquility(tranquility)
-				)))
+						.await?;
-			}
+					Ok(AdminRpc::Ok("Resync tranquility updated".into()))
+				}
+			},
 		}
 	}
 }
@@ -1196,8 +923,7 @@ impl EndpointHandler<AdminRpc> for AdminRpcHandler {
 			AdminRpc::Migrate(opt) => self.handle_migrate(opt.clone()).await,
 			AdminRpc::LaunchRepair(opt) => self.handle_launch_repair(opt.clone()).await,
 			AdminRpc::Stats(opt) => self.handle_stats(opt.clone()).await,
-			AdminRpc::Worker(wo) => self.handle_worker_cmd(wo).await,
+			AdminRpc::Worker(opt) => self.handle_worker_cmd(opt.clone()).await,
-			AdminRpc::BlockOperation(bo) => self.handle_block_cmd(bo).await,
 			m => Err(GarageError::unexpected_rpc_message(m).into()),
 		}
 	}

src/garage/cli/cmd.rs

@@ -41,9 +41,6 @@ pub async fn cli_command_dispatch(
 		}
 		Command::Stats(so) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Stats(so)).await,
 		Command::Worker(wo) => cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::Worker(wo)).await,
-		Command::Block(bo) => {
-			cmd_admin(admin_rpc_endpoint, rpc_host, AdminRpc::BlockOperation(bo)).await
-		}
 		_ => unreachable!(),
 	}
 }
@@ -189,23 +186,7 @@ pub async fn cmd_admin(
 			print_key_info(&key, &rb);
 		}
 		AdminRpc::WorkerList(wi, wlo) => {
-			print_worker_list(wi, wlo);
+			print_worker_info(wi, wlo);
-		}
-		AdminRpc::WorkerVars(wv) => {
-			print_worker_vars(wv);
-		}
-		AdminRpc::WorkerInfo(tid, wi) => {
-			print_worker_info(tid, wi);
-		}
-		AdminRpc::BlockErrorList(el) => {
-			print_block_error_list(el);
-		}
-		AdminRpc::BlockInfo {
-			hash,
-			refcount,
-			versions,
-		} => {
-			print_block_info(hash, refcount, versions);
 		}
 		r => {
 			error!("Unexpected response: {:?}", r);

src/garage/cli/structs.rs

@@ -49,11 +49,7 @@ pub enum Command {
 
 	/// Manage background workers
 	#[structopt(name = "worker", version = garage_version())]
-	Worker(WorkerOperation),
+	Worker(WorkerOpt),
-
-	/// Low-level debug operations on data blocks
-	#[structopt(name = "block", version = garage_version())]
-	Block(BlockOperation),
 }
 
 #[derive(StructOpt, Debug)]
@@ -506,36 +502,25 @@ pub struct StatsOpt {
 	pub detailed: bool,
 }
 
+#[derive(Serialize, Deserialize, StructOpt, Debug, Clone)]
+pub struct WorkerOpt {
+	#[structopt(subcommand)]
+	pub cmd: WorkerCmd,
+}
+
 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
-pub enum WorkerOperation {
+pub enum WorkerCmd {
 	/// List all workers on Garage node
 	#[structopt(name = "list", version = garage_version())]
 	List {
 		#[structopt(flatten)]
 		opt: WorkerListOpt,
 	},
-	/// Get detailed information about a worker
-	#[structopt(name = "info", version = garage_version())]
-	Info { tid: usize },
-	/// Get worker parameter
-	#[structopt(name = "get", version = garage_version())]
-	Get {
-		/// Gather variable values from all nodes
-		#[structopt(short = "a", long = "all-nodes")]
-		all_nodes: bool,
-		/// Variable name to get, or none to get all variables
-		variable: Option<String>,
-	},
 	/// Set worker parameter
 	#[structopt(name = "set", version = garage_version())]
 	Set {
-		/// Set variable values on all nodes
+		#[structopt(subcommand)]
-		#[structopt(short = "a", long = "all-nodes")]
+		opt: WorkerSetCmd,
-		all_nodes: bool,
-		/// Variable node to set
-		variable: String,
-		/// Value to set the variable to
-		value: String,
 	},
 }
 
@@ -550,33 +535,14 @@ pub struct WorkerListOpt {
 }
 
 #[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
-pub enum BlockOperation {
+pub enum WorkerSetCmd {
-	/// List all blocks that currently have a resync error
+	/// Set tranquility of scrub operations
-	#[structopt(name = "list-errors", version = garage_version())]
+	#[structopt(name = "scrub-tranquility", version = garage_version())]
-	ListErrors,
+	ScrubTranquility { tranquility: u32 },
-	/// Get detailed information about a single block
+	/// Set number of concurrent block resync workers
-	#[structopt(name = "info", version = garage_version())]
+	#[structopt(name = "resync-n-workers", version = garage_version())]
-	Info {
+	ResyncNWorkers { n_workers: usize },
-		/// Hash of the block for which to retrieve information
+	/// Set tranquility of block resync operations
-		hash: String,
+	#[structopt(name = "resync-tranquility", version = garage_version())]
-	},
+	ResyncTranquility { tranquility: u32 },
-	/// Retry now the resync of one or many blocks
-	#[structopt(name = "retry-now", version = garage_version())]
-	RetryNow {
-		/// Retry all blocks that have a resync error
-		#[structopt(long = "all")]
-		all: bool,
-		/// Hashes of the block to retry to resync now
-		blocks: Vec<String>,
-	},
-	/// Delete all objects referencing a missing block
-	#[structopt(name = "purge", version = garage_version())]
-	Purge {
-		/// Mandatory to confirm this operation
-		#[structopt(long = "yes")]
-		yes: bool,
-		/// Hashes of the block to purge
-		#[structopt(required = true)]
-		blocks: Vec<String>,
-	},
 }

src/garage/cli/util.rs

@@ -3,17 +3,14 @@ use std::time::Duration;
 
 use garage_util::background::*;
 use garage_util::crdt::*;
-use garage_util::data::*;
+use garage_util::data::Uuid;
 use garage_util::error::*;
 use garage_util::formater::format_table;
 use garage_util::time::*;
 
-use garage_block::manager::BlockResyncErrorInfo;
-
 use garage_model::bucket_table::*;
 use garage_model::key_table::*;
 use garage_model::s3::object_table::{BYTES, OBJECTS, UNFINISHED_UPLOADS};
-use garage_model::s3::version_table::Version;
 
 use crate::cli::structs::WorkerListOpt;
 
@@ -244,7 +241,7 @@ pub fn find_matching_node(
 	}
 }
 
-pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
+pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 	let mut wi = wi.into_iter().collect::<Vec<_>>();
 	wi.sort_by_key(|(tid, info)| {
 		(
@@ -257,7 +254,7 @@ pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 		)
 	});
 
-	let mut table = vec!["TID\tState\tName\tTranq\tDone\tQueue\tErrors\tConsec\tLast".to_string()];
+	let mut table = vec![];
 	for (tid, info) in wi.iter() {
 		if wlo.busy && !matches!(info.state, WorkerState::Busy | WorkerState::Throttled(_)) {
 			continue;
@@ -266,155 +263,33 @@ pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 			continue;
 		}
 
+		table.push(format!("{}\t{}\t{}", tid, info.state, info.name));
+		if let Some(i) = &info.info {
+			table.push(format!("\t\t  {}", i));
+		}
 		let tf = timeago::Formatter::new();
-		let err_ago = info
+		let (err_ago, err_msg) = info
 			.last_error
 			.as_ref()
-			.map(|(_, t)| tf.convert(Duration::from_millis(now_msec() - t)))
+			.map(|(m, t)| {
-			.unwrap_or_default();
+				(
-		let (total_err, consec_err) = if info.errors > 0 {
+					tf.convert(Duration::from_millis(now_msec() - t)),
-			(info.errors.to_string(), info.consecutive_errors.to_string())
+					m.as_str(),
-		} else {
+				)
-			("-".into(), "-".into())
+			})
-		};
+			.unwrap_or(("(?) ago".into(), "(?)"));
-
+		if info.consecutive_errors > 0 {
-		table.push(format!(
-			"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}",
-			tid,
-			info.state,
-			info.name,
-			info.status
-				.tranquility
-				.as_ref()
-				.map(ToString::to_string)
-				.unwrap_or_else(|| "-".into()),
-			info.status.progress.as_deref().unwrap_or("-"),
-			info.status
-				.queue_length
-				.as_ref()
-				.map(ToString::to_string)
-				.unwrap_or_else(|| "-".into()),
-			total_err,
-			consec_err,
-			err_ago,
-		));
-	}
-	format_table(table);
-}
-
-pub fn print_worker_info(tid: usize, info: WorkerInfo) {
-	let mut table = vec![];
-	table.push(format!("Task id:\t{}", tid));
-	table.push(format!("Worker name:\t{}", info.name));
-	match info.state {
-		WorkerState::Throttled(t) => {
 			table.push(format!(
-				"Worker state:\tBusy (throttled, paused for {:.3}s)",
+				"\t\t  {} consecutive errors ({} total), last {}",
-				t
+				info.consecutive_errors, info.errors, err_ago,
 			));
-		}
+			table.push(format!("\t\t  {}", err_msg));
-		s => {
+		} else if info.errors > 0 {
-			table.push(format!("Worker state:\t{}", s));
+			table.push(format!("\t\t  ({} errors, last {})", info.errors, err_ago,));
-		}
+			if wlo.errors {
-	};
+				table.push(format!("\t\t  {}", err_msg));
-	if let Some(tql) = info.status.tranquility {
-		table.push(format!("Tranquility:\t{}", tql));
-	}
-
-	table.push("".into());
-	table.push(format!("Total errors:\t{}", info.errors));
-	table.push(format!("Consecutive errs:\t{}", info.consecutive_errors));
-	if let Some((s, t)) = info.last_error {
-		table.push(format!("Last error:\t{}", s));
-		let tf = timeago::Formatter::new();
-		table.push(format!(
-			"Last error time:\t{}",
-			tf.convert(Duration::from_millis(now_msec() - t))
-		));
-	}
-
-	table.push("".into());
-	if let Some(p) = info.status.progress {
-		table.push(format!("Progress:\t{}", p));
-	}
-	if let Some(ql) = info.status.queue_length {
-		table.push(format!("Queue length:\t{}", ql));
-	}
-	if let Some(pe) = info.status.persistent_errors {
-		table.push(format!("Persistent errors:\t{}", pe));
-	}
-
-	for (i, s) in info.status.freeform.iter().enumerate() {
-		if i == 0 {
-			if table.last() != Some(&"".into()) {
-				table.push("".into());
-			}
-			table.push(format!("Message:\t{}", s));
-		} else {
-			table.push(format!("\t{}", s));
-		}
-	}
-	format_table(table);
-}
-
-pub fn print_worker_vars(wv: Vec<(Uuid, String, String)>) {
-	let table = wv
-		.into_iter()
-		.map(|(n, k, v)| format!("{:?}\t{}\t{}", n, k, v))
-		.collect::<Vec<_>>();
-	format_table(table);
-}
-
-pub fn print_block_error_list(el: Vec<BlockResyncErrorInfo>) {
-	let now = now_msec();
-	let tf = timeago::Formatter::new();
-	let mut tf2 = timeago::Formatter::new();
-	tf2.ago("");
-
-	let mut table = vec!["Hash\tRC\tErrors\tLast error\tNext try".into()];
-	for e in el {
-		table.push(format!(
-			"{}\t{}\t{}\t{}\tin {}",
-			hex::encode(e.hash.as_slice()),
-			e.refcount,
-			e.error_count,
-			tf.convert(Duration::from_millis(now - e.last_try)),
-			tf2.convert(Duration::from_millis(e.next_try - now))
-		));
-	}
-	format_table(table);
-}
-
-pub fn print_block_info(hash: Hash, refcount: u64, versions: Vec<Result<Version, Uuid>>) {
-	println!("Block hash: {}", hex::encode(hash.as_slice()));
-	println!("Refcount: {}", refcount);
-	println!();
-
-	let mut table = vec!["Version\tBucket\tKey\tDeleted".into()];
-	let mut nondeleted_count = 0;
-	for v in versions.iter() {
-		match v {
-			Ok(ver) => {
-				table.push(format!(
-					"{:?}\t{:?}\t{}\t{:?}",
-					ver.uuid,
-					ver.bucket_id,
-					ver.key,
-					ver.deleted.get()
-				));
-				if !ver.deleted.get() {
-					nondeleted_count += 1;
-				}
-			}
-			Err(vh) => {
-				table.push(format!("{:?}\t\t\tyes", vh));
 			}
 		}
 	}
 	format_table(table);
-
-	if refcount != nondeleted_count {
-		println!();
-		println!("Warning: refcount does not match number of non-deleted versions");
-	}
 }

src/garage/main.rs

@@ -127,16 +127,9 @@ async fn main() {
 		std::process::abort();
 	}));
 
-	// Parse arguments and dispatch command line
-	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
-
 	// Initialize logging as well as other libraries used in Garage
 	if std::env::var("RUST_LOG").is_err() {
-		let default_log = match &opt.cmd {
+		std::env::set_var("RUST_LOG", "netapp=info,garage=info")
-			Command::Server => "netapp=info,garage=info",
-			_ => "netapp=warn,garage=warn",
-		};
-		std::env::set_var("RUST_LOG", default_log)
 	}
 	tracing_subscriber::fmt()
 		.with_writer(std::io::stderr)
@@ -144,6 +137,9 @@ async fn main() {
 		.init();
 	sodiumoxide::init().expect("Unable to init sodiumoxide");
 
+	// Parse arguments and dispatch command line
+	let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
+
 	let res = match opt.cmd {
 		Command::Server => server::run_server(opt.config_file).await,
 		Command::OfflineRepair(repair_opt) => {
@@ -173,7 +169,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let net_key_hex_str = opt
 		.rpc_secret
 		.as_ref()
-		.or_else(|| config.as_ref().and_then(|c| c.rpc_secret.as_ref()))
+		.or_else(|| config.as_ref().map(|c| &c.rpc_secret))
 		.ok_or("No RPC secret provided")?;
 	let network_key = NetworkKey::from_slice(
 		&hex::decode(net_key_hex_str).err_context("Invalid RPC secret key (bad hex)")?[..],
@@ -186,9 +182,9 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk);
 
 	// Find and parse the address of the target host
-	let (id, addr, is_default_addr) = if let Some(h) = opt.rpc_host {
+	let (id, addr) = if let Some(h) = opt.rpc_host {
 		let (id, addrs) = parse_and_resolve_peer_addr(&h).ok_or_else(|| format!("Invalid RPC remote node identifier: {}. Expected format is <pubkey>@<IP or hostname>:<port>.", h))?;
-		(id, addrs[0], false)
+		(id, addrs[0])
 	} else {
 		let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
 			.err_context(READ_KEY_ERROR)?;
@@ -199,26 +195,24 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?
 				.next()
 				.ok_or_message("unable to resolve rpc_public_addr specified in config file")?;
-			(node_id, a, false)
+			(node_id, a)
 		} else {
 			let default_addr = SocketAddr::new(
 				"127.0.0.1".parse().unwrap(),
 				config.as_ref().unwrap().rpc_bind_addr.port(),
 			);
-			(node_id, default_addr, true)
+			warn!(
+				"Trying to contact Garage node at default address {}",
+				default_addr
+			);
+			warn!("If this doesn't work, consider adding rpc_public_addr in your config file or specifying the -h command line parameter.");
+			(node_id, default_addr)
 		}
 	};
 
 	// Connect to target host
-	if let Err(e) = netapp.clone().try_connect(addr, id).await {
+	netapp.clone().try_connect(addr, id).await
-		if is_default_addr {
+		.err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
-			warn!(
-				"Tried to contact Garage node at default address {}, which didn't work. If that address is wrong, consider setting rpc_public_addr in your config file.",
-				addr
-			);
-		}
-		Err(e).err_context("Unable to connect to destination RPC host. Check that you are using the same value of rpc_secret as them, and that you have their correct public key.")?;
-	}
 
 	let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
 	let admin_rpc_endpoint = netapp.endpoint::<AdminRpc, ()>(ADMIN_RPC_PATH.into());

src/garage/repair/offline.rs

@@ -1,5 +1,8 @@
 use std::path::PathBuf;
 
+use tokio::sync::watch;
+
+use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;
 
@@ -17,8 +20,12 @@ pub async fn offline_repair(config_file: PathBuf, opt: OfflineRepairOpt) -> Resu
 	info!("Loading configuration...");
 	let config = read_config(config_file)?;
 
+	info!("Initializing background runner...");
+	let (done_tx, done_rx) = watch::channel(false);
+	let (background, await_background_done) = BackgroundRunner::new(16, done_rx);
+
 	info!("Initializing Garage main data store...");
-	let garage = Garage::new(config)?;
+	let garage = Garage::new(config.clone(), background)?;
 
 	info!("Launching repair operation...");
 	match opt.what {
@@ -36,7 +43,13 @@ pub async fn offline_repair(config_file: PathBuf, opt: OfflineRepairOpt) -> Resu
 		}
 	}
 
-	info!("Repair operation finished, shutting down...");
+	info!("Repair operation finished, shutting down Garage internals...");
+	done_tx.send(true).unwrap();
+	drop(garage);
+
+	await_background_done.await?;
+
+	info!("Cleaning up...");
 
 	Ok(())
 }

src/garage/repair/online.rs

@@ -12,37 +12,38 @@ use garage_model::s3::version_table::*;
 use garage_table::*;
 use garage_util::background::*;
 use garage_util::error::Error;
-use garage_util::migrate::Migrate;
 
 use crate::*;
 
-pub async fn launch_online_repair(
+pub async fn launch_online_repair(garage: Arc<Garage>, opt: RepairOpt) {
-	garage: &Arc<Garage>,
-	bg: &BackgroundRunner,
-	opt: RepairOpt,
-) -> Result<(), Error> {
 	match opt.what {
 		RepairWhat::Tables => {
 			info!("Launching a full sync of tables");
-			garage.bucket_table.syncer.add_full_sync()?;
+			garage.bucket_table.syncer.add_full_sync();
-			garage.object_table.syncer.add_full_sync()?;
+			garage.object_table.syncer.add_full_sync();
-			garage.version_table.syncer.add_full_sync()?;
+			garage.version_table.syncer.add_full_sync();
-			garage.block_ref_table.syncer.add_full_sync()?;
+			garage.block_ref_table.syncer.add_full_sync();
-			garage.key_table.syncer.add_full_sync()?;
+			garage.key_table.syncer.add_full_sync();
 		}
 		RepairWhat::Versions => {
 			info!("Repairing the versions table");
-			bg.spawn_worker(RepairVersionsWorker::new(garage.clone()));
+			garage
+				.background
+				.spawn_worker(RepairVersionsWorker::new(garage.clone()));
 		}
 		RepairWhat::BlockRefs => {
 			info!("Repairing the block refs table");
-			bg.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
+			garage
+				.background
+				.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
 		}
 		RepairWhat::Blocks => {
 			info!("Repairing the stored blocks");
-			bg.spawn_worker(garage_block::repair::RepairWorker::new(
+			garage
-				garage.block_manager.clone(),
+				.background
-			));
+				.spawn_worker(garage_block::repair::RepairWorker::new(
+					garage.block_manager.clone(),
+				));
 		}
 		RepairWhat::Scrub { cmd } => {
 			let cmd = match cmd {
@@ -51,18 +52,13 @@ pub async fn launch_online_repair(
 				ScrubCmd::Resume => ScrubWorkerCommand::Resume,
 				ScrubCmd::Cancel => ScrubWorkerCommand::Cancel,
 				ScrubCmd::SetTranquility { tranquility } => {
-					garage
+					ScrubWorkerCommand::SetTranquility(tranquility)
-						.block_manager
-						.scrub_persister
-						.set_with(|x| x.tranquility = tranquility)?;
-					return Ok(());
 				}
 			};
 			info!("Sending command to scrub worker: {:?}", cmd);
-			garage.block_manager.send_scrub_command(cmd).await?;
+			garage.block_manager.send_scrub_command(cmd).await;
 		}
 	}
-	Ok(())
 }
 
 // ----
@@ -89,23 +85,25 @@ impl Worker for RepairVersionsWorker {
 		"Version repair worker".into()
 	}
 
-	fn status(&self) -> WorkerStatus {
+	fn info(&self) -> Option<String> {
-		WorkerStatus {
+		Some(format!("{} items done", self.counter))
-			progress: Some(self.counter.to_string()),
-			..Default::default()
-		}
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let (item_bytes, next_pos) = match self.garage.version_table.data.store.get_gt(&self.pos)? {
+		let item_bytes = match self.garage.version_table.data.store.get_gt(&self.pos)? {
-			Some((k, v)) => (v, k),
+			Some((k, v)) => {
+				self.pos = k;
+				v
+			}
 			None => {
 				info!("repair_versions: finished, done {}", self.counter);
 				return Ok(WorkerState::Done);
 			}
 		};
 
-		let version = Version::decode(&item_bytes).ok_or_message("Cannot decode Version")?;
+		self.counter += 1;
+
+		let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
 		if !version.deleted.get() {
 			let object = self
 				.garage
@@ -133,13 +131,10 @@ impl Worker for RepairVersionsWorker {
 			}
 		}
 
-		self.counter += 1;
-		self.pos = next_pos;
-
 		Ok(WorkerState::Busy)
 	}
 
-	async fn wait_for_work(&mut self) -> WorkerState {
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
 		unreachable!()
 	}
 }
@@ -168,24 +163,25 @@ impl Worker for RepairBlockrefsWorker {
 		"Block refs repair worker".into()
 	}
 
-	fn status(&self) -> WorkerStatus {
+	fn info(&self) -> Option<String> {
-		WorkerStatus {
+		Some(format!("{} items done", self.counter))
-			progress: Some(self.counter.to_string()),
-			..Default::default()
-		}
 	}
 
 	async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
-		let (item_bytes, next_pos) =
+		let item_bytes = match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
-			match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
+			Some((k, v)) => {
-				Some((k, v)) => (v, k),
+				self.pos = k;
-				None => {
+				v
-					info!("repair_block_ref: finished, done {}", self.counter);
+			}
-					return Ok(WorkerState::Done);
+			None => {
-				}
+				info!("repair_block_ref: finished, done {}", self.counter);
-			};
+				return Ok(WorkerState::Done);
+			}
+		};
 
-		let block_ref = BlockRef::decode(&item_bytes).ok_or_message("Cannot decode BlockRef")?;
+		self.counter += 1;
+
+		let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
 		if !block_ref.deleted.get() {
 			let version = self
 				.garage
@@ -210,13 +206,10 @@ impl Worker for RepairBlockrefsWorker {
 			}
 		}
 
-		self.counter += 1;
-		self.pos = next_pos;
-
 		Ok(WorkerState::Busy)
 	}
 
-	async fn wait_for_work(&mut self) -> WorkerState {
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
 		unreachable!()
 	}
 }

src/garage/server.rs

@@ -35,15 +35,12 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	#[cfg(feature = "metrics")]
 	let metrics_exporter = opentelemetry_prometheus::exporter().init();
 
-	info!("Initializing Garage main data store...");
-	let garage = Garage::new(config.clone())?;
-
 	info!("Initializing background runner...");
 	let watch_cancel = watch_shutdown_signal();
-	let (background, await_background_done) = BackgroundRunner::new(watch_cancel.clone());
+	let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
 
-	info!("Spawning Garage workers...");
+	info!("Initializing Garage main data store...");
-	garage.spawn_workers(&background);
+	let garage = Garage::new(config.clone(), background)?;
 
 	if config.admin.trace_sink.is_some() {
 		info!("Initialize tracing...");
@@ -66,7 +63,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone()));
 
 	info!("Create admin RPC handler...");
-	AdminRpcHandler::new(garage.clone(), background.clone());
+	AdminRpcHandler::new(garage.clone());
 
 	// ---- Launch public-facing API servers ----
 

src/garage/tests/bucket.rs

@@ -1,5 +1,4 @@
 use crate::common;
-use crate::common::ext::CommandExt;
 use aws_sdk_s3::model::BucketLocationConstraint;
 use aws_sdk_s3::output::DeleteBucketOutput;
 
@@ -9,27 +8,6 @@ async fn test_bucket_all() {
 	let bucket_name = "hello";
 
 	{
-		// Check bucket cannot be created if not authorized
-		ctx.garage
-			.command()
-			.args(["key", "deny"])
-			.args(["--create-bucket", &ctx.garage.key.id])
-			.quiet()
-			.expect_success_output("Could not deny key to create buckets");
-
-		// Try create bucket, should fail
-		let r = ctx.client.create_bucket().bucket(bucket_name).send().await;
-		assert!(r.is_err());
-	}
-	{
-		// Now allow key to create bucket
-		ctx.garage
-			.command()
-			.args(["key", "allow"])
-			.args(["--create-bucket", &ctx.garage.key.id])
-			.quiet()
-			.expect_success_output("Could not deny key to create buckets");
-
 		// Create bucket
 		//@TODO check with an invalid bucket name + with an already existing bucket
 		let r = ctx

src/garage/tests/s3/streaming_signature.rs

@@ -1,7 +1,6 @@
 use std::collections::HashMap;
 
 use crate::common;
-use crate::common::ext::CommandExt;
 use common::custom_requester::BodySignature;
 use hyper::Method;
 
@@ -106,13 +105,6 @@ async fn test_create_bucket_streaming() {
 	let ctx = common::context();
 	let bucket = "createbucket-streaming";
 
-	ctx.garage
-		.command()
-		.args(["key", "allow"])
-		.args(["--create-bucket", &ctx.garage.key.id])
-		.quiet()
-		.expect_success_output("Could not allow key to create buckets");
-
 	{
 		// create bucket
 		let _ = ctx

src/k2v-client/Cargo.toml

@@ -22,7 +22,7 @@ tokio = "1.17.0"
 
 # cli deps
 clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
-garage_util = { version = "0.8.1", path = "../util", optional = true }
+garage_util = { version = "0.8.0", path = "../util", optional = true }
 
 
 [features]

src/model/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_model"
-version = "0.8.1"
+version = "0.8.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,11 +14,11 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_db = { version = "0.8.1", default-features = false, path = "../db" }
+garage_db = { version = "0.8.0", path = "../db" }
-garage_rpc = { version = "0.8.1", path = "../rpc" }
+garage_rpc = { version = "0.8.0", path = "../rpc" }
-garage_table = { version = "0.8.1", path = "../table" }
+garage_table = { version = "0.8.0", path = "../table" }
-garage_block = { version = "0.8.1", path = "../block" }
+garage_block = { version = "0.8.0", path = "../block" }
-garage_util = { version = "0.8.1", path = "../util" }
+garage_util = { version = "0.8.0", path = "../util" }
 
 async-trait = "0.1.7"
 arc-swap = "1.0"
@@ -30,6 +30,7 @@ tracing = "0.1.30"
 rand = "0.8"
 zstd = { version = "0.9", default-features = false }
 
+rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 
@@ -41,7 +42,6 @@ opentelemetry = "0.17"
 netapp = "0.5"
 
 [features]
-default = [ "sled" ]
 k2v = [ "garage_util/k2v" ]
 lmdb = [ "garage_db/lmdb" ]
 sled = [ "garage_db/sled" ]

src/model/bucket_alias_table.rs

@@ -1,26 +1,18 @@
+use serde::{Deserialize, Serialize};
+
 use garage_util::data::*;
 
 use garage_table::crdt::*;
 use garage_table::*;
 
-mod v08 {
+/// The bucket alias table holds the names given to buckets
-	use garage_util::crdt;
+/// in the global namespace.
-	use garage_util::data::Uuid;
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	use serde::{Deserialize, Serialize};
+pub struct BucketAlias {
-
+	name: String,
-	/// The bucket alias table holds the names given to buckets
+	pub state: crdt::Lww<Option<Uuid>>,
-	/// in the global namespace.
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct BucketAlias {
-		pub(super) name: String,
-		pub state: crdt::Lww<Option<Uuid>>,
-	}
-
-	impl garage_util::migrate::InitialFormat for BucketAlias {}
 }
 
-pub use v08::*;
-
 impl BucketAlias {
 	pub fn new(name: String, ts: u64, bucket_id: Option<Uuid>) -> Option<Self> {
 		if !is_valid_bucket_name(&name) {

src/model/bucket_table.rs

@@ -1,3 +1,5 @@
+use serde::{Deserialize, Serialize};
+
 use garage_table::crdt::*;
 use garage_table::*;
 use garage_util::data::*;
@@ -5,82 +7,71 @@ use garage_util::time::*;
 
 use crate::permission::BucketKeyPerm;
 
-mod v08 {
+/// A bucket is a collection of objects
-	use crate::permission::BucketKeyPerm;
+///
-	use garage_util::crdt;
+/// Its parameters are not directly accessible as:
-	use garage_util::data::Uuid;
+///  - It must be possible to merge paramaters, hence the use of a LWW CRDT.
-	use serde::{Deserialize, Serialize};
+///  - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
-
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	/// A bucket is a collection of objects
+pub struct Bucket {
-	///
+	/// ID of the bucket
-	/// Its parameters are not directly accessible as:
+	pub id: Uuid,
-	///  - It must be possible to merge paramaters, hence the use of a LWW CRDT.
+	/// State, and configuration if not deleted, of the bucket
-	///  - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
+	pub state: crdt::Deletable<BucketParams>,
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct Bucket {
-		/// ID of the bucket
-		pub id: Uuid,
-		/// State, and configuration if not deleted, of the bucket
-		pub state: crdt::Deletable<BucketParams>,
-	}
-
-	/// Configuration for a bucket
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct BucketParams {
-		/// Bucket's creation date
-		pub creation_date: u64,
-		/// Map of key with access to the bucket, and what kind of access they give
-		pub authorized_keys: crdt::Map<String, BucketKeyPerm>,
-
-		/// Map of aliases that are or have been given to this bucket
-		/// in the global namespace
-		/// (not authoritative: this is just used as an indication to
-		/// map back to aliases when doing ListBuckets)
-		pub aliases: crdt::LwwMap<String, bool>,
-		/// Map of aliases that are or have been given to this bucket
-		/// in namespaces local to keys
-		/// key = (access key id, alias name)
-		pub local_aliases: crdt::LwwMap<(String, String), bool>,
-
-		/// Whether this bucket is allowed for website access
-		/// (under all of its global alias names),
-		/// and if so, the website configuration XML document
-		pub website_config: crdt::Lww<Option<WebsiteConfig>>,
-		/// CORS rules
-		pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
-		/// Bucket quotas
-		#[serde(default)]
-		pub quotas: crdt::Lww<BucketQuotas>,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct WebsiteConfig {
-		pub index_document: String,
-		pub error_document: Option<String>,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct CorsRule {
-		pub id: Option<String>,
-		pub max_age_seconds: Option<u64>,
-		pub allow_origins: Vec<String>,
-		pub allow_methods: Vec<String>,
-		pub allow_headers: Vec<String>,
-		pub expose_headers: Vec<String>,
-	}
-
-	#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-	pub struct BucketQuotas {
-		/// Maximum size in bytes (bucket size = sum of sizes of objects in the bucket)
-		pub max_size: Option<u64>,
-		/// Maximum number of non-deleted objects in the bucket
-		pub max_objects: Option<u64>,
-	}
-
-	impl garage_util::migrate::InitialFormat for Bucket {}
 }
 
-pub use v08::*;
+/// Configuration for a bucket
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct BucketParams {
+	/// Bucket's creation date
+	pub creation_date: u64,
+	/// Map of key with access to the bucket, and what kind of access they give
+	pub authorized_keys: crdt::Map<String, BucketKeyPerm>,
+
+	/// Map of aliases that are or have been given to this bucket
+	/// in the global namespace
+	/// (not authoritative: this is just used as an indication to
+	/// map back to aliases when doing ListBuckets)
+	pub aliases: crdt::LwwMap<String, bool>,
+	/// Map of aliases that are or have been given to this bucket
+	/// in namespaces local to keys
+	/// key = (access key id, alias name)
+	pub local_aliases: crdt::LwwMap<(String, String), bool>,
+
+	/// Whether this bucket is allowed for website access
+	/// (under all of its global alias names),
+	/// and if so, the website configuration XML document
+	pub website_config: crdt::Lww<Option<WebsiteConfig>>,
+	/// CORS rules
+	pub cors_config: crdt::Lww<Option<Vec<CorsRule>>>,
+	/// Bucket quotas
+	#[serde(default)]
+	pub quotas: crdt::Lww<BucketQuotas>,
+}
+
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct WebsiteConfig {
+	pub index_document: String,
+	pub error_document: Option<String>,
+}
+
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct CorsRule {
+	pub id: Option<String>,
+	pub max_age_seconds: Option<u64>,
+	pub allow_origins: Vec<String>,
+	pub allow_methods: Vec<String>,
+	pub allow_headers: Vec<String>,
+	pub expose_headers: Vec<String>,
+}
+
+#[derive(Default, PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+pub struct BucketQuotas {
+	/// Maximum size in bytes (bucket size = sum of sizes of objects in the bucket)
+	pub max_size: Option<u64>,
+	/// Maximum number of non-deleted objects in the bucket
+	pub max_objects: Option<u64>,
+}
 
 impl AutoCrdt for BucketQuotas {
 	const WARN_IF_DIFFERENT: bool = true;

src/model/garage.rs

@@ -8,10 +8,10 @@ use garage_util::background::*;
 use garage_util::config::*;
 use garage_util::error::*;
 
-use garage_rpc::replication_mode::ReplicationMode;
 use garage_rpc::system::System;
 
 use garage_block::manager::*;
+use garage_table::replication::ReplicationMode;
 use garage_table::replication::TableFullReplication;
 use garage_table::replication::TableShardedReplication;
 use garage_table::*;
@@ -33,14 +33,11 @@ use crate::k2v::{item_table::*, poll::*, rpc::*};
 pub struct Garage {
 	/// The parsed configuration Garage is running
 	pub config: Config,
-	/// The set of background variables that can be viewed/modified at runtime
-	pub bg_vars: vars::BgVars,
-
-	/// The replication mode of this cluster
-	pub replication_mode: ReplicationMode,
 
 	/// The local database
 	pub db: db::Db,
+	/// A background job runner
+	pub background: Arc<BackgroundRunner>,
 	/// The membership manager
 	pub system: Arc<System>,
 	/// The block manager
@@ -78,7 +75,7 @@ pub struct GarageK2V {
 
 impl Garage {
 	/// Create and run garage
-	pub fn new(config: Config) -> Result<Arc<Self>, Error> {
+	pub fn new(config: Config, background: Arc<BackgroundRunner>) -> Result<Arc<Self>, Error> {
 		// Create meta dir and data dir if they don't exist already
 		std::fs::create_dir_all(&config.metadata_dir)
 			.ok_or_message("Unable to create Garage metadata directory")?;
@@ -159,7 +156,7 @@ impl Garage {
 		};
 
 		let network_key = NetworkKey::from_slice(
-			&hex::decode(&config.rpc_secret.as_ref().unwrap()).expect("Invalid RPC secret key")[..],
+			&hex::decode(&config.rpc_secret).expect("Invalid RPC secret key")[..],
 		)
 		.expect("Invalid RPC secret key");
 
@@ -167,7 +164,12 @@ impl Garage {
 			.expect("Invalid replication_mode in config file.");
 
 		info!("Initialize membership management system...");
-		let system = System::new(network_key, replication_mode, &config)?;
+		let system = System::new(
+			network_key,
+			background.clone(),
+			replication_mode.replication_factor(),
+			&config,
+		)?;
 
 		let data_rep_param = TableShardedReplication {
 			system: system.clone(),
@@ -225,6 +227,7 @@ impl Garage {
 		info!("Initialize version_table...");
 		let version_table = Table::new(
 			VersionTable {
+				background: background.clone(),
 				block_ref_table: block_ref_table.clone(),
 			},
 			meta_rep_param.clone(),
@@ -239,6 +242,7 @@ impl Garage {
 		#[allow(clippy::redundant_clone)]
 		let object_table = Table::new(
 			ObjectTable {
+				background: background.clone(),
 				version_table: version_table.clone(),
 				object_counter_table: object_counter_table.clone(),
 			},
@@ -251,16 +255,11 @@ impl Garage {
 		#[cfg(feature = "k2v")]
 		let k2v = GarageK2V::new(system.clone(), &db, meta_rep_param);
 
-		// Initialize bg vars
-		let mut bg_vars = vars::BgVars::new();
-		block_manager.register_bg_vars(&mut bg_vars);
-
 		// -- done --
 		Ok(Arc::new(Self {
 			config,
-			bg_vars,
-			replication_mode,
 			db,
+			background,
 			system,
 			block_manager,
 			bucket_table,
@@ -275,22 +274,6 @@ impl Garage {
 		}))
 	}
 
-	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
-		self.block_manager.spawn_workers(bg);
-
-		self.bucket_table.spawn_workers(bg);
-		self.bucket_alias_table.spawn_workers(bg);
-		self.key_table.spawn_workers(bg);
-
-		self.object_table.spawn_workers(bg);
-		self.object_counter_table.spawn_workers(bg);
-		self.version_table.spawn_workers(bg);
-		self.block_ref_table.spawn_workers(bg);
-
-		#[cfg(feature = "k2v")]
-		self.k2v.spawn_workers(bg);
-	}
-
 	pub fn bucket_helper(&self) -> helper::bucket::BucketHelper {
 		helper::bucket::BucketHelper(self)
 	}
@@ -325,9 +308,4 @@ impl GarageK2V {
 			rpc,
 		}
 	}
-
-	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
-		self.item_table.spawn_workers(bg);
-		self.counter_table.spawn_workers(bg);
-	}
 }

src/model/index_counter.rs

@@ -1,18 +1,19 @@
 use core::ops::Bound;
-use std::collections::{BTreeMap, HashMap};
+use std::collections::{hash_map, BTreeMap, HashMap};
 use std::marker::PhantomData;
 use std::sync::Arc;
 
+use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
+use tokio::sync::{mpsc, watch};
 
 use garage_db as db;
 
 use garage_rpc::ring::Ring;
 use garage_rpc::system::System;
-use garage_util::background::BackgroundRunner;
+use garage_util::background::*;
 use garage_util::data::*;
 use garage_util::error::*;
-use garage_util::migrate::Migrate;
 use garage_util::time::*;
 
 use garage_table::crdt::*;
@@ -30,44 +31,14 @@ pub trait CountedItem: Clone + PartialEq + Send + Sync + 'static {
 	fn counts(&self) -> Vec<(&'static str, i64)>;
 }
 
-mod v08 {
+/// A counter entry in the global table
-	use super::CountedItem;
+#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
-	use garage_util::data::Uuid;
+pub struct CounterEntry<T: CountedItem> {
-	use serde::{Deserialize, Serialize};
+	pub pk: T::CP,
-	use std::collections::BTreeMap;
+	pub sk: T::CS,
-
+	pub values: BTreeMap<String, CounterValue>,
-	// ---- Global part (the table everyone queries) ----
-
-	/// A counter entry in the global table
-	#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]
-	pub struct CounterEntry<T: CountedItem> {
-		pub pk: T::CP,
-		pub sk: T::CS,
-		pub values: BTreeMap<String, CounterValue>,
-	}
-
-	/// A counter entry in the global table
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct CounterValue {
-		pub node_values: BTreeMap<Uuid, (u64, i64)>,
-	}
-
-	impl<T: CountedItem> garage_util::migrate::InitialFormat for CounterEntry<T> {}
-
-	// ---- Local part (the counter we maintain transactionnaly on each node) ----
-
-	#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
-	pub(super) struct LocalCounterEntry<T: CountedItem> {
-		pub(super) pk: T::CP,
-		pub(super) sk: T::CS,
-		pub(super) values: BTreeMap<String, (u64, i64)>,
-	}
-
-	impl<T: CountedItem> garage_util::migrate::InitialFormat for LocalCounterEntry<T> {}
 }
 
-pub use v08::*;
-
 impl<T: CountedItem> Entry<T::CP, T::CS> for CounterEntry<T> {
 	fn partition_key(&self) -> &T::CP {
 		&self.pk
@@ -109,6 +80,12 @@ impl<T: CountedItem> CounterEntry<T> {
 	}
 }
 
+/// A counter entry in the global table
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct CounterValue {
+	pub node_values: BTreeMap<Uuid, (u64, i64)>,
+}
+
 impl<T: CountedItem> Crdt for CounterEntry<T> {
 	fn merge(&mut self, other: &Self) {
 		for (name, e2) in other.values.iter() {
@@ -165,6 +142,7 @@ impl<T: CountedItem> TableSchema for CounterTable<T> {
 pub struct IndexCounter<T: CountedItem> {
 	this_node: Uuid,
 	local_counter: db::Tree,
+	propagate_tx: mpsc::UnboundedSender<(T::CP, T::CS, LocalCounterEntry<T>)>,
 	pub table: Arc<Table<CounterTable<T>, TableShardedReplication>>,
 }
 
@@ -174,11 +152,16 @@ impl<T: CountedItem> IndexCounter<T> {
 		replication: TableShardedReplication,
 		db: &db::Db,
 	) -> Arc<Self> {
-		Arc::new(Self {
+		let background = system.background.clone();
+
+		let (propagate_tx, propagate_rx) = mpsc::unbounded_channel();
+
+		let this = Arc::new(Self {
 			this_node: system.id,
 			local_counter: db
 				.open_tree(format!("local_counter_v2:{}", T::COUNTER_TABLE_NAME))
 				.expect("Unable to open local counter tree"),
+			propagate_tx,
 			table: Table::new(
 				CounterTable {
 					_phantom_t: Default::default(),
@@ -187,11 +170,16 @@ impl<T: CountedItem> IndexCounter<T> {
 				system,
 				db,
 			),
-		})
+		});
-	}
 
-	pub fn spawn_workers(&self, bg: &BackgroundRunner) {
+		background.spawn_worker(IndexPropagatorWorker {
-		self.table.spawn_workers(bg);
+			index_counter: this.clone(),
+			propagate_rx,
+			buf: HashMap::new(),
+			errors: 0,
+		});
+
+		this
 	}
 
 	pub fn count(
@@ -220,9 +208,11 @@ impl<T: CountedItem> IndexCounter<T> {
 		let tree_key = self.table.data.tree_key(pk, sk);
 
 		let mut entry = match tx.get(&self.local_counter, &tree_key[..])? {
-			Some(old_bytes) => LocalCounterEntry::<T>::decode(&old_bytes)
+			Some(old_bytes) => {
-				.ok_or_message("Cannot decode local counter entry")
+				rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(&old_bytes)
-				.map_err(db::TxError::Abort)?,
+					.map_err(Error::RmpDecode)
+					.map_err(db::TxError::Abort)?
+			}
 			None => LocalCounterEntry {
 				pk: pk.clone(),
 				sk: sk.clone(),
@@ -237,14 +227,17 @@ impl<T: CountedItem> IndexCounter<T> {
 			ent.1 += *inc;
 		}
 
-		let new_entry_bytes = entry
+		let new_entry_bytes = rmp_to_vec_all_named(&entry)
-			.encode()
 			.map_err(Error::RmpEncode)
 			.map_err(db::TxError::Abort)?;
 		tx.insert(&self.local_counter, &tree_key[..], new_entry_bytes)?;
 
-		let dist_entry = entry.into_counter_entry(self.this_node);
+		if let Err(e) = self.propagate_tx.send((pk.clone(), sk.clone(), entry)) {
-		self.table.queue_insert(tx, &dist_entry)?;
+			error!(
+				"Could not propagate updated counter values, failed to send to channel: {}",
+				e
+			);
+		}
 
 		Ok(())
 	}
@@ -257,6 +250,23 @@ impl<T: CountedItem> IndexCounter<T> {
 		TS: TableSchema<E = T>,
 		TR: TableReplication,
 	{
+		let save_counter_entry = |entry: CounterEntry<T>| -> Result<(), Error> {
+			let entry_k = self
+				.table
+				.data
+				.tree_key(entry.partition_key(), entry.sort_key());
+			self.table
+				.data
+				.update_entry_with(&entry_k, |ent| match ent {
+					Some(mut ent) => {
+						ent.merge(&entry);
+						ent
+					}
+					None => entry.clone(),
+				})?;
+			Ok(())
+		};
+
 		// 1. Set all old local counters to zero
 		let now = now_msec();
 		let mut next_start: Option<Vec<u8>> = None;
@@ -279,22 +289,20 @@ impl<T: CountedItem> IndexCounter<T> {
 
 			info!("zeroing old counters... ({})", hex::encode(&batch[0].0));
 			for (local_counter_k, local_counter) in batch {
-				let mut local_counter = LocalCounterEntry::<T>::decode(&local_counter)
+				let mut local_counter =
-					.ok_or_message("Cannot decode local counter entry")?;
+					rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(&local_counter)?;
 
 				for (_, tv) in local_counter.values.iter_mut() {
 					tv.0 = std::cmp::max(tv.0 + 1, now);
 					tv.1 = 0;
 				}
 
-				let local_counter_bytes = local_counter.encode()?;
+				let local_counter_bytes = rmp_to_vec_all_named(&local_counter)?;
 				self.local_counter
 					.insert(&local_counter_k, &local_counter_bytes)?;
 
 				let counter_entry = local_counter.into_counter_entry(self.this_node);
-				self.local_counter
+				save_counter_entry(counter_entry)?;
-					.db()
-					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;
 
 				next_start = Some(local_counter_k);
 			}
@@ -335,8 +343,9 @@ impl<T: CountedItem> IndexCounter<T> {
 				let local_counter_key = self.table.data.tree_key(pk, sk);
 				let mut local_counter = match self.local_counter.get(&local_counter_key)? {
 					Some(old_bytes) => {
-						let ent = LocalCounterEntry::<T>::decode(&old_bytes)
+						let ent = rmp_serde::decode::from_read_ref::<_, LocalCounterEntry<T>>(
-							.ok_or_message("Cannot decode local counter entry")?;
+							&old_bytes,
+						)?;
 						assert!(ent.pk == *pk);
 						assert!(ent.sk == *sk);
 						ent
@@ -353,14 +362,12 @@ impl<T: CountedItem> IndexCounter<T> {
 					tv.1 += v;
 				}
 
-				let local_counter_bytes = local_counter.encode()?;
+				let local_counter_bytes = rmp_to_vec_all_named(&local_counter)?;
 				self.local_counter
 					.insert(&local_counter_key, local_counter_bytes)?;
 
 				let counter_entry = local_counter.into_counter_entry(self.this_node);
-				self.local_counter
+				save_counter_entry(counter_entry)?;
-					.db()
-					.transaction(|mut tx| self.table.queue_insert(&mut tx, &counter_entry))?;
 
 				next_start = Some(counted_entry_k);
 			}
@@ -371,7 +378,104 @@ impl<T: CountedItem> IndexCounter<T> {
 	}
 }
 
-// ----
+struct IndexPropagatorWorker<T: CountedItem> {
+	index_counter: Arc<IndexCounter<T>>,
+	propagate_rx: mpsc::UnboundedReceiver<(T::CP, T::CS, LocalCounterEntry<T>)>,
+
+	buf: HashMap<Vec<u8>, CounterEntry<T>>,
+	errors: usize,
+}
+
+impl<T: CountedItem> IndexPropagatorWorker<T> {
+	fn add_ent(&mut self, pk: T::CP, sk: T::CS, counters: LocalCounterEntry<T>) {
+		let tree_key = self.index_counter.table.data.tree_key(&pk, &sk);
+		let dist_entry = counters.into_counter_entry(self.index_counter.this_node);
+		match self.buf.entry(tree_key) {
+			hash_map::Entry::Vacant(e) => {
+				e.insert(dist_entry);
+			}
+			hash_map::Entry::Occupied(mut e) => {
+				e.get_mut().merge(&dist_entry);
+			}
+		}
+	}
+}
+
+#[async_trait]
+impl<T: CountedItem> Worker for IndexPropagatorWorker<T> {
+	fn name(&self) -> String {
+		format!("{} index counter propagator", T::COUNTER_TABLE_NAME)
+	}
+
+	fn info(&self) -> Option<String> {
+		if !self.buf.is_empty() {
+			Some(format!("{} items in queue", self.buf.len()))
+		} else {
+			None
+		}
+	}
+
+	async fn work(&mut self, must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
+		// This loop batches updates to counters to be sent all at once.
+		// They are sent once the propagate_rx channel has been emptied (or is closed).
+		let closed = loop {
+			match self.propagate_rx.try_recv() {
+				Ok((pk, sk, counters)) => {
+					self.add_ent(pk, sk, counters);
+				}
+				Err(mpsc::error::TryRecvError::Empty) => break false,
+				Err(mpsc::error::TryRecvError::Disconnected) => break true,
+			}
+		};
+
+		if !self.buf.is_empty() {
+			let entries_k = self.buf.keys().take(100).cloned().collect::<Vec<_>>();
+			let entries = entries_k.iter().map(|k| self.buf.get(k).unwrap());
+			if let Err(e) = self.index_counter.table.insert_many(entries).await {
+				self.errors += 1;
+				if self.errors >= 2 && *must_exit.borrow() {
+					error!("({}) Could not propagate {} counter values: {}, these counters will not be updated correctly.", T::COUNTER_TABLE_NAME, self.buf.len(), e);
+					return Ok(WorkerState::Done);
+				}
+				// Propagate error up to worker manager, it will log it, increment a counter,
+				// and sleep for a certain delay (with exponential backoff), waiting for
+				// things to go back to normal
+				return Err(e);
+			} else {
+				for k in entries_k {
+					self.buf.remove(&k);
+				}
+				self.errors = 0;
+			}
+
+			return Ok(WorkerState::Busy);
+		} else if closed {
+			return Ok(WorkerState::Done);
+		} else {
+			return Ok(WorkerState::Idle);
+		}
+	}
+
+	async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerState {
+		match self.propagate_rx.recv().await {
+			Some((pk, sk, counters)) => {
+				self.add_ent(pk, sk, counters);
+				WorkerState::Busy
+			}
+			None => match self.buf.is_empty() {
+				false => WorkerState::Busy,
+				true => WorkerState::Done,
+			},
+		}
+	}
+}
+
+#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
+struct LocalCounterEntry<T: CountedItem> {
+	pk: T::CP,
+	sk: T::CS,
+	values: BTreeMap<String, (u64, i64)>,
+}
 
 impl<T: CountedItem> LocalCounterEntry<T> {
 	fn into_counter_entry(self, this_node: Uuid) -> CounterEntry<T> {

src/model/k2v/item_table.rs

@@ -1,8 +1,7 @@
+use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 use std::sync::Arc;
 
-use serde::{Deserialize, Serialize};
-
 use garage_db as db;
 use garage_util::data::*;
 
@@ -18,42 +17,31 @@ pub const CONFLICTS: &str = "conflicts";
 pub const VALUES: &str = "values";
 pub const BYTES: &str = "bytes";
 
-mod v08 {
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	use crate::k2v::causality::K2VNodeId;
+pub struct K2VItem {
-	use garage_util::data::Uuid;
+	pub partition: K2VItemPartition,
-	use serde::{Deserialize, Serialize};
+	pub sort_key: String,
-	use std::collections::BTreeMap;
 
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	items: BTreeMap<K2VNodeId, DvvsEntry>,
-	pub struct K2VItem {
-		pub partition: K2VItemPartition,
-		pub sort_key: String,
-
-		pub(super) items: BTreeMap<K2VNodeId, DvvsEntry>,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Hash)]
-	pub struct K2VItemPartition {
-		pub bucket_id: Uuid,
-		pub partition_key: String,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct DvvsEntry {
-		pub(super) t_discard: u64,
-		pub(super) values: Vec<(u64, DvvsValue)>,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub enum DvvsValue {
-		Value(#[serde(with = "serde_bytes")] Vec<u8>),
-		Deleted,
-	}
-
-	impl garage_util::migrate::InitialFormat for K2VItem {}
 }
 
-pub use v08::*;
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Hash)]
+pub struct K2VItemPartition {
+	pub bucket_id: Uuid,
+	pub partition_key: String,
+}
+
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+struct DvvsEntry {
+	t_discard: u64,
+	values: Vec<(u64, DvvsValue)>,
+}
+
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub enum DvvsValue {
+	Value(#[serde(with = "serde_bytes")] Vec<u8>),
+	Deleted,
+}
 
 impl K2VItem {
 	/// Creates a new K2VItem when no previous entry existed in the db

src/model/k2v/rpc.rs

@@ -273,9 +273,14 @@ impl K2VRpcHandler {
 	}
 
 	fn local_insert(&self, item: &InsertedItem) -> Result<Option<K2VItem>, Error> {
+		let tree_key = self
+			.item_table
+			.data
+			.tree_key(&item.partition, &item.sort_key);
+
 		self.item_table
 			.data
-			.update_entry_with(&item.partition, &item.sort_key, |ent| {
+			.update_entry_with(&tree_key[..], |ent| {
 				let mut ent = ent.unwrap_or_else(|| {
 					K2VItem::new(
 						item.partition.bucket_id,

src/model/key_table.rs

@@ -1,121 +1,45 @@
 use serde::{Deserialize, Serialize};
 
-use garage_util::crdt::{self, Crdt};
+use garage_table::crdt::*;
+use garage_table::*;
 use garage_util::data::*;
 
-use garage_table::{DeletedFilter, EmptyKey, Entry, TableSchema};
-
 use crate::permission::BucketKeyPerm;
 
-pub(crate) mod v05 {
+use crate::prev::v051::key_table as old;
-	use garage_util::crdt;
-	use serde::{Deserialize, Serialize};
 
-	/// An api key
+/// An api key
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct Key {
+pub struct Key {
-		/// The id of the key (immutable), used as partition key
+	/// The id of the key (immutable), used as partition key
-		pub key_id: String,
+	pub key_id: String,
 
-		/// The secret_key associated
+	/// Internal state of the key
-		pub secret_key: String,
+	pub state: crdt::Deletable<KeyParams>,
-
-		/// Name for the key
-		pub name: crdt::Lww<String>,
-
-		/// Is the key deleted
-		pub deleted: crdt::Bool,
-
-		/// Buckets in which the key is authorized. Empty if `Key` is deleted
-		// CRDT interaction: deleted implies authorized_buckets is empty
-		pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
-	}
-
-	/// Permission given to a key in a bucket
-	#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct PermissionSet {
-		/// The key can be used to read the bucket
-		pub allow_read: bool,
-		/// The key can be used to write in the bucket
-		pub allow_write: bool,
-	}
-
-	impl crdt::AutoCrdt for PermissionSet {
-		const WARN_IF_DIFFERENT: bool = true;
-	}
-
-	impl garage_util::migrate::InitialFormat for Key {}
 }
 
-mod v08 {
+/// Configuration for a key
-	use super::v05;
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	use crate::permission::BucketKeyPerm;
+pub struct KeyParams {
-	use garage_util::crdt;
+	/// The secret_key associated (immutable)
-	use garage_util::data::Uuid;
+	pub secret_key: String,
-	use serde::{Deserialize, Serialize};
 
-	/// An api key
+	/// Name for the key
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub name: crdt::Lww<String>,
-	pub struct Key {
-		/// The id of the key (immutable), used as partition key
-		pub key_id: String,
 
-		/// Internal state of the key
+	/// Flag to allow users having this key to create buckets
-		pub state: crdt::Deletable<KeyParams>,
+	pub allow_create_bucket: crdt::Lww<bool>,
-	}
 
-	/// Configuration for a key
+	/// If the key is present: it gives some permissions,
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	/// a map of bucket IDs (uuids) to permissions.
-	pub struct KeyParams {
+	/// Otherwise no permissions are granted to key
-		/// The secret_key associated (immutable)
+	pub authorized_buckets: crdt::Map<Uuid, BucketKeyPerm>,
-		pub secret_key: String,
 
-		/// Name for the key
+	/// A key can have a local view of buckets names it is
-		pub name: crdt::Lww<String>,
+	/// the only one to see, this is the namespace for these aliases
-
+	pub local_aliases: crdt::LwwMap<String, Option<Uuid>>,
-		/// Flag to allow users having this key to create buckets
-		pub allow_create_bucket: crdt::Lww<bool>,
-
-		/// If the key is present: it gives some permissions,
-		/// a map of bucket IDs (uuids) to permissions.
-		/// Otherwise no permissions are granted to key
-		pub authorized_buckets: crdt::Map<Uuid, BucketKeyPerm>,
-
-		/// A key can have a local view of buckets names it is
-		/// the only one to see, this is the namespace for these aliases
-		pub local_aliases: crdt::LwwMap<String, Option<Uuid>>,
-	}
-
-	impl garage_util::migrate::Migrate for Key {
-		type Previous = v05::Key;
-
-		fn migrate(old_k: v05::Key) -> Key {
-			let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone());
-
-			let state = if old_k.deleted.get() {
-				crdt::Deletable::Deleted
-			} else {
-				// Authorized buckets is ignored here,
-				// migration is performed in specific migration code in
-				// garage/migrate.rs
-				crdt::Deletable::Present(KeyParams {
-					secret_key: old_k.secret_key,
-					name,
-					allow_create_bucket: crdt::Lww::new(false),
-					authorized_buckets: crdt::Map::new(),
-					local_aliases: crdt::LwwMap::new(),
-				})
-			};
-			Key {
-				key_id: old_k.key_id,
-				state,
-			}
-		}
-	}
 }
 
-pub use v08::*;
-
 impl KeyParams {
 	fn new(secret_key: &str, name: &str) -> Self {
 		KeyParams {
@@ -249,4 +173,28 @@ impl TableSchema for KeyTable {
 			}
 		}
 	}
+
+	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
+		let old_k = rmp_serde::decode::from_read_ref::<_, old::Key>(bytes).ok()?;
+		let name = crdt::Lww::raw(old_k.name.timestamp(), old_k.name.get().clone());
+
+		let state = if old_k.deleted.get() {
+			crdt::Deletable::Deleted
+		} else {
+			// Authorized buckets is ignored here,
+			// migration is performed in specific migration code in
+			// garage/migrate.rs
+			crdt::Deletable::Present(KeyParams {
+				secret_key: old_k.secret_key,
+				name,
+				allow_create_bucket: crdt::Lww::new(false),
+				authorized_buckets: crdt::Map::new(),
+				local_aliases: crdt::LwwMap::new(),
+			})
+		};
+		Some(Key {
+			key_id: old_k.key_id,
+			state,
+		})
+	}
 }

src/model/migrate.rs

@@ -2,7 +2,6 @@ use std::sync::Arc;
 
 use garage_util::crdt::*;
 use garage_util::data::*;
-use garage_util::encode::nonversioned_decode;
 use garage_util::error::Error as GarageError;
 use garage_util::time::*;
 
@@ -29,8 +28,8 @@ impl Migrate {
 		let mut old_buckets = vec![];
 		for res in tree.iter().map_err(GarageError::from)? {
 			let (_k, v) = res.map_err(GarageError::from)?;
-			let bucket =
+			let bucket = rmp_serde::decode::from_read_ref::<_, old_bucket::Bucket>(&v[..])
-				nonversioned_decode::<old_bucket::Bucket>(&v[..]).map_err(GarageError::from)?;
+				.map_err(GarageError::from)?;
 			old_buckets.push(bucket);
 		}
 

src/model/prev/v051/bucket_table.rs

@@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize};
 use garage_table::crdt::Crdt;
 use garage_table::*;
 
-use crate::key_table::v05::PermissionSet;
+use super::key_table::PermissionSet;
 
 /// A bucket is a collection of objects
 ///

src/model/prev/v051/key_table.rs

@@ -0,0 +1,50 @@
+use serde::{Deserialize, Serialize};
+
+use garage_table::crdt::*;
+use garage_table::*;
+
+/// An api key
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct Key {
+	/// The id of the key (immutable), used as partition key
+	pub key_id: String,
+
+	/// The secret_key associated
+	pub secret_key: String,
+
+	/// Name for the key
+	pub name: crdt::Lww<String>,
+
+	/// Is the key deleted
+	pub deleted: crdt::Bool,
+
+	/// Buckets in which the key is authorized. Empty if `Key` is deleted
+	// CRDT interaction: deleted implies authorized_buckets is empty
+	pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
+}
+
+/// Permission given to a key in a bucket
+#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct PermissionSet {
+	/// The key can be used to read the bucket
+	pub allow_read: bool,
+	/// The key can be used to write in the bucket
+	pub allow_write: bool,
+}
+
+impl AutoCrdt for PermissionSet {
+	const WARN_IF_DIFFERENT: bool = true;
+}
+
+impl Crdt for Key {
+	fn merge(&mut self, other: &Self) {
+		self.name.merge(&other.name);
+		self.deleted.merge(&other.deleted);
+
+		if self.deleted.get() {
+			self.authorized_buckets.clear();
+		} else {
+			self.authorized_buckets.merge(&other.authorized_buckets);
+		}
+	}
+}

src/model/prev/v051/mod.rs

@@ -1 +1,4 @@
 pub(crate) mod bucket_table;
+pub(crate) mod key_table;
+pub(crate) mod object_table;
+pub(crate) mod version_table;

src/model/prev/v051/object_table.rs

@@ -0,0 +1,149 @@
+use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
+
+use garage_util::data::*;
+
+use garage_table::crdt::*;
+
+/// An object
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct Object {
+	/// The bucket in which the object is stored, used as partition key
+	pub bucket: String,
+
+	/// The key at which the object is stored in its bucket, used as sorting key
+	pub key: String,
+
+	/// The list of currenty stored versions of the object
+	versions: Vec<ObjectVersion>,
+}
+
+impl Object {
+	/// Get a list of currently stored versions of `Object`
+	pub fn versions(&self) -> &[ObjectVersion] {
+		&self.versions[..]
+	}
+}
+
+/// Informations about a version of an object
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct ObjectVersion {
+	/// Id of the version
+	pub uuid: Uuid,
+	/// Timestamp of when the object was created
+	pub timestamp: u64,
+	/// State of the version
+	pub state: ObjectVersionState,
+}
+
+/// State of an object version
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub enum ObjectVersionState {
+	/// The version is being received
+	Uploading(ObjectVersionHeaders),
+	/// The version is fully received
+	Complete(ObjectVersionData),
+	/// The version uploaded containded errors or the upload was explicitly aborted
+	Aborted,
+}
+
+impl Crdt for ObjectVersionState {
+	fn merge(&mut self, other: &Self) {
+		use ObjectVersionState::*;
+		match other {
+			Aborted => {
+				*self = Aborted;
+			}
+			Complete(b) => match self {
+				Aborted => {}
+				Complete(a) => {
+					a.merge(b);
+				}
+				Uploading(_) => {
+					*self = Complete(b.clone());
+				}
+			},
+			Uploading(_) => {}
+		}
+	}
+}
+
+/// Data stored in object version
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+pub enum ObjectVersionData {
+	/// The object was deleted, this Version is a tombstone to mark it as such
+	DeleteMarker,
+	/// The object is short, it's stored inlined
+	Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
+	/// The object is not short, Hash of first block is stored here, next segments hashes are
+	/// stored in the version table
+	FirstBlock(ObjectVersionMeta, Hash),
+}
+
+impl AutoCrdt for ObjectVersionData {
+	const WARN_IF_DIFFERENT: bool = true;
+}
+
+/// Metadata about the object version
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+pub struct ObjectVersionMeta {
+	/// Headers to send to the client
+	pub headers: ObjectVersionHeaders,
+	/// Size of the object
+	pub size: u64,
+	/// etag of the object
+	pub etag: String,
+}
+
+/// Additional headers for an object
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+pub struct ObjectVersionHeaders {
+	/// Content type of the object
+	pub content_type: String,
+	/// Any other http headers to send
+	pub other: BTreeMap<String, String>,
+}
+
+impl ObjectVersion {
+	fn cmp_key(&self) -> (u64, Uuid) {
+		(self.timestamp, self.uuid)
+	}
+
+	/// Is the object version completely received
+	pub fn is_complete(&self) -> bool {
+		matches!(self.state, ObjectVersionState::Complete(_))
+	}
+}
+
+impl Crdt for Object {
+	fn merge(&mut self, other: &Self) {
+		// Merge versions from other into here
+		for other_v in other.versions.iter() {
+			match self
+				.versions
+				.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
+			{
+				Ok(i) => {
+					self.versions[i].state.merge(&other_v.state);
+				}
+				Err(i) => {
+					self.versions.insert(i, other_v.clone());
+				}
+			}
+		}
+
+		// Remove versions which are obsolete, i.e. those that come
+		// before the last version which .is_complete().
+		let last_complete = self
+			.versions
+			.iter()
+			.enumerate()
+			.rev()
+			.find(|(_, v)| v.is_complete())
+			.map(|(vi, _)| vi);
+
+		if let Some(last_vi) = last_complete {
+			self.versions = self.versions.drain(last_vi..).collect::<Vec<_>>();
+		}
+	}
+}

src/model/prev/v051/version_table.rs

@@ -0,0 +1,79 @@
+use serde::{Deserialize, Serialize};
+
+use garage_util::data::*;
+
+use garage_table::crdt::*;
+use garage_table::*;
+
+/// A version of an object
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct Version {
+	/// UUID of the version, used as partition key
+	pub uuid: Uuid,
+
+	// Actual data: the blocks for this version
+	// In the case of a multipart upload, also store the etags
+	// of individual parts and check them when doing CompleteMultipartUpload
+	/// Is this version deleted
+	pub deleted: crdt::Bool,
+	/// list of blocks of data composing the version
+	pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
+	/// Etag of each part in case of a multipart upload, empty otherwise
+	pub parts_etags: crdt::Map<u64, String>,
+
+	// Back link to bucket+key so that we can figure if
+	// this was deleted later on
+	/// Bucket in which the related object is stored
+	pub bucket: String,
+	/// Key in which the related object is stored
+	pub key: String,
+}
+
+#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct VersionBlockKey {
+	/// Number of the part
+	pub part_number: u64,
+	/// Offset of this sub-segment in its part
+	pub offset: u64,
+}
+
+impl Ord for VersionBlockKey {
+	fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+		self.part_number
+			.cmp(&other.part_number)
+			.then(self.offset.cmp(&other.offset))
+	}
+}
+
+impl PartialOrd for VersionBlockKey {
+	fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+		Some(self.cmp(other))
+	}
+}
+
+/// Informations about a single block
+#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct VersionBlock {
+	/// Blake2 sum of the block
+	pub hash: Hash,
+	/// Size of the block
+	pub size: u64,
+}
+
+impl AutoCrdt for VersionBlock {
+	const WARN_IF_DIFFERENT: bool = true;
+}
+
+impl Crdt for Version {
+	fn merge(&mut self, other: &Self) {
+		self.deleted.merge(&other.deleted);
+
+		if self.deleted.get() {
+			self.blocks.clear();
+			self.parts_etags.clear();
+		} else {
+			self.blocks.merge(&other.blocks);
+			self.parts_etags.merge(&other.parts_etags);
+		}
+	}
+}

src/model/s3/block_ref_table.rs

@@ -1,3 +1,4 @@
+use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 
 use garage_db as db;
@@ -9,29 +10,19 @@ use garage_table::*;
 
 use garage_block::manager::*;
 
-mod v08 {
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	use garage_util::crdt;
+pub struct BlockRef {
-	use garage_util::data::{Hash, Uuid};
+	/// Hash (blake2 sum) of the block, used as partition key
-	use serde::{Deserialize, Serialize};
+	pub block: Hash,
 
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	/// Id of the Version for the object containing this block, used as sorting key
-	pub struct BlockRef {
+	pub version: Uuid,
-		/// Hash (blake2 sum) of the block, used as partition key
-		pub block: Hash,
 
-		/// Id of the Version for the object containing this block, used as sorting key
+	// Keep track of deleted status
-		pub version: Uuid,
+	/// Is the Version that contains this block deleted
-
+	pub deleted: crdt::Bool,
-		// Keep track of deleted status
-		/// Is the Version that contains this block deleted
-		pub deleted: crdt::Bool,
-	}
-
-	impl garage_util::migrate::InitialFormat for BlockRef {}
 }
 
-pub use v08::*;
-
 impl Entry<Hash, Uuid> for BlockRef {
 	fn partition_key(&self) -> &Hash {
 		&self.block

src/model/s3/object_table.rs

@@ -1,8 +1,10 @@
 use serde::{Deserialize, Serialize};
+use std::collections::BTreeMap;
 use std::sync::Arc;
 
 use garage_db as db;
 
+use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 
 use garage_table::crdt::*;
@@ -12,126 +14,25 @@ use garage_table::*;
 use crate::index_counter::*;
 use crate::s3::version_table::*;
 
+use crate::prev::v051::object_table as old;
+
 pub const OBJECTS: &str = "objects";
 pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads";
 pub const BYTES: &str = "bytes";
 
-mod v05 {
+/// An object
-	use garage_util::data::{Hash, Uuid};
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	use serde::{Deserialize, Serialize};
+pub struct Object {
-	use std::collections::BTreeMap;
+	/// The bucket in which the object is stored, used as partition key
+	pub bucket_id: Uuid,
 
-	/// An object
+	/// The key at which the object is stored in its bucket, used as sorting key
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+	pub key: String,
-	pub struct Object {
-		/// The bucket in which the object is stored, used as partition key
-		pub bucket: String,
 
-		/// The key at which the object is stored in its bucket, used as sorting key
+	/// The list of currenty stored versions of the object
-		pub key: String,
+	versions: Vec<ObjectVersion>,
-
-		/// The list of currenty stored versions of the object
-		pub(super) versions: Vec<ObjectVersion>,
-	}
-
-	/// Informations about a version of an object
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct ObjectVersion {
-		/// Id of the version
-		pub uuid: Uuid,
-		/// Timestamp of when the object was created
-		pub timestamp: u64,
-		/// State of the version
-		pub state: ObjectVersionState,
-	}
-
-	/// State of an object version
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub enum ObjectVersionState {
-		/// The version is being received
-		Uploading(ObjectVersionHeaders),
-		/// The version is fully received
-		Complete(ObjectVersionData),
-		/// The version uploaded containded errors or the upload was explicitly aborted
-		Aborted,
-	}
-
-	/// Data stored in object version
-	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-	pub enum ObjectVersionData {
-		/// The object was deleted, this Version is a tombstone to mark it as such
-		DeleteMarker,
-		/// The object is short, it's stored inlined
-		Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
-		/// The object is not short, Hash of first block is stored here, next segments hashes are
-		/// stored in the version table
-		FirstBlock(ObjectVersionMeta, Hash),
-	}
-
-	/// Metadata about the object version
-	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-	pub struct ObjectVersionMeta {
-		/// Headers to send to the client
-		pub headers: ObjectVersionHeaders,
-		/// Size of the object
-		pub size: u64,
-		/// etag of the object
-		pub etag: String,
-	}
-
-	/// Additional headers for an object
-	#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
-	pub struct ObjectVersionHeaders {
-		/// Content type of the object
-		pub content_type: String,
-		/// Any other http headers to send
-		pub other: BTreeMap<String, String>,
-	}
-
-	impl garage_util::migrate::InitialFormat for Object {}
 }
 
-mod v08 {
-	use garage_util::data::Uuid;
-	use serde::{Deserialize, Serialize};
-
-	use super::v05;
-
-	pub use v05::{
-		ObjectVersion, ObjectVersionData, ObjectVersionHeaders, ObjectVersionMeta,
-		ObjectVersionState,
-	};
-
-	/// An object
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct Object {
-		/// The bucket in which the object is stored, used as partition key
-		pub bucket_id: Uuid,
-
-		/// The key at which the object is stored in its bucket, used as sorting key
-		pub key: String,
-
-		/// The list of currenty stored versions of the object
-		pub(super) versions: Vec<ObjectVersion>,
-	}
-
-	impl garage_util::migrate::Migrate for Object {
-		type Previous = v05::Object;
-
-		fn migrate(old: v05::Object) -> Object {
-			use garage_util::data::blake2sum;
-
-			Object {
-				bucket_id: blake2sum(old.bucket.as_bytes()),
-				key: old.key,
-				versions: old.versions,
-			}
-		}
-	}
-}
-
-pub use v08::*;
-
 impl Object {
 	/// Initialize an Object struct from parts
 	pub fn new(bucket_id: Uuid, key: String, versions: Vec<ObjectVersion>) -> Self {
@@ -168,6 +69,28 @@ impl Object {
 	}
 }
 
+/// Informations about a version of an object
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub struct ObjectVersion {
+	/// Id of the version
+	pub uuid: Uuid,
+	/// Timestamp of when the object was created
+	pub timestamp: u64,
+	/// State of the version
+	pub state: ObjectVersionState,
+}
+
+/// State of an object version
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+pub enum ObjectVersionState {
+	/// The version is being received
+	Uploading(ObjectVersionHeaders),
+	/// The version is fully received
+	Complete(ObjectVersionData),
+	/// The version uploaded containded errors or the upload was explicitly aborted
+	Aborted,
+}
+
 impl Crdt for ObjectVersionState {
 	fn merge(&mut self, other: &Self) {
 		use ObjectVersionState::*;
@@ -189,10 +112,42 @@ impl Crdt for ObjectVersionState {
 	}
 }
 
+/// Data stored in object version
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+pub enum ObjectVersionData {
+	/// The object was deleted, this Version is a tombstone to mark it as such
+	DeleteMarker,
+	/// The object is short, it's stored inlined
+	Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
+	/// The object is not short, Hash of first block is stored here, next segments hashes are
+	/// stored in the version table
+	FirstBlock(ObjectVersionMeta, Hash),
+}
+
 impl AutoCrdt for ObjectVersionData {
 	const WARN_IF_DIFFERENT: bool = true;
 }
 
+/// Metadata about the object version
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+pub struct ObjectVersionMeta {
+	/// Headers to send to the client
+	pub headers: ObjectVersionHeaders,
+	/// Size of the object
+	pub size: u64,
+	/// etag of the object
+	pub etag: String,
+}
+
+/// Additional headers for an object
+#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
+pub struct ObjectVersionHeaders {
+	/// Content type of the object
+	pub content_type: String,
+	/// Any other http headers to send
+	pub other: BTreeMap<String, String>,
+}
+
 impl ObjectVersion {
 	fn cmp_key(&self) -> (u64, Uuid) {
 		(self.timestamp, self.uuid)
@@ -266,6 +221,7 @@ impl Crdt for Object {
 }
 
 pub struct ObjectTable {
+	pub background: Arc<BackgroundRunner>,
 	pub version_table: Arc<Table<VersionTable, TableShardedReplication>>,
 	pub object_counter_table: Arc<IndexCounter<Object>>,
 }
@@ -299,34 +255,34 @@ impl TableSchema for ObjectTable {
 			);
 		}
 
-		// 2. Enqueue propagation deletions to version table
+		// 2. Spawn threads that propagates deletions to version table
-		if let (Some(old_v), Some(new_v)) = (old, new) {
+		let version_table = self.version_table.clone();
-			// Propagate deletion of old versions
+		let old = old.cloned();
-			for v in old_v.versions.iter() {
+		let new = new.cloned();
-				let newly_deleted = match new_v
+
-					.versions
+		self.background.spawn(async move {
-					.binary_search_by(|nv| nv.cmp_key().cmp(&v.cmp_key()))
+			if let (Some(old_v), Some(new_v)) = (old, new) {
-				{
+				// Propagate deletion of old versions
-					Err(_) => true,
+				for v in old_v.versions.iter() {
-					Ok(i) => {
+					let newly_deleted = match new_v
-						new_v.versions[i].state == ObjectVersionState::Aborted
+						.versions
-							&& v.state != ObjectVersionState::Aborted
+						.binary_search_by(|nv| nv.cmp_key().cmp(&v.cmp_key()))
-					}
+					{
-				};
+						Err(_) => true,
-				if newly_deleted {
+						Ok(i) => {
-					let deleted_version =
+							new_v.versions[i].state == ObjectVersionState::Aborted
-						Version::new(v.uuid, old_v.bucket_id, old_v.key.clone(), true);
+								&& v.state != ObjectVersionState::Aborted
-					let res = self.version_table.queue_insert(tx, &deleted_version);
+						}
-					if let Err(e) = db::unabort(res)? {
+					};
-						error!(
+					if newly_deleted {
-							"Unable to enqueue version deletion propagation: {}. A repair will be needed.",
+						let deleted_version =
-							e
+							Version::new(v.uuid, old_v.bucket_id, old_v.key.clone(), true);
-						);
+						version_table.insert(&deleted_version).await?;
 					}
 				}
 			}
-		}
+			Ok(())
-
+		});
 		Ok(())
 	}
 
@@ -336,6 +292,11 @@ impl TableSchema for ObjectTable {
 			ObjectFilter::IsUploading => entry.versions.iter().any(|v| v.is_uploading()),
 		}
 	}
+
+	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
+		let old_obj = rmp_serde::decode::from_read_ref::<_, old::Object>(bytes).ok()?;
+		Some(migrate_object(old_obj))
+	}
 }
 
 impl CountedItem for Object {
@@ -380,3 +341,64 @@ impl CountedItem for Object {
 		]
 	}
 }
+
+// vvvvvvvv migration code, stupid stuff vvvvvvvvvvvv
+// (we just want to change bucket into bucket_id by hashing it)
+
+fn migrate_object(o: old::Object) -> Object {
+	let versions = o
+		.versions()
+		.iter()
+		.cloned()
+		.map(migrate_object_version)
+		.collect();
+	Object {
+		bucket_id: blake2sum(o.bucket.as_bytes()),
+		key: o.key,
+		versions,
+	}
+}
+
+fn migrate_object_version(v: old::ObjectVersion) -> ObjectVersion {
+	ObjectVersion {
+		uuid: Uuid::try_from(v.uuid.as_slice()).unwrap(),
+		timestamp: v.timestamp,
+		state: match v.state {
+			old::ObjectVersionState::Uploading(h) => {
+				ObjectVersionState::Uploading(migrate_object_version_headers(h))
+			}
+			old::ObjectVersionState::Complete(d) => {
+				ObjectVersionState::Complete(migrate_object_version_data(d))
+			}
+			old::ObjectVersionState::Aborted => ObjectVersionState::Aborted,
+		},
+	}
+}
+
+fn migrate_object_version_headers(h: old::ObjectVersionHeaders) -> ObjectVersionHeaders {
+	ObjectVersionHeaders {
+		content_type: h.content_type,
+		other: h.other,
+	}
+}
+
+fn migrate_object_version_data(d: old::ObjectVersionData) -> ObjectVersionData {
+	match d {
+		old::ObjectVersionData::DeleteMarker => ObjectVersionData::DeleteMarker,
+		old::ObjectVersionData::Inline(m, b) => {
+			ObjectVersionData::Inline(migrate_object_version_meta(m), b)
+		}
+		old::ObjectVersionData::FirstBlock(m, h) => ObjectVersionData::FirstBlock(
+			migrate_object_version_meta(m),
+			Hash::try_from(h.as_slice()).unwrap(),
+		),
+	}
+}
+
+fn migrate_object_version_meta(m: old::ObjectVersionMeta) -> ObjectVersionMeta {
+	ObjectVersionMeta {
+		headers: migrate_object_version_headers(m.headers),
+		size: m.size,
+		etag: m.etag,
+	}
+}

src/model/s3/version_table.rs

@@ -1,7 +1,9 @@
+use serde::{Deserialize, Serialize};
 use std::sync::Arc;
 
 use garage_db as db;
 
+use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 
 use garage_table::crdt::*;
@@ -10,108 +12,32 @@ use garage_table::*;
 
 use crate::s3::block_ref_table::*;
 
-mod v05 {
+use crate::prev::v051::version_table as old;
-	use garage_util::crdt;
-	use garage_util::data::{Hash, Uuid};
-	use serde::{Deserialize, Serialize};
 
-	/// A version of an object
+/// A version of an object
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
+#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct Version {
+pub struct Version {
-		/// UUID of the version, used as partition key
+	/// UUID of the version, used as partition key
-		pub uuid: Uuid,
+	pub uuid: Uuid,
 
-		// Actual data: the blocks for this version
+	// Actual data: the blocks for this version
-		// In the case of a multipart upload, also store the etags
+	// In the case of a multipart upload, also store the etags
-		// of individual parts and check them when doing CompleteMultipartUpload
+	// of individual parts and check them when doing CompleteMultipartUpload
-		/// Is this version deleted
+	/// Is this version deleted
-		pub deleted: crdt::Bool,
+	pub deleted: crdt::Bool,
-		/// list of blocks of data composing the version
+	/// list of blocks of data composing the version
-		pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
+	pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
-		/// Etag of each part in case of a multipart upload, empty otherwise
+	/// Etag of each part in case of a multipart upload, empty otherwise
-		pub parts_etags: crdt::Map<u64, String>,
+	pub parts_etags: crdt::Map<u64, String>,
 
-		// Back link to bucket+key so that we can figure if
+	// Back link to bucket+key so that we can figure if
-		// this was deleted later on
+	// this was deleted later on
-		/// Bucket in which the related object is stored
+	/// Bucket in which the related object is stored
-		pub bucket: String,
+	pub bucket_id: Uuid,
-		/// Key in which the related object is stored
+	/// Key in which the related object is stored
-		pub key: String,
+	pub key: String,
-	}
-
-	#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
-	pub struct VersionBlockKey {
-		/// Number of the part
-		pub part_number: u64,
-		/// Offset of this sub-segment in its part
-		pub offset: u64,
-	}
-
-	/// Informations about a single block
-	#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
-	pub struct VersionBlock {
-		/// Blake2 sum of the block
-		pub hash: Hash,
-		/// Size of the block
-		pub size: u64,
-	}
-
-	impl garage_util::migrate::InitialFormat for Version {}
 }
 
-mod v08 {
-	use garage_util::crdt;
-	use garage_util::data::Uuid;
-	use serde::{Deserialize, Serialize};
-
-	use super::v05;
-
-	/// A version of an object
-	#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
-	pub struct Version {
-		/// UUID of the version, used as partition key
-		pub uuid: Uuid,
-
-		// Actual data: the blocks for this version
-		// In the case of a multipart upload, also store the etags
-		// of individual parts and check them when doing CompleteMultipartUpload
-		/// Is this version deleted
-		pub deleted: crdt::Bool,
-		/// list of blocks of data composing the version
-		pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
-		/// Etag of each part in case of a multipart upload, empty otherwise
-		pub parts_etags: crdt::Map<u64, String>,
-
-		// Back link to bucket+key so that we can figure if
-		// this was deleted later on
-		/// Bucket in which the related object is stored
-		pub bucket_id: Uuid,
-		/// Key in which the related object is stored
-		pub key: String,
-	}
-
-	pub use v05::{VersionBlock, VersionBlockKey};
-
-	impl garage_util::migrate::Migrate for Version {
-		type Previous = v05::Version;
-
-		fn migrate(old: v05::Version) -> Version {
-			use garage_util::data::blake2sum;
-
-			Version {
-				uuid: old.uuid,
-				deleted: old.deleted,
-				blocks: old.blocks,
-				parts_etags: old.parts_etags,
-				bucket_id: blake2sum(old.bucket.as_bytes()),
-				key: old.key,
-			}
-		}
-	}
-}
-
-pub use v08::*;
-
 impl Version {
 	pub fn new(uuid: Uuid, bucket_id: Uuid, key: String, deleted: bool) -> Self {
 		Self {
@@ -139,6 +65,14 @@ impl Version {
 	}
 }
 
+#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct VersionBlockKey {
+	/// Number of the part
+	pub part_number: u64,
+	/// Offset of this sub-segment in its part
+	pub offset: u64,
+}
+
 impl Ord for VersionBlockKey {
 	fn cmp(&self, other: &Self) -> std::cmp::Ordering {
 		self.part_number
@@ -153,6 +87,15 @@ impl PartialOrd for VersionBlockKey {
 	}
 }
 
+/// Informations about a single block
+#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
+pub struct VersionBlock {
+	/// Blake2 sum of the block
+	pub hash: Hash,
+	/// Size of the block
+	pub size: u64,
+}
+
 impl AutoCrdt for VersionBlock {
 	const WARN_IF_DIFFERENT: bool = true;
 }
@@ -184,6 +127,7 @@ impl Crdt for Version {
 }
 
 pub struct VersionTable {
+	pub background: Arc<BackgroundRunner>,
 	pub block_ref_table: Arc<Table<BlockRefTable, TableShardedReplication>>,
 }
 
@@ -197,26 +141,33 @@ impl TableSchema for VersionTable {
 
 	fn updated(
 		&self,
-		tx: &mut db::Transaction,
+		_tx: &mut db::Transaction,
 		old: Option<&Self::E>,
 		new: Option<&Self::E>,
 	) -> db::TxOpResult<()> {
-		if let (Some(old_v), Some(new_v)) = (old, new) {
+		let block_ref_table = self.block_ref_table.clone();
-			// Propagate deletion of version blocks
+		let old = old.cloned();
-			if new_v.deleted.get() && !old_v.deleted.get() {
+		let new = new.cloned();
-				let deleted_block_refs = old_v.blocks.items().iter().map(|(_k, vb)| BlockRef {
+
-					block: vb.hash,
+		self.background.spawn(async move {
-					version: old_v.uuid,
+			if let (Some(old_v), Some(new_v)) = (old, new) {
-					deleted: true.into(),
+				// Propagate deletion of version blocks
-				});
+				if new_v.deleted.get() && !old_v.deleted.get() {
-				for block_ref in deleted_block_refs {
+					let deleted_block_refs = old_v
-					let res = self.block_ref_table.queue_insert(tx, &block_ref);
+						.blocks
-					if let Err(e) = db::unabort(res)? {
+						.items()
-						error!("Unable to enqueue block ref deletion propagation: {}. A repair will be needed.", e);
+						.iter()
-					}
+						.map(|(_k, vb)| BlockRef {
+							block: vb.hash,
+							version: old_v.uuid,
+							deleted: true.into(),
+						})
+						.collect::<Vec<_>>();
+					block_ref_table.insert_many(&deleted_block_refs[..]).await?;
 				}
 			}
-		}
+			Ok(())
+		});
 
 		Ok(())
 	}
@@ -224,4 +175,42 @@ impl TableSchema for VersionTable {
 	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
 		filter.apply(entry.deleted.get())
 	}
+
+	fn try_migrate(bytes: &[u8]) -> Option<Self::E> {
+		let old = rmp_serde::decode::from_read_ref::<_, old::Version>(bytes).ok()?;
+
+		let blocks = old
+			.blocks
+			.items()
+			.iter()
+			.map(|(k, v)| {
+				(
+					VersionBlockKey {
+						part_number: k.part_number,
+						offset: k.offset,
+					},
+					VersionBlock {
+						hash: Hash::try_from(v.hash.as_slice()).unwrap(),
+						size: v.size,
+					},
+				)
+			})
+			.collect::<crdt::Map<_, _>>();
+
+		let parts_etags = old
+			.parts_etags
+			.items()
+			.iter()
+			.map(|(k, v)| (*k, v.clone()))
+			.collect::<crdt::Map<_, _>>();
+
+		Some(Version {
+			uuid: Hash::try_from(old.uuid.as_slice()).unwrap(),
+			deleted: crdt::Bool::new(old.deleted.get()),
+			blocks,
+			parts_etags,
+			bucket_id: blake2sum(old.bucket.as_bytes()),
+			key: old.key,
+		})
+	}
 }

src/rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "garage_rpc"
-version = "0.8.1"
+version = "0.8.0"
 authors = ["Alex Auvolat <alex@adnab.me>"]
 edition = "2018"
 license = "AGPL-3.0"
@@ -14,7 +14,7 @@ path = "lib.rs"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-garage_util = { version = "0.8.1", path = "../util" }
+garage_util = { version = "0.8.0", path = "../util" }
 
 arc-swap = "1.0"
 bytes = "1.0"
@@ -25,6 +25,7 @@ rand = "0.8"
 sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
 
 async-trait = "0.1.7"
+rmp-serde = "0.15"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_bytes = "0.11"
 serde_json = "1.0"

src/rpc/layout.rs

@@ -5,7 +5,6 @@ use serde::{Deserialize, Serialize};
 
 use garage_util::crdt::{AutoCrdt, Crdt, LwwMap};
 use garage_util::data::*;
-use garage_util::encode::nonversioned_encode;
 use garage_util::error::*;
 
 use crate::ring::*;
@@ -36,8 +35,6 @@ pub struct ClusterLayout {
 	pub staging_hash: Hash,
 }
 
-impl garage_util::migrate::InitialFormat for ClusterLayout {}
-
 #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
 pub struct NodeRoleV(pub Option<NodeRole>);
 
@@ -71,7 +68,7 @@ impl NodeRole {
 impl ClusterLayout {
 	pub fn new(replication_factor: usize) -> Self {
 		let empty_lwwmap = LwwMap::new();
-		let empty_lwwmap_hash = blake2sum(&nonversioned_encode(&empty_lwwmap).unwrap()[..]);
+		let empty_lwwmap_hash = blake2sum(&rmp_to_vec_all_named(&empty_lwwmap).unwrap()[..]);
 
 		ClusterLayout {
 			version: 0,
@@ -93,7 +90,7 @@ impl ClusterLayout {
 			Ordering::Equal => {
 				self.staging.merge(&other.staging);
 
-				let new_staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
+				let new_staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
 				let changed = new_staging_hash != self.staging_hash;
 
 				self.staging_hash = new_staging_hash;
@@ -128,7 +125,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		}
 
 		self.staging.clear();
-		self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
+		self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
 
 		self.version += 1;
 
@@ -152,7 +149,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		}
 
 		self.staging.clear();
-		self.staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
+		self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
 
 		self.version += 1;
 
@@ -181,7 +178,7 @@ To know the correct value of the new layout version, invoke `garage layout show`
 	/// returns true if consistent, false if error
 	pub fn check(&self) -> bool {
 		// Check that the hash of the staging data is correct
-		let staging_hash = blake2sum(&nonversioned_encode(&self.staging).unwrap()[..]);
+		let staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]);
 		if staging_hash != self.staging_hash {
 			return false;
 		}
@@ -257,11 +254,9 @@ To know the correct value of the new layout version, invoke `garage layout show`
 		match self.initial_partition_assignation() {
 			Some(initial_partitions) => {
 				for (part, ipart) in partitions.iter_mut().zip(initial_partitions.iter()) {
-					for _ in 0..2 {
+					for (id, info) in ipart.nodes.iter() {
-						for (id, info) in ipart.nodes.iter() {
+						if part.nodes.len() < self.replication_factor {
-							if part.nodes.len() < self.replication_factor {
+							part.add(None, n_zones, id, info.unwrap());
-								part.add(None, n_zones, id, info.unwrap());
-							}
 						}
 					}
 					assert!(part.nodes.len() == self.replication_factor);

src/rpc/lib.rs

@@ -9,7 +9,6 @@ mod consul;
 mod kubernetes;
 
 pub mod layout;
-pub mod replication_mode;
 pub mod ring;
 pub mod system;
 
@@ -17,5 +16,3 @@ mod metrics;
 pub mod rpc_helper;
 
 pub use rpc_helper::*;
-
-pub mod system_metrics;

src/rpc/rpc_helper.rs

@@ -5,6 +5,7 @@ use std::time::Duration;
 use futures::future::join_all;
 use futures::stream::futures_unordered::FuturesUnordered;
 use futures::stream::StreamExt;
+use futures_util::future::FutureExt;
 use tokio::select;
 use tokio::sync::watch;
 
@@ -23,6 +24,7 @@ pub use netapp::message::{
 use netapp::peering::fullmesh::FullMeshPeeringStrategy;
 pub use netapp::{self, NetApp, NodeID};
 
+use garage_util::background::BackgroundRunner;
 use garage_util::data::*;
 use garage_util::error::Error;
 use garage_util::metrics::RecordDuration;
@@ -92,6 +94,7 @@ pub struct RpcHelper(Arc<RpcHelperInner>);
 struct RpcHelperInner {
 	our_node_id: Uuid,
 	fullmesh: Arc<FullMeshPeeringStrategy>,
+	background: Arc<BackgroundRunner>,
 	ring: watch::Receiver<Arc<Ring>>,
 	metrics: RpcMetrics,
 	rpc_timeout: Duration,
@@ -101,6 +104,7 @@ impl RpcHelper {
 	pub(crate) fn new(
 		our_node_id: Uuid,
 		fullmesh: Arc<FullMeshPeeringStrategy>,
+		background: Arc<BackgroundRunner>,
 		ring: watch::Receiver<Arc<Ring>>,
 		rpc_timeout: Option<Duration>,
 	) -> Self {
@@ -109,6 +113,7 @@ impl RpcHelper {
 		Self(Arc::new(RpcHelperInner {
 			our_node_id,
 			fullmesh,
+			background,
 			ring,
 			metrics,
 			rpc_timeout: rpc_timeout.unwrap_or(DEFAULT_TIMEOUT),
@@ -372,13 +377,16 @@ impl RpcHelper {
 
 			if !resp_stream.is_empty() {
 				// Continue remaining requests in background.
-				// Note: these requests can get interrupted on process shutdown,
+				// Continue the remaining requests immediately using tokio::spawn
-				// we must not count on them being executed for certain.
+				// but enqueue a task in the background runner
-				// For all background things that have to happen with certainty,
+				// to ensure that the process won't exit until the requests are done
-				// they have to be put in a proper queue that is persisted to disk.
+				// (if we had just enqueued the resp_stream.collect directly in the background runner,
-				tokio::spawn(async move {
+				// the requests might have been put on hold in the background runner's queue,
+				// in which case they might timeout or otherwise fail)
+				let wait_finished_fut = tokio::spawn(async move {
 					resp_stream.collect::<Vec<Result<_, _>>>().await;
 				});
+				self.0.background.spawn(wait_finished_fut.map(|_| Ok(())));
 			}
 		}