Add tracing integration with opentelemetry

2022-02-17 23:28:23 +01:00 · 2022-02-17 23:28:23 +01:00 · 8c2fb0c066
parent b6561f6e1b
commit 8c2fb0c066
30 changed files with 1020 additions and 198 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@ -11,6 +11,12 @@ dependencies = [
 "memchr",
 ]
 [[package]]
 name = "anyhow"
 version = "1.0.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"
 [[package]]
 name = "arc-swap"
 version = "1.5.0"
@ -23,6 +29,27 @@ version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
 [[package]]
 name = "async-stream"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"
 dependencies = [
 "async-stream-impl",
 "futures-core",
 ]
 [[package]]
 name = "async-stream-impl"
 version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"
 dependencies = [
 "proc-macro2",
 "quote",
 "syn",
 ]
 [[package]]
 name = "async-trait"
 version = "0.1.52"
@ -628,6 +655,12 @@ dependencies = [
 "instant",
 ]
 [[package]]
 name = "fixedbitset"
 version = "0.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e"
 [[package]]
 name = "fnv"
 version = "1.0.7"
@ -788,7 +821,6 @@ dependencies = [
 "http",
 "hyper",
 "kuska-sodiumoxide",
 "log",
 "netapp",
 "pretty_env_logger",
 "rand",
@ -800,6 +832,7 @@ dependencies = [
 "structopt",
 "tokio",
 "toml",
 "tracing",
 ]
 [[package]]
@ -810,12 +843,14 @@ dependencies = [
 "futures-util",
 "garage_model 0.6.0",
 "garage_util 0.6.0",
 "hex",
 "http",
 "hyper",
 "log",
 "opentelemetry",
 "opentelemetry-otlp",
 "opentelemetry-prometheus",
 "prometheus",
 "tracing",
 ]
 [[package]]
@ -840,10 +875,10 @@ dependencies = [
 "httpdate 0.3.2",
 "hyper",
 "idna",
 "log",
 "md-5",
 "multer",
 "nom",
 "opentelemetry",
 "percent-encoding",
 "pin-project 1.0.10",
 "quick-xml",
@ -852,7 +887,10 @@ dependencies = [
 "serde_bytes",
 "serde_json",
 "sha2",
 "strum",
 "strum_macros",
 "tokio",
 "tracing",
 "url",
 ]
@ -895,7 +933,6 @@ dependencies = [
 "garage_table 0.6.0",
 "garage_util 0.6.0",
 "hex",
 "log",
 "netapp",
 "opentelemetry",
 "rand",
@ -904,6 +941,7 @@ dependencies = [
 "serde_bytes",
 "sled",
 "tokio",
 "tracing",
 "zstd",
 ]
@ -950,7 +988,6 @@ dependencies = [
 "k8s-openapi",
 "kube",
 "kuska-sodiumoxide",
 "log",
 "netapp",
 "openssl",
 "opentelemetry",
@ -963,6 +1000,7 @@ dependencies = [
 "serde_json",
 "tokio",
 "tokio-stream",
 "tracing",
 ]
 [[package]]
@ -998,7 +1036,6 @@ dependencies = [
 "garage_rpc 0.6.0",
 "garage_util 0.6.0",
 "hexdump",
 "log",
 "opentelemetry",
 "rand",
 "rmp-serde 0.15.5",
@ -1006,6 +1043,7 @@ dependencies = [
 "serde_bytes",
 "sled",
 "tokio",
 "tracing",
 ]
 [[package]]
@ -1047,7 +1085,6 @@ dependencies = [
 "hyper",
 "k8s-openapi",
 "kube",
 "log",
 "netapp",
 "rand",
 "rmp-serde 0.15.5",
@ -1057,6 +1094,7 @@ dependencies = [
 "sled",
 "tokio",
 "toml",
 "tracing",
 "xxhash-rust",
 ]
@ -1072,8 +1110,8 @@ dependencies = [
 "garage_util 0.6.0",
 "http",
 "hyper",
 "log",
 "percent-encoding",
 "tracing",
 ]
 [[package]]
@ -1191,7 +1229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e40283dadb02f3af778878be1d717b17b4e4ab92e1d935ab03a730b0542905f2"
 dependencies = [
 "arrayvec",
- "itertools",
+ "itertools 0.4.19",
 ]
 [[package]]
@ -1376,6 +1414,15 @@ version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f"
 [[package]]
 name = "itertools"
 version = "0.10.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"
 dependencies = [
 "either",
 ]
 [[package]]
 name = "itoa"
 version = "1.0.1"
@ -1701,6 +1748,12 @@ dependencies = [
 "version_check",
 ]
 [[package]]
 name = "multimap"
 version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 [[package]]
 name = "native-tls"
 version = "0.2.8"
@ -1873,6 +1926,26 @@ dependencies = [
 "pin-project 1.0.10",
 "rand",
 "thiserror",
 "tokio",
 "tokio-stream",
 ]
 [[package]]
 name = "opentelemetry-otlp"
 version = "0.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a"
 dependencies = [
 "async-trait",
 "futures",
 "futures-util",
 "http",
 "opentelemetry",
 "prost",
 "thiserror",
 "tokio",
 "tonic",
 "tonic-build",
 ]
 [[package]]
@ -1903,7 +1976,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
 dependencies = [
 "instant",
 "lock_api",
- "parking_lot_core",
+ "parking_lot_core 0.8.5",
 ]
 [[package]]
 name = "parking_lot"
 version = "0.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"
 dependencies = [
 "lock_api",
 "parking_lot_core 0.9.1",
 ]
 [[package]]
@ -1920,6 +2003,19 @@ dependencies = [
 "winapi",
 ]
 [[package]]
 name = "parking_lot_core"
 version = "0.9.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954"
 dependencies = [
 "cfg-if",
 "libc",
 "redox_syscall",
 "smallvec",
 "windows-sys",
 ]
 [[package]]
 name = "pem"
 version = "0.8.3"
@ -1937,6 +2033,16 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 [[package]]
 name = "petgraph"
 version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f"
 dependencies = [
 "fixedbitset",
 "indexmap",
 ]
 [[package]]
 name = "pin-project"
 version = "0.4.29"
@ -2148,11 +2254,64 @@ dependencies = [
 "fnv",
 "lazy_static",
 "memchr",
- "parking_lot",
+ "parking_lot 0.11.2",
 "protobuf",
 "thiserror",
 ]
 [[package]]
 name = "prost"
 version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
 dependencies = [
 "bytes 1.1.0",
 "prost-derive",
 ]
 [[package]]
 name = "prost-build"
 version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
 dependencies = [
 "bytes 1.1.0",
 "heck",
 "itertools 0.10.3",
 "lazy_static",
 "log",
 "multimap",
 "petgraph",
 "prost",
 "prost-types",
 "regex",
 "tempfile",
 "which",
 ]
 [[package]]
 name = "prost-derive"
 version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"
 dependencies = [
 "anyhow",
 "itertools 0.10.3",
 "proc-macro2",
 "quote",
 "syn",
 ]
 [[package]]
 name = "prost-types"
 version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
 dependencies = [
 "bytes 1.1.0",
 "prost",
 ]
 [[package]]
 name = "protobuf"
 version = "2.27.1"
@ -2564,7 +2723,7 @@ dependencies = [
 "fxhash",
 "libc",
 "log",
- "parking_lot",
+ "parking_lot 0.11.2",
 ]
 [[package]]
@ -2627,8 +2786,8 @@ dependencies = [
 "bitflags",
 "cfg_aliases",
 "libc",
- "parking_lot",
+ "parking_lot 0.11.2",
- "parking_lot_core",
+ "parking_lot_core 0.8.5",
 "static_init_macro",
 "winapi",
 ]
@ -2676,6 +2835,25 @@ dependencies = [
 "syn",
 ]
 [[package]]
 name = "strum"
 version = "0.23.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"
 [[package]]
 name = "strum_macros"
 version = "0.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38"
 dependencies = [
 "heck",
 "proc-macro2",
 "quote",
 "rustversion",
 "syn",
 ]
 [[package]]
 name = "subtle"
 version = "2.4.1"
@ -2805,6 +2983,7 @@ dependencies = [
 "mio",
 "num_cpus",
 "once_cell",
 "parking_lot 0.12.0",
 "pin-project-lite",
 "signal-hook-registry",
 "socket2",
@ -2904,6 +3083,49 @@ dependencies = [
 "serde",
 ]
 [[package]]
 name = "tonic"
 version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"
 dependencies = [
 "async-stream",
 "async-trait",
 "base64",
 "bytes 1.1.0",
 "futures-core",
 "futures-util",
 "h2",
 "http",
 "http-body",
 "hyper",
 "hyper-timeout",
 "percent-encoding",
 "pin-project 1.0.10",
 "prost",
 "prost-derive",
 "tokio",
 "tokio-stream",
 "tokio-util 0.6.9",
 "tower",
 "tower-layer",
 "tower-service",
 "tracing",
 "tracing-futures",
 ]
 [[package]]
 name = "tonic-build"
 version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"
 dependencies = [
 "proc-macro2",
 "prost-build",
 "quote",
 "syn",
 ]
 [[package]]
 name = "tower"
 version = "0.4.12"
@ -2912,8 +3134,11 @@ checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e"
 dependencies = [
 "futures-core",
 "futures-util",
 "indexmap",
 "pin-project 1.0.10",
 "pin-project-lite",
 "rand",
 "slab",
 "tokio",
 "tokio-util 0.7.0",
 "tower-layer",
@ -2984,6 +3209,16 @@ dependencies = [
 "lazy_static",
 ]
 [[package]]
 name = "tracing-futures"
 version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
 dependencies = [
 "pin-project 1.0.10",
 "tracing",
 ]
 [[package]]
 name = "treediff"
 version = "3.0.2"
@ -3175,6 +3410,17 @@ dependencies = [
 "untrusted",
 ]
 [[package]]
 name = "which"
 version = "4.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a5a7e487e921cf220206864a94a89b6c6905bfc19f1057fa26a4cb360e5c1d2"
 dependencies = [
 "either",
 "lazy_static",
 "libc",
 ]
 [[package]]
 name = "winapi"
 version = "0.3.9"
@ -3206,6 +3452,49 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 [[package]]
 name = "windows-sys"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6"
 dependencies = [
 "windows_aarch64_msvc",
 "windows_i686_gnu",
 "windows_i686_msvc",
 "windows_x86_64_gnu",
 "windows_x86_64_msvc",
 ]
 [[package]]
 name = "windows_aarch64_msvc"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5"
 [[package]]
 name = "windows_i686_gnu"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615"
 [[package]]
 name = "windows_i686_msvc"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172"
 [[package]]
 name = "windows_x86_64_gnu"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc"
 [[package]]
 name = "windows_x86_64_msvc"
 version = "0.32.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"
 [[package]]
 name = "xmlparser"
 version = "0.13.3"
--- a/Cargo.nix
+++ b/Cargo.nix
@ -67,6 +67,17 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".anyhow."1.0.56" = overridableMkRustCrate (profileName: rec {
    name = "anyhow";
    version = "1.0.56";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"; };
    features = builtins.concatLists [
      [ "default" ]
      [ "std" ]
    ];
  });
  "registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" = overridableMkRustCrate (profileName: rec {
    name = "arc-swap";
    version = "1.5.0";
@ -85,6 +96,29 @@ in
    ];
  });
  "registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.3" = overridableMkRustCrate (profileName: rec {
    name = "async-stream";
    version = "0.3.3";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"; };
    dependencies = {
      async_stream_impl = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-stream-impl."0.3.3" { profileName = "__noProfile"; };
      futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".async-stream-impl."0.3.3" = overridableMkRustCrate (profileName: rec {
    name = "async-stream-impl";
    version = "0.3.3";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"; };
    dependencies = {
      proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; };
      quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.15" { inherit profileName; };
      syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.86" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" = overridableMkRustCrate (profileName: rec {
    name = "async-trait";
    version = "0.1.52";
@ -564,7 +598,7 @@ in
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; };
    dependencies = {
-      ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; };
+      ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; };
    };
  });
@ -890,6 +924,13 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".fixedbitset."0.4.1" = overridableMkRustCrate (profileName: rec {
    name = "fixedbitset";
    version = "0.4.1";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" = overridableMkRustCrate (profileName: rec {
    name = "fnv";
    version = "1.0.7";
@ -1121,7 +1162,6 @@ in
      git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; };
      hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
      sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; };
      pretty_env_logger = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; };
      rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
@ -1132,6 +1172,7 @@ in
      structopt = rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; };
      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      toml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
    };
    devDependencies = {
      aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" { inherit profileName; };
@ -1151,12 +1192,14 @@ in
      futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
      garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; };
      garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; };
      hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
      http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
      hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
      opentelemetry_otlp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; };
      opentelemetry_prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; };
      prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
    };
  });
@ -1184,10 +1227,10 @@ in
      httpdate = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."0.3.2" { inherit profileName; };
      hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; };
      idna = rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.2.3" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.9.1" { inherit profileName; };
      multer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".multer."2.0.2" { inherit profileName; };
      nom = rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.0" { inherit profileName; };
      opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
      percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
      pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; };
      quick_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.21.0" { inherit profileName; };
@ -1196,7 +1239,10 @@ in
      serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; };
      serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.79" { inherit profileName; };
      sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; };
      strum = rustPackages."registry+https://github.com/rust-lang/crates.io-index".strum."0.23.0" { inherit profileName; };
      strum_macros = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".strum_macros."0.23.1" { profileName = "__noProfile"; };
      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
      url = rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.2.2" { inherit profileName; };
    };
  });
@ -1243,7 +1289,6 @@ in
      garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; };
      garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; };
      hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; };
      opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
      rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
@ -1252,6 +1297,7 @@ in
      serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; };
      sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; };
      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
      zstd = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.2+zstd.1.5.1" { inherit profileName; };
    };
  });
@ -1302,7 +1348,6 @@ in
      k8s_openapi = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; };
      kube = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; };
      sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; };
      openssl = rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl."0.10.38" { inherit profileName; };
      opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
@ -1315,6 +1360,7 @@ in
      serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.79" { inherit profileName; };
      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
    };
  });
@ -1354,7 +1400,6 @@ in
      garage_rpc = rustPackages."unknown".garage_rpc."0.6.0" { inherit profileName; };
      garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; };
      hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
      rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
      rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
@ -1362,6 +1407,7 @@ in
      serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; };
      sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; };
      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
    };
  });
@ -1407,7 +1453,6 @@ in
      hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; };
      k8s_openapi = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; };
      kube = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; };
      rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
      rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
@ -1417,6 +1462,7 @@ in
      sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; };
      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      toml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
      xxhash_rust = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.3" { inherit profileName; };
    };
  });
@ -1435,8 +1481,8 @@ in
      garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; };
      http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
      hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
    };
  });
@ -1671,11 +1717,12 @@ in
    features = builtins.concatLists [
      [ "client" ]
      [ "default" ]
-      (lib.optional (rootFeatures' ? "garage") "h2")
+      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "full")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "h2")
      [ "http1" ]
-      (lib.optional (rootFeatures' ? "garage") "http2")
+      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "http2")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "runtime")
-      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "server")
+      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "server")
      [ "socket2" ]
      [ "stream" ]
      [ "tcp" ]
@ -1685,7 +1732,7 @@ in
      futures_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; };
      futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
      futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
-      ${ if rootFeatures' ? "garage" then "h2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; };
+      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "h2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; };
      http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
      http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; };
      httparse = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.6.0" { inherit profileName; };
@ -1777,7 +1824,7 @@ in
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"; };
    features = builtins.concatLists [
-      (lib.optional (rootFeatures' ? "garage") "std")
+      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "std")
    ];
    dependencies = {
      hashbrown = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.11.2" { inherit profileName; };
@ -1818,6 +1865,21 @@ in
    src = fetchCratesIo { inherit name version; sha256 = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.3" = overridableMkRustCrate (profileName: rec {
    name = "itertools";
    version = "0.10.3";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"; };
    features = builtins.concatLists [
      [ "default" ]
      [ "use_alloc" ]
      [ "use_std" ]
    ];
    dependencies = {
      either = rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.6.1" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" = overridableMkRustCrate (profileName: rec {
    name = "itoa";
    version = "1.0.1";
@ -2239,7 +2301,7 @@ in
      [ "os-poll" ]
    ];
    dependencies = {
-      ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      ${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; };
      ${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; };
@ -2282,6 +2344,13 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".multimap."0.8.3" = overridableMkRustCrate (profileName: rec {
    name = "multimap";
    version = "0.8.3";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".native-tls."0.2.8" = overridableMkRustCrate (profileName: rec {
    name = "native-tls";
    version = "0.2.8";
@ -2506,6 +2575,9 @@ in
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "percent-encoding")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "pin-project")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rand")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "rt-tokio")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "tokio")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "tokio-stream")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "trace")
    ];
    dependencies = {
@ -2522,6 +2594,36 @@ in
      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "pin_project" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; };
      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "thiserror" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "tokio_stream" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" = overridableMkRustCrate (profileName: rec {
    name = "opentelemetry-otlp";
    version = "0.10.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a"; };
    features = builtins.concatLists [
      [ "default" ]
      [ "prost" ]
      [ "tokio" ]
      [ "tonic" ]
      [ "tonic-build" ]
    ];
    dependencies = {
      async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
      futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
      futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
      http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
      opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
      prost = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; };
      thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; };
      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      tonic = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tonic."0.6.2" { inherit profileName; };
    };
    buildDependencies = {
      tonic_build = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tonic-build."0.6.2" { profileName = "__noProfile"; };
    };
  });
@ -2566,6 +2668,20 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.0" = overridableMkRustCrate (profileName: rec {
    name = "parking_lot";
    version = "0.12.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"; };
    features = builtins.concatLists [
      [ "default" ]
    ];
    dependencies = {
      lock_api = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.6" { inherit profileName; };
      parking_lot_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.1" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" = overridableMkRustCrate (profileName: rec {
    name = "parking_lot_core";
    version = "0.8.5";
@ -2581,6 +2697,20 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.1" = overridableMkRustCrate (profileName: rec {
    name = "parking_lot_core";
    version = "0.9.1";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954"; };
    dependencies = {
      cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; };
      ${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; };
      ${ if hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.11" { inherit profileName; };
      smallvec = rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.8.0" { inherit profileName; };
      ${ if hostPlatform.isWindows then "windows_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.32.0" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".pem."0.8.3" = overridableMkRustCrate (profileName: rec {
    name = "pem";
    version = "0.8.3";
@ -2600,6 +2730,17 @@ in
    src = fetchCratesIo { inherit name version; sha256 = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".petgraph."0.6.0" = overridableMkRustCrate (profileName: rec {
    name = "petgraph";
    version = "0.6.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f"; };
    dependencies = {
      fixedbitset = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fixedbitset."0.4.1" { inherit profileName; };
      indexmap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.8.0" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".pin-project."0.4.29" = overridableMkRustCrate (profileName: rec {
    name = "pin-project";
    version = "0.4.29";
@ -2864,6 +3005,70 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" = overridableMkRustCrate (profileName: rec {
    name = "prost";
    version = "0.9.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"; };
    features = builtins.concatLists [
      [ "default" ]
      [ "prost-derive" ]
      [ "std" ]
    ];
    dependencies = {
      bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
      prost_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" { profileName = "__noProfile"; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".prost-build."0.9.0" = overridableMkRustCrate (profileName: rec {
    name = "prost-build";
    version = "0.9.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"; };
    dependencies = {
      bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
      heck = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; };
      itertools = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.3" { inherit profileName; };
      lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; };
      log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; };
      multimap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".multimap."0.8.3" { inherit profileName; };
      petgraph = rustPackages."registry+https://github.com/rust-lang/crates.io-index".petgraph."0.6.0" { inherit profileName; };
      prost = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; };
      prost_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-types."0.9.0" { inherit profileName; };
      regex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.5" { inherit profileName; };
      tempfile = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tempfile."3.3.0" { inherit profileName; };
    };
    buildDependencies = {
      which = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".which."4.2.4" { profileName = "__noProfile"; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" = overridableMkRustCrate (profileName: rec {
    name = "prost-derive";
    version = "0.9.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"; };
    dependencies = {
      anyhow = rustPackages."registry+https://github.com/rust-lang/crates.io-index".anyhow."1.0.56" { inherit profileName; };
      itertools = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.3" { inherit profileName; };
      proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; };
      quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.15" { inherit profileName; };
      syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.86" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".prost-types."0.9.0" = overridableMkRustCrate (profileName: rec {
    name = "prost-types";
    version = "0.9.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"; };
    dependencies = {
      bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
      prost = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".protobuf."2.27.1" = overridableMkRustCrate (profileName: rec {
    name = "protobuf";
    version = "2.27.1";
@ -2919,6 +3124,7 @@ in
      [ "getrandom" ]
      [ "libc" ]
      [ "rand_chacha" ]
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "small_rng")
      [ "std" ]
      [ "std_rng" ]
    ];
@ -3051,7 +3257,7 @@ in
    ];
    dependencies = {
      ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; };
-      ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; };
+      ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; };
      ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; };
      untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; };
      ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; };
@ -3588,6 +3794,31 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".strum."0.23.0" = overridableMkRustCrate (profileName: rec {
    name = "strum";
    version = "0.23.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"; };
    features = builtins.concatLists [
      [ "default" ]
      [ "std" ]
    ];
  });
  "registry+https://github.com/rust-lang/crates.io-index".strum_macros."0.23.1" = overridableMkRustCrate (profileName: rec {
    name = "strum_macros";
    version = "0.23.1";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38"; };
    dependencies = {
      heck = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; };
      proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; };
      quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.15" { inherit profileName; };
      rustversion = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustversion."1.0.6" { profileName = "__noProfile"; };
      syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.86" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".subtle."2.4.1" = overridableMkRustCrate (profileName: rec {
    name = "subtle";
    version = "2.4.1";
@ -3754,6 +3985,8 @@ in
      [ "bytes" ]
      [ "default" ]
      [ "fs" ]
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "full")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "io-std")
      [ "io-util" ]
      [ "libc" ]
      [ "macros" ]
@ -3762,6 +3995,8 @@ in
      [ "net" ]
      [ "num_cpus" ]
      [ "once_cell" ]
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "parking_lot")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "process")
      [ "rt" ]
      [ "rt-multi-thread" ]
      [ "signal" ]
@ -3779,6 +4014,7 @@ in
      mio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mio."0.8.1" { inherit profileName; };
      num_cpus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.13.1" { inherit profileName; };
      once_cell = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; };
      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.0" { inherit profileName; };
      pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; };
      ${ if hostPlatform.isUnix then "signal_hook_registry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.0" { inherit profileName; };
      socket2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.4" { inherit profileName; };
@ -3904,6 +4140,71 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".tonic."0.6.2" = overridableMkRustCrate (profileName: rec {
    name = "tonic";
    version = "0.6.2";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"; };
    features = builtins.concatLists [
      [ "async-trait" ]
      [ "codegen" ]
      [ "default" ]
      [ "h2" ]
      [ "hyper" ]
      [ "hyper-timeout" ]
      [ "prost" ]
      [ "prost-derive" ]
      [ "prost1" ]
      [ "tokio" ]
      [ "tower" ]
      [ "tracing-futures" ]
      [ "transport" ]
    ];
    dependencies = {
      async_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.3" { inherit profileName; };
      async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
      base64 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; };
      bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
      futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
      futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
      h2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; };
      http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
      http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; };
      hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; };
      hyper_timeout = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-timeout."0.4.1" { inherit profileName; };
      percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
      pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; };
      prost1 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; };
      prost_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" { profileName = "__noProfile"; };
      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; };
      tokio_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.9" { inherit profileName; };
      tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.12" { inherit profileName; };
      tower_layer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.1" { inherit profileName; };
      tower_service = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.1" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
      tracing_futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-futures."0.2.5" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".tonic-build."0.6.2" = overridableMkRustCrate (profileName: rec {
    name = "tonic-build";
    version = "0.6.2";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"; };
    features = builtins.concatLists [
      [ "prost" ]
      [ "prost-build" ]
      [ "transport" ]
    ];
    dependencies = {
      proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; };
      prost_build = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-build."0.9.0" { inherit profileName; };
      quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.15" { inherit profileName; };
      syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.86" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".tower."0.4.12" = overridableMkRustCrate (profileName: rec {
    name = "tower";
    version = "0.4.12";
@ -3911,14 +4212,24 @@ in
    src = fetchCratesIo { inherit name version; sha256 = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e"; };
    features = builtins.concatLists [
      [ "__common" ]
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "balance")
      [ "buffer" ]
      [ "default" ]
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "discover")
      [ "futures-core" ]
      [ "futures-util" ]
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "indexmap")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "limit")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "load")
      [ "log" ]
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "make")
      [ "pin-project" ]
      [ "pin-project-lite" ]
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "rand")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "ready-cache")
      (lib.optional (rootFeatures' ? "garage") "retry")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "slab")
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "timeout")
      [ "tokio" ]
      [ "tokio-util" ]
      [ "tracing" ]
@ -3927,8 +4238,11 @@ in
    dependencies = {
      futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
      futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "indexmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.8.0" { inherit profileName; };
      pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; };
      pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; };
      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
      ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "slab" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; };
      tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
      tokio_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.0" { inherit profileName; };
      tower_layer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.1" { inherit profileName; };
@ -3987,6 +4301,7 @@ in
      [ "attributes" ]
      [ "default" ]
      [ "log" ]
      (lib.optional (rootFeatures' ? "garage") "log-always")
      [ "std" ]
      [ "tracing-attributes" ]
    ];
@ -4025,6 +4340,23 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".tracing-futures."0.2.5" = overridableMkRustCrate (profileName: rec {
    name = "tracing-futures";
    version = "0.2.5";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"; };
    features = builtins.concatLists [
      [ "default" ]
      [ "pin-project" ]
      [ "std" ]
      [ "std-future" ]
    ];
    dependencies = {
      pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; };
      tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".treediff."3.0.2" = overridableMkRustCrate (profileName: rec {
    name = "treediff";
    version = "3.0.2";
@ -4289,6 +4621,18 @@ in
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".which."4.2.4" = overridableMkRustCrate (profileName: rec {
    name = "which";
    version = "4.2.4";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "2a5a7e487e921cf220206864a94a89b6c6905bfc19f1057fa26a4cb360e5c1d2"; };
    dependencies = {
      either = rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.6.1" { inherit profileName; };
      ${ if hostPlatform.isWindows then "lazy_static" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; };
      libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" = overridableMkRustCrate (profileName: rec {
    name = "winapi";
    version = "0.3.9";
@ -4325,6 +4669,7 @@ in
      [ "std" ]
      [ "synchapi" ]
      [ "sysinfoapi" ]
      (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "threadpoollegacyapiset")
      [ "timezoneapi" ]
      [ "winbase" ]
      [ "wincon" ]
@ -4369,6 +4714,64 @@ in
    src = fetchCratesIo { inherit name version; sha256 = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.32.0" = overridableMkRustCrate (profileName: rec {
    name = "windows-sys";
    version = "0.32.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6"; };
    features = builtins.concatLists [
      [ "Win32" ]
      [ "Win32_Foundation" ]
      [ "Win32_System" ]
      [ "Win32_System_LibraryLoader" ]
      [ "Win32_System_SystemServices" ]
      [ "Win32_System_WindowsProgramming" ]
      [ "default" ]
    ];
    dependencies = {
      ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; };
      ${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
      ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; };
      ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; };
      ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; };
    };
  });
  "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" = overridableMkRustCrate (profileName: rec {
    name = "windows_aarch64_msvc";
    version = "0.32.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" = overridableMkRustCrate (profileName: rec {
    name = "windows_i686_gnu";
    version = "0.32.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" = overridableMkRustCrate (profileName: rec {
    name = "windows_i686_msvc";
    version = "0.32.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" = overridableMkRustCrate (profileName: rec {
    name = "windows_x86_64_gnu";
    version = "0.32.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" = overridableMkRustCrate (profileName: rec {
    name = "windows_x86_64_msvc";
    version = "0.32.0";
    registry = "registry+https://github.com/rust-lang/crates.io-index";
    src = fetchCratesIo { inherit name version; sha256 = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"; };
  });
  "registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" = overridableMkRustCrate (profileName: rec {
    name = "xmlparser";
    version = "0.13.3";
--- a/src/admin/Cargo.toml
+++ b/src/admin/Cargo.toml
@ -16,12 +16,15 @@ path = "lib.rs"
 garage_model = { version = "0.6.0", path = "../model" }
 garage_util = { version = "0.6.0", path = "../util" }
 hex = "0.4"
 futures = "0.3"
 futures-util = "0.3"
 http = "0.2"
 hyper = "0.14"
-log = "0.4"
+tracing = "0.1.30"
-opentelemetry = "0.17"
+opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
 opentelemetry-prometheus = "0.10"
 opentelemetry-otlp = "0.10"
 prometheus = "0.13"
--- a/src/admin/lib.rs
+++ b/src/admin/lib.rs
@ -1,5 +1,6 @@
 //! Crate for handling the admin and metric HTTP APIs
 #[macro_use]
-extern crate log;
+extern crate tracing;
 pub mod metrics;
 pub mod tracing_setup;
--- a/src/admin/metrics.rs
+++ b/src/admin/metrics.rs
@ -1,20 +1,26 @@
 use hyper::{
 	header::CONTENT_TYPE,
 	service::{make_service_fn, service_fn},
 	Body, Method, Request, Response, Server,
 };
 use opentelemetry::{
 	global,
 	metrics::{BoundCounter, BoundValueRecorder},
 };
 use opentelemetry_prometheus::PrometheusExporter;
 use prometheus::{Encoder, TextEncoder};
 use std::convert::Infallible;
 use std::sync::Arc;
 use std::time::SystemTime;
 use futures::future::*;
 use hyper::{
 	header::CONTENT_TYPE,
 	service::{make_service_fn, service_fn},
 	Body, Method, Request, Response, Server,
 };
 use opentelemetry::{
 	global,
 	metrics::{BoundCounter, BoundValueRecorder},
 	trace::{FutureExt, TraceContextExt, Tracer},
 	Context,
 };
 use opentelemetry_prometheus::PrometheusExporter;
 use prometheus::{Encoder, TextEncoder};
 use garage_model::garage::Garage;
 use garage_util::data::*;
 use garage_util::error::Error as GarageError;
 // serve_req on metric endpoint
@ -22,7 +28,7 @@ async fn serve_req(
 	req: Request<Body>,
 	admin_server: Arc<AdminServer>,
 ) -> Result<Response<Body>, hyper::Error> {
-	println!("Receiving request at path {}", req.uri());
+	info!("Receiving request at path {}", req.uri());
 	let request_start = SystemTime::now();
 	admin_server.metrics.http_counter.add(1);
@ -31,7 +37,12 @@ async fn serve_req(
 		(&Method::GET, "/metrics") => {
 			let mut buffer = vec![];
 			let encoder = TextEncoder::new();
-			let metric_families = admin_server.exporter.registry().gather();
+
 			let tracer = opentelemetry::global::tracer("garage");
 			let metric_families = tracer.in_span("admin/gather_metrics", |_| {
 				admin_server.exporter.registry().gather()
 			});
 			encoder.encode(&metric_families, &mut buffer).unwrap();
 			admin_server
 				.metrics
@ -112,7 +123,22 @@ impl AdminServer {
 			// `service_fn` is a helper to convert a function that
 			// returns a Response into a `Service`.
 			async move {
-				Ok::<_, Infallible>(service_fn(move |req| serve_req(req, admin_server.clone())))
+				Ok::<_, Infallible>(service_fn(move |req| {
 					let tracer = opentelemetry::global::tracer("garage");
 					let uuid = gen_uuid();
 					let span = tracer
 						.span_builder("admin/request")
 						.with_trace_id(
 							opentelemetry::trace::TraceId::from_hex(&hex::encode(
 								&uuid.as_slice()[..16],
 							))
 							.unwrap(),
 						)
 						.start(&tracer);
 					serve_req(req, admin_server.clone())
 						.with_context(Context::current_with_span(span))
 				}))
 			}
 		});
--- a/src/admin/tracing_setup.rs
+++ b/src/admin/tracing_setup.rs
@ -0,0 +1,37 @@
 use std::time::Duration;
 use opentelemetry::sdk::{
 	trace::{self, IdGenerator, Sampler},
 	Resource,
 };
 use opentelemetry::KeyValue;
 use opentelemetry_otlp::WithExportConfig;
 use garage_util::data::*;
 use garage_util::error::*;
 pub fn init_tracing(export_to: &str, node_id: Uuid) -> Result<(), Error> {
 	let node_id = hex::encode(&node_id.as_slice()[..8]);
 	opentelemetry_otlp::new_pipeline()
 		.tracing()
 		.with_exporter(
 			opentelemetry_otlp::new_exporter()
 				.tonic()
 				.with_endpoint(export_to)
 				.with_timeout(Duration::from_secs(3)),
 		)
 		.with_trace_config(
 			trace::config()
 				.with_id_generator(IdGenerator::default())
 				.with_sampler(Sampler::TraceIdRatioBased(0.01f64))
 				.with_resource(Resource::new(vec![
 					KeyValue::new("service.name", "garage"),
 					KeyValue::new("service.instance.id", node_id),
 				])),
 		)
 		.install_batch(opentelemetry::runtime::Tokio)
 		.ok_or_message("Unable to initialize tracing")?;
 	Ok(())
 }
--- a/src/api/Cargo.toml
+++ b/src/api/Cargo.toml
@ -26,11 +26,14 @@ err-derive = "0.3"
 hex = "0.4"
 hmac = "0.10"
 idna = "0.2"
-log = "0.4"
+tracing = "0.1.30"
 md-5 = "0.9"
 nom = "7.1"
 sha2 = "0.9"
 strum = "0.23"
 strum_macros = "0.23"
 futures = "0.3"
 futures-util = "0.3"
 pin-project = "1.0"
@ -49,3 +52,5 @@ serde_bytes = "0.11"
 serde_json = "1.0"
 quick-xml = { version = "0.21", features = [ "serialize" ] }
 url = "2.1"
 opentelemetry = "0.17"
--- a/src/api/api_server.rs
+++ b/src/api/api_server.rs
@ -7,6 +7,11 @@ use hyper::server::conn::AddrStream;
 use hyper::service::{make_service_fn, service_fn};
 use hyper::{Body, Method, Request, Response, Server};
 use opentelemetry::{
 	trace::{FutureExt, TraceContextExt, Tracer},
 	Context,
 };
 use garage_util::data::*;
 use garage_util::error::Error as GarageError;
@ -43,7 +48,20 @@ pub async fn run_api_server(
 		async move {
 			Ok::<_, GarageError>(service_fn(move |req: Request<Body>| {
 				let garage = garage.clone();
-				handler(garage, req, client_addr)
+
 				let tracer = opentelemetry::global::tracer("garage");
 				let uuid = gen_uuid();
 				let span = tracer
 					.span_builder("S3 API call (unknown)")
 					.with_trace_id(
 						opentelemetry::trace::TraceId::from_hex(&hex::encode(
 							&uuid.as_slice()[..16],
 						))
 						.unwrap(),
 					)
 					.start(&tracer);
 				handler(garage, req, client_addr).with_context(Context::current_with_span(span))
 			}))
 		}
 	});
@ -111,6 +129,10 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
 	let (endpoint, bucket_name) = Endpoint::from_request(&req, bucket_name.map(ToOwned::to_owned))?;
 	debug!("Endpoint: {:?}", endpoint);
 	Context::current()
 		.span()
 		.update_name::<String>(format!("S3 API {}", endpoint));
 	// Some endpoints are processed early, before we even check for an API key
 	if let Endpoint::PostObject = endpoint {
 		return handle_post_object(garage, req, bucket_name.unwrap()).await;
--- a/src/api/lib.rs
+++ b/src/api/lib.rs
@ -1,6 +1,6 @@
 //! Crate for serving a S3 compatible API
 #[macro_use]
-extern crate log;
+extern crate tracing;
 pub mod error;
 pub use error::Error;
--- a/src/api/s3_router.rs
+++ b/src/api/s3_router.rs
@ -5,6 +5,8 @@ use std::borrow::Cow;
 use hyper::header::HeaderValue;
 use hyper::{HeaderMap, Method, Request};
 use strum_macros::Display;
 /// This macro is used to generate very repetitive match {} blocks in this module
 /// It is _not_ made to be used anywhere else
 macro_rules! s3_match {
@ -133,7 +135,7 @@ s3_match! {@func
 /// query parameters). Parameters it may receive by header are left out, however headers are
 /// considered when required to determine between one endpoint or another (for CopyObject and
 /// UploadObject, for instance).
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq, Display)]
 pub enum Endpoint {
 	AbortMultipartUpload {
 		key: String,
--- a/src/garage/Cargo.toml
+++ b/src/garage/Cargo.toml
@ -32,7 +32,7 @@ garage_admin = { version = "0.6.0", path = "../admin" }
 bytes = "1.0"
 git-version = "0.3.4"
 hex = "0.4"
-log = "0.4"
+tracing = { version = "0.1.30", features = ["log-always"] }
 pretty_env_logger = "0.4"
 rand = "0.8"
 async-trait = "0.1.7"
--- a/src/garage/cli/init.rs
+++ b/src/garage/cli/init.rs
@ -1,7 +1,5 @@
 use std::path::PathBuf;
 use log::warn;
 use garage_util::error::*;
 pub const READ_KEY_ERROR: &str = "Unable to read node key. It will be generated by your garage node the first time is it launched. Ensure that your garage node is currently running. (The node key is supposed to be stored in your metadata directory.)";
--- a/src/garage/main.rs
+++ b/src/garage/main.rs
@ -2,7 +2,7 @@
 //! Garage CLI, used to interact with a running Garage instance, and to launch a Garage instance
 #[macro_use]
-extern crate log;
+extern crate tracing;
 mod admin;
 mod cli;
--- a/src/garage/server.rs
+++ b/src/garage/server.rs
@ -7,6 +7,7 @@ use garage_util::config::*;
 use garage_util::error::Error;
 use garage_admin::metrics::*;
 use garage_admin::tracing_setup::*;
 use garage_api::run_api_server;
 use garage_model::garage::Garage;
 use garage_web::run_web_server;
@ -45,6 +46,11 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
 	info!("Initializing Garage main data store...");
 	let garage = Garage::new(config.clone(), db, background);
 	info!("Initialize tracing...");
 	if let Some(export_to) = config.admin_api.otlp_export_traces_to {
 		init_tracing(&export_to, garage.system.id)?;
 	}
 	let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone()));
 	info!("Create admin RPC handler...");
--- a/src/model/Cargo.toml
+++ b/src/model/Cargo.toml
@ -23,7 +23,7 @@ async-trait = "0.1.7"
 arc-swap = "1.0"
 err-derive = "0.3"
 hex = "0.4"
-log = "0.4"
+tracing = "0.1.30"
 rand = "0.8"
 zstd = { version = "0.9", default-features = false }
--- a/src/model/block.rs
+++ b/src/model/block.rs
@ -5,14 +5,16 @@ use std::time::{Duration, SystemTime};
 use arc_swap::ArcSwapOption;
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 use zstd::stream::{decode_all as zstd_decode, Encoder};
 use futures::future::*;
 use futures::select;
 use opentelemetry::KeyValue;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 use tokio::sync::{watch, Mutex, Notify};
-use zstd::stream::{decode_all as zstd_decode, Encoder};
+
 use opentelemetry::KeyValue;
 use garage_util::data::*;
 use garage_util::error::*;
--- a/src/model/lib.rs
+++ b/src/model/lib.rs
@ -1,5 +1,5 @@
 #[macro_use]
-extern crate log;
+extern crate tracing;
 pub mod permission;
--- a/src/rpc/Cargo.toml
+++ b/src/rpc/Cargo.toml
@ -20,7 +20,7 @@ arc-swap = "1.0"
 bytes = "1.0"
 gethostname = "0.2"
 hex = "0.4"
-log = "0.4"
+tracing = "0.1.30"
 rand = "0.8"
 sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" }
--- a/src/rpc/consul.rs
+++ b/src/rpc/consul.rs
@ -139,10 +139,10 @@ pub async fn publish_consul_service(
 	let resp = client.request(req).await?;
 	debug!("Response of advertising to Consul: {:?}", resp);
 	let resp_code = resp.status();
 	let resp_bytes = &hyper::body::to_bytes(resp.into_body()).await?;
 	debug!(
 		"{}",
-		std::str::from_utf8(&hyper::body::to_bytes(resp.into_body()).await?)
+		std::str::from_utf8(resp_bytes).unwrap_or("<invalid utf8>")
 			.unwrap_or("<invalid utf8>")
 	);
 	if resp_code != StatusCode::OK {
--- a/src/rpc/lib.rs
+++ b/src/rpc/lib.rs
@ -1,7 +1,7 @@
 //! Crate containing rpc related functions and types used in Garage
 #[macro_use]
-extern crate log;
+extern crate tracing;
 mod consul;
 mod kubernetes;
--- a/src/rpc/rpc_helper.rs
+++ b/src/rpc/rpc_helper.rs
@ -6,10 +6,15 @@ use futures::future::join_all;
 use futures::stream::futures_unordered::FuturesUnordered;
 use futures::stream::StreamExt;
 use futures_util::future::FutureExt;
 use opentelemetry::KeyValue;
 use tokio::select;
 use tokio::sync::{watch, Semaphore};
 use opentelemetry::KeyValue;
 use opentelemetry::{
 	trace::{FutureExt as OtelFutureExt, Span, TraceContextExt, Tracer},
 	Context,
 };
 pub use netapp::endpoint::{Endpoint, EndpointHandler, Message as Rpc};
 use netapp::peering::fullmesh::FullMeshPeeringStrategy;
 pub use netapp::proto::*;
@ -147,9 +152,17 @@ impl RpcHelper {
 		self.0.metrics.rpc_counter.add(1, &metric_tags);
 		let rpc_start_time = SystemTime::now();
 		let tracer = opentelemetry::global::tracer("garage");
 		let mut span = tracer.start(format!("RPC {}", endpoint.path()));
 		span.set_attribute(KeyValue::new("to", format!("{:?}", to)));
 		let node_id = to.into();
 		let rpc_call = endpoint
 			.call(&node_id, &msg, strat.rs_priority)
 			.with_context(Context::current_with_span(span));
 		select! {
-			res = endpoint.call(&node_id, &msg, strat.rs_priority) => {
+			res = rpc_call => {
 				drop(permit);
 				if res.is_err() {
@ -228,149 +241,164 @@ impl RpcHelper {
 	where
 		M: Rpc<Response = Result<S, Error>> + 'static,
 		H: EndpointHandler<M> + 'static,
-		S: Send,
+		S: Send + 'static,
 	{
 		let msg = Arc::new(msg);
 		// Build future for each request
 		// They are not started now: they are added below in a FuturesUnordered
 		// object that will take care of polling them (see below)
 		let requests = to.iter().cloned().map(|to| {
 			let self2 = self.clone();
 			let msg = msg.clone();
 			let endpoint2 = endpoint.clone();
 			(to, async move {
 				self2.call_arc(&endpoint2, to, msg, strategy).await
 			})
 		});
 		let quorum = strategy.rs_quorum.unwrap_or(to.len());
-		// Vectors in which success results and errors will be collected
+		let tracer = opentelemetry::global::tracer("garage");
-		let mut successes = vec![];
+		let mut span = tracer.start(format!("RPC {} to {:?}", endpoint.path(), to));
-		let mut errors = vec![];
+		span.set_attribute(KeyValue::new("to", format!("{:?}", to)));
 		span.set_attribute(KeyValue::new("quorum", quorum as i64));
-		if strategy.rs_interrupt_after_quorum {
+		async {
-			// Case 1: once quorum is reached, other requests don't matter.
+			let msg = Arc::new(msg);
 			// What we do here is only send the required number of requests
 			// to reach a quorum, priorizing nodes with the lowest latency.
 			// When there are errors, we start new requests to compensate.
-			// Retrieve some status variables that we will use to sort requests
+			// Build future for each request
-			let peer_list = self.0.fullmesh.get_peer_list();
+			// They are not started now: they are added below in a FuturesUnordered
-			let ring: Arc<Ring> = self.0.ring.borrow().clone();
+			// object that will take care of polling them (see below)
-			let our_zone = match ring.layout.node_role(&self.0.our_node_id) {
+			let requests = to.iter().cloned().map(|to| {
-				Some(pc) => &pc.zone,
+				let self2 = self.clone();
-				None => "",
+				let msg = msg.clone();
-			};
+				let endpoint2 = endpoint.clone();
-
+				(to, async move {
-			// Augment requests with some information used to sort them.
+					self2.call_arc(&endpoint2, to, msg, strategy).await
 			// The tuples are as follows:
 			//         (is another node?, is another zone?, latency, node ID, request future)
 			// We store all of these tuples in a vec that we can sort.
 			// By sorting this vec, we priorize ourself, then nodes in the same zone,
 			// and within a same zone we priorize nodes with the lowest latency.
 			let mut requests = requests
 				.map(|(to, fut)| {
 					let peer_zone = match ring.layout.node_role(&to) {
 						Some(pc) => &pc.zone,
 						None => "",
 					};
 					let peer_avg_ping = peer_list
 						.iter()
 						.find(|x| x.id.as_ref() == to.as_slice())
 						.map(|pi| pi.avg_ping)
 						.flatten()
 						.unwrap_or_else(|| Duration::from_secs(1));
 					(
 						to != self.0.our_node_id,
 						peer_zone != our_zone,
 						peer_avg_ping,
 						to,
 						fut,
 					)
 				})
-				.collect::<Vec<_>>();
+			});
-			// Sort requests by (priorize ourself, priorize same zone, priorize low latency)
+			// Vectors in which success results and errors will be collected
-			requests
+			let mut successes = vec![];
-				.sort_by_key(|(diffnode, diffzone, ping, _to, _fut)| (*diffnode, *diffzone, *ping));
+			let mut errors = vec![];
-			// Make an iterator to take requests in their sorted order
+			if strategy.rs_interrupt_after_quorum {
-			let mut requests = requests.into_iter();
+				// Case 1: once quorum is reached, other requests don't matter.
 				// What we do here is only send the required number of requests
 				// to reach a quorum, priorizing nodes with the lowest latency.
 				// When there are errors, we start new requests to compensate.
-			// resp_stream will contain all of the requests that are currently in flight.
+				// Retrieve some status variables that we will use to sort requests
-			// (for the moment none, they will be added in the loop below)
+				let peer_list = self.0.fullmesh.get_peer_list();
-			let mut resp_stream = FuturesUnordered::new();
+				let ring: Arc<Ring> = self.0.ring.borrow().clone();
 				let our_zone = match ring.layout.node_role(&self.0.our_node_id) {
 					Some(pc) => &pc.zone,
 					None => "",
 				};
-			// Do some requests and collect results
+				// Augment requests with some information used to sort them.
-			'request_loop: while successes.len() < quorum {
+				// The tuples are as follows:
-				// If the current set of requests that are running is not enough to possibly
+				//         (is another node?, is another zone?, latency, node ID, request future)
-				// reach quorum, start some new requests.
+				// We store all of these tuples in a vec that we can sort.
-				while successes.len() + resp_stream.len() < quorum {
+				// By sorting this vec, we priorize ourself, then nodes in the same zone,
-					if let Some((_, _, _, _to, fut)) = requests.next() {
+				// and within a same zone we priorize nodes with the lowest latency.
-						resp_stream.push(fut);
+				let mut requests = requests
-					} else {
+					.map(|(to, fut)| {
-						// If we have no request to add, we know that we won't ever
+						let peer_zone = match ring.layout.node_role(&to) {
-						// reach quorum: bail out now.
+							Some(pc) => &pc.zone,
-						break 'request_loop;
+							None => "",
-					}
+						};
-				}
+						let peer_avg_ping = peer_list
-				assert!(!resp_stream.is_empty()); // because of loop invariants
+							.iter()
 							.find(|x| x.id.as_ref() == to.as_slice())
 							.map(|pi| pi.avg_ping)
 							.flatten()
 							.unwrap_or_else(|| Duration::from_secs(1));
 						(
 							to != self.0.our_node_id,
 							peer_zone != our_zone,
 							peer_avg_ping,
 							to,
 							fut,
 						)
 					})
 					.collect::<Vec<_>>();
-				// Wait for one request to terminate
+				// Sort requests by (priorize ourself, priorize same zone, priorize low latency)
-				match resp_stream.next().await.unwrap() {
+				requests.sort_by_key(|(diffnode, diffzone, ping, _to, _fut)| {
-					Ok(msg) => {
+					(*diffnode, *diffzone, *ping)
-						successes.push(msg);
+				});
 					}
 					Err(e) => {
 						errors.push(e);
 					}
 				}
 			}
 		} else {
 			// Case 2: all of the requests need to be sent in all cases,
 			// and need to terminate. (this is the case for writes that
 			// must be spread to n nodes)
 			// Just start all the requests in parallel and return as soon
 			// as the quorum is reached.
 			let mut resp_stream = requests
 				.map(|(_, fut)| fut)
 				.collect::<FuturesUnordered<_>>();
-			while let Some(resp) = resp_stream.next().await {
+				// Make an iterator to take requests in their sorted order
-				match resp {
+				let mut requests = requests.into_iter();
-					Ok(msg) => {
+
-						successes.push(msg);
+				// resp_stream will contain all of the requests that are currently in flight.
-						if successes.len() >= quorum {
+				// (for the moment none, they will be added in the loop below)
-							break;
+				let mut resp_stream = FuturesUnordered::new();
 				// Do some requests and collect results
 				'request_loop: while successes.len() < quorum {
 					// If the current set of requests that are running is not enough to possibly
 					// reach quorum, start some new requests.
 					while successes.len() + resp_stream.len() < quorum {
 						if let Some((_, _, _, req_to, fut)) = requests.next() {
 							let tracer = opentelemetry::global::tracer("garage");
 							let span = tracer.start(format!("RPC to {:?}", req_to));
 							resp_stream.push(tokio::spawn(
 								fut.with_context(Context::current_with_span(span)),
 							));
 						} else {
 							// If we have no request to add, we know that we won't ever
 							// reach quorum: bail out now.
 							break 'request_loop;
 						}
 					}
-					Err(e) => {
+					assert!(!resp_stream.is_empty()); // because of loop invariants
-						errors.push(e);
+
 					// Wait for one request to terminate
 					match resp_stream.next().await.unwrap().unwrap() {
 						Ok(msg) => {
 							successes.push(msg);
 						}
 						Err(e) => {
 							errors.push(e);
 						}
 					}
 				}
 			} else {
 				// Case 2: all of the requests need to be sent in all cases,
 				// and need to terminate. (this is the case for writes that
 				// must be spread to n nodes)
 				// Just start all the requests in parallel and return as soon
 				// as the quorum is reached.
 				let mut resp_stream = requests
 					.map(|(_, fut)| fut)
 					.collect::<FuturesUnordered<_>>();
 				while let Some(resp) = resp_stream.next().await {
 					match resp {
 						Ok(msg) => {
 							successes.push(msg);
 							if successes.len() >= quorum {
 								break;
 							}
 						}
 						Err(e) => {
 							errors.push(e);
 						}
 					}
 				}
 				if !resp_stream.is_empty() {
 					// Continue remaining requests in background.
 					// Continue the remaining requests immediately using tokio::spawn
 					// but enqueue a task in the background runner
 					// to ensure that the process won't exit until the requests are done
 					// (if we had just enqueued the resp_stream.collect directly in the background runner,
 					// the requests might have been put on hold in the background runner's queue,
 					// in which case they might timeout or otherwise fail)
 					let wait_finished_fut = tokio::spawn(async move {
 						resp_stream.collect::<Vec<Result<_, _>>>().await;
 					});
 					self.0.background.spawn(wait_finished_fut.map(|_| Ok(())));
 				}
 			}
-			if !resp_stream.is_empty() {
+			if successes.len() >= quorum {
-				// Continue remaining requests in background.
+				Ok(successes)
-				// Continue the remaining requests immediately using tokio::spawn
+			} else {
-				// but enqueue a task in the background runner
+				let errors = errors.iter().map(|e| format!("{}", e)).collect::<Vec<_>>();
-				// to ensure that the process won't exit until the requests are done
+				Err(Error::Quorum(quorum, successes.len(), to.len(), errors))
 				// (if we had just enqueued the resp_stream.collect directly in the background runner,
 				// the requests might have been put on hold in the background runner's queue,
 				// in which case they might timeout or otherwise fail)
 				let wait_finished_fut = tokio::spawn(async move {
 					resp_stream.collect::<Vec<Result<_, _>>>().await;
 				});
 				self.0.background.spawn(wait_finished_fut.map(|_| Ok(())));
 			}
 		}
-
+		.with_context(Context::current_with_span(span))
-		if successes.len() >= quorum {
+		.await
 			Ok(successes)
 		} else {
 			let errors = errors.iter().map(|e| format!("{}", e)).collect::<Vec<_>>();
 			Err(Error::Quorum(quorum, successes.len(), to.len(), errors))
 		}
 	}
 }
--- a/src/table/Cargo.toml
+++ b/src/table/Cargo.toml
@ -22,7 +22,7 @@ opentelemetry = "0.17"
 async-trait = "0.1.7"
 bytes = "1.0"
 hexdump = "0.1"
-log = "0.4"
+tracing = "0.1.30"
 rand = "0.8"
 sled = "0.34"
--- a/src/table/data.rs
+++ b/src/table/data.rs
@ -1,7 +1,6 @@
 use core::borrow::Borrow;
 use std::sync::Arc;
 use log::warn;
 use serde_bytes::ByteBuf;
 use sled::Transactional;
 use tokio::sync::Notify;
--- a/src/table/lib.rs
+++ b/src/table/lib.rs
@ -2,7 +2,7 @@
 #![allow(clippy::comparison_chain)]
 #[macro_use]
-extern crate log;
+extern crate tracing;
 mod metrics;
 pub mod schema;
--- a/src/table/merkle.rs
+++ b/src/table/merkle.rs
@ -3,7 +3,6 @@ use std::time::Duration;
 use futures::select;
 use futures_util::future::*;
 use log::{debug, warn};
 use serde::{Deserialize, Serialize};
 use sled::transaction::{
 	ConflictableTransactionError, ConflictableTransactionResult, TransactionalTree,
--- a/src/util/Cargo.toml
+++ b/src/util/Cargo.toml
@ -18,7 +18,7 @@ blake2 = "0.9"
 err-derive = "0.3"
 xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
 hex = "0.4"
-log = "0.4"
+tracing = "0.1.30"
 rand = "0.8"
 sha2 = "0.9"
--- a/src/util/config.rs
+++ b/src/util/config.rs
@ -104,6 +104,8 @@ pub struct WebConfig {
 pub struct AdminConfig {
 	/// Address and port to bind for admin API serving
 	pub bind_addr: SocketAddr,
 	/// OTLP server to where to export traces
 	pub otlp_export_traces_to: Option<String>,
 }
 fn default_sled_cache_capacity() -> u64 {
--- a/src/util/lib.rs
+++ b/src/util/lib.rs
@ -1,7 +1,7 @@
 //! Crate containing common functions and types used in Garage
 #[macro_use]
-extern crate log;
+extern crate tracing;
 pub mod background;
 pub mod config;
--- a/src/web/Cargo.toml
+++ b/src/web/Cargo.toml
@ -20,7 +20,7 @@ garage_util = { version = "0.6.0", path = "../util" }
 garage_table = { version = "0.6.0", path = "../table" }
 err-derive = "0.3"
-log = "0.4"
+tracing = "0.1.30"
 percent-encoding = "2.1.0"
 futures = "0.3"
--- a/src/web/lib.rs
+++ b/src/web/lib.rs
@ -1,6 +1,6 @@
 //! Crate for handling web serving of s3 bucket
 #[macro_use]
-extern crate log;
+extern crate tracing;
 mod error;
 pub use error::Error;