forked from Deuxfleurs/garage
Merge pull request 'use netapp streaming body' (#343) from netapp-stream-body into main
Reviewed-on: Deuxfleurs/garage#343
This commit is contained in:
commit
11bdc971e2
24 changed files with 615 additions and 349 deletions
25
Cargo.lock
generated
25
Cargo.lock
generated
|
|
@ -59,6 +59,20 @@ dependencies = [
|
|||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-compression"
|
||||
version = "0.3.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00461f243d703f6999c8e7494f077799f1362720a55ae49a90ffe6214032fc0b"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"zstd",
|
||||
"zstd-safe",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async-stream"
|
||||
version = "0.3.3"
|
||||
|
|
@ -394,9 +408,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
|||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
|
||||
checksum = "f0b3de4a0c5e67e16066a0715723abd91edc2f9001d09c46e1dca929351e130e"
|
||||
|
||||
[[package]]
|
||||
name = "bytes-utils"
|
||||
|
|
@ -1058,6 +1072,7 @@ name = "garage_block"
|
|||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-compression",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"futures",
|
||||
|
|
@ -1073,6 +1088,7 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_bytes",
|
||||
"tokio",
|
||||
"tokio-util 0.6.9",
|
||||
"tracing",
|
||||
"zstd",
|
||||
]
|
||||
|
|
@ -2016,9 +2032,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "netapp"
|
||||
version = "0.4.5"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38e390e559aff9e3865c7cd326c691cbf25c580a96f259d2debc35bd4e14018e"
|
||||
checksum = "14fa05cce4c65d4cc5277b2525c5054fca0e51db82416d9f43fb916575233774"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
|
|
@ -2032,6 +2048,7 @@ dependencies = [
|
|||
"log",
|
||||
"opentelemetry",
|
||||
"opentelemetry-contrib",
|
||||
"pin-project 1.0.10",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
|
|
|
|||
135
Cargo.nix
135
Cargo.nix
|
|
@ -135,6 +135,28 @@ in
|
|||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".async-compression."0.3.10" = overridableMkRustCrate (profileName: rec {
|
||||
name = "async-compression";
|
||||
version = "0.3.10";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "00461f243d703f6999c8e7494f077799f1362720a55ae49a90ffe6214032fc0b"; };
|
||||
features = builtins.concatLists [
|
||||
[ "default" ]
|
||||
[ "libzstd" ]
|
||||
[ "tokio" ]
|
||||
[ "zstd" ]
|
||||
[ "zstd-safe" ]
|
||||
];
|
||||
dependencies = {
|
||||
futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
memchr = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; };
|
||||
pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; };
|
||||
tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
|
||||
libzstd = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.2+zstd.1.5.1" { inherit profileName; };
|
||||
zstd_safe = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd-safe."4.1.3+zstd.1.5.1" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.3" = overridableMkRustCrate (profileName: rec {
|
||||
name = "async-stream";
|
||||
version = "0.3.3";
|
||||
|
|
@ -252,7 +274,7 @@ in
|
|||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
aws_smithy_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-xml."0.38.0" { inherit profileName; };
|
||||
aws_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-types."0.8.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md5."0.7.0" { inherit profileName; };
|
||||
tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; };
|
||||
|
|
@ -298,7 +320,7 @@ in
|
|||
dependencies = {
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
form_urlencoded = rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
|
|
@ -345,7 +367,7 @@ in
|
|||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
aws_smithy_http_tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http-tower."0.38.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
fastrand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fastrand."1.7.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; };
|
||||
|
|
@ -367,7 +389,7 @@ in
|
|||
src = fetchCratesIo { inherit name version; sha256 = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"; };
|
||||
dependencies = {
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
crc32fast = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
|
@ -387,7 +409,7 @@ in
|
|||
dependencies = {
|
||||
aws_smithy_eventstream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-eventstream."0.38.0" { inherit profileName; };
|
||||
aws_smithy_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-types."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
bytes_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes-utils."0.1.2" { inherit profileName; };
|
||||
futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
|
|
@ -408,7 +430,7 @@ in
|
|||
src = fetchCratesIo { inherit name version; sha256 = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"; };
|
||||
dependencies = {
|
||||
aws_smithy_http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-smithy-http."0.38.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; };
|
||||
pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; };
|
||||
|
|
@ -557,11 +579,11 @@ in
|
|||
];
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "bytes";
|
||||
version = "1.1.0";
|
||||
version = "1.2.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "f0b3de4a0c5e67e16066a0715723abd91edc2f9001d09c46e1dca929351e130e"; };
|
||||
features = builtins.concatLists [
|
||||
[ "default" ]
|
||||
[ "std" ]
|
||||
|
|
@ -578,7 +600,7 @@ in
|
|||
[ "std" ]
|
||||
];
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
either = rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.6.1" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
|
@ -758,7 +780,7 @@ in
|
|||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"; };
|
||||
dependencies = {
|
||||
${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-linux-android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.config == "aarch64-linux-android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
|
|
@ -1392,7 +1414,7 @@ in
|
|||
];
|
||||
dependencies = {
|
||||
async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
bytesize = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
|
||||
|
|
@ -1406,7 +1428,7 @@ in
|
|||
garage_web = rustPackages."unknown".garage_web."0.8.0" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
|
||||
netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.5" { inherit profileName; };
|
||||
netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" { inherit profileName; };
|
||||
opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
opentelemetry_otlp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; };
|
||||
opentelemetry_prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; };
|
||||
|
|
@ -1450,7 +1472,7 @@ in
|
|||
dependencies = {
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "async_trait" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "base64" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "chrono" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "crypto_common" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crypto-common."0.1.6" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "err_derive" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; };
|
||||
|
|
@ -1499,8 +1521,9 @@ in
|
|||
];
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "arc_swap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "async_compression" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-compression."0.3.10" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "async_trait" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "futures" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_db" else null } = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; };
|
||||
|
|
@ -1514,6 +1537,7 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "serde_bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "tokio_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.9" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "tracing" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "zstd" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.2+zstd.1.5.1" { inherit profileName; };
|
||||
};
|
||||
|
|
@ -1575,7 +1599,7 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_table" else null } = rustPackages."unknown".garage_table."0.8.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_util" else null } = rustPackages."unknown".garage_util."0.8.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "hex" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "rmp_serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
|
||||
|
|
@ -1603,7 +1627,7 @@ in
|
|||
dependencies = {
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "arc_swap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "async_trait" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "garage_util" else null } = rustPackages."unknown".garage_util."0.8.0" { inherit profileName; };
|
||||
|
|
@ -1613,7 +1637,7 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "k8s_openapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "kube" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "sodiumoxide" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "openssl" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl."0.10.38" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "pnet_datalink" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; };
|
||||
|
|
@ -1636,7 +1660,7 @@ in
|
|||
src = fetchCrateLocal (workspaceSrc + "/src/table");
|
||||
dependencies = {
|
||||
async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
|
||||
garage_db = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; };
|
||||
|
|
@ -1665,7 +1689,7 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "arc_swap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "async_trait" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "blake2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.9.2" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "chrono" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "digest" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "err_derive" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; };
|
||||
|
|
@ -1676,7 +1700,7 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "http" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "hyper" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "lazy_static" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rmp_serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
|
||||
|
|
@ -1782,7 +1806,7 @@ in
|
|||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"; };
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
fnv = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; };
|
||||
futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
futures_sink = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.21" { inherit profileName; };
|
||||
|
|
@ -1958,7 +1982,7 @@ in
|
|||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"; };
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
fnv = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; };
|
||||
itoa = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; };
|
||||
};
|
||||
|
|
@ -1970,7 +1994,7 @@ in
|
|||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"; };
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; };
|
||||
};
|
||||
|
|
@ -2037,7 +2061,7 @@ in
|
|||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "tcp")
|
||||
];
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_channel" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
|
||||
|
|
@ -2100,7 +2124,7 @@ in
|
|||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"; };
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; };
|
||||
native_tls = rustPackages."registry+https://github.com/rust-lang/crates.io-index".native-tls."0.2.8" { inherit profileName; };
|
||||
tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
|
||||
|
|
@ -2302,7 +2326,7 @@ in
|
|||
];
|
||||
dependencies = {
|
||||
base64 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; };
|
||||
|
|
@ -2373,7 +2397,7 @@ in
|
|||
];
|
||||
dependencies = {
|
||||
base64 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
|
||||
dirs = rustPackages."registry+https://github.com/rust-lang/crates.io-index".dirs-next."2.0.0" { inherit profileName; };
|
||||
either = rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.6.1" { inherit profileName; };
|
||||
|
|
@ -2756,7 +2780,7 @@ in
|
|||
[ "default" ]
|
||||
];
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
encoding_rs = rustPackages."registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.30" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
|
|
@ -2797,22 +2821,21 @@ in
|
|||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.5" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "netapp";
|
||||
version = "0.4.5";
|
||||
version = "0.5.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "38e390e559aff9e3865c7cd326c691cbf25c580a96f259d2debc35bd4e14018e"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "14fa05cce4c65d4cc5277b2525c5054fca0e51db82416d9f43fb916575233774"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "opentelemetry")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "opentelemetry-contrib")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rand")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "telemetry")
|
||||
];
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "arc_swap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "async_trait" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "cfg_if" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "err_derive" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
|
||||
|
|
@ -2822,7 +2845,8 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "log" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "opentelemetry_contrib" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-contrib."0.9.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_project" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rmp_serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.137" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
|
||||
|
|
@ -3456,7 +3480,7 @@ in
|
|||
[ "std" ]
|
||||
];
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
prost_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" { profileName = "__noProfile"; };
|
||||
};
|
||||
});
|
||||
|
|
@ -3467,7 +3491,7 @@ in
|
|||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"; };
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
heck = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; };
|
||||
itertools = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.3" { inherit profileName; };
|
||||
lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; };
|
||||
|
|
@ -3504,7 +3528,7 @@ in
|
|||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"; };
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
prost = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
|
@ -3923,7 +3947,7 @@ in
|
|||
dependencies = {
|
||||
async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
|
||||
base64 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
crc32fast = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crc32fast."1.3.2" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
|
|
@ -3969,7 +3993,7 @@ in
|
|||
src = fetchCratesIo { inherit name version; sha256 = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"; };
|
||||
dependencies = {
|
||||
base64 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; };
|
||||
digest = rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.9.0" { inherit profileName; };
|
||||
futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; };
|
||||
|
|
@ -4472,7 +4496,7 @@ in
|
|||
];
|
||||
dependencies = {
|
||||
bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; };
|
||||
${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" { inherit profileName; };
|
||||
static_init_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; };
|
||||
|
|
@ -4765,7 +4789,7 @@ in
|
|||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "winapi")
|
||||
];
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
${ if (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") && hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "memchr" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "mio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mio."0.8.2" { inherit profileName; };
|
||||
|
|
@ -4850,19 +4874,19 @@ in
|
|||
src = fetchCratesIo { inherit name version; sha256 = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "codec")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client") "default")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc") "io")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "io")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc") "slab")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc") "time")
|
||||
];
|
||||
dependencies = {
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client" then "futures_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client" then "futures_sink" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client" then "log" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "futures_sink" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.21" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "log" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "slab" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "k2v-client" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
|
|
@ -4874,9 +4898,10 @@ in
|
|||
features = builtins.concatLists [
|
||||
[ "compat" ]
|
||||
[ "futures-io" ]
|
||||
[ "io" ]
|
||||
];
|
||||
dependencies = {
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
futures_io = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-io."0.3.21" { inherit profileName; };
|
||||
futures_sink = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-sink."0.3.21" { inherit profileName; };
|
||||
|
|
@ -4923,7 +4948,7 @@ in
|
|||
async_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.3" { inherit profileName; };
|
||||
async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; };
|
||||
base64 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
|
||||
h2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; };
|
||||
|
|
@ -5025,7 +5050,7 @@ in
|
|||
];
|
||||
dependencies = {
|
||||
base64 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; };
|
||||
bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; };
|
||||
futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; };
|
||||
futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; };
|
||||
http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
|
|
@ -5572,10 +5597,10 @@ in
|
|||
];
|
||||
dependencies = {
|
||||
${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -8,10 +8,10 @@ rec {
|
|||
sha256 = "1xy9zpypqfxs5gcq5dcla4bfkhxmh5nzn9dyqkr03lqycm9wg5cr";
|
||||
};
|
||||
cargo2nixSrc = fetchGit {
|
||||
# As of 2022-03-17
|
||||
url = "https://github.com/superboum/cargo2nix";
|
||||
ref = "dedup_propagate";
|
||||
rev = "486675c67249e735dd7eb68e1b9feac9db102be7";
|
||||
# As of 2022-08-29, stacking two patches: superboum@dedup_propagate and Alexis211@fix_fetchcrategit
|
||||
url = "https://github.com/Alexis211/cargo2nix";
|
||||
ref = "fix_fetchcrategit";
|
||||
rev = "4b31c0cc05b6394916d46e9289f51263d81973b9";
|
||||
};
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
|||
FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m")
|
||||
|
||||
export RUST_BACKTRACE=1
|
||||
export RUST_LOG=garage=info,garage_api=debug
|
||||
export RUST_LOG=garage=info,garage_api=debug,netapp=trace
|
||||
MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
|
||||
|
||||
WHICH_GARAGE=$(which garage || exit 1)
|
||||
|
|
|
|||
|
|
@ -5,9 +5,12 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|||
use futures::{stream, stream::Stream, StreamExt, TryFutureExt};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
|
||||
use bytes::Bytes;
|
||||
use hyper::{Body, Request, Response};
|
||||
use serde::Serialize;
|
||||
|
||||
use garage_rpc::netapp::bytes_buf::BytesBuf;
|
||||
use garage_rpc::rpc_helper::OrderTag;
|
||||
use garage_table::*;
|
||||
use garage_util::data::*;
|
||||
use garage_util::time::*;
|
||||
|
|
@ -305,13 +308,18 @@ pub async fn handle_upload_part_copy(
|
|||
// if and only if the block returned is a block that already existed
|
||||
// in the Garage data store (thus we don't need to save it again).
|
||||
let garage2 = garage.clone();
|
||||
let order_stream = OrderTag::stream();
|
||||
let source_blocks = stream::iter(blocks_to_copy)
|
||||
.flat_map(|(block_hash, range_to_copy)| {
|
||||
.enumerate()
|
||||
.flat_map(|(i, (block_hash, range_to_copy))| {
|
||||
let garage3 = garage2.clone();
|
||||
stream::once(async move {
|
||||
let data = garage3.block_manager.rpc_get_block(&block_hash).await?;
|
||||
let data = garage3
|
||||
.block_manager
|
||||
.rpc_get_block(&block_hash, Some(order_stream.order(i as u64)))
|
||||
.await?;
|
||||
match range_to_copy {
|
||||
Some(r) => Ok((data[r].to_vec(), None)),
|
||||
Some(r) => Ok((data.slice(r), None)),
|
||||
None => Ok((data, Some(block_hash))),
|
||||
}
|
||||
})
|
||||
|
|
@ -365,10 +373,7 @@ pub async fn handle_upload_part_copy(
|
|||
// we need to insert that data as a new block.
|
||||
async move {
|
||||
if must_upload {
|
||||
garage2
|
||||
.block_manager
|
||||
.rpc_put_block(final_hash, data.into())
|
||||
.await
|
||||
garage2.block_manager.rpc_put_block(final_hash, data).await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -556,13 +561,13 @@ impl CopyPreconditionHeaders {
|
|||
}
|
||||
}
|
||||
|
||||
type BlockStreamItemOk = (Vec<u8>, Option<Hash>);
|
||||
type BlockStreamItemOk = (Bytes, Option<Hash>);
|
||||
type BlockStreamItem = Result<BlockStreamItemOk, garage_util::error::Error>;
|
||||
|
||||
struct Defragmenter<S: Stream<Item = BlockStreamItem>> {
|
||||
block_size: usize,
|
||||
block_stream: Pin<Box<stream::Peekable<S>>>,
|
||||
buffer: Vec<u8>,
|
||||
buffer: BytesBuf,
|
||||
hash: Option<Hash>,
|
||||
}
|
||||
|
||||
|
|
@ -571,7 +576,7 @@ impl<S: Stream<Item = BlockStreamItem>> Defragmenter<S> {
|
|||
Self {
|
||||
block_size,
|
||||
block_stream,
|
||||
buffer: vec![],
|
||||
buffer: BytesBuf::new(),
|
||||
hash: None,
|
||||
}
|
||||
}
|
||||
|
|
@ -589,7 +594,7 @@ impl<S: Stream<Item = BlockStreamItem>> Defragmenter<S> {
|
|||
|
||||
if self.buffer.is_empty() {
|
||||
let (next_block, next_block_hash) = self.block_stream.next().await.unwrap()?;
|
||||
self.buffer = next_block;
|
||||
self.buffer.extend(next_block);
|
||||
self.hash = next_block_hash;
|
||||
} else if self.buffer.len() + peeked_next_block.len() > self.block_size {
|
||||
break;
|
||||
|
|
@ -600,7 +605,7 @@ impl<S: Stream<Item = BlockStreamItem>> Defragmenter<S> {
|
|||
}
|
||||
}
|
||||
|
||||
Ok((std::mem::take(&mut self.buffer), self.hash.take()))
|
||||
Ok((self.buffer.take_all(), self.hash.take()))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,9 +7,9 @@ use http::header::{
|
|||
ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE,
|
||||
IF_NONE_MATCH, LAST_MODIFIED, RANGE,
|
||||
};
|
||||
use hyper::body::Bytes;
|
||||
use hyper::{Body, Request, Response, StatusCode};
|
||||
|
||||
use garage_rpc::rpc_helper::{netapp::stream::ByteStream, OrderTag};
|
||||
use garage_table::EmptyKey;
|
||||
use garage_util::data::*;
|
||||
|
||||
|
|
@ -242,10 +242,15 @@ pub async fn handle_get(
|
|||
Ok(resp_builder.body(body)?)
|
||||
}
|
||||
ObjectVersionData::FirstBlock(_, first_block_hash) => {
|
||||
let read_first_block = garage.block_manager.rpc_get_block(first_block_hash);
|
||||
let order_stream = OrderTag::stream();
|
||||
|
||||
let read_first_block = garage
|
||||
.block_manager
|
||||
.rpc_get_block_streaming(first_block_hash, Some(order_stream.order(0)));
|
||||
let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);
|
||||
|
||||
let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;
|
||||
let (first_block_stream, version) =
|
||||
futures::try_join!(read_first_block, get_next_blocks)?;
|
||||
let version = version.ok_or(Error::NoSuchKey)?;
|
||||
|
||||
let mut blocks = version
|
||||
|
|
@ -254,24 +259,26 @@ pub async fn handle_get(
|
|||
.iter()
|
||||
.map(|(_, vb)| (vb.hash, None))
|
||||
.collect::<Vec<_>>();
|
||||
blocks[0].1 = Some(first_block);
|
||||
blocks[0].1 = Some(first_block_stream);
|
||||
|
||||
let body_stream = futures::stream::iter(blocks)
|
||||
.map(move |(hash, data_opt)| {
|
||||
.enumerate()
|
||||
.map(move |(i, (hash, stream_opt))| {
|
||||
let garage = garage.clone();
|
||||
async move {
|
||||
if let Some(data) = data_opt {
|
||||
Ok(Bytes::from(data))
|
||||
if let Some(stream) = stream_opt {
|
||||
stream
|
||||
} else {
|
||||
garage
|
||||
.block_manager
|
||||
.rpc_get_block(&hash)
|
||||
.rpc_get_block_streaming(&hash, Some(order_stream.order(i as u64)))
|
||||
.await
|
||||
.map(Bytes::from)
|
||||
.unwrap_or_else(|e| error_stream(i, e))
|
||||
}
|
||||
}
|
||||
})
|
||||
.buffered(2);
|
||||
.buffered(2)
|
||||
.flatten();
|
||||
|
||||
let body = hyper::body::Body::wrap_stream(body_stream);
|
||||
Ok(resp_builder.body(body)?)
|
||||
|
|
@ -422,40 +429,79 @@ fn body_from_blocks_range(
|
|||
all_blocks.len(),
|
||||
4 + ((end - begin) / std::cmp::max(all_blocks[0].1.size as u64, 1024)) as usize,
|
||||
));
|
||||
let mut true_offset = 0;
|
||||
let mut block_offset: u64 = 0;
|
||||
for (_, b) in all_blocks.iter() {
|
||||
if true_offset >= end {
|
||||
if block_offset >= end {
|
||||
break;
|
||||
}
|
||||
// Keep only blocks that have an intersection with the requested range
|
||||
if true_offset < end && true_offset + b.size > begin {
|
||||
blocks.push((*b, true_offset));
|
||||
if block_offset < end && block_offset + b.size > begin {
|
||||
blocks.push((*b, block_offset));
|
||||
}
|
||||
true_offset += b.size;
|
||||
block_offset += b.size as u64;
|
||||
}
|
||||
|
||||
let order_stream = OrderTag::stream();
|
||||
let body_stream = futures::stream::iter(blocks)
|
||||
.map(move |(block, true_offset)| {
|
||||
.enumerate()
|
||||
.map(move |(i, (block, block_offset))| {
|
||||
let garage = garage.clone();
|
||||
async move {
|
||||
let data = garage.block_manager.rpc_get_block(&block.hash).await?;
|
||||
let data = Bytes::from(data);
|
||||
let start_in_block = if true_offset > begin {
|
||||
0
|
||||
} else {
|
||||
begin - true_offset
|
||||
};
|
||||
let end_in_block = if true_offset + block.size < end {
|
||||
block.size
|
||||
} else {
|
||||
end - true_offset
|
||||
};
|
||||
Result::<Bytes, Error>::Ok(
|
||||
data.slice(start_in_block as usize..end_in_block as usize),
|
||||
)
|
||||
garage
|
||||
.block_manager
|
||||
.rpc_get_block_streaming(&block.hash, Some(order_stream.order(i as u64)))
|
||||
.await
|
||||
.unwrap_or_else(|e| error_stream(i, e))
|
||||
.scan(block_offset, move |chunk_offset, chunk| {
|
||||
let r = match chunk {
|
||||
Ok(chunk_bytes) => {
|
||||
let chunk_len = chunk_bytes.len() as u64;
|
||||
let r = if *chunk_offset >= end {
|
||||
// The current chunk is after the part we want to read.
|
||||
// Returning None here will stop the scan, the rest of the
|
||||
// stream will be ignored
|
||||
None
|
||||
} else if *chunk_offset + chunk_len <= begin {
|
||||
// The current chunk is before the part we want to read.
|
||||
// We return a None that will be removed by the filter_map
|
||||
// below.
|
||||
Some(None)
|
||||
} else {
|
||||
// The chunk has an intersection with the requested range
|
||||
let start_in_chunk = if *chunk_offset > begin {
|
||||
0
|
||||
} else {
|
||||
begin - *chunk_offset
|
||||
};
|
||||
let end_in_chunk = if *chunk_offset + chunk_len < end {
|
||||
chunk_len
|
||||
} else {
|
||||
end - *chunk_offset
|
||||
};
|
||||
Some(Some(Ok(chunk_bytes
|
||||
.slice(start_in_chunk as usize..end_in_chunk as usize))))
|
||||
};
|
||||
*chunk_offset += chunk_bytes.len() as u64;
|
||||
r
|
||||
}
|
||||
Err(e) => Some(Some(Err(e))),
|
||||
};
|
||||
futures::future::ready(r)
|
||||
})
|
||||
.filter_map(futures::future::ready)
|
||||
}
|
||||
})
|
||||
.buffered(2);
|
||||
.buffered(2)
|
||||
.flatten();
|
||||
|
||||
hyper::body::Body::wrap_stream(body_stream)
|
||||
}
|
||||
|
||||
fn error_stream(i: usize, e: garage_util::error::Error) -> ByteStream {
|
||||
Box::pin(futures::stream::once(async move {
|
||||
Err(std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
format!("Could not get block {}: {}", i, e),
|
||||
))
|
||||
}))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use std::collections::{BTreeMap, BTreeSet, HashMap, VecDeque};
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::sync::Arc;
|
||||
|
||||
use futures::prelude::*;
|
||||
|
|
@ -13,6 +13,7 @@ use opentelemetry::{
|
|||
Context,
|
||||
};
|
||||
|
||||
use garage_rpc::netapp::bytes_buf::BytesBuf;
|
||||
use garage_table::*;
|
||||
use garage_util::async_hash::*;
|
||||
use garage_util::data::*;
|
||||
|
|
@ -108,7 +109,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
|||
size,
|
||||
etag: data_md5sum_hex.clone(),
|
||||
},
|
||||
first_block,
|
||||
first_block.to_vec(),
|
||||
)),
|
||||
};
|
||||
|
||||
|
|
@ -136,7 +137,6 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
|||
garage.version_table.insert(&version).await?;
|
||||
|
||||
// Transfer data and verify checksum
|
||||
let first_block = Bytes::from(first_block);
|
||||
let first_block_hash = async_blake2sum(first_block.clone()).await;
|
||||
|
||||
let tx_result = (|| async {
|
||||
|
|
@ -318,7 +318,6 @@ async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
|
|||
chunker.next(),
|
||||
)?;
|
||||
if let Some(block) = next_block {
|
||||
let block = Bytes::from(block);
|
||||
let (_, _, block_hash) = futures::future::join3(
|
||||
md5hasher.update(block.clone()),
|
||||
sha256hasher.update(block.clone()),
|
||||
|
|
@ -387,8 +386,7 @@ struct StreamChunker<S: Stream<Item = Result<Bytes, Error>>> {
|
|||
stream: S,
|
||||
read_all: bool,
|
||||
block_size: usize,
|
||||
buf: VecDeque<Bytes>,
|
||||
buf_len: usize,
|
||||
buf: BytesBuf,
|
||||
}
|
||||
|
||||
impl<S: Stream<Item = Result<Bytes, Error>> + Unpin> StreamChunker<S> {
|
||||
|
|
@ -397,45 +395,25 @@ impl<S: Stream<Item = Result<Bytes, Error>> + Unpin> StreamChunker<S> {
|
|||
stream,
|
||||
read_all: false,
|
||||
block_size,
|
||||
buf: VecDeque::with_capacity(8),
|
||||
buf_len: 0,
|
||||
buf: BytesBuf::new(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn next(&mut self) -> Result<Option<Vec<u8>>, Error> {
|
||||
while !self.read_all && self.buf_len < self.block_size {
|
||||
async fn next(&mut self) -> Result<Option<Bytes>, Error> {
|
||||
while !self.read_all && self.buf.len() < self.block_size {
|
||||
if let Some(block) = self.stream.next().await {
|
||||
let bytes = block?;
|
||||
trace!("Body next: {} bytes", bytes.len());
|
||||
self.buf_len += bytes.len();
|
||||
self.buf.push_back(bytes);
|
||||
self.buf.extend(bytes);
|
||||
} else {
|
||||
self.read_all = true;
|
||||
}
|
||||
}
|
||||
|
||||
if self.buf_len == 0 {
|
||||
if self.buf.is_empty() {
|
||||
Ok(None)
|
||||
} else {
|
||||
let mut slices = Vec::with_capacity(self.buf.len());
|
||||
let mut taken = 0;
|
||||
while self.buf_len > 0 && taken < self.block_size {
|
||||
let front = self.buf.pop_front().unwrap();
|
||||
if taken + front.len() <= self.block_size {
|
||||
taken += front.len();
|
||||
self.buf_len -= front.len();
|
||||
slices.push(front);
|
||||
} else {
|
||||
let front_take = self.block_size - taken;
|
||||
slices.push(front.slice(..front_take));
|
||||
self.buf.push_front(front.slice(front_take..));
|
||||
self.buf_len -= front_take;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(Some(
|
||||
slices.iter().map(|x| &x[..]).collect::<Vec<_>>().concat(),
|
||||
))
|
||||
Ok(Some(self.buf.take_max(self.block_size)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -545,7 +523,6 @@ pub async fn handle_put_part(
|
|||
// Copy block to store
|
||||
let version = Version::new(version_uuid, bucket_id, key, false);
|
||||
|
||||
let first_block = Bytes::from(first_block);
|
||||
let first_block_hash = async_blake2sum(first_block.clone()).await;
|
||||
|
||||
let (_, data_md5sum, data_sha256sum) = read_and_put_blocks(
|
||||
|
|
|
|||
|
|
@ -27,6 +27,8 @@ bytes = "1.0"
|
|||
hex = "0.4"
|
||||
tracing = "0.1.30"
|
||||
rand = "0.8"
|
||||
|
||||
async-compression = { version = "0.3", features = ["tokio", "zstd"] }
|
||||
zstd = { version = "0.9", default-features = false }
|
||||
|
||||
rmp-serde = "0.15"
|
||||
|
|
@ -36,7 +38,7 @@ serde_bytes = "0.11"
|
|||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
|
||||
tokio-util = { version = "0.6", features = ["io"] }
|
||||
|
||||
[features]
|
||||
system-libs = [ "zstd/pkg-config" ]
|
||||
|
|
|
|||
|
|
@ -5,13 +5,18 @@ use zstd::stream::{decode_all as zstd_decode, Encoder};
|
|||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
|
||||
pub enum DataBlockHeader {
|
||||
Plain,
|
||||
Compressed,
|
||||
}
|
||||
|
||||
/// A possibly compressed block of data
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub enum DataBlock {
|
||||
/// Uncompressed data
|
||||
Plain(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||
Plain(Bytes),
|
||||
/// Data compressed with zstd
|
||||
Compressed(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||
Compressed(Bytes),
|
||||
}
|
||||
|
||||
impl DataBlock {
|
||||
|
|
@ -31,7 +36,7 @@ impl DataBlock {
|
|||
/// Get the buffer, possibly decompressing it, and verify it's integrity.
|
||||
/// For Plain block, data is compared to hash, for Compressed block, zstd checksumming system
|
||||
/// is used instead.
|
||||
pub fn verify_get(self, hash: Hash) -> Result<Vec<u8>, Error> {
|
||||
pub fn verify_get(self, hash: Hash) -> Result<Bytes, Error> {
|
||||
match self {
|
||||
DataBlock::Plain(data) => {
|
||||
if blake2sum(&data) == hash {
|
||||
|
|
@ -40,9 +45,9 @@ impl DataBlock {
|
|||
Err(Error::CorruptData(hash))
|
||||
}
|
||||
}
|
||||
DataBlock::Compressed(data) => {
|
||||
zstd_decode(&data[..]).map_err(|_| Error::CorruptData(hash))
|
||||
}
|
||||
DataBlock::Compressed(data) => zstd_decode(&data[..])
|
||||
.map_err(|_| Error::CorruptData(hash))
|
||||
.map(Bytes::from),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -66,14 +71,28 @@ impl DataBlock {
|
|||
tokio::task::spawn_blocking(move || {
|
||||
if let Some(level) = level {
|
||||
if let Ok(data) = zstd_encode(&data[..], level) {
|
||||
return DataBlock::Compressed(data);
|
||||
return DataBlock::Compressed(data.into());
|
||||
}
|
||||
}
|
||||
DataBlock::Plain(data.to_vec()) // TODO: remove to_vec here
|
||||
DataBlock::Plain(data)
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub fn into_parts(self) -> (DataBlockHeader, Bytes) {
|
||||
match self {
|
||||
DataBlock::Plain(data) => (DataBlockHeader::Plain, data),
|
||||
DataBlock::Compressed(data) => (DataBlockHeader::Compressed, data),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_parts(h: DataBlockHeader, bytes: Bytes) -> Self {
|
||||
match h {
|
||||
DataBlockHeader::Plain => DataBlock::Plain(bytes),
|
||||
DataBlockHeader::Compressed => DataBlock::Compressed(bytes),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn zstd_encode<R: std::io::Read>(mut source: R, level: i32) -> std::io::Result<Vec<u8>> {
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use std::path::PathBuf;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
|
|
@ -6,21 +7,26 @@ use async_trait::async_trait;
|
|||
use bytes::Bytes;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use futures::Stream;
|
||||
use futures_util::stream::StreamExt;
|
||||
use tokio::fs;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::sync::{mpsc, Mutex};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt, BufReader};
|
||||
use tokio::sync::{mpsc, Mutex, MutexGuard};
|
||||
|
||||
use opentelemetry::{
|
||||
trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer},
|
||||
Context,
|
||||
};
|
||||
|
||||
use garage_rpc::rpc_helper::netapp::stream::{stream_asyncread, ByteStream};
|
||||
|
||||
use garage_db as db;
|
||||
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::*;
|
||||
use garage_util::metrics::RecordDuration;
|
||||
|
||||
use garage_rpc::rpc_helper::OrderTag;
|
||||
use garage_rpc::system::System;
|
||||
use garage_rpc::*;
|
||||
|
||||
|
|
@ -36,7 +42,7 @@ use crate::resync::*;
|
|||
pub const INLINE_THRESHOLD: usize = 3072;
|
||||
|
||||
// Timeout for RPCs that read and write blocks to remote nodes
|
||||
pub(crate) const BLOCK_RW_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
pub(crate) const BLOCK_RW_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
// The delay between the moment when the reference counter
|
||||
// drops to zero, and the moment where we allow ourselves
|
||||
|
|
@ -48,12 +54,12 @@ pub(crate) const BLOCK_GC_DELAY: Duration = Duration::from_secs(600);
|
|||
pub enum BlockRpc {
|
||||
Ok,
|
||||
/// Message to ask for a block of data, by hash
|
||||
GetBlock(Hash),
|
||||
GetBlock(Hash, Option<OrderTag>),
|
||||
/// Message to send a block of data, either because requested, of for first delivery of new
|
||||
/// block
|
||||
PutBlock {
|
||||
hash: Hash,
|
||||
data: DataBlock,
|
||||
header: DataBlockHeader,
|
||||
},
|
||||
/// Ask other node if they should have this block, but don't actually have it
|
||||
NeedBlockQuery(Hash),
|
||||
|
|
@ -143,56 +149,162 @@ impl BlockManager {
|
|||
}
|
||||
|
||||
/// Ask nodes that might have a (possibly compressed) block for it
|
||||
pub(crate) async fn rpc_get_raw_block(&self, hash: &Hash) -> Result<DataBlock, Error> {
|
||||
/// Return it as a stream with a header
|
||||
async fn rpc_get_raw_block_streaming(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
order_tag: Option<OrderTag>,
|
||||
) -> Result<(DataBlockHeader, ByteStream), Error> {
|
||||
let who = self.replication.read_nodes(hash);
|
||||
let resps = self
|
||||
.system
|
||||
.rpc
|
||||
.try_call_many(
|
||||
&self.endpoint,
|
||||
&who[..],
|
||||
BlockRpc::GetBlock(*hash),
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(1)
|
||||
.with_timeout(BLOCK_RW_TIMEOUT)
|
||||
.interrupt_after_quorum(true),
|
||||
)
|
||||
.await?;
|
||||
let who = self.system.rpc.request_order(&who);
|
||||
|
||||
for resp in resps {
|
||||
if let BlockRpc::PutBlock { data, .. } = resp {
|
||||
return Ok(data);
|
||||
}
|
||||
for node in who.iter() {
|
||||
let node_id = NodeID::from(*node);
|
||||
let rpc = self.endpoint.call_streaming(
|
||||
&node_id,
|
||||
BlockRpc::GetBlock(*hash, order_tag),
|
||||
PRIO_NORMAL | PRIO_SECONDARY,
|
||||
);
|
||||
tokio::select! {
|
||||
res = rpc => {
|
||||
let res = match res {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
debug!("Node {:?} returned error: {}", node, e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let (header, stream) = match res.into_parts() {
|
||||
(Ok(BlockRpc::PutBlock { hash: _, header }), Some(stream)) => (header, stream),
|
||||
_ => {
|
||||
debug!("Node {:?} returned a malformed response", node);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
return Ok((header, stream));
|
||||
}
|
||||
_ = tokio::time::sleep(BLOCK_RW_TIMEOUT) => {
|
||||
debug!("Node {:?} didn't return block in time, trying next.", node);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
Err(Error::Message(format!(
|
||||
"Unable to read block {:?}: no valid blocks returned",
|
||||
"Unable to read block {:?}: no node returned a valid block",
|
||||
hash
|
||||
)))
|
||||
}
|
||||
|
||||
/// Ask nodes that might have a (possibly compressed) block for it
|
||||
/// Return its entire body
|
||||
pub(crate) async fn rpc_get_raw_block(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
order_tag: Option<OrderTag>,
|
||||
) -> Result<DataBlock, Error> {
|
||||
let who = self.replication.read_nodes(hash);
|
||||
let who = self.system.rpc.request_order(&who);
|
||||
|
||||
for node in who.iter() {
|
||||
let node_id = NodeID::from(*node);
|
||||
let rpc = self.endpoint.call_streaming(
|
||||
&node_id,
|
||||
BlockRpc::GetBlock(*hash, order_tag),
|
||||
PRIO_NORMAL | PRIO_SECONDARY,
|
||||
);
|
||||
tokio::select! {
|
||||
res = rpc => {
|
||||
let res = match res {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
debug!("Node {:?} returned error: {}", node, e);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
let (header, stream) = match res.into_parts() {
|
||||
(Ok(BlockRpc::PutBlock { hash: _, header }), Some(stream)) => (header, stream),
|
||||
_ => {
|
||||
debug!("Node {:?} returned a malformed response", node);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
match read_stream_to_end(stream).await {
|
||||
Ok(bytes) => return Ok(DataBlock::from_parts(header, bytes)),
|
||||
Err(e) => {
|
||||
debug!("Error reading stream from node {:?}: {}", node, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep(BLOCK_RW_TIMEOUT) => {
|
||||
debug!("Node {:?} didn't return block in time, trying next.", node);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
Err(Error::Message(format!(
|
||||
"Unable to read block {:?}: no node returned a valid block",
|
||||
hash
|
||||
)))
|
||||
}
|
||||
|
||||
// ---- Public interface ----
|
||||
|
||||
/// Ask nodes that might have a block for it,
|
||||
/// return it as a stream
|
||||
pub async fn rpc_get_block_streaming(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
order_tag: Option<OrderTag>,
|
||||
) -> Result<
|
||||
Pin<Box<dyn Stream<Item = Result<Bytes, std::io::Error>> + Send + Sync + 'static>>,
|
||||
Error,
|
||||
> {
|
||||
let (header, stream) = self.rpc_get_raw_block_streaming(hash, order_tag).await?;
|
||||
match header {
|
||||
DataBlockHeader::Plain => Ok(stream),
|
||||
DataBlockHeader::Compressed => {
|
||||
// Too many things, I hate it.
|
||||
let reader = stream_asyncread(stream);
|
||||
let reader = BufReader::new(reader);
|
||||
let reader = async_compression::tokio::bufread::ZstdDecoder::new(reader);
|
||||
Ok(Box::pin(tokio_util::io::ReaderStream::new(reader)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Ask nodes that might have a block for it
|
||||
pub async fn rpc_get_block(&self, hash: &Hash) -> Result<Vec<u8>, Error> {
|
||||
self.rpc_get_raw_block(hash).await?.verify_get(*hash)
|
||||
pub async fn rpc_get_block(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
order_tag: Option<OrderTag>,
|
||||
) -> Result<Bytes, Error> {
|
||||
self.rpc_get_raw_block(hash, order_tag)
|
||||
.await?
|
||||
.verify_get(*hash)
|
||||
}
|
||||
|
||||
/// Send block to nodes that should have it
|
||||
pub async fn rpc_put_block(&self, hash: Hash, data: Bytes) -> Result<(), Error> {
|
||||
let who = self.replication.write_nodes(&hash);
|
||||
let data = DataBlock::from_buffer(data, self.compression_level).await;
|
||||
|
||||
let (header, bytes) = DataBlock::from_buffer(data, self.compression_level)
|
||||
.await
|
||||
.into_parts();
|
||||
let put_block_rpc =
|
||||
Req::new(BlockRpc::PutBlock { hash, header })?.with_stream_from_buffer(bytes);
|
||||
|
||||
self.system
|
||||
.rpc
|
||||
.try_call_many(
|
||||
&self.endpoint,
|
||||
&who[..],
|
||||
// TODO: remove to_vec() here
|
||||
BlockRpc::PutBlock { hash, data },
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
put_block_rpc,
|
||||
RequestStrategy::with_priority(PRIO_NORMAL | PRIO_SECONDARY)
|
||||
.with_quorum(self.replication.write_quorum())
|
||||
.with_timeout(BLOCK_RW_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
|
@ -259,21 +371,25 @@ impl BlockManager {
|
|||
|
||||
// ---- Reading and writing blocks locally ----
|
||||
|
||||
/// Write a block to disk
|
||||
pub(crate) async fn write_block(
|
||||
async fn handle_put_block(
|
||||
&self,
|
||||
hash: &Hash,
|
||||
data: &DataBlock,
|
||||
) -> Result<BlockRpc, Error> {
|
||||
hash: Hash,
|
||||
header: DataBlockHeader,
|
||||
stream: Option<ByteStream>,
|
||||
) -> Result<(), Error> {
|
||||
let stream = stream.ok_or_message("missing stream")?;
|
||||
let bytes = read_stream_to_end(stream).await?;
|
||||
let data = DataBlock::from_parts(header, bytes);
|
||||
self.write_block(&hash, &data).await
|
||||
}
|
||||
|
||||
/// Write a block to disk
|
||||
pub(crate) async fn write_block(&self, hash: &Hash, data: &DataBlock) -> Result<(), Error> {
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
|
||||
let write_size = data.inner_buffer().len() as u64;
|
||||
|
||||
let res = self.mutation_lock[hash.as_slice()[0] as usize]
|
||||
.lock()
|
||||
.with_context(Context::current_with_span(
|
||||
tracer.start("Acquire mutation_lock"),
|
||||
))
|
||||
self.lock_mutate(hash)
|
||||
.await
|
||||
.write_block(hash, data, self)
|
||||
.bound_record_duration(&self.metrics.block_write_duration)
|
||||
|
|
@ -284,11 +400,32 @@ impl BlockManager {
|
|||
|
||||
self.metrics.bytes_written.add(write_size);
|
||||
|
||||
Ok(res)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handle_get_block(&self, hash: &Hash, order_tag: Option<OrderTag>) -> Resp<BlockRpc> {
|
||||
let block = match self.read_block(hash).await {
|
||||
Ok(data) => data,
|
||||
Err(e) => return Resp::new(Err(e)),
|
||||
};
|
||||
|
||||
let (header, data) = block.into_parts();
|
||||
|
||||
let resp = Resp::new(Ok(BlockRpc::PutBlock {
|
||||
hash: *hash,
|
||||
header,
|
||||
}))
|
||||
.with_stream_from_buffer(data);
|
||||
|
||||
if let Some(order_tag) = order_tag {
|
||||
resp.with_order_tag(order_tag)
|
||||
} else {
|
||||
resp
|
||||
}
|
||||
}
|
||||
|
||||
/// Read block from disk, verifying it's integrity
|
||||
pub(crate) async fn read_block(&self, hash: &Hash) -> Result<BlockRpc, Error> {
|
||||
pub(crate) async fn read_block(&self, hash: &Hash) -> Result<DataBlock, Error> {
|
||||
let data = self
|
||||
.read_block_internal(hash)
|
||||
.bound_record_duration(&self.metrics.block_read_duration)
|
||||
|
|
@ -298,7 +435,7 @@ impl BlockManager {
|
|||
.bytes_read
|
||||
.add(data.inner_buffer().len() as u64);
|
||||
|
||||
Ok(BlockRpc::PutBlock { hash: *hash, data })
|
||||
Ok(data)
|
||||
}
|
||||
|
||||
async fn read_block_internal(&self, hash: &Hash) -> Result<DataBlock, Error> {
|
||||
|
|
@ -321,16 +458,15 @@ impl BlockManager {
|
|||
drop(f);
|
||||
|
||||
let data = if compressed {
|
||||
DataBlock::Compressed(data)
|
||||
DataBlock::Compressed(data.into())
|
||||
} else {
|
||||
DataBlock::Plain(data)
|
||||
DataBlock::Plain(data.into())
|
||||
};
|
||||
|
||||
if data.verify(*hash).is_err() {
|
||||
self.metrics.corruption_counter.add(1);
|
||||
|
||||
self.mutation_lock[hash.as_slice()[0] as usize]
|
||||
.lock()
|
||||
self.lock_mutate(hash)
|
||||
.await
|
||||
.move_block_to_corrupted(hash, self)
|
||||
.await?;
|
||||
|
|
@ -343,8 +479,7 @@ impl BlockManager {
|
|||
|
||||
/// Check if this node has a block and whether it needs it
|
||||
pub(crate) async fn check_block_status(&self, hash: &Hash) -> Result<BlockStatus, Error> {
|
||||
self.mutation_lock[hash.as_slice()[0] as usize]
|
||||
.lock()
|
||||
self.lock_mutate(hash)
|
||||
.await
|
||||
.check_block_status(hash, self)
|
||||
.await
|
||||
|
|
@ -358,8 +493,7 @@ impl BlockManager {
|
|||
|
||||
/// Delete block if it is not needed anymore
|
||||
pub(crate) async fn delete_if_unneeded(&self, hash: &Hash) -> Result<(), Error> {
|
||||
self.mutation_lock[hash.as_slice()[0] as usize]
|
||||
.lock()
|
||||
self.lock_mutate(hash)
|
||||
.await
|
||||
.delete_if_unneeded(hash, self)
|
||||
.await
|
||||
|
|
@ -391,20 +525,32 @@ impl BlockManager {
|
|||
path.set_extension("");
|
||||
fs::metadata(&path).await.map(|_| false).map_err(Into::into)
|
||||
}
|
||||
|
||||
async fn lock_mutate(&self, hash: &Hash) -> MutexGuard<'_, BlockManagerLocked> {
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
self.mutation_lock[hash.as_slice()[0] as usize]
|
||||
.lock()
|
||||
.with_context(Context::current_with_span(
|
||||
tracer.start("Acquire mutation_lock"),
|
||||
))
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl EndpointHandler<BlockRpc> for BlockManager {
|
||||
async fn handle(
|
||||
self: &Arc<Self>,
|
||||
message: &BlockRpc,
|
||||
_from: NodeID,
|
||||
) -> Result<BlockRpc, Error> {
|
||||
match message {
|
||||
BlockRpc::PutBlock { hash, data } => self.write_block(hash, data).await,
|
||||
BlockRpc::GetBlock(h) => self.read_block(h).await,
|
||||
BlockRpc::NeedBlockQuery(h) => self.need_block(h).await.map(BlockRpc::NeedBlockReply),
|
||||
m => Err(Error::unexpected_rpc_message(m)),
|
||||
impl StreamingEndpointHandler<BlockRpc> for BlockManager {
|
||||
async fn handle(self: &Arc<Self>, mut message: Req<BlockRpc>, _from: NodeID) -> Resp<BlockRpc> {
|
||||
match message.msg() {
|
||||
BlockRpc::PutBlock { hash, header } => Resp::new(
|
||||
self.handle_put_block(*hash, *header, message.take_stream())
|
||||
.await
|
||||
.map(|_| BlockRpc::Ok),
|
||||
),
|
||||
BlockRpc::GetBlock(h, order_tag) => self.handle_get_block(h, *order_tag).await,
|
||||
BlockRpc::NeedBlockQuery(h) => {
|
||||
Resp::new(self.need_block(h).await.map(BlockRpc::NeedBlockReply))
|
||||
}
|
||||
m => Resp::new(Err(Error::unexpected_rpc_message(m))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -431,7 +577,7 @@ impl BlockManagerLocked {
|
|||
hash: &Hash,
|
||||
data: &DataBlock,
|
||||
mgr: &BlockManager,
|
||||
) -> Result<BlockRpc, Error> {
|
||||
) -> Result<(), Error> {
|
||||
let compressed = data.is_compressed();
|
||||
let data = data.inner_buffer();
|
||||
|
||||
|
|
@ -442,8 +588,8 @@ impl BlockManagerLocked {
|
|||
fs::create_dir_all(&directory).await?;
|
||||
|
||||
let to_delete = match (mgr.is_block_compressed(hash).await, compressed) {
|
||||
(Ok(true), _) => return Ok(BlockRpc::Ok),
|
||||
(Ok(false), false) => return Ok(BlockRpc::Ok),
|
||||
(Ok(true), _) => return Ok(()),
|
||||
(Ok(false), false) => return Ok(()),
|
||||
(Ok(false), true) => {
|
||||
let path_to_delete = path.clone();
|
||||
path.set_extension("zst");
|
||||
|
|
@ -482,7 +628,7 @@ impl BlockManagerLocked {
|
|||
dir.sync_all().await?;
|
||||
drop(dir);
|
||||
|
||||
Ok(BlockRpc::Ok)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn move_block_to_corrupted(&self, hash: &Hash, mgr: &BlockManager) -> Result<(), Error> {
|
||||
|
|
@ -516,3 +662,17 @@ impl BlockManagerLocked {
|
|||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_stream_to_end(mut stream: ByteStream) -> Result<Bytes, Error> {
|
||||
let mut parts: Vec<Bytes> = vec![];
|
||||
while let Some(part) = stream.next().await {
|
||||
parts.push(part.ok_or_message("error in stream")?);
|
||||
}
|
||||
|
||||
Ok(parts
|
||||
.iter()
|
||||
.map(|x| &x[..])
|
||||
.collect::<Vec<_>>()
|
||||
.concat()
|
||||
.into())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ use arc_swap::ArcSwap;
|
|||
use async_trait::async_trait;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use futures::future::*;
|
||||
use tokio::select;
|
||||
use tokio::sync::{watch, Notify};
|
||||
|
||||
|
|
@ -36,7 +35,11 @@ use crate::manager::*;
|
|||
|
||||
// Timeout for RPCs that ask other nodes whether they need a copy
|
||||
// of a given block before we delete it locally
|
||||
pub(crate) const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
// The timeout here is relatively low because we don't want to block
|
||||
// the entire resync loop when some nodes are not responding.
|
||||
// Nothing will be deleted if the nodes don't answer the queries,
|
||||
// we will just retry later.
|
||||
const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
// The delay between the time where a resync operation fails
|
||||
// and the time when it is retried, with exponential backoff
|
||||
|
|
@ -336,24 +339,24 @@ impl BlockResyncManager {
|
|||
}
|
||||
who.retain(|id| *id != manager.system.id);
|
||||
|
||||
let msg = Arc::new(BlockRpc::NeedBlockQuery(*hash));
|
||||
let who_needs_fut = who.iter().map(|to| {
|
||||
manager.system.rpc.call_arc(
|
||||
let who_needs_resps = manager
|
||||
.system
|
||||
.rpc
|
||||
.call_many(
|
||||
&manager.endpoint,
|
||||
*to,
|
||||
msg.clone(),
|
||||
&who,
|
||||
BlockRpc::NeedBlockQuery(*hash),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_timeout(NEED_BLOCK_QUERY_TIMEOUT),
|
||||
)
|
||||
});
|
||||
let who_needs_resps = join_all(who_needs_fut).await;
|
||||
.await?;
|
||||
|
||||
let mut need_nodes = vec![];
|
||||
for (node, needed) in who.iter().zip(who_needs_resps.into_iter()) {
|
||||
for (node, needed) in who_needs_resps {
|
||||
match needed.err_context("NeedBlockQuery RPC")? {
|
||||
BlockRpc::NeedBlockReply(needed) => {
|
||||
if needed {
|
||||
need_nodes.push(*node);
|
||||
need_nodes.push(node);
|
||||
}
|
||||
}
|
||||
m => {
|
||||
|
|
@ -376,7 +379,13 @@ impl BlockResyncManager {
|
|||
.add(1, &[KeyValue::new("to", format!("{:?}", node))]);
|
||||
}
|
||||
|
||||
let put_block_message = manager.read_block(hash).await?;
|
||||
let block = manager.read_block(hash).await?;
|
||||
let (header, bytes) = block.into_parts();
|
||||
let put_block_message = Req::new(BlockRpc::PutBlock {
|
||||
hash: *hash,
|
||||
header,
|
||||
})?
|
||||
.with_stream_from_buffer(bytes);
|
||||
manager
|
||||
.system
|
||||
.rpc
|
||||
|
|
@ -409,7 +418,7 @@ impl BlockResyncManager {
|
|||
hash
|
||||
);
|
||||
|
||||
let block_data = manager.rpc_get_raw_block(hash).await?;
|
||||
let block_data = manager.rpc_get_raw_block(hash, None).await?;
|
||||
|
||||
manager.metrics.resync_recv_counter.add(1);
|
||||
|
||||
|
|
|
|||
|
|
@ -50,9 +50,7 @@ futures = "0.3"
|
|||
futures-util = "0.3"
|
||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
|
||||
#netapp = { version = "0.3.2", git = "https://git.deuxfleurs.fr/lx/netapp" }
|
||||
#netapp = { version = "0.4", path = "../../../netapp" }
|
||||
netapp = "0.4"
|
||||
netapp = "0.5"
|
||||
|
||||
opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
|
||||
opentelemetry-prometheus = { version = "0.10", optional = true }
|
||||
|
|
|
|||
|
|
@ -683,7 +683,7 @@ impl AdminRpcHandler {
|
|||
.endpoint
|
||||
.call(
|
||||
&node,
|
||||
&AdminRpc::LaunchRepair(opt_to_send.clone()),
|
||||
AdminRpc::LaunchRepair(opt_to_send.clone()),
|
||||
PRIO_NORMAL,
|
||||
)
|
||||
.await;
|
||||
|
|
@ -723,7 +723,7 @@ impl AdminRpcHandler {
|
|||
let node_id = (*node).into();
|
||||
match self
|
||||
.endpoint
|
||||
.call(&node_id, &AdminRpc::Stats(opt), PRIO_NORMAL)
|
||||
.call(&node_id, AdminRpc::Stats(opt), PRIO_NORMAL)
|
||||
.await?
|
||||
{
|
||||
Ok(AdminRpc::Ok(s)) => writeln!(&mut ret, "{}", s).unwrap(),
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ pub async fn cli_command_dispatch(
|
|||
|
||||
pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) -> Result<(), Error> {
|
||||
let status = match rpc_cli
|
||||
.call(&rpc_host, &SystemRpc::GetKnownNodes, PRIO_NORMAL)
|
||||
.call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL)
|
||||
.await??
|
||||
{
|
||||
SystemRpc::ReturnKnownNodes(nodes) => nodes,
|
||||
|
|
@ -149,7 +149,7 @@ pub async fn cmd_connect(
|
|||
args: ConnectNodeOpt,
|
||||
) -> Result<(), Error> {
|
||||
match rpc_cli
|
||||
.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL)
|
||||
.call(&rpc_host, SystemRpc::Connect(args.node), PRIO_NORMAL)
|
||||
.await??
|
||||
{
|
||||
SystemRpc::Ok => {
|
||||
|
|
@ -165,7 +165,7 @@ pub async fn cmd_admin(
|
|||
rpc_host: NodeID,
|
||||
args: AdminRpc,
|
||||
) -> Result<(), HelperError> {
|
||||
match rpc_cli.call(&rpc_host, &args, PRIO_NORMAL).await?? {
|
||||
match rpc_cli.call(&rpc_host, args, PRIO_NORMAL).await?? {
|
||||
AdminRpc::Ok(msg) => {
|
||||
println!("{}", msg);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ pub async fn cmd_assign_role(
|
|||
args: AssignRoleOpt,
|
||||
) -> Result<(), Error> {
|
||||
let status = match rpc_cli
|
||||
.call(&rpc_host, &SystemRpc::GetKnownNodes, PRIO_NORMAL)
|
||||
.call(&rpc_host, SystemRpc::GetKnownNodes, PRIO_NORMAL)
|
||||
.await??
|
||||
{
|
||||
SystemRpc::ReturnKnownNodes(nodes) => nodes,
|
||||
|
|
@ -245,7 +245,7 @@ pub async fn fetch_layout(
|
|||
rpc_host: NodeID,
|
||||
) -> Result<ClusterLayout, Error> {
|
||||
match rpc_cli
|
||||
.call(&rpc_host, &SystemRpc::PullClusterLayout, PRIO_NORMAL)
|
||||
.call(&rpc_host, SystemRpc::PullClusterLayout, PRIO_NORMAL)
|
||||
.await??
|
||||
{
|
||||
SystemRpc::AdvertiseClusterLayout(t) => Ok(t),
|
||||
|
|
@ -261,7 +261,7 @@ pub async fn send_layout(
|
|||
rpc_cli
|
||||
.call(
|
||||
&rpc_host,
|
||||
&SystemRpc::AdvertiseClusterLayout(layout),
|
||||
SystemRpc::AdvertiseClusterLayout(layout),
|
||||
PRIO_NORMAL,
|
||||
)
|
||||
.await??;
|
||||
|
|
|
|||
|
|
@ -39,9 +39,7 @@ futures-util = "0.3"
|
|||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
opentelemetry = "0.17"
|
||||
|
||||
#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
|
||||
#netapp = { version = "0.4", path = "../../../netapp" }
|
||||
netapp = "0.4"
|
||||
netapp = "0.5"
|
||||
|
||||
[features]
|
||||
k2v = [ "garage_util/k2v" ]
|
||||
|
|
|
|||
|
|
@ -45,9 +45,7 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
|
|||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
opentelemetry = "0.17"
|
||||
|
||||
#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
|
||||
#netapp = { version = "0.4", path = "../../../netapp", features = ["telemetry"] }
|
||||
netapp = { version = "0.4.5", features = ["telemetry"] }
|
||||
netapp = { version = "0.5.0", features = ["telemetry"] }
|
||||
|
||||
hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] }
|
||||
|
||||
|
|
|
|||
|
|
@ -15,10 +15,14 @@ use opentelemetry::{
|
|||
Context,
|
||||
};
|
||||
|
||||
pub use netapp::endpoint::{Endpoint, EndpointHandler, Message as Rpc};
|
||||
pub use netapp::endpoint::{Endpoint, EndpointHandler, StreamingEndpointHandler};
|
||||
use netapp::message::IntoReq;
|
||||
pub use netapp::message::{
|
||||
Message as Rpc, OrderTag, Req, RequestPriority, Resp, PRIO_BACKGROUND, PRIO_HIGH, PRIO_NORMAL,
|
||||
PRIO_SECONDARY,
|
||||
};
|
||||
use netapp::peering::fullmesh::FullMeshPeeringStrategy;
|
||||
pub use netapp::proto::*;
|
||||
pub use netapp::{NetApp, NodeID};
|
||||
pub use netapp::{self, NetApp, NodeID};
|
||||
|
||||
use garage_util::background::BackgroundRunner;
|
||||
use garage_util::data::*;
|
||||
|
|
@ -28,12 +32,10 @@ use garage_util::metrics::RecordDuration;
|
|||
use crate::metrics::RpcMetrics;
|
||||
use crate::ring::Ring;
|
||||
|
||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
// Try to never have more than 200MB of outgoing requests
|
||||
// buffered at the same time. Other requests are queued until
|
||||
// space is freed.
|
||||
const REQUEST_BUFFER_SIZE: usize = 200 * 1024 * 1024;
|
||||
// Don't allow more than 100 concurrent outgoing RPCs.
|
||||
const MAX_CONCURRENT_REQUESTS: usize = 100;
|
||||
|
||||
/// Strategy to apply when making RPC
|
||||
#[derive(Copy, Clone)]
|
||||
|
|
@ -95,7 +97,7 @@ impl RpcHelper {
|
|||
background: Arc<BackgroundRunner>,
|
||||
ring: watch::Receiver<Arc<Ring>>,
|
||||
) -> Self {
|
||||
let sem = Arc::new(Semaphore::new(REQUEST_BUFFER_SIZE));
|
||||
let sem = Arc::new(Semaphore::new(MAX_CONCURRENT_REQUESTS));
|
||||
|
||||
let metrics = RpcMetrics::new(sem.clone());
|
||||
|
||||
|
|
@ -109,30 +111,17 @@ impl RpcHelper {
|
|||
}))
|
||||
}
|
||||
|
||||
pub async fn call<M, H, S>(
|
||||
pub async fn call<M, N, H, S>(
|
||||
&self,
|
||||
endpoint: &Endpoint<M, H>,
|
||||
to: Uuid,
|
||||
msg: M,
|
||||
msg: N,
|
||||
strat: RequestStrategy,
|
||||
) -> Result<S, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>>,
|
||||
H: EndpointHandler<M>,
|
||||
{
|
||||
self.call_arc(endpoint, to, Arc::new(msg), strat).await
|
||||
}
|
||||
|
||||
pub async fn call_arc<M, H, S>(
|
||||
&self,
|
||||
endpoint: &Endpoint<M, H>,
|
||||
to: Uuid,
|
||||
msg: Arc<M>,
|
||||
strat: RequestStrategy,
|
||||
) -> Result<S, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>>,
|
||||
H: EndpointHandler<M>,
|
||||
N: IntoReq<M> + Send,
|
||||
H: StreamingEndpointHandler<M>,
|
||||
{
|
||||
let metric_tags = [
|
||||
KeyValue::new("rpc_endpoint", endpoint.path().to_string()),
|
||||
|
|
@ -140,11 +129,10 @@ impl RpcHelper {
|
|||
KeyValue::new("to", format!("{:?}", to)),
|
||||
];
|
||||
|
||||
let msg_size = rmp_to_vec_all_named(&msg)?.len() as u32;
|
||||
let permit = self
|
||||
.0
|
||||
.request_buffer_semaphore
|
||||
.acquire_many(msg_size)
|
||||
.acquire()
|
||||
.record_duration(&self.0.metrics.rpc_queueing_time, &metric_tags)
|
||||
.await?;
|
||||
|
||||
|
|
@ -152,7 +140,7 @@ impl RpcHelper {
|
|||
|
||||
let node_id = to.into();
|
||||
let rpc_call = endpoint
|
||||
.call(&node_id, msg, strat.rs_priority)
|
||||
.call_streaming(&node_id, msg, strat.rs_priority)
|
||||
.record_duration(&self.0.metrics.rpc_duration, &metric_tags);
|
||||
|
||||
select! {
|
||||
|
|
@ -162,7 +150,7 @@ impl RpcHelper {
|
|||
if res.is_err() {
|
||||
self.0.metrics.rpc_netapp_error_counter.add(1, &metric_tags);
|
||||
}
|
||||
let res = res?;
|
||||
let res = res?.into_msg();
|
||||
|
||||
if res.is_err() {
|
||||
self.0.metrics.rpc_garage_error_counter.add(1, &metric_tags);
|
||||
|
|
@ -178,38 +166,42 @@ impl RpcHelper {
|
|||
}
|
||||
}
|
||||
|
||||
pub async fn call_many<M, H, S>(
|
||||
pub async fn call_many<M, N, H, S>(
|
||||
&self,
|
||||
endpoint: &Endpoint<M, H>,
|
||||
to: &[Uuid],
|
||||
msg: M,
|
||||
msg: N,
|
||||
strat: RequestStrategy,
|
||||
) -> Vec<(Uuid, Result<S, Error>)>
|
||||
) -> Result<Vec<(Uuid, Result<S, Error>)>, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>>,
|
||||
H: EndpointHandler<M>,
|
||||
N: IntoReq<M>,
|
||||
H: StreamingEndpointHandler<M>,
|
||||
{
|
||||
let msg = Arc::new(msg);
|
||||
let msg = msg.into_req().map_err(netapp::error::Error::from)?;
|
||||
|
||||
let resps = join_all(
|
||||
to.iter()
|
||||
.map(|to| self.call_arc(endpoint, *to, msg.clone(), strat)),
|
||||
.map(|to| self.call(endpoint, *to, msg.clone(), strat)),
|
||||
)
|
||||
.await;
|
||||
to.iter()
|
||||
Ok(to
|
||||
.iter()
|
||||
.cloned()
|
||||
.zip(resps.into_iter())
|
||||
.collect::<Vec<_>>()
|
||||
.collect::<Vec<_>>())
|
||||
}
|
||||
|
||||
pub async fn broadcast<M, H, S>(
|
||||
pub async fn broadcast<M, N, H, S>(
|
||||
&self,
|
||||
endpoint: &Endpoint<M, H>,
|
||||
msg: M,
|
||||
msg: N,
|
||||
strat: RequestStrategy,
|
||||
) -> Vec<(Uuid, Result<S, Error>)>
|
||||
) -> Result<Vec<(Uuid, Result<S, Error>)>, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>>,
|
||||
H: EndpointHandler<M>,
|
||||
N: IntoReq<M>,
|
||||
H: StreamingEndpointHandler<M>,
|
||||
{
|
||||
let to = self
|
||||
.0
|
||||
|
|
@ -223,16 +215,17 @@ impl RpcHelper {
|
|||
|
||||
/// Make a RPC call to multiple servers, returning either a Vec of responses,
|
||||
/// or an error if quorum could not be reached due to too many errors
|
||||
pub async fn try_call_many<M, H, S>(
|
||||
pub async fn try_call_many<M, N, H, S>(
|
||||
&self,
|
||||
endpoint: &Arc<Endpoint<M, H>>,
|
||||
to: &[Uuid],
|
||||
msg: M,
|
||||
msg: N,
|
||||
strategy: RequestStrategy,
|
||||
) -> Result<Vec<S>, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>> + 'static,
|
||||
H: EndpointHandler<M> + 'static,
|
||||
N: IntoReq<M>,
|
||||
H: StreamingEndpointHandler<M> + 'static,
|
||||
S: Send + 'static,
|
||||
{
|
||||
let quorum = strategy.rs_quorum.unwrap_or(to.len());
|
||||
|
|
@ -262,20 +255,21 @@ impl RpcHelper {
|
|||
.await
|
||||
}
|
||||
|
||||
async fn try_call_many_internal<M, H, S>(
|
||||
async fn try_call_many_internal<M, N, H, S>(
|
||||
&self,
|
||||
endpoint: &Arc<Endpoint<M, H>>,
|
||||
to: &[Uuid],
|
||||
msg: M,
|
||||
msg: N,
|
||||
strategy: RequestStrategy,
|
||||
quorum: usize,
|
||||
) -> Result<Vec<S>, Error>
|
||||
where
|
||||
M: Rpc<Response = Result<S, Error>> + 'static,
|
||||
H: EndpointHandler<M> + 'static,
|
||||
N: IntoReq<M>,
|
||||
H: StreamingEndpointHandler<M> + 'static,
|
||||
S: Send + 'static,
|
||||
{
|
||||
let msg = Arc::new(msg);
|
||||
let msg = msg.into_req().map_err(netapp::error::Error::from)?;
|
||||
|
||||
// Build future for each request
|
||||
// They are not started now: they are added below in a FuturesUnordered
|
||||
|
|
@ -285,7 +279,7 @@ impl RpcHelper {
|
|||
let msg = msg.clone();
|
||||
let endpoint2 = endpoint.clone();
|
||||
(to, async move {
|
||||
self2.call_arc(&endpoint2, to, msg, strategy).await
|
||||
self2.call(&endpoint2, to, msg, strategy).await
|
||||
})
|
||||
});
|
||||
|
||||
|
|
@ -299,47 +293,19 @@ impl RpcHelper {
|
|||
// to reach a quorum, priorizing nodes with the lowest latency.
|
||||
// When there are errors, we start new requests to compensate.
|
||||
|
||||
// Retrieve some status variables that we will use to sort requests
|
||||
let peer_list = self.0.fullmesh.get_peer_list();
|
||||
let ring: Arc<Ring> = self.0.ring.borrow().clone();
|
||||
let our_zone = match ring.layout.node_role(&self.0.our_node_id) {
|
||||
Some(pc) => &pc.zone,
|
||||
None => "",
|
||||
};
|
||||
|
||||
// Augment requests with some information used to sort them.
|
||||
// The tuples are as follows:
|
||||
// (is another node?, is another zone?, latency, node ID, request future)
|
||||
// We store all of these tuples in a vec that we can sort.
|
||||
// By sorting this vec, we priorize ourself, then nodes in the same zone,
|
||||
// and within a same zone we priorize nodes with the lowest latency.
|
||||
let mut requests = requests
|
||||
.map(|(to, fut)| {
|
||||
let peer_zone = match ring.layout.node_role(&to) {
|
||||
Some(pc) => &pc.zone,
|
||||
None => "",
|
||||
};
|
||||
let peer_avg_ping = peer_list
|
||||
.iter()
|
||||
.find(|x| x.id.as_ref() == to.as_slice())
|
||||
.and_then(|pi| pi.avg_ping)
|
||||
.unwrap_or_else(|| Duration::from_secs(1));
|
||||
(
|
||||
to != self.0.our_node_id,
|
||||
peer_zone != our_zone,
|
||||
peer_avg_ping,
|
||||
to,
|
||||
fut,
|
||||
)
|
||||
})
|
||||
// Reorder requests to priorize closeness / low latency
|
||||
let request_order = self.request_order(to);
|
||||
let mut ord_requests = vec![(); request_order.len()]
|
||||
.into_iter()
|
||||
.map(|_| None)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// Sort requests by (priorize ourself, priorize same zone, priorize low latency)
|
||||
requests
|
||||
.sort_by_key(|(diffnode, diffzone, ping, _to, _fut)| (*diffnode, *diffzone, *ping));
|
||||
for (to, fut) in requests {
|
||||
let i = request_order.iter().position(|x| *x == to).unwrap();
|
||||
ord_requests[i] = Some((to, fut));
|
||||
}
|
||||
|
||||
// Make an iterator to take requests in their sorted order
|
||||
let mut requests = requests.into_iter();
|
||||
let mut requests = ord_requests.into_iter().map(Option::unwrap);
|
||||
|
||||
// resp_stream will contain all of the requests that are currently in flight.
|
||||
// (for the moment none, they will be added in the loop below)
|
||||
|
|
@ -350,7 +316,7 @@ impl RpcHelper {
|
|||
// If the current set of requests that are running is not enough to possibly
|
||||
// reach quorum, start some new requests.
|
||||
while successes.len() + resp_stream.len() < quorum {
|
||||
if let Some((_, _, _, req_to, fut)) = requests.next() {
|
||||
if let Some((req_to, fut)) = requests.next() {
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let span = tracer.start(format!("RPC to {:?}", req_to));
|
||||
resp_stream.push(tokio::spawn(
|
||||
|
|
@ -420,4 +386,49 @@ impl RpcHelper {
|
|||
Err(Error::Quorum(quorum, successes.len(), to.len(), errors))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn request_order(&self, nodes: &[Uuid]) -> Vec<Uuid> {
|
||||
// Retrieve some status variables that we will use to sort requests
|
||||
let peer_list = self.0.fullmesh.get_peer_list();
|
||||
let ring: Arc<Ring> = self.0.ring.borrow().clone();
|
||||
let our_zone = match ring.layout.node_role(&self.0.our_node_id) {
|
||||
Some(pc) => &pc.zone,
|
||||
None => "",
|
||||
};
|
||||
|
||||
// Augment requests with some information used to sort them.
|
||||
// The tuples are as follows:
|
||||
// (is another node?, is another zone?, latency, node ID, request future)
|
||||
// We store all of these tuples in a vec that we can sort.
|
||||
// By sorting this vec, we priorize ourself, then nodes in the same zone,
|
||||
// and within a same zone we priorize nodes with the lowest latency.
|
||||
let mut nodes = nodes
|
||||
.iter()
|
||||
.map(|to| {
|
||||
let peer_zone = match ring.layout.node_role(to) {
|
||||
Some(pc) => &pc.zone,
|
||||
None => "",
|
||||
};
|
||||
let peer_avg_ping = peer_list
|
||||
.iter()
|
||||
.find(|x| x.id.as_ref() == to.as_slice())
|
||||
.and_then(|pi| pi.avg_ping)
|
||||
.unwrap_or_else(|| Duration::from_secs(1));
|
||||
(
|
||||
*to != self.0.our_node_id,
|
||||
peer_zone != our_zone,
|
||||
peer_avg_ping,
|
||||
*to,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
// Sort requests by (priorize ourself, priorize same zone, priorize low latency)
|
||||
nodes.sort_by_key(|(diffnode, diffzone, ping, _to)| (*diffnode, *diffzone, *ping));
|
||||
|
||||
nodes
|
||||
.into_iter()
|
||||
.map(|(_, _, _, to)| to)
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,8 +16,8 @@ use tokio::sync::watch;
|
|||
use tokio::sync::Mutex;
|
||||
|
||||
use netapp::endpoint::{Endpoint, EndpointHandler};
|
||||
use netapp::message::*;
|
||||
use netapp::peering::fullmesh::FullMeshPeeringStrategy;
|
||||
use netapp::proto::*;
|
||||
use netapp::util::parse_and_resolve_peer_addr;
|
||||
use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
|
||||
|
||||
|
|
@ -37,7 +37,7 @@ use crate::rpc_helper::*;
|
|||
|
||||
const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const PING_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
const SYSTEM_RPC_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
/// Version tag used for version check upon Netapp connection.
|
||||
/// Cluster nodes with different version tags are deemed
|
||||
|
|
@ -538,7 +538,7 @@ impl System {
|
|||
SystemRpc::AdvertiseClusterLayout(layout),
|
||||
RequestStrategy::with_priority(PRIO_HIGH),
|
||||
)
|
||||
.await;
|
||||
.await?;
|
||||
Ok(())
|
||||
});
|
||||
self.background.spawn(self.clone().save_cluster_layout());
|
||||
|
|
@ -553,11 +553,12 @@ impl System {
|
|||
|
||||
self.update_local_status();
|
||||
let local_status: NodeStatus = self.local_status.load().as_ref().clone();
|
||||
self.rpc
|
||||
let _ = self
|
||||
.rpc
|
||||
.broadcast(
|
||||
&self.system_endpoint,
|
||||
SystemRpc::AdvertiseStatus(local_status),
|
||||
RequestStrategy::with_priority(PRIO_HIGH).with_timeout(PING_TIMEOUT),
|
||||
RequestStrategy::with_priority(PRIO_HIGH).with_timeout(SYSTEM_RPC_TIMEOUT),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -681,7 +682,7 @@ impl System {
|
|||
&self.system_endpoint,
|
||||
peer,
|
||||
SystemRpc::PullClusterLayout,
|
||||
RequestStrategy::with_priority(PRIO_HIGH).with_timeout(PING_TIMEOUT),
|
||||
RequestStrategy::with_priority(PRIO_HIGH).with_timeout(SYSTEM_RPC_TIMEOUT),
|
||||
)
|
||||
.await;
|
||||
if let Ok(SystemRpc::AdvertiseClusterLayout(layout)) = resp {
|
||||
|
|
|
|||
|
|
@ -25,7 +25,8 @@ use crate::replication::*;
|
|||
use crate::schema::*;
|
||||
|
||||
const TABLE_GC_BATCH_SIZE: usize = 1024;
|
||||
const TABLE_GC_RPC_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
// Same timeout as NEED_BLOCK_QUERY_TIMEOUT in block manager
|
||||
const TABLE_GC_RPC_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
// GC delay for table entries: 1 day (24 hours)
|
||||
// (the delay before the entry is added in the GC todo list
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ pub trait Entry<P: PartitionKey, S: SortKey>:
|
|||
}
|
||||
|
||||
/// Trait for the schema used in a table
|
||||
pub trait TableSchema: Send + Sync {
|
||||
pub trait TableSchema: Send + Sync + 'static {
|
||||
/// The name of the table in the database
|
||||
const TABLE_NAME: &'static str;
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ use crate::merkle::*;
|
|||
use crate::replication::*;
|
||||
use crate::*;
|
||||
|
||||
const TABLE_SYNC_RPC_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
// Sync RPC can contain a lot of data, so have a 1min timeout
|
||||
const TABLE_SYNC_RPC_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
// Do anti-entropy every 10 minutes
|
||||
const ANTI_ENTROPY_INTERVAL: Duration = Duration::from_secs(10 * 60);
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ use crate::schema::*;
|
|||
use crate::sync::*;
|
||||
use crate::util::*;
|
||||
|
||||
pub const TABLE_RPC_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
pub const TABLE_RPC_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub struct Table<F: TableSchema + 'static, R: TableReplication + 'static> {
|
||||
pub system: Arc<System>,
|
||||
|
|
|
|||
|
|
@ -39,9 +39,7 @@ toml = "0.5"
|
|||
futures = "0.3"
|
||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
|
||||
#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
|
||||
#netapp = { version = "0.4", path = "../../../netapp" }
|
||||
netapp = "0.4"
|
||||
netapp = "0.5"
|
||||
|
||||
http = "0.2"
|
||||
hyper = "0.14"
|
||||
|
|
|
|||
Loading…
Reference in a new issue