Deuxfleurs/garage
48
56
Fork 73
Code Issues 100 Pull requests 13 Releases 30 Wiki Activity

Some improvements to Garage internals #451

Merged
lx merged 14 commits from internals-rework into main 2023-01-03 11:37:32 +00:00
Conversation 0 Commits 14 Files changed 27 +332 -268
18 changed files with 332 additions and 268 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 939a6d67e8 - Show all commits

18
Cargo.lock generated
View file

@ -1048,7 +1048,7 @@ dependencies = [
[[package]] [[package]]
name = "garage" name = "garage"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"assert-json-diff", "assert-json-diff",
"async-trait", "async-trait",
@ -1096,7 +1096,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_api" name = "garage_api"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"base64", "base64",
@ -1141,7 +1141,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_block" name = "garage_block"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-compression", "async-compression",
@ -1167,7 +1167,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_db" name = "garage_db"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"clap 3.1.18", "clap 3.1.18",
"err-derive", "err-derive",
@ -1182,7 +1182,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_model" name = "garage_model"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@ -1210,7 +1210,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_rpc" name = "garage_rpc"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@ -1241,7 +1241,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_table" name = "garage_table"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@ -1264,7 +1264,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_util" name = "garage_util"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap",
"async-trait", "async-trait",
@ -1295,7 +1295,7 @@ dependencies = [
[[package]] [[package]]
name = "garage_web" name = "garage_web"
version = "0.8.0" version = "0.8.1"
dependencies = [ dependencies = [
"err-derive", "err-derive",
"futures", "futures",

120
Cargo.nix
View file

@ -32,7 +32,7 @@ args@{
ignoreLockHash, ignoreLockHash,
}: }:
let let
nixifiedLockHash = "a1d84930f23d3d8abc8abbed59b8ce3c9adf9f25d06bc1f39cbdf5bd90aceead"; nixifiedLockHash = "4639f63ff4c54c01f66ec3d0d362f6905456dd768d6e94df1a7367c763721fd7";
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
lockHashIgnored = if ignoreLockHash lockHashIgnored = if ignoreLockHash
@ -56,15 +56,15 @@ in
{ {
cargo2nixVersion = "0.11.0"; cargo2nixVersion = "0.11.0";
workspace = { workspace = {
garage_db = rustPackages.unknown.garage_db."0.8.0"; garage_db = rustPackages.unknown.garage_db."0.8.1";
garage_util = rustPackages.unknown.garage_util."0.8.0"; garage_util = rustPackages.unknown.garage_util."0.8.1";
garage_rpc = rustPackages.unknown.garage_rpc."0.8.0"; garage_rpc = rustPackages.unknown.garage_rpc."0.8.1";
garage_table = rustPackages.unknown.garage_table."0.8.0"; garage_table = rustPackages.unknown.garage_table."0.8.1";
garage_block = rustPackages.unknown.garage_block."0.8.0"; garage_block = rustPackages.unknown.garage_block."0.8.1";
garage_model = rustPackages.unknown.garage_model."0.8.0"; garage_model = rustPackages.unknown.garage_model."0.8.1";
garage_api = rustPackages.unknown.garage_api."0.8.0"; garage_api = rustPackages.unknown.garage_api."0.8.1";
garage_web = rustPackages.unknown.garage_web."0.8.0"; garage_web = rustPackages.unknown.garage_web."0.8.1";
garage = rustPackages.unknown.garage."0.8.0"; garage = rustPackages.unknown.garage."0.8.1";
k2v-client = rustPackages.unknown.k2v-client."0.0.1"; k2v-client = rustPackages.unknown.k2v-client."0.0.1";
}; };
"registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec {
@ -1494,9 +1494,9 @@ in
}; };
}); });
"unknown".garage."0.8.0" = overridableMkRustCrate (profileName: rec { "unknown".garage."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage"; name = "garage";
version = "0.8.0"; version = "0.8.1";
registry = "unknown"; registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/garage"); src = fetchCrateLocal (workspaceSrc + "/src/garage");
features = builtins.concatLists [ features = builtins.concatLists [
@ -1522,14 +1522,14 @@ in
bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out; bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out; garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out; garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out; garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out; garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out; garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out; garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out; garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
garage_web = (rustPackages."unknown".garage_web."0.8.0" { inherit profileName; }).out; garage_web = (rustPackages."unknown".garage_web."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
@ -1563,9 +1563,9 @@ in
}; };
}); });
"unknown".garage_api."0.8.0" = overridableMkRustCrate (profileName: rec { "unknown".garage_api."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_api"; name = "garage_api";
version = "0.8.0"; version = "0.8.1";
registry = "unknown"; registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/api"); src = fetchCrateLocal (workspaceSrc + "/src/api");
features = builtins.concatLists [ features = builtins.concatLists [
@ -1584,11 +1584,11 @@ in
form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }).out; form_urlencoded = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out; garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out; garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out; garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out; garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out; garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out; hmac = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.12.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
@ -1617,9 +1617,9 @@ in
}; };
}); });
"unknown".garage_block."0.8.0" = overridableMkRustCrate (profileName: rec { "unknown".garage_block."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_block"; name = "garage_block";
version = "0.8.0"; version = "0.8.1";
registry = "unknown"; registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/block"); src = fetchCrateLocal (workspaceSrc + "/src/block");
features = builtins.concatLists [ features = builtins.concatLists [
@ -1632,10 +1632,10 @@ in
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out; bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out; garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out; garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out; garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out; garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
@ -1649,9 +1649,9 @@ in
}; };
}); });
"unknown".garage_db."0.8.0" = overridableMkRustCrate (profileName: rec { "unknown".garage_db."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_db"; name = "garage_db";
version = "0.8.0"; version = "0.8.1";
registry = "unknown"; registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/db"); src = fetchCrateLocal (workspaceSrc + "/src/db");
features = builtins.concatLists [ features = builtins.concatLists [
@ -1681,9 +1681,9 @@ in
}; };
}); });
"unknown".garage_model."0.8.0" = overridableMkRustCrate (profileName: rec { "unknown".garage_model."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_model"; name = "garage_model";
version = "0.8.0"; version = "0.8.1";
registry = "unknown"; registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/model"); src = fetchCrateLocal (workspaceSrc + "/src/model");
features = builtins.concatLists [ features = builtins.concatLists [
@ -1701,11 +1701,11 @@ in
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_block = (rustPackages."unknown".garage_block."0.8.0" { inherit profileName; }).out; garage_block = (rustPackages."unknown".garage_block."0.8.1" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out; garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out; garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out; garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out; garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@ -1719,9 +1719,9 @@ in
}; };
}); });
"unknown".garage_rpc."0.8.0" = overridableMkRustCrate (profileName: rec { "unknown".garage_rpc."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_rpc"; name = "garage_rpc";
version = "0.8.0"; version = "0.8.1";
registry = "unknown"; registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/rpc"); src = fetchCrateLocal (workspaceSrc + "/src/rpc");
features = builtins.concatLists [ features = builtins.concatLists [
@ -1741,7 +1741,7 @@ in
${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive" then "err_derive" else null } = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out; garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out; gethostname = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/k8s-openapi" || rootFeatures' ? "garage_rpc/kubernetes-discovery" then "k8s_openapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.16.0" { inherit profileName; }).out;
@ -1763,9 +1763,9 @@ in
}; };
}); });
"unknown".garage_table."0.8.0" = overridableMkRustCrate (profileName: rec { "unknown".garage_table."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_table"; name = "garage_table";
version = "0.8.0"; version = "0.8.1";
registry = "unknown"; registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/table"); src = fetchCrateLocal (workspaceSrc + "/src/table");
dependencies = { dependencies = {
@ -1774,9 +1774,9 @@ in
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out; bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out; garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
garage_rpc = (rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; }).out; garage_rpc = (rustPackages."unknown".garage_rpc."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out; garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out; hexdump = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@ -1789,9 +1789,9 @@ in
}; };
}); });
"unknown".garage_util."0.8.0" = overridableMkRustCrate (profileName: rec { "unknown".garage_util."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_util"; name = "garage_util";
version = "0.8.0"; version = "0.8.1";
registry = "unknown"; registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/util"); src = fetchCrateLocal (workspaceSrc + "/src/util");
features = builtins.concatLists [ features = builtins.concatLists [
@ -1806,7 +1806,7 @@ in
digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; }).out; digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.3" { inherit profileName; }).out;
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
garage_db = (rustPackages."unknown".garage_db."0.8.0" { inherit profileName; }).out; garage_db = (rustPackages."unknown".garage_db."0.8.1" { inherit profileName; }).out;
git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out; git_version = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }).out;
hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out; hex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
@ -1826,18 +1826,18 @@ in
}; };
}); });
"unknown".garage_web."0.8.0" = overridableMkRustCrate (profileName: rec { "unknown".garage_web."0.8.1" = overridableMkRustCrate (profileName: rec {
name = "garage_web"; name = "garage_web";
version = "0.8.0"; version = "0.8.1";
registry = "unknown"; registry = "unknown";
src = fetchCrateLocal (workspaceSrc + "/src/web"); src = fetchCrateLocal (workspaceSrc + "/src/web");
dependencies = { dependencies = {
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
garage_api = (rustPackages."unknown".garage_api."0.8.0" { inherit profileName; }).out; garage_api = (rustPackages."unknown".garage_api."0.8.1" { inherit profileName; }).out;
garage_model = (rustPackages."unknown".garage_model."0.8.0" { inherit profileName; }).out; garage_model = (rustPackages."unknown".garage_model."0.8.1" { inherit profileName; }).out;
garage_table = (rustPackages."unknown".garage_table."0.8.0" { inherit profileName; }).out; garage_table = (rustPackages."unknown".garage_table."0.8.1" { inherit profileName; }).out;
garage_util = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out; garage_util = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out; hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
@ -2451,7 +2451,7 @@ in
dependencies = { dependencies = {
base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }).out; base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }).out;
${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }).out; ${ if rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli" then "clap" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".clap."3.1.18" { inherit profileName; }).out;
${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.0" { inherit profileName; }).out; ${ if rootFeatures' ? "k2v-client/cli" || rootFeatures' ? "k2v-client/garage_util" then "garage_util" else null } = (rustPackages."unknown".garage_util."0.8.1" { inherit profileName; }).out;
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out; http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.8" { inherit profileName; }).out;
log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out; log = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }).out;
rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out; rusoto_core = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rusoto_core."0.48.0" { inherit profileName; }).out;

59
doc/book/build/python.md
View file

@ -5,16 +5,59 @@ weight = 20
## S3 ## S3
### Using Minio SDK
First install the SDK:
```bash
pip3 install minio
```
Then instantiate a client object using garage root domain, api key and secret:
```python
import minio
client = minio.Minio(
"your.domain.tld",
"GKyourapikey",
"abcd[...]1234",
# Force the region, this is specific to garage
region="region",
)
```
Then use all the standard S3 endpoints as implemented by the Minio SDK:
```
# List buckets
print(client.list_buckets())
# Put an object containing 'content' to /path in bucket named 'bucket':
content = b"content"
client.put_object(
"bucket",
"path",
io.BytesIO(content),
len(content),
)
# Read the object back and check contents
data = client.get_object("bucket", "path").read()
assert data == content
```
For further documentation, see the Minio SDK
[Reference](https://docs.min.io/docs/python-client-api-reference.html)
### Using Amazon boto3
*Coming soon* *Coming soon*
Some refs: See the official documentation:
- Minio SDK - [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
- [Reference](https://docs.min.io/docs/python-client-api-reference.html) - [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
- [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
- Amazon boto3
- [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
- [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
- [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
## K2V ## K2V

11
doc/book/connect/apps/index.md
View file

@ -8,7 +8,7 @@ In this section, we cover the following web applications:
| Name | Status | Note | | Name | Status | Note |
|------|--------|------| |------|--------|------|
| [Nextcloud](#nextcloud) | ✅ | Both Primary Storage and External Storage are supported | | [Nextcloud](#nextcloud) | ✅ | Both Primary Storage and External Storage are supported |
| [Peertube](#peertube) | ✅ | Must be configured with the website endpoint | | [Peertube](#peertube) | ✅ | Supported with the website endpoint, proxifying private videos unsupported |
| [Mastodon](#mastodon) | ✅ | Natively supported | | [Mastodon](#mastodon) | ✅ | Natively supported |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` | | [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [Pixelfed](#pixelfed) | ❓ | Not yet tested | | [Pixelfed](#pixelfed) | ❓ | Not yet tested |
@ -128,6 +128,10 @@ In other words, Peertube is only responsible of the "control plane" and offload
In return, this system is a bit harder to configure. In return, this system is a bit harder to configure.
We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster. We show how it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandwidth usage on the Garage cluster.
Starting from version 5.0, Peertube also supports improving the security for private videos by not exposing them directly
but relying on a single control point in the Peertube instance. This is based on S3 per-object and prefix ACL, which are not currently supported
in Garage, so this feature is unsupported. While this technically impedes security for private videos, it is not a blocking issue and could be
a reasonable trade-off for some instances.
### Create resources in Garage ### Create resources in Garage
@ -195,6 +199,11 @@ object_storage:
max_upload_part: 2GB max_upload_part: 2GB
proxy:
# You may enable this feature, yet it will not provide any security benefit, so
# you should rather benefit from Garage public endpoint for all videos
proxify_private_files: false
streaming_playlists: streaming_playlists:
bucket_name: 'peertube-playlist' bucket_name: 'peertube-playlist'

12
src/api/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_api" name = "garage_api"
version = "0.8.0" version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -14,11 +14,11 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_model = { version = "0.8.0", path = "../model" } garage_model = { version = "0.8.1", path = "../model" }
garage_table = { version = "0.8.0", path = "../table" } garage_table = { version = "0.8.1", path = "../table" }
garage_block = { version = "0.8.0", path = "../block" } garage_block = { version = "0.8.1", path = "../block" }
garage_util = { version = "0.8.0", path = "../util" } garage_util = { version = "0.8.1", path = "../util" }
garage_rpc = { version = "0.8.0", path = "../rpc" } garage_rpc = { version = "0.8.1", path = "../rpc" }
async-trait = "0.1.7" async-trait = "0.1.7"
base64 = "0.13" base64 = "0.13"

15
src/api/admin/router.rs
View file

@ -143,10 +143,13 @@ impl Endpoint {
} }
generateQueryParameters! { generateQueryParameters! {
"format" => format, keywords: [],
"id" => id, fields: [
"search" => search, "format" => format,
"globalAlias" => global_alias, "id" => id,
"alias" => alias, "search" => search,
"accessKeyId" => access_key_id "globalAlias" => global_alias,
"alias" => alias,
"accessKeyId" => access_key_id
]
} }

43
src/api/k2v/router.rs
View file

@ -96,7 +96,7 @@ impl Endpoint {
fn from_get(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_get(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None), (query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [ key: [
EMPTY if causality_token => PollItem (query::sort_key, query::causality_token, opt_parse::timeout), EMPTY if causality_token => PollItem (query::sort_key, query::causality_token, opt_parse::timeout),
EMPTY => ReadItem (query::sort_key), EMPTY => ReadItem (query::sort_key),
@ -111,7 +111,7 @@ impl Endpoint {
fn from_search(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_search(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None), (query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [ key: [
], ],
no_key: [ no_key: [
@ -125,7 +125,7 @@ impl Endpoint {
fn from_head(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_head(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None), (query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [ key: [
EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id), EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
], ],
@ -140,7 +140,7 @@ impl Endpoint {
fn from_post(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_post(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None), (query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [ key: [
], ],
no_key: [ no_key: [
@ -155,7 +155,7 @@ impl Endpoint {
fn from_put(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_put(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None), (query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [ key: [
EMPTY => InsertItem (query::sort_key), EMPTY => InsertItem (query::sort_key),
@ -169,7 +169,7 @@ impl Endpoint {
fn from_delete(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_delete(partition_key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), partition_key, query, None), (query.keyword.take().unwrap_or_default(), partition_key, query, None),
key: [ key: [
EMPTY => DeleteItem (query::sort_key), EMPTY => DeleteItem (query::sort_key),
], ],
@ -232,21 +232,18 @@ impl Endpoint {
// parameter name => struct field // parameter name => struct field
generateQueryParameters! { generateQueryParameters! {
"prefix" => prefix, keywords: [
"start" => start, "delete" => DELETE,
"causality_token" => causality_token, "search" => SEARCH
"end" => end, ],
"limit" => limit, fields: [
"reverse" => reverse, "prefix" => prefix,
"sort_key" => sort_key, "start" => start,
"timeout" => timeout "causality_token" => causality_token,
} "end" => end,
"limit" => limit,
mod keywords { "reverse" => reverse,
//! This module contain all query parameters with no associated value "sort_key" => sort_key,
//! used to differentiate endpoints. "timeout" => timeout
pub const EMPTY: &str = ""; ]
pub const DELETE: &str = "delete";
pub const SEARCH: &str = "search";
} }

131
src/api/router_macros.rs
View file

@ -4,10 +4,9 @@ macro_rules! router_match {
(@match $enum:expr , [ $($endpoint:ident,)* ]) => {{ (@match $enum:expr , [ $($endpoint:ident,)* ]) => {{
// usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] } // usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] }
// returns true if the variant was one of the listed variants, false otherwise. // returns true if the variant was one of the listed variants, false otherwise.
use Endpoint::*;
match $enum { match $enum {
$( $(
$endpoint { .. } => true, Endpoint::$endpoint { .. } => true,
)* )*
_ => false _ => false
} }
@ -15,37 +14,35 @@ macro_rules! router_match {
(@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{ (@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{
// usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] } // usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] }
// returns Some(field_value), or None if the variant was not one of the listed variants. // returns Some(field_value), or None if the variant was not one of the listed variants.
use Endpoint::*;
match $enum { match $enum {
$( $(
$endpoint {$param, ..} => Some($param), Endpoint::$endpoint {$param, ..} => Some($param),
)* )*
_ => None _ => None
} }
}}; }};
(@gen_path_parser ($method:expr, $reqpath:expr, $query:expr) (@gen_path_parser ($method:expr, $reqpath:expr, $query:expr)
[ [
$($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)* $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)*
]) => {{ ]) => {{
{ {
use Endpoint::*; match ($method, $reqpath) {
match ($method, $reqpath) { $(
$( (&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api {
(&Method::$meth, $path) if true $(&& $query.$required.is_some())? => $api { $($(
$($( $param: router_match!(@@parse_param $query, $conv, $param),
$param: router_match!(@@parse_param $query, $conv, $param), )*)?
)*)? },
}, )*
)* (m, p) => {
(m, p) => { return Err(Error::bad_request(format!(
return Err(Error::bad_request(format!( "Unknown API endpoint: {} {}",
"Unknown API endpoint: {} {}", m, p
m, p )))
))) }
} }
} }
} }};
}};
(@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr), (@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr),
key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*], key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*],
no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{ no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{
@ -60,11 +57,9 @@ macro_rules! router_match {
// ] // ]
// } // }
// See in from_{method} for more detailed usage. // See in from_{method} for more detailed usage.
use Endpoint::*;
use keywords::*;
match ($keyword, !$key.is_empty()){ match ($keyword, !$key.is_empty()){
$( $(
($kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok($api_k { (Keyword::$kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok(Endpoint::$api_k {
$key, $key,
$($( $($(
$param_k: router_match!(@@parse_param $query, $conv_k, $param_k), $param_k: router_match!(@@parse_param $query, $conv_k, $param_k),
@ -72,7 +67,7 @@ macro_rules! router_match {
}), }),
)* )*
$( $(
($kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok($api_nk { (Keyword::$kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok(Endpoint::$api_nk {
$($( $($(
$param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk), $param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk),
)*)? )*)?
@ -84,7 +79,7 @@ macro_rules! router_match {
(@@parse_param $query:expr, query_opt, $param:ident) => {{ (@@parse_param $query:expr, query_opt, $param:ident) => {{
// extract optional query parameter // extract optional query parameter
$query.$param.take().map(|param| param.into_owned()) $query.$param.take().map(|param| param.into_owned())
}}; }};
(@@parse_param $query:expr, query, $param:ident) => {{ (@@parse_param $query:expr, query, $param:ident) => {{
// extract mendatory query parameter // extract mendatory query parameter
@ -93,7 +88,7 @@ macro_rules! router_match {
(@@parse_param $query:expr, opt_parse, $param:ident) => {{ (@@parse_param $query:expr, opt_parse, $param:ident) => {{
// extract and parse optional query parameter // extract and parse optional query parameter
// missing parameter is file, however parse error is reported as an error // missing parameter is file, however parse error is reported as an error
$query.$param $query.$param
.take() .take()
.map(|param| param.parse()) .map(|param| param.parse())
.transpose() .transpose()
@ -144,14 +139,39 @@ macro_rules! router_match {
/// This macro is used to generate part of the code in this module. It must be called only one, and /// This macro is used to generate part of the code in this module. It must be called only one, and
/// is useless outside of this module. /// is useless outside of this module.
macro_rules! generateQueryParameters { macro_rules! generateQueryParameters {
( $($rest:expr => $name:ident),* ) => { (
keywords: [ $($kw_param:expr => $kw_name: ident),* ],
fields: [ $($f_param:expr => $f_name:ident),* ]
) => {
#[derive(Debug)]
#[allow(non_camel_case_types)]
enum Keyword {
EMPTY,
$( $kw_name, )*
}
impl std::fmt::Display for Keyword {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
Keyword::EMPTY => write!(f, "``"),
$( Keyword::$kw_name => write!(f, "`{}`", $kw_param), )*
}
}
}
impl Default for Keyword {
fn default() -> Self {
Keyword::EMPTY
}
}
/// Struct containing all query parameters used in endpoints. Think of it as an HashMap, /// Struct containing all query parameters used in endpoints. Think of it as an HashMap,
/// but with keys statically known. /// but with keys statically known.
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct QueryParameters<'a> { struct QueryParameters<'a> {
keyword: Option<Cow<'a, str>>, keyword: Option<Keyword>,
$( $(
$name: Option<Cow<'a, str>>, $f_name: Option<Cow<'a, str>>,
)* )*
} }
@ -160,34 +180,29 @@ macro_rules! generateQueryParameters {
fn from_query(query: &'a str) -> Result<Self, Error> { fn from_query(query: &'a str) -> Result<Self, Error> {
let mut res: Self = Default::default(); let mut res: Self = Default::default();
for (k, v) in url::form_urlencoded::parse(query.as_bytes()) { for (k, v) in url::form_urlencoded::parse(query.as_bytes()) {
let repeated = match k.as_ref() { match k.as_ref() {
$( $(
$rest => if !v.is_empty() { $kw_param => if let Some(prev_kw) = res.keyword.replace(Keyword::$kw_name) {
res.$name.replace(v).is_some() return Err(Error::bad_request(format!(
} else { "Multiple keywords: '{}' and '{}'", prev_kw, $kw_param
false )));
},
)*
$(
$f_param => if !v.is_empty() {
if res.$f_name.replace(v).is_some() {
return Err(Error::bad_request(format!(
"Query parameter repeated: '{}'", k
)));
}
}, },
)* )*
_ => { _ => {
if k.starts_with("response-") || k.starts_with("X-Amz-") { if !(k.starts_with("response-") || k.starts_with("X-Amz-")) {
false
} else if v.as_ref().is_empty() {
if res.keyword.replace(k).is_some() {
return Err(Error::bad_request("Multiple keywords"));
}
continue;
} else {
debug!("Received an unknown query parameter: '{}'", k); debug!("Received an unknown query parameter: '{}'", k);
false
} }
} }
}; };
if repeated {
return Err(Error::bad_request(format!(
"Query parameter repeated: '{}'",
k
)));
}
} }
Ok(res) Ok(res)
} }
@ -198,8 +213,8 @@ macro_rules! generateQueryParameters {
if self.keyword.is_some() { if self.keyword.is_some() {
Some("Keyword not used") Some("Keyword not used")
} $( } $(
else if self.$name.is_some() { else if self.$f_name.is_some() {
Some(concat!("'", $rest, "'")) Some(concat!("'", $f_param, "'"))
} }
)* else { )* else {
None None

121
src/api/s3/router.rs
View file

@ -355,7 +355,7 @@ impl Endpoint {
fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_get(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None), (query.keyword.take().unwrap_or_default(), key, query, None),
key: [ key: [
EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker), EMPTY if upload_id => ListParts (query::upload_id, opt_parse::max_parts, opt_parse::part_number_marker),
EMPTY => GetObject (query_opt::version_id, opt_parse::part_number), EMPTY => GetObject (query_opt::version_id, opt_parse::part_number),
@ -412,7 +412,7 @@ impl Endpoint {
fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_head(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None), (query.keyword.take().unwrap_or_default(), key, query, None),
key: [ key: [
EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id), EMPTY => HeadObject(opt_parse::part_number, query_opt::version_id),
], ],
@ -426,7 +426,7 @@ impl Endpoint {
fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_post(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None), (query.keyword.take().unwrap_or_default(), key, query, None),
key: [ key: [
EMPTY if upload_id => CompleteMultipartUpload (query::upload_id), EMPTY if upload_id => CompleteMultipartUpload (query::upload_id),
RESTORE => RestoreObject (query_opt::version_id), RESTORE => RestoreObject (query_opt::version_id),
@ -448,7 +448,7 @@ impl Endpoint {
) -> Result<Self, Error> { ) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, headers), (query.keyword.take().unwrap_or_default(), key, query, headers),
key: [ key: [
EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id), EMPTY if part_number header "x-amz-copy-source" => UploadPartCopy (parse::part_number, query::upload_id),
EMPTY header "x-amz-copy-source" => CopyObject, EMPTY header "x-amz-copy-source" => CopyObject,
@ -490,7 +490,7 @@ impl Endpoint {
fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> { fn from_delete(key: String, query: &mut QueryParameters<'_>) -> Result<Self, Error> {
router_match! { router_match! {
@gen_parser @gen_parser
(query.keyword.take().unwrap_or_default().as_ref(), key, query, None), (query.keyword.take().unwrap_or_default(), key, query, None),
key: [ key: [
EMPTY if upload_id => AbortMultipartUpload (query::upload_id), EMPTY if upload_id => AbortMultipartUpload (query::upload_id),
EMPTY => DeleteObject (query_opt::version_id), EMPTY => DeleteObject (query_opt::version_id),
@ -624,63 +624,60 @@ impl Endpoint {
// parameter name => struct field // parameter name => struct field
generateQueryParameters! { generateQueryParameters! {
"continuation-token" => continuation_token, keywords: [
"delimiter" => delimiter, "accelerate" => ACCELERATE,
"encoding-type" => encoding_type, "acl" => ACL,
"fetch-owner" => fetch_owner, "analytics" => ANALYTICS,
"id" => id, "cors" => CORS,
"key-marker" => key_marker, "delete" => DELETE,
"list-type" => list_type, "encryption" => ENCRYPTION,
"marker" => marker, "intelligent-tiering" => INTELLIGENT_TIERING,
"max-keys" => max_keys, "inventory" => INVENTORY,
"max-parts" => max_parts, "legal-hold" => LEGAL_HOLD,
"max-uploads" => max_uploads, "lifecycle" => LIFECYCLE,
"partNumber" => part_number, "location" => LOCATION,
"part-number-marker" => part_number_marker, "logging" => LOGGING,
"prefix" => prefix, "metrics" => METRICS,
"select-type" => select_type, "notification" => NOTIFICATION,
"start-after" => start_after, "object-lock" => OBJECT_LOCK,
"uploadId" => upload_id, "ownershipControls" => OWNERSHIP_CONTROLS,
"upload-id-marker" => upload_id_marker, "policy" => POLICY,
"versionId" => version_id, "policyStatus" => POLICY_STATUS,
"version-id-marker" => version_id_marker "publicAccessBlock" => PUBLIC_ACCESS_BLOCK,
} "replication" => REPLICATION,
"requestPayment" => REQUEST_PAYMENT,
mod keywords { "restore" => RESTORE,
//! This module contain all query parameters with no associated value S3 uses to differentiate "retention" => RETENTION,
//! endpoints. "select" => SELECT,
pub const EMPTY: &str = ""; "tagging" => TAGGING,
"torrent" => TORRENT,
pub const ACCELERATE: &str = "accelerate"; "uploads" => UPLOADS,
pub const ACL: &str = "acl"; "versioning" => VERSIONING,
pub const ANALYTICS: &str = "analytics"; "versions" => VERSIONS,
pub const CORS: &str = "cors"; "website" => WEBSITE
pub const DELETE: &str = "delete"; ],
pub const ENCRYPTION: &str = "encryption"; fields: [
pub const INTELLIGENT_TIERING: &str = "intelligent-tiering"; "continuation-token" => continuation_token,
pub const INVENTORY: &str = "inventory"; "delimiter" => delimiter,
pub const LEGAL_HOLD: &str = "legal-hold"; "encoding-type" => encoding_type,
pub const LIFECYCLE: &str = "lifecycle"; "fetch-owner" => fetch_owner,
pub const LOCATION: &str = "location"; "id" => id,
pub const LOGGING: &str = "logging"; "key-marker" => key_marker,
pub const METRICS: &str = "metrics"; "list-type" => list_type,
pub const NOTIFICATION: &str = "notification"; "marker" => marker,
pub const OBJECT_LOCK: &str = "object-lock"; "max-keys" => max_keys,
pub const OWNERSHIP_CONTROLS: &str = "ownershipControls"; "max-parts" => max_parts,
pub const POLICY: &str = "policy"; "max-uploads" => max_uploads,
pub const POLICY_STATUS: &str = "policyStatus"; "partNumber" => part_number,
pub const PUBLIC_ACCESS_BLOCK: &str = "publicAccessBlock"; "part-number-marker" => part_number_marker,
pub const REPLICATION: &str = "replication"; "prefix" => prefix,
pub const REQUEST_PAYMENT: &str = "requestPayment"; "select-type" => select_type,
pub const RESTORE: &str = "restore"; "start-after" => start_after,
pub const RETENTION: &str = "retention"; "uploadId" => upload_id,
pub const SELECT: &str = "select"; "upload-id-marker" => upload_id_marker,
pub const TAGGING: &str = "tagging"; "versionId" => version_id,
pub const TORRENT: &str = "torrent"; "version-id-marker" => version_id_marker
pub const UPLOADS: &str = "uploads"; ]
pub const VERSIONING: &str = "versioning";
pub const VERSIONS: &str = "versions";
pub const WEBSITE: &str = "website";
} }
#[cfg(test)] #[cfg(test)]

10
src/block/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_block" name = "garage_block"
version = "0.8.0" version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -14,10 +14,10 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_db = { version = "0.8.0", path = "../db" } garage_db = { version = "0.8.1", path = "../db" }
garage_rpc = { version = "0.8.0", path = "../rpc" } garage_rpc = { version = "0.8.1", path = "../rpc" }
garage_util = { version = "0.8.0", path = "../util" } garage_util = { version = "0.8.1", path = "../util" }
garage_table = { version = "0.8.0", path = "../table" } garage_table = { version = "0.8.1", path = "../table" }
opentelemetry = "0.17" opentelemetry = "0.17"

2
src/db/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_db" name = "garage_db"
version = "0.8.0" version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"

18
src/garage/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage" name = "garage"
version = "0.8.0" version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -21,14 +21,14 @@ path = "tests/lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_db = { version = "0.8.0", path = "../db" } garage_db = { version = "0.8.1", path = "../db" }
garage_api = { version = "0.8.0", path = "../api" } garage_api = { version = "0.8.1", path = "../api" }
garage_block = { version = "0.8.0", path = "../block" } garage_block = { version = "0.8.1", path = "../block" }
garage_model = { version = "0.8.0", path = "../model" } garage_model = { version = "0.8.1", path = "../model" }
garage_rpc = { version = "0.8.0", path = "../rpc" } garage_rpc = { version = "0.8.1", path = "../rpc" }
garage_table = { version = "0.8.0", path = "../table" } garage_table = { version = "0.8.1", path = "../table" }
garage_util = { version = "0.8.0", path = "../util" } garage_util = { version = "0.8.1", path = "../util" }
garage_web = { version = "0.8.0", path = "../web" } garage_web = { version = "0.8.1", path = "../web" }
backtrace = "0.3" backtrace = "0.3"
bytes = "1.0" bytes = "1.0"

2
src/k2v-client/Cargo.toml
View file

@ -22,7 +22,7 @@ tokio = "1.17.0"
# cli deps # cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] } clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
garage_util = { version = "0.8.0", path = "../util", optional = true } garage_util = { version = "0.8.1", path = "../util", optional = true }
[features] [features]

12
src/model/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_model" name = "garage_model"
version = "0.8.0" version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -14,11 +14,11 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_db = { version = "0.8.0", default-features = false, path = "../db" } garage_db = { version = "0.8.1", default-features = false, path = "../db" }
garage_rpc = { version = "0.8.0", path = "../rpc" } garage_rpc = { version = "0.8.1", path = "../rpc" }
garage_table = { version = "0.8.0", path = "../table" } garage_table = { version = "0.8.1", path = "../table" }
garage_block = { version = "0.8.0", path = "../block" } garage_block = { version = "0.8.1", path = "../block" }
garage_util = { version = "0.8.0", path = "../util" } garage_util = { version = "0.8.1", path = "../util" }
async-trait = "0.1.7" async-trait = "0.1.7"
arc-swap = "1.0" arc-swap = "1.0"

4
src/rpc/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_rpc" name = "garage_rpc"
version = "0.8.0" version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -14,7 +14,7 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_util = { version = "0.8.0", path = "../util" } garage_util = { version = "0.8.1", path = "../util" }
arc-swap = "1.0" arc-swap = "1.0"
bytes = "1.0" bytes = "1.0"

8
src/table/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_table" name = "garage_table"
version = "0.8.0" version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -14,9 +14,9 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_db = { version = "0.8.0", path = "../db" } garage_db = { version = "0.8.1", path = "../db" }
garage_rpc = { version = "0.8.0", path = "../rpc" } garage_rpc = { version = "0.8.1", path = "../rpc" }
garage_util = { version = "0.8.0", path = "../util" } garage_util = { version = "0.8.1", path = "../util" }
opentelemetry = "0.17" opentelemetry = "0.17"

4
src/util/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_util" name = "garage_util"
version = "0.8.0" version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -14,7 +14,7 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_db = { version = "0.8.0", path = "../db" } garage_db = { version = "0.8.1", path = "../db" }
arc-swap = "1.0" arc-swap = "1.0"
async-trait = "0.1" async-trait = "0.1"

10
src/web/Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_web" name = "garage_web"
version = "0.8.0" version = "0.8.1"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"] authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -14,10 +14,10 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_api = { version = "0.8.0", path = "../api" } garage_api = { version = "0.8.1", path = "../api" }
garage_model = { version = "0.8.0", path = "../model" } garage_model = { version = "0.8.1", path = "../model" }
garage_util = { version = "0.8.0", path = "../util" } garage_util = { version = "0.8.1", path = "../util" }
garage_table = { version = "0.8.0", path = "../table" } garage_table = { version = "0.8.1", path = "../table" }
err-derive = "0.3" err-derive = "0.3"
tracing = "0.1.30" tracing = "0.1.30"