forked from Deuxfleurs/garage
Compare commits
34 commits
bf5868a71d
...
4787685912
Author | SHA1 | Date | |
---|---|---|---|
4787685912 | |||
17a0ba9f7c | |||
462655188c | |||
a53e6271bb | |||
7fafd14a25 | |||
555a54ec40 | |||
fc8f795bba | |||
a7af0c8af9 | |||
bcc9772470 | |||
c4e4cc1156 | |||
05547f2ba6 | |||
39ac295eb7 | |||
cf23aee183 | |||
74ea449f4b | |||
eabb37b53f | |||
e7824faa17 | |||
|
8dfc909759 | ||
485109ea60 | |||
ebe8a41f2d | |||
dc50fa3b34 | |||
a976c9190c | |||
72a0f90070 | |||
d814deb806 | |||
6a09f16da7 | |||
23207d18a0 | |||
3024405a65 | |||
5f0928f89c | |||
0a01b34e81 | |||
66f2daa025 | |||
|
26b3295aaa | ||
0d279918b7 | |||
e03d9062f7 | |||
8d3bbf5703 | |||
5b18fd8201 |
50 changed files with 5660 additions and 723 deletions
131
Cargo.lock
generated
131
Cargo.lock
generated
|
@ -2,6 +2,21 @@
|
||||||
# It is not intended for manual editing.
|
# It is not intended for manual editing.
|
||||||
version = 3
|
version = 3
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "addr2line"
|
||||||
|
version = "0.17.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
|
||||||
|
dependencies = [
|
||||||
|
"gimli",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "adler"
|
||||||
|
version = "1.0.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ahash"
|
name = "ahash"
|
||||||
version = "0.7.6"
|
version = "0.7.6"
|
||||||
|
@ -389,6 +404,21 @@ dependencies = [
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "backtrace"
|
||||||
|
version = "0.3.66"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"
|
||||||
|
dependencies = [
|
||||||
|
"addr2line",
|
||||||
|
"cc",
|
||||||
|
"cfg-if 1.0.0",
|
||||||
|
"libc",
|
||||||
|
"miniz_oxide",
|
||||||
|
"object",
|
||||||
|
"rustc-demangle",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "base64"
|
name = "base64"
|
||||||
version = "0.13.0"
|
version = "0.13.0"
|
||||||
|
@ -1023,6 +1053,7 @@ dependencies = [
|
||||||
"assert-json-diff",
|
"assert-json-diff",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"aws-sdk-s3",
|
"aws-sdk-s3",
|
||||||
|
"backtrace",
|
||||||
"base64",
|
"base64",
|
||||||
"bytes",
|
"bytes",
|
||||||
"bytesize",
|
"bytesize",
|
||||||
|
@ -1046,6 +1077,7 @@ dependencies = [
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"opentelemetry-otlp",
|
"opentelemetry-otlp",
|
||||||
"opentelemetry-prometheus",
|
"opentelemetry-prometheus",
|
||||||
|
"parse_duration",
|
||||||
"prometheus",
|
"prometheus",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"rmp-serde",
|
"rmp-serde",
|
||||||
|
@ -1308,6 +1340,12 @@ dependencies = [
|
||||||
"wasi 0.10.0+wasi-snapshot-preview1",
|
"wasi 0.10.0+wasi-snapshot-preview1",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "gimli"
|
||||||
|
version = "0.26.2"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "git-version"
|
name = "git-version"
|
||||||
version = "0.3.5"
|
version = "0.3.5"
|
||||||
|
@ -2010,6 +2048,15 @@ version = "0.2.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "miniz_oxide"
|
||||||
|
version = "0.5.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"
|
||||||
|
dependencies = [
|
||||||
|
"adler",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mio"
|
name = "mio"
|
||||||
version = "0.8.2"
|
version = "0.8.2"
|
||||||
|
@ -2112,6 +2159,41 @@ dependencies = [
|
||||||
"winapi",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num"
|
||||||
|
version = "0.2.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"
|
||||||
|
dependencies = [
|
||||||
|
"num-bigint",
|
||||||
|
"num-complex",
|
||||||
|
"num-integer",
|
||||||
|
"num-iter",
|
||||||
|
"num-rational",
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-bigint"
|
||||||
|
version = "0.2.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg 1.1.0",
|
||||||
|
"num-integer",
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-complex"
|
||||||
|
version = "0.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg 1.1.0",
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-integer"
|
name = "num-integer"
|
||||||
version = "0.1.44"
|
version = "0.1.44"
|
||||||
|
@ -2122,6 +2204,29 @@ dependencies = [
|
||||||
"num-traits",
|
"num-traits",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-iter"
|
||||||
|
version = "0.1.43"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg 1.1.0",
|
||||||
|
"num-integer",
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "num-rational"
|
||||||
|
version = "0.2.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"
|
||||||
|
dependencies = [
|
||||||
|
"autocfg 1.1.0",
|
||||||
|
"num-bigint",
|
||||||
|
"num-integer",
|
||||||
|
"num-traits",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-traits"
|
name = "num-traits"
|
||||||
version = "0.2.14"
|
version = "0.2.14"
|
||||||
|
@ -2150,6 +2255,15 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "object"
|
||||||
|
version = "0.29.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"
|
||||||
|
dependencies = [
|
||||||
|
"memchr",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "once_cell"
|
name = "once_cell"
|
||||||
version = "1.10.0"
|
version = "1.10.0"
|
||||||
|
@ -2303,6 +2417,17 @@ dependencies = [
|
||||||
"windows-sys",
|
"windows-sys",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "parse_duration"
|
||||||
|
version = "2.1.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d"
|
||||||
|
dependencies = [
|
||||||
|
"lazy_static",
|
||||||
|
"num",
|
||||||
|
"regex",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pem"
|
name = "pem"
|
||||||
version = "1.1.0"
|
version = "1.1.0"
|
||||||
|
@ -2954,6 +3079,12 @@ dependencies = [
|
||||||
"smallvec",
|
"smallvec",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rustc-demangle"
|
||||||
|
version = "0.1.21"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc_version"
|
name = "rustc_version"
|
||||||
version = "0.4.0"
|
version = "0.4.0"
|
||||||
|
|
192
Cargo.nix
192
Cargo.nix
|
@ -32,7 +32,7 @@ args@{
|
||||||
ignoreLockHash,
|
ignoreLockHash,
|
||||||
}:
|
}:
|
||||||
let
|
let
|
||||||
nixifiedLockHash = "45b04ec226341a714068633cf7a3a6b421aa70ac3f78c72931d43ab8301b3c7b";
|
nixifiedLockHash = "90b29705f5037c7e1b33f4650841f1266f2e86fa03d5d0c87ad80be7619985c7";
|
||||||
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
|
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
|
||||||
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
|
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
|
||||||
lockHashIgnored = if ignoreLockHash
|
lockHashIgnored = if ignoreLockHash
|
||||||
|
@ -67,6 +67,23 @@ in
|
||||||
garage = rustPackages.unknown.garage."0.8.0";
|
garage = rustPackages.unknown.garage."0.8.0";
|
||||||
k2v-client = rustPackages.unknown.k2v-client."0.0.1";
|
k2v-client = rustPackages.unknown.k2v-client."0.0.1";
|
||||||
};
|
};
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "addr2line";
|
||||||
|
version = "0.17.0";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"; };
|
||||||
|
dependencies = {
|
||||||
|
gimli = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gimli."0.26.2" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".adler."1.0.2" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "adler";
|
||||||
|
version = "1.0.2";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"; };
|
||||||
|
});
|
||||||
|
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".ahash."0.7.6" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".ahash."0.7.6" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "ahash";
|
name = "ahash";
|
||||||
version = "0.7.6";
|
version = "0.7.6";
|
||||||
|
@ -555,6 +572,28 @@ in
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".backtrace."0.3.66" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "backtrace";
|
||||||
|
version = "0.3.66";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"; };
|
||||||
|
features = builtins.concatLists [
|
||||||
|
[ "default" ]
|
||||||
|
[ "std" ]
|
||||||
|
];
|
||||||
|
dependencies = {
|
||||||
|
addr2line = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" { inherit profileName; }).out;
|
||||||
|
cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out;
|
||||||
|
libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out;
|
||||||
|
miniz_oxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".miniz_oxide."0.5.4" { inherit profileName; }).out;
|
||||||
|
object = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".object."0.29.0" { inherit profileName; }).out;
|
||||||
|
rustc_demangle = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc-demangle."0.1.21" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
buildDependencies = {
|
||||||
|
cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.73" { profileName = "__noProfile"; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "base64";
|
name = "base64";
|
||||||
version = "0.13.0";
|
version = "0.13.0";
|
||||||
|
@ -1478,6 +1517,7 @@ in
|
||||||
];
|
];
|
||||||
dependencies = {
|
dependencies = {
|
||||||
async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }).out;
|
async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }).out;
|
||||||
|
backtrace = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".backtrace."0.3.66" { inherit profileName; }).out;
|
||||||
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
|
bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out;
|
||||||
bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
|
bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out;
|
||||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out;
|
||||||
|
@ -1496,6 +1536,7 @@ in
|
||||||
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
|
||||||
${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "opentelemetry_otlp" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }).out;
|
${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "opentelemetry_otlp" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }).out;
|
||||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" then "opentelemetry_prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }).out;
|
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" then "opentelemetry_prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }).out;
|
||||||
|
parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out;
|
||||||
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }).out;
|
${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }).out;
|
||||||
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
|
||||||
rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
|
rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out;
|
||||||
|
@ -1844,6 +1885,17 @@ in
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".gimli."0.26.2" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "gimli";
|
||||||
|
version = "0.26.2";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"; };
|
||||||
|
features = builtins.concatLists [
|
||||||
|
[ "read" ]
|
||||||
|
[ "read-core" ]
|
||||||
|
];
|
||||||
|
});
|
||||||
|
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "git-version";
|
name = "git-version";
|
||||||
version = "0.3.5";
|
version = "0.3.5";
|
||||||
|
@ -2818,6 +2870,16 @@ in
|
||||||
];
|
];
|
||||||
});
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".miniz_oxide."0.5.4" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "miniz_oxide";
|
||||||
|
version = "0.5.4";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"; };
|
||||||
|
dependencies = {
|
||||||
|
adler = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".adler."1.0.2" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".mio."0.8.2" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".mio."0.8.2" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "mio";
|
name = "mio";
|
||||||
version = "0.8.2";
|
version = "0.8.2";
|
||||||
|
@ -2954,6 +3016,59 @@ in
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".num."0.2.1" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "num";
|
||||||
|
version = "0.2.1";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"; };
|
||||||
|
features = builtins.concatLists [
|
||||||
|
[ "default" ]
|
||||||
|
[ "num-bigint" ]
|
||||||
|
[ "std" ]
|
||||||
|
];
|
||||||
|
dependencies = {
|
||||||
|
num_bigint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" { inherit profileName; }).out;
|
||||||
|
num_complex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-complex."0.2.4" { inherit profileName; }).out;
|
||||||
|
num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }).out;
|
||||||
|
num_iter = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-iter."0.1.43" { inherit profileName; }).out;
|
||||||
|
num_rational = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-rational."0.2.4" { inherit profileName; }).out;
|
||||||
|
num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "num-bigint";
|
||||||
|
version = "0.2.6";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"; };
|
||||||
|
features = builtins.concatLists [
|
||||||
|
[ "std" ]
|
||||||
|
];
|
||||||
|
dependencies = {
|
||||||
|
num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }).out;
|
||||||
|
num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
buildDependencies = {
|
||||||
|
autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".num-complex."0.2.4" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "num-complex";
|
||||||
|
version = "0.2.4";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"; };
|
||||||
|
features = builtins.concatLists [
|
||||||
|
[ "std" ]
|
||||||
|
];
|
||||||
|
dependencies = {
|
||||||
|
num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
buildDependencies = {
|
||||||
|
autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "num-integer";
|
name = "num-integer";
|
||||||
version = "0.1.44";
|
version = "0.1.44";
|
||||||
|
@ -2971,6 +3086,43 @@ in
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".num-iter."0.1.43" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "num-iter";
|
||||||
|
version = "0.1.43";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"; };
|
||||||
|
features = builtins.concatLists [
|
||||||
|
[ "std" ]
|
||||||
|
];
|
||||||
|
dependencies = {
|
||||||
|
num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }).out;
|
||||||
|
num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
buildDependencies = {
|
||||||
|
autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".num-rational."0.2.4" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "num-rational";
|
||||||
|
version = "0.2.4";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"; };
|
||||||
|
features = builtins.concatLists [
|
||||||
|
[ "bigint" ]
|
||||||
|
[ "num-bigint" ]
|
||||||
|
[ "std" ]
|
||||||
|
];
|
||||||
|
dependencies = {
|
||||||
|
num_bigint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" { inherit profileName; }).out;
|
||||||
|
num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }).out;
|
||||||
|
num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
buildDependencies = {
|
||||||
|
autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "num-traits";
|
name = "num-traits";
|
||||||
version = "0.2.14";
|
version = "0.2.14";
|
||||||
|
@ -3006,6 +3158,25 @@ in
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".object."0.29.0" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "object";
|
||||||
|
version = "0.29.0";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"; };
|
||||||
|
features = builtins.concatLists [
|
||||||
|
[ "archive" ]
|
||||||
|
[ "coff" ]
|
||||||
|
[ "elf" ]
|
||||||
|
[ "macho" ]
|
||||||
|
[ "pe" ]
|
||||||
|
[ "read_core" ]
|
||||||
|
[ "unaligned" ]
|
||||||
|
];
|
||||||
|
dependencies = {
|
||||||
|
memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "once_cell";
|
name = "once_cell";
|
||||||
version = "1.10.0";
|
version = "1.10.0";
|
||||||
|
@ -3220,6 +3391,18 @@ in
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "parse_duration";
|
||||||
|
version = "2.1.1";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d"; };
|
||||||
|
dependencies = {
|
||||||
|
lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out;
|
||||||
|
num = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num."0.2.1" { inherit profileName; }).out;
|
||||||
|
regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.5" { inherit profileName; }).out;
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".pem."1.1.0" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".pem."1.1.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "pem";
|
name = "pem";
|
||||||
version = "1.1.0";
|
version = "1.1.0";
|
||||||
|
@ -4078,6 +4261,13 @@ in
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
|
"registry+https://github.com/rust-lang/crates.io-index".rustc-demangle."0.1.21" = overridableMkRustCrate (profileName: rec {
|
||||||
|
name = "rustc-demangle";
|
||||||
|
version = "0.1.21";
|
||||||
|
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||||
|
src = fetchCratesIo { inherit name version; sha256 = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"; };
|
||||||
|
});
|
||||||
|
|
||||||
"registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" = overridableMkRustCrate (profileName: rec {
|
"registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" = overridableMkRustCrate (profileName: rec {
|
||||||
name = "rustc_version";
|
name = "rustc_version";
|
||||||
version = "0.4.0";
|
version = "0.4.0";
|
||||||
|
|
17
doc/api/README.md
Normal file
17
doc/api/README.md
Normal file
|
@ -0,0 +1,17 @@
|
||||||
|
# Browse doc
|
||||||
|
|
||||||
|
Run in this directory:
|
||||||
|
|
||||||
|
```
|
||||||
|
python3 -m http.server
|
||||||
|
```
|
||||||
|
|
||||||
|
And open in your browser:
|
||||||
|
- http://localhost:8000/garage-admin-v0.html
|
||||||
|
|
||||||
|
# Validate doc
|
||||||
|
|
||||||
|
```
|
||||||
|
wget https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/6.1.0/openapi-generator-cli-6.1.0.jar -O openapi-generator-cli.jar
|
||||||
|
java -jar openapi-generator-cli.jar validate -i garage-admin-v0.yml
|
||||||
|
```
|
59
doc/api/css/redoc.css
Normal file
59
doc/api/css/redoc.css
Normal file
|
@ -0,0 +1,59 @@
|
||||||
|
/* montserrat-300 - latin */
|
||||||
|
@font-face {
|
||||||
|
font-family: 'Montserrat';
|
||||||
|
font-style: normal;
|
||||||
|
font-weight: 300;
|
||||||
|
src: local(''),
|
||||||
|
url('../fonts/montserrat-v25-latin-300.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
|
||||||
|
url('../fonts/montserrat-v25-latin-300.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
|
||||||
|
}
|
||||||
|
|
||||||
|
/* montserrat-regular - latin */
|
||||||
|
@font-face {
|
||||||
|
font-family: 'Montserrat';
|
||||||
|
font-style: normal;
|
||||||
|
font-weight: 400;
|
||||||
|
src: local(''),
|
||||||
|
url('../fonts/montserrat-v25-latin-regular.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
|
||||||
|
url('../fonts/montserrat-v25-latin-regular.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
|
||||||
|
}
|
||||||
|
|
||||||
|
/* montserrat-700 - latin */
|
||||||
|
@font-face {
|
||||||
|
font-family: 'Montserrat';
|
||||||
|
font-style: normal;
|
||||||
|
font-weight: 700;
|
||||||
|
src: local(''),
|
||||||
|
url('../fonts/montserrat-v25-latin-700.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
|
||||||
|
url('../fonts/montserrat-v25-latin-700.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
|
||||||
|
}
|
||||||
|
/* roboto-300 - latin */
|
||||||
|
@font-face {
|
||||||
|
font-family: 'Roboto';
|
||||||
|
font-style: normal;
|
||||||
|
font-weight: 300;
|
||||||
|
src: local(''),
|
||||||
|
url('../fonts/roboto-v30-latin-300.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
|
||||||
|
url('../fonts/roboto-v30-latin-300.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
|
||||||
|
}
|
||||||
|
|
||||||
|
/* roboto-regular - latin */
|
||||||
|
@font-face {
|
||||||
|
font-family: 'Roboto';
|
||||||
|
font-style: normal;
|
||||||
|
font-weight: 400;
|
||||||
|
src: local(''),
|
||||||
|
url('../fonts/roboto-v30-latin-regular.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
|
||||||
|
url('../fonts/roboto-v30-latin-regular.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
|
||||||
|
}
|
||||||
|
|
||||||
|
/* roboto-700 - latin */
|
||||||
|
@font-face {
|
||||||
|
font-family: 'Roboto';
|
||||||
|
font-style: normal;
|
||||||
|
font-weight: 700;
|
||||||
|
src: local(''),
|
||||||
|
url('../fonts/roboto-v30-latin-700.woff2') format('woff2'), /* Chrome 26+, Opera 23+, Firefox 39+ */
|
||||||
|
url('../fonts/roboto-v30-latin-700.woff') format('woff'); /* Chrome 6+, Firefox 3.6+, IE 9+, Safari 5.1+ */
|
||||||
|
}
|
||||||
|
|
BIN
doc/api/fonts/montserrat-v25-latin-300.woff
Normal file
BIN
doc/api/fonts/montserrat-v25-latin-300.woff
Normal file
Binary file not shown.
BIN
doc/api/fonts/montserrat-v25-latin-300.woff2
Normal file
BIN
doc/api/fonts/montserrat-v25-latin-300.woff2
Normal file
Binary file not shown.
BIN
doc/api/fonts/montserrat-v25-latin-700.woff
Normal file
BIN
doc/api/fonts/montserrat-v25-latin-700.woff
Normal file
Binary file not shown.
BIN
doc/api/fonts/montserrat-v25-latin-700.woff2
Normal file
BIN
doc/api/fonts/montserrat-v25-latin-700.woff2
Normal file
Binary file not shown.
BIN
doc/api/fonts/montserrat-v25-latin-regular.woff
Normal file
BIN
doc/api/fonts/montserrat-v25-latin-regular.woff
Normal file
Binary file not shown.
BIN
doc/api/fonts/montserrat-v25-latin-regular.woff2
Normal file
BIN
doc/api/fonts/montserrat-v25-latin-regular.woff2
Normal file
Binary file not shown.
BIN
doc/api/fonts/roboto-v30-latin-300.woff
Normal file
BIN
doc/api/fonts/roboto-v30-latin-300.woff
Normal file
Binary file not shown.
BIN
doc/api/fonts/roboto-v30-latin-300.woff2
Normal file
BIN
doc/api/fonts/roboto-v30-latin-300.woff2
Normal file
Binary file not shown.
BIN
doc/api/fonts/roboto-v30-latin-700.woff
Normal file
BIN
doc/api/fonts/roboto-v30-latin-700.woff
Normal file
Binary file not shown.
BIN
doc/api/fonts/roboto-v30-latin-700.woff2
Normal file
BIN
doc/api/fonts/roboto-v30-latin-700.woff2
Normal file
Binary file not shown.
BIN
doc/api/fonts/roboto-v30-latin-regular.woff
Normal file
BIN
doc/api/fonts/roboto-v30-latin-regular.woff
Normal file
Binary file not shown.
BIN
doc/api/fonts/roboto-v30-latin-regular.woff2
Normal file
BIN
doc/api/fonts/roboto-v30-latin-regular.woff2
Normal file
Binary file not shown.
24
doc/api/garage-admin-v0.html
Normal file
24
doc/api/garage-admin-v0.html
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Garage Adminstration API v0</title>
|
||||||
|
<!-- needed for adaptive design -->
|
||||||
|
<meta charset="utf-8"/>
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||||
|
<link href="./css/redoc.css" rel="stylesheet">
|
||||||
|
|
||||||
|
<!--
|
||||||
|
Redoc doesn't change outer page styles
|
||||||
|
-->
|
||||||
|
<style>
|
||||||
|
body {
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<redoc spec-url='./garage-admin-v0.yml'></redoc>
|
||||||
|
<script src="./redoc.standalone.js"> </script>
|
||||||
|
</body>
|
||||||
|
</html>
|
1212
doc/api/garage-admin-v0.yml
Normal file
1212
doc/api/garage-admin-v0.yml
Normal file
File diff suppressed because it is too large
Load diff
1806
doc/api/redoc.standalone.js
Normal file
1806
doc/api/redoc.standalone.js
Normal file
File diff suppressed because one or more lines are too long
54
doc/book/build/_index.md
Normal file
54
doc/book/build/_index.md
Normal file
|
@ -0,0 +1,54 @@
|
||||||
|
+++
|
||||||
|
title = "Build your own app"
|
||||||
|
weight = 4
|
||||||
|
sort_by = "weight"
|
||||||
|
template = "documentation.html"
|
||||||
|
+++
|
||||||
|
|
||||||
|
Garage has many API that you can rely on to build complex applications.
|
||||||
|
In this section, we reference the existing SDKs and give some code examples.
|
||||||
|
|
||||||
|
|
||||||
|
## ⚠️ DISCLAIMER
|
||||||
|
|
||||||
|
**K2V AND ADMIN SDK ARE TECHNICAL PREVIEWS**. The following limitations apply:
|
||||||
|
- The API is not complete, some actions are possible only through the `garage` binary
|
||||||
|
- The underlying admin API is not yet stable nor complete, it can breaks at any time
|
||||||
|
- The generator configuration is currently tweaked, the library might break at any time due to a generator change
|
||||||
|
- Because the API and the library are not stable, none of them are published in a package manager (npm, pypi, etc.)
|
||||||
|
- This code has not been extensively tested, some things might not work (please report!)
|
||||||
|
|
||||||
|
To have the best experience possible, please consider:
|
||||||
|
- Make sure that the version of the library you are using is pinned (`go.sum`, `package-lock.json`, `requirements.txt`).
|
||||||
|
- Before upgrading your Garage cluster, make sure that you can find a version of this SDK that works with your targeted version and that you are able to update your own code to work with this new version of the library.
|
||||||
|
- Join our Matrix channel at `#garage:deuxfleurs.fr`, say that you are interested by this SDK, and report any friction.
|
||||||
|
- If stability is critical, mirror this repository on your own infrastructure, regenerate the SDKs and upgrade them at your own pace.
|
||||||
|
|
||||||
|
|
||||||
|
## About the APIs
|
||||||
|
|
||||||
|
Code can interact with Garage through 3 different APIs: S3, K2V, and Admin.
|
||||||
|
Each of them has a specific scope.
|
||||||
|
|
||||||
|
### S3
|
||||||
|
|
||||||
|
De-facto standard, introduced by Amazon, designed to store blobs of data.
|
||||||
|
|
||||||
|
### K2V
|
||||||
|
|
||||||
|
A simple database API similar to RiakKV or DynamoDB.
|
||||||
|
Think a key value store with some additional operations.
|
||||||
|
Its design is inspired by Distributed Hash Tables (DHT).
|
||||||
|
|
||||||
|
More information:
|
||||||
|
- [In the reference manual](@/documentation/reference-manual/k2v.md)
|
||||||
|
|
||||||
|
|
||||||
|
### Administration
|
||||||
|
|
||||||
|
Garage operations can also be automated through a REST API.
|
||||||
|
We are currently building this SDK for [Python](@/documentation/build/python.md#admin-api), [Javascript](@/documentation/build/javascript.md#administration) and [Golang](@/documentation/build/golang.md#administration).
|
||||||
|
|
||||||
|
More information:
|
||||||
|
- [In the reference manual](@/documentation/reference-manual/admin-api.md)
|
||||||
|
- [Full specifiction](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html)
|
69
doc/book/build/golang.md
Normal file
69
doc/book/build/golang.md
Normal file
|
@ -0,0 +1,69 @@
|
||||||
|
+++
|
||||||
|
title = "Golang"
|
||||||
|
weight = 30
|
||||||
|
+++
|
||||||
|
|
||||||
|
## S3
|
||||||
|
|
||||||
|
*Coming soon*
|
||||||
|
|
||||||
|
Some refs:
|
||||||
|
- Minio minio-go-sdk
|
||||||
|
- [Reference](https://docs.min.io/docs/golang-client-api-reference.html)
|
||||||
|
|
||||||
|
- Amazon aws-sdk-go-v2
|
||||||
|
- [Installation](https://aws.github.io/aws-sdk-go-v2/docs/getting-started/)
|
||||||
|
- [Reference](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3)
|
||||||
|
- [Example](https://aws.github.io/aws-sdk-go-v2/docs/code-examples/s3/putobject/)
|
||||||
|
|
||||||
|
## K2V
|
||||||
|
|
||||||
|
*Coming soon*
|
||||||
|
|
||||||
|
## Administration
|
||||||
|
|
||||||
|
Install the SDK with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
go get git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang
|
||||||
|
```
|
||||||
|
|
||||||
|
A short example:
|
||||||
|
|
||||||
|
```go
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
garage "git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
// Set Host and other parameters
|
||||||
|
configuration := garage.NewConfiguration()
|
||||||
|
configuration.Host = "127.0.0.1:3903"
|
||||||
|
|
||||||
|
|
||||||
|
// We can now generate a client
|
||||||
|
client := garage.NewAPIClient(configuration)
|
||||||
|
|
||||||
|
// Authentication is handled through the context pattern
|
||||||
|
ctx := context.WithValue(context.Background(), garage.ContextAccessToken, "s3cr3t")
|
||||||
|
|
||||||
|
// Send a request
|
||||||
|
resp, r, err := client.NodesApi.GetNodes(ctx).Execute()
|
||||||
|
if err != nil {
|
||||||
|
fmt.Fprintf(os.Stderr, "Error when calling `NodesApi.GetNodes``: %v\n", err)
|
||||||
|
fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process the response
|
||||||
|
fmt.Fprintf(os.Stdout, "Target hostname: %v\n", resp.KnownNodes[resp.Node].Hostname)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
See also:
|
||||||
|
- [generated doc](https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang)
|
||||||
|
- [examples](https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-generator/src/branch/main/example/golang)
|
55
doc/book/build/javascript.md
Normal file
55
doc/book/build/javascript.md
Normal file
|
@ -0,0 +1,55 @@
|
||||||
|
+++
|
||||||
|
title = "Javascript"
|
||||||
|
weight = 10
|
||||||
|
+++
|
||||||
|
|
||||||
|
## S3
|
||||||
|
|
||||||
|
*Coming soon*.
|
||||||
|
|
||||||
|
Some refs:
|
||||||
|
- Minio SDK
|
||||||
|
- [Reference](https://docs.min.io/docs/javascript-client-api-reference.html)
|
||||||
|
|
||||||
|
- Amazon aws-sdk-js
|
||||||
|
- [Installation](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/getting-started.html)
|
||||||
|
- [Reference](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html)
|
||||||
|
- [Example](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/s3-example-creating-buckets.html)
|
||||||
|
|
||||||
|
## K2V
|
||||||
|
|
||||||
|
*Coming soon*
|
||||||
|
|
||||||
|
## Administration
|
||||||
|
|
||||||
|
Install the SDK with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
npm install --save git+https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-js.git
|
||||||
|
```
|
||||||
|
|
||||||
|
A short example:
|
||||||
|
|
||||||
|
```javascript
|
||||||
|
const garage = require('garage_administration_api_v0garage_v0_8_0');
|
||||||
|
|
||||||
|
const api = new garage.ApiClient("http://127.0.0.1:3903/v0");
|
||||||
|
api.authentications['bearerAuth'].accessToken = "s3cr3t";
|
||||||
|
|
||||||
|
const [node, layout, key, bucket] = [
|
||||||
|
new garage.NodesApi(api),
|
||||||
|
new garage.LayoutApi(api),
|
||||||
|
new garage.KeyApi(api),
|
||||||
|
new garage.BucketApi(api),
|
||||||
|
];
|
||||||
|
|
||||||
|
node.getNodes().then((data) => {
|
||||||
|
console.log(`nodes: ${Object.values(data.knownNodes).map(n => n.hostname)}`)
|
||||||
|
}, (error) => {
|
||||||
|
console.error(error);
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
See also:
|
||||||
|
- [sdk repository](https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-js)
|
||||||
|
- [examples](https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-generator/src/branch/main/example/javascript)
|
|
@ -1,8 +1,10 @@
|
||||||
+++
|
+++
|
||||||
title = "Your code (PHP, JS, Go...)"
|
title = "Others"
|
||||||
weight = 30
|
weight = 99
|
||||||
+++
|
+++
|
||||||
|
|
||||||
|
## S3
|
||||||
|
|
||||||
If you are developping a new application, you may want to use Garage to store your user's media.
|
If you are developping a new application, you may want to use Garage to store your user's media.
|
||||||
|
|
||||||
The S3 API that Garage uses is a standard REST API, so as long as you can make HTTP requests,
|
The S3 API that Garage uses is a standard REST API, so as long as you can make HTTP requests,
|
||||||
|
@ -13,44 +15,14 @@ Instead, there are some libraries already avalaible.
|
||||||
|
|
||||||
Some of them are maintained by Amazon, some by Minio, others by the community.
|
Some of them are maintained by Amazon, some by Minio, others by the community.
|
||||||
|
|
||||||
## PHP
|
### PHP
|
||||||
|
|
||||||
- Amazon aws-sdk-php
|
- Amazon aws-sdk-php
|
||||||
- [Installation](https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/getting-started_installation.html)
|
- [Installation](https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/getting-started_installation.html)
|
||||||
- [Reference](https://docs.aws.amazon.com/aws-sdk-php/v3/api/api-s3-2006-03-01.html)
|
- [Reference](https://docs.aws.amazon.com/aws-sdk-php/v3/api/api-s3-2006-03-01.html)
|
||||||
- [Example](https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/s3-examples-creating-buckets.html)
|
- [Example](https://docs.aws.amazon.com/sdk-for-php/v3/developer-guide/s3-examples-creating-buckets.html)
|
||||||
|
|
||||||
## Javascript
|
### Java
|
||||||
|
|
||||||
- Minio SDK
|
|
||||||
- [Reference](https://docs.min.io/docs/javascript-client-api-reference.html)
|
|
||||||
|
|
||||||
- Amazon aws-sdk-js
|
|
||||||
- [Installation](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/getting-started.html)
|
|
||||||
- [Reference](https://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/S3.html)
|
|
||||||
- [Example](https://docs.aws.amazon.com/sdk-for-javascript/v3/developer-guide/s3-example-creating-buckets.html)
|
|
||||||
|
|
||||||
## Golang
|
|
||||||
|
|
||||||
- Minio minio-go-sdk
|
|
||||||
- [Reference](https://docs.min.io/docs/golang-client-api-reference.html)
|
|
||||||
|
|
||||||
- Amazon aws-sdk-go-v2
|
|
||||||
- [Installation](https://aws.github.io/aws-sdk-go-v2/docs/getting-started/)
|
|
||||||
- [Reference](https://pkg.go.dev/github.com/aws/aws-sdk-go-v2/service/s3)
|
|
||||||
- [Example](https://aws.github.io/aws-sdk-go-v2/docs/code-examples/s3/putobject/)
|
|
||||||
|
|
||||||
## Python
|
|
||||||
|
|
||||||
- Minio SDK
|
|
||||||
- [Reference](https://docs.min.io/docs/python-client-api-reference.html)
|
|
||||||
|
|
||||||
- Amazon boto3
|
|
||||||
- [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
|
|
||||||
- [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
|
|
||||||
- [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
|
|
||||||
|
|
||||||
## Java
|
|
||||||
|
|
||||||
- Minio SDK
|
- Minio SDK
|
||||||
- [Reference](https://docs.min.io/docs/java-client-api-reference.html)
|
- [Reference](https://docs.min.io/docs/java-client-api-reference.html)
|
||||||
|
@ -60,23 +32,18 @@ Some of them are maintained by Amazon, some by Minio, others by the community.
|
||||||
- [Reference](https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Client.html)
|
- [Reference](https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Client.html)
|
||||||
- [Example](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/examples-s3-objects.html)
|
- [Example](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/examples-s3-objects.html)
|
||||||
|
|
||||||
## Rust
|
### .NET
|
||||||
|
|
||||||
- Amazon aws-rust-sdk
|
|
||||||
- [Github](https://github.com/awslabs/aws-sdk-rust)
|
|
||||||
|
|
||||||
## .NET
|
|
||||||
|
|
||||||
- Minio SDK
|
- Minio SDK
|
||||||
- [Reference](https://docs.min.io/docs/dotnet-client-api-reference.html)
|
- [Reference](https://docs.min.io/docs/dotnet-client-api-reference.html)
|
||||||
|
|
||||||
- Amazon aws-dotnet-sdk
|
- Amazon aws-dotnet-sdk
|
||||||
|
|
||||||
## C++
|
### C++
|
||||||
|
|
||||||
- Amazon aws-cpp-sdk
|
- Amazon aws-cpp-sdk
|
||||||
|
|
||||||
## Haskell
|
### Haskell
|
||||||
|
|
||||||
- Minio SDK
|
- Minio SDK
|
||||||
- [Reference](https://docs.min.io/docs/haskell-client-api-reference.html)
|
- [Reference](https://docs.min.io/docs/haskell-client-api-reference.html)
|
95
doc/book/build/python.md
Normal file
95
doc/book/build/python.md
Normal file
|
@ -0,0 +1,95 @@
|
||||||
|
+++
|
||||||
|
title = "Python"
|
||||||
|
weight = 20
|
||||||
|
+++
|
||||||
|
|
||||||
|
## S3
|
||||||
|
|
||||||
|
*Coming soon*
|
||||||
|
|
||||||
|
Some refs:
|
||||||
|
- Minio SDK
|
||||||
|
- [Reference](https://docs.min.io/docs/python-client-api-reference.html)
|
||||||
|
|
||||||
|
- Amazon boto3
|
||||||
|
- [Installation](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/quickstart.html)
|
||||||
|
- [Reference](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html)
|
||||||
|
- [Example](https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html)
|
||||||
|
|
||||||
|
## K2V
|
||||||
|
|
||||||
|
*Coming soon*
|
||||||
|
|
||||||
|
## Admin API
|
||||||
|
|
||||||
|
You need at least Python 3.6, pip, and setuptools.
|
||||||
|
Because the python package is in a subfolder, the command is a bit more complicated than usual:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
pip3 install --user 'git+https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-python'
|
||||||
|
```
|
||||||
|
|
||||||
|
Now, let imagine you have a fresh Garage instance running on localhost, with the admin API configured on port 3903 with the bearer `s3cr3t`:
|
||||||
|
|
||||||
|
```python
|
||||||
|
import garage_admin_sdk
|
||||||
|
from garage_admin_sdk.apis import *
|
||||||
|
from garage_admin_sdk.models import *
|
||||||
|
|
||||||
|
configuration = garage_admin_sdk.Configuration(
|
||||||
|
host = "http://localhost:3903/v0",
|
||||||
|
access_token = "s3cr3t"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Init APIs
|
||||||
|
api = garage_admin_sdk.ApiClient(configuration)
|
||||||
|
nodes, layout, keys, buckets = NodesApi(api), LayoutApi(api), KeyApi(api), BucketApi(api)
|
||||||
|
|
||||||
|
# Display some info on the node
|
||||||
|
status = nodes.get_nodes()
|
||||||
|
print(f"running garage {status.garage_version}, node_id {status.node}")
|
||||||
|
|
||||||
|
# Change layout of this node
|
||||||
|
current = layout.get_layout()
|
||||||
|
layout.add_layout({
|
||||||
|
status.node: NodeClusterInfo(
|
||||||
|
zone = "dc1",
|
||||||
|
capacity = 1,
|
||||||
|
tags = [ "dev" ],
|
||||||
|
)
|
||||||
|
})
|
||||||
|
layout.apply_layout(LayoutVersion(
|
||||||
|
version = current.version + 1
|
||||||
|
))
|
||||||
|
|
||||||
|
# Create key, allow it to create buckets
|
||||||
|
kinfo = keys.add_key(AddKeyRequest(name="openapi"))
|
||||||
|
|
||||||
|
allow_create = UpdateKeyRequestAllow(create_bucket=True)
|
||||||
|
keys.update_key(kinfo.access_key_id, UpdateKeyRequest(allow=allow_create))
|
||||||
|
|
||||||
|
# Create a bucket, allow key, set quotas
|
||||||
|
binfo = buckets.create_bucket(CreateBucketRequest(global_alias="documentation"))
|
||||||
|
binfo = buckets.allow_bucket_key(AllowBucketKeyRequest(
|
||||||
|
bucket_id=binfo.id,
|
||||||
|
access_key_id=kinfo.access_key_id,
|
||||||
|
permissions=AllowBucketKeyRequestPermissions(read=True, write=True, owner=True),
|
||||||
|
))
|
||||||
|
binfo = buckets.update_bucket(binfo.id, UpdateBucketRequest(
|
||||||
|
quotas=UpdateBucketRequestQuotas(max_size=19029801,max_objects=1500)))
|
||||||
|
|
||||||
|
# Display key
|
||||||
|
print(f"""
|
||||||
|
cluster ready
|
||||||
|
key id is {kinfo.access_key_id}
|
||||||
|
secret key is {kinfo.secret_access_key}
|
||||||
|
bucket {binfo.global_aliases[0]} contains {binfo.objects}/{binfo.quotas.max_objects} objects
|
||||||
|
""")
|
||||||
|
```
|
||||||
|
|
||||||
|
*This example is named `short.py` in the example folder. Other python examples are also available.*
|
||||||
|
|
||||||
|
See also:
|
||||||
|
- [sdk repo](https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-python)
|
||||||
|
- [examples](https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-generator/src/branch/main/example/python)
|
||||||
|
|
47
doc/book/build/rust.md
Normal file
47
doc/book/build/rust.md
Normal file
|
@ -0,0 +1,47 @@
|
||||||
|
+++
|
||||||
|
title = "Rust"
|
||||||
|
weight = 40
|
||||||
|
+++
|
||||||
|
|
||||||
|
## S3
|
||||||
|
|
||||||
|
*Coming soon*
|
||||||
|
|
||||||
|
Some refs:
|
||||||
|
- Amazon aws-rust-sdk
|
||||||
|
- [Github](https://github.com/awslabs/aws-sdk-rust)
|
||||||
|
|
||||||
|
## K2V
|
||||||
|
|
||||||
|
*Coming soon*
|
||||||
|
|
||||||
|
Some refs: https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/main/src/k2v-client
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# all these values can be provided on the cli instead
|
||||||
|
export AWS_ACCESS_KEY_ID=GK123456
|
||||||
|
export AWS_SECRET_ACCESS_KEY=0123..789
|
||||||
|
export AWS_REGION=garage
|
||||||
|
export K2V_ENDPOINT=http://172.30.2.1:3903
|
||||||
|
export K2V_BUCKET=my-bucket
|
||||||
|
|
||||||
|
cargo run --features=cli -- read-range my-partition-key --all
|
||||||
|
|
||||||
|
cargo run --features=cli -- insert my-partition-key my-sort-key --text "my string1"
|
||||||
|
cargo run --features=cli -- insert my-partition-key my-sort-key --text "my string2"
|
||||||
|
cargo run --features=cli -- insert my-partition-key my-sort-key2 --text "my string"
|
||||||
|
|
||||||
|
cargo run --features=cli -- read-range my-partition-key --all
|
||||||
|
|
||||||
|
causality=$(cargo run --features=cli -- read my-partition-key my-sort-key2 -b | head -n1)
|
||||||
|
cargo run --features=cli -- delete my-partition-key my-sort-key2 -c $causality
|
||||||
|
|
||||||
|
causality=$(cargo run --features=cli -- read my-partition-key my-sort-key -b | head -n1)
|
||||||
|
cargo run --features=cli -- insert my-partition-key my-sort-key --text "my string3" -c $causality
|
||||||
|
|
||||||
|
cargo run --features=cli -- read-range my-partition-key --all
|
||||||
|
```
|
||||||
|
|
||||||
|
## Admin API
|
||||||
|
|
||||||
|
*Coming soon*
|
|
@ -1,5 +1,5 @@
|
||||||
+++
|
+++
|
||||||
title = "Integrations"
|
title = "Existing integrations"
|
||||||
weight = 3
|
weight = 3
|
||||||
sort_by = "weight"
|
sort_by = "weight"
|
||||||
template = "documentation.html"
|
template = "documentation.html"
|
||||||
|
@ -14,7 +14,6 @@ In particular, you will find here instructions to connect it with:
|
||||||
- [Applications](@/documentation/connect/apps/index.md)
|
- [Applications](@/documentation/connect/apps/index.md)
|
||||||
- [Website hosting](@/documentation/connect/websites.md)
|
- [Website hosting](@/documentation/connect/websites.md)
|
||||||
- [Software repositories](@/documentation/connect/repositories.md)
|
- [Software repositories](@/documentation/connect/repositories.md)
|
||||||
- [Your own code](@/documentation/connect/code.md)
|
|
||||||
- [FUSE](@/documentation/connect/fs.md)
|
- [FUSE](@/documentation/connect/fs.md)
|
||||||
|
|
||||||
### Generic instructions
|
### Generic instructions
|
||||||
|
|
|
@ -9,7 +9,7 @@ In this section, we cover the following web applications:
|
||||||
|------|--------|------|
|
|------|--------|------|
|
||||||
| [Nextcloud](#nextcloud) | ✅ | Both Primary Storage and External Storage are supported |
|
| [Nextcloud](#nextcloud) | ✅ | Both Primary Storage and External Storage are supported |
|
||||||
| [Peertube](#peertube) | ✅ | Must be configured with the website endpoint |
|
| [Peertube](#peertube) | ✅ | Must be configured with the website endpoint |
|
||||||
| [Mastodon](#mastodon) | ❓ | Not yet tested |
|
| [Mastodon](#mastodon) | ✅ | Natively supported |
|
||||||
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
|
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
|
||||||
| [Pixelfed](#pixelfed) | ❓ | Not yet tested |
|
| [Pixelfed](#pixelfed) | ❓ | Not yet tested |
|
||||||
| [Pleroma](#pleroma) | ❓ | Not yet tested |
|
| [Pleroma](#pleroma) | ❓ | Not yet tested |
|
||||||
|
@ -224,7 +224,135 @@ You can now reload the page and see in your browser console that data are fetche
|
||||||
|
|
||||||
## Mastodon
|
## Mastodon
|
||||||
|
|
||||||
https://docs.joinmastodon.org/admin/config/#cdn
|
Mastodon natively supports the S3 protocol to store media files, and it works out-of-the-box with Garage.
|
||||||
|
You will need to expose your Garage bucket as a website: that way, media files will be served directly from Garage.
|
||||||
|
|
||||||
|
### Performance considerations
|
||||||
|
|
||||||
|
Mastodon tends to store many small objects over time: expect hundreds of thousands of objects,
|
||||||
|
with average object size ranging from 50 KB to 150 KB.
|
||||||
|
|
||||||
|
As such, your Garage cluster should be configured appropriately for good performance:
|
||||||
|
|
||||||
|
- use Garage v0.8.0 or higher with the [LMDB database engine](@documentation/reference-manual/configuration.md#db-engine-since-v0-8-0).
|
||||||
|
With the default Sled database engine, your database could quickly end up taking tens of GB of disk space.
|
||||||
|
- the Garage database should be stored on a SSD
|
||||||
|
|
||||||
|
### Creating your bucket
|
||||||
|
|
||||||
|
This is the usual Garage setup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
garage key new --name mastodon-key
|
||||||
|
garage bucket create mastodon-data
|
||||||
|
garage bucket allow mastodon-data --read --write --key mastodon-key
|
||||||
|
```
|
||||||
|
|
||||||
|
Note the Key ID and Secret Key.
|
||||||
|
|
||||||
|
### Exposing your bucket as a website
|
||||||
|
|
||||||
|
Create a DNS name to serve your media files, such as `my-social-media.mydomain.tld`.
|
||||||
|
This name will be publicly exposed to the users of your Mastodon instance: they
|
||||||
|
will load images directly from this DNS name.
|
||||||
|
|
||||||
|
As [documented here](@/documentation/cookbook/exposing-websites.md),
|
||||||
|
add this DNS name as alias to your bucket, and expose it as a website:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
garage bucket alias mastodon-data my-social-media.mydomain.tld
|
||||||
|
garage bucket website --allow mastodon-data
|
||||||
|
```
|
||||||
|
|
||||||
|
Then you will likely need to [setup a reverse proxy](@/documentation/cookbook/reverse-proxy.md)
|
||||||
|
in front of it to serve your media files over HTTPS.
|
||||||
|
|
||||||
|
### Cleaning up old media files before migration
|
||||||
|
|
||||||
|
Mastodon instance quickly accumulate a lot of media files from the federation.
|
||||||
|
Most of them are not strictly necessary because they can be fetched again from
|
||||||
|
other servers. As such, it is highly recommended to clean them up before
|
||||||
|
migration, this will greatly reduce the migration time.
|
||||||
|
|
||||||
|
From the [official Mastodon documentation](https://docs.joinmastodon.org/admin/tootctl/#media):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ RAILS_ENV=production bin/tootctl media remove --days 3
|
||||||
|
$ RAILS_ENV=production bin/tootctl media remove-orphans
|
||||||
|
$ RAILS_ENV=production bin/tootctl preview_cards remove --days 15
|
||||||
|
```
|
||||||
|
|
||||||
|
Here is a typical disk usage for a small but multi-year instance after cleanup:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ RAILS_ENV=production bin/tootctl media usage
|
||||||
|
Attachments: 5.67 GB (1.14 GB local)
|
||||||
|
Custom emoji: 295 MB (0 Bytes local)
|
||||||
|
Preview cards: 154 MB
|
||||||
|
Avatars: 3.77 GB (127 KB local)
|
||||||
|
Headers: 8.72 GB (242 KB local)
|
||||||
|
Backups: 0 Bytes
|
||||||
|
Imports: 1.7 KB
|
||||||
|
Settings: 0 Bytes
|
||||||
|
```
|
||||||
|
|
||||||
|
Unfortunately, [old avatars and headers cannot currently be cleaned up](https://github.com/mastodon/mastodon/issues/9567).
|
||||||
|
|
||||||
|
### Migrating your data
|
||||||
|
|
||||||
|
Data migration should be done with an efficient S3 client.
|
||||||
|
The [minio client](@documentation/connect/cli.md#minio-client) is a good choice
|
||||||
|
thanks to its mirror mode:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mc mirror ./public/system/ garage/mastodon-data
|
||||||
|
```
|
||||||
|
|
||||||
|
Here is a typical bucket usage after all data has been migrated:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
$ garage bucket info mastodon-data
|
||||||
|
|
||||||
|
Size: 20.3 GiB (21.8 GB)
|
||||||
|
Objects: 175968
|
||||||
|
```
|
||||||
|
|
||||||
|
### Configuring Mastodon
|
||||||
|
|
||||||
|
In your `.env.production` configuration file:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
S3_ENABLED=true
|
||||||
|
# Internal access to Garage
|
||||||
|
S3_ENDPOINT=http://my-garage-instance.mydomain.tld:3900
|
||||||
|
S3_REGION=garage
|
||||||
|
S3_BUCKET=mastodon-data
|
||||||
|
# Change this (Key ID and Secret Key of your Garage key)
|
||||||
|
AWS_ACCESS_KEY_ID=GKe88df__CHANGETHIS__c5145
|
||||||
|
AWS_SECRET_ACCESS_KEY=a2f7__CHANGETHIS__77fcfcf7a58f47a4aa4431f2e675c56da37821a1070000
|
||||||
|
# What name gets exposed to users (HTTPS is implicit)
|
||||||
|
S3_ALIAS_HOST=my-social-media.mydomain.tld
|
||||||
|
```
|
||||||
|
|
||||||
|
For more details, see the [reference Mastodon documentation](https://docs.joinmastodon.org/admin/config/#cdn).
|
||||||
|
|
||||||
|
Restart all Mastodon services and everything should now be using Garage!
|
||||||
|
You can check the URLs of images in the Mastodon web client, they should start
|
||||||
|
with `https://my-social-media.mydomain.tld`.
|
||||||
|
|
||||||
|
### Last migration sync
|
||||||
|
|
||||||
|
After Mastodon is successfully using Garage, you can run a last sync from the local filesystem to Garage:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mc mirror --newer-than "3h" ./public/system/ garage/mastodon-data
|
||||||
|
```
|
||||||
|
|
||||||
|
### References
|
||||||
|
|
||||||
|
[cybrespace's guide to migrate to S3](https://github.com/cybrespace/cybrespace-meta/blob/master/s3.md)
|
||||||
|
(the guide is for Amazon S3, so the configuration is a bit different, but the rest is similar)
|
||||||
|
|
||||||
|
|
||||||
## Matrix
|
## Matrix
|
||||||
|
|
||||||
|
|
|
@ -48,7 +48,8 @@ garage:
|
||||||
replicationMode: "2"
|
replicationMode: "2"
|
||||||
|
|
||||||
# Start 4 instances (StatefulSets) of garage
|
# Start 4 instances (StatefulSets) of garage
|
||||||
replicaCount: 4
|
deployment:
|
||||||
|
replicaCount: 4
|
||||||
|
|
||||||
# Override default storage class and size
|
# Override default storage class and size
|
||||||
persistence:
|
persistence:
|
||||||
|
|
306
doc/book/cookbook/monitoring.md
Normal file
306
doc/book/cookbook/monitoring.md
Normal file
|
@ -0,0 +1,306 @@
|
||||||
|
+++
|
||||||
|
title = "Monitoring Garage"
|
||||||
|
weight = 40
|
||||||
|
+++
|
||||||
|
|
||||||
|
Garage exposes some internal metrics in the Prometheus data format.
|
||||||
|
This page explains how to exploit these metrics.
|
||||||
|
|
||||||
|
## Setting up monitoring
|
||||||
|
|
||||||
|
### Enabling the Admin API endpoint
|
||||||
|
|
||||||
|
If you have not already enabled the [administration API endpoint](@/documentation/reference-manual/admin-api.md), do so by adding the following lines to your configuration file:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[admin]
|
||||||
|
api_bind_addr = "0.0.0.0:3903"
|
||||||
|
```
|
||||||
|
|
||||||
|
This will allow anyone to scrape Prometheus metrics by fetching
|
||||||
|
`http://localhost:3093/metrics`. If you want to restrict access
|
||||||
|
to the exported metrics, set the `metrics_token` configuration value
|
||||||
|
to a bearer token to be used when fetching the metrics endpoint.
|
||||||
|
|
||||||
|
### Setting up Prometheus and Grafana
|
||||||
|
|
||||||
|
Add a scrape config to your Prometheus daemon to scrape metrics from
|
||||||
|
all of your nodes:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: 'garage'
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 'node1.mycluster:3903'
|
||||||
|
- 'node2.mycluster:3903'
|
||||||
|
- 'node3.mycluster:3903'
|
||||||
|
```
|
||||||
|
|
||||||
|
If you have set a metrics token in your Garage configuration file,
|
||||||
|
add the following lines in your Prometheus scrape config:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
authorization:
|
||||||
|
type: Bearer
|
||||||
|
credentials: 'your metrics token'
|
||||||
|
```
|
||||||
|
|
||||||
|
To visualize the scraped data in Grafana,
|
||||||
|
you can either import our [Grafana dashboard for Garage](https://git.deuxfleurs.fr/Deuxfleurs/garage/raw/branch/main/script/telemetry/grafana-garage-dashboard-prometheus.json)
|
||||||
|
or make your own.
|
||||||
|
We detail below the list of exposed metrics and their meaning.
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## List of exported metrics
|
||||||
|
|
||||||
|
|
||||||
|
### Metrics of the API endpoints
|
||||||
|
|
||||||
|
#### `api_admin_request_counter` (counter)
|
||||||
|
|
||||||
|
Counts the number of requests to a given endpoint of the administration API. Example:
|
||||||
|
|
||||||
|
```
|
||||||
|
api_admin_request_counter{api_endpoint="Metrics"} 127041
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `api_admin_request_duration` (histogram)
|
||||||
|
|
||||||
|
Evaluates the duration of API calls to the various administration API endpoint. Example:
|
||||||
|
|
||||||
|
```
|
||||||
|
api_admin_request_duration_bucket{api_endpoint="Metrics",le="0.5"} 127041
|
||||||
|
api_admin_request_duration_sum{api_endpoint="Metrics"} 605.250344830999
|
||||||
|
api_admin_request_duration_count{api_endpoint="Metrics"} 127041
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `api_s3_request_counter` (counter)
|
||||||
|
|
||||||
|
Counts the number of requests to a given endpoint of the S3 API. Example:
|
||||||
|
|
||||||
|
```
|
||||||
|
api_s3_request_counter{api_endpoint="CreateMultipartUpload"} 1
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `api_s3_error_counter` (counter)
|
||||||
|
|
||||||
|
Counts the number of requests to a given endpoint of the S3 API that returned an error. Example:
|
||||||
|
|
||||||
|
```
|
||||||
|
api_s3_error_counter{api_endpoint="GetObject",status_code="404"} 39
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `api_s3_request_duration` (histogram)
|
||||||
|
|
||||||
|
Evaluates the duration of API calls to the various S3 API endpoints. Example:
|
||||||
|
|
||||||
|
```
|
||||||
|
api_s3_request_duration_bucket{api_endpoint="CreateMultipartUpload",le="0.5"} 1
|
||||||
|
api_s3_request_duration_sum{api_endpoint="CreateMultipartUpload"} 0.046340762
|
||||||
|
api_s3_request_duration_count{api_endpoint="CreateMultipartUpload"} 1
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `api_k2v_request_counter` (counter), `api_k2v_error_counter` (counter), `api_k2v_error_duration` (histogram)
|
||||||
|
|
||||||
|
Same as for S3, for the K2V API.
|
||||||
|
|
||||||
|
|
||||||
|
### Metrics of the Web endpoint
|
||||||
|
|
||||||
|
|
||||||
|
#### `web_request_counter` (counter)
|
||||||
|
|
||||||
|
Number of requests to the web endpoint
|
||||||
|
|
||||||
|
```
|
||||||
|
web_request_counter{method="GET"} 80
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `web_request_duration` (histogram)
|
||||||
|
|
||||||
|
Duration of requests to the web endpoint
|
||||||
|
|
||||||
|
```
|
||||||
|
web_request_duration_bucket{method="GET",le="0.5"} 80
|
||||||
|
web_request_duration_sum{method="GET"} 1.0528433229999998
|
||||||
|
web_request_duration_count{method="GET"} 80
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `web_error_counter` (counter)
|
||||||
|
|
||||||
|
Number of requests to the web endpoint resulting in errors
|
||||||
|
|
||||||
|
```
|
||||||
|
web_error_counter{method="GET",status_code="404 Not Found"} 64
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
### Metrics of the data block manager
|
||||||
|
|
||||||
|
#### `block_bytes_read`, `block_bytes_written` (counter)
|
||||||
|
|
||||||
|
Number of bytes read/written to/from disk in the data storage directory.
|
||||||
|
|
||||||
|
```
|
||||||
|
block_bytes_read 120586322022
|
||||||
|
block_bytes_written 3386618077
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `block_read_duration`, `block_write_duration` (histograms)
|
||||||
|
|
||||||
|
Evaluates the duration of the reading/writing of individual data blocks in the data storage directory.
|
||||||
|
|
||||||
|
```
|
||||||
|
block_read_duration_bucket{le="0.5"} 169229
|
||||||
|
block_read_duration_sum 2761.6902550310056
|
||||||
|
block_read_duration_count 169240
|
||||||
|
block_write_duration_bucket{le="0.5"} 3559
|
||||||
|
block_write_duration_sum 195.59170078500006
|
||||||
|
block_write_duration_count 3571
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `block_delete_counter` (counter)
|
||||||
|
|
||||||
|
Counts the number of data blocks that have been deleted from storage.
|
||||||
|
|
||||||
|
```
|
||||||
|
block_delete_counter 122
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `block_resync_counter` (counter), `block_resync_duration` (histogram)
|
||||||
|
|
||||||
|
Counts the number of resync operations the node has executed, and evaluates their duration.
|
||||||
|
|
||||||
|
```
|
||||||
|
block_resync_counter 308897
|
||||||
|
block_resync_duration_bucket{le="0.5"} 308892
|
||||||
|
block_resync_duration_sum 139.64204196100016
|
||||||
|
block_resync_duration_count 308897
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `block_resync_queue_length` (gauge)
|
||||||
|
|
||||||
|
The number of block hashes currently queued for a resync.
|
||||||
|
This is normal to be nonzero for long periods of time.
|
||||||
|
|
||||||
|
```
|
||||||
|
block_resync_queue_length 0
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `block_resync_errored_blocks` (gauge)
|
||||||
|
|
||||||
|
The number of block hashes that we were unable to resync last time we tried.
|
||||||
|
**THIS SHOULD BE ZERO, OR FALL BACK TO ZERO RAPIDLY, IN A HEALTHY CLUSTER.**
|
||||||
|
Persistent nonzero values indicate that some data is likely to be lost.
|
||||||
|
|
||||||
|
```
|
||||||
|
block_resync_errored_blocks 0
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
### Metrics related to RPCs (remote procedure calls) between nodes
|
||||||
|
|
||||||
|
#### `rpc_netapp_request_counter` (counter)
|
||||||
|
|
||||||
|
Number of RPC requests emitted
|
||||||
|
|
||||||
|
```
|
||||||
|
rpc_request_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 176
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `rpc_netapp_error_counter` (counter)
|
||||||
|
|
||||||
|
Number of communication errors (errors in the Netapp library, generally due to disconnected nodes)
|
||||||
|
|
||||||
|
```
|
||||||
|
rpc_netapp_error_counter{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 354
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `rpc_timeout_counter` (counter)
|
||||||
|
|
||||||
|
Number of RPC timeouts, should be close to zero in a healthy cluster.
|
||||||
|
|
||||||
|
```
|
||||||
|
rpc_timeout_counter{from="<this node>",rpc_endpoint="garage_rpc/membership.rs/SystemRpc",to="<remote node>"} 1
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `rpc_duration` (histogram)
|
||||||
|
|
||||||
|
The duration of internal RPC calls between Garage nodes.
|
||||||
|
|
||||||
|
```
|
||||||
|
rpc_duration_bucket{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>",le="0.5"} 166
|
||||||
|
rpc_duration_sum{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 35.172253716
|
||||||
|
rpc_duration_count{from="<this node>",rpc_endpoint="garage_block/manager.rs/Rpc",to="<remote node>"} 174
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
### Metrics of the metadata table manager
|
||||||
|
|
||||||
|
#### `table_gc_todo_queue_length` (gauge)
|
||||||
|
|
||||||
|
Table garbage collector TODO queue length
|
||||||
|
|
||||||
|
```
|
||||||
|
table_gc_todo_queue_length{table_name="block_ref"} 0
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `table_get_request_counter` (counter), `table_get_request_duration` (histogram)
|
||||||
|
|
||||||
|
Number of get/get_range requests internally made on each table, and their duration.
|
||||||
|
|
||||||
|
```
|
||||||
|
table_get_request_counter{table_name="bucket_alias"} 315
|
||||||
|
table_get_request_duration_bucket{table_name="bucket_alias",le="0.5"} 315
|
||||||
|
table_get_request_duration_sum{table_name="bucket_alias"} 0.048509778000000024
|
||||||
|
table_get_request_duration_count{table_name="bucket_alias"} 315
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
#### `table_put_request_counter` (counter), `table_put_request_duration` (histogram)
|
||||||
|
|
||||||
|
Number of insert/insert_many requests internally made on this table, and their duration
|
||||||
|
|
||||||
|
```
|
||||||
|
table_put_request_counter{table_name="block_ref"} 677
|
||||||
|
table_put_request_duration_bucket{table_name="block_ref",le="0.5"} 677
|
||||||
|
table_put_request_duration_sum{table_name="block_ref"} 61.617528636
|
||||||
|
table_put_request_duration_count{table_name="block_ref"} 677
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `table_internal_delete_counter` (counter)
|
||||||
|
|
||||||
|
Number of value deletions in the tree (due to GC or repartitioning)
|
||||||
|
|
||||||
|
```
|
||||||
|
table_internal_delete_counter{table_name="block_ref"} 2296
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `table_internal_update_counter` (counter)
|
||||||
|
|
||||||
|
Number of value updates where the value actually changes (includes creation of new key and update of existing key)
|
||||||
|
|
||||||
|
```
|
||||||
|
table_internal_update_counter{table_name="block_ref"} 5996
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `table_merkle_updater_todo_queue_length` (gauge)
|
||||||
|
|
||||||
|
Merkle tree updater TODO queue length (should fall to zero rapidly)
|
||||||
|
|
||||||
|
```
|
||||||
|
table_merkle_updater_todo_queue_length{table_name="block_ref"} 0
|
||||||
|
```
|
||||||
|
|
||||||
|
#### `table_sync_items_received`, `table_sync_items_sent` (counters)
|
||||||
|
|
||||||
|
Number of data items sent to/recieved from other nodes during resync procedures
|
||||||
|
|
||||||
|
```
|
||||||
|
table_sync_items_received{from="<remote node>",table_name="bucket_v2"} 3
|
||||||
|
table_sync_items_sent{table_name="block_ref",to="<remote node>"} 2
|
||||||
|
```
|
||||||
|
|
||||||
|
|
|
@ -11,8 +11,9 @@ We recommend first following the [quick start guide](@/documentation/quick-start
|
||||||
to get familiar with Garage's command line and usage patterns.
|
to get familiar with Garage's command line and usage patterns.
|
||||||
|
|
||||||
|
|
||||||
|
## Preparing your environment
|
||||||
|
|
||||||
## Prerequisites
|
### Prerequisites
|
||||||
|
|
||||||
To run a real-world deployment, make sure the following conditions are met:
|
To run a real-world deployment, make sure the following conditions are met:
|
||||||
|
|
||||||
|
@ -21,10 +22,6 @@ To run a real-world deployment, make sure the following conditions are met:
|
||||||
- Each machine has a public IP address which is reachable by other machines.
|
- Each machine has a public IP address which is reachable by other machines.
|
||||||
Running behind a NAT is likely to be possible but hasn't been tested for the latest version (TODO).
|
Running behind a NAT is likely to be possible but hasn't been tested for the latest version (TODO).
|
||||||
|
|
||||||
- Ideally, each machine should have a SSD available in addition to the HDD you are dedicating
|
|
||||||
to Garage. This will allow for faster access to metadata and has the potential
|
|
||||||
to significantly reduce Garage's response times.
|
|
||||||
|
|
||||||
- This guide will assume you are using Docker containers to deploy Garage on each node.
|
- This guide will assume you are using Docker containers to deploy Garage on each node.
|
||||||
Garage can also be run independently, for instance as a [Systemd service](@/documentation/cookbook/systemd.md).
|
Garage can also be run independently, for instance as a [Systemd service](@/documentation/cookbook/systemd.md).
|
||||||
You can also use an orchestrator such as Nomad or Kubernetes to automatically manage
|
You can also use an orchestrator such as Nomad or Kubernetes to automatically manage
|
||||||
|
@ -49,6 +46,42 @@ available in the different locations of your cluster is roughly the same.
|
||||||
For instance, here, the Mercury node could be moved to Brussels; this would allow the cluster
|
For instance, here, the Mercury node could be moved to Brussels; this would allow the cluster
|
||||||
to store 2 TB of data in total.
|
to store 2 TB of data in total.
|
||||||
|
|
||||||
|
### Best practices
|
||||||
|
|
||||||
|
- If you have fast dedicated networking between all your nodes, and are planing to store
|
||||||
|
very large files, bump the `block_size` configuration parameter to 10 MB
|
||||||
|
(`block_size = 10485760`).
|
||||||
|
|
||||||
|
- Garage stores its files in two locations: it uses a metadata directory to store frequently-accessed
|
||||||
|
small metadata items, and a data directory to store data blocks of uploaded objects.
|
||||||
|
Ideally, the metadata directory would be stored on an SSD (smaller but faster),
|
||||||
|
and the data directory would be stored on an HDD (larger but slower).
|
||||||
|
|
||||||
|
- For the data directory, Garage already does checksumming and integrity verification,
|
||||||
|
so there is no need to use a filesystem such as BTRFS or ZFS that does it.
|
||||||
|
We recommend using XFS for the data partition, as it has the best performance.
|
||||||
|
EXT4 is not recommended as it has more strict limitations on the number of inodes,
|
||||||
|
which might cause issues with Garage when large numbers of objects are stored.
|
||||||
|
|
||||||
|
- If you only have an HDD and no SSD, it's fine to put your metadata alongside the data
|
||||||
|
on the same drive. Having lots of RAM for your kernel to cache the metadata will
|
||||||
|
help a lot with performance. Make sure to use the LMDB database engine,
|
||||||
|
instead of Sled, which suffers from quite bad performance degradation on HDDs.
|
||||||
|
Sled is still the default for legacy reasons, but is not recommended anymore.
|
||||||
|
|
||||||
|
- For the metadata storage, Garage does not do checksumming and integrity
|
||||||
|
verification on its own. If you are afraid of bitrot/data corruption,
|
||||||
|
put your metadata directory on a BTRFS partition. Otherwise, just use regular
|
||||||
|
EXT4 or XFS.
|
||||||
|
|
||||||
|
- Having a single server with several storage drives is currently not very well
|
||||||
|
supported in Garage ([#218](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/218)).
|
||||||
|
For an easy setup, just put all your drives in a RAID0 or a ZFS RAIDZ array.
|
||||||
|
If you're adventurous, you can try to format each of your disk as
|
||||||
|
a separate XFS partition, and then run one `garage` daemon per disk drive,
|
||||||
|
or use something like [`mergerfs`](https://github.com/trapexit/mergerfs) to merge
|
||||||
|
all your disks in a single union filesystem that spreads load over them.
|
||||||
|
|
||||||
## Get a Docker image
|
## Get a Docker image
|
||||||
|
|
||||||
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
|
Our docker image is currently named `dxflrs/garage` and is stored on the [Docker Hub](https://hub.docker.com/r/dxflrs/garage/tags?page=1&ordering=last_updated).
|
||||||
|
@ -81,6 +114,7 @@ A valid `/etc/garage/garage.toml` for our cluster would look as follows:
|
||||||
```toml
|
```toml
|
||||||
metadata_dir = "/var/lib/garage/meta"
|
metadata_dir = "/var/lib/garage/meta"
|
||||||
data_dir = "/var/lib/garage/data"
|
data_dir = "/var/lib/garage/data"
|
||||||
|
db_engine = "lmdb"
|
||||||
|
|
||||||
replication_mode = "3"
|
replication_mode = "3"
|
||||||
|
|
||||||
|
@ -90,8 +124,6 @@ rpc_bind_addr = "[::]:3901"
|
||||||
rpc_public_addr = "<this node's public IP>:3901"
|
rpc_public_addr = "<this node's public IP>:3901"
|
||||||
rpc_secret = "<RPC secret>"
|
rpc_secret = "<RPC secret>"
|
||||||
|
|
||||||
bootstrap_peers = []
|
|
||||||
|
|
||||||
[s3_api]
|
[s3_api]
|
||||||
s3_region = "garage"
|
s3_region = "garage"
|
||||||
api_bind_addr = "[::]:3900"
|
api_bind_addr = "[::]:3900"
|
||||||
|
@ -132,6 +164,21 @@ It should be restarted automatically at each reboot.
|
||||||
Please note that we use host networking as otherwise Docker containers
|
Please note that we use host networking as otherwise Docker containers
|
||||||
can not communicate with IPv6.
|
can not communicate with IPv6.
|
||||||
|
|
||||||
|
If you want to use `docker-compose`, you may use the following `docker-compose.yml` file as a reference:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
version: "3"
|
||||||
|
services:
|
||||||
|
garage:
|
||||||
|
image: dxflrs/garage:v0.8.0
|
||||||
|
network_mode: "host"
|
||||||
|
restart: unless-stopped
|
||||||
|
volumes:
|
||||||
|
- /etc/garage.toml:/etc/garage.toml
|
||||||
|
- /var/lib/garage/meta:/var/lib/garage/meta
|
||||||
|
- /var/lib/garage/data:/var/lib/garage/data
|
||||||
|
```
|
||||||
|
|
||||||
Upgrading between Garage versions should be supported transparently,
|
Upgrading between Garage versions should be supported transparently,
|
||||||
but please check the relase notes before doing so!
|
but please check the relase notes before doing so!
|
||||||
To upgrade, simply stop and remove this container and
|
To upgrade, simply stop and remove this container and
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
+++
|
+++
|
||||||
title = "Recovering from failures"
|
title = "Recovering from failures"
|
||||||
weight = 35
|
weight = 50
|
||||||
+++
|
+++
|
||||||
|
|
||||||
Garage is meant to work on old, second-hand hardware.
|
Garage is meant to work on old, second-hand hardware.
|
||||||
|
|
|
@ -70,14 +70,16 @@ A possible configuration:
|
||||||
|
|
||||||
```nginx
|
```nginx
|
||||||
upstream s3_backend {
|
upstream s3_backend {
|
||||||
# if you have a garage instance locally
|
# If you have a garage instance locally.
|
||||||
server 127.0.0.1:3900;
|
server 127.0.0.1:3900;
|
||||||
# you can also put your other instances
|
# You can also put your other instances.
|
||||||
server 192.168.1.3:3900;
|
server 192.168.1.3:3900;
|
||||||
# domain names also work
|
# Domain names also work.
|
||||||
server garage1.example.com:3900;
|
server garage1.example.com:3900;
|
||||||
# you can assign weights if you have some servers
|
# A "backup" server is only used if all others have failed.
|
||||||
# that are more powerful than others
|
server garage-remote.example.com:3900 backup;
|
||||||
|
# You can assign weights if you have some servers
|
||||||
|
# that can serve more requests than others.
|
||||||
server garage2.example.com:3900 weight=2;
|
server garage2.example.com:3900 weight=2;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -96,6 +98,8 @@ server {
|
||||||
proxy_pass http://s3_backend;
|
proxy_pass http://s3_backend;
|
||||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
|
# Disable buffering to a temporary file.
|
||||||
|
proxy_max_temp_file_size 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
+++
|
+++
|
||||||
title = "Upgrading Garage"
|
title = "Upgrading Garage"
|
||||||
weight = 40
|
weight = 60
|
||||||
+++
|
+++
|
||||||
|
|
||||||
Garage is a stateful clustered application, where all nodes are communicating together and share data structures.
|
Garage is a stateful clustered application, where all nodes are communicating together and share data structures.
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
+++
|
+++
|
||||||
title = "Design"
|
title = "Design"
|
||||||
weight = 5
|
weight = 6
|
||||||
sort_by = "weight"
|
sort_by = "weight"
|
||||||
template = "documentation.html"
|
template = "documentation.html"
|
||||||
+++
|
+++
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
+++
|
+++
|
||||||
title = "Development"
|
title = "Development"
|
||||||
weight = 6
|
weight = 7
|
||||||
sort_by = "weight"
|
sort_by = "weight"
|
||||||
template = "documentation.html"
|
template = "documentation.html"
|
||||||
+++
|
+++
|
||||||
|
|
|
@ -42,25 +42,25 @@ you can [build Garage from source](@/documentation/cookbook/from-source.md).
|
||||||
|
|
||||||
## Configuring and starting Garage
|
## Configuring and starting Garage
|
||||||
|
|
||||||
### Writing a first configuration file
|
### Generating a first configuration file
|
||||||
|
|
||||||
This first configuration file should allow you to get started easily with the simplest
|
This first configuration file should allow you to get started easily with the simplest
|
||||||
possible Garage deployment.
|
possible Garage deployment.
|
||||||
**Save it as `/etc/garage.toml`.**
|
|
||||||
You can also store it somewhere else, but you will have to specify `-c path/to/garage.toml`
|
|
||||||
at each invocation of the `garage` binary (for example: `garage -c ./garage.toml server`, `garage -c ./garage.toml status`).
|
|
||||||
|
|
||||||
```toml
|
We will create it with the following command line
|
||||||
|
to generate unique and private secrets for security reasons:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cat > garage.toml <<EOF
|
||||||
metadata_dir = "/tmp/meta"
|
metadata_dir = "/tmp/meta"
|
||||||
data_dir = "/tmp/data"
|
data_dir = "/tmp/data"
|
||||||
|
db_engine = "lmdb"
|
||||||
|
|
||||||
replication_mode = "none"
|
replication_mode = "none"
|
||||||
|
|
||||||
rpc_bind_addr = "[::]:3901"
|
rpc_bind_addr = "[::]:3901"
|
||||||
rpc_public_addr = "127.0.0.1:3901"
|
rpc_public_addr = "127.0.0.1:3901"
|
||||||
rpc_secret = "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
|
rpc_secret = "$(openssl rand -hex 32)"
|
||||||
|
|
||||||
bootstrap_peers = []
|
|
||||||
|
|
||||||
[s3_api]
|
[s3_api]
|
||||||
s3_region = "garage"
|
s3_region = "garage"
|
||||||
|
@ -71,12 +71,26 @@ root_domain = ".s3.garage.localhost"
|
||||||
bind_addr = "[::]:3902"
|
bind_addr = "[::]:3902"
|
||||||
root_domain = ".web.garage.localhost"
|
root_domain = ".web.garage.localhost"
|
||||||
index = "index.html"
|
index = "index.html"
|
||||||
|
|
||||||
|
[k2v_api]
|
||||||
|
api_bind_addr = "[::]:3904"
|
||||||
|
|
||||||
|
[admin]
|
||||||
|
api_bind_addr = "0.0.0.0:3903"
|
||||||
|
admin_token = "$(openssl rand -base64 32)"
|
||||||
|
EOF
|
||||||
```
|
```
|
||||||
|
|
||||||
The `rpc_secret` value provided above is just an example. It will work, but in
|
Now that your configuration file has been created, you can put
|
||||||
order to secure your cluster you will need to use another one. You can generate
|
it in the right place. By default, garage looks at **`/etc/garage.toml`.**
|
||||||
such a value with `openssl rand -hex 32`.
|
|
||||||
|
|
||||||
|
You can also store it somewhere else, but you will have to specify `-c path/to/garage.toml`
|
||||||
|
at each invocation of the `garage` binary (for example: `garage -c ./garage.toml server`, `garage -c ./garage.toml status`).
|
||||||
|
|
||||||
|
As you can see, the `rpc_secret` is a 32 bytes hexadecimal string.
|
||||||
|
You can regenerate it with `openssl rand -hex 32`.
|
||||||
|
If you target a cluster deployment with multiple nodes, make sure that
|
||||||
|
you use the same value for all nodes.
|
||||||
|
|
||||||
As you can see in the `metadata_dir` and `data_dir` parameters, we are saving Garage's data
|
As you can see in the `metadata_dir` and `data_dir` parameters, we are saving Garage's data
|
||||||
in `/tmp` which gets erased when your system reboots. This means that data stored on this
|
in `/tmp` which gets erased when your system reboots. This means that data stored on this
|
||||||
|
@ -219,6 +233,7 @@ Now that we have a bucket and a key, we need to give permissions to the key on t
|
||||||
garage bucket allow \
|
garage bucket allow \
|
||||||
--read \
|
--read \
|
||||||
--write \
|
--write \
|
||||||
|
--owner \
|
||||||
nextcloud-bucket \
|
nextcloud-bucket \
|
||||||
--key nextcloud-app-key
|
--key nextcloud-app-key
|
||||||
```
|
```
|
||||||
|
@ -232,54 +247,73 @@ garage bucket info nextcloud-bucket
|
||||||
|
|
||||||
## Uploading and downlading from Garage
|
## Uploading and downlading from Garage
|
||||||
|
|
||||||
We recommend the use of MinIO Client to interact with Garage files (`mc`).
|
To download and upload files on garage, we can use a third-party tool named `awscli`.
|
||||||
Instructions to install it and use it are provided on the
|
|
||||||
[MinIO website](https://docs.min.io/docs/minio-client-quickstart-guide.html).
|
|
||||||
Before reading the following, you need a working `mc` command on your path.
|
|
||||||
|
|
||||||
Note that on certain Linux distributions such as Arch Linux, the Minio client binary
|
|
||||||
is called `mcli` instead of `mc` (to avoid name clashes with the Midnight Commander).
|
|
||||||
|
|
||||||
### Configure `mc`
|
### Install and configure `awscli`
|
||||||
|
|
||||||
You need your access key and secret key created above.
|
If you have python on your system, you can install it with:
|
||||||
We will assume you are invoking `mc` on the same machine as the Garage server,
|
|
||||||
your S3 API endpoint is therefore `http://127.0.0.1:3900`.
|
|
||||||
For this whole configuration, you must set an alias name: we chose `my-garage`, that you will used for all commands.
|
|
||||||
|
|
||||||
Adapt the following command accordingly and run it:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mc alias set \
|
python -m pip install --user awscli
|
||||||
my-garage \
|
|
||||||
http://127.0.0.1:3900 \
|
|
||||||
<access key> \
|
|
||||||
<secret key> \
|
|
||||||
--api S3v4
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Use `mc`
|
Now that `awscli` is installed, you must configure it to talk to your Garage instance,
|
||||||
|
with your key. There are multiple ways to do that, the simplest one is to create a file
|
||||||
You can not list buckets from `mc` currently.
|
named `~/.awsrc` with this content:
|
||||||
|
|
||||||
But the following commands and many more should work:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
mc cp image.png my-garage/nextcloud-bucket
|
export AWS_ACCESS_KEY_ID=xxxx # put your Key ID here
|
||||||
mc cp my-garage/nextcloud-bucket/image.png .
|
export AWS_SECRET_ACCESS_KEY=xxxx # put your Secret key here
|
||||||
mc ls my-garage/nextcloud-bucket
|
export AWS_DEFAULT_REGION='garage'
|
||||||
mc mirror localdir/ my-garage/another-bucket
|
export AWS_ENDPOINT='http://localhost:3900'
|
||||||
|
|
||||||
|
function aws { command aws --endpoint-url $AWS_ENDPOINT $@ ; }
|
||||||
|
aws --version
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Now, each time you want to use `awscli` on this target, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
source ~/.awsrc
|
||||||
|
```
|
||||||
|
|
||||||
|
*You can create multiple files with different names if you
|
||||||
|
have multiple Garage clusters or different keys.
|
||||||
|
Switching from one cluster to another is as simple as
|
||||||
|
sourcing the right file.*
|
||||||
|
|
||||||
|
### Example usage of `awscli`
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# list buckets
|
||||||
|
aws s3 ls
|
||||||
|
|
||||||
|
# list objects of a bucket
|
||||||
|
aws s3 ls s3://my_files
|
||||||
|
|
||||||
|
# copy from your filesystem to garage
|
||||||
|
aws s3 cp /proc/cpuinfo s3://my_files/cpuinfo.txt
|
||||||
|
|
||||||
|
# copy from garage to your filesystem
|
||||||
|
aws s3 cp s3/my_files/cpuinfo.txt /tmp/cpuinfo.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
Note that you can use `awscli` for more advanced operations like
|
||||||
|
creating a bucket, pre-signing a request or managing your website.
|
||||||
|
[Read the full documentation to know more](https://awscli.amazonaws.com/v2/documentation/api/latest/reference/s3/index.html).
|
||||||
|
|
||||||
|
Some features are however not implemented like ACL or policy.
|
||||||
|
Check [our s3 compatibility list](@/documentation/reference-manual/s3-compatibility.md).
|
||||||
|
|
||||||
### Other tools for interacting with Garage
|
### Other tools for interacting with Garage
|
||||||
|
|
||||||
The following tools can also be used to send and recieve files from/to Garage:
|
The following tools can also be used to send and recieve files from/to Garage:
|
||||||
|
|
||||||
- the [AWS CLI](https://aws.amazon.com/cli/)
|
- [minio-client](@/documentation/connect/cli.md#minio-client)
|
||||||
- [`rclone`](https://rclone.org/)
|
- [s3cmd](@/documentation/connect/cli.md#s3cmd)
|
||||||
- [Cyberduck](https://cyberduck.io/)
|
- [rclone](@/documentation/connect/cli.md#rclone)
|
||||||
- [`s3cmd`](https://s3tools.org/s3cmd)
|
- [Cyberduck](@/documentation/connect/cli.md#cyberduck)
|
||||||
|
- [WinSCP](@/documentation/connect/cli.md#winscp)
|
||||||
|
|
||||||
Refer to the ["Integrations" section](@/documentation/connect/_index.md) to learn how to
|
An exhaustive list is maintained in the ["Integrations" > "Browsing tools" section](@/documentation/connect/_index.md).
|
||||||
configure application and command line utilities to integrate with Garage.
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
+++
|
+++
|
||||||
title = "Reference Manual"
|
title = "Reference Manual"
|
||||||
weight = 4
|
weight = 5
|
||||||
sort_by = "weight"
|
sort_by = "weight"
|
||||||
template = "documentation.html"
|
template = "documentation.html"
|
||||||
+++
|
+++
|
||||||
|
|
|
@ -47,598 +47,13 @@ Returns internal Garage metrics in Prometheus format.
|
||||||
|
|
||||||
### Cluster operations
|
### Cluster operations
|
||||||
|
|
||||||
#### GetClusterStatus `GET /v0/status`
|
These endpoints are defined on a dedicated [Redocly page](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html). You can also download its [OpenAPI specification](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml).
|
||||||
|
|
||||||
Returns the cluster's current status in JSON, including:
|
Requesting the API from the command line can be as simple as running:
|
||||||
|
|
||||||
- ID of the node being queried and its version of the Garage daemon
|
```bash
|
||||||
- Live nodes
|
curl -H 'Authorization: Bearer s3cr3t' http://localhost:3903/v0/status | jq
|
||||||
- Currently configured cluster layout
|
|
||||||
- Staged changes to the cluster layout
|
|
||||||
|
|
||||||
Example response body:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f",
|
|
||||||
"garage_version": "git:v0.8.0",
|
|
||||||
"knownNodes": {
|
|
||||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
|
|
||||||
"addr": "10.0.0.11:3901",
|
|
||||||
"is_up": true,
|
|
||||||
"last_seen_secs_ago": 9,
|
|
||||||
"hostname": "node1"
|
|
||||||
},
|
|
||||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
|
|
||||||
"addr": "10.0.0.12:3901",
|
|
||||||
"is_up": true,
|
|
||||||
"last_seen_secs_ago": 1,
|
|
||||||
"hostname": "node2"
|
|
||||||
},
|
|
||||||
"23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
|
|
||||||
"addr": "10.0.0.21:3901",
|
|
||||||
"is_up": true,
|
|
||||||
"last_seen_secs_ago": 7,
|
|
||||||
"hostname": "node3"
|
|
||||||
},
|
|
||||||
"e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
|
|
||||||
"addr": "10.0.0.22:3901",
|
|
||||||
"is_up": true,
|
|
||||||
"last_seen_secs_ago": 1,
|
|
||||||
"hostname": "node4"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"layout": {
|
|
||||||
"version": 12,
|
|
||||||
"roles": {
|
|
||||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
|
|
||||||
"zone": "dc1",
|
|
||||||
"capacity": 4,
|
|
||||||
"tags": [
|
|
||||||
"node1"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
|
|
||||||
"zone": "dc1",
|
|
||||||
"capacity": 6,
|
|
||||||
"tags": [
|
|
||||||
"node2"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
|
|
||||||
"zone": "dc2",
|
|
||||||
"capacity": 10,
|
|
||||||
"tags": [
|
|
||||||
"node3"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"stagedRoleChanges": {
|
|
||||||
"e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
|
|
||||||
"zone": "dc2",
|
|
||||||
"capacity": 5,
|
|
||||||
"tags": [
|
|
||||||
"node4"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
#### ConnectClusterNodes `POST /v0/connect`
|
For more advanced use cases, we recommend using a SDK.
|
||||||
|
[Go to the "Build your own app" section to know how to use our SDKs](@/documentation/build/_index.md)
|
||||||
Instructs this Garage node to connect to other Garage nodes at specified addresses.
|
|
||||||
|
|
||||||
Example request body:
|
|
||||||
|
|
||||||
```json
|
|
||||||
[
|
|
||||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901",
|
|
||||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901"
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
The format of the string for a node to connect to is: `<node ID>@<ip address>:<port>`, same as in the `garage node connect` CLI call.
|
|
||||||
|
|
||||||
Example response:
|
|
||||||
|
|
||||||
```json
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"success": true,
|
|
||||||
"error": null
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"success": false,
|
|
||||||
"error": "Handshake error"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
#### GetClusterLayout `GET /v0/layout`
|
|
||||||
|
|
||||||
Returns the cluster's current layout in JSON, including:
|
|
||||||
|
|
||||||
- Currently configured cluster layout
|
|
||||||
- Staged changes to the cluster layout
|
|
||||||
|
|
||||||
(the info returned by this endpoint is a subset of the info returned by GetClusterStatus)
|
|
||||||
|
|
||||||
Example response body:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"version": 12,
|
|
||||||
"roles": {
|
|
||||||
"ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": {
|
|
||||||
"zone": "dc1",
|
|
||||||
"capacity": 4,
|
|
||||||
"tags": [
|
|
||||||
"node1"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": {
|
|
||||||
"zone": "dc1",
|
|
||||||
"capacity": 6,
|
|
||||||
"tags": [
|
|
||||||
"node2"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": {
|
|
||||||
"zone": "dc2",
|
|
||||||
"capacity": 10,
|
|
||||||
"tags": [
|
|
||||||
"node3"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"stagedRoleChanges": {
|
|
||||||
"e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": {
|
|
||||||
"zone": "dc2",
|
|
||||||
"capacity": 5,
|
|
||||||
"tags": [
|
|
||||||
"node4"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### UpdateClusterLayout `POST /v0/layout`
|
|
||||||
|
|
||||||
Send modifications to the cluster layout. These modifications will
|
|
||||||
be included in the staged role changes, visible in subsequent calls
|
|
||||||
of `GetClusterLayout`. Once the set of staged changes is satisfactory,
|
|
||||||
the user may call `ApplyClusterLayout` to apply the changed changes,
|
|
||||||
or `Revert ClusterLayout` to clear all of the staged changes in
|
|
||||||
the layout.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
<node_id>: {
|
|
||||||
"capacity": <new_capacity>,
|
|
||||||
"zone": <new_zone>,
|
|
||||||
"tags": [
|
|
||||||
<new_tag>,
|
|
||||||
...
|
|
||||||
]
|
|
||||||
},
|
|
||||||
<node_id_to_remove>: null,
|
|
||||||
...
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Contrary to the CLI that may update only a subset of the fields
|
|
||||||
`capacity`, `zone` and `tags`, when calling this API all of these
|
|
||||||
values must be specified.
|
|
||||||
|
|
||||||
|
|
||||||
#### ApplyClusterLayout `POST /v0/layout/apply`
|
|
||||||
|
|
||||||
Applies to the cluster the layout changes currently registered as
|
|
||||||
staged layout changes.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"version": 13
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Similarly to the CLI, the body must include the version of the new layout
|
|
||||||
that will be created, which MUST be 1 + the value of the currently
|
|
||||||
existing layout in the cluster.
|
|
||||||
|
|
||||||
#### RevertClusterLayout `POST /v0/layout/revert`
|
|
||||||
|
|
||||||
Clears all of the staged layout changes.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"version": 13
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Reverting the staged changes is done by incrementing the version number
|
|
||||||
and clearing the contents of the staged change list.
|
|
||||||
Similarly to the CLI, the body must include the incremented
|
|
||||||
version number, which MUST be 1 + the value of the currently
|
|
||||||
existing layout in the cluster.
|
|
||||||
|
|
||||||
|
|
||||||
### Access key operations
|
|
||||||
|
|
||||||
#### ListKeys `GET /v0/key`
|
|
||||||
|
|
||||||
Returns all API access keys in the cluster.
|
|
||||||
|
|
||||||
Example response:
|
|
||||||
|
|
||||||
```json
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"id": "GK31c2f218a2e44f485b94239e",
|
|
||||||
"name": "test"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "GKe10061ac9c2921f09e4c5540",
|
|
||||||
"name": "test2"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
#### CreateKey `POST /v0/key`
|
|
||||||
|
|
||||||
Creates a new API access key.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"name": "NameOfMyKey"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### ImportKey `POST /v0/key/import`
|
|
||||||
|
|
||||||
Imports an existing API key.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
|
||||||
"secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
|
|
||||||
"name": "NameOfMyKey"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### GetKeyInfo `GET /v0/key?id=<acces key id>`
|
|
||||||
#### GetKeyInfo `GET /v0/key?search=<pattern>`
|
|
||||||
|
|
||||||
Returns information about the requested API access key.
|
|
||||||
|
|
||||||
If `id` is set, the key is looked up using its exact identifier (faster).
|
|
||||||
If `search` is set, the key is looked up using its name or prefix
|
|
||||||
of identifier (slower, all keys are enumerated to do this).
|
|
||||||
|
|
||||||
Example response:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"name": "test",
|
|
||||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
|
||||||
"secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835",
|
|
||||||
"permissions": {
|
|
||||||
"createBucket": false
|
|
||||||
},
|
|
||||||
"buckets": [
|
|
||||||
{
|
|
||||||
"id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
|
|
||||||
"globalAliases": [
|
|
||||||
"test2"
|
|
||||||
],
|
|
||||||
"localAliases": [],
|
|
||||||
"permissions": {
|
|
||||||
"read": true,
|
|
||||||
"write": true,
|
|
||||||
"owner": false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
|
|
||||||
"globalAliases": [
|
|
||||||
"test3"
|
|
||||||
],
|
|
||||||
"localAliases": [],
|
|
||||||
"permissions": {
|
|
||||||
"read": true,
|
|
||||||
"write": true,
|
|
||||||
"owner": false
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
|
||||||
"globalAliases": [],
|
|
||||||
"localAliases": [
|
|
||||||
"test"
|
|
||||||
],
|
|
||||||
"permissions": {
|
|
||||||
"read": true,
|
|
||||||
"write": true,
|
|
||||||
"owner": true
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
|
|
||||||
"globalAliases": [
|
|
||||||
"alex"
|
|
||||||
],
|
|
||||||
"localAliases": [],
|
|
||||||
"permissions": {
|
|
||||||
"read": true,
|
|
||||||
"write": true,
|
|
||||||
"owner": true
|
|
||||||
}
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### DeleteKey `DELETE /v0/key?id=<acces key id>`
|
|
||||||
|
|
||||||
Deletes an API access key.
|
|
||||||
|
|
||||||
#### UpdateKey `POST /v0/key?id=<acces key id>`
|
|
||||||
|
|
||||||
Updates information about the specified API access key.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"name": "NameOfMyKey",
|
|
||||||
"allow": {
|
|
||||||
"createBucket": true,
|
|
||||||
},
|
|
||||||
"deny": {}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
All fields (`name`, `allow` and `deny`) are optionnal.
|
|
||||||
If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed.
|
|
||||||
The possible flags in `allow` and `deny` are: `createBucket`.
|
|
||||||
|
|
||||||
|
|
||||||
### Bucket operations
|
|
||||||
|
|
||||||
#### ListBuckets `GET /v0/bucket`
|
|
||||||
|
|
||||||
Returns all storage buckets in the cluster.
|
|
||||||
|
|
||||||
Example response:
|
|
||||||
|
|
||||||
```json
|
|
||||||
[
|
|
||||||
{
|
|
||||||
"id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033",
|
|
||||||
"globalAliases": [
|
|
||||||
"test2"
|
|
||||||
],
|
|
||||||
"localAliases": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95",
|
|
||||||
"globalAliases": [
|
|
||||||
"alex"
|
|
||||||
],
|
|
||||||
"localAliases": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995",
|
|
||||||
"globalAliases": [
|
|
||||||
"test3"
|
|
||||||
],
|
|
||||||
"localAliases": []
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
|
||||||
"globalAliases": [],
|
|
||||||
"localAliases": [
|
|
||||||
{
|
|
||||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
|
||||||
"alias": "test"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
#### GetBucketInfo `GET /v0/bucket?id=<bucket id>`
|
|
||||||
#### GetBucketInfo `GET /v0/bucket?globalAlias=<alias>`
|
|
||||||
|
|
||||||
Returns information about the requested storage bucket.
|
|
||||||
|
|
||||||
If `id` is set, the bucket is looked up using its exact identifier.
|
|
||||||
If `globalAlias` is set, the bucket is looked up using its global alias.
|
|
||||||
(both are fast)
|
|
||||||
|
|
||||||
Example response:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"id": "afa8f0a22b40b1247ccd0affb869b0af5cff980924a20e4b5e0720a44deb8d39",
|
|
||||||
"globalAliases": [],
|
|
||||||
"websiteAccess": false,
|
|
||||||
"websiteConfig": null,
|
|
||||||
"keys": [
|
|
||||||
{
|
|
||||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
|
||||||
"name": "Imported key",
|
|
||||||
"permissions": {
|
|
||||||
"read": true,
|
|
||||||
"write": true,
|
|
||||||
"owner": true
|
|
||||||
},
|
|
||||||
"bucketLocalAliases": [
|
|
||||||
"debug"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"objects": 14827,
|
|
||||||
"bytes": 13189855625,
|
|
||||||
"unfinshedUploads": 0,
|
|
||||||
"quotas": {
|
|
||||||
"maxSize": null,
|
|
||||||
"maxObjects": null
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
#### CreateBucket `POST /v0/bucket`
|
|
||||||
|
|
||||||
Creates a new storage bucket.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"globalAlias": "NameOfMyBucket"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
OR
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"localAlias": {
|
|
||||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
|
||||||
"alias": "NameOfMyBucket",
|
|
||||||
"allow": {
|
|
||||||
"read": true,
|
|
||||||
"write": true,
|
|
||||||
"owner": false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
OR
|
|
||||||
|
|
||||||
```json
|
|
||||||
{}
|
|
||||||
```
|
|
||||||
|
|
||||||
Creates a new bucket, either with a global alias, a local one,
|
|
||||||
or no alias at all.
|
|
||||||
|
|
||||||
Technically, you can also specify both `globalAlias` and `localAlias` and that would create
|
|
||||||
two aliases, but I don't see why you would want to do that.
|
|
||||||
|
|
||||||
#### DeleteBucket `DELETE /v0/bucket?id=<bucket id>`
|
|
||||||
|
|
||||||
Deletes a storage bucket. A bucket cannot be deleted if it is not empty.
|
|
||||||
|
|
||||||
Warning: this will delete all aliases associated with the bucket!
|
|
||||||
|
|
||||||
#### UpdateBucket `PUT /v0/bucket?id=<bucket id>`
|
|
||||||
|
|
||||||
Updates configuration of the given bucket.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"websiteAccess": {
|
|
||||||
"enabled": true,
|
|
||||||
"indexDocument": "index.html",
|
|
||||||
"errorDocument": "404.html"
|
|
||||||
},
|
|
||||||
"quotas": {
|
|
||||||
"maxSize": 19029801,
|
|
||||||
"maxObjects": null,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
All fields (`websiteAccess` and `quotas`) are optionnal.
|
|
||||||
If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
|
|
||||||
|
|
||||||
In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.
|
|
||||||
The field `errorDocument` is optional, if no error document is set a generic
|
|
||||||
error message is displayed when errors happen. Conversely, if `enabled` is
|
|
||||||
`false`, neither `indexDocument` nor `errorDocument` must be specified.
|
|
||||||
|
|
||||||
In `quotas`: new values of `maxSize` and `maxObjects` must both be specified, or set to `null`
|
|
||||||
to remove the quotas. An absent value will be considered the same as a `null`. It is not possible
|
|
||||||
to change only one of the two quotas.
|
|
||||||
|
|
||||||
### Operations on permissions for keys on buckets
|
|
||||||
|
|
||||||
#### BucketAllowKey `POST /v0/bucket/allow`
|
|
||||||
|
|
||||||
Allows a key to do read/write/owner operations on a bucket.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
|
||||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
|
||||||
"permissions": {
|
|
||||||
"read": true,
|
|
||||||
"write": true,
|
|
||||||
"owner": true
|
|
||||||
},
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Flags in `permissions` which have the value `true` will be activated.
|
|
||||||
Other flags will remain unchanged.
|
|
||||||
|
|
||||||
#### BucketDenyKey `POST /v0/bucket/deny`
|
|
||||||
|
|
||||||
Denies a key from doing read/write/owner operations on a bucket.
|
|
||||||
|
|
||||||
Request body format:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b",
|
|
||||||
"accessKeyId": "GK31c2f218a2e44f485b94239e",
|
|
||||||
"permissions": {
|
|
||||||
"read": false,
|
|
||||||
"write": false,
|
|
||||||
"owner": true
|
|
||||||
},
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
Flags in `permissions` which have the value `true` will be deactivated.
|
|
||||||
Other flags will remain unchanged.
|
|
||||||
|
|
||||||
|
|
||||||
### Operations on bucket aliases
|
|
||||||
|
|
||||||
#### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
|
|
||||||
|
|
||||||
Empty body. Creates a global alias for a bucket.
|
|
||||||
|
|
||||||
#### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=<bucket id>&alias=<global alias>`
|
|
||||||
|
|
||||||
Removes a global alias for a bucket.
|
|
||||||
|
|
||||||
#### LocalAliasBucket `PUT /v0/bucket/alias/local?id=<bucket id>&accessKeyId=<access key ID>&alias=<local alias>`
|
|
||||||
|
|
||||||
Empty body. Creates a local alias for a bucket in the namespace of a specific access key.
|
|
||||||
|
|
||||||
#### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=<bucket id>&accessKeyId<access key ID>&alias=<local alias>`
|
|
||||||
|
|
||||||
Removes a local alias for a bucket in the namespace of a specific access key.
|
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
+++
|
+++
|
||||||
title = "Working Documents"
|
title = "Working Documents"
|
||||||
weight = 7
|
weight = 8
|
||||||
sort_by = "weight"
|
sort_by = "weight"
|
||||||
template = "documentation.html"
|
template = "documentation.html"
|
||||||
+++
|
+++
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
+++
|
+++
|
||||||
title = "Design draft (obsolete)"
|
title = "Design draft (obsolete)"
|
||||||
weight = 50
|
weight = 900
|
||||||
+++
|
+++
|
||||||
|
|
||||||
**WARNING: this documentation is a design draft which was written before Garage's actual implementation.
|
**WARNING: this documentation is a design draft which was written before Garage's actual implementation.
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
+++
|
+++
|
||||||
title = "Load balancing data (obsolete)"
|
title = "Load balancing data (obsolete)"
|
||||||
weight = 60
|
weight = 910
|
||||||
+++
|
+++
|
||||||
|
|
||||||
**This is being yet improved in release 0.5. The working document has not been updated yet, it still only applies to Garage 0.2 through 0.4.**
|
**This is being yet improved in release 0.5. The working document has not been updated yet, it still only applies to Garage 0.2 through 0.4.**
|
||||||
|
|
75
doc/book/working-documents/testing-strategy.md
Normal file
75
doc/book/working-documents/testing-strategy.md
Normal file
|
@ -0,0 +1,75 @@
|
||||||
|
+++
|
||||||
|
title = "Testing strategy"
|
||||||
|
weight = 30
|
||||||
|
+++
|
||||||
|
|
||||||
|
|
||||||
|
## Testing Garage
|
||||||
|
|
||||||
|
Currently, we have the following tests:
|
||||||
|
|
||||||
|
- some unit tests spread around the codebase
|
||||||
|
- integration tests written in Rust (`src/garage/test`) to check that Garage operations perform correctly
|
||||||
|
- integration test for compatibility with external tools (`script/test-smoke.sh`)
|
||||||
|
|
||||||
|
We have also tried `minio/mint` but it fails a lot and for now we haven't gotten a lot from it.
|
||||||
|
|
||||||
|
In the future:
|
||||||
|
|
||||||
|
1. We'd like to have a systematic way of testing with `minio/mint`,
|
||||||
|
it would add value to Garage by providing a compatibility score and reference that can be trusted.
|
||||||
|
2. We'd also like to do testing with Jepsen in some way.
|
||||||
|
|
||||||
|
## How to instrument Garagae
|
||||||
|
|
||||||
|
We should try to test in least invasive ways, i.e. minimize the impact of the testing framework on Garage's source code. This means for example:
|
||||||
|
|
||||||
|
- Not abstracting IO/nondeterminism in the source code
|
||||||
|
- Not making `garage` a shared library (launch using `execve`, it's perfectly fine)
|
||||||
|
|
||||||
|
Instead, we should focus on building a clean outer interface for the `garage` binary,
|
||||||
|
for example loading configuration using environnement variables instead of the configuration file if that's helpfull for writing the tests.
|
||||||
|
|
||||||
|
There are two reasons for this:
|
||||||
|
|
||||||
|
- Keep the soure code clean and focused
|
||||||
|
- Test something that is as close as possible as the true garage that will actually be running
|
||||||
|
|
||||||
|
Reminder: rules of simplicity, concerning changes to Garage's source code.
|
||||||
|
Always question what we are doing.
|
||||||
|
Never do anything just because it looks nice or because we "think" it might be usefull at some later point but without knowing precisely why/when.
|
||||||
|
Only do things that make perfect sense in the context of what we currently know.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
Testing is a research field on its own.
|
||||||
|
About testing distributed systems:
|
||||||
|
|
||||||
|
- [Jepsen](https://jepsen.io/) is a testing framework designed to test distributed systems. It can mock some part of the system like the time and the network.
|
||||||
|
- [FoundationDB Testing Approach](https://www.micahlerner.com/2021/06/12/foundationdb-a-distributed-unbundled-transactional-key-value-store.html#what-is-unique-about-foundationdbs-testing-framework). They chose to abstract "all sources of nondeterminism and communication are abstracted, including network, disk, time, and pseudo random number generator" to be able to run tests by simulating faults.
|
||||||
|
- [Testing Distributed Systems](https://asatarin.github.io/testing-distributed-systems/) - Curated list of resources on testing distributed systems
|
||||||
|
|
||||||
|
About S3 compatibility:
|
||||||
|
- [ceph/s3-tests](https://github.com/ceph/s3-tests)
|
||||||
|
- (deprecated) [minio/s3verify](https://blog.min.io/s3verify-a-simple-tool-to-verify-aws-s3-api-compatibility/)
|
||||||
|
- [minio/mint](https://github.com/minio/mint)
|
||||||
|
|
||||||
|
About benchmarking S3 (I think it is not necessarily very relevant for this iteration):
|
||||||
|
- [minio/warp](https://github.com/minio/warp)
|
||||||
|
- [wasabi-tech/s3-benchmark](https://github.com/wasabi-tech/s3-benchmark)
|
||||||
|
- [dvassallo/s3-benchmark](https://github.com/dvassallo/s3-benchmark)
|
||||||
|
- [intel-cloud/cosbench](https://github.com/intel-cloud/cosbench) - used by Ceph
|
||||||
|
|
||||||
|
Engineering blog posts:
|
||||||
|
- [Quincy @ Scale: A Tale of Three Large-Scale Clusters](https://ceph.io/en/news/blog/2022/three-large-scale-clusters/)
|
||||||
|
|
||||||
|
Interesting blog posts on the blog of the Sled database:
|
||||||
|
|
||||||
|
- <https://sled.rs/simulation.html>
|
||||||
|
- <https://sled.rs/perf.html>
|
||||||
|
|
||||||
|
Misc:
|
||||||
|
- [mutagen](https://github.com/llogiq/mutagen) - mutation testing is a way to assert our test quality by mutating the code and see if the mutation makes the tests fail
|
||||||
|
- [fuzzing](https://rust-fuzz.github.io/book/) - cargo supports fuzzing, it could be a way to test our software reliability in presence of garbage data.
|
||||||
|
|
||||||
|
|
|
@ -86,9 +86,11 @@ spec:
|
||||||
- name: meta
|
- name: meta
|
||||||
hostPath:
|
hostPath:
|
||||||
path: {{ .Values.persistence.meta.hostPath }}
|
path: {{ .Values.persistence.meta.hostPath }}
|
||||||
|
type: DirectoryOrCreate
|
||||||
- name: data
|
- name: data
|
||||||
hostPath:
|
hostPath:
|
||||||
path: {{ .Values.persistence.data.hostPath }}
|
path: {{ .Values.persistence.data.hostPath }}
|
||||||
|
type: DirectoryOrCreate
|
||||||
{{- end }}
|
{{- end }}
|
||||||
{{- else }}
|
{{- else }}
|
||||||
- name: meta
|
- name: meta
|
||||||
|
|
1053
script/telemetry/grafana-garage-dashboard-prometheus.json
Normal file
1053
script/telemetry/grafana-garage-dashboard-prometheus.json
Normal file
File diff suppressed because it is too large
Load diff
|
@ -210,7 +210,7 @@ async fn bucket_info_results(
|
||||||
.collect::<Vec<_>>(),
|
.collect::<Vec<_>>(),
|
||||||
objects: counters.get(OBJECTS).cloned().unwrap_or_default(),
|
objects: counters.get(OBJECTS).cloned().unwrap_or_default(),
|
||||||
bytes: counters.get(BYTES).cloned().unwrap_or_default(),
|
bytes: counters.get(BYTES).cloned().unwrap_or_default(),
|
||||||
unfinshed_uploads: counters
|
unfinished_uploads: counters
|
||||||
.get(UNFINISHED_UPLOADS)
|
.get(UNFINISHED_UPLOADS)
|
||||||
.cloned()
|
.cloned()
|
||||||
.unwrap_or_default(),
|
.unwrap_or_default(),
|
||||||
|
@ -234,7 +234,7 @@ struct GetBucketInfoResult {
|
||||||
keys: Vec<GetBucketInfoKey>,
|
keys: Vec<GetBucketInfoKey>,
|
||||||
objects: i64,
|
objects: i64,
|
||||||
bytes: i64,
|
bytes: i64,
|
||||||
unfinshed_uploads: i64,
|
unfinished_uploads: i64,
|
||||||
quotas: ApiBucketQuotas,
|
quotas: ApiBucketQuotas,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -30,9 +30,11 @@ garage_table = { version = "0.8.0", path = "../table" }
|
||||||
garage_util = { version = "0.8.0", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_web = { version = "0.8.0", path = "../web" }
|
garage_web = { version = "0.8.0", path = "../web" }
|
||||||
|
|
||||||
|
backtrace = "0.3"
|
||||||
bytes = "1.0"
|
bytes = "1.0"
|
||||||
bytesize = "1.1"
|
bytesize = "1.1"
|
||||||
timeago = "0.3"
|
timeago = "0.3"
|
||||||
|
parse_duration = "2.1"
|
||||||
hex = "0.4"
|
hex = "0.4"
|
||||||
tracing = { version = "0.1.30", features = ["log-always"] }
|
tracing = { version = "0.1.30", features = ["log-always"] }
|
||||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||||
|
|
|
@ -85,6 +85,9 @@ impl AdminRpcHandler {
|
||||||
BucketOperation::Deny(query) => self.handle_bucket_deny(query).await,
|
BucketOperation::Deny(query) => self.handle_bucket_deny(query).await,
|
||||||
BucketOperation::Website(query) => self.handle_bucket_website(query).await,
|
BucketOperation::Website(query) => self.handle_bucket_website(query).await,
|
||||||
BucketOperation::SetQuotas(query) => self.handle_bucket_set_quotas(query).await,
|
BucketOperation::SetQuotas(query) => self.handle_bucket_set_quotas(query).await,
|
||||||
|
BucketOperation::CleanupIncompleteUploads(query) => {
|
||||||
|
self.handle_bucket_cleanup_incomplete_uploads(query).await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -512,6 +515,42 @@ impl AdminRpcHandler {
|
||||||
)))
|
)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn handle_bucket_cleanup_incomplete_uploads(
|
||||||
|
&self,
|
||||||
|
query: &CleanupIncompleteUploadsOpt,
|
||||||
|
) -> Result<AdminRpc, Error> {
|
||||||
|
let mut bucket_ids = vec![];
|
||||||
|
for b in query.buckets.iter() {
|
||||||
|
bucket_ids.push(
|
||||||
|
self.garage
|
||||||
|
.bucket_helper()
|
||||||
|
.resolve_global_bucket_name(b)
|
||||||
|
.await?
|
||||||
|
.ok_or_bad_request(format!("Bucket not found: {}", b))?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let duration = parse_duration::parse::parse(&query.older_than)
|
||||||
|
.ok_or_bad_request("Invalid duration passed for --older-than parameter")?;
|
||||||
|
|
||||||
|
let mut ret = String::new();
|
||||||
|
for bucket in bucket_ids {
|
||||||
|
let count = self
|
||||||
|
.garage
|
||||||
|
.bucket_helper()
|
||||||
|
.cleanup_incomplete_uploads(&bucket, duration)
|
||||||
|
.await?;
|
||||||
|
writeln!(
|
||||||
|
&mut ret,
|
||||||
|
"Bucket {:?}: {} incomplete uploads aborted",
|
||||||
|
bucket, count
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(AdminRpc::Ok(ret))
|
||||||
|
}
|
||||||
|
|
||||||
async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result<AdminRpc, Error> {
|
async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result<AdminRpc, Error> {
|
||||||
match cmd {
|
match cmd {
|
||||||
KeyOperation::List => self.handle_list_keys().await,
|
KeyOperation::List => self.handle_list_keys().await,
|
||||||
|
|
|
@ -189,6 +189,10 @@ pub enum BucketOperation {
|
||||||
/// Set the quotas for this bucket
|
/// Set the quotas for this bucket
|
||||||
#[structopt(name = "set-quotas", version = garage_version())]
|
#[structopt(name = "set-quotas", version = garage_version())]
|
||||||
SetQuotas(SetQuotasOpt),
|
SetQuotas(SetQuotasOpt),
|
||||||
|
|
||||||
|
/// Clean up (abort) old incomplete multipart uploads
|
||||||
|
#[structopt(name = "cleanup-incomplete-uploads", version = garage_version())]
|
||||||
|
CleanupIncompleteUploads(CleanupIncompleteUploadsOpt),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||||
|
@ -290,6 +294,17 @@ pub struct SetQuotasOpt {
|
||||||
pub max_objects: Option<String>,
|
pub max_objects: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||||
|
pub struct CleanupIncompleteUploadsOpt {
|
||||||
|
/// Abort multipart uploads older than this value
|
||||||
|
#[structopt(long = "older-than", default_value = "1d")]
|
||||||
|
pub older_than: String,
|
||||||
|
|
||||||
|
/// Name of bucket(s) to clean up
|
||||||
|
#[structopt(required = true)]
|
||||||
|
pub buckets: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||||
pub enum KeyOperation {
|
pub enum KeyOperation {
|
||||||
/// List keys
|
/// List keys
|
||||||
|
|
|
@ -65,21 +65,6 @@ struct Opt {
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() {
|
async fn main() {
|
||||||
if std::env::var("RUST_LOG").is_err() {
|
|
||||||
std::env::set_var("RUST_LOG", "netapp=info,garage=info")
|
|
||||||
}
|
|
||||||
tracing_subscriber::fmt()
|
|
||||||
.with_writer(std::io::stderr)
|
|
||||||
.with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env())
|
|
||||||
.init();
|
|
||||||
sodiumoxide::init().expect("Unable to init sodiumoxide");
|
|
||||||
|
|
||||||
// Abort on panic (same behavior as in Go)
|
|
||||||
std::panic::set_hook(Box::new(|panic_info| {
|
|
||||||
error!("{}", panic_info.to_string());
|
|
||||||
std::process::abort();
|
|
||||||
}));
|
|
||||||
|
|
||||||
// Initialize version and features info
|
// Initialize version and features info
|
||||||
let features = &[
|
let features = &[
|
||||||
#[cfg(feature = "k2v")]
|
#[cfg(feature = "k2v")]
|
||||||
|
@ -108,12 +93,51 @@ async fn main() {
|
||||||
}
|
}
|
||||||
garage_util::version::init_features(features);
|
garage_util::version::init_features(features);
|
||||||
|
|
||||||
// Parse arguments
|
|
||||||
let version = format!(
|
let version = format!(
|
||||||
"{} [features: {}]",
|
"{} [features: {}]",
|
||||||
garage_util::version::garage_version(),
|
garage_util::version::garage_version(),
|
||||||
features.join(", ")
|
features.join(", ")
|
||||||
);
|
);
|
||||||
|
|
||||||
|
// Initialize panic handler that aborts on panic and shows a nice message.
|
||||||
|
// By default, Tokio continues runing normally when a task panics. We want
|
||||||
|
// to avoid this behavior in Garage as this would risk putting the process in an
|
||||||
|
// unknown/uncontrollable state. We prefer to exit the process and restart it
|
||||||
|
// from scratch, so that it boots back into a fresh, known state.
|
||||||
|
let panic_version_info = version.clone();
|
||||||
|
std::panic::set_hook(Box::new(move |panic_info| {
|
||||||
|
eprintln!("======== PANIC (internal Garage error) ========");
|
||||||
|
eprintln!("{}", panic_info);
|
||||||
|
eprintln!();
|
||||||
|
eprintln!("Panics are internal errors that Garage is unable to handle on its own.");
|
||||||
|
eprintln!("They can be caused by bugs in Garage's code, or by corrupted data in");
|
||||||
|
eprintln!("the node's storage. If you feel that this error is likely to be a bug");
|
||||||
|
eprintln!("in Garage, please report it on our issue tracker a the following address:");
|
||||||
|
eprintln!();
|
||||||
|
eprintln!(" https://git.deuxfleurs.fr/Deuxfleurs/garage/issues");
|
||||||
|
eprintln!();
|
||||||
|
eprintln!("Please include the last log messages and the the full backtrace below in");
|
||||||
|
eprintln!("your bug report, as well as any relevant information on the context in");
|
||||||
|
eprintln!("which Garage was running when this error occurred.");
|
||||||
|
eprintln!();
|
||||||
|
eprintln!("GARAGE VERSION: {}", panic_version_info);
|
||||||
|
eprintln!();
|
||||||
|
eprintln!("BACKTRACE:");
|
||||||
|
eprintln!("{:?}", backtrace::Backtrace::new());
|
||||||
|
std::process::abort();
|
||||||
|
}));
|
||||||
|
|
||||||
|
// Initialize logging as well as other libraries used in Garage
|
||||||
|
if std::env::var("RUST_LOG").is_err() {
|
||||||
|
std::env::set_var("RUST_LOG", "netapp=info,garage=info")
|
||||||
|
}
|
||||||
|
tracing_subscriber::fmt()
|
||||||
|
.with_writer(std::io::stderr)
|
||||||
|
.with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env())
|
||||||
|
.init();
|
||||||
|
sodiumoxide::init().expect("Unable to init sodiumoxide");
|
||||||
|
|
||||||
|
// Parse arguments and dispatch command line
|
||||||
let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
|
let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
|
||||||
|
|
||||||
let res = match opt.cmd {
|
let res = match opt.cmd {
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use garage_util::crdt::*;
|
use garage_util::crdt::*;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
use garage_util::error::{Error as GarageError, OkOrMessage};
|
use garage_util::error::{Error as GarageError, OkOrMessage};
|
||||||
|
@ -12,7 +14,7 @@ use crate::helper::error::*;
|
||||||
use crate::helper::key::KeyHelper;
|
use crate::helper::key::KeyHelper;
|
||||||
use crate::key_table::*;
|
use crate::key_table::*;
|
||||||
use crate::permission::BucketKeyPerm;
|
use crate::permission::BucketKeyPerm;
|
||||||
use crate::s3::object_table::ObjectFilter;
|
use crate::s3::object_table::*;
|
||||||
|
|
||||||
pub struct BucketHelper<'a>(pub(crate) &'a Garage);
|
pub struct BucketHelper<'a>(pub(crate) &'a Garage);
|
||||||
|
|
||||||
|
@ -472,4 +474,69 @@ impl<'a> BucketHelper<'a> {
|
||||||
|
|
||||||
Ok(true)
|
Ok(true)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
/// Deletes all incomplete multipart uploads that are older than a certain time.
|
||||||
|
/// Returns the number of uploads aborted
|
||||||
|
pub async fn cleanup_incomplete_uploads(
|
||||||
|
&self,
|
||||||
|
bucket_id: &Uuid,
|
||||||
|
older_than: Duration,
|
||||||
|
) -> Result<usize, Error> {
|
||||||
|
let older_than = now_msec() - older_than.as_millis() as u64;
|
||||||
|
|
||||||
|
let mut ret = 0usize;
|
||||||
|
let mut start = None;
|
||||||
|
|
||||||
|
loop {
|
||||||
|
let objects = self
|
||||||
|
.0
|
||||||
|
.object_table
|
||||||
|
.get_range(
|
||||||
|
bucket_id,
|
||||||
|
start,
|
||||||
|
Some(ObjectFilter::IsUploading),
|
||||||
|
1000,
|
||||||
|
EnumerationOrder::Forward,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
|
let abortions = objects
|
||||||
|
.iter()
|
||||||
|
.filter_map(|object| {
|
||||||
|
let aborted_versions = object
|
||||||
|
.versions()
|
||||||
|
.iter()
|
||||||
|
.filter(|v| v.is_uploading() && v.timestamp < older_than)
|
||||||
|
.map(|v| ObjectVersion {
|
||||||
|
state: ObjectVersionState::Aborted,
|
||||||
|
uuid: v.uuid,
|
||||||
|
timestamp: v.timestamp,
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if !aborted_versions.is_empty() {
|
||||||
|
Some(Object::new(
|
||||||
|
object.bucket_id,
|
||||||
|
object.key.clone(),
|
||||||
|
aborted_versions,
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
|
||||||
|
ret += abortions.len();
|
||||||
|
self.0.object_table.insert_many(abortions).await?;
|
||||||
|
|
||||||
|
if objects.len() < 1000 {
|
||||||
|
break;
|
||||||
|
} else {
|
||||||
|
start = Some(objects.last().unwrap().key.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(ret)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue