Compare commits
1 commit
672ba550a8
...
ab79a4aae2
| Author | SHA1 | Date | |
|---|---|---|---|
ab79a4aae2
|
38 changed files with 327 additions and 496 deletions
6
Cargo.lock
generated
6
Cargo.lock
generated
|
|
@ -1063,7 +1063,6 @@ dependencies = [
|
|||
"serde_json",
|
||||
"sha2 0.10.2",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
|
@ -1179,7 +1178,6 @@ dependencies = [
|
|||
"garage_db",
|
||||
"garage_rpc",
|
||||
"garage_util",
|
||||
"hex",
|
||||
"hexdump",
|
||||
"opentelemetry",
|
||||
"rand 0.8.5",
|
||||
|
|
@ -2034,9 +2032,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "netapp"
|
||||
version = "0.5.2"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ffe47ac46d3b2ce2f736a70865492df082e042eb2bfdddfca3b8dd66bd9469d"
|
||||
checksum = "14fa05cce4c65d4cc5277b2525c5054fca0e51db82416d9f43fb916575233774"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
|
|
|
|||
22
Cargo.nix
22
Cargo.nix
|
|
@ -780,7 +780,7 @@ in
|
|||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"; };
|
||||
dependencies = {
|
||||
${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.config == "aarch64-linux-android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
};
|
||||
});
|
||||
|
||||
|
|
@ -1428,7 +1428,7 @@ in
|
|||
garage_web = rustPackages."unknown".garage_web."0.8.0" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
|
||||
netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; };
|
||||
netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" { inherit profileName; };
|
||||
opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
opentelemetry_otlp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; };
|
||||
opentelemetry_prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; };
|
||||
|
|
@ -1506,7 +1506,6 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "serde_json" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.81" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "sha2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.2" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "tokio_stream" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "tracing" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "url" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.2.2" { inherit profileName; };
|
||||
};
|
||||
|
|
@ -1600,7 +1599,7 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_table" else null } = rustPackages."unknown".garage_table."0.8.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "garage_util" else null } = rustPackages."unknown".garage_util."0.8.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "hex" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_web" then "rmp_serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
|
||||
|
|
@ -1638,7 +1637,7 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "k8s_openapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "kube" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "sodiumoxide" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "openssl" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl."0.10.38" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "pnet_datalink" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; };
|
||||
|
|
@ -1667,7 +1666,6 @@ in
|
|||
garage_db = rustPackages."unknown".garage_db."0.8.0" { inherit profileName; };
|
||||
garage_rpc = rustPackages."unknown".garage_rpc."0.8.0" { inherit profileName; };
|
||||
garage_util = rustPackages."unknown".garage_util."0.8.0" { inherit profileName; };
|
||||
hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; };
|
||||
hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; };
|
||||
opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
|
||||
|
|
@ -1702,7 +1700,7 @@ in
|
|||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "http" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "hyper" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "lazy_static" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "netapp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; };
|
||||
${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client" then "rmp_serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; };
|
||||
|
|
@ -2823,11 +2821,11 @@ in
|
|||
};
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" = overridableMkRustCrate (profileName: rec {
|
||||
"registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.0" = overridableMkRustCrate (profileName: rec {
|
||||
name = "netapp";
|
||||
version = "0.5.2";
|
||||
version = "0.5.0";
|
||||
registry = "registry+https://github.com/rust-lang/crates.io-index";
|
||||
src = fetchCratesIo { inherit name version; sha256 = "4ffe47ac46d3b2ce2f736a70865492df082e042eb2bfdddfca3b8dd66bd9469d"; };
|
||||
src = fetchCratesIo { inherit name version; sha256 = "14fa05cce4c65d4cc5277b2525c5054fca0e51db82416d9f43fb916575233774"; };
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_util" || rootFeatures' ? "garage_web" || rootFeatures' ? "k2v-client") "default")
|
||||
(lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "opentelemetry")
|
||||
|
|
@ -3888,7 +3886,7 @@ in
|
|||
];
|
||||
dependencies = {
|
||||
${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; };
|
||||
untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; };
|
||||
${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; };
|
||||
|
|
@ -5598,7 +5596,7 @@ in
|
|||
[ "default" ]
|
||||
];
|
||||
dependencies = {
|
||||
${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; };
|
||||
${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; };
|
||||
|
|
|
|||
|
|
@ -5,14 +5,12 @@ weight = 25
|
|||
|
||||
## Configuring a bucket for website access
|
||||
|
||||
There are three methods to expose buckets as website:
|
||||
There are two methods to expose buckets as website:
|
||||
|
||||
1. using the PutBucketWebsite S3 API call, which is allowed for access keys that have the owner permission bit set
|
||||
|
||||
2. from the Garage CLI, by an adminstrator of the cluster
|
||||
|
||||
3. using the Garage administration API
|
||||
|
||||
The `PutBucketWebsite` API endpoint [is documented](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketWebsite.html) in the official AWS docs.
|
||||
This endpoint can also be called [using `aws s3api`](https://docs.aws.amazon.com/cli/latest/reference/s3api/put-bucket-website.html) on the command line.
|
||||
The website configuration supported by Garage is only a subset of the possibilities on Amazon S3: redirections are not supported, only the index document and error document can be specified.
|
||||
|
|
|
|||
|
|
@ -20,76 +20,57 @@ sudo apt-get update
|
|||
sudo apt-get install build-essential
|
||||
```
|
||||
|
||||
## Building from source from the Gitea repository
|
||||
## Using source from the Gitea repository (recommended)
|
||||
|
||||
The primary location for Garage's source code is the
|
||||
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage),
|
||||
which contains all of the released versions as well as the code
|
||||
for the developpement of the next version.
|
||||
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage).
|
||||
|
||||
Clone the repository and enter it as follows:
|
||||
Clone the repository and build Garage with the following commands:
|
||||
|
||||
```bash
|
||||
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage.git
|
||||
cd garage
|
||||
cargo build
|
||||
```
|
||||
|
||||
If you wish to build a specific version of Garage, check out the corresponding tag. For instance:
|
||||
Be careful, as this will make a debug build of Garage, which will be extremely slow!
|
||||
To make a release build, invoke `cargo build --release` (this takes much longer).
|
||||
|
||||
The binaries built this way are found in `target/{debug,release}/garage`.
|
||||
|
||||
## Using source from `crates.io`
|
||||
|
||||
Garage's source code is published on `crates.io`, Rust's official package repository.
|
||||
This means you can simply ask `cargo` to download and build this source code for you:
|
||||
|
||||
```bash
|
||||
git tag # List available tags
|
||||
git checkout v0.8.0 # Change v0.8.0 with the version you wish to build
|
||||
cargo install garage
|
||||
```
|
||||
|
||||
Otherwise you will be building a developpement build from the `main` branch
|
||||
that includes all of the changes to be released in the next version.
|
||||
Be careful that such a build might be unstable or contain bugs,
|
||||
and could be incompatible with nodes that run stable versions of Garage.
|
||||
That's all, `garage` should be in `$HOME/.cargo/bin`.
|
||||
|
||||
Finally, build Garage with the following command:
|
||||
You can add this folder to your `$PATH` or copy the binary somewhere else on your system.
|
||||
For instance:
|
||||
|
||||
```bash
|
||||
cargo build --release
|
||||
sudo cp $HOME/.cargo/bin/garage /usr/local/bin/garage
|
||||
```
|
||||
|
||||
The binary built this way can now be found in `target/release/garage`.
|
||||
You may simply copy this binary to somewhere in your `$PATH` in order to
|
||||
have the `garage` command available in your shell, for instance:
|
||||
|
||||
```bash
|
||||
sudo cp target/release/garage /usr/local/bin/garage
|
||||
```
|
||||
## Selecting features to activate in your build
|
||||
|
||||
If you are planning to develop Garage,
|
||||
you might be interested in producing debug builds, which compile faster but run slower:
|
||||
this can be done by removing the `--release` flag, and the resulting build can then
|
||||
be found in `target/debug/garage`.
|
||||
|
||||
## List of available Cargo feature flags
|
||||
|
||||
Garage supports a number of compilation options in the form of Cargo feature flags,
|
||||
Garage supports a number of compilation options in the form of Cargo features,
|
||||
which can be used to provide builds adapted to your system and your use case.
|
||||
To produce a build with a given set of features, invoke the `cargo build` command
|
||||
as follows:
|
||||
The following features are available:
|
||||
|
||||
```bash
|
||||
# This will build the default feature set plus feature1, feature2 and feature3
|
||||
cargo build --release --features feature1,feature2,feature3
|
||||
# This will build ONLY feature1, feature2 and feature3
|
||||
cargo build --release --no-default-features \
|
||||
--features feature1,feature2,feature3
|
||||
```
|
||||
|
||||
The following feature flags are available in v0.8.0:
|
||||
|
||||
| Feature flag | Enabled | Description |
|
||||
| ------------ | ------- | ----------- |
|
||||
| `bundled-libs` | *by default* | Use bundled version of sqlite3, zstd, lmdb and libsodium |
|
||||
| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium<br>if available (exclusive with `bundled-libs`, build using<br>`cargo build --no-default-features --features system-libs`) |
|
||||
| `k2v` | optional | Enable the experimental K2V API (if used, all nodes on your<br>Garage cluster must have it enabled as well) |
|
||||
| `kubernetes-discovery` | optional | Enable automatic registration and discovery<br>of cluster nodes through the Kubernetes API |
|
||||
| `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API |
|
||||
| Feature | Enabled | Description |
|
||||
| ------- | ------- | ----------- |
|
||||
| `bundled-libs` | BY DEFAULT | Use bundled version of sqlite3, zstd, lmdb and libsodium |
|
||||
| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium if available (exclusive with `bundled-libs`, build using `cargo build --no-default-features --features system-libs`) |
|
||||
| `k2v` | optional | Enable the experimental K2V API (if used, all nodes on your Garage cluster must have it enabled as well) |
|
||||
| `kubernetes-discovery` | optional | Enable automatic registration and discovery of cluster nodes through the Kubernetes API |
|
||||
| `metrics` | BY DEFAULT | Enable collection of metrics in Prometheus format on the admin API |
|
||||
| `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
|
||||
| `sled` | *by default* | Enable using Sled to store Garage's metadata |
|
||||
| `sled` | BY DEFAULT | Enable using Sled to store Garage's metadata |
|
||||
| `lmdb` | optional | Enable using LMDB to store Garage's metadata |
|
||||
| `sqlite` | optional | Enable using Sqlite3 to store Garage's metadata |
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Benchmarks"
|
||||
weight = 40
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
With Garage, we wanted to build a software defined storage service that follow the [KISS principle](https://en.wikipedia.org/wiki/KISS_principle),
|
||||
|
|
|
|||
|
|
@ -1,13 +1,13 @@
|
|||
+++
|
||||
title = "Goals and use cases"
|
||||
weight = 10
|
||||
weight = 5
|
||||
+++
|
||||
|
||||
## Goals and non-goals
|
||||
|
||||
Garage is a lightweight geo-distributed data store that implements the
|
||||
[Amazon S3](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html)
|
||||
object storage protocol. It enables applications to store large blobs such
|
||||
object storage protocole. It enables applications to store large blobs such
|
||||
as pictures, video, images, documents, etc., in a redundant multi-node
|
||||
setting. S3 is versatile enough to also be used to publish a static
|
||||
website.
|
||||
|
|
|
|||
|
|
@ -20,49 +20,6 @@ In the meantime, you can find some information at the following links:
|
|||
- [an old design draft](@/documentation/working-documents/design-draft.md)
|
||||
|
||||
|
||||
## Request routing logic
|
||||
|
||||
Data retrieval requests to Garage endpoints (S3 API and websites) are resolved
|
||||
to an individual object in a bucket. Since objects are replicated to multiple nodes
|
||||
Garage must ensure consistency before answering the request.
|
||||
|
||||
### Using quorum to ensure consistency
|
||||
|
||||
Garage ensures consistency by attempting to establish a quorum with the
|
||||
data nodes responsible for the object. When a majority of the data nodes
|
||||
have provided metadata on a object Garage can then answer the request.
|
||||
|
||||
When a request arrives Garage will, assuming the recommended 3 replicas, perform the following actions:
|
||||
|
||||
- Make a request to the two preferred nodes for object metadata
|
||||
- Try the third node if one of the two initial requests fail
|
||||
- Check that the metadata from at least 2 nodes match
|
||||
- Check that the object hasn't been marked deleted
|
||||
- Answer the request with inline data from metadata if object is small enough
|
||||
- Or get data blocks from the preferred nodes and answer using the assembled object
|
||||
|
||||
Garage dynamically determines which nodes to query based on health, preference, and
|
||||
which nodes actually host a given data. Garage has no concept of "primary" so any
|
||||
healthy node with the data can be used as long as a quorum is reached for the metadata.
|
||||
|
||||
### Node health
|
||||
|
||||
Garage keeps a TCP session open to each node in the cluster and periodically pings them. If a connection
|
||||
cannot be established, or a node fails to answer a number of pings, the target node is marked as failed.
|
||||
Failed nodes are not used for quorum or other internal requests.
|
||||
|
||||
### Node preference
|
||||
|
||||
Garage prioritizes which nodes to query according to a few criteria:
|
||||
|
||||
- A node always prefers itself if it can answer the request
|
||||
- Then the node prioritizes nodes in the same zone
|
||||
- Finally the nodes with the lowest latency are prioritized
|
||||
|
||||
|
||||
For further reading on the cluster structure look at the [gateway](@/documentation/cookbook/gateways.md)
|
||||
and [cluster layout management](@/documentation/reference-manual/layout.md) pages.
|
||||
|
||||
## Garbage collection
|
||||
|
||||
A faulty garbage collection procedure has been the cause of
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Related work"
|
||||
weight = 50
|
||||
weight = 15
|
||||
+++
|
||||
|
||||
## Context
|
||||
|
|
|
|||
|
|
@ -9,15 +9,6 @@ Let's start your Garage journey!
|
|||
In this chapter, we explain how to deploy Garage as a single-node server
|
||||
and how to interact with it.
|
||||
|
||||
## What is Garage?
|
||||
|
||||
Before jumping in, you might be interested in reading the following pages:
|
||||
|
||||
- [Goals and use cases](@/documentation/design/goals.md)
|
||||
- [List of features](@/documentation/reference-manual/features.md)
|
||||
|
||||
## Scope of this tutorial
|
||||
|
||||
Our goal is to introduce you to Garage's workflows.
|
||||
Following this guide is recommended before moving on to
|
||||
[configuring a multi-node cluster](@/documentation/cookbook/real-world.md).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Administration API"
|
||||
weight = 60
|
||||
weight = 16
|
||||
+++
|
||||
|
||||
The Garage administration API is accessible through a dedicated server whose
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Garage CLI"
|
||||
weight = 30
|
||||
weight = 15
|
||||
+++
|
||||
|
||||
The Garage CLI is mostly self-documented. Make use of the `help` subcommand
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Configuration file format"
|
||||
weight = 20
|
||||
weight = 5
|
||||
+++
|
||||
|
||||
Here is an example `garage.toml` configuration file that illustrates all of the possible options:
|
||||
|
|
@ -10,6 +10,7 @@ metadata_dir = "/var/lib/garage/meta"
|
|||
data_dir = "/var/lib/garage/data"
|
||||
|
||||
block_size = 1048576
|
||||
block_manager_background_tranquility = 2
|
||||
|
||||
replication_mode = "3"
|
||||
|
||||
|
|
@ -86,6 +87,17 @@ files will remain available. This however means that chunks from existing files
|
|||
will not be deduplicated with chunks from newly uploaded files, meaning you
|
||||
might use more storage space that is optimally possible.
|
||||
|
||||
### `block_manager_background_tranquility`
|
||||
|
||||
This parameter tunes the activity of the background worker responsible for
|
||||
resyncing data blocks between nodes. The higher the tranquility value is set,
|
||||
the more the background worker will wait between iterations, meaning the load
|
||||
on the system (including network usage between nodes) will be reduced. The
|
||||
minimal value for this parameter is `0`, where the background worker will
|
||||
allways work at maximal throughput to resynchronize blocks. The default value
|
||||
is `2`, where the background worker will try to spend at most 1/3 of its time
|
||||
working, and 2/3 sleeping in order to reduce system load.
|
||||
|
||||
### `replication_mode`
|
||||
|
||||
Garage supports the following replication modes:
|
||||
|
|
|
|||
|
|
@ -1,125 +0,0 @@
|
|||
+++
|
||||
title = "List of Garage features"
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
|
||||
### S3 API
|
||||
|
||||
The main goal of Garage is to provide an object storage service that is compatible with the
|
||||
[S3 API](https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html) from Amazon Web Services.
|
||||
We try to adhere as strictly as possible to the semantics of the API as implemented by Amazon
|
||||
and other vendors such as Minio or CEPH.
|
||||
|
||||
Of course Garage does not implement the full span of API endpoints that AWS S3 does;
|
||||
the exact list of S3 features implemented by Garage can be found [on our S3 compatibility page](@/documentation/reference-manual/s3-compatibility.md).
|
||||
|
||||
### Geo-distribution
|
||||
|
||||
Garage allows you to store copies of your data in multiple geographical locations in order to maximize resilience
|
||||
to adverse events, such as network/power outages or hardware failures.
|
||||
This allows Garage to run very well even at home, using consumer-grade Internet connectivity
|
||||
(such as FTTH) and power, as long as cluster nodes can be spawned at several physical locations.
|
||||
Garage exploits knowledge of the capacity and physical location of each storage node to design
|
||||
a storage plan that best exploits the available storage capacity while satisfying the geo-distributed replication constraint.
|
||||
|
||||
To learn more about geo-distributed Garage clusters,
|
||||
read our documentation on [setting up a real-world deployment](@/documentation/cookbook/real-world.md).
|
||||
|
||||
### Standalone/self-contained
|
||||
|
||||
Garage is extremely simple to deploy, and does not depend on any external service to run.
|
||||
This makes setting up and administering storage clusters, we hope, as easy as it could be.
|
||||
|
||||
### Flexible topology
|
||||
|
||||
A Garage cluster can very easily evolve over time, as storage nodes are added or removed.
|
||||
Garage will automatically rebalance data between nodes as needed to ensure the desired number of copies.
|
||||
Read about cluster layout management [here](@/documentation/reference-manual/layout.md).
|
||||
|
||||
### No RAFT slowing you down
|
||||
|
||||
It might seem strange to tout the absence of something as a desirable feature,
|
||||
but this is in fact a very important point! Garage does not use RAFT or another
|
||||
consensus algorithm internally to order incoming requests: this means that all requests
|
||||
directed to a Garage cluster can be handled independently of one another instead
|
||||
of going through a central bottleneck (the leader node).
|
||||
As a consequence, requests can be handled much faster, even in cases where latency
|
||||
between cluster nodes is important (see our [benchmarks](@/documentation/design/benchmarks/index.md) for data on this).
|
||||
This is particularly usefull when nodes are far from one another and talk to one other through standard Internet connections.
|
||||
|
||||
### Several replication modes
|
||||
|
||||
Garage supports a variety of replication modes, with 1 copy, 2 copies or 3 copies of your data,
|
||||
and with various levels of consistency, in order to adapt to a variety of usage scenarios.
|
||||
Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication-mode)
|
||||
to select the replication mode best suited to your use case (hint: in most cases, `replication_mode = "3"` is what you want).
|
||||
|
||||
### Web server for static websites
|
||||
|
||||
A storage bucket can easily be configured to be served directly by Garage as a static web site.
|
||||
Domain names for multiple websites directly map to bucket names, making it easy to build
|
||||
a platform for your users to autonomously build and host their websites over Garage.
|
||||
Surprisingly, none of the other alternative S3 implementations we surveyed (such as Minio
|
||||
or CEPH) support publishing static websites from S3 buckets, a feature that is however
|
||||
directly inherited from S3 on AWS.
|
||||
Read more on our [dedicated documentation page](@/documentation/cookbook/exposing-websites.md).
|
||||
|
||||
### Bucket names as aliases
|
||||
|
||||
In Garage, a bucket may have several names, known as aliases.
|
||||
Aliases can easily be added and removed on demand:
|
||||
this allows to easily rename buckets if needed
|
||||
without having to copy all of their content, something that cannot be done on AWS.
|
||||
For buckets served as static websites, having multiple aliases for a bucket can allow
|
||||
exposing the same content under different domain names.
|
||||
|
||||
Garage also supports bucket aliases which are local to a single user:
|
||||
this allows different users to have different buckets with the same name, thus avoiding naming collisions.
|
||||
This can be helpfull for instance if you want to write an application that creates per-user buckets with always the same name.
|
||||
|
||||
This feature is totally invisible to S3 clients and does not break compatibility with AWS.
|
||||
|
||||
### Cluster administration API
|
||||
|
||||
Garage provides a fully-fledged REST API to administer your cluster programatically.
|
||||
Functionnality included in the admin API include: setting up and monitoring
|
||||
cluster nodes, managing access credentials, and managing storage buckets and bucket aliases.
|
||||
A full reference of the administration API is available [here](@/documentation/reference-manual/admin-api.md).
|
||||
|
||||
### Metrics and traces
|
||||
|
||||
Garage makes some internal metrics available in the Prometheus data format,
|
||||
which allows you to build interactive dashboards to visualize the load and internal state of your storage cluster.
|
||||
|
||||
For developpers and performance-savvy administrators,
|
||||
Garage also supports exporting traces of what it does internally in OpenTelemetry format.
|
||||
This allows to monitor the time spent at various steps of the processing of requests,
|
||||
in order to detect potential performance bottlenecks.
|
||||
|
||||
### Kubernetes and Nomad integrations
|
||||
|
||||
Garage can automatically discover other nodes in the cluster thanks to integration
|
||||
with orchestrators such as Kubernetes and Nomad (when used with Consul).
|
||||
This eases the configuration of your cluster as it removes one step where nodes need
|
||||
to be manually connected to one another.
|
||||
|
||||
### Support for changing IP addresses
|
||||
|
||||
As long as all of your nodes don't thange their IP address at the same time,
|
||||
Garage should be able to tolerate nodes with changing/dynamic IP addresses,
|
||||
as nodes will regularly exchange the IP addresses of their peers and try to
|
||||
reconnect using newer addresses when existing connections are broken.
|
||||
|
||||
### K2V API (experimental)
|
||||
|
||||
As part of an ongoing research project, Garage can expose an experimental key/value storage API called K2V.
|
||||
K2V is made for the storage and retrieval of many small key/value pairs that need to be processed in bulk.
|
||||
This completes the S3 API with an alternative that can be used to easily store and access metadata
|
||||
related to objects stored in an S3 bucket.
|
||||
|
||||
In the context of our research project, [Aérogramme](https://aerogramme.deuxfleurs.fr),
|
||||
K2V is used to provide metadata and log storage for operations on encrypted e-mail storage.
|
||||
|
||||
Learn more on the specification of K2V [here](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md)
|
||||
and on how to enable it in Garage [here](@/documentation/reference-manual/k2v.md).
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "K2V"
|
||||
weight = 70
|
||||
weight = 30
|
||||
+++
|
||||
|
||||
Starting with version 0.7.2, Garage introduces an optionnal feature, K2V,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Cluster layout management"
|
||||
weight = 50
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
The cluster layout in Garage is a table that assigns to each node a role in
|
||||
|
|
|
|||
45
doc/book/reference-manual/routing.md
Normal file
45
doc/book/reference-manual/routing.md
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
+++
|
||||
title = "Request routing logic"
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
Data retrieval requests to Garage endpoints (S3 API and websites) are resolved
|
||||
to an individual object in a bucket. Since objects are replicated to multiple nodes
|
||||
Garage must ensure consistency before answering the request.
|
||||
|
||||
## Using quorum to ensure consistency
|
||||
|
||||
Garage ensures consistency by attempting to establish a quorum with the
|
||||
data nodes responsible for the object. When a majority of the data nodes
|
||||
have provided metadata on a object Garage can then answer the request.
|
||||
|
||||
When a request arrives Garage will, assuming the recommended 3 replicas, perform the following actions:
|
||||
|
||||
- Make a request to the two preferred nodes for object metadata
|
||||
- Try the third node if one of the two initial requests fail
|
||||
- Check that the metadata from at least 2 nodes match
|
||||
- Check that the object hasn't been marked deleted
|
||||
- Answer the request with inline data from metadata if object is small enough
|
||||
- Or get data blocks from the preferred nodes and answer using the assembled object
|
||||
|
||||
Garage dynamically determines which nodes to query based on health, preference, and
|
||||
which nodes actually host a given data. Garage has no concept of "primary" so any
|
||||
healthy node with the data can be used as long as a quorum is reached for the metadata.
|
||||
|
||||
## Node health
|
||||
|
||||
Garage keeps a TCP session open to each node in the cluster and periodically pings them. If a connection
|
||||
cannot be established, or a node fails to answer a number of pings, the target node is marked as failed.
|
||||
Failed nodes are not used for quorum or other internal requests.
|
||||
|
||||
## Node preference
|
||||
|
||||
Garage prioritizes which nodes to query according to a few criteria:
|
||||
|
||||
- A node always prefers itself if it can answer the request
|
||||
- Then the node prioritizes nodes in the same zone
|
||||
- Finally the nodes with the lowest latency are prioritized
|
||||
|
||||
|
||||
For further reading on the cluster structure look at the [gateway](@/documentation/cookbook/gateways.md)
|
||||
and [cluster layout management](@/documentation/reference-manual/layout.md) pages.
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "S3 Compatibility status"
|
||||
weight = 40
|
||||
weight = 20
|
||||
+++
|
||||
|
||||
## DISCLAIMER
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Design draft (obsolete)"
|
||||
weight = 50
|
||||
title = "Design draft"
|
||||
weight = 25
|
||||
+++
|
||||
|
||||
**WARNING: this documentation is a design draft which was written before Garage's actual implementation.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
+++
|
||||
title = "Load balancing data (obsolete)"
|
||||
weight = 60
|
||||
title = "Load balancing data"
|
||||
weight = 10
|
||||
+++
|
||||
|
||||
**This is being yet improved in release 0.5. The working document has not been updated yet, it still only applies to Garage 0.2 through 0.4.**
|
||||
|
|
|
|||
|
|
@ -38,7 +38,6 @@ futures = "0.3"
|
|||
futures-util = "0.3"
|
||||
pin-project = "1.0"
|
||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
tokio-stream = "0.1"
|
||||
|
||||
form_urlencoded = "1.0.0"
|
||||
http = "0.2"
|
||||
|
|
|
|||
|
|
@ -64,13 +64,14 @@ pub async fn handle_delete(
|
|||
bucket_id: Uuid,
|
||||
key: &str,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
match handle_delete_internal(&garage, bucket_id, key).await {
|
||||
Ok(_) | Err(Error::NoSuchKey) => Ok(Response::builder()
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.body(Body::from(vec![]))
|
||||
.unwrap()),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
let (_deleted_version, delete_marker_version) =
|
||||
handle_delete_internal(&garage, bucket_id, key).await?;
|
||||
|
||||
Ok(Response::builder()
|
||||
.header("x-amz-version-id", hex::encode(delete_marker_version))
|
||||
.status(StatusCode::NO_CONTENT)
|
||||
.body(Body::from(vec![]))
|
||||
.unwrap())
|
||||
}
|
||||
|
||||
pub async fn handle_delete_objects(
|
||||
|
|
|
|||
|
|
@ -2,19 +2,16 @@
|
|||
use std::sync::Arc;
|
||||
use std::time::{Duration, UNIX_EPOCH};
|
||||
|
||||
use futures::future;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use futures::stream::*;
|
||||
use http::header::{
|
||||
ACCEPT_RANGES, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, IF_MODIFIED_SINCE,
|
||||
IF_NONE_MATCH, LAST_MODIFIED, RANGE,
|
||||
};
|
||||
use hyper::{Body, Request, Response, StatusCode};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
use garage_rpc::rpc_helper::{netapp::stream::ByteStream, OrderTag};
|
||||
use garage_table::EmptyKey;
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::OkOrMessage;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::s3::object_table::*;
|
||||
|
|
@ -245,56 +242,43 @@ pub async fn handle_get(
|
|||
Ok(resp_builder.body(body)?)
|
||||
}
|
||||
ObjectVersionData::FirstBlock(_, first_block_hash) => {
|
||||
let (tx, rx) = mpsc::channel(2);
|
||||
|
||||
let order_stream = OrderTag::stream();
|
||||
let first_block_hash = *first_block_hash;
|
||||
let version_uuid = last_v.uuid;
|
||||
|
||||
tokio::spawn(async move {
|
||||
match async {
|
||||
let garage2 = garage.clone();
|
||||
let version_fut = tokio::spawn(async move {
|
||||
garage2.version_table.get(&version_uuid, &EmptyKey).await
|
||||
});
|
||||
let read_first_block = garage
|
||||
.block_manager
|
||||
.rpc_get_block_streaming(first_block_hash, Some(order_stream.order(0)));
|
||||
let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);
|
||||
|
||||
let stream_block_0 = garage
|
||||
.block_manager
|
||||
.rpc_get_block_streaming(&first_block_hash, Some(order_stream.order(0)))
|
||||
.await?;
|
||||
tx.send(stream_block_0)
|
||||
.await
|
||||
.ok_or_message("channel closed")?;
|
||||
let (first_block_stream, version) =
|
||||
futures::try_join!(read_first_block, get_next_blocks)?;
|
||||
let version = version.ok_or(Error::NoSuchKey)?;
|
||||
|
||||
let version = version_fut.await.unwrap()?.ok_or(Error::NoSuchKey)?;
|
||||
for (i, (_, vb)) in version.blocks.items().iter().enumerate().skip(1) {
|
||||
let stream_block_i = garage
|
||||
.block_manager
|
||||
.rpc_get_block_streaming(&vb.hash, Some(order_stream.order(i as u64)))
|
||||
.await?;
|
||||
tx.send(stream_block_i)
|
||||
.await
|
||||
.ok_or_message("channel closed")?;
|
||||
let mut blocks = version
|
||||
.blocks
|
||||
.items()
|
||||
.iter()
|
||||
.map(|(_, vb)| (vb.hash, None))
|
||||
.collect::<Vec<_>>();
|
||||
blocks[0].1 = Some(first_block_stream);
|
||||
|
||||
let body_stream = futures::stream::iter(blocks)
|
||||
.enumerate()
|
||||
.map(move |(i, (hash, stream_opt))| {
|
||||
let garage = garage.clone();
|
||||
async move {
|
||||
if let Some(stream) = stream_opt {
|
||||
stream
|
||||
} else {
|
||||
garage
|
||||
.block_manager
|
||||
.rpc_get_block_streaming(&hash, Some(order_stream.order(i as u64)))
|
||||
.await
|
||||
.unwrap_or_else(|e| error_stream(i, e))
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<(), Error>(())
|
||||
}
|
||||
.await
|
||||
{
|
||||
Ok(()) => (),
|
||||
Err(e) => {
|
||||
let err = std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
format!("Error while getting object data: {}", e),
|
||||
);
|
||||
let _ = tx
|
||||
.send(Box::pin(stream::once(future::ready(Err(err)))))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let body_stream = tokio_stream::wrappers::ReceiverStream::new(rx).flatten();
|
||||
})
|
||||
.buffered(2)
|
||||
.flatten();
|
||||
|
||||
let body = hyper::body::Body::wrap_stream(body_stream);
|
||||
Ok(resp_builder.body(body)?)
|
||||
|
|
|
|||
|
|
@ -41,6 +41,9 @@ use crate::resync::*;
|
|||
/// Size under which data will be stored inlined in database instead of as files
|
||||
pub const INLINE_THRESHOLD: usize = 3072;
|
||||
|
||||
// Timeout for RPCs that read and write blocks to remote nodes
|
||||
pub(crate) const BLOCK_RW_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
// The delay between the moment when the reference counter
|
||||
// drops to zero, and the moment where we allow ourselves
|
||||
// to delete the block locally.
|
||||
|
|
@ -180,7 +183,7 @@ impl BlockManager {
|
|||
};
|
||||
return Ok((header, stream));
|
||||
}
|
||||
_ = tokio::time::sleep(self.system.rpc.rpc_timeout()) => {
|
||||
_ = tokio::time::sleep(BLOCK_RW_TIMEOUT) => {
|
||||
debug!("Node {:?} didn't return block in time, trying next.", node);
|
||||
}
|
||||
};
|
||||
|
|
@ -232,7 +235,7 @@ impl BlockManager {
|
|||
}
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep(self.system.rpc.rpc_timeout()) => {
|
||||
_ = tokio::time::sleep(BLOCK_RW_TIMEOUT) => {
|
||||
debug!("Node {:?} didn't return block in time, trying next.", node);
|
||||
}
|
||||
};
|
||||
|
|
@ -297,7 +300,8 @@ impl BlockManager {
|
|||
&who[..],
|
||||
put_block_rpc,
|
||||
RequestStrategy::with_priority(PRIO_NORMAL | PRIO_SECONDARY)
|
||||
.with_quorum(self.replication.write_quorum()),
|
||||
.with_quorum(self.replication.write_quorum())
|
||||
.with_timeout(BLOCK_RW_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -332,10 +336,7 @@ impl BlockManager {
|
|||
// we will fecth it from someone.
|
||||
let this = self.clone();
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = this
|
||||
.resync
|
||||
.put_to_resync(&hash, 2 * this.system.rpc.rpc_timeout())
|
||||
{
|
||||
if let Err(e) = this.resync.put_to_resync(&hash, 2 * BLOCK_RW_TIMEOUT) {
|
||||
error!("Block {:?} could not be put in resync queue: {}.", hash, e);
|
||||
}
|
||||
});
|
||||
|
|
@ -443,8 +444,7 @@ impl BlockManager {
|
|||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
// Not found but maybe we should have had it ??
|
||||
self.resync
|
||||
.put_to_resync(hash, 2 * self.system.rpc.rpc_timeout())?;
|
||||
self.resync.put_to_resync(hash, 2 * BLOCK_RW_TIMEOUT)?;
|
||||
return Err(Into::into(e));
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -33,6 +33,14 @@ use garage_table::replication::TableReplication;
|
|||
|
||||
use crate::manager::*;
|
||||
|
||||
// Timeout for RPCs that ask other nodes whether they need a copy
|
||||
// of a given block before we delete it locally
|
||||
// The timeout here is relatively low because we don't want to block
|
||||
// the entire resync loop when some nodes are not responding.
|
||||
// Nothing will be deleted if the nodes don't answer the queries,
|
||||
// we will just retry later.
|
||||
const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
// The delay between the time where a resync operation fails
|
||||
// and the time when it is retried, with exponential backoff
|
||||
// (multiplied by 2, 4, 8, 16, etc. for every consecutive failure).
|
||||
|
|
@ -338,7 +346,8 @@ impl BlockResyncManager {
|
|||
&manager.endpoint,
|
||||
&who,
|
||||
BlockRpc::NeedBlockQuery(*hash),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_timeout(NEED_BLOCK_QUERY_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -385,7 +394,8 @@ impl BlockResyncManager {
|
|||
&need_nodes[..],
|
||||
put_block_message,
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_quorum(need_nodes.len()),
|
||||
.with_quorum(need_nodes.len())
|
||||
.with_timeout(BLOCK_RW_TIMEOUT),
|
||||
)
|
||||
.await
|
||||
.err_context("PutBlock RPC")?;
|
||||
|
|
|
|||
|
|
@ -162,13 +162,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
|
|||
} else {
|
||||
let node_id = garage_rpc::system::read_node_id(&config.as_ref().unwrap().metadata_dir)
|
||||
.err_context(READ_KEY_ERROR)?;
|
||||
if let Some(a) = config.as_ref().and_then(|c| c.rpc_public_addr.as_ref()) {
|
||||
use std::net::ToSocketAddrs;
|
||||
let a = a
|
||||
.to_socket_addrs()
|
||||
.ok_or_message("unable to resolve rpc_public_addr specified in config file")?
|
||||
.next()
|
||||
.ok_or_message("unable to resolve rpc_public_addr specified in config file")?;
|
||||
if let Some(a) = config.as_ref().and_then(|c| c.rpc_public_addr) {
|
||||
(node_id, a)
|
||||
} else {
|
||||
let default_addr = SocketAddr::new(
|
||||
|
|
|
|||
|
|
@ -263,13 +263,4 @@ async fn test_deleteobject() {
|
|||
.unwrap();
|
||||
|
||||
assert!(l.contents.is_none());
|
||||
|
||||
// Deleting a non-existing object shouldn't be a problem
|
||||
ctx.client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("l-0")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ pub fn make_node_id(node_id: Uuid) -> K2VNodeId {
|
|||
u64::from_be_bytes(tmp)
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Debug, Serialize, Deserialize)]
|
||||
#[derive(PartialEq, Debug, Serialize, Deserialize)]
|
||||
pub struct CausalContext {
|
||||
pub vector_clock: BTreeMap<K2VNodeId, u64>,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ pub const CONFLICTS: &str = "conflicts";
|
|||
pub const VALUES: &str = "values";
|
||||
pub const BYTES: &str = "bytes";
|
||||
|
||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct K2VItem {
|
||||
pub partition: K2VItemPartition,
|
||||
pub sort_key: String,
|
||||
|
|
@ -25,19 +25,19 @@ pub struct K2VItem {
|
|||
items: BTreeMap<K2VNodeId, DvvsEntry>,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, Hash)]
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize, Hash, Eq)]
|
||||
pub struct K2VItemPartition {
|
||||
pub bucket_id: Uuid,
|
||||
pub partition_key: String,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
struct DvvsEntry {
|
||||
t_discard: u64,
|
||||
values: Vec<(u64, DvvsValue)>,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub enum DvvsValue {
|
||||
Value(#[serde(with = "serde_bytes")] Vec<u8>),
|
||||
Deleted,
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ use garage_rpc::system::System;
|
|||
use garage_rpc::*;
|
||||
|
||||
use garage_table::replication::{TableReplication, TableShardedReplication};
|
||||
use garage_table::table::TABLE_RPC_TIMEOUT;
|
||||
use garage_table::{PartitionKey, Table};
|
||||
|
||||
use crate::k2v::causality::*;
|
||||
|
|
@ -116,6 +117,7 @@ impl K2VRpcHandler {
|
|||
}),
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(1)
|
||||
.with_timeout(TABLE_RPC_TIMEOUT)
|
||||
.interrupt_after_quorum(true),
|
||||
)
|
||||
.await?;
|
||||
|
|
@ -167,6 +169,7 @@ impl K2VRpcHandler {
|
|||
K2VRpc::InsertManyItems(items),
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(1)
|
||||
.with_timeout(TABLE_RPC_TIMEOUT)
|
||||
.interrupt_after_quorum(true),
|
||||
)
|
||||
.await?;
|
||||
|
|
@ -202,23 +205,22 @@ impl K2VRpcHandler {
|
|||
.replication
|
||||
.write_nodes(&poll_key.partition.hash());
|
||||
|
||||
let rpc = self.system.rpc.try_call_many(
|
||||
&self.endpoint,
|
||||
&nodes[..],
|
||||
K2VRpc::PollItem {
|
||||
key: poll_key,
|
||||
causal_context,
|
||||
timeout_msec,
|
||||
},
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(self.item_table.data.replication.read_quorum())
|
||||
.without_timeout(),
|
||||
);
|
||||
let timeout_duration = Duration::from_millis(timeout_msec) + self.system.rpc.rpc_timeout();
|
||||
let resps = select! {
|
||||
r = rpc => r?,
|
||||
_ = tokio::time::sleep(timeout_duration) => return Ok(None),
|
||||
};
|
||||
let resps = self
|
||||
.system
|
||||
.rpc
|
||||
.try_call_many(
|
||||
&self.endpoint,
|
||||
&nodes[..],
|
||||
K2VRpc::PollItem {
|
||||
key: poll_key,
|
||||
causal_context,
|
||||
timeout_msec,
|
||||
},
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(self.item_table.data.replication.read_quorum())
|
||||
.with_timeout(Duration::from_millis(timeout_msec) + TABLE_RPC_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut resp: Option<K2VItem> = None;
|
||||
for v in resps {
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
|
|||
tokio-stream = { version = "0.1", features = ["net"] }
|
||||
opentelemetry = "0.17"
|
||||
|
||||
netapp = { version = "0.5.2", features = ["telemetry"] }
|
||||
netapp = { version = "0.5.0", features = ["telemetry"] }
|
||||
|
||||
hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] }
|
||||
|
||||
|
|
|
|||
|
|
@ -1,18 +1,31 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use opentelemetry::{global, metrics::*};
|
||||
use tokio::sync::Semaphore;
|
||||
|
||||
/// TableMetrics reference all counter used for metrics
|
||||
pub struct RpcMetrics {
|
||||
pub(crate) _rpc_available_permits: ValueObserver<u64>,
|
||||
|
||||
pub(crate) rpc_counter: Counter<u64>,
|
||||
pub(crate) rpc_timeout_counter: Counter<u64>,
|
||||
pub(crate) rpc_netapp_error_counter: Counter<u64>,
|
||||
pub(crate) rpc_garage_error_counter: Counter<u64>,
|
||||
|
||||
pub(crate) rpc_duration: ValueRecorder<f64>,
|
||||
pub(crate) rpc_queueing_time: ValueRecorder<f64>,
|
||||
}
|
||||
impl RpcMetrics {
|
||||
pub fn new() -> Self {
|
||||
pub fn new(sem: Arc<Semaphore>) -> Self {
|
||||
let meter = global::meter("garage_rpc");
|
||||
RpcMetrics {
|
||||
_rpc_available_permits: meter
|
||||
.u64_value_observer("rpc.available_permits", move |observer| {
|
||||
observer.observe(sem.available_permits() as u64, &[])
|
||||
})
|
||||
.with_description("Number of available RPC permits")
|
||||
.init(),
|
||||
|
||||
rpc_counter: meter
|
||||
.u64_counter("rpc.request_counter")
|
||||
.with_description("Number of RPC requests emitted")
|
||||
|
|
@ -33,6 +46,10 @@ impl RpcMetrics {
|
|||
.f64_value_recorder("rpc.duration")
|
||||
.with_description("Duration of RPCs")
|
||||
.init(),
|
||||
rpc_queueing_time: meter
|
||||
.f64_value_recorder("rpc.queueing_time")
|
||||
.with_description("Time RPC requests were queued for before being sent")
|
||||
.init(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use futures::stream::futures_unordered::FuturesUnordered;
|
|||
use futures::stream::StreamExt;
|
||||
use futures_util::future::FutureExt;
|
||||
use tokio::select;
|
||||
use tokio::sync::watch;
|
||||
use tokio::sync::{watch, Semaphore};
|
||||
|
||||
use opentelemetry::KeyValue;
|
||||
use opentelemetry::{
|
||||
|
|
@ -32,37 +32,32 @@ use garage_util::metrics::RecordDuration;
|
|||
use crate::metrics::RpcMetrics;
|
||||
use crate::ring::Ring;
|
||||
|
||||
// Default RPC timeout = 5 minutes
|
||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300);
|
||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
// Don't allow more than 100 concurrent outgoing RPCs.
|
||||
const MAX_CONCURRENT_REQUESTS: usize = 100;
|
||||
|
||||
/// Strategy to apply when making RPC
|
||||
#[derive(Copy, Clone)]
|
||||
pub struct RequestStrategy {
|
||||
/// Max time to wait for reponse
|
||||
pub rs_timeout: Duration,
|
||||
/// Min number of response to consider the request successful
|
||||
pub rs_quorum: Option<usize>,
|
||||
/// Should requests be dropped after enough response are received
|
||||
pub rs_interrupt_after_quorum: bool,
|
||||
/// Request priority
|
||||
pub rs_priority: RequestPriority,
|
||||
/// Custom timeout for this request
|
||||
rs_timeout: Timeout,
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone)]
|
||||
enum Timeout {
|
||||
None,
|
||||
Default,
|
||||
Custom(Duration),
|
||||
}
|
||||
|
||||
impl RequestStrategy {
|
||||
/// Create a RequestStrategy with default timeout and not interrupting when quorum reached
|
||||
pub fn with_priority(prio: RequestPriority) -> Self {
|
||||
RequestStrategy {
|
||||
rs_timeout: DEFAULT_TIMEOUT,
|
||||
rs_quorum: None,
|
||||
rs_interrupt_after_quorum: false,
|
||||
rs_priority: prio,
|
||||
rs_timeout: Timeout::Default,
|
||||
}
|
||||
}
|
||||
/// Set quorum to be reached for request
|
||||
|
|
@ -70,22 +65,17 @@ impl RequestStrategy {
|
|||
self.rs_quorum = Some(quorum);
|
||||
self
|
||||
}
|
||||
/// Set timeout of the strategy
|
||||
pub fn with_timeout(mut self, timeout: Duration) -> Self {
|
||||
self.rs_timeout = timeout;
|
||||
self
|
||||
}
|
||||
/// Set if requests can be dropped after quorum has been reached
|
||||
/// In general true for read requests, and false for write
|
||||
pub fn interrupt_after_quorum(mut self, interrupt: bool) -> Self {
|
||||
self.rs_interrupt_after_quorum = interrupt;
|
||||
self
|
||||
}
|
||||
/// Deactivate timeout for this request
|
||||
pub fn without_timeout(mut self) -> Self {
|
||||
self.rs_timeout = Timeout::None;
|
||||
self
|
||||
}
|
||||
/// Set custom timeout for this request
|
||||
pub fn with_custom_timeout(mut self, timeout: Duration) -> Self {
|
||||
self.rs_timeout = Timeout::Custom(timeout);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
|
|
@ -96,8 +86,8 @@ struct RpcHelperInner {
|
|||
fullmesh: Arc<FullMeshPeeringStrategy>,
|
||||
background: Arc<BackgroundRunner>,
|
||||
ring: watch::Receiver<Arc<Ring>>,
|
||||
request_buffer_semaphore: Arc<Semaphore>,
|
||||
metrics: RpcMetrics,
|
||||
rpc_timeout: Duration,
|
||||
}
|
||||
|
||||
impl RpcHelper {
|
||||
|
|
@ -106,24 +96,21 @@ impl RpcHelper {
|
|||
fullmesh: Arc<FullMeshPeeringStrategy>,
|
||||
background: Arc<BackgroundRunner>,
|
||||
ring: watch::Receiver<Arc<Ring>>,
|
||||
rpc_timeout: Option<Duration>,
|
||||
) -> Self {
|
||||
let metrics = RpcMetrics::new();
|
||||
let sem = Arc::new(Semaphore::new(MAX_CONCURRENT_REQUESTS));
|
||||
|
||||
let metrics = RpcMetrics::new(sem.clone());
|
||||
|
||||
Self(Arc::new(RpcHelperInner {
|
||||
our_node_id,
|
||||
fullmesh,
|
||||
background,
|
||||
ring,
|
||||
request_buffer_semaphore: sem,
|
||||
metrics,
|
||||
rpc_timeout: rpc_timeout.unwrap_or(DEFAULT_TIMEOUT),
|
||||
}))
|
||||
}
|
||||
|
||||
pub fn rpc_timeout(&self) -> Duration {
|
||||
self.0.rpc_timeout
|
||||
}
|
||||
|
||||
pub async fn call<M, N, H, S>(
|
||||
&self,
|
||||
endpoint: &Endpoint<M, H>,
|
||||
|
|
@ -142,6 +129,13 @@ impl RpcHelper {
|
|||
KeyValue::new("to", format!("{:?}", to)),
|
||||
];
|
||||
|
||||
let permit = self
|
||||
.0
|
||||
.request_buffer_semaphore
|
||||
.acquire()
|
||||
.record_duration(&self.0.metrics.rpc_queueing_time, &metric_tags)
|
||||
.await?;
|
||||
|
||||
self.0.metrics.rpc_counter.add(1, &metric_tags);
|
||||
|
||||
let node_id = to.into();
|
||||
|
|
@ -149,16 +143,10 @@ impl RpcHelper {
|
|||
.call_streaming(&node_id, msg, strat.rs_priority)
|
||||
.record_duration(&self.0.metrics.rpc_duration, &metric_tags);
|
||||
|
||||
let timeout = async {
|
||||
match strat.rs_timeout {
|
||||
Timeout::None => futures::future::pending().await,
|
||||
Timeout::Default => tokio::time::sleep(self.0.rpc_timeout).await,
|
||||
Timeout::Custom(t) => tokio::time::sleep(t).await,
|
||||
}
|
||||
};
|
||||
|
||||
select! {
|
||||
res = rpc_call => {
|
||||
drop(permit);
|
||||
|
||||
if res.is_err() {
|
||||
self.0.metrics.rpc_netapp_error_counter.add(1, &metric_tags);
|
||||
}
|
||||
|
|
@ -170,7 +158,8 @@ impl RpcHelper {
|
|||
|
||||
Ok(res?)
|
||||
}
|
||||
() = timeout => {
|
||||
_ = tokio::time::sleep(strat.rs_timeout) => {
|
||||
drop(permit);
|
||||
self.0.metrics.rpc_timeout_counter.add(1, &metric_tags);
|
||||
Err(Error::Timeout)
|
||||
}
|
||||
|
|
@ -424,7 +413,7 @@ impl RpcHelper {
|
|||
.iter()
|
||||
.find(|x| x.id.as_ref() == to.as_slice())
|
||||
.and_then(|pi| pi.avg_ping)
|
||||
.unwrap_or_else(|| Duration::from_secs(10));
|
||||
.unwrap_or_else(|| Duration::from_secs(1));
|
||||
(
|
||||
*to != self.0.our_node_id,
|
||||
peer_zone != our_zone,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ use tokio::sync::Mutex;
|
|||
use netapp::endpoint::{Endpoint, EndpointHandler};
|
||||
use netapp::message::*;
|
||||
use netapp::peering::fullmesh::FullMeshPeeringStrategy;
|
||||
use netapp::util::parse_and_resolve_peer_addr_async;
|
||||
use netapp::util::parse_and_resolve_peer_addr;
|
||||
use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
|
||||
|
||||
use garage_util::background::BackgroundRunner;
|
||||
|
|
@ -37,6 +37,7 @@ use crate::rpc_helper::*;
|
|||
|
||||
const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
||||
const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const SYSTEM_RPC_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
/// Version tag used for version check upon Netapp connection.
|
||||
/// Cluster nodes with different version tags are deemed
|
||||
|
|
@ -91,7 +92,7 @@ pub struct System {
|
|||
|
||||
rpc_listen_addr: SocketAddr,
|
||||
rpc_public_addr: Option<SocketAddr>,
|
||||
bootstrap_peers: Vec<String>,
|
||||
bootstrap_peers: Vec<(NodeID, SocketAddr)>,
|
||||
|
||||
consul_discovery: Option<ConsulDiscoveryParam>,
|
||||
#[cfg(feature = "kubernetes-discovery")]
|
||||
|
|
@ -241,29 +242,8 @@ impl System {
|
|||
let ring = Ring::new(cluster_layout, replication_factor);
|
||||
let (update_ring, ring) = watch::channel(Arc::new(ring));
|
||||
|
||||
let rpc_public_addr = match &config.rpc_public_addr {
|
||||
Some(a_str) => {
|
||||
use std::net::ToSocketAddrs;
|
||||
match a_str.to_socket_addrs() {
|
||||
Err(e) => {
|
||||
error!(
|
||||
"Cannot resolve rpc_public_addr {} from config file: {}.",
|
||||
a_str, e
|
||||
);
|
||||
None
|
||||
}
|
||||
Ok(a) => {
|
||||
let a = a.collect::<Vec<_>>();
|
||||
if a.is_empty() {
|
||||
error!("rpc_public_addr {} resolve to no known IP address", a_str);
|
||||
}
|
||||
if a.len() > 1 {
|
||||
warn!("Multiple possible resolutions for rpc_public_addr: {:?}. Taking the first one.", a);
|
||||
}
|
||||
a.into_iter().next()
|
||||
}
|
||||
}
|
||||
}
|
||||
let rpc_public_addr = match config.rpc_public_addr {
|
||||
Some(a) => Some(a),
|
||||
None => {
|
||||
let addr =
|
||||
get_default_ip().map(|ip| SocketAddr::new(ip, config.rpc_bind_addr.port()));
|
||||
|
|
@ -273,15 +253,13 @@ impl System {
|
|||
addr
|
||||
}
|
||||
};
|
||||
if rpc_public_addr.is_none() {
|
||||
warn!("This Garage node does not know its publicly reachable RPC address, this might hamper intra-cluster communication.");
|
||||
}
|
||||
|
||||
let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, node_key);
|
||||
let fullmesh = FullMeshPeeringStrategy::new(netapp.clone(), vec![], rpc_public_addr);
|
||||
if let Some(ping_timeout) = config.rpc_ping_timeout_msec {
|
||||
fullmesh.set_ping_timeout_millis(ping_timeout);
|
||||
}
|
||||
let fullmesh = FullMeshPeeringStrategy::new(
|
||||
netapp.clone(),
|
||||
config.bootstrap_peers.clone(),
|
||||
rpc_public_addr,
|
||||
);
|
||||
|
||||
let system_endpoint = netapp.endpoint(SYSTEM_RPC_PATH.into());
|
||||
|
||||
|
|
@ -319,13 +297,7 @@ impl System {
|
|||
node_status: RwLock::new(HashMap::new()),
|
||||
netapp: netapp.clone(),
|
||||
fullmesh: fullmesh.clone(),
|
||||
rpc: RpcHelper::new(
|
||||
netapp.id.into(),
|
||||
fullmesh,
|
||||
background.clone(),
|
||||
ring.clone(),
|
||||
config.rpc_timeout_msec.map(Duration::from_millis),
|
||||
),
|
||||
rpc: RpcHelper::new(netapp.id.into(), fullmesh, background.clone(), ring.clone()),
|
||||
system_endpoint,
|
||||
replication_factor,
|
||||
rpc_listen_addr: config.rpc_bind_addr,
|
||||
|
|
@ -398,14 +370,12 @@ impl System {
|
|||
}
|
||||
|
||||
pub async fn connect(&self, node: &str) -> Result<(), Error> {
|
||||
let (pubkey, addrs) = parse_and_resolve_peer_addr_async(node)
|
||||
.await
|
||||
.ok_or_else(|| {
|
||||
Error::Message(format!(
|
||||
"Unable to parse or resolve node specification: {}",
|
||||
node
|
||||
))
|
||||
})?;
|
||||
let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| {
|
||||
Error::Message(format!(
|
||||
"Unable to parse or resolve node specification: {}",
|
||||
node
|
||||
))
|
||||
})?;
|
||||
let mut errors = vec![];
|
||||
for ip in addrs.iter() {
|
||||
match self
|
||||
|
|
@ -608,7 +578,7 @@ impl System {
|
|||
.broadcast(
|
||||
&self.system_endpoint,
|
||||
SystemRpc::AdvertiseStatus(local_status),
|
||||
RequestStrategy::with_priority(PRIO_HIGH),
|
||||
RequestStrategy::with_priority(PRIO_HIGH).with_timeout(SYSTEM_RPC_TIMEOUT),
|
||||
)
|
||||
.await;
|
||||
|
||||
|
|
@ -634,7 +604,7 @@ impl System {
|
|||
if not_configured || no_peers || bad_peers {
|
||||
info!("Doing a bootstrap/discovery step (not_configured: {}, no_peers: {}, bad_peers: {})", not_configured, no_peers, bad_peers);
|
||||
|
||||
let mut ping_list = resolve_peers(&self.bootstrap_peers).await;
|
||||
let mut ping_list = self.bootstrap_peers.clone();
|
||||
|
||||
// Add peer list from list stored on disk
|
||||
if let Ok(peers) = self.persist_peer_list.load_async().await {
|
||||
|
|
@ -732,7 +702,7 @@ impl System {
|
|||
&self.system_endpoint,
|
||||
peer,
|
||||
SystemRpc::PullClusterLayout,
|
||||
RequestStrategy::with_priority(PRIO_HIGH),
|
||||
RequestStrategy::with_priority(PRIO_HIGH).with_timeout(SYSTEM_RPC_TIMEOUT),
|
||||
)
|
||||
.await;
|
||||
if let Ok(SystemRpc::AdvertiseClusterLayout(layout)) = resp {
|
||||
|
|
@ -765,25 +735,6 @@ fn get_default_ip() -> Option<IpAddr> {
|
|||
.map(|a| a.ip())
|
||||
}
|
||||
|
||||
async fn resolve_peers(peers: &[String]) -> Vec<(NodeID, SocketAddr)> {
|
||||
let mut ret = vec![];
|
||||
|
||||
for peer in peers.iter() {
|
||||
match parse_and_resolve_peer_addr_async(peer).await {
|
||||
Some((pubkey, addrs)) => {
|
||||
for ip in addrs {
|
||||
ret.push((pubkey, ip));
|
||||
}
|
||||
}
|
||||
None => {
|
||||
warn!("Unable to parse and/or resolve peer hostname {}", peer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
ret
|
||||
}
|
||||
|
||||
struct ConsulDiscoveryParam {
|
||||
consul_host: String,
|
||||
service_name: String,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ opentelemetry = "0.17"
|
|||
|
||||
async-trait = "0.1.7"
|
||||
bytes = "1.0"
|
||||
hex = "0.4"
|
||||
hexdump = "0.1"
|
||||
tracing = "0.1.30"
|
||||
rand = "0.8"
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ use crate::replication::*;
|
|||
use crate::schema::*;
|
||||
|
||||
const TABLE_GC_BATCH_SIZE: usize = 1024;
|
||||
// Same timeout as NEED_BLOCK_QUERY_TIMEOUT in block manager
|
||||
const TABLE_GC_RPC_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
|
||||
// GC delay for table entries: 1 day (24 hours)
|
||||
// (the delay before the entry is added in the GC todo list
|
||||
|
|
@ -235,7 +237,9 @@ where
|
|||
&self.endpoint,
|
||||
&nodes[..],
|
||||
GcRpc::Update(updates),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND).with_quorum(nodes.len()),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_quorum(nodes.len())
|
||||
.with_timeout(TABLE_GC_RPC_TIMEOUT),
|
||||
)
|
||||
.await
|
||||
.err_context("GC: send tombstones")?;
|
||||
|
|
@ -256,7 +260,9 @@ where
|
|||
&self.endpoint,
|
||||
&nodes[..],
|
||||
GcRpc::DeleteIfEqualHash(deletes),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND).with_quorum(nodes.len()),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_quorum(nodes.len())
|
||||
.with_timeout(TABLE_GC_RPC_TIMEOUT),
|
||||
)
|
||||
.await
|
||||
.err_context("GC: remote delete tombstones")?;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,9 @@ use crate::merkle::*;
|
|||
use crate::replication::*;
|
||||
use crate::*;
|
||||
|
||||
// Sync RPC can contain a lot of data, so have a 1min timeout
|
||||
const TABLE_SYNC_RPC_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
// Do anti-entropy every 10 minutes
|
||||
const ANTI_ENTROPY_INTERVAL: Duration = Duration::from_secs(10 * 60);
|
||||
|
||||
|
|
@ -245,7 +248,9 @@ where
|
|||
&self.endpoint,
|
||||
nodes,
|
||||
SyncRpc::Items(values),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND).with_quorum(nodes.len()),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_quorum(nodes.len())
|
||||
.with_timeout(TABLE_SYNC_RPC_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -306,7 +311,8 @@ where
|
|||
&self.endpoint,
|
||||
who,
|
||||
SyncRpc::RootCkHash(partition.partition, root_ck_hash),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_timeout(TABLE_SYNC_RPC_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -345,11 +351,11 @@ where
|
|||
// Just send that item directly
|
||||
if let Some(val) = self.data.store.get(&ik[..])? {
|
||||
if blake2sum(&val[..]) != ivhash {
|
||||
debug!("({}) Hashes differ between stored value and Merkle tree, key: {} (if your server is very busy, don't worry, this happens when the Merkle tree can't be updated fast enough)", F::TABLE_NAME, hex::encode(ik));
|
||||
warn!("({}) Hashes differ between stored value and Merkle tree, key: {:?} (if your server is very busy, don't worry, this happens when the Merkle tree can't be updated fast enough)", F::TABLE_NAME, ik);
|
||||
}
|
||||
todo_items.push(val.to_vec());
|
||||
} else {
|
||||
debug!("({}) Item from Merkle tree not found in store: {} (if your server is very busy, don't worry, this happens when the Merkle tree can't be updated fast enough)", F::TABLE_NAME, hex::encode(ik));
|
||||
warn!("({}) Item from Merkle tree not found in store: {:?} (if your server is very busy, don't worry, this happens when the Merkle tree can't be updated fast enough)", F::TABLE_NAME, ik);
|
||||
}
|
||||
}
|
||||
MerkleNode::Intermediate(l) => {
|
||||
|
|
@ -362,7 +368,8 @@ where
|
|||
&self.endpoint,
|
||||
who,
|
||||
SyncRpc::GetNode(key.clone()),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_timeout(TABLE_SYNC_RPC_TIMEOUT),
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
|
@ -438,7 +445,8 @@ where
|
|||
&self.endpoint,
|
||||
who,
|
||||
SyncRpc::Items(values),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND),
|
||||
RequestStrategy::with_priority(PRIO_BACKGROUND)
|
||||
.with_timeout(TABLE_SYNC_RPC_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
if let SyncRpc::Ok = rpc_resp {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::borrow::Borrow;
|
||||
use std::collections::{BTreeMap, BTreeSet, HashMap};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use futures::stream::*;
|
||||
|
|
@ -30,6 +31,8 @@ use crate::schema::*;
|
|||
use crate::sync::*;
|
||||
use crate::util::*;
|
||||
|
||||
pub const TABLE_RPC_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
pub struct Table<F: TableSchema + 'static, R: TableReplication + 'static> {
|
||||
pub system: Arc<System>,
|
||||
pub data: Arc<TableData<F, R>>,
|
||||
|
|
@ -121,7 +124,8 @@ where
|
|||
&who[..],
|
||||
rpc,
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(self.data.replication.write_quorum()),
|
||||
.with_quorum(self.data.replication.write_quorum())
|
||||
.with_timeout(TABLE_RPC_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
|
@ -173,7 +177,7 @@ where
|
|||
&self.endpoint,
|
||||
node,
|
||||
rpc,
|
||||
RequestStrategy::with_priority(PRIO_NORMAL),
|
||||
RequestStrategy::with_priority(PRIO_NORMAL).with_timeout(TABLE_RPC_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
Ok::<_, Error>((node, resp))
|
||||
|
|
@ -230,6 +234,7 @@ where
|
|||
rpc,
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(self.data.replication.read_quorum())
|
||||
.with_timeout(TABLE_RPC_TIMEOUT)
|
||||
.interrupt_after_quorum(true),
|
||||
)
|
||||
.await?;
|
||||
|
|
@ -324,6 +329,7 @@ where
|
|||
rpc,
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(self.data.replication.read_quorum())
|
||||
.with_timeout(TABLE_RPC_TIMEOUT)
|
||||
.interrupt_after_quorum(true),
|
||||
)
|
||||
.await?;
|
||||
|
|
@ -400,7 +406,9 @@ where
|
|||
&self.endpoint,
|
||||
who,
|
||||
TableRpc::<F>::Update(vec![what_enc]),
|
||||
RequestStrategy::with_priority(PRIO_NORMAL).with_quorum(who.len()),
|
||||
RequestStrategy::with_priority(PRIO_NORMAL)
|
||||
.with_quorum(who.len())
|
||||
.with_timeout(TABLE_RPC_TIMEOUT),
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -3,8 +3,12 @@ use std::io::Read;
|
|||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde::de::Error as SerdeError;
|
||||
use serde::{de, Deserialize};
|
||||
|
||||
use netapp::util::parse_and_resolve_peer_addr;
|
||||
use netapp::NodeID;
|
||||
|
||||
use crate::error::Error;
|
||||
|
||||
/// Represent the whole configuration
|
||||
|
|
@ -39,16 +43,11 @@ pub struct Config {
|
|||
/// Address to bind for RPC
|
||||
pub rpc_bind_addr: SocketAddr,
|
||||
/// Public IP address of this node
|
||||
pub rpc_public_addr: Option<String>,
|
||||
|
||||
/// Timeout for Netapp's ping messagess
|
||||
pub rpc_ping_timeout_msec: Option<u64>,
|
||||
/// Timeout for Netapp RPC calls
|
||||
pub rpc_timeout_msec: Option<u64>,
|
||||
pub rpc_public_addr: Option<SocketAddr>,
|
||||
|
||||
/// Bootstrap peers RPC address
|
||||
#[serde(default)]
|
||||
pub bootstrap_peers: Vec<String>,
|
||||
#[serde(deserialize_with = "deserialize_vec_addr", default)]
|
||||
pub bootstrap_peers: Vec<(NodeID, SocketAddr)>,
|
||||
/// Consul host to connect to to discover more peers
|
||||
pub consul_host: Option<String>,
|
||||
/// Consul service name to use
|
||||
|
|
@ -155,6 +154,24 @@ pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
|
|||
Ok(toml::from_str(&config)?)
|
||||
}
|
||||
|
||||
fn deserialize_vec_addr<'de, D>(deserializer: D) -> Result<Vec<(NodeID, SocketAddr)>, D::Error>
|
||||
where
|
||||
D: de::Deserializer<'de>,
|
||||
{
|
||||
let mut ret = vec![];
|
||||
|
||||
for peer in <Vec<&str>>::deserialize(deserializer)? {
|
||||
let (pubkey, addrs) = parse_and_resolve_peer_addr(peer).ok_or_else(|| {
|
||||
D::Error::custom(format!("Unable to parse or resolve peer: {}", peer))
|
||||
})?;
|
||||
for ip in addrs {
|
||||
ret.push((pubkey, ip));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ret)
|
||||
}
|
||||
|
||||
fn default_compression() -> Option<i32> {
|
||||
Some(1)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue