performance improvements #342

Merged
lx merged 21 commits from lx-perf-improvements into main 2022-09-12 14:38:44 +00:00
42 changed files with 1337 additions and 1072 deletions
Showing only changes of commit d9d199a6c9 - Show all commits

316
Cargo.lock generated
View file

@ -163,7 +163,7 @@ dependencies = [
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
"bytes 1.1.0",
"bytes",
"http",
"md5",
"tokio-stream",
@ -193,7 +193,7 @@ checksum = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-http",
"bytes 1.1.0",
"bytes",
"form_urlencoded",
"hex",
"http",
@ -227,7 +227,7 @@ dependencies = [
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-types",
"bytes 1.1.0",
"bytes",
"fastrand",
"http",
"http-body",
@ -248,7 +248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"
dependencies = [
"aws-smithy-types",
"bytes 1.1.0",
"bytes",
"crc32fast",
]
@ -260,7 +260,7 @@ checksum = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-types",
"bytes 1.1.0",
"bytes",
"bytes-utils",
"futures-core",
"http",
@ -280,7 +280,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"
dependencies = [
"aws-smithy-http",
"bytes 1.1.0",
"bytes",
"http",
"http-body",
"pin-project 1.0.10",
@ -392,12 +392,6 @@ version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16"
[[package]]
name = "bytes"
version = "1.1.0"
@ -410,7 +404,7 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1934a3ef9cac8efde4966a92781e77713e1ba329f1d42e446c7d7eba340d8ef1"
dependencies = [
"bytes 1.1.0",
"bytes",
"either",
]
@ -783,20 +777,6 @@ dependencies = [
"termcolor",
]
[[package]]
name = "err-derive"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4"
dependencies = [
"proc-macro-error",
"proc-macro2",
"quote",
"rustversion",
"syn",
"synstructure",
]
[[package]]
name = "err-derive"
version = "0.3.1"
@ -985,13 +965,13 @@ dependencies = [
[[package]]
name = "garage"
version = "0.7.0"
version = "0.8.0"
dependencies = [
"assert-json-diff",
"async-trait",
"aws-sdk-s3",
"base64",
"bytes 1.1.0",
"bytes",
"bytesize",
"chrono",
"futures",
@ -999,23 +979,23 @@ dependencies = [
"garage_api",
"garage_block",
"garage_db",
"garage_model 0.7.0",
"garage_rpc 0.7.0",
"garage_table 0.7.0",
"garage_util 0.7.0",
"garage_model",
"garage_rpc",
"garage_table",
"garage_util",
"garage_web",
"hex",
"hmac 0.12.1",
"http",
"hyper",
"kuska-sodiumoxide",
"netapp 0.4.5",
"netapp",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry-prometheus",
"prometheus",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde",
"serde",
"serde_bytes",
"serde_json",
@ -1031,22 +1011,22 @@ dependencies = [
[[package]]
name = "garage_api"
version = "0.7.0"
version = "0.8.0"
dependencies = [
"async-trait",
"base64",
"bytes 1.1.0",
"bytes",
"chrono",
"crypto-common",
"err-derive 0.3.1",
"err-derive",
"form_urlencoded",
"futures",
"futures-util",
"garage_block",
"garage_model 0.7.0",
"garage_rpc 0.7.0",
"garage_table 0.7.0",
"garage_util 0.7.0",
"garage_model",
"garage_rpc",
"garage_table",
"garage_util",
"hex",
"hmac 0.12.1",
"http",
@ -1058,7 +1038,6 @@ dependencies = [
"multer",
"nom",
"opentelemetry",
"opentelemetry-otlp",
"opentelemetry-prometheus",
"percent-encoding",
"pin-project 1.0.10",
@ -1076,21 +1055,21 @@ dependencies = [
[[package]]
name = "garage_block"
version = "0.7.0"
version = "0.8.0"
dependencies = [
"arc-swap",
"async-trait",
"bytes 1.1.0",
"bytes",
"futures",
"futures-util",
"garage_db",
"garage_rpc 0.7.0",
"garage_table 0.7.0",
"garage_util 0.7.0",
"garage_rpc",
"garage_table",
"garage_util",
"hex",
"opentelemetry",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde",
"serde",
"serde_bytes",
"tokio",
@ -1103,7 +1082,7 @@ name = "garage_db"
version = "0.8.0"
dependencies = [
"clap 3.1.18",
"err-derive 0.3.1",
"err-derive",
"heed",
"hexdump",
"mktemp",
@ -1115,51 +1094,25 @@ dependencies = [
[[package]]
name = "garage_model"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "584619e8999713d73761775591ad6f01ff8c9d724f3b20984f5932f1fc7f9988"
dependencies = [
"arc-swap",
"async-trait",
"futures",
"futures-util",
"garage_rpc 0.5.1",
"garage_table 0.5.1",
"garage_util 0.5.1",
"hex",
"log",
"netapp 0.3.1",
"rand 0.8.5",
"rmp-serde 0.15.5",
"serde",
"serde_bytes",
"sled",
"tokio",
"zstd",
]
[[package]]
name = "garage_model"
version = "0.7.0"
version = "0.8.0"
dependencies = [
"arc-swap",
"async-trait",
"base64",
"blake2",
"err-derive 0.3.1",
"err-derive",
"futures",
"futures-util",
"garage_block",
"garage_db",
"garage_model 0.5.1",
"garage_rpc 0.7.0",
"garage_table 0.7.0",
"garage_util 0.7.0",
"garage_rpc",
"garage_table",
"garage_util",
"hex",
"netapp 0.4.5",
"netapp",
"opentelemetry",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde",
"serde",
"serde_bytes",
"tokio",
@ -1169,53 +1122,26 @@ dependencies = [
[[package]]
name = "garage_rpc"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81e693aa4582cfe7a7ce70c07880e3662544b5d0cd68bc4b59c53febfbb8d1ec"
version = "0.8.0"
dependencies = [
"arc-swap",
"async-trait",
"bytes 1.1.0",
"bytes",
"futures",
"futures-util",
"garage_util 0.5.1",
"gethostname",
"hex",
"hyper",
"kuska-sodiumoxide",
"log",
"netapp 0.3.1",
"rand 0.8.5",
"rmp-serde 0.15.5",
"serde",
"serde_bytes",
"serde_json",
"tokio",
"tokio-stream",
]
[[package]]
name = "garage_rpc"
version = "0.7.0"
dependencies = [
"arc-swap",
"async-trait",
"bytes 1.1.0",
"futures",
"futures-util",
"garage_util 0.7.0",
"garage_util",
"gethostname",
"hex",
"hyper",
"k8s-openapi",
"kube",
"kuska-sodiumoxide",
"netapp 0.4.5",
"netapp",
"openssl",
"opentelemetry",
"pnet_datalink",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde",
"schemars",
"serde",
"serde_bytes",
@ -1227,41 +1153,19 @@ dependencies = [
[[package]]
name = "garage_table"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c3557f3757e2acd29eaee86804d4e6c38d2abda81b4b349d8a0d2277044265c"
version = "0.8.0"
dependencies = [
"async-trait",
"bytes 1.1.0",
"futures",
"futures-util",
"garage_rpc 0.5.1",
"garage_util 0.5.1",
"hexdump",
"log",
"rand 0.8.5",
"rmp-serde 0.15.5",
"serde",
"serde_bytes",
"sled",
"tokio",
]
[[package]]
name = "garage_table"
version = "0.7.0"
dependencies = [
"async-trait",
"bytes 1.1.0",
"bytes",
"futures",
"futures-util",
"garage_db",
"garage_rpc 0.7.0",
"garage_util 0.7.0",
"garage_rpc",
"garage_util",
"hexdump",
"opentelemetry",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde",
"serde",
"serde_bytes",
"tokio",
@ -1270,50 +1174,26 @@ dependencies = [
[[package]]
name = "garage_util"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e096994382447431e2f3c70e3685eb8b24c00eceff8667bb22a2a27ff17832f"
dependencies = [
"blake2",
"chrono",
"err-derive 0.3.1",
"futures",
"hex",
"http",
"hyper",
"log",
"netapp 0.3.1",
"rand 0.8.5",
"rmp-serde 0.15.5",
"serde",
"serde_json",
"sha2 0.9.9",
"sled",
"tokio",
"toml",
"xxhash-rust",
]
[[package]]
name = "garage_util"
version = "0.7.0"
version = "0.8.0"
dependencies = [
"arc-swap",
"async-trait",
"blake2",
"bytes 1.1.0",
"bytes",
"chrono",
"digest 0.10.3",
"err-derive 0.3.1",
"err-derive",
"futures",
"garage_db",
"git-version",
"hex",
"http",
"hyper",
"netapp 0.4.5",
"lazy_static",
"netapp",
"opentelemetry",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde",
"serde",
"serde_json",
"sha2 0.10.2",
@ -1325,14 +1205,14 @@ dependencies = [
[[package]]
name = "garage_web"
version = "0.7.0"
version = "0.8.0"
dependencies = [
"err-derive 0.3.1",
"err-derive",
"futures",
"garage_api",
"garage_model 0.7.0",
"garage_table 0.7.0",
"garage_util 0.7.0",
"garage_model",
"garage_table",
"garage_util",
"http",
"hyper",
"opentelemetry",
@ -1399,7 +1279,7 @@ version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"
dependencies = [
"bytes 1.1.0",
"bytes",
"fnv",
"futures-core",
"futures-sink",
@ -1459,7 +1339,6 @@ dependencies = [
"lmdb-rkv-sys",
"once_cell",
"page_size",
"serde",
"synchronoise",
"url",
]
@ -1534,7 +1413,7 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
dependencies = [
"bytes 1.1.0",
"bytes",
"fnv",
"itoa",
]
@ -1545,7 +1424,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
dependencies = [
"bytes 1.1.0",
"bytes",
"http",
"pin-project-lite",
]
@ -1589,7 +1468,7 @@ version = "0.14.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2"
dependencies = [
"bytes 1.1.0",
"bytes",
"futures-channel",
"futures-core",
"futures-util",
@ -1642,7 +1521,7 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes 1.1.0",
"bytes",
"hyper",
"native-tls",
"tokio",
@ -1771,7 +1650,7 @@ version = "0.0.1"
dependencies = [
"base64",
"clap 3.1.18",
"garage_util 0.7.0",
"garage_util",
"http",
"log",
"rusoto_core",
@ -1790,7 +1669,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c"
dependencies = [
"base64",
"bytes 1.1.0",
"bytes",
"chrono",
"http",
"percent-encoding",
@ -1819,7 +1698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cd12d68768b54bbd50547f4c7b57b73cff680ef8da3ba409463ee995cf0d707"
dependencies = [
"base64",
"bytes 1.1.0",
"bytes",
"chrono",
"dirs-next",
"either",
@ -2099,7 +1978,7 @@ version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f8f35e687561d5c1667590911e6698a8cb714a134a7505718a182e7bc9d3836"
dependencies = [
"bytes 1.1.0",
"bytes",
"encoding_rs",
"futures-util",
"http",
@ -2135,28 +2014,6 @@ dependencies = [
"tempfile",
]
[[package]]
name = "netapp"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac7dcd2c6c5a9d5ea88ffc17d3339d49d308e68c4d8190c494b55ddbf78d80ad"
dependencies = [
"arc-swap",
"async-trait",
"bytes 0.6.0",
"err-derive 0.2.4",
"futures",
"hex",
"kuska-handshake",
"kuska-sodiumoxide",
"log",
"rmp-serde 0.14.4",
"serde",
"tokio",
"tokio-stream",
"tokio-util 0.6.9",
]
[[package]]
name = "netapp"
version = "0.4.5"
@ -2165,9 +2022,9 @@ checksum = "38e390e559aff9e3865c7cd326c691cbf25c580a96f259d2debc35bd4e14018e"
dependencies = [
"arc-swap",
"async-trait",
"bytes 1.1.0",
"bytes",
"cfg-if 1.0.0",
"err-derive 0.3.1",
"err-derive",
"futures",
"hex",
"kuska-handshake",
@ -2176,7 +2033,7 @@ dependencies = [
"opentelemetry",
"opentelemetry-contrib",
"rand 0.8.5",
"rmp-serde 0.15.5",
"rmp-serde",
"serde",
"tokio",
"tokio-stream",
@ -2658,7 +2515,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
dependencies = [
"bytes 1.1.0",
"bytes",
"prost-derive",
]
@ -2668,7 +2525,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
dependencies = [
"bytes 1.1.0",
"bytes",
"heck 0.3.3",
"itertools 0.10.3",
"lazy_static",
@ -2701,7 +2558,7 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
dependencies = [
"bytes 1.1.0",
"bytes",
"prost",
]
@ -2961,17 +2818,6 @@ dependencies = [
"num-traits",
]
[[package]]
name = "rmp-serde"
version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ce7d70c926fe472aed493b902010bccc17fa9f7284145cb8772fd22fdb052d8"
dependencies = [
"byteorder",
"rmp",
"serde",
]
[[package]]
name = "rmp-serde"
version = "0.15.5"
@ -3000,7 +2846,7 @@ checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"
dependencies = [
"async-trait",
"base64",
"bytes 1.1.0",
"bytes",
"crc32fast",
"futures",
"http",
@ -3042,7 +2888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"
dependencies = [
"base64",
"bytes 1.1.0",
"bytes",
"chrono",
"digest 0.9.0",
"futures",
@ -3626,7 +3472,7 @@ version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"
dependencies = [
"bytes 1.1.0",
"bytes",
"libc",
"memchr",
"mio",
@ -3699,9 +3545,8 @@ version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"
dependencies = [
"bytes 1.1.0",
"bytes",
"futures-core",
"futures-io",
"futures-sink",
"log",
"pin-project-lite",
@ -3715,7 +3560,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1"
dependencies = [
"bytes 1.1.0",
"bytes",
"futures-core",
"futures-io",
"futures-sink",
@ -3742,7 +3587,7 @@ dependencies = [
"async-stream",
"async-trait",
"base64",
"bytes 1.1.0",
"bytes",
"futures-core",
"futures-util",
"h2",
@ -3803,7 +3648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81eca72647e58054bbfa41e6f297c23436f1c60aff6e5eb38455a0f9ca420bb5"
dependencies = [
"base64",
"bytes 1.1.0",
"bytes",
"futures-core",
"futures-util",
"http",
@ -4250,4 +4095,5 @@ checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f"
dependencies = [
"cc",
"libc",
"pkg-config",
]

627
Cargo.nix

File diff suppressed because it is too large Load diff

View file

@ -1,4 +1,5 @@
[workspace]
resolver = "2"
members = [
"src/db",
"src/util",

View file

@ -20,6 +20,24 @@ sudo apt-get update
sudo apt-get install build-essential
```
## Using source from the Gitea repository (recommended)
The primary location for Garage's source code is the
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage).
Clone the repository and build Garage with the following commands:
```bash
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage.git
cd garage
cargo build
```
Be careful, as this will make a debug build of Garage, which will be extremely slow!
To make a release build, invoke `cargo build --release` (this takes much longer).
The binaries built this way are found in `target/{debug,release}/garage`.
## Using source from `crates.io`
Garage's source code is published on `crates.io`, Rust's official package repository.
@ -39,21 +57,20 @@ sudo cp $HOME/.cargo/bin/garage /usr/local/bin/garage
```
## Using source from the Gitea repository
## Selecting features to activate in your build
The primary location for Garage's source code is the
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage).
Clone the repository and build Garage with the following commands:
```bash
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage.git
cd garage
cargo build
```
Be careful, as this will make a debug build of Garage, which will be extremely slow!
To make a release build, invoke `cargo build --release` (this takes much longer).
The binaries built this way are found in `target/{debug,release}/garage`.
Garage supports a number of compilation options in the form of Cargo features,
which can be used to provide builds adapted to your system and your use case.
The following features are available:
| Feature | Enabled | Description |
| ------- | ------- | ----------- |
| `bundled-libs` | BY DEFAULT | Use bundled version of sqlite3, zstd, lmdb and libsodium |
| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium if available (exclusive with `bundled-libs`, build using `cargo build --no-default-features --features system-libs`) |
| `k2v` | optional | Enable the experimental K2V API (if used, all nodes on your Garage cluster must have it enabled as well) |
| `kubernetes-discovery` | optional | Enable automatic registration and discovery of cluster nodes through the Kubernetes API |
| `metrics` | BY DEFAULT | Enable collection of metrics in Prometheus format on the admin API |
| `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
| `sled` | BY DEFAULT | Enable using Sled to store Garage's metadata |
| `lmdb` | optional | Enable using LMDB to store Garage's metadata |
| `sqlite` | optional | Enable using Sqlite3 to store Garage's metadata |

View file

@ -117,22 +117,34 @@ let
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
But we want to ship these additional features when we release Garage.
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate.
[5] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library.
However the features to do so get activated for some reason (due to a bug in cargo2nix?),
so disable them manually here.
*/
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage";
overrideAttrs = drv: {
overrideAttrs = drv:
(if git_version != null then {
/* [3] */ preConfigure = ''
${drv.preConfigure or ""}
export GIT_VERSION="${git_version}"
'';
} else {})
//
{
/* [1] */ setBuildEnv = (buildEnv drv);
/* [2] */ hardeningDisable = [ "pie" ];
};
overrideArgs = old: {
/* [4] */ features = [ "bundled-libs" "sled" ]
++ (if release then [ "kubernetes-discovery" "telemetry-otlp" "metrics" "lmdb" "sqlite" ] else []);
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_rpc";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
overrideArgs = old: {
/* [4] */ features = if release then [ "kubernetes-discovery" ] else [];
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
@ -142,15 +154,7 @@ let
(pkgs.rustBuilder.rustLib.makeOverride {
name = "garage_util";
overrideAttrs = drv:
(if git_version != null then {
/* [3] */ preConfigure = ''
${drv.preConfigure or ""}
export GIT_VERSION="${git_version}"
'';
} else {})
//
{ /* [1] */ setBuildEnv = (buildEnv drv); };
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
@ -182,6 +186,20 @@ let
name = "k2v-client";
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "libsodium-sys";
overrideArgs = old: {
features = [ ]; /* [5] */
};
})
(pkgs.rustBuilder.rustLib.makeOverride {
name = "zstd-sys";
overrideArgs = old: {
features = [ ]; /* [5] */
};
})
];
packageFun = import ../Cargo.nix;

View file

@ -1,6 +1,6 @@
[package]
name = "garage_api"
version = "0.7.0"
version = "0.8.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,11 +14,11 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_model = { version = "0.7.0", path = "../model" }
garage_table = { version = "0.7.0", path = "../table" }
garage_block = { version = "0.7.0", path = "../block" }
garage_util = { version = "0.7.0", path = "../util" }
garage_rpc = { version = "0.7.0", path = "../rpc" }
garage_model = { version = "0.8.0", path = "../model" }
garage_table = { version = "0.8.0", path = "../table" }
garage_block = { version = "0.8.0", path = "../block" }
garage_util = { version = "0.8.0", path = "../util" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
async-trait = "0.1.7"
base64 = "0.13"
@ -54,9 +54,9 @@ quick-xml = { version = "0.21", features = [ "serialize" ] }
url = "2.1"
opentelemetry = "0.17"
opentelemetry-prometheus = "0.10"
opentelemetry-otlp = "0.10"
prometheus = "0.13"
opentelemetry-prometheus = { version = "0.10", optional = true }
prometheus = { version = "0.13", optional = true }
[features]
k2v = [ "garage_util/k2v", "garage_model/k2v" ]
metrics = [ "opentelemetry-prometheus", "prometheus" ]

View file

@ -1,15 +1,17 @@
use std::net::SocketAddr;
use std::sync::Arc;
use async_trait::async_trait;
use futures::future::Future;
use http::header::{
ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW, CONTENT_TYPE,
};
use http::header::{ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW};
use hyper::{Body, Request, Response};
use opentelemetry::trace::{SpanRef, Tracer};
use opentelemetry::trace::SpanRef;
#[cfg(feature = "metrics")]
use opentelemetry_prometheus::PrometheusExporter;
#[cfg(feature = "metrics")]
use prometheus::{Encoder, TextEncoder};
use garage_model::garage::Garage;
@ -25,6 +27,7 @@ use crate::admin::router::{Authorization, Endpoint};
pub struct AdminApiServer {
garage: Arc<Garage>,
#[cfg(feature = "metrics")]
exporter: PrometheusExporter,
metrics_token: Option<String>,
admin_token: Option<String>,
@ -32,7 +35,6 @@ pub struct AdminApiServer {
impl AdminApiServer {
pub fn new(garage: Arc<Garage>) -> Self {
let exporter = opentelemetry_prometheus::exporter().init();
let cfg = &garage.config.admin;
let metrics_token = cfg
.metrics_token
@ -44,21 +46,22 @@ impl AdminApiServer {
.map(|tok| format!("Bearer {}", tok));
Self {
garage,
exporter,
#[cfg(feature = "metrics")]
exporter: opentelemetry_prometheus::exporter().init(),
metrics_token,
admin_token,
}
}
pub async fn run(self, shutdown_signal: impl Future<Output = ()>) -> Result<(), GarageError> {
if let Some(bind_addr) = self.garage.config.admin.api_bind_addr {
pub async fn run(
self,
bind_addr: SocketAddr,
shutdown_signal: impl Future<Output = ()>,
) -> Result<(), GarageError> {
let region = self.garage.config.s3_api.s3_region.clone();
ApiServer::new(region, self)
.run_server(bind_addr, shutdown_signal)
.await
} else {
Ok(())
}
}
fn handle_options(&self, _req: &Request<Body>) -> Result<Response<Body>, Error> {
@ -71,6 +74,10 @@ impl AdminApiServer {
}
fn handle_metrics(&self) -> Result<Response<Body>, Error> {
#[cfg(feature = "metrics")]
{
use opentelemetry::trace::Tracer;
let mut buffer = vec![];
let encoder = TextEncoder::new();
@ -85,9 +92,14 @@ impl AdminApiServer {
Ok(Response::builder()
.status(200)
.header(CONTENT_TYPE, encoder.format_type())
.header(http::header::CONTENT_TYPE, encoder.format_type())
.body(Body::from(buffer))?)
}
#[cfg(not(feature = "metrics"))]
Err(Error::bad_request(
"Garage was built without the metrics feature".to_string(),
))
}
}
#[async_trait]

View file

@ -18,7 +18,8 @@ use crate::helpers::{json_ok_response, parse_json_body};
pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
let res = GetClusterStatusResponse {
node: hex::encode(garage.system.id),
garage_version: garage.system.garage_version(),
garage_version: garage_util::version::garage_version(),
garage_features: garage_util::version::garage_features(),
db_engine: garage.db.engine(),
known_nodes: garage
.system
@ -99,6 +100,7 @@ fn get_cluster_layout(garage: &Arc<Garage>) -> GetClusterLayoutResponse {
struct GetClusterStatusResponse {
node: String,
garage_version: &'static str,
garage_features: Option<&'static [&'static str]>,
db_engine: String,
known_nodes: HashMap<String, KnownNodeResp>,
layout: GetClusterLayoutResponse,

View file

@ -1,3 +1,4 @@
use std::net::SocketAddr;
use std::sync::Arc;
use async_trait::async_trait;
@ -36,20 +37,13 @@ pub(crate) struct K2VApiEndpoint {
impl K2VApiServer {
pub async fn run(
garage: Arc<Garage>,
bind_addr: SocketAddr,
s3_region: String,
shutdown_signal: impl Future<Output = ()>,
) -> Result<(), GarageError> {
if let Some(cfg) = &garage.config.k2v_api {
let bind_addr = cfg.api_bind_addr;
ApiServer::new(
garage.config.s3_api.s3_region.clone(),
K2VApiServer { garage },
)
ApiServer::new(s3_region, K2VApiServer { garage })
.run_server(bind_addr, shutdown_signal)
.await
} else {
Ok(())
}
}
}

View file

@ -1,3 +1,4 @@
use std::net::SocketAddr;
use std::sync::Arc;
use async_trait::async_trait;
@ -43,14 +44,11 @@ pub(crate) struct S3ApiEndpoint {
impl S3ApiServer {
pub async fn run(
garage: Arc<Garage>,
addr: SocketAddr,
s3_region: String,
shutdown_signal: impl Future<Output = ()>,
) -> Result<(), GarageError> {
let addr = garage.config.s3_api.api_bind_addr;
ApiServer::new(
garage.config.s3_api.s3_region.clone(),
S3ApiServer { garage },
)
ApiServer::new(s3_region, S3ApiServer { garage })
.run_server(addr, shutdown_signal)
.await
}

View file

@ -1,6 +1,6 @@
[package]
name = "garage_block"
version = "0.7.0"
version = "0.8.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -15,9 +15,9 @@ path = "lib.rs"
[dependencies]
garage_db = { version = "0.8.0", path = "../db" }
garage_rpc = { version = "0.7.0", path = "../rpc" }
garage_util = { version = "0.7.0", path = "../util" }
garage_table = { version = "0.7.0", path = "../table" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
garage_util = { version = "0.8.0", path = "../util" }
garage_table = { version = "0.8.0", path = "../table" }
opentelemetry = "0.17"
@ -36,3 +36,7 @@ serde_bytes = "0.11"
futures = "0.3"
futures-util = "0.3"
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
[features]
system-libs = [ "zstd/pkg-config" ]

View file

@ -21,9 +21,9 @@ err-derive = "0.3"
hexdump = "0.1"
tracing = "0.1.30"
heed = "0.11"
rusqlite = { version = "0.27", features = ["bundled"] }
sled = "0.34"
heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true }
rusqlite = { version = "0.27", optional = true }
sled = { version = "0.34", optional = true }
# cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
@ -33,4 +33,7 @@ pretty_env_logger = { version = "0.4", optional = true }
mktemp = "0.4"
[features]
bundled-libs = [ "rusqlite/bundled" ]
cli = ["clap", "pretty_env_logger"]
lmdb = [ "heed" ]
sqlite = [ "rusqlite" ]

View file

@ -1,8 +1,15 @@
#[macro_use]
#[cfg(feature = "sqlite")]
extern crate tracing;
#[cfg(not(any(feature = "lmdb", feature = "sled", feature = "sqlite")))]
compile_error!("Must activate the Cargo feature for at least one DB engine: lmdb, sled or sqlite.");
#[cfg(feature = "lmdb")]
pub mod lmdb_adapter;
#[cfg(feature = "sled")]
pub mod sled_adapter;
#[cfg(feature = "sqlite")]
pub mod sqlite_adapter;
pub mod counted_tree_hack;

View file

@ -1,6 +1,6 @@
[package]
name = "garage"
version = "0.7.0"
version = "0.8.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -22,13 +22,13 @@ path = "tests/lib.rs"
[dependencies]
garage_db = { version = "0.8.0", path = "../db" }
garage_api = { version = "0.7.0", path = "../api" }
garage_block = { version = "0.7.0", path = "../block" }
garage_model = { version = "0.7.0", path = "../model" }
garage_rpc = { version = "0.7.0", path = "../rpc" }
garage_table = { version = "0.7.0", path = "../table" }
garage_util = { version = "0.7.0", path = "../util" }
garage_web = { version = "0.7.0", path = "../web" }
garage_api = { version = "0.8.0", path = "../api" }
garage_block = { version = "0.8.0", path = "../block" }
garage_model = { version = "0.8.0", path = "../model" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
garage_table = { version = "0.8.0", path = "../table" }
garage_util = { version = "0.8.0", path = "../util" }
garage_web = { version = "0.8.0", path = "../web" }
bytes = "1.0"
bytesize = "1.1"
@ -55,9 +55,9 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
netapp = "0.4"
opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
opentelemetry-prometheus = "0.10"
opentelemetry-otlp = "0.10"
prometheus = "0.13"
opentelemetry-prometheus = { version = "0.10", optional = true }
opentelemetry-otlp = { version = "0.10", optional = true }
prometheus = { version = "0.13", optional = true }
[dev-dependencies]
aws-sdk-s3 = "0.8"
@ -74,5 +74,26 @@ base64 = "0.13"
[features]
kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
default = [ "bundled-libs", "metrics", "sled" ]
k2v = [ "garage_util/k2v", "garage_api/k2v" ]
# Database engines, Sled is still our default even though we don't like it
sled = [ "garage_model/sled" ]
lmdb = [ "garage_model/lmdb" ]
sqlite = [ "garage_model/sqlite" ]
# Automatic registration and discovery via Kubernetes API
kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
# Prometheus exporter (/metrics endpoint).
metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ]
# Exporter for the OpenTelemetry Collector.
telemetry-otlp = [ "opentelemetry-otlp" ]
# NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
# exactly one of them should be enabled.
# Use bundled libsqlite instead of linking against system-provided.
bundled-libs = [ "garage_db/bundled-libs" ]
# Link against system-provided libsodium and libzstd.
system-libs = [ "garage_block/system-libs", "garage_rpc/system-libs", "sodiumoxide/use-pkg-config" ]

View file

@ -741,8 +741,11 @@ impl AdminRpcHandler {
let mut ret = String::new();
writeln!(
&mut ret,
"\nGarage version: {}",
self.garage.system.garage_version(),
"\nGarage version: {} [features: {}]",
garage_util::version::garage_version(),
garage_util::version::garage_features()
.map(|list| list.join(", "))
.unwrap_or_else(|| "(unknown)".into()),
)
.unwrap();
writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();

View file

@ -1,65 +1,65 @@
use serde::{Deserialize, Serialize};
use garage_util::version;
use structopt::StructOpt;
use garage_util::version::garage_version;
#[derive(StructOpt, Debug)]
pub enum Command {
/// Run Garage server
#[structopt(name = "server", version = version::garage())]
#[structopt(name = "server", version = garage_version())]
Server,
/// Get network status
#[structopt(name = "status", version = version::garage())]
#[structopt(name = "status", version = garage_version())]
Status,
/// Operations on individual Garage nodes
#[structopt(name = "node", version = version::garage())]
#[structopt(name = "node", version = garage_version())]
Node(NodeOperation),
/// Operations on the assignation of node roles in the cluster layout
#[structopt(name = "layout", version = version::garage())]
#[structopt(name = "layout", version = garage_version())]
Layout(LayoutOperation),
/// Operations on buckets
#[structopt(name = "bucket", version = version::garage())]
#[structopt(name = "bucket", version = garage_version())]
Bucket(BucketOperation),
/// Operations on S3 access keys
#[structopt(name = "key", version = version::garage())]
#[structopt(name = "key", version = garage_version())]
Key(KeyOperation),
/// Run migrations from previous Garage version
/// (DO NOT USE WITHOUT READING FULL DOCUMENTATION)
#[structopt(name = "migrate", version = version::garage())]
#[structopt(name = "migrate", version = garage_version())]
Migrate(MigrateOpt),
/// Start repair of node data on remote node
#[structopt(name = "repair", version = version::garage())]
#[structopt(name = "repair", version = garage_version())]
Repair(RepairOpt),
/// Offline reparation of node data (these repairs must be run offline
/// directly on the server node)
#[structopt(name = "offline-repair", version = version::garage())]
#[structopt(name = "offline-repair", version = garage_version())]
OfflineRepair(OfflineRepairOpt),
/// Gather node statistics
#[structopt(name = "stats", version = version::garage())]
#[structopt(name = "stats", version = garage_version())]
Stats(StatsOpt),
/// Manage background workers
#[structopt(name = "worker", version = version::garage())]
#[structopt(name = "worker", version = garage_version())]
Worker(WorkerOpt),
}
#[derive(StructOpt, Debug)]
pub enum NodeOperation {
/// Print identifier (public key) of this Garage node
#[structopt(name = "id", version = version::garage())]
#[structopt(name = "id", version = garage_version())]
NodeId(NodeIdOpt),
/// Connect to Garage node that is currently isolated from the system
#[structopt(name = "connect", version = version::garage())]
#[structopt(name = "connect", version = garage_version())]
Connect(ConnectNodeOpt),
}
@ -80,23 +80,23 @@ pub struct ConnectNodeOpt {
#[derive(StructOpt, Debug)]
pub enum LayoutOperation {
/// Assign role to Garage node
#[structopt(name = "assign", version = version::garage())]
#[structopt(name = "assign", version = garage_version())]
Assign(AssignRoleOpt),
/// Remove role from Garage cluster node
#[structopt(name = "remove", version = version::garage())]
#[structopt(name = "remove", version = garage_version())]
Remove(RemoveRoleOpt),
/// Show roles currently assigned to nodes and changes staged for commit
#[structopt(name = "show", version = version::garage())]
#[structopt(name = "show", version = garage_version())]
Show,
/// Apply staged changes to cluster layout
#[structopt(name = "apply", version = version::garage())]
#[structopt(name = "apply", version = garage_version())]
Apply(ApplyLayoutOpt),
/// Revert staged changes to cluster layout
#[structopt(name = "revert", version = version::garage())]
#[structopt(name = "revert", version = garage_version())]
Revert(RevertLayoutOpt),
}
@ -151,43 +151,43 @@ pub struct RevertLayoutOpt {
#[derive(Serialize, Deserialize, StructOpt, Debug)]
pub enum BucketOperation {
/// List buckets
#[structopt(name = "list", version = version::garage())]
#[structopt(name = "list", version = garage_version())]
List,
/// Get bucket info
#[structopt(name = "info", version = version::garage())]
#[structopt(name = "info", version = garage_version())]
Info(BucketOpt),
/// Create bucket
#[structopt(name = "create", version = version::garage())]
#[structopt(name = "create", version = garage_version())]
Create(BucketOpt),
/// Delete bucket
#[structopt(name = "delete", version = version::garage())]
#[structopt(name = "delete", version = garage_version())]
Delete(DeleteBucketOpt),
/// Alias bucket under new name
#[structopt(name = "alias", version = version::garage())]
#[structopt(name = "alias", version = garage_version())]
Alias(AliasBucketOpt),
/// Remove bucket alias
#[structopt(name = "unalias", version = version::garage())]
#[structopt(name = "unalias", version = garage_version())]
Unalias(UnaliasBucketOpt),
/// Allow key to read or write to bucket
#[structopt(name = "allow", version = version::garage())]
#[structopt(name = "allow", version = garage_version())]
Allow(PermBucketOpt),
/// Deny key from reading or writing to bucket
#[structopt(name = "deny", version = version::garage())]
#[structopt(name = "deny", version = garage_version())]
Deny(PermBucketOpt),
/// Expose as website or not
#[structopt(name = "website", version = version::garage())]
#[structopt(name = "website", version = garage_version())]
Website(WebsiteOpt),
/// Set the quotas for this bucket
#[structopt(name = "set-quotas", version = version::garage())]
#[structopt(name = "set-quotas", version = garage_version())]
SetQuotas(SetQuotasOpt),
}
@ -293,35 +293,35 @@ pub struct SetQuotasOpt {
#[derive(Serialize, Deserialize, StructOpt, Debug)]
pub enum KeyOperation {
/// List keys
#[structopt(name = "list", version = version::garage())]
#[structopt(name = "list", version = garage_version())]
List,
/// Get key info
#[structopt(name = "info", version = version::garage())]
#[structopt(name = "info", version = garage_version())]
Info(KeyOpt),
/// Create new key
#[structopt(name = "new", version = version::garage())]
#[structopt(name = "new", version = garage_version())]
New(KeyNewOpt),
/// Rename key
#[structopt(name = "rename", version = version::garage())]
#[structopt(name = "rename", version = garage_version())]
Rename(KeyRenameOpt),
/// Delete key
#[structopt(name = "delete", version = version::garage())]
#[structopt(name = "delete", version = garage_version())]
Delete(KeyDeleteOpt),
/// Set permission flags for key
#[structopt(name = "allow", version = version::garage())]
#[structopt(name = "allow", version = garage_version())]
Allow(KeyPermOpt),
/// Unset permission flags for key
#[structopt(name = "deny", version = version::garage())]
#[structopt(name = "deny", version = garage_version())]
Deny(KeyPermOpt),
/// Import key
#[structopt(name = "import", version = version::garage())]
#[structopt(name = "import", version = garage_version())]
Import(KeyImportOpt),
}
@ -393,7 +393,7 @@ pub struct MigrateOpt {
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
pub enum MigrateWhat {
/// Migrate buckets and permissions from v0.5.0
#[structopt(name = "buckets050", version = version::garage())]
#[structopt(name = "buckets050", version = garage_version())]
Buckets050,
}
@ -414,19 +414,19 @@ pub struct RepairOpt {
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
pub enum RepairWhat {
/// Only do a full sync of metadata tables
#[structopt(name = "tables", version = version::garage())]
#[structopt(name = "tables", version = garage_version())]
Tables,
/// Only repair (resync/rebalance) the set of stored blocks
#[structopt(name = "blocks", version = version::garage())]
#[structopt(name = "blocks", version = garage_version())]
Blocks,
/// Only redo the propagation of object deletions to the version table (slow)
#[structopt(name = "versions", version = version::garage())]
#[structopt(name = "versions", version = garage_version())]
Versions,
/// Only redo the propagation of version deletions to the block ref table (extremely slow)
#[structopt(name = "block_refs", version = version::garage())]
#[structopt(name = "block_refs", version = garage_version())]
BlockRefs,
/// Verify integrity of all blocks on disc (extremely slow, i/o intensive)
#[structopt(name = "scrub", version = version::garage())]
#[structopt(name = "scrub", version = garage_version())]
Scrub {
#[structopt(subcommand)]
cmd: ScrubCmd,
@ -436,19 +436,19 @@ pub enum RepairWhat {
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
pub enum ScrubCmd {
/// Start scrub
#[structopt(name = "start", version = version::garage())]
#[structopt(name = "start", version = garage_version())]
Start,
/// Pause scrub (it will resume automatically after 24 hours)
#[structopt(name = "pause", version = version::garage())]
#[structopt(name = "pause", version = garage_version())]
Pause,
/// Resume paused scrub
#[structopt(name = "resume", version = version::garage())]
#[structopt(name = "resume", version = garage_version())]
Resume,
/// Cancel scrub in progress
#[structopt(name = "cancel", version = version::garage())]
#[structopt(name = "cancel", version = garage_version())]
Cancel,
/// Set tranquility level for in-progress and future scrubs
#[structopt(name = "set-tranquility", version = version::garage())]
#[structopt(name = "set-tranquility", version = garage_version())]
SetTranquility {
#[structopt()]
tranquility: u32,
@ -469,10 +469,10 @@ pub struct OfflineRepairOpt {
pub enum OfflineRepairWhat {
/// Repair K2V item counters
#[cfg(feature = "k2v")]
#[structopt(name = "k2v_item_counters", version = version::garage())]
#[structopt(name = "k2v_item_counters", version = garage_version())]
K2VItemCounters,
/// Repair object counters
#[structopt(name = "object_counters", version = version::garage())]
#[structopt(name = "object_counters", version = garage_version())]
ObjectCounters,
}
@ -496,13 +496,13 @@ pub struct WorkerOpt {
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
pub enum WorkerCmd {
/// List all workers on Garage node
#[structopt(name = "list", version = version::garage())]
#[structopt(name = "list", version = garage_version())]
List {
#[structopt(flatten)]
opt: WorkerListOpt,
},
/// Set worker parameter
#[structopt(name = "set", version = version::garage())]
#[structopt(name = "set", version = garage_version())]
Set {
#[structopt(subcommand)]
opt: WorkerSetCmd,
@ -522,12 +522,12 @@ pub struct WorkerListOpt {
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
pub enum WorkerSetCmd {
/// Set tranquility of scrub operations
#[structopt(name = "scrub-tranquility", version = version::garage())]
#[structopt(name = "scrub-tranquility", version = garage_version())]
ScrubTranquility { tranquility: u32 },
/// Set number of concurrent block resync workers
#[structopt(name = "resync-n-workers", version = version::garage())]
#[structopt(name = "resync-n-workers", version = garage_version())]
ResyncNWorkers { n_workers: usize },
/// Set tranquility of block resync operations
#[structopt(name = "resync-tranquility", version = version::garage())]
#[structopt(name = "resync-tranquility", version = garage_version())]
ResyncTranquility { tranquility: u32 },
}

View file

@ -8,8 +8,15 @@ mod admin;
mod cli;
mod repair;
mod server;
#[cfg(feature = "telemetry-otlp")]
mod tracing_setup;
#[cfg(not(any(feature = "bundled-libs", feature = "system-libs")))]
compile_error!("Either bundled-libs or system-libs Cargo feature must be enabled");
#[cfg(all(feature = "bundled-libs", feature = "system-libs"))]
compile_error!("Only one of bundled-libs and system-libs Cargo features must be enabled");
use std::net::SocketAddr;
use std::path::PathBuf;
@ -22,7 +29,6 @@ use garage_util::error::*;
use garage_rpc::system::*;
use garage_rpc::*;
use garage_util::version;
use garage_model::helper::error::Error as HelperError;
@ -30,7 +36,10 @@ use admin::*;
use cli::*;
#[derive(StructOpt, Debug)]
#[structopt(name = "garage", version = version::garage(), about = "S3-compatible object store for self-hosted geo-distributed deployments")]
#[structopt(
name = "garage",
about = "S3-compatible object store for self-hosted geo-distributed deployments"
)]
struct Opt {
/// Host to connect to for admin operations, in the format:
/// <public-key>@<ip>:<port>
@ -71,7 +80,40 @@ async fn main() {
std::process::abort();
}));
let opt = Opt::from_args();
// Initialize version and features info
let features = &[
#[cfg(feature = "k2v")]
"k2v",
#[cfg(feature = "sled")]
"sled",
#[cfg(feature = "lmdb")]
"lmdb",
#[cfg(feature = "sqlite")]
"sqlite",
#[cfg(feature = "kubernetes-discovery")]
"kubernetes-discovery",
#[cfg(feature = "metrics")]
"metrics",
#[cfg(feature = "telemetry-otlp")]
"telemetry-otlp",
#[cfg(feature = "bundled-libs")]
"bundled-libs",
#[cfg(feature = "system-libs")]
"system-libs",
][..];
if let Some(git_version) = option_env!("GIT_VERSION") {
garage_util::version::init_version(git_version);
}
garage_util::version::init_features(features);
// Parse arguments
let version = format!(
"{} [features: {}]",
garage_util::version::garage_version(),
features.join(", ")
);
let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
let res = match opt.cmd {
Command::Server => server::run_server(opt.config_file).await,
Command::OfflineRepair(repair_opt) => {

View file

@ -9,12 +9,13 @@ use garage_util::error::Error;
use garage_api::admin::api_server::AdminApiServer;
use garage_api::s3::api_server::S3ApiServer;
use garage_model::garage::Garage;
use garage_web::run_web_server;
use garage_web::WebServer;
#[cfg(feature = "k2v")]
use garage_api::k2v::api_server::K2VApiServer;
use crate::admin::*;
#[cfg(feature = "telemetry-otlp")]
use crate::tracing_setup::*;
async fn wait_from(mut chan: watch::Receiver<bool>) {
@ -29,6 +30,8 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
info!("Loading configuration...");
let config = read_config(config_file)?;
// ---- Initialize Garage internals ----
info!("Initializing background runner...");
let watch_cancel = netapp::util::watch_ctrl_c();
let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
@ -36,9 +39,14 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
info!("Initializing Garage main data store...");
let garage = Garage::new(config.clone(), background)?;
if config.admin.trace_sink.is_some() {
info!("Initialize tracing...");
if let Some(export_to) = config.admin.trace_sink {
init_tracing(&export_to, garage.system.id)?;
#[cfg(feature = "telemetry-otlp")]
init_tracing(config.admin.trace_sink.as_ref().unwrap(), garage.system.id)?;
#[cfg(not(feature = "telemetry-otlp"))]
error!("Garage was built without OTLP exporter, admin.trace_sink is ignored.");
}
info!("Initialize Admin API server and metrics collector...");
@ -50,53 +58,78 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
info!("Create admin RPC handler...");
AdminRpcHandler::new(garage.clone());
info!("Initializing S3 API server...");
let s3_api_server = tokio::spawn(S3ApiServer::run(
garage.clone(),
wait_from(watch_cancel.clone()),
));
// ---- Launch public-facing API servers ----
let mut servers = vec![];
if let Some(s3_bind_addr) = &config.s3_api.api_bind_addr {
info!("Initializing S3 API server...");
servers.push((
"S3 API",
tokio::spawn(S3ApiServer::run(
garage.clone(),
*s3_bind_addr,
config.s3_api.s3_region.clone(),
wait_from(watch_cancel.clone()),
)),
));
}
if config.k2v_api.is_some() {
#[cfg(feature = "k2v")]
let k2v_api_server = {
{
info!("Initializing K2V API server...");
servers.push((
"K2V API",
tokio::spawn(K2VApiServer::run(
garage.clone(),
config.k2v_api.as_ref().unwrap().api_bind_addr,
config.s3_api.s3_region.clone(),
wait_from(watch_cancel.clone()),
))
};
info!("Initializing web server...");
let web_server = tokio::spawn(run_web_server(
garage.clone(),
wait_from(watch_cancel.clone()),
)),
));
}
#[cfg(not(feature = "k2v"))]
error!("K2V is not enabled in this build, cannot start K2V API server");
}
if let Some(web_config) = &config.s3_web {
info!("Initializing web server...");
servers.push((
"Web",
tokio::spawn(WebServer::run(
garage.clone(),
web_config.bind_addr,
web_config.root_domain.clone(),
wait_from(watch_cancel.clone()),
)),
));
}
if let Some(admin_bind_addr) = &config.admin.api_bind_addr {
info!("Launching Admin API server...");
let admin_server = tokio::spawn(admin_server.run(wait_from(watch_cancel.clone())));
servers.push((
"Admin",
tokio::spawn(admin_server.run(*admin_bind_addr, wait_from(watch_cancel.clone()))),
));
}
#[cfg(not(feature = "metrics"))]
if config.admin.metrics_token.is_some() {
warn!("This Garage version is built without the metrics feature");
}
// Stuff runs
// When a cancel signal is sent, stuff stops
if let Err(e) = s3_api_server.await? {
warn!("S3 API server exited with error: {}", e);
// Collect stuff
for (desc, join_handle) in servers {
if let Err(e) = join_handle.await? {
error!("{} server exited with error: {}", desc, e);
} else {
info!("S3 API server exited without error.");
info!("{} server exited without error.", desc);
}
#[cfg(feature = "k2v")]
if let Err(e) = k2v_api_server.await? {
warn!("K2V API server exited with error: {}", e);
} else {
info!("K2V API server exited without error.");
}
if let Err(e) = web_server.await? {
warn!("Web server exited with error: {}", e);
} else {
info!("Web server exited without error.");
}
if let Err(e) = admin_server.await? {
warn!("Admin web server exited with error: {}", e);
} else {
info!("Admin API server exited without error.");
}
// Remove RPC handlers for system to break reference cycles

View file

@ -4,7 +4,7 @@ mod common;
mod admin;
mod bucket;
mod s3;
#[cfg(feature = "k2v")]
mod k2v;
mod s3;

View file

@ -22,7 +22,7 @@ tokio = "1.17.0"
# cli deps
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
garage_util = { version = "0.7.0", path = "../util", optional = true }
garage_util = { version = "0.8.0", path = "../util", optional = true }
[features]

View file

@ -1,6 +1,6 @@
[package]
name = "garage_model"
version = "0.7.0"
version = "0.8.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -15,11 +15,10 @@ path = "lib.rs"
[dependencies]
garage_db = { version = "0.8.0", path = "../db" }
garage_rpc = { version = "0.7.0", path = "../rpc" }
garage_table = { version = "0.7.0", path = "../table" }
garage_block = { version = "0.7.0", path = "../block" }
garage_util = { version = "0.7.0", path = "../util" }
garage_model_050 = { package = "garage_model", version = "0.5.1" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
garage_table = { version = "0.8.0", path = "../table" }
garage_block = { version = "0.8.0", path = "../block" }
garage_util = { version = "0.8.0", path = "../util" }
async-trait = "0.1.7"
arc-swap = "1.0"
@ -46,3 +45,6 @@ netapp = "0.4"
[features]
k2v = [ "garage_util/k2v" ]
lmdb = [ "garage_db/lmdb" ]
sled = [ "garage_db/sled" ]
sqlite = [ "garage_db/sqlite" ]

View file

@ -80,6 +80,8 @@ impl Garage {
let mut db_path = config.metadata_dir.clone();
std::fs::create_dir_all(&db_path).expect("Unable to create Garage meta data directory");
let db = match config.db_engine.as_str() {
// ---- Sled DB ----
#[cfg(feature = "sled")]
"sled" => {
db_path.push("db");
info!("Opening Sled database at: {}", db_path.display());
@ -91,6 +93,10 @@ impl Garage {
.expect("Unable to open sled DB");
db::sled_adapter::SledDb::init(db)
}
#[cfg(not(feature = "sled"))]
"sled" => return Err(Error::Message("sled db not available in this build".into())),
// ---- Sqlite DB ----
#[cfg(feature = "sqlite")]
"sqlite" | "sqlite3" | "rusqlite" => {
db_path.push("db.sqlite");
info!("Opening Sqlite database at: {}", db_path.display());
@ -98,6 +104,14 @@ impl Garage {
.expect("Unable to open sqlite DB");
db::sqlite_adapter::SqliteDb::init(db)
}
#[cfg(not(feature = "sqlite"))]
"sqlite" | "sqlite3" | "rusqlite" => {
return Err(Error::Message(
"sqlite db not available in this build".into(),
))
}
// ---- LMDB DB ----
#[cfg(feature = "lmdb")]
"lmdb" | "heed" => {
db_path.push("db.lmdb");
info!("Opening LMDB database at: {}", db_path.display());
@ -116,10 +130,22 @@ impl Garage {
let db = env_builder.open(&db_path).expect("Unable to open LMDB DB");
db::lmdb_adapter::LmdbDb::init(db)
}
#[cfg(not(feature = "lmdb"))]
"lmdb" | "heed" => return Err(Error::Message("lmdb db not available in this build".into())),
// ---- Unavailable DB engine ----
e => {
return Err(Error::Message(format!(
"Unsupported DB engine: {} (options: sled, sqlite, lmdb)",
e
"Unsupported DB engine: {} (options: {})",
e,
vec![
#[cfg(feature = "sled")]
"sled",
#[cfg(feature = "sqlite")]
"sqlite",
#[cfg(feature = "lmdb")]
"lmdb",
]
.join(", ")
)));
}
};

View file

@ -6,7 +6,7 @@ use garage_util::data::*;
use crate::permission::BucketKeyPerm;
use garage_model_050::key_table as old;
use crate::prev::v051::key_table as old;
/// An api key
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]

View file

@ -1,6 +1,9 @@
#[macro_use]
extern crate tracing;
// For migration from previous versions
pub(crate) mod prev;
pub mod permission;
pub mod index_counter;

View file

@ -5,7 +5,7 @@ use garage_util::data::*;
use garage_util::error::Error as GarageError;
use garage_util::time::*;
use garage_model_050::bucket_table as old_bucket;
use crate::prev::v051::bucket_table as old_bucket;
use crate::bucket_alias_table::*;
use crate::bucket_table::*;

1
src/model/prev/mod.rs Normal file
View file

@ -0,0 +1 @@
pub(crate) mod v051;

View file

@ -0,0 +1,63 @@
use serde::{Deserialize, Serialize};
use garage_table::crdt::Crdt;
use garage_table::*;
use super::key_table::PermissionSet;
/// A bucket is a collection of objects
///
/// Its parameters are not directly accessible as:
/// - It must be possible to merge paramaters, hence the use of a LWW CRDT.
/// - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct Bucket {
/// Name of the bucket
pub name: String,
/// State, and configuration if not deleted, of the bucket
pub state: crdt::Lww<BucketState>,
}
/// State of a bucket
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub enum BucketState {
/// The bucket is deleted
Deleted,
/// The bucket exists
Present(BucketParams),
}
impl Crdt for BucketState {
fn merge(&mut self, o: &Self) {
match o {
BucketState::Deleted => *self = BucketState::Deleted,
BucketState::Present(other_params) => {
if let BucketState::Present(params) = self {
params.merge(other_params);
}
}
}
}
}
/// Configuration for a bucket
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct BucketParams {
/// Map of key with access to the bucket, and what kind of access they give
pub authorized_keys: crdt::LwwMap<String, PermissionSet>,
/// Is the bucket served as http
pub website: crdt::Lww<bool>,
}
impl Crdt for BucketParams {
fn merge(&mut self, o: &Self) {
self.authorized_keys.merge(&o.authorized_keys);
self.website.merge(&o.website);
}
}
impl Crdt for Bucket {
fn merge(&mut self, other: &Self) {
self.state.merge(&other.state);
}
}

View file

@ -0,0 +1,50 @@
use serde::{Deserialize, Serialize};
use garage_table::crdt::*;
use garage_table::*;
/// An api key
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct Key {
/// The id of the key (immutable), used as partition key
pub key_id: String,
/// The secret_key associated
pub secret_key: String,
/// Name for the key
pub name: crdt::Lww<String>,
/// Is the key deleted
pub deleted: crdt::Bool,
/// Buckets in which the key is authorized. Empty if `Key` is deleted
// CRDT interaction: deleted implies authorized_buckets is empty
pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
}
/// Permission given to a key in a bucket
#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
pub struct PermissionSet {
/// The key can be used to read the bucket
pub allow_read: bool,
/// The key can be used to write in the bucket
pub allow_write: bool,
}
impl AutoCrdt for PermissionSet {
const WARN_IF_DIFFERENT: bool = true;
}
impl Crdt for Key {
fn merge(&mut self, other: &Self) {
self.name.merge(&other.name);
self.deleted.merge(&other.deleted);
if self.deleted.get() {
self.authorized_buckets.clear();
} else {
self.authorized_buckets.merge(&other.authorized_buckets);
}
}
}

View file

@ -0,0 +1,4 @@
pub(crate) mod bucket_table;
pub(crate) mod key_table;
pub(crate) mod object_table;
pub(crate) mod version_table;

View file

@ -0,0 +1,149 @@
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use garage_util::data::*;
use garage_table::crdt::*;
/// An object
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct Object {
/// The bucket in which the object is stored, used as partition key
pub bucket: String,
/// The key at which the object is stored in its bucket, used as sorting key
pub key: String,
/// The list of currenty stored versions of the object
versions: Vec<ObjectVersion>,
}
impl Object {
/// Get a list of currently stored versions of `Object`
pub fn versions(&self) -> &[ObjectVersion] {
&self.versions[..]
}
}
/// Informations about a version of an object
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct ObjectVersion {
/// Id of the version
pub uuid: Uuid,
/// Timestamp of when the object was created
pub timestamp: u64,
/// State of the version
pub state: ObjectVersionState,
}
/// State of an object version
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub enum ObjectVersionState {
/// The version is being received
Uploading(ObjectVersionHeaders),
/// The version is fully received
Complete(ObjectVersionData),
/// The version uploaded containded errors or the upload was explicitly aborted
Aborted,
}
impl Crdt for ObjectVersionState {
fn merge(&mut self, other: &Self) {
use ObjectVersionState::*;
match other {
Aborted => {
*self = Aborted;
}
Complete(b) => match self {
Aborted => {}
Complete(a) => {
a.merge(b);
}
Uploading(_) => {
*self = Complete(b.clone());
}
},
Uploading(_) => {}
}
}
}
/// Data stored in object version
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
pub enum ObjectVersionData {
/// The object was deleted, this Version is a tombstone to mark it as such
DeleteMarker,
/// The object is short, it's stored inlined
Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
/// The object is not short, Hash of first block is stored here, next segments hashes are
/// stored in the version table
FirstBlock(ObjectVersionMeta, Hash),
}
impl AutoCrdt for ObjectVersionData {
const WARN_IF_DIFFERENT: bool = true;
}
/// Metadata about the object version
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
pub struct ObjectVersionMeta {
/// Headers to send to the client
pub headers: ObjectVersionHeaders,
/// Size of the object
pub size: u64,
/// etag of the object
pub etag: String,
}
/// Additional headers for an object
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
pub struct ObjectVersionHeaders {
/// Content type of the object
pub content_type: String,
/// Any other http headers to send
pub other: BTreeMap<String, String>,
}
impl ObjectVersion {
fn cmp_key(&self) -> (u64, Uuid) {
(self.timestamp, self.uuid)
}
/// Is the object version completely received
pub fn is_complete(&self) -> bool {
matches!(self.state, ObjectVersionState::Complete(_))
}
}
impl Crdt for Object {
fn merge(&mut self, other: &Self) {
// Merge versions from other into here
for other_v in other.versions.iter() {
match self
.versions
.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
{
Ok(i) => {
self.versions[i].state.merge(&other_v.state);
}
Err(i) => {
self.versions.insert(i, other_v.clone());
}
}
}
// Remove versions which are obsolete, i.e. those that come
// before the last version which .is_complete().
let last_complete = self
.versions
.iter()
.enumerate()
.rev()
.find(|(_, v)| v.is_complete())
.map(|(vi, _)| vi);
if let Some(last_vi) = last_complete {
self.versions = self.versions.drain(last_vi..).collect::<Vec<_>>();
}
}
}

View file

@ -0,0 +1,79 @@
use serde::{Deserialize, Serialize};
use garage_util::data::*;
use garage_table::crdt::*;
use garage_table::*;
/// A version of an object
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct Version {
/// UUID of the version, used as partition key
pub uuid: Uuid,
// Actual data: the blocks for this version
// In the case of a multipart upload, also store the etags
// of individual parts and check them when doing CompleteMultipartUpload
/// Is this version deleted
pub deleted: crdt::Bool,
/// list of blocks of data composing the version
pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
/// Etag of each part in case of a multipart upload, empty otherwise
pub parts_etags: crdt::Map<u64, String>,
// Back link to bucket+key so that we can figure if
// this was deleted later on
/// Bucket in which the related object is stored
pub bucket: String,
/// Key in which the related object is stored
pub key: String,
}
#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
pub struct VersionBlockKey {
/// Number of the part
pub part_number: u64,
/// Offset of this sub-segment in its part
pub offset: u64,
}
impl Ord for VersionBlockKey {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.part_number
.cmp(&other.part_number)
.then(self.offset.cmp(&other.offset))
}
}
impl PartialOrd for VersionBlockKey {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
/// Informations about a single block
#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
pub struct VersionBlock {
/// Blake2 sum of the block
pub hash: Hash,
/// Size of the block
pub size: u64,
}
impl AutoCrdt for VersionBlock {
const WARN_IF_DIFFERENT: bool = true;
}
impl Crdt for Version {
fn merge(&mut self, other: &Self) {
self.deleted.merge(&other.deleted);
if self.deleted.get() {
self.blocks.clear();
self.parts_etags.clear();
} else {
self.blocks.merge(&other.blocks);
self.parts_etags.merge(&other.parts_etags);
}
}
}

View file

@ -14,7 +14,7 @@ use garage_table::*;
use crate::index_counter::*;
use crate::s3::version_table::*;
use garage_model_050::object_table as old;
use crate::prev::v051::object_table as old;
pub const OBJECTS: &str = "objects";
pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads";

View file

@ -12,7 +12,7 @@ use garage_table::*;
use crate::s3::block_ref_table::*;
use garage_model_050::version_table as old;
use crate::prev::v051::version_table as old;
/// A version of an object
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]

View file

@ -1,6 +1,6 @@
[package]
name = "garage_rpc"
version = "0.7.0"
version = "0.8.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,7 +14,7 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_util = { version = "0.7.0", path = "../util" }
garage_util = { version = "0.8.0", path = "../util" }
arc-swap = "1.0"
bytes = "1.0"
@ -54,3 +54,4 @@ hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] }
[features]
kubernetes-discovery = [ "kube", "k8s-openapi", "openssl", "schemars" ]
system-libs = [ "sodiumoxide/use-pkg-config" ]

View file

@ -27,7 +27,6 @@ use garage_util::data::*;
use garage_util::error::*;
use garage_util::persister::Persister;
use garage_util::time::*;
use garage_util::version;
use crate::consul::*;
#[cfg(feature = "kubernetes-discovery")]
@ -40,8 +39,10 @@ const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60);
const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10);
const PING_TIMEOUT: Duration = Duration::from_secs(2);
/// Version tag used for version check upon Netapp connection
pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650007; // garage 0x0007
/// Version tag used for version check upon Netapp connection.
/// Cluster nodes with different version tags are deemed
/// incompatible and will refuse to connect.
pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650008; // garage 0x0008
/// RPC endpoint used for calls related to membership
pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc";
@ -320,10 +321,6 @@ impl System {
// ---- Administrative operations (directly available and
// also available through RPC) ----
pub fn garage_version(&self) -> &'static str {
version::garage()
}
pub fn get_known_nodes(&self) -> Vec<KnownNodeInfo> {
let node_status = self.node_status.read().unwrap();
let known_nodes = self

View file

@ -1,6 +1,6 @@
[package]
name = "garage_table"
version = "0.7.0"
version = "0.8.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -15,8 +15,8 @@ path = "lib.rs"
[dependencies]
garage_db = { version = "0.8.0", path = "../db" }
garage_rpc = { version = "0.7.0", path = "../rpc" }
garage_util = { version = "0.7.0", path = "../util" }
garage_rpc = { version = "0.8.0", path = "../rpc" }
garage_util = { version = "0.8.0", path = "../util" }
opentelemetry = "0.17"

View file

@ -1,6 +1,6 @@
[package]
name = "garage_util"
version = "0.7.0"
version = "0.8.0"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
license = "AGPL-3.0"
@ -16,17 +16,19 @@ path = "lib.rs"
[dependencies]
garage_db = { version = "0.8.0", path = "../db" }
arc-swap = "1.0"
async-trait = "0.1"
blake2 = "0.9"
bytes = "1.0"
digest = "0.10"
err-derive = "0.3"
git-version = "0.3.4"
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
hex = "0.4"
lazy_static = "1.4"
tracing = "0.1.30"
rand = "0.8"
sha2 = "0.10"
git-version = "0.3.4"
chrono = "0.4"
rmp-serde = "0.15"

View file

@ -77,11 +77,10 @@ pub struct Config {
pub s3_api: S3ApiConfig,
/// Configuration for K2V api
#[cfg(feature = "k2v")]
pub k2v_api: Option<K2VApiConfig>,
/// Configuration for serving files as normal web server
pub s3_web: WebConfig,
pub s3_web: Option<WebConfig>,
/// Configuration for the admin API endpoint
#[serde(default = "Default::default")]
@ -92,7 +91,7 @@ pub struct Config {
#[derive(Deserialize, Debug, Clone)]
pub struct S3ApiConfig {
/// Address and port to bind for api serving
pub api_bind_addr: SocketAddr,
pub api_bind_addr: Option<SocketAddr>,
/// S3 region to use
pub s3_region: String,
/// Suffix to remove from domain name to find bucket. If None,
@ -101,7 +100,6 @@ pub struct S3ApiConfig {
}
/// Configuration for K2V api
#[cfg(feature = "k2v")]
#[derive(Deserialize, Debug, Clone)]
pub struct K2VApiConfig {
/// Address and port to bind for api serving

View file

@ -1,7 +1,28 @@
pub fn garage() -> &'static str {
option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
use std::sync::Arc;
use arc_swap::{ArcSwap, ArcSwapOption};
lazy_static::lazy_static! {
static ref VERSION: ArcSwap<&'static str> = ArcSwap::new(Arc::new(git_version::git_version!(
prefix = "git:",
cargo_prefix = "cargo:",
fallback = "unknown"
))
)));
static ref FEATURES: ArcSwapOption<&'static [&'static str]> = ArcSwapOption::new(None);
}
pub fn garage_version() -> &'static str {
&VERSION.load()
}
pub fn garage_features() -> Option<&'static [&'static str]> {
FEATURES.load().as_ref().map(|f| &f[..])
}
pub fn init_version(version: &'static str) {
VERSION.store(Arc::new(version));
}
pub fn init_features(features: &'static [&'static str]) {
FEATURES.store(Some(Arc::new(features)));
}

View file

@ -1,6 +1,6 @@
[package]
name = "garage_web"
version = "0.7.0"
version = "0.8.0"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018"
license = "AGPL-3.0"
@ -14,10 +14,10 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
garage_api = { version = "0.7.0", path = "../api" }
garage_model = { version = "0.7.0", path = "../model" }
garage_util = { version = "0.7.0", path = "../util" }
garage_table = { version = "0.7.0", path = "../table" }
garage_api = { version = "0.8.0", path = "../api" }
garage_model = { version = "0.8.0", path = "../model" }
garage_util = { version = "0.8.0", path = "../util" }
garage_table = { version = "0.8.0", path = "../table" }
err-derive = "0.3"
tracing = "0.1.30"

View file

@ -6,4 +6,4 @@ mod error;
pub use error::Error;
mod web_server;
pub use web_server::run_web_server;
pub use web_server::WebServer;

View file

@ -57,44 +57,53 @@ impl WebMetrics {
}
}
/// Run a web server
pub async fn run_web_server(
pub struct WebServer {
garage: Arc<Garage>,
shutdown_signal: impl Future<Output = ()>,
) -> Result<(), GarageError> {
let addr = &garage.config.s3_web.bind_addr;
metrics: Arc<WebMetrics>,
root_domain: String,
}
impl WebServer {
/// Run a web server
pub async fn run(
garage: Arc<Garage>,
addr: SocketAddr,
root_domain: String,
shutdown_signal: impl Future<Output = ()>,
) -> Result<(), GarageError> {
let metrics = Arc::new(WebMetrics::new());
let web_server = Arc::new(WebServer {
garage,
metrics,
root_domain,
});
let service = make_service_fn(|conn: &AddrStream| {
let garage = garage.clone();
let metrics = metrics.clone();
let web_server = web_server.clone();
let client_addr = conn.remote_addr();
async move {
Ok::<_, Error>(service_fn(move |req: Request<Body>| {
let garage = garage.clone();
let metrics = metrics.clone();
let web_server = web_server.clone();
handle_request(garage, metrics, req, client_addr)
web_server.handle_request(req, client_addr)
}))
}
});
let server = Server::bind(addr).serve(service);
let server = Server::bind(&addr).serve(service);
let graceful = server.with_graceful_shutdown(shutdown_signal);
info!("Web server listening on http://{}", addr);
graceful.await?;
Ok(())
}
}
async fn handle_request(
garage: Arc<Garage>,
metrics: Arc<WebMetrics>,
async fn handle_request(
self: Arc<Self>,
req: Request<Body>,
addr: SocketAddr,
) -> Result<Response<Body>, Infallible> {
) -> Result<Response<Body>, Infallible> {
info!("{} {} {}", addr, req.method(), req.uri());
// Lots of instrumentation
@ -111,13 +120,14 @@ async fn handle_request(
let metrics_tags = &[KeyValue::new("method", req.method().to_string())];
// The actual handler
let res = serve_file(garage, &req)
let res = self
.serve_file(&req)
.with_context(Context::current_with_span(span))
.record_duration(&metrics.request_duration, &metrics_tags[..])
.record_duration(&self.metrics.request_duration, &metrics_tags[..])
.await;
// More instrumentation
metrics.request_counter.add(1, &metrics_tags[..]);
self.metrics.request_counter.add(1, &metrics_tags[..]);
// Returning the result
match res {
@ -133,7 +143,7 @@ async fn handle_request(
req.uri(),
error
);
metrics.error_counter.add(
self.metrics.error_counter.add(
1,
&[
metrics_tags[0].clone(),
@ -143,24 +153,9 @@ async fn handle_request(
Ok(error_to_res(error))
}
}
}
}
fn error_to_res(e: Error) -> Response<Body> {
// If we are here, it is either that:
// - there was an error before trying to get the requested URL
// from the bucket (e.g. bucket not found)
// - there was an error processing the request and (the request
// was a HEAD request or we couldn't get the error document)
// We do NOT enter this code path when returning the bucket's
// error document (this is handled in serve_file)
let body = Body::from(format!("{}\n", e));
let mut http_error = Response::new(body);
*http_error.status_mut() = e.http_status_code();
e.add_headers(http_error.headers_mut());
http_error
}
async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response<Body>, Error> {
async fn serve_file(self: &Arc<Self>, req: &Request<Body>) -> Result<Response<Body>, Error> {
// Get http authority string (eg. [::1]:3902 or garage.tld:80)
let authority = req
.headers()
@ -170,10 +165,10 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
// Get bucket
let host = authority_to_host(authority)?;
let root = &garage.config.s3_web.root_domain;
let bucket_name = host_to_bucket(&host, root).unwrap_or(&host);
let bucket_id = garage
let bucket_name = host_to_bucket(&host, &self.root_domain).unwrap_or(&host);
let bucket_id = self
.garage
.bucket_alias_table
.get(&EmptyKey, &bucket_name.to_string())
.await?
@ -181,7 +176,8 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
.ok_or(Error::NotFound)?;
// Check bucket isn't deleted and has website access enabled
let bucket = garage
let bucket = self
.garage
.bucket_table
.get(&EmptyKey, &bucket_id)
.await?
@ -207,8 +203,8 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
let ret_doc = match *req.method() {
Method::OPTIONS => handle_options_for_bucket(req, &bucket),
Method::HEAD => handle_head(garage.clone(), req, bucket_id, &key, None).await,
Method::GET => handle_get(garage.clone(), req, bucket_id, &key, None).await,
Method::HEAD => handle_head(self.garage.clone(), req, bucket_id, &key, None).await,
Method::GET => handle_get(self.garage.clone(), req, bucket_id, &key, None).await,
_ => Err(ApiError::bad_request("HTTP method not supported")),
}
.map_err(Error::from);
@ -239,7 +235,8 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
.body(Body::empty())
.unwrap();
match handle_get(garage, &req2, bucket_id, &error_document, None).await {
match handle_get(self.garage.clone(), &req2, bucket_id, &error_document, None).await
{
Ok(mut error_doc) => {
// The error won't be logged back in handle_request,
// so log it here
@ -281,6 +278,22 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
Ok(resp)
}
}
}
}
fn error_to_res(e: Error) -> Response<Body> {
// If we are here, it is either that:
// - there was an error before trying to get the requested URL
// from the bucket (e.g. bucket not found)
// - there was an error processing the request and (the request
// was a HEAD request or we couldn't get the error document)
// We do NOT enter this code path when returning the bucket's
// error document (this is handled in serve_file)
let body = Body::from(format!("{}\n", e));
let mut http_error = Response::new(body);
*http_error.status_mut() = e.http_status_code();
e.add_headers(http_error.headers_mut());
http_error
}
/// Path to key