Merge branch 'lx-perf-improvements' into netapp-stream-body
This commit is contained in:
commit
7f54706b95
GPG key ID:
0E496D15096376BE
42 changed files with 1328 additions and 1064 deletions
316
Cargo.lock
generated
316
Cargo.lock
generated
|
|
@ -177,7 +177,7 @@ dependencies = [
|
||||||
"aws-smithy-types",
|
"aws-smithy-types",
|
||||||
"aws-smithy-xml",
|
"aws-smithy-xml",
|
||||||
"aws-types",
|
"aws-types",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"http",
|
"http",
|
||||||
"md5",
|
"md5",
|
||||||
"tokio-stream",
|
"tokio-stream",
|
||||||
|
|
@ -207,7 +207,7 @@ checksum = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-smithy-eventstream",
|
"aws-smithy-eventstream",
|
||||||
"aws-smithy-http",
|
"aws-smithy-http",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"form_urlencoded",
|
"form_urlencoded",
|
||||||
"hex",
|
"hex",
|
||||||
"http",
|
"http",
|
||||||
|
|
@ -241,7 +241,7 @@ dependencies = [
|
||||||
"aws-smithy-http",
|
"aws-smithy-http",
|
||||||
"aws-smithy-http-tower",
|
"aws-smithy-http-tower",
|
||||||
"aws-smithy-types",
|
"aws-smithy-types",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"fastrand",
|
"fastrand",
|
||||||
"http",
|
"http",
|
||||||
"http-body",
|
"http-body",
|
||||||
|
|
@ -262,7 +262,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"
|
checksum = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-smithy-types",
|
"aws-smithy-types",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"crc32fast",
|
"crc32fast",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -274,7 +274,7 @@ checksum = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-smithy-eventstream",
|
"aws-smithy-eventstream",
|
||||||
"aws-smithy-types",
|
"aws-smithy-types",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"bytes-utils",
|
"bytes-utils",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"http",
|
"http",
|
||||||
|
|
@ -294,7 +294,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"
|
checksum = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-smithy-http",
|
"aws-smithy-http",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"http",
|
"http",
|
||||||
"http-body",
|
"http-body",
|
||||||
"pin-project 1.0.10",
|
"pin-project 1.0.10",
|
||||||
|
|
@ -406,12 +406,6 @@ version = "1.4.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "bytes"
|
|
||||||
version = "0.6.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bytes"
|
name = "bytes"
|
||||||
version = "1.2.0"
|
version = "1.2.0"
|
||||||
|
|
@ -424,7 +418,7 @@ version = "0.1.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1934a3ef9cac8efde4966a92781e77713e1ba329f1d42e446c7d7eba340d8ef1"
|
checksum = "1934a3ef9cac8efde4966a92781e77713e1ba329f1d42e446c7d7eba340d8ef1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"either",
|
"either",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -797,20 +791,6 @@ dependencies = [
|
||||||
"termcolor",
|
"termcolor",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "err-derive"
|
|
||||||
version = "0.2.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro-error",
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"rustversion",
|
|
||||||
"syn",
|
|
||||||
"synstructure",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "err-derive"
|
name = "err-derive"
|
||||||
version = "0.3.1"
|
version = "0.3.1"
|
||||||
|
|
@ -999,13 +979,13 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage"
|
name = "garage"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"assert-json-diff",
|
"assert-json-diff",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"aws-sdk-s3",
|
"aws-sdk-s3",
|
||||||
"base64",
|
"base64",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"bytesize",
|
"bytesize",
|
||||||
"chrono",
|
"chrono",
|
||||||
"futures",
|
"futures",
|
||||||
|
|
@ -1013,23 +993,23 @@ dependencies = [
|
||||||
"garage_api",
|
"garage_api",
|
||||||
"garage_block",
|
"garage_block",
|
||||||
"garage_db",
|
"garage_db",
|
||||||
"garage_model 0.7.0",
|
"garage_model",
|
||||||
"garage_rpc 0.7.0",
|
"garage_rpc",
|
||||||
"garage_table 0.7.0",
|
"garage_table",
|
||||||
"garage_util 0.7.0",
|
"garage_util",
|
||||||
"garage_web",
|
"garage_web",
|
||||||
"hex",
|
"hex",
|
||||||
"hmac 0.12.1",
|
"hmac 0.12.1",
|
||||||
"http",
|
"http",
|
||||||
"hyper",
|
"hyper",
|
||||||
"kuska-sodiumoxide",
|
"kuska-sodiumoxide",
|
||||||
"netapp 0.5.0",
|
"netapp",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"opentelemetry-otlp",
|
"opentelemetry-otlp",
|
||||||
"opentelemetry-prometheus",
|
"opentelemetry-prometheus",
|
||||||
"prometheus",
|
"prometheus",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"rmp-serde 0.15.5",
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
|
@ -1045,22 +1025,22 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_api"
|
name = "garage_api"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64",
|
"base64",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
"crypto-common",
|
"crypto-common",
|
||||||
"err-derive 0.3.1",
|
"err-derive",
|
||||||
"form_urlencoded",
|
"form_urlencoded",
|
||||||
"futures",
|
"futures",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"garage_block",
|
"garage_block",
|
||||||
"garage_model 0.7.0",
|
"garage_model",
|
||||||
"garage_rpc 0.7.0",
|
"garage_rpc",
|
||||||
"garage_table 0.7.0",
|
"garage_table",
|
||||||
"garage_util 0.7.0",
|
"garage_util",
|
||||||
"hex",
|
"hex",
|
||||||
"hmac 0.12.1",
|
"hmac 0.12.1",
|
||||||
"http",
|
"http",
|
||||||
|
|
@ -1072,7 +1052,6 @@ dependencies = [
|
||||||
"multer",
|
"multer",
|
||||||
"nom",
|
"nom",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"opentelemetry-otlp",
|
|
||||||
"opentelemetry-prometheus",
|
"opentelemetry-prometheus",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
"pin-project 1.0.10",
|
"pin-project 1.0.10",
|
||||||
|
|
@ -1090,22 +1069,22 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_block"
|
name = "garage_block"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-compression",
|
"async-compression",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"futures",
|
"futures",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"garage_db",
|
"garage_db",
|
||||||
"garage_rpc 0.7.0",
|
"garage_rpc",
|
||||||
"garage_table 0.7.0",
|
"garage_table",
|
||||||
"garage_util 0.7.0",
|
"garage_util",
|
||||||
"hex",
|
"hex",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"rmp-serde 0.15.5",
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|
@ -1119,7 +1098,7 @@ name = "garage_db"
|
||||||
version = "0.8.0"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"clap 3.1.18",
|
"clap 3.1.18",
|
||||||
"err-derive 0.3.1",
|
"err-derive",
|
||||||
"heed",
|
"heed",
|
||||||
"hexdump",
|
"hexdump",
|
||||||
"mktemp",
|
"mktemp",
|
||||||
|
|
@ -1131,51 +1110,25 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_model"
|
name = "garage_model"
|
||||||
version = "0.5.1"
|
version = "0.8.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "584619e8999713d73761775591ad6f01ff8c9d724f3b20984f5932f1fc7f9988"
|
|
||||||
dependencies = [
|
|
||||||
"arc-swap",
|
|
||||||
"async-trait",
|
|
||||||
"futures",
|
|
||||||
"futures-util",
|
|
||||||
"garage_rpc 0.5.1",
|
|
||||||
"garage_table 0.5.1",
|
|
||||||
"garage_util 0.5.1",
|
|
||||||
"hex",
|
|
||||||
"log",
|
|
||||||
"netapp 0.3.1",
|
|
||||||
"rand 0.8.5",
|
|
||||||
"rmp-serde 0.15.5",
|
|
||||||
"serde",
|
|
||||||
"serde_bytes",
|
|
||||||
"sled",
|
|
||||||
"tokio",
|
|
||||||
"zstd",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "garage_model"
|
|
||||||
version = "0.7.0"
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64",
|
"base64",
|
||||||
"blake2",
|
"blake2",
|
||||||
"err-derive 0.3.1",
|
"err-derive",
|
||||||
"futures",
|
"futures",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"garage_block",
|
"garage_block",
|
||||||
"garage_db",
|
"garage_db",
|
||||||
"garage_model 0.5.1",
|
"garage_rpc",
|
||||||
"garage_rpc 0.7.0",
|
"garage_table",
|
||||||
"garage_table 0.7.0",
|
"garage_util",
|
||||||
"garage_util 0.7.0",
|
|
||||||
"hex",
|
"hex",
|
||||||
"netapp 0.5.0",
|
"netapp",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"rmp-serde 0.15.5",
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|
@ -1185,53 +1138,26 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_rpc"
|
name = "garage_rpc"
|
||||||
version = "0.5.1"
|
version = "0.8.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "81e693aa4582cfe7a7ce70c07880e3662544b5d0cd68bc4b59c53febfbb8d1ec"
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"futures",
|
"futures",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"garage_util 0.5.1",
|
"garage_util",
|
||||||
"gethostname",
|
|
||||||
"hex",
|
|
||||||
"hyper",
|
|
||||||
"kuska-sodiumoxide",
|
|
||||||
"log",
|
|
||||||
"netapp 0.3.1",
|
|
||||||
"rand 0.8.5",
|
|
||||||
"rmp-serde 0.15.5",
|
|
||||||
"serde",
|
|
||||||
"serde_bytes",
|
|
||||||
"serde_json",
|
|
||||||
"tokio",
|
|
||||||
"tokio-stream",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "garage_rpc"
|
|
||||||
version = "0.7.0"
|
|
||||||
dependencies = [
|
|
||||||
"arc-swap",
|
|
||||||
"async-trait",
|
|
||||||
"bytes 1.2.0",
|
|
||||||
"futures",
|
|
||||||
"futures-util",
|
|
||||||
"garage_util 0.7.0",
|
|
||||||
"gethostname",
|
"gethostname",
|
||||||
"hex",
|
"hex",
|
||||||
"hyper",
|
"hyper",
|
||||||
"k8s-openapi",
|
"k8s-openapi",
|
||||||
"kube",
|
"kube",
|
||||||
"kuska-sodiumoxide",
|
"kuska-sodiumoxide",
|
||||||
"netapp 0.5.0",
|
"netapp",
|
||||||
"openssl",
|
"openssl",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"pnet_datalink",
|
"pnet_datalink",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"rmp-serde 0.15.5",
|
"rmp-serde",
|
||||||
"schemars",
|
"schemars",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
|
|
@ -1243,41 +1169,19 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_table"
|
name = "garage_table"
|
||||||
version = "0.5.1"
|
version = "0.8.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "5c3557f3757e2acd29eaee86804d4e6c38d2abda81b4b349d8a0d2277044265c"
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"futures",
|
|
||||||
"futures-util",
|
|
||||||
"garage_rpc 0.5.1",
|
|
||||||
"garage_util 0.5.1",
|
|
||||||
"hexdump",
|
|
||||||
"log",
|
|
||||||
"rand 0.8.5",
|
|
||||||
"rmp-serde 0.15.5",
|
|
||||||
"serde",
|
|
||||||
"serde_bytes",
|
|
||||||
"sled",
|
|
||||||
"tokio",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "garage_table"
|
|
||||||
version = "0.7.0"
|
|
||||||
dependencies = [
|
|
||||||
"async-trait",
|
|
||||||
"bytes 1.2.0",
|
|
||||||
"futures",
|
"futures",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"garage_db",
|
"garage_db",
|
||||||
"garage_rpc 0.7.0",
|
"garage_rpc",
|
||||||
"garage_util 0.7.0",
|
"garage_util",
|
||||||
"hexdump",
|
"hexdump",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"rmp-serde 0.15.5",
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_bytes",
|
"serde_bytes",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|
@ -1286,50 +1190,26 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_util"
|
name = "garage_util"
|
||||||
version = "0.5.1"
|
version = "0.8.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "1e096994382447431e2f3c70e3685eb8b24c00eceff8667bb22a2a27ff17832f"
|
|
||||||
dependencies = [
|
|
||||||
"blake2",
|
|
||||||
"chrono",
|
|
||||||
"err-derive 0.3.1",
|
|
||||||
"futures",
|
|
||||||
"hex",
|
|
||||||
"http",
|
|
||||||
"hyper",
|
|
||||||
"log",
|
|
||||||
"netapp 0.3.1",
|
|
||||||
"rand 0.8.5",
|
|
||||||
"rmp-serde 0.15.5",
|
|
||||||
"serde",
|
|
||||||
"serde_json",
|
|
||||||
"sha2 0.9.9",
|
|
||||||
"sled",
|
|
||||||
"tokio",
|
|
||||||
"toml",
|
|
||||||
"xxhash-rust",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "garage_util"
|
|
||||||
version = "0.7.0"
|
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"blake2",
|
"blake2",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
"digest 0.10.3",
|
"digest 0.10.3",
|
||||||
"err-derive 0.3.1",
|
"err-derive",
|
||||||
"futures",
|
"futures",
|
||||||
"garage_db",
|
"garage_db",
|
||||||
"git-version",
|
"git-version",
|
||||||
"hex",
|
"hex",
|
||||||
"http",
|
"http",
|
||||||
"hyper",
|
"hyper",
|
||||||
"netapp 0.5.0",
|
"lazy_static",
|
||||||
|
"netapp",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"rmp-serde 0.15.5",
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"sha2 0.10.2",
|
"sha2 0.10.2",
|
||||||
|
|
@ -1341,14 +1221,14 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "garage_web"
|
name = "garage_web"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"err-derive 0.3.1",
|
"err-derive",
|
||||||
"futures",
|
"futures",
|
||||||
"garage_api",
|
"garage_api",
|
||||||
"garage_model 0.7.0",
|
"garage_model",
|
||||||
"garage_table 0.7.0",
|
"garage_table",
|
||||||
"garage_util 0.7.0",
|
"garage_util",
|
||||||
"http",
|
"http",
|
||||||
"hyper",
|
"hyper",
|
||||||
"opentelemetry",
|
"opentelemetry",
|
||||||
|
|
@ -1415,7 +1295,7 @@ version = "0.3.12"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"
|
checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"fnv",
|
"fnv",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
|
|
@ -1475,7 +1355,6 @@ dependencies = [
|
||||||
"lmdb-rkv-sys",
|
"lmdb-rkv-sys",
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"page_size",
|
"page_size",
|
||||||
"serde",
|
|
||||||
"synchronoise",
|
"synchronoise",
|
||||||
"url",
|
"url",
|
||||||
]
|
]
|
||||||
|
|
@ -1550,7 +1429,7 @@ version = "0.2.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
|
checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"fnv",
|
"fnv",
|
||||||
"itoa",
|
"itoa",
|
||||||
]
|
]
|
||||||
|
|
@ -1561,7 +1440,7 @@ version = "0.4.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
|
checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"http",
|
"http",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
]
|
]
|
||||||
|
|
@ -1605,7 +1484,7 @@ version = "0.14.18"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2"
|
checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"futures-channel",
|
"futures-channel",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
|
|
@ -1658,7 +1537,7 @@ version = "0.5.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"hyper",
|
"hyper",
|
||||||
"native-tls",
|
"native-tls",
|
||||||
"tokio",
|
"tokio",
|
||||||
|
|
@ -1787,7 +1666,7 @@ version = "0.0.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64",
|
"base64",
|
||||||
"clap 3.1.18",
|
"clap 3.1.18",
|
||||||
"garage_util 0.7.0",
|
"garage_util",
|
||||||
"http",
|
"http",
|
||||||
"log",
|
"log",
|
||||||
"rusoto_core",
|
"rusoto_core",
|
||||||
|
|
@ -1806,7 +1685,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c"
|
checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64",
|
"base64",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
"http",
|
"http",
|
||||||
"percent-encoding",
|
"percent-encoding",
|
||||||
|
|
@ -1835,7 +1714,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8cd12d68768b54bbd50547f4c7b57b73cff680ef8da3ba409463ee995cf0d707"
|
checksum = "8cd12d68768b54bbd50547f4c7b57b73cff680ef8da3ba409463ee995cf0d707"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64",
|
"base64",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
"dirs-next",
|
"dirs-next",
|
||||||
"either",
|
"either",
|
||||||
|
|
@ -2115,7 +1994,7 @@ version = "2.0.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5f8f35e687561d5c1667590911e6698a8cb714a134a7505718a182e7bc9d3836"
|
checksum = "5f8f35e687561d5c1667590911e6698a8cb714a134a7505718a182e7bc9d3836"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"encoding_rs",
|
"encoding_rs",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"http",
|
"http",
|
||||||
|
|
@ -2151,28 +2030,6 @@ dependencies = [
|
||||||
"tempfile",
|
"tempfile",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "netapp"
|
|
||||||
version = "0.3.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ac7dcd2c6c5a9d5ea88ffc17d3339d49d308e68c4d8190c494b55ddbf78d80ad"
|
|
||||||
dependencies = [
|
|
||||||
"arc-swap",
|
|
||||||
"async-trait",
|
|
||||||
"bytes 0.6.0",
|
|
||||||
"err-derive 0.2.4",
|
|
||||||
"futures",
|
|
||||||
"hex",
|
|
||||||
"kuska-handshake",
|
|
||||||
"kuska-sodiumoxide",
|
|
||||||
"log",
|
|
||||||
"rmp-serde 0.14.4",
|
|
||||||
"serde",
|
|
||||||
"tokio",
|
|
||||||
"tokio-stream",
|
|
||||||
"tokio-util 0.6.9",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "netapp"
|
name = "netapp"
|
||||||
version = "0.5.0"
|
version = "0.5.0"
|
||||||
|
|
@ -2180,9 +2037,9 @@ source = "git+https://git.deuxfleurs.fr/lx/netapp?branch=stream-body#0f799a77689
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"cfg-if 1.0.0",
|
"cfg-if 1.0.0",
|
||||||
"err-derive 0.3.1",
|
"err-derive",
|
||||||
"futures",
|
"futures",
|
||||||
"hex",
|
"hex",
|
||||||
"kuska-handshake",
|
"kuska-handshake",
|
||||||
|
|
@ -2192,7 +2049,7 @@ dependencies = [
|
||||||
"opentelemetry-contrib",
|
"opentelemetry-contrib",
|
||||||
"pin-project 1.0.10",
|
"pin-project 1.0.10",
|
||||||
"rand 0.8.5",
|
"rand 0.8.5",
|
||||||
"rmp-serde 0.15.5",
|
"rmp-serde",
|
||||||
"serde",
|
"serde",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-stream",
|
"tokio-stream",
|
||||||
|
|
@ -2674,7 +2531,7 @@ version = "0.9.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
|
checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"prost-derive",
|
"prost-derive",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -2684,7 +2541,7 @@ version = "0.9.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
|
checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"heck 0.3.3",
|
"heck 0.3.3",
|
||||||
"itertools 0.10.3",
|
"itertools 0.10.3",
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
|
|
@ -2717,7 +2574,7 @@ version = "0.9.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
|
checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"prost",
|
"prost",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -2977,17 +2834,6 @@ dependencies = [
|
||||||
"num-traits",
|
"num-traits",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "rmp-serde"
|
|
||||||
version = "0.14.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "4ce7d70c926fe472aed493b902010bccc17fa9f7284145cb8772fd22fdb052d8"
|
|
||||||
dependencies = [
|
|
||||||
"byteorder",
|
|
||||||
"rmp",
|
|
||||||
"serde",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rmp-serde"
|
name = "rmp-serde"
|
||||||
version = "0.15.5"
|
version = "0.15.5"
|
||||||
|
|
@ -3016,7 +2862,7 @@ checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64",
|
"base64",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"crc32fast",
|
"crc32fast",
|
||||||
"futures",
|
"futures",
|
||||||
"http",
|
"http",
|
||||||
|
|
@ -3058,7 +2904,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"
|
checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64",
|
"base64",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"chrono",
|
"chrono",
|
||||||
"digest 0.9.0",
|
"digest 0.9.0",
|
||||||
"futures",
|
"futures",
|
||||||
|
|
@ -3642,7 +3488,7 @@ version = "1.17.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"
|
checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"libc",
|
"libc",
|
||||||
"memchr",
|
"memchr",
|
||||||
"mio",
|
"mio",
|
||||||
|
|
@ -3715,9 +3561,8 @@ version = "0.6.9"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"
|
checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-io",
|
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
"log",
|
"log",
|
||||||
"pin-project-lite",
|
"pin-project-lite",
|
||||||
|
|
@ -3731,7 +3576,7 @@ version = "0.7.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1"
|
checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-io",
|
"futures-io",
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
|
|
@ -3758,7 +3603,7 @@ dependencies = [
|
||||||
"async-stream",
|
"async-stream",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"base64",
|
"base64",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"h2",
|
"h2",
|
||||||
|
|
@ -3819,7 +3664,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "81eca72647e58054bbfa41e6f297c23436f1c60aff6e5eb38455a0f9ca420bb5"
|
checksum = "81eca72647e58054bbfa41e6f297c23436f1c60aff6e5eb38455a0f9ca420bb5"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64",
|
"base64",
|
||||||
"bytes 1.2.0",
|
"bytes",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"http",
|
"http",
|
||||||
|
|
@ -4266,4 +4111,5 @@ checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cc",
|
"cc",
|
||||||
"libc",
|
"libc",
|
||||||
|
"pkg-config",
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
[workspace]
|
[workspace]
|
||||||
|
resolver = "2"
|
||||||
members = [
|
members = [
|
||||||
"src/db",
|
"src/db",
|
||||||
"src/util",
|
"src/util",
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,24 @@ sudo apt-get update
|
||||||
sudo apt-get install build-essential
|
sudo apt-get install build-essential
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Using source from the Gitea repository (recommended)
|
||||||
|
|
||||||
|
The primary location for Garage's source code is the
|
||||||
|
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage).
|
||||||
|
|
||||||
|
Clone the repository and build Garage with the following commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage.git
|
||||||
|
cd garage
|
||||||
|
cargo build
|
||||||
|
```
|
||||||
|
|
||||||
|
Be careful, as this will make a debug build of Garage, which will be extremely slow!
|
||||||
|
To make a release build, invoke `cargo build --release` (this takes much longer).
|
||||||
|
|
||||||
|
The binaries built this way are found in `target/{debug,release}/garage`.
|
||||||
|
|
||||||
## Using source from `crates.io`
|
## Using source from `crates.io`
|
||||||
|
|
||||||
Garage's source code is published on `crates.io`, Rust's official package repository.
|
Garage's source code is published on `crates.io`, Rust's official package repository.
|
||||||
|
|
@ -39,21 +57,20 @@ sudo cp $HOME/.cargo/bin/garage /usr/local/bin/garage
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
## Using source from the Gitea repository
|
## Selecting features to activate in your build
|
||||||
|
|
||||||
The primary location for Garage's source code is the
|
Garage supports a number of compilation options in the form of Cargo features,
|
||||||
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage).
|
which can be used to provide builds adapted to your system and your use case.
|
||||||
|
The following features are available:
|
||||||
Clone the repository and build Garage with the following commands:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage.git
|
|
||||||
cd garage
|
|
||||||
cargo build
|
|
||||||
```
|
|
||||||
|
|
||||||
Be careful, as this will make a debug build of Garage, which will be extremely slow!
|
|
||||||
To make a release build, invoke `cargo build --release` (this takes much longer).
|
|
||||||
|
|
||||||
The binaries built this way are found in `target/{debug,release}/garage`.
|
|
||||||
|
|
||||||
|
| Feature | Enabled | Description |
|
||||||
|
| ------- | ------- | ----------- |
|
||||||
|
| `bundled-libs` | BY DEFAULT | Use bundled version of sqlite3, zstd, lmdb and libsodium |
|
||||||
|
| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium if available (exclusive with `bundled-libs`, build using `cargo build --no-default-features --features system-libs`) |
|
||||||
|
| `k2v` | optional | Enable the experimental K2V API (if used, all nodes on your Garage cluster must have it enabled as well) |
|
||||||
|
| `kubernetes-discovery` | optional | Enable automatic registration and discovery of cluster nodes through the Kubernetes API |
|
||||||
|
| `metrics` | BY DEFAULT | Enable collection of metrics in Prometheus format on the admin API |
|
||||||
|
| `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
|
||||||
|
| `sled` | BY DEFAULT | Enable using Sled to store Garage's metadata |
|
||||||
|
| `lmdb` | optional | Enable using LMDB to store Garage's metadata |
|
||||||
|
| `sqlite` | optional | Enable using Sqlite3 to store Garage's metadata |
|
||||||
|
|
|
||||||
|
|
@ -117,22 +117,34 @@ let
|
||||||
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
|
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
|
||||||
But we want to ship these additional features when we release Garage.
|
But we want to ship these additional features when we release Garage.
|
||||||
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
|
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
|
||||||
Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate.
|
|
||||||
|
[5] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library.
|
||||||
|
However the features to do so get activated for some reason (due to a bug in cargo2nix?),
|
||||||
|
so disable them manually here.
|
||||||
*/
|
*/
|
||||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
name = "garage";
|
name = "garage";
|
||||||
overrideAttrs = drv: {
|
overrideAttrs = drv:
|
||||||
|
(if git_version != null then {
|
||||||
|
/* [3] */ preConfigure = ''
|
||||||
|
${drv.preConfigure or ""}
|
||||||
|
export GIT_VERSION="${git_version}"
|
||||||
|
'';
|
||||||
|
} else {})
|
||||||
|
//
|
||||||
|
{
|
||||||
/* [1] */ setBuildEnv = (buildEnv drv);
|
/* [1] */ setBuildEnv = (buildEnv drv);
|
||||||
/* [2] */ hardeningDisable = [ "pie" ];
|
/* [2] */ hardeningDisable = [ "pie" ];
|
||||||
};
|
};
|
||||||
|
overrideArgs = old: {
|
||||||
|
/* [4] */ features = [ "bundled-libs" "sled" ]
|
||||||
|
++ (if release then [ "kubernetes-discovery" "telemetry-otlp" "metrics" "lmdb" "sqlite" ] else []);
|
||||||
|
};
|
||||||
})
|
})
|
||||||
|
|
||||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
name = "garage_rpc";
|
name = "garage_rpc";
|
||||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||||
overrideArgs = old: {
|
|
||||||
/* [4] */ features = if release then [ "kubernetes-discovery" ] else [];
|
|
||||||
};
|
|
||||||
})
|
})
|
||||||
|
|
||||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
|
|
@ -142,15 +154,7 @@ let
|
||||||
|
|
||||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
name = "garage_util";
|
name = "garage_util";
|
||||||
overrideAttrs = drv:
|
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||||
(if git_version != null then {
|
|
||||||
/* [3] */ preConfigure = ''
|
|
||||||
${drv.preConfigure or ""}
|
|
||||||
export GIT_VERSION="${git_version}"
|
|
||||||
'';
|
|
||||||
} else {})
|
|
||||||
//
|
|
||||||
{ /* [1] */ setBuildEnv = (buildEnv drv); };
|
|
||||||
})
|
})
|
||||||
|
|
||||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
|
|
@ -182,6 +186,20 @@ let
|
||||||
name = "k2v-client";
|
name = "k2v-client";
|
||||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||||
})
|
})
|
||||||
|
|
||||||
|
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
|
name = "libsodium-sys";
|
||||||
|
overrideArgs = old: {
|
||||||
|
features = [ ]; /* [5] */
|
||||||
|
};
|
||||||
|
})
|
||||||
|
|
||||||
|
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
|
name = "zstd-sys";
|
||||||
|
overrideArgs = old: {
|
||||||
|
features = [ ]; /* [5] */
|
||||||
|
};
|
||||||
|
})
|
||||||
];
|
];
|
||||||
|
|
||||||
packageFun = import ../Cargo.nix;
|
packageFun = import ../Cargo.nix;
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "garage_api"
|
name = "garage_api"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|
@ -14,11 +14,11 @@ path = "lib.rs"
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_model = { version = "0.7.0", path = "../model" }
|
garage_model = { version = "0.8.0", path = "../model" }
|
||||||
garage_table = { version = "0.7.0", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
garage_block = { version = "0.7.0", path = "../block" }
|
garage_block = { version = "0.8.0", path = "../block" }
|
||||||
garage_util = { version = "0.7.0", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
|
|
||||||
async-trait = "0.1.7"
|
async-trait = "0.1.7"
|
||||||
base64 = "0.13"
|
base64 = "0.13"
|
||||||
|
|
@ -54,9 +54,9 @@ quick-xml = { version = "0.21", features = [ "serialize" ] }
|
||||||
url = "2.1"
|
url = "2.1"
|
||||||
|
|
||||||
opentelemetry = "0.17"
|
opentelemetry = "0.17"
|
||||||
opentelemetry-prometheus = "0.10"
|
opentelemetry-prometheus = { version = "0.10", optional = true }
|
||||||
opentelemetry-otlp = "0.10"
|
prometheus = { version = "0.13", optional = true }
|
||||||
prometheus = "0.13"
|
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
k2v = [ "garage_util/k2v", "garage_model/k2v" ]
|
k2v = [ "garage_util/k2v", "garage_model/k2v" ]
|
||||||
|
metrics = [ "opentelemetry-prometheus", "prometheus" ]
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,17 @@
|
||||||
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
||||||
use futures::future::Future;
|
use futures::future::Future;
|
||||||
use http::header::{
|
use http::header::{ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW};
|
||||||
ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW, CONTENT_TYPE,
|
|
||||||
};
|
|
||||||
use hyper::{Body, Request, Response};
|
use hyper::{Body, Request, Response};
|
||||||
|
|
||||||
use opentelemetry::trace::{SpanRef, Tracer};
|
use opentelemetry::trace::SpanRef;
|
||||||
|
|
||||||
|
#[cfg(feature = "metrics")]
|
||||||
use opentelemetry_prometheus::PrometheusExporter;
|
use opentelemetry_prometheus::PrometheusExporter;
|
||||||
|
#[cfg(feature = "metrics")]
|
||||||
use prometheus::{Encoder, TextEncoder};
|
use prometheus::{Encoder, TextEncoder};
|
||||||
|
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
|
|
@ -25,6 +27,7 @@ use crate::admin::router::{Authorization, Endpoint};
|
||||||
|
|
||||||
pub struct AdminApiServer {
|
pub struct AdminApiServer {
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
|
#[cfg(feature = "metrics")]
|
||||||
exporter: PrometheusExporter,
|
exporter: PrometheusExporter,
|
||||||
metrics_token: Option<String>,
|
metrics_token: Option<String>,
|
||||||
admin_token: Option<String>,
|
admin_token: Option<String>,
|
||||||
|
|
@ -32,7 +35,6 @@ pub struct AdminApiServer {
|
||||||
|
|
||||||
impl AdminApiServer {
|
impl AdminApiServer {
|
||||||
pub fn new(garage: Arc<Garage>) -> Self {
|
pub fn new(garage: Arc<Garage>) -> Self {
|
||||||
let exporter = opentelemetry_prometheus::exporter().init();
|
|
||||||
let cfg = &garage.config.admin;
|
let cfg = &garage.config.admin;
|
||||||
let metrics_token = cfg
|
let metrics_token = cfg
|
||||||
.metrics_token
|
.metrics_token
|
||||||
|
|
@ -44,21 +46,22 @@ impl AdminApiServer {
|
||||||
.map(|tok| format!("Bearer {}", tok));
|
.map(|tok| format!("Bearer {}", tok));
|
||||||
Self {
|
Self {
|
||||||
garage,
|
garage,
|
||||||
exporter,
|
#[cfg(feature = "metrics")]
|
||||||
|
exporter: opentelemetry_prometheus::exporter().init(),
|
||||||
metrics_token,
|
metrics_token,
|
||||||
admin_token,
|
admin_token,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn run(self, shutdown_signal: impl Future<Output = ()>) -> Result<(), GarageError> {
|
pub async fn run(
|
||||||
if let Some(bind_addr) = self.garage.config.admin.api_bind_addr {
|
self,
|
||||||
let region = self.garage.config.s3_api.s3_region.clone();
|
bind_addr: SocketAddr,
|
||||||
ApiServer::new(region, self)
|
shutdown_signal: impl Future<Output = ()>,
|
||||||
.run_server(bind_addr, shutdown_signal)
|
) -> Result<(), GarageError> {
|
||||||
.await
|
let region = self.garage.config.s3_api.s3_region.clone();
|
||||||
} else {
|
ApiServer::new(region, self)
|
||||||
Ok(())
|
.run_server(bind_addr, shutdown_signal)
|
||||||
}
|
.await
|
||||||
}
|
}
|
||||||
|
|
||||||
fn handle_options(&self, _req: &Request<Body>) -> Result<Response<Body>, Error> {
|
fn handle_options(&self, _req: &Request<Body>) -> Result<Response<Body>, Error> {
|
||||||
|
|
@ -71,22 +74,31 @@ impl AdminApiServer {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn handle_metrics(&self) -> Result<Response<Body>, Error> {
|
fn handle_metrics(&self) -> Result<Response<Body>, Error> {
|
||||||
let mut buffer = vec![];
|
#[cfg(feature = "metrics")]
|
||||||
let encoder = TextEncoder::new();
|
{
|
||||||
|
use opentelemetry::trace::Tracer;
|
||||||
|
|
||||||
let tracer = opentelemetry::global::tracer("garage");
|
let mut buffer = vec![];
|
||||||
let metric_families = tracer.in_span("admin/gather_metrics", |_| {
|
let encoder = TextEncoder::new();
|
||||||
self.exporter.registry().gather()
|
|
||||||
});
|
|
||||||
|
|
||||||
encoder
|
let tracer = opentelemetry::global::tracer("garage");
|
||||||
.encode(&metric_families, &mut buffer)
|
let metric_families = tracer.in_span("admin/gather_metrics", |_| {
|
||||||
.ok_or_internal_error("Could not serialize metrics")?;
|
self.exporter.registry().gather()
|
||||||
|
});
|
||||||
|
|
||||||
Ok(Response::builder()
|
encoder
|
||||||
.status(200)
|
.encode(&metric_families, &mut buffer)
|
||||||
.header(CONTENT_TYPE, encoder.format_type())
|
.ok_or_internal_error("Could not serialize metrics")?;
|
||||||
.body(Body::from(buffer))?)
|
|
||||||
|
Ok(Response::builder()
|
||||||
|
.status(200)
|
||||||
|
.header(http::header::CONTENT_TYPE, encoder.format_type())
|
||||||
|
.body(Body::from(buffer))?)
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "metrics"))]
|
||||||
|
Err(Error::bad_request(
|
||||||
|
"Garage was built without the metrics feature".to_string(),
|
||||||
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,8 @@ use crate::helpers::{json_ok_response, parse_json_body};
|
||||||
pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
|
pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
|
||||||
let res = GetClusterStatusResponse {
|
let res = GetClusterStatusResponse {
|
||||||
node: hex::encode(garage.system.id),
|
node: hex::encode(garage.system.id),
|
||||||
garage_version: garage.system.garage_version(),
|
garage_version: garage_util::version::garage_version(),
|
||||||
|
garage_features: garage_util::version::garage_features(),
|
||||||
db_engine: garage.db.engine(),
|
db_engine: garage.db.engine(),
|
||||||
known_nodes: garage
|
known_nodes: garage
|
||||||
.system
|
.system
|
||||||
|
|
@ -99,6 +100,7 @@ fn get_cluster_layout(garage: &Arc<Garage>) -> GetClusterLayoutResponse {
|
||||||
struct GetClusterStatusResponse {
|
struct GetClusterStatusResponse {
|
||||||
node: String,
|
node: String,
|
||||||
garage_version: &'static str,
|
garage_version: &'static str,
|
||||||
|
garage_features: Option<&'static [&'static str]>,
|
||||||
db_engine: String,
|
db_engine: String,
|
||||||
known_nodes: HashMap<String, KnownNodeResp>,
|
known_nodes: HashMap<String, KnownNodeResp>,
|
||||||
layout: GetClusterLayoutResponse,
|
layout: GetClusterLayoutResponse,
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
@ -36,20 +37,13 @@ pub(crate) struct K2VApiEndpoint {
|
||||||
impl K2VApiServer {
|
impl K2VApiServer {
|
||||||
pub async fn run(
|
pub async fn run(
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
|
bind_addr: SocketAddr,
|
||||||
|
s3_region: String,
|
||||||
shutdown_signal: impl Future<Output = ()>,
|
shutdown_signal: impl Future<Output = ()>,
|
||||||
) -> Result<(), GarageError> {
|
) -> Result<(), GarageError> {
|
||||||
if let Some(cfg) = &garage.config.k2v_api {
|
ApiServer::new(s3_region, K2VApiServer { garage })
|
||||||
let bind_addr = cfg.api_bind_addr;
|
|
||||||
|
|
||||||
ApiServer::new(
|
|
||||||
garage.config.s3_api.s3_region.clone(),
|
|
||||||
K2VApiServer { garage },
|
|
||||||
)
|
|
||||||
.run_server(bind_addr, shutdown_signal)
|
.run_server(bind_addr, shutdown_signal)
|
||||||
.await
|
.await
|
||||||
} else {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,4 @@
|
||||||
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
|
|
@ -43,16 +44,13 @@ pub(crate) struct S3ApiEndpoint {
|
||||||
impl S3ApiServer {
|
impl S3ApiServer {
|
||||||
pub async fn run(
|
pub async fn run(
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
|
addr: SocketAddr,
|
||||||
|
s3_region: String,
|
||||||
shutdown_signal: impl Future<Output = ()>,
|
shutdown_signal: impl Future<Output = ()>,
|
||||||
) -> Result<(), GarageError> {
|
) -> Result<(), GarageError> {
|
||||||
let addr = garage.config.s3_api.api_bind_addr;
|
ApiServer::new(s3_region, S3ApiServer { garage })
|
||||||
|
.run_server(addr, shutdown_signal)
|
||||||
ApiServer::new(
|
.await
|
||||||
garage.config.s3_api.s3_region.clone(),
|
|
||||||
S3ApiServer { garage },
|
|
||||||
)
|
|
||||||
.run_server(addr, shutdown_signal)
|
|
||||||
.await
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn handle_request_without_bucket(
|
async fn handle_request_without_bucket(
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "garage_block"
|
name = "garage_block"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|
@ -15,9 +15,9 @@ path = "lib.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.0", path = "../db" }
|
garage_db = { version = "0.8.0", path = "../db" }
|
||||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
garage_util = { version = "0.7.0", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_table = { version = "0.7.0", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
|
|
||||||
opentelemetry = "0.17"
|
opentelemetry = "0.17"
|
||||||
|
|
||||||
|
|
@ -39,3 +39,6 @@ futures = "0.3"
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||||
tokio-util = { version = "0.6", features = ["io"] }
|
tokio-util = { version = "0.6", features = ["io"] }
|
||||||
|
|
||||||
|
[features]
|
||||||
|
system-libs = [ "zstd/pkg-config" ]
|
||||||
|
|
|
||||||
|
|
@ -21,9 +21,9 @@ err-derive = "0.3"
|
||||||
hexdump = "0.1"
|
hexdump = "0.1"
|
||||||
tracing = "0.1.30"
|
tracing = "0.1.30"
|
||||||
|
|
||||||
heed = "0.11"
|
heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true }
|
||||||
rusqlite = { version = "0.27", features = ["bundled"] }
|
rusqlite = { version = "0.27", optional = true }
|
||||||
sled = "0.34"
|
sled = { version = "0.34", optional = true }
|
||||||
|
|
||||||
# cli deps
|
# cli deps
|
||||||
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
||||||
|
|
@ -33,4 +33,7 @@ pretty_env_logger = { version = "0.4", optional = true }
|
||||||
mktemp = "0.4"
|
mktemp = "0.4"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
|
bundled-libs = [ "rusqlite/bundled" ]
|
||||||
cli = ["clap", "pretty_env_logger"]
|
cli = ["clap", "pretty_env_logger"]
|
||||||
|
lmdb = [ "heed" ]
|
||||||
|
sqlite = [ "rusqlite" ]
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,15 @@
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
|
#[cfg(feature = "sqlite")]
|
||||||
extern crate tracing;
|
extern crate tracing;
|
||||||
|
|
||||||
|
#[cfg(not(any(feature = "lmdb", feature = "sled", feature = "sqlite")))]
|
||||||
|
compile_error!("Must activate the Cargo feature for at least one DB engine: lmdb, sled or sqlite.");
|
||||||
|
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
pub mod lmdb_adapter;
|
pub mod lmdb_adapter;
|
||||||
|
#[cfg(feature = "sled")]
|
||||||
pub mod sled_adapter;
|
pub mod sled_adapter;
|
||||||
|
#[cfg(feature = "sqlite")]
|
||||||
pub mod sqlite_adapter;
|
pub mod sqlite_adapter;
|
||||||
|
|
||||||
pub mod counted_tree_hack;
|
pub mod counted_tree_hack;
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "garage"
|
name = "garage"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|
@ -22,13 +22,13 @@ path = "tests/lib.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.0", path = "../db" }
|
garage_db = { version = "0.8.0", path = "../db" }
|
||||||
garage_api = { version = "0.7.0", path = "../api" }
|
garage_api = { version = "0.8.0", path = "../api" }
|
||||||
garage_block = { version = "0.7.0", path = "../block" }
|
garage_block = { version = "0.8.0", path = "../block" }
|
||||||
garage_model = { version = "0.7.0", path = "../model" }
|
garage_model = { version = "0.8.0", path = "../model" }
|
||||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
garage_table = { version = "0.7.0", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
garage_util = { version = "0.7.0", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_web = { version = "0.7.0", path = "../web" }
|
garage_web = { version = "0.8.0", path = "../web" }
|
||||||
|
|
||||||
bytes = "1.0"
|
bytes = "1.0"
|
||||||
bytesize = "1.1"
|
bytesize = "1.1"
|
||||||
|
|
@ -54,9 +54,9 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
|
||||||
netapp = { version = "0.5", git = "https://git.deuxfleurs.fr/lx/netapp", branch = "stream-body", features = ["telemetry"] }
|
netapp = { version = "0.5", git = "https://git.deuxfleurs.fr/lx/netapp", branch = "stream-body", features = ["telemetry"] }
|
||||||
|
|
||||||
opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
|
opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
|
||||||
opentelemetry-prometheus = "0.10"
|
opentelemetry-prometheus = { version = "0.10", optional = true }
|
||||||
opentelemetry-otlp = "0.10"
|
opentelemetry-otlp = { version = "0.10", optional = true }
|
||||||
prometheus = "0.13"
|
prometheus = { version = "0.13", optional = true }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
aws-sdk-s3 = "0.8"
|
aws-sdk-s3 = "0.8"
|
||||||
|
|
@ -73,5 +73,26 @@ base64 = "0.13"
|
||||||
|
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
|
default = [ "bundled-libs", "metrics", "sled" ]
|
||||||
|
|
||||||
k2v = [ "garage_util/k2v", "garage_api/k2v" ]
|
k2v = [ "garage_util/k2v", "garage_api/k2v" ]
|
||||||
|
|
||||||
|
# Database engines, Sled is still our default even though we don't like it
|
||||||
|
sled = [ "garage_model/sled" ]
|
||||||
|
lmdb = [ "garage_model/lmdb" ]
|
||||||
|
sqlite = [ "garage_model/sqlite" ]
|
||||||
|
|
||||||
|
# Automatic registration and discovery via Kubernetes API
|
||||||
|
kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
|
||||||
|
# Prometheus exporter (/metrics endpoint).
|
||||||
|
metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ]
|
||||||
|
# Exporter for the OpenTelemetry Collector.
|
||||||
|
telemetry-otlp = [ "opentelemetry-otlp" ]
|
||||||
|
|
||||||
|
# NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
|
||||||
|
# exactly one of them should be enabled.
|
||||||
|
|
||||||
|
# Use bundled libsqlite instead of linking against system-provided.
|
||||||
|
bundled-libs = [ "garage_db/bundled-libs" ]
|
||||||
|
# Link against system-provided libsodium and libzstd.
|
||||||
|
system-libs = [ "garage_block/system-libs", "garage_rpc/system-libs", "sodiumoxide/use-pkg-config" ]
|
||||||
|
|
|
||||||
|
|
@ -741,8 +741,11 @@ impl AdminRpcHandler {
|
||||||
let mut ret = String::new();
|
let mut ret = String::new();
|
||||||
writeln!(
|
writeln!(
|
||||||
&mut ret,
|
&mut ret,
|
||||||
"\nGarage version: {}",
|
"\nGarage version: {} [features: {}]",
|
||||||
self.garage.system.garage_version(),
|
garage_util::version::garage_version(),
|
||||||
|
garage_util::version::garage_features()
|
||||||
|
.map(|list| list.join(", "))
|
||||||
|
.unwrap_or_else(|| "(unknown)".into()),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();
|
writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();
|
||||||
|
|
|
||||||
|
|
@ -1,65 +1,65 @@
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use garage_util::version;
|
|
||||||
use structopt::StructOpt;
|
use structopt::StructOpt;
|
||||||
|
|
||||||
|
use garage_util::version::garage_version;
|
||||||
|
|
||||||
#[derive(StructOpt, Debug)]
|
#[derive(StructOpt, Debug)]
|
||||||
pub enum Command {
|
pub enum Command {
|
||||||
/// Run Garage server
|
/// Run Garage server
|
||||||
#[structopt(name = "server", version = version::garage())]
|
#[structopt(name = "server", version = garage_version())]
|
||||||
Server,
|
Server,
|
||||||
|
|
||||||
/// Get network status
|
/// Get network status
|
||||||
#[structopt(name = "status", version = version::garage())]
|
#[structopt(name = "status", version = garage_version())]
|
||||||
Status,
|
Status,
|
||||||
|
|
||||||
/// Operations on individual Garage nodes
|
/// Operations on individual Garage nodes
|
||||||
#[structopt(name = "node", version = version::garage())]
|
#[structopt(name = "node", version = garage_version())]
|
||||||
Node(NodeOperation),
|
Node(NodeOperation),
|
||||||
|
|
||||||
/// Operations on the assignation of node roles in the cluster layout
|
/// Operations on the assignation of node roles in the cluster layout
|
||||||
#[structopt(name = "layout", version = version::garage())]
|
#[structopt(name = "layout", version = garage_version())]
|
||||||
Layout(LayoutOperation),
|
Layout(LayoutOperation),
|
||||||
|
|
||||||
/// Operations on buckets
|
/// Operations on buckets
|
||||||
#[structopt(name = "bucket", version = version::garage())]
|
#[structopt(name = "bucket", version = garage_version())]
|
||||||
Bucket(BucketOperation),
|
Bucket(BucketOperation),
|
||||||
|
|
||||||
/// Operations on S3 access keys
|
/// Operations on S3 access keys
|
||||||
#[structopt(name = "key", version = version::garage())]
|
#[structopt(name = "key", version = garage_version())]
|
||||||
Key(KeyOperation),
|
Key(KeyOperation),
|
||||||
|
|
||||||
/// Run migrations from previous Garage version
|
/// Run migrations from previous Garage version
|
||||||
/// (DO NOT USE WITHOUT READING FULL DOCUMENTATION)
|
/// (DO NOT USE WITHOUT READING FULL DOCUMENTATION)
|
||||||
#[structopt(name = "migrate", version = version::garage())]
|
#[structopt(name = "migrate", version = garage_version())]
|
||||||
Migrate(MigrateOpt),
|
Migrate(MigrateOpt),
|
||||||
|
|
||||||
/// Start repair of node data on remote node
|
/// Start repair of node data on remote node
|
||||||
#[structopt(name = "repair", version = version::garage())]
|
#[structopt(name = "repair", version = garage_version())]
|
||||||
Repair(RepairOpt),
|
Repair(RepairOpt),
|
||||||
|
|
||||||
/// Offline reparation of node data (these repairs must be run offline
|
/// Offline reparation of node data (these repairs must be run offline
|
||||||
/// directly on the server node)
|
/// directly on the server node)
|
||||||
#[structopt(name = "offline-repair", version = version::garage())]
|
#[structopt(name = "offline-repair", version = garage_version())]
|
||||||
OfflineRepair(OfflineRepairOpt),
|
OfflineRepair(OfflineRepairOpt),
|
||||||
|
|
||||||
/// Gather node statistics
|
/// Gather node statistics
|
||||||
#[structopt(name = "stats", version = version::garage())]
|
#[structopt(name = "stats", version = garage_version())]
|
||||||
Stats(StatsOpt),
|
Stats(StatsOpt),
|
||||||
|
|
||||||
/// Manage background workers
|
/// Manage background workers
|
||||||
#[structopt(name = "worker", version = version::garage())]
|
#[structopt(name = "worker", version = garage_version())]
|
||||||
Worker(WorkerOpt),
|
Worker(WorkerOpt),
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(StructOpt, Debug)]
|
#[derive(StructOpt, Debug)]
|
||||||
pub enum NodeOperation {
|
pub enum NodeOperation {
|
||||||
/// Print identifier (public key) of this Garage node
|
/// Print identifier (public key) of this Garage node
|
||||||
#[structopt(name = "id", version = version::garage())]
|
#[structopt(name = "id", version = garage_version())]
|
||||||
NodeId(NodeIdOpt),
|
NodeId(NodeIdOpt),
|
||||||
|
|
||||||
/// Connect to Garage node that is currently isolated from the system
|
/// Connect to Garage node that is currently isolated from the system
|
||||||
#[structopt(name = "connect", version = version::garage())]
|
#[structopt(name = "connect", version = garage_version())]
|
||||||
Connect(ConnectNodeOpt),
|
Connect(ConnectNodeOpt),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -80,23 +80,23 @@ pub struct ConnectNodeOpt {
|
||||||
#[derive(StructOpt, Debug)]
|
#[derive(StructOpt, Debug)]
|
||||||
pub enum LayoutOperation {
|
pub enum LayoutOperation {
|
||||||
/// Assign role to Garage node
|
/// Assign role to Garage node
|
||||||
#[structopt(name = "assign", version = version::garage())]
|
#[structopt(name = "assign", version = garage_version())]
|
||||||
Assign(AssignRoleOpt),
|
Assign(AssignRoleOpt),
|
||||||
|
|
||||||
/// Remove role from Garage cluster node
|
/// Remove role from Garage cluster node
|
||||||
#[structopt(name = "remove", version = version::garage())]
|
#[structopt(name = "remove", version = garage_version())]
|
||||||
Remove(RemoveRoleOpt),
|
Remove(RemoveRoleOpt),
|
||||||
|
|
||||||
/// Show roles currently assigned to nodes and changes staged for commit
|
/// Show roles currently assigned to nodes and changes staged for commit
|
||||||
#[structopt(name = "show", version = version::garage())]
|
#[structopt(name = "show", version = garage_version())]
|
||||||
Show,
|
Show,
|
||||||
|
|
||||||
/// Apply staged changes to cluster layout
|
/// Apply staged changes to cluster layout
|
||||||
#[structopt(name = "apply", version = version::garage())]
|
#[structopt(name = "apply", version = garage_version())]
|
||||||
Apply(ApplyLayoutOpt),
|
Apply(ApplyLayoutOpt),
|
||||||
|
|
||||||
/// Revert staged changes to cluster layout
|
/// Revert staged changes to cluster layout
|
||||||
#[structopt(name = "revert", version = version::garage())]
|
#[structopt(name = "revert", version = garage_version())]
|
||||||
Revert(RevertLayoutOpt),
|
Revert(RevertLayoutOpt),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -151,43 +151,43 @@ pub struct RevertLayoutOpt {
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||||
pub enum BucketOperation {
|
pub enum BucketOperation {
|
||||||
/// List buckets
|
/// List buckets
|
||||||
#[structopt(name = "list", version = version::garage())]
|
#[structopt(name = "list", version = garage_version())]
|
||||||
List,
|
List,
|
||||||
|
|
||||||
/// Get bucket info
|
/// Get bucket info
|
||||||
#[structopt(name = "info", version = version::garage())]
|
#[structopt(name = "info", version = garage_version())]
|
||||||
Info(BucketOpt),
|
Info(BucketOpt),
|
||||||
|
|
||||||
/// Create bucket
|
/// Create bucket
|
||||||
#[structopt(name = "create", version = version::garage())]
|
#[structopt(name = "create", version = garage_version())]
|
||||||
Create(BucketOpt),
|
Create(BucketOpt),
|
||||||
|
|
||||||
/// Delete bucket
|
/// Delete bucket
|
||||||
#[structopt(name = "delete", version = version::garage())]
|
#[structopt(name = "delete", version = garage_version())]
|
||||||
Delete(DeleteBucketOpt),
|
Delete(DeleteBucketOpt),
|
||||||
|
|
||||||
/// Alias bucket under new name
|
/// Alias bucket under new name
|
||||||
#[structopt(name = "alias", version = version::garage())]
|
#[structopt(name = "alias", version = garage_version())]
|
||||||
Alias(AliasBucketOpt),
|
Alias(AliasBucketOpt),
|
||||||
|
|
||||||
/// Remove bucket alias
|
/// Remove bucket alias
|
||||||
#[structopt(name = "unalias", version = version::garage())]
|
#[structopt(name = "unalias", version = garage_version())]
|
||||||
Unalias(UnaliasBucketOpt),
|
Unalias(UnaliasBucketOpt),
|
||||||
|
|
||||||
/// Allow key to read or write to bucket
|
/// Allow key to read or write to bucket
|
||||||
#[structopt(name = "allow", version = version::garage())]
|
#[structopt(name = "allow", version = garage_version())]
|
||||||
Allow(PermBucketOpt),
|
Allow(PermBucketOpt),
|
||||||
|
|
||||||
/// Deny key from reading or writing to bucket
|
/// Deny key from reading or writing to bucket
|
||||||
#[structopt(name = "deny", version = version::garage())]
|
#[structopt(name = "deny", version = garage_version())]
|
||||||
Deny(PermBucketOpt),
|
Deny(PermBucketOpt),
|
||||||
|
|
||||||
/// Expose as website or not
|
/// Expose as website or not
|
||||||
#[structopt(name = "website", version = version::garage())]
|
#[structopt(name = "website", version = garage_version())]
|
||||||
Website(WebsiteOpt),
|
Website(WebsiteOpt),
|
||||||
|
|
||||||
/// Set the quotas for this bucket
|
/// Set the quotas for this bucket
|
||||||
#[structopt(name = "set-quotas", version = version::garage())]
|
#[structopt(name = "set-quotas", version = garage_version())]
|
||||||
SetQuotas(SetQuotasOpt),
|
SetQuotas(SetQuotasOpt),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -293,35 +293,35 @@ pub struct SetQuotasOpt {
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||||
pub enum KeyOperation {
|
pub enum KeyOperation {
|
||||||
/// List keys
|
/// List keys
|
||||||
#[structopt(name = "list", version = version::garage())]
|
#[structopt(name = "list", version = garage_version())]
|
||||||
List,
|
List,
|
||||||
|
|
||||||
/// Get key info
|
/// Get key info
|
||||||
#[structopt(name = "info", version = version::garage())]
|
#[structopt(name = "info", version = garage_version())]
|
||||||
Info(KeyOpt),
|
Info(KeyOpt),
|
||||||
|
|
||||||
/// Create new key
|
/// Create new key
|
||||||
#[structopt(name = "new", version = version::garage())]
|
#[structopt(name = "new", version = garage_version())]
|
||||||
New(KeyNewOpt),
|
New(KeyNewOpt),
|
||||||
|
|
||||||
/// Rename key
|
/// Rename key
|
||||||
#[structopt(name = "rename", version = version::garage())]
|
#[structopt(name = "rename", version = garage_version())]
|
||||||
Rename(KeyRenameOpt),
|
Rename(KeyRenameOpt),
|
||||||
|
|
||||||
/// Delete key
|
/// Delete key
|
||||||
#[structopt(name = "delete", version = version::garage())]
|
#[structopt(name = "delete", version = garage_version())]
|
||||||
Delete(KeyDeleteOpt),
|
Delete(KeyDeleteOpt),
|
||||||
|
|
||||||
/// Set permission flags for key
|
/// Set permission flags for key
|
||||||
#[structopt(name = "allow", version = version::garage())]
|
#[structopt(name = "allow", version = garage_version())]
|
||||||
Allow(KeyPermOpt),
|
Allow(KeyPermOpt),
|
||||||
|
|
||||||
/// Unset permission flags for key
|
/// Unset permission flags for key
|
||||||
#[structopt(name = "deny", version = version::garage())]
|
#[structopt(name = "deny", version = garage_version())]
|
||||||
Deny(KeyPermOpt),
|
Deny(KeyPermOpt),
|
||||||
|
|
||||||
/// Import key
|
/// Import key
|
||||||
#[structopt(name = "import", version = version::garage())]
|
#[structopt(name = "import", version = garage_version())]
|
||||||
Import(KeyImportOpt),
|
Import(KeyImportOpt),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -393,7 +393,7 @@ pub struct MigrateOpt {
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||||
pub enum MigrateWhat {
|
pub enum MigrateWhat {
|
||||||
/// Migrate buckets and permissions from v0.5.0
|
/// Migrate buckets and permissions from v0.5.0
|
||||||
#[structopt(name = "buckets050", version = version::garage())]
|
#[structopt(name = "buckets050", version = garage_version())]
|
||||||
Buckets050,
|
Buckets050,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -414,19 +414,19 @@ pub struct RepairOpt {
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||||
pub enum RepairWhat {
|
pub enum RepairWhat {
|
||||||
/// Only do a full sync of metadata tables
|
/// Only do a full sync of metadata tables
|
||||||
#[structopt(name = "tables", version = version::garage())]
|
#[structopt(name = "tables", version = garage_version())]
|
||||||
Tables,
|
Tables,
|
||||||
/// Only repair (resync/rebalance) the set of stored blocks
|
/// Only repair (resync/rebalance) the set of stored blocks
|
||||||
#[structopt(name = "blocks", version = version::garage())]
|
#[structopt(name = "blocks", version = garage_version())]
|
||||||
Blocks,
|
Blocks,
|
||||||
/// Only redo the propagation of object deletions to the version table (slow)
|
/// Only redo the propagation of object deletions to the version table (slow)
|
||||||
#[structopt(name = "versions", version = version::garage())]
|
#[structopt(name = "versions", version = garage_version())]
|
||||||
Versions,
|
Versions,
|
||||||
/// Only redo the propagation of version deletions to the block ref table (extremely slow)
|
/// Only redo the propagation of version deletions to the block ref table (extremely slow)
|
||||||
#[structopt(name = "block_refs", version = version::garage())]
|
#[structopt(name = "block_refs", version = garage_version())]
|
||||||
BlockRefs,
|
BlockRefs,
|
||||||
/// Verify integrity of all blocks on disc (extremely slow, i/o intensive)
|
/// Verify integrity of all blocks on disc (extremely slow, i/o intensive)
|
||||||
#[structopt(name = "scrub", version = version::garage())]
|
#[structopt(name = "scrub", version = garage_version())]
|
||||||
Scrub {
|
Scrub {
|
||||||
#[structopt(subcommand)]
|
#[structopt(subcommand)]
|
||||||
cmd: ScrubCmd,
|
cmd: ScrubCmd,
|
||||||
|
|
@ -436,19 +436,19 @@ pub enum RepairWhat {
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||||
pub enum ScrubCmd {
|
pub enum ScrubCmd {
|
||||||
/// Start scrub
|
/// Start scrub
|
||||||
#[structopt(name = "start", version = version::garage())]
|
#[structopt(name = "start", version = garage_version())]
|
||||||
Start,
|
Start,
|
||||||
/// Pause scrub (it will resume automatically after 24 hours)
|
/// Pause scrub (it will resume automatically after 24 hours)
|
||||||
#[structopt(name = "pause", version = version::garage())]
|
#[structopt(name = "pause", version = garage_version())]
|
||||||
Pause,
|
Pause,
|
||||||
/// Resume paused scrub
|
/// Resume paused scrub
|
||||||
#[structopt(name = "resume", version = version::garage())]
|
#[structopt(name = "resume", version = garage_version())]
|
||||||
Resume,
|
Resume,
|
||||||
/// Cancel scrub in progress
|
/// Cancel scrub in progress
|
||||||
#[structopt(name = "cancel", version = version::garage())]
|
#[structopt(name = "cancel", version = garage_version())]
|
||||||
Cancel,
|
Cancel,
|
||||||
/// Set tranquility level for in-progress and future scrubs
|
/// Set tranquility level for in-progress and future scrubs
|
||||||
#[structopt(name = "set-tranquility", version = version::garage())]
|
#[structopt(name = "set-tranquility", version = garage_version())]
|
||||||
SetTranquility {
|
SetTranquility {
|
||||||
#[structopt()]
|
#[structopt()]
|
||||||
tranquility: u32,
|
tranquility: u32,
|
||||||
|
|
@ -469,10 +469,10 @@ pub struct OfflineRepairOpt {
|
||||||
pub enum OfflineRepairWhat {
|
pub enum OfflineRepairWhat {
|
||||||
/// Repair K2V item counters
|
/// Repair K2V item counters
|
||||||
#[cfg(feature = "k2v")]
|
#[cfg(feature = "k2v")]
|
||||||
#[structopt(name = "k2v_item_counters", version = version::garage())]
|
#[structopt(name = "k2v_item_counters", version = garage_version())]
|
||||||
K2VItemCounters,
|
K2VItemCounters,
|
||||||
/// Repair object counters
|
/// Repair object counters
|
||||||
#[structopt(name = "object_counters", version = version::garage())]
|
#[structopt(name = "object_counters", version = garage_version())]
|
||||||
ObjectCounters,
|
ObjectCounters,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -496,13 +496,13 @@ pub struct WorkerOpt {
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||||
pub enum WorkerCmd {
|
pub enum WorkerCmd {
|
||||||
/// List all workers on Garage node
|
/// List all workers on Garage node
|
||||||
#[structopt(name = "list", version = version::garage())]
|
#[structopt(name = "list", version = garage_version())]
|
||||||
List {
|
List {
|
||||||
#[structopt(flatten)]
|
#[structopt(flatten)]
|
||||||
opt: WorkerListOpt,
|
opt: WorkerListOpt,
|
||||||
},
|
},
|
||||||
/// Set worker parameter
|
/// Set worker parameter
|
||||||
#[structopt(name = "set", version = version::garage())]
|
#[structopt(name = "set", version = garage_version())]
|
||||||
Set {
|
Set {
|
||||||
#[structopt(subcommand)]
|
#[structopt(subcommand)]
|
||||||
opt: WorkerSetCmd,
|
opt: WorkerSetCmd,
|
||||||
|
|
@ -522,12 +522,12 @@ pub struct WorkerListOpt {
|
||||||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||||
pub enum WorkerSetCmd {
|
pub enum WorkerSetCmd {
|
||||||
/// Set tranquility of scrub operations
|
/// Set tranquility of scrub operations
|
||||||
#[structopt(name = "scrub-tranquility", version = version::garage())]
|
#[structopt(name = "scrub-tranquility", version = garage_version())]
|
||||||
ScrubTranquility { tranquility: u32 },
|
ScrubTranquility { tranquility: u32 },
|
||||||
/// Set number of concurrent block resync workers
|
/// Set number of concurrent block resync workers
|
||||||
#[structopt(name = "resync-n-workers", version = version::garage())]
|
#[structopt(name = "resync-n-workers", version = garage_version())]
|
||||||
ResyncNWorkers { n_workers: usize },
|
ResyncNWorkers { n_workers: usize },
|
||||||
/// Set tranquility of block resync operations
|
/// Set tranquility of block resync operations
|
||||||
#[structopt(name = "resync-tranquility", version = version::garage())]
|
#[structopt(name = "resync-tranquility", version = garage_version())]
|
||||||
ResyncTranquility { tranquility: u32 },
|
ResyncTranquility { tranquility: u32 },
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,8 +8,15 @@ mod admin;
|
||||||
mod cli;
|
mod cli;
|
||||||
mod repair;
|
mod repair;
|
||||||
mod server;
|
mod server;
|
||||||
|
#[cfg(feature = "telemetry-otlp")]
|
||||||
mod tracing_setup;
|
mod tracing_setup;
|
||||||
|
|
||||||
|
#[cfg(not(any(feature = "bundled-libs", feature = "system-libs")))]
|
||||||
|
compile_error!("Either bundled-libs or system-libs Cargo feature must be enabled");
|
||||||
|
|
||||||
|
#[cfg(all(feature = "bundled-libs", feature = "system-libs"))]
|
||||||
|
compile_error!("Only one of bundled-libs and system-libs Cargo features must be enabled");
|
||||||
|
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
|
@ -22,7 +29,6 @@ use garage_util::error::*;
|
||||||
|
|
||||||
use garage_rpc::system::*;
|
use garage_rpc::system::*;
|
||||||
use garage_rpc::*;
|
use garage_rpc::*;
|
||||||
use garage_util::version;
|
|
||||||
|
|
||||||
use garage_model::helper::error::Error as HelperError;
|
use garage_model::helper::error::Error as HelperError;
|
||||||
|
|
||||||
|
|
@ -30,7 +36,10 @@ use admin::*;
|
||||||
use cli::*;
|
use cli::*;
|
||||||
|
|
||||||
#[derive(StructOpt, Debug)]
|
#[derive(StructOpt, Debug)]
|
||||||
#[structopt(name = "garage", version = version::garage(), about = "S3-compatible object store for self-hosted geo-distributed deployments")]
|
#[structopt(
|
||||||
|
name = "garage",
|
||||||
|
about = "S3-compatible object store for self-hosted geo-distributed deployments"
|
||||||
|
)]
|
||||||
struct Opt {
|
struct Opt {
|
||||||
/// Host to connect to for admin operations, in the format:
|
/// Host to connect to for admin operations, in the format:
|
||||||
/// <public-key>@<ip>:<port>
|
/// <public-key>@<ip>:<port>
|
||||||
|
|
@ -71,7 +80,40 @@ async fn main() {
|
||||||
std::process::abort();
|
std::process::abort();
|
||||||
}));
|
}));
|
||||||
|
|
||||||
let opt = Opt::from_args();
|
// Initialize version and features info
|
||||||
|
let features = &[
|
||||||
|
#[cfg(feature = "k2v")]
|
||||||
|
"k2v",
|
||||||
|
#[cfg(feature = "sled")]
|
||||||
|
"sled",
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
|
"lmdb",
|
||||||
|
#[cfg(feature = "sqlite")]
|
||||||
|
"sqlite",
|
||||||
|
#[cfg(feature = "kubernetes-discovery")]
|
||||||
|
"kubernetes-discovery",
|
||||||
|
#[cfg(feature = "metrics")]
|
||||||
|
"metrics",
|
||||||
|
#[cfg(feature = "telemetry-otlp")]
|
||||||
|
"telemetry-otlp",
|
||||||
|
#[cfg(feature = "bundled-libs")]
|
||||||
|
"bundled-libs",
|
||||||
|
#[cfg(feature = "system-libs")]
|
||||||
|
"system-libs",
|
||||||
|
][..];
|
||||||
|
if let Some(git_version) = option_env!("GIT_VERSION") {
|
||||||
|
garage_util::version::init_version(git_version);
|
||||||
|
}
|
||||||
|
garage_util::version::init_features(features);
|
||||||
|
|
||||||
|
// Parse arguments
|
||||||
|
let version = format!(
|
||||||
|
"{} [features: {}]",
|
||||||
|
garage_util::version::garage_version(),
|
||||||
|
features.join(", ")
|
||||||
|
);
|
||||||
|
let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
|
||||||
|
|
||||||
let res = match opt.cmd {
|
let res = match opt.cmd {
|
||||||
Command::Server => server::run_server(opt.config_file).await,
|
Command::Server => server::run_server(opt.config_file).await,
|
||||||
Command::OfflineRepair(repair_opt) => {
|
Command::OfflineRepair(repair_opt) => {
|
||||||
|
|
|
||||||
|
|
@ -9,12 +9,13 @@ use garage_util::error::Error;
|
||||||
use garage_api::admin::api_server::AdminApiServer;
|
use garage_api::admin::api_server::AdminApiServer;
|
||||||
use garage_api::s3::api_server::S3ApiServer;
|
use garage_api::s3::api_server::S3ApiServer;
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_web::run_web_server;
|
use garage_web::WebServer;
|
||||||
|
|
||||||
#[cfg(feature = "k2v")]
|
#[cfg(feature = "k2v")]
|
||||||
use garage_api::k2v::api_server::K2VApiServer;
|
use garage_api::k2v::api_server::K2VApiServer;
|
||||||
|
|
||||||
use crate::admin::*;
|
use crate::admin::*;
|
||||||
|
#[cfg(feature = "telemetry-otlp")]
|
||||||
use crate::tracing_setup::*;
|
use crate::tracing_setup::*;
|
||||||
|
|
||||||
async fn wait_from(mut chan: watch::Receiver<bool>) {
|
async fn wait_from(mut chan: watch::Receiver<bool>) {
|
||||||
|
|
@ -29,6 +30,8 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
||||||
info!("Loading configuration...");
|
info!("Loading configuration...");
|
||||||
let config = read_config(config_file)?;
|
let config = read_config(config_file)?;
|
||||||
|
|
||||||
|
// ---- Initialize Garage internals ----
|
||||||
|
|
||||||
info!("Initializing background runner...");
|
info!("Initializing background runner...");
|
||||||
let watch_cancel = netapp::util::watch_ctrl_c();
|
let watch_cancel = netapp::util::watch_ctrl_c();
|
||||||
let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
|
let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
|
||||||
|
|
@ -36,9 +39,14 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
||||||
info!("Initializing Garage main data store...");
|
info!("Initializing Garage main data store...");
|
||||||
let garage = Garage::new(config.clone(), background)?;
|
let garage = Garage::new(config.clone(), background)?;
|
||||||
|
|
||||||
info!("Initialize tracing...");
|
if config.admin.trace_sink.is_some() {
|
||||||
if let Some(export_to) = config.admin.trace_sink {
|
info!("Initialize tracing...");
|
||||||
init_tracing(&export_to, garage.system.id)?;
|
|
||||||
|
#[cfg(feature = "telemetry-otlp")]
|
||||||
|
init_tracing(config.admin.trace_sink.as_ref().unwrap(), garage.system.id)?;
|
||||||
|
|
||||||
|
#[cfg(not(feature = "telemetry-otlp"))]
|
||||||
|
error!("Garage was built without OTLP exporter, admin.trace_sink is ignored.");
|
||||||
}
|
}
|
||||||
|
|
||||||
info!("Initialize Admin API server and metrics collector...");
|
info!("Initialize Admin API server and metrics collector...");
|
||||||
|
|
@ -50,53 +58,78 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
||||||
info!("Create admin RPC handler...");
|
info!("Create admin RPC handler...");
|
||||||
AdminRpcHandler::new(garage.clone());
|
AdminRpcHandler::new(garage.clone());
|
||||||
|
|
||||||
info!("Initializing S3 API server...");
|
// ---- Launch public-facing API servers ----
|
||||||
let s3_api_server = tokio::spawn(S3ApiServer::run(
|
|
||||||
garage.clone(),
|
|
||||||
wait_from(watch_cancel.clone()),
|
|
||||||
));
|
|
||||||
|
|
||||||
#[cfg(feature = "k2v")]
|
let mut servers = vec![];
|
||||||
let k2v_api_server = {
|
|
||||||
info!("Initializing K2V API server...");
|
|
||||||
tokio::spawn(K2VApiServer::run(
|
|
||||||
garage.clone(),
|
|
||||||
wait_from(watch_cancel.clone()),
|
|
||||||
))
|
|
||||||
};
|
|
||||||
|
|
||||||
info!("Initializing web server...");
|
if let Some(s3_bind_addr) = &config.s3_api.api_bind_addr {
|
||||||
let web_server = tokio::spawn(run_web_server(
|
info!("Initializing S3 API server...");
|
||||||
garage.clone(),
|
servers.push((
|
||||||
wait_from(watch_cancel.clone()),
|
"S3 API",
|
||||||
));
|
tokio::spawn(S3ApiServer::run(
|
||||||
|
garage.clone(),
|
||||||
|
*s3_bind_addr,
|
||||||
|
config.s3_api.s3_region.clone(),
|
||||||
|
wait_from(watch_cancel.clone()),
|
||||||
|
)),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
info!("Launching Admin API server...");
|
if config.k2v_api.is_some() {
|
||||||
let admin_server = tokio::spawn(admin_server.run(wait_from(watch_cancel.clone())));
|
#[cfg(feature = "k2v")]
|
||||||
|
{
|
||||||
|
info!("Initializing K2V API server...");
|
||||||
|
servers.push((
|
||||||
|
"K2V API",
|
||||||
|
tokio::spawn(K2VApiServer::run(
|
||||||
|
garage.clone(),
|
||||||
|
config.k2v_api.as_ref().unwrap().api_bind_addr,
|
||||||
|
config.s3_api.s3_region.clone(),
|
||||||
|
wait_from(watch_cancel.clone()),
|
||||||
|
)),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "k2v"))]
|
||||||
|
error!("K2V is not enabled in this build, cannot start K2V API server");
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(web_config) = &config.s3_web {
|
||||||
|
info!("Initializing web server...");
|
||||||
|
servers.push((
|
||||||
|
"Web",
|
||||||
|
tokio::spawn(WebServer::run(
|
||||||
|
garage.clone(),
|
||||||
|
web_config.bind_addr,
|
||||||
|
web_config.root_domain.clone(),
|
||||||
|
wait_from(watch_cancel.clone()),
|
||||||
|
)),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(admin_bind_addr) = &config.admin.api_bind_addr {
|
||||||
|
info!("Launching Admin API server...");
|
||||||
|
servers.push((
|
||||||
|
"Admin",
|
||||||
|
tokio::spawn(admin_server.run(*admin_bind_addr, wait_from(watch_cancel.clone()))),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(feature = "metrics"))]
|
||||||
|
if config.admin.metrics_token.is_some() {
|
||||||
|
warn!("This Garage version is built without the metrics feature");
|
||||||
|
}
|
||||||
|
|
||||||
// Stuff runs
|
// Stuff runs
|
||||||
|
|
||||||
// When a cancel signal is sent, stuff stops
|
// When a cancel signal is sent, stuff stops
|
||||||
if let Err(e) = s3_api_server.await? {
|
|
||||||
warn!("S3 API server exited with error: {}", e);
|
// Collect stuff
|
||||||
} else {
|
for (desc, join_handle) in servers {
|
||||||
info!("S3 API server exited without error.");
|
if let Err(e) = join_handle.await? {
|
||||||
}
|
error!("{} server exited with error: {}", desc, e);
|
||||||
#[cfg(feature = "k2v")]
|
} else {
|
||||||
if let Err(e) = k2v_api_server.await? {
|
info!("{} server exited without error.", desc);
|
||||||
warn!("K2V API server exited with error: {}", e);
|
}
|
||||||
} else {
|
|
||||||
info!("K2V API server exited without error.");
|
|
||||||
}
|
|
||||||
if let Err(e) = web_server.await? {
|
|
||||||
warn!("Web server exited with error: {}", e);
|
|
||||||
} else {
|
|
||||||
info!("Web server exited without error.");
|
|
||||||
}
|
|
||||||
if let Err(e) = admin_server.await? {
|
|
||||||
warn!("Admin web server exited with error: {}", e);
|
|
||||||
} else {
|
|
||||||
info!("Admin API server exited without error.");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Remove RPC handlers for system to break reference cycles
|
// Remove RPC handlers for system to break reference cycles
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ mod common;
|
||||||
mod admin;
|
mod admin;
|
||||||
mod bucket;
|
mod bucket;
|
||||||
|
|
||||||
|
mod s3;
|
||||||
|
|
||||||
#[cfg(feature = "k2v")]
|
#[cfg(feature = "k2v")]
|
||||||
mod k2v;
|
mod k2v;
|
||||||
|
|
||||||
mod s3;
|
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ tokio = "1.17.0"
|
||||||
|
|
||||||
# cli deps
|
# cli deps
|
||||||
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
||||||
garage_util = { version = "0.7.0", path = "../util", optional = true }
|
garage_util = { version = "0.8.0", path = "../util", optional = true }
|
||||||
|
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "garage_model"
|
name = "garage_model"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|
@ -15,11 +15,10 @@ path = "lib.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.0", path = "../db" }
|
garage_db = { version = "0.8.0", path = "../db" }
|
||||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
garage_table = { version = "0.7.0", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
garage_block = { version = "0.7.0", path = "../block" }
|
garage_block = { version = "0.8.0", path = "../block" }
|
||||||
garage_util = { version = "0.7.0", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_model_050 = { package = "garage_model", version = "0.5.1" }
|
|
||||||
|
|
||||||
async-trait = "0.1.7"
|
async-trait = "0.1.7"
|
||||||
arc-swap = "1.0"
|
arc-swap = "1.0"
|
||||||
|
|
@ -45,3 +44,6 @@ netapp = { version = "0.5", git = "https://git.deuxfleurs.fr/lx/netapp", branch
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
k2v = [ "garage_util/k2v" ]
|
k2v = [ "garage_util/k2v" ]
|
||||||
|
lmdb = [ "garage_db/lmdb" ]
|
||||||
|
sled = [ "garage_db/sled" ]
|
||||||
|
sqlite = [ "garage_db/sqlite" ]
|
||||||
|
|
|
||||||
|
|
@ -80,6 +80,8 @@ impl Garage {
|
||||||
let mut db_path = config.metadata_dir.clone();
|
let mut db_path = config.metadata_dir.clone();
|
||||||
std::fs::create_dir_all(&db_path).expect("Unable to create Garage meta data directory");
|
std::fs::create_dir_all(&db_path).expect("Unable to create Garage meta data directory");
|
||||||
let db = match config.db_engine.as_str() {
|
let db = match config.db_engine.as_str() {
|
||||||
|
// ---- Sled DB ----
|
||||||
|
#[cfg(feature = "sled")]
|
||||||
"sled" => {
|
"sled" => {
|
||||||
db_path.push("db");
|
db_path.push("db");
|
||||||
info!("Opening Sled database at: {}", db_path.display());
|
info!("Opening Sled database at: {}", db_path.display());
|
||||||
|
|
@ -91,6 +93,10 @@ impl Garage {
|
||||||
.expect("Unable to open sled DB");
|
.expect("Unable to open sled DB");
|
||||||
db::sled_adapter::SledDb::init(db)
|
db::sled_adapter::SledDb::init(db)
|
||||||
}
|
}
|
||||||
|
#[cfg(not(feature = "sled"))]
|
||||||
|
"sled" => return Err(Error::Message("sled db not available in this build".into())),
|
||||||
|
// ---- Sqlite DB ----
|
||||||
|
#[cfg(feature = "sqlite")]
|
||||||
"sqlite" | "sqlite3" | "rusqlite" => {
|
"sqlite" | "sqlite3" | "rusqlite" => {
|
||||||
db_path.push("db.sqlite");
|
db_path.push("db.sqlite");
|
||||||
info!("Opening Sqlite database at: {}", db_path.display());
|
info!("Opening Sqlite database at: {}", db_path.display());
|
||||||
|
|
@ -98,6 +104,14 @@ impl Garage {
|
||||||
.expect("Unable to open sqlite DB");
|
.expect("Unable to open sqlite DB");
|
||||||
db::sqlite_adapter::SqliteDb::init(db)
|
db::sqlite_adapter::SqliteDb::init(db)
|
||||||
}
|
}
|
||||||
|
#[cfg(not(feature = "sqlite"))]
|
||||||
|
"sqlite" | "sqlite3" | "rusqlite" => {
|
||||||
|
return Err(Error::Message(
|
||||||
|
"sqlite db not available in this build".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
// ---- LMDB DB ----
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
"lmdb" | "heed" => {
|
"lmdb" | "heed" => {
|
||||||
db_path.push("db.lmdb");
|
db_path.push("db.lmdb");
|
||||||
info!("Opening LMDB database at: {}", db_path.display());
|
info!("Opening LMDB database at: {}", db_path.display());
|
||||||
|
|
@ -116,10 +130,22 @@ impl Garage {
|
||||||
let db = env_builder.open(&db_path).expect("Unable to open LMDB DB");
|
let db = env_builder.open(&db_path).expect("Unable to open LMDB DB");
|
||||||
db::lmdb_adapter::LmdbDb::init(db)
|
db::lmdb_adapter::LmdbDb::init(db)
|
||||||
}
|
}
|
||||||
|
#[cfg(not(feature = "lmdb"))]
|
||||||
|
"lmdb" | "heed" => return Err(Error::Message("lmdb db not available in this build".into())),
|
||||||
|
// ---- Unavailable DB engine ----
|
||||||
e => {
|
e => {
|
||||||
return Err(Error::Message(format!(
|
return Err(Error::Message(format!(
|
||||||
"Unsupported DB engine: {} (options: sled, sqlite, lmdb)",
|
"Unsupported DB engine: {} (options: {})",
|
||||||
e
|
e,
|
||||||
|
vec![
|
||||||
|
#[cfg(feature = "sled")]
|
||||||
|
"sled",
|
||||||
|
#[cfg(feature = "sqlite")]
|
||||||
|
"sqlite",
|
||||||
|
#[cfg(feature = "lmdb")]
|
||||||
|
"lmdb",
|
||||||
|
]
|
||||||
|
.join(", ")
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,7 @@ use garage_util::data::*;
|
||||||
|
|
||||||
use crate::permission::BucketKeyPerm;
|
use crate::permission::BucketKeyPerm;
|
||||||
|
|
||||||
use garage_model_050::key_table as old;
|
use crate::prev::v051::key_table as old;
|
||||||
|
|
||||||
/// An api key
|
/// An api key
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,9 @@
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate tracing;
|
extern crate tracing;
|
||||||
|
|
||||||
|
// For migration from previous versions
|
||||||
|
pub(crate) mod prev;
|
||||||
|
|
||||||
pub mod permission;
|
pub mod permission;
|
||||||
|
|
||||||
pub mod index_counter;
|
pub mod index_counter;
|
||||||
|
|
|
||||||
|
|
@ -5,7 +5,7 @@ use garage_util::data::*;
|
||||||
use garage_util::error::Error as GarageError;
|
use garage_util::error::Error as GarageError;
|
||||||
use garage_util::time::*;
|
use garage_util::time::*;
|
||||||
|
|
||||||
use garage_model_050::bucket_table as old_bucket;
|
use crate::prev::v051::bucket_table as old_bucket;
|
||||||
|
|
||||||
use crate::bucket_alias_table::*;
|
use crate::bucket_alias_table::*;
|
||||||
use crate::bucket_table::*;
|
use crate::bucket_table::*;
|
||||||
|
|
|
||||||
1
src/model/prev/mod.rs
Normal file
1
src/model/prev/mod.rs
Normal file
|
|
@ -0,0 +1 @@
|
||||||
|
pub(crate) mod v051;
|
||||||
63
src/model/prev/v051/bucket_table.rs
Normal file
63
src/model/prev/v051/bucket_table.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use garage_table::crdt::Crdt;
|
||||||
|
use garage_table::*;
|
||||||
|
|
||||||
|
use super::key_table::PermissionSet;
|
||||||
|
|
||||||
|
/// A bucket is a collection of objects
|
||||||
|
///
|
||||||
|
/// Its parameters are not directly accessible as:
|
||||||
|
/// - It must be possible to merge paramaters, hence the use of a LWW CRDT.
|
||||||
|
/// - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct Bucket {
|
||||||
|
/// Name of the bucket
|
||||||
|
pub name: String,
|
||||||
|
/// State, and configuration if not deleted, of the bucket
|
||||||
|
pub state: crdt::Lww<BucketState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// State of a bucket
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum BucketState {
|
||||||
|
/// The bucket is deleted
|
||||||
|
Deleted,
|
||||||
|
/// The bucket exists
|
||||||
|
Present(BucketParams),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for BucketState {
|
||||||
|
fn merge(&mut self, o: &Self) {
|
||||||
|
match o {
|
||||||
|
BucketState::Deleted => *self = BucketState::Deleted,
|
||||||
|
BucketState::Present(other_params) => {
|
||||||
|
if let BucketState::Present(params) = self {
|
||||||
|
params.merge(other_params);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Configuration for a bucket
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct BucketParams {
|
||||||
|
/// Map of key with access to the bucket, and what kind of access they give
|
||||||
|
pub authorized_keys: crdt::LwwMap<String, PermissionSet>,
|
||||||
|
/// Is the bucket served as http
|
||||||
|
pub website: crdt::Lww<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for BucketParams {
|
||||||
|
fn merge(&mut self, o: &Self) {
|
||||||
|
self.authorized_keys.merge(&o.authorized_keys);
|
||||||
|
self.website.merge(&o.website);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for Bucket {
|
||||||
|
fn merge(&mut self, other: &Self) {
|
||||||
|
self.state.merge(&other.state);
|
||||||
|
}
|
||||||
|
}
|
||||||
50
src/model/prev/v051/key_table.rs
Normal file
50
src/model/prev/v051/key_table.rs
Normal file
|
|
@ -0,0 +1,50 @@
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use garage_table::crdt::*;
|
||||||
|
use garage_table::*;
|
||||||
|
|
||||||
|
/// An api key
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct Key {
|
||||||
|
/// The id of the key (immutable), used as partition key
|
||||||
|
pub key_id: String,
|
||||||
|
|
||||||
|
/// The secret_key associated
|
||||||
|
pub secret_key: String,
|
||||||
|
|
||||||
|
/// Name for the key
|
||||||
|
pub name: crdt::Lww<String>,
|
||||||
|
|
||||||
|
/// Is the key deleted
|
||||||
|
pub deleted: crdt::Bool,
|
||||||
|
|
||||||
|
/// Buckets in which the key is authorized. Empty if `Key` is deleted
|
||||||
|
// CRDT interaction: deleted implies authorized_buckets is empty
|
||||||
|
pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Permission given to a key in a bucket
|
||||||
|
#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct PermissionSet {
|
||||||
|
/// The key can be used to read the bucket
|
||||||
|
pub allow_read: bool,
|
||||||
|
/// The key can be used to write in the bucket
|
||||||
|
pub allow_write: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AutoCrdt for PermissionSet {
|
||||||
|
const WARN_IF_DIFFERENT: bool = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for Key {
|
||||||
|
fn merge(&mut self, other: &Self) {
|
||||||
|
self.name.merge(&other.name);
|
||||||
|
self.deleted.merge(&other.deleted);
|
||||||
|
|
||||||
|
if self.deleted.get() {
|
||||||
|
self.authorized_buckets.clear();
|
||||||
|
} else {
|
||||||
|
self.authorized_buckets.merge(&other.authorized_buckets);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
4
src/model/prev/v051/mod.rs
Normal file
4
src/model/prev/v051/mod.rs
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
pub(crate) mod bucket_table;
|
||||||
|
pub(crate) mod key_table;
|
||||||
|
pub(crate) mod object_table;
|
||||||
|
pub(crate) mod version_table;
|
||||||
149
src/model/prev/v051/object_table.rs
Normal file
149
src/model/prev/v051/object_table.rs
Normal file
|
|
@ -0,0 +1,149 @@
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use std::collections::BTreeMap;
|
||||||
|
|
||||||
|
use garage_util::data::*;
|
||||||
|
|
||||||
|
use garage_table::crdt::*;
|
||||||
|
|
||||||
|
/// An object
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct Object {
|
||||||
|
/// The bucket in which the object is stored, used as partition key
|
||||||
|
pub bucket: String,
|
||||||
|
|
||||||
|
/// The key at which the object is stored in its bucket, used as sorting key
|
||||||
|
pub key: String,
|
||||||
|
|
||||||
|
/// The list of currenty stored versions of the object
|
||||||
|
versions: Vec<ObjectVersion>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Object {
|
||||||
|
/// Get a list of currently stored versions of `Object`
|
||||||
|
pub fn versions(&self) -> &[ObjectVersion] {
|
||||||
|
&self.versions[..]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Informations about a version of an object
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectVersion {
|
||||||
|
/// Id of the version
|
||||||
|
pub uuid: Uuid,
|
||||||
|
/// Timestamp of when the object was created
|
||||||
|
pub timestamp: u64,
|
||||||
|
/// State of the version
|
||||||
|
pub state: ObjectVersionState,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// State of an object version
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum ObjectVersionState {
|
||||||
|
/// The version is being received
|
||||||
|
Uploading(ObjectVersionHeaders),
|
||||||
|
/// The version is fully received
|
||||||
|
Complete(ObjectVersionData),
|
||||||
|
/// The version uploaded containded errors or the upload was explicitly aborted
|
||||||
|
Aborted,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for ObjectVersionState {
|
||||||
|
fn merge(&mut self, other: &Self) {
|
||||||
|
use ObjectVersionState::*;
|
||||||
|
match other {
|
||||||
|
Aborted => {
|
||||||
|
*self = Aborted;
|
||||||
|
}
|
||||||
|
Complete(b) => match self {
|
||||||
|
Aborted => {}
|
||||||
|
Complete(a) => {
|
||||||
|
a.merge(b);
|
||||||
|
}
|
||||||
|
Uploading(_) => {
|
||||||
|
*self = Complete(b.clone());
|
||||||
|
}
|
||||||
|
},
|
||||||
|
Uploading(_) => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Data stored in object version
|
||||||
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub enum ObjectVersionData {
|
||||||
|
/// The object was deleted, this Version is a tombstone to mark it as such
|
||||||
|
DeleteMarker,
|
||||||
|
/// The object is short, it's stored inlined
|
||||||
|
Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
|
||||||
|
/// The object is not short, Hash of first block is stored here, next segments hashes are
|
||||||
|
/// stored in the version table
|
||||||
|
FirstBlock(ObjectVersionMeta, Hash),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AutoCrdt for ObjectVersionData {
|
||||||
|
const WARN_IF_DIFFERENT: bool = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Metadata about the object version
|
||||||
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectVersionMeta {
|
||||||
|
/// Headers to send to the client
|
||||||
|
pub headers: ObjectVersionHeaders,
|
||||||
|
/// Size of the object
|
||||||
|
pub size: u64,
|
||||||
|
/// etag of the object
|
||||||
|
pub etag: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Additional headers for an object
|
||||||
|
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct ObjectVersionHeaders {
|
||||||
|
/// Content type of the object
|
||||||
|
pub content_type: String,
|
||||||
|
/// Any other http headers to send
|
||||||
|
pub other: BTreeMap<String, String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ObjectVersion {
|
||||||
|
fn cmp_key(&self) -> (u64, Uuid) {
|
||||||
|
(self.timestamp, self.uuid)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Is the object version completely received
|
||||||
|
pub fn is_complete(&self) -> bool {
|
||||||
|
matches!(self.state, ObjectVersionState::Complete(_))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for Object {
|
||||||
|
fn merge(&mut self, other: &Self) {
|
||||||
|
// Merge versions from other into here
|
||||||
|
for other_v in other.versions.iter() {
|
||||||
|
match self
|
||||||
|
.versions
|
||||||
|
.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
|
||||||
|
{
|
||||||
|
Ok(i) => {
|
||||||
|
self.versions[i].state.merge(&other_v.state);
|
||||||
|
}
|
||||||
|
Err(i) => {
|
||||||
|
self.versions.insert(i, other_v.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Remove versions which are obsolete, i.e. those that come
|
||||||
|
// before the last version which .is_complete().
|
||||||
|
let last_complete = self
|
||||||
|
.versions
|
||||||
|
.iter()
|
||||||
|
.enumerate()
|
||||||
|
.rev()
|
||||||
|
.find(|(_, v)| v.is_complete())
|
||||||
|
.map(|(vi, _)| vi);
|
||||||
|
|
||||||
|
if let Some(last_vi) = last_complete {
|
||||||
|
self.versions = self.versions.drain(last_vi..).collect::<Vec<_>>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
79
src/model/prev/v051/version_table.rs
Normal file
79
src/model/prev/v051/version_table.rs
Normal file
|
|
@ -0,0 +1,79 @@
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use garage_util::data::*;
|
||||||
|
|
||||||
|
use garage_table::crdt::*;
|
||||||
|
use garage_table::*;
|
||||||
|
|
||||||
|
/// A version of an object
|
||||||
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct Version {
|
||||||
|
/// UUID of the version, used as partition key
|
||||||
|
pub uuid: Uuid,
|
||||||
|
|
||||||
|
// Actual data: the blocks for this version
|
||||||
|
// In the case of a multipart upload, also store the etags
|
||||||
|
// of individual parts and check them when doing CompleteMultipartUpload
|
||||||
|
/// Is this version deleted
|
||||||
|
pub deleted: crdt::Bool,
|
||||||
|
/// list of blocks of data composing the version
|
||||||
|
pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
|
||||||
|
/// Etag of each part in case of a multipart upload, empty otherwise
|
||||||
|
pub parts_etags: crdt::Map<u64, String>,
|
||||||
|
|
||||||
|
// Back link to bucket+key so that we can figure if
|
||||||
|
// this was deleted later on
|
||||||
|
/// Bucket in which the related object is stored
|
||||||
|
pub bucket: String,
|
||||||
|
/// Key in which the related object is stored
|
||||||
|
pub key: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct VersionBlockKey {
|
||||||
|
/// Number of the part
|
||||||
|
pub part_number: u64,
|
||||||
|
/// Offset of this sub-segment in its part
|
||||||
|
pub offset: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Ord for VersionBlockKey {
|
||||||
|
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||||
|
self.part_number
|
||||||
|
.cmp(&other.part_number)
|
||||||
|
.then(self.offset.cmp(&other.offset))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl PartialOrd for VersionBlockKey {
|
||||||
|
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||||
|
Some(self.cmp(other))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Informations about a single block
|
||||||
|
#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct VersionBlock {
|
||||||
|
/// Blake2 sum of the block
|
||||||
|
pub hash: Hash,
|
||||||
|
/// Size of the block
|
||||||
|
pub size: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AutoCrdt for VersionBlock {
|
||||||
|
const WARN_IF_DIFFERENT: bool = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Crdt for Version {
|
||||||
|
fn merge(&mut self, other: &Self) {
|
||||||
|
self.deleted.merge(&other.deleted);
|
||||||
|
|
||||||
|
if self.deleted.get() {
|
||||||
|
self.blocks.clear();
|
||||||
|
self.parts_etags.clear();
|
||||||
|
} else {
|
||||||
|
self.blocks.merge(&other.blocks);
|
||||||
|
self.parts_etags.merge(&other.parts_etags);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -14,7 +14,7 @@ use garage_table::*;
|
||||||
use crate::index_counter::*;
|
use crate::index_counter::*;
|
||||||
use crate::s3::version_table::*;
|
use crate::s3::version_table::*;
|
||||||
|
|
||||||
use garage_model_050::object_table as old;
|
use crate::prev::v051::object_table as old;
|
||||||
|
|
||||||
pub const OBJECTS: &str = "objects";
|
pub const OBJECTS: &str = "objects";
|
||||||
pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads";
|
pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads";
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,7 @@ use garage_table::*;
|
||||||
|
|
||||||
use crate::s3::block_ref_table::*;
|
use crate::s3::block_ref_table::*;
|
||||||
|
|
||||||
use garage_model_050::version_table as old;
|
use crate::prev::v051::version_table as old;
|
||||||
|
|
||||||
/// A version of an object
|
/// A version of an object
|
||||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "garage_rpc"
|
name = "garage_rpc"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|
@ -14,7 +14,7 @@ path = "lib.rs"
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_util = { version = "0.7.0", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
|
|
||||||
arc-swap = "1.0"
|
arc-swap = "1.0"
|
||||||
bytes = "1.0"
|
bytes = "1.0"
|
||||||
|
|
@ -53,3 +53,4 @@ hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] }
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
kubernetes-discovery = [ "kube", "k8s-openapi", "openssl", "schemars" ]
|
kubernetes-discovery = [ "kube", "k8s-openapi", "openssl", "schemars" ]
|
||||||
|
system-libs = [ "sodiumoxide/use-pkg-config" ]
|
||||||
|
|
|
||||||
|
|
@ -27,7 +27,6 @@ use garage_util::data::*;
|
||||||
use garage_util::error::*;
|
use garage_util::error::*;
|
||||||
use garage_util::persister::Persister;
|
use garage_util::persister::Persister;
|
||||||
use garage_util::time::*;
|
use garage_util::time::*;
|
||||||
use garage_util::version;
|
|
||||||
|
|
||||||
use crate::consul::*;
|
use crate::consul::*;
|
||||||
#[cfg(feature = "kubernetes-discovery")]
|
#[cfg(feature = "kubernetes-discovery")]
|
||||||
|
|
@ -40,8 +39,10 @@ const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
||||||
const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10);
|
const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10);
|
||||||
const SYSTEM_RPC_TIMEOUT: Duration = Duration::from_secs(15);
|
const SYSTEM_RPC_TIMEOUT: Duration = Duration::from_secs(15);
|
||||||
|
|
||||||
/// Version tag used for version check upon Netapp connection
|
/// Version tag used for version check upon Netapp connection.
|
||||||
pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650007; // garage 0x0007
|
/// Cluster nodes with different version tags are deemed
|
||||||
|
/// incompatible and will refuse to connect.
|
||||||
|
pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650008; // garage 0x0008
|
||||||
|
|
||||||
/// RPC endpoint used for calls related to membership
|
/// RPC endpoint used for calls related to membership
|
||||||
pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc";
|
pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc";
|
||||||
|
|
@ -320,10 +321,6 @@ impl System {
|
||||||
// ---- Administrative operations (directly available and
|
// ---- Administrative operations (directly available and
|
||||||
// also available through RPC) ----
|
// also available through RPC) ----
|
||||||
|
|
||||||
pub fn garage_version(&self) -> &'static str {
|
|
||||||
version::garage()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn get_known_nodes(&self) -> Vec<KnownNodeInfo> {
|
pub fn get_known_nodes(&self) -> Vec<KnownNodeInfo> {
|
||||||
let node_status = self.node_status.read().unwrap();
|
let node_status = self.node_status.read().unwrap();
|
||||||
let known_nodes = self
|
let known_nodes = self
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "garage_table"
|
name = "garage_table"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|
@ -15,8 +15,8 @@ path = "lib.rs"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.0", path = "../db" }
|
garage_db = { version = "0.8.0", path = "../db" }
|
||||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||||
garage_util = { version = "0.7.0", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
|
|
||||||
opentelemetry = "0.17"
|
opentelemetry = "0.17"
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "garage_util"
|
name = "garage_util"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|
@ -16,17 +16,19 @@ path = "lib.rs"
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_db = { version = "0.8.0", path = "../db" }
|
garage_db = { version = "0.8.0", path = "../db" }
|
||||||
|
|
||||||
|
arc-swap = "1.0"
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
blake2 = "0.9"
|
blake2 = "0.9"
|
||||||
bytes = "1.0"
|
bytes = "1.0"
|
||||||
digest = "0.10"
|
digest = "0.10"
|
||||||
err-derive = "0.3"
|
err-derive = "0.3"
|
||||||
|
git-version = "0.3.4"
|
||||||
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
|
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
|
||||||
hex = "0.4"
|
hex = "0.4"
|
||||||
|
lazy_static = "1.4"
|
||||||
tracing = "0.1.30"
|
tracing = "0.1.30"
|
||||||
rand = "0.8"
|
rand = "0.8"
|
||||||
sha2 = "0.10"
|
sha2 = "0.10"
|
||||||
git-version = "0.3.4"
|
|
||||||
|
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
rmp-serde = "0.15"
|
rmp-serde = "0.15"
|
||||||
|
|
|
||||||
|
|
@ -77,11 +77,10 @@ pub struct Config {
|
||||||
pub s3_api: S3ApiConfig,
|
pub s3_api: S3ApiConfig,
|
||||||
|
|
||||||
/// Configuration for K2V api
|
/// Configuration for K2V api
|
||||||
#[cfg(feature = "k2v")]
|
|
||||||
pub k2v_api: Option<K2VApiConfig>,
|
pub k2v_api: Option<K2VApiConfig>,
|
||||||
|
|
||||||
/// Configuration for serving files as normal web server
|
/// Configuration for serving files as normal web server
|
||||||
pub s3_web: WebConfig,
|
pub s3_web: Option<WebConfig>,
|
||||||
|
|
||||||
/// Configuration for the admin API endpoint
|
/// Configuration for the admin API endpoint
|
||||||
#[serde(default = "Default::default")]
|
#[serde(default = "Default::default")]
|
||||||
|
|
@ -92,7 +91,7 @@ pub struct Config {
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
pub struct S3ApiConfig {
|
pub struct S3ApiConfig {
|
||||||
/// Address and port to bind for api serving
|
/// Address and port to bind for api serving
|
||||||
pub api_bind_addr: SocketAddr,
|
pub api_bind_addr: Option<SocketAddr>,
|
||||||
/// S3 region to use
|
/// S3 region to use
|
||||||
pub s3_region: String,
|
pub s3_region: String,
|
||||||
/// Suffix to remove from domain name to find bucket. If None,
|
/// Suffix to remove from domain name to find bucket. If None,
|
||||||
|
|
@ -101,7 +100,6 @@ pub struct S3ApiConfig {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Configuration for K2V api
|
/// Configuration for K2V api
|
||||||
#[cfg(feature = "k2v")]
|
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
pub struct K2VApiConfig {
|
pub struct K2VApiConfig {
|
||||||
/// Address and port to bind for api serving
|
/// Address and port to bind for api serving
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,28 @@
|
||||||
pub fn garage() -> &'static str {
|
use std::sync::Arc;
|
||||||
option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
|
|
||||||
|
use arc_swap::{ArcSwap, ArcSwapOption};
|
||||||
|
|
||||||
|
lazy_static::lazy_static! {
|
||||||
|
static ref VERSION: ArcSwap<&'static str> = ArcSwap::new(Arc::new(git_version::git_version!(
|
||||||
prefix = "git:",
|
prefix = "git:",
|
||||||
cargo_prefix = "cargo:",
|
cargo_prefix = "cargo:",
|
||||||
fallback = "unknown"
|
fallback = "unknown"
|
||||||
))
|
)));
|
||||||
|
static ref FEATURES: ArcSwapOption<&'static [&'static str]> = ArcSwapOption::new(None);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn garage_version() -> &'static str {
|
||||||
|
&VERSION.load()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn garage_features() -> Option<&'static [&'static str]> {
|
||||||
|
FEATURES.load().as_ref().map(|f| &f[..])
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn init_version(version: &'static str) {
|
||||||
|
VERSION.store(Arc::new(version));
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn init_features(features: &'static [&'static str]) {
|
||||||
|
FEATURES.store(Some(Arc::new(features)));
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
[package]
|
[package]
|
||||||
name = "garage_web"
|
name = "garage_web"
|
||||||
version = "0.7.0"
|
version = "0.8.0"
|
||||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
license = "AGPL-3.0"
|
license = "AGPL-3.0"
|
||||||
|
|
@ -14,10 +14,10 @@ path = "lib.rs"
|
||||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
garage_api = { version = "0.7.0", path = "../api" }
|
garage_api = { version = "0.8.0", path = "../api" }
|
||||||
garage_model = { version = "0.7.0", path = "../model" }
|
garage_model = { version = "0.8.0", path = "../model" }
|
||||||
garage_util = { version = "0.7.0", path = "../util" }
|
garage_util = { version = "0.8.0", path = "../util" }
|
||||||
garage_table = { version = "0.7.0", path = "../table" }
|
garage_table = { version = "0.8.0", path = "../table" }
|
||||||
|
|
||||||
err-derive = "0.3"
|
err-derive = "0.3"
|
||||||
tracing = "0.1.30"
|
tracing = "0.1.30"
|
||||||
|
|
|
||||||
|
|
@ -6,4 +6,4 @@ mod error;
|
||||||
pub use error::Error;
|
pub use error::Error;
|
||||||
|
|
||||||
mod web_server;
|
mod web_server;
|
||||||
pub use web_server::run_web_server;
|
pub use web_server::WebServer;
|
||||||
|
|
|
||||||
|
|
@ -57,90 +57,226 @@ impl WebMetrics {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Run a web server
|
pub struct WebServer {
|
||||||
pub async fn run_web_server(
|
|
||||||
garage: Arc<Garage>,
|
|
||||||
shutdown_signal: impl Future<Output = ()>,
|
|
||||||
) -> Result<(), GarageError> {
|
|
||||||
let addr = &garage.config.s3_web.bind_addr;
|
|
||||||
|
|
||||||
let metrics = Arc::new(WebMetrics::new());
|
|
||||||
|
|
||||||
let service = make_service_fn(|conn: &AddrStream| {
|
|
||||||
let garage = garage.clone();
|
|
||||||
let metrics = metrics.clone();
|
|
||||||
|
|
||||||
let client_addr = conn.remote_addr();
|
|
||||||
async move {
|
|
||||||
Ok::<_, Error>(service_fn(move |req: Request<Body>| {
|
|
||||||
let garage = garage.clone();
|
|
||||||
let metrics = metrics.clone();
|
|
||||||
|
|
||||||
handle_request(garage, metrics, req, client_addr)
|
|
||||||
}))
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
let server = Server::bind(addr).serve(service);
|
|
||||||
let graceful = server.with_graceful_shutdown(shutdown_signal);
|
|
||||||
info!("Web server listening on http://{}", addr);
|
|
||||||
|
|
||||||
graceful.await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn handle_request(
|
|
||||||
garage: Arc<Garage>,
|
garage: Arc<Garage>,
|
||||||
metrics: Arc<WebMetrics>,
|
metrics: Arc<WebMetrics>,
|
||||||
req: Request<Body>,
|
root_domain: String,
|
||||||
addr: SocketAddr,
|
}
|
||||||
) -> Result<Response<Body>, Infallible> {
|
|
||||||
info!("{} {} {}", addr, req.method(), req.uri());
|
|
||||||
|
|
||||||
// Lots of instrumentation
|
impl WebServer {
|
||||||
let tracer = opentelemetry::global::tracer("garage");
|
/// Run a web server
|
||||||
let span = tracer
|
pub async fn run(
|
||||||
.span_builder(format!("Web {} request", req.method()))
|
garage: Arc<Garage>,
|
||||||
.with_trace_id(gen_trace_id())
|
addr: SocketAddr,
|
||||||
.with_attributes(vec![
|
root_domain: String,
|
||||||
KeyValue::new("method", format!("{}", req.method())),
|
shutdown_signal: impl Future<Output = ()>,
|
||||||
KeyValue::new("uri", req.uri().to_string()),
|
) -> Result<(), GarageError> {
|
||||||
])
|
let metrics = Arc::new(WebMetrics::new());
|
||||||
.start(&tracer);
|
let web_server = Arc::new(WebServer {
|
||||||
|
garage,
|
||||||
|
metrics,
|
||||||
|
root_domain,
|
||||||
|
});
|
||||||
|
|
||||||
let metrics_tags = &[KeyValue::new("method", req.method().to_string())];
|
let service = make_service_fn(|conn: &AddrStream| {
|
||||||
|
let web_server = web_server.clone();
|
||||||
|
|
||||||
// The actual handler
|
let client_addr = conn.remote_addr();
|
||||||
let res = serve_file(garage, &req)
|
async move {
|
||||||
.with_context(Context::current_with_span(span))
|
Ok::<_, Error>(service_fn(move |req: Request<Body>| {
|
||||||
.record_duration(&metrics.request_duration, &metrics_tags[..])
|
let web_server = web_server.clone();
|
||||||
.await;
|
|
||||||
|
|
||||||
// More instrumentation
|
web_server.handle_request(req, client_addr)
|
||||||
metrics.request_counter.add(1, &metrics_tags[..]);
|
}))
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
// Returning the result
|
let server = Server::bind(&addr).serve(service);
|
||||||
match res {
|
let graceful = server.with_graceful_shutdown(shutdown_signal);
|
||||||
Ok(res) => {
|
info!("Web server listening on http://{}", addr);
|
||||||
debug!("{} {} {}", req.method(), res.status(), req.uri());
|
|
||||||
Ok(res)
|
graceful.await?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handle_request(
|
||||||
|
self: Arc<Self>,
|
||||||
|
req: Request<Body>,
|
||||||
|
addr: SocketAddr,
|
||||||
|
) -> Result<Response<Body>, Infallible> {
|
||||||
|
info!("{} {} {}", addr, req.method(), req.uri());
|
||||||
|
|
||||||
|
// Lots of instrumentation
|
||||||
|
let tracer = opentelemetry::global::tracer("garage");
|
||||||
|
let span = tracer
|
||||||
|
.span_builder(format!("Web {} request", req.method()))
|
||||||
|
.with_trace_id(gen_trace_id())
|
||||||
|
.with_attributes(vec![
|
||||||
|
KeyValue::new("method", format!("{}", req.method())),
|
||||||
|
KeyValue::new("uri", req.uri().to_string()),
|
||||||
|
])
|
||||||
|
.start(&tracer);
|
||||||
|
|
||||||
|
let metrics_tags = &[KeyValue::new("method", req.method().to_string())];
|
||||||
|
|
||||||
|
// The actual handler
|
||||||
|
let res = self
|
||||||
|
.serve_file(&req)
|
||||||
|
.with_context(Context::current_with_span(span))
|
||||||
|
.record_duration(&self.metrics.request_duration, &metrics_tags[..])
|
||||||
|
.await;
|
||||||
|
|
||||||
|
// More instrumentation
|
||||||
|
self.metrics.request_counter.add(1, &metrics_tags[..]);
|
||||||
|
|
||||||
|
// Returning the result
|
||||||
|
match res {
|
||||||
|
Ok(res) => {
|
||||||
|
debug!("{} {} {}", req.method(), res.status(), req.uri());
|
||||||
|
Ok(res)
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
info!(
|
||||||
|
"{} {} {} {}",
|
||||||
|
req.method(),
|
||||||
|
error.http_status_code(),
|
||||||
|
req.uri(),
|
||||||
|
error
|
||||||
|
);
|
||||||
|
self.metrics.error_counter.add(
|
||||||
|
1,
|
||||||
|
&[
|
||||||
|
metrics_tags[0].clone(),
|
||||||
|
KeyValue::new("status_code", error.http_status_code().to_string()),
|
||||||
|
],
|
||||||
|
);
|
||||||
|
Ok(error_to_res(error))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
Err(error) => {
|
}
|
||||||
info!(
|
|
||||||
"{} {} {} {}",
|
async fn serve_file(self: &Arc<Self>, req: &Request<Body>) -> Result<Response<Body>, Error> {
|
||||||
req.method(),
|
// Get http authority string (eg. [::1]:3902 or garage.tld:80)
|
||||||
error.http_status_code(),
|
let authority = req
|
||||||
req.uri(),
|
.headers()
|
||||||
error
|
.get(HOST)
|
||||||
);
|
.ok_or_bad_request("HOST header required")?
|
||||||
metrics.error_counter.add(
|
.to_str()?;
|
||||||
1,
|
|
||||||
&[
|
// Get bucket
|
||||||
metrics_tags[0].clone(),
|
let host = authority_to_host(authority)?;
|
||||||
KeyValue::new("status_code", error.http_status_code().to_string()),
|
|
||||||
],
|
let bucket_name = host_to_bucket(&host, &self.root_domain).unwrap_or(&host);
|
||||||
);
|
let bucket_id = self
|
||||||
Ok(error_to_res(error))
|
.garage
|
||||||
|
.bucket_alias_table
|
||||||
|
.get(&EmptyKey, &bucket_name.to_string())
|
||||||
|
.await?
|
||||||
|
.and_then(|x| x.state.take())
|
||||||
|
.ok_or(Error::NotFound)?;
|
||||||
|
|
||||||
|
// Check bucket isn't deleted and has website access enabled
|
||||||
|
let bucket = self
|
||||||
|
.garage
|
||||||
|
.bucket_table
|
||||||
|
.get(&EmptyKey, &bucket_id)
|
||||||
|
.await?
|
||||||
|
.ok_or(Error::NotFound)?;
|
||||||
|
|
||||||
|
let website_config = bucket
|
||||||
|
.params()
|
||||||
|
.ok_or(Error::NotFound)?
|
||||||
|
.website_config
|
||||||
|
.get()
|
||||||
|
.as_ref()
|
||||||
|
.ok_or(Error::NotFound)?;
|
||||||
|
|
||||||
|
// Get path
|
||||||
|
let path = req.uri().path().to_string();
|
||||||
|
let index = &website_config.index_document;
|
||||||
|
let key = path_to_key(&path, index)?;
|
||||||
|
|
||||||
|
debug!(
|
||||||
|
"Selected bucket: \"{}\" {:?}, selected key: \"{}\"",
|
||||||
|
bucket_name, bucket_id, key
|
||||||
|
);
|
||||||
|
|
||||||
|
let ret_doc = match *req.method() {
|
||||||
|
Method::OPTIONS => handle_options_for_bucket(req, &bucket),
|
||||||
|
Method::HEAD => handle_head(self.garage.clone(), req, bucket_id, &key, None).await,
|
||||||
|
Method::GET => handle_get(self.garage.clone(), req, bucket_id, &key, None).await,
|
||||||
|
_ => Err(ApiError::bad_request("HTTP method not supported")),
|
||||||
|
}
|
||||||
|
.map_err(Error::from);
|
||||||
|
|
||||||
|
match ret_doc {
|
||||||
|
Err(error) => {
|
||||||
|
// For a HEAD or OPTIONS method, and for non-4xx errors,
|
||||||
|
// we don't return the error document as content,
|
||||||
|
// we return above and just return the error message
|
||||||
|
// by relying on err_to_res that is called when we return an Err.
|
||||||
|
if *req.method() == Method::HEAD
|
||||||
|
|| *req.method() == Method::OPTIONS
|
||||||
|
|| !error.http_status_code().is_client_error()
|
||||||
|
{
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
|
||||||
|
// If no error document is set: just return the error directly
|
||||||
|
let error_document = match &website_config.error_document {
|
||||||
|
Some(ed) => ed.trim_start_matches('/').to_owned(),
|
||||||
|
None => return Err(error),
|
||||||
|
};
|
||||||
|
|
||||||
|
// We want to return the error document
|
||||||
|
// Create a fake HTTP request with path = the error document
|
||||||
|
let req2 = Request::builder()
|
||||||
|
.uri(format!("http://{}/{}", host, &error_document))
|
||||||
|
.body(Body::empty())
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
match handle_get(self.garage.clone(), &req2, bucket_id, &error_document, None).await
|
||||||
|
{
|
||||||
|
Ok(mut error_doc) => {
|
||||||
|
// The error won't be logged back in handle_request,
|
||||||
|
// so log it here
|
||||||
|
info!(
|
||||||
|
"{} {} {} {}",
|
||||||
|
req.method(),
|
||||||
|
req.uri(),
|
||||||
|
error.http_status_code(),
|
||||||
|
error
|
||||||
|
);
|
||||||
|
|
||||||
|
*error_doc.status_mut() = error.http_status_code();
|
||||||
|
error.add_headers(error_doc.headers_mut());
|
||||||
|
|
||||||
|
// Preserve error message in a special header
|
||||||
|
for error_line in error.to_string().split('\n') {
|
||||||
|
if let Ok(v) = HeaderValue::from_bytes(error_line.as_bytes()) {
|
||||||
|
error_doc.headers_mut().append("X-Garage-Error", v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(error_doc)
|
||||||
|
}
|
||||||
|
Err(error_doc_error) => {
|
||||||
|
warn!(
|
||||||
|
"Couldn't get error document {} for bucket {:?}: {}",
|
||||||
|
error_document, bucket_id, error_doc_error
|
||||||
|
);
|
||||||
|
Err(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(mut resp) => {
|
||||||
|
// Maybe add CORS headers
|
||||||
|
if let Some(rule) = find_matching_cors_rule(&bucket, req)? {
|
||||||
|
add_cors_headers(&mut resp, rule)
|
||||||
|
.ok_or_internal_error("Invalid bucket CORS configuration")?;
|
||||||
|
}
|
||||||
|
Ok(resp)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -160,129 +296,6 @@ fn error_to_res(e: Error) -> Response<Body> {
|
||||||
http_error
|
http_error
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response<Body>, Error> {
|
|
||||||
// Get http authority string (eg. [::1]:3902 or garage.tld:80)
|
|
||||||
let authority = req
|
|
||||||
.headers()
|
|
||||||
.get(HOST)
|
|
||||||
.ok_or_bad_request("HOST header required")?
|
|
||||||
.to_str()?;
|
|
||||||
|
|
||||||
// Get bucket
|
|
||||||
let host = authority_to_host(authority)?;
|
|
||||||
let root = &garage.config.s3_web.root_domain;
|
|
||||||
|
|
||||||
let bucket_name = host_to_bucket(&host, root).unwrap_or(&host);
|
|
||||||
let bucket_id = garage
|
|
||||||
.bucket_alias_table
|
|
||||||
.get(&EmptyKey, &bucket_name.to_string())
|
|
||||||
.await?
|
|
||||||
.and_then(|x| x.state.take())
|
|
||||||
.ok_or(Error::NotFound)?;
|
|
||||||
|
|
||||||
// Check bucket isn't deleted and has website access enabled
|
|
||||||
let bucket = garage
|
|
||||||
.bucket_table
|
|
||||||
.get(&EmptyKey, &bucket_id)
|
|
||||||
.await?
|
|
||||||
.ok_or(Error::NotFound)?;
|
|
||||||
|
|
||||||
let website_config = bucket
|
|
||||||
.params()
|
|
||||||
.ok_or(Error::NotFound)?
|
|
||||||
.website_config
|
|
||||||
.get()
|
|
||||||
.as_ref()
|
|
||||||
.ok_or(Error::NotFound)?;
|
|
||||||
|
|
||||||
// Get path
|
|
||||||
let path = req.uri().path().to_string();
|
|
||||||
let index = &website_config.index_document;
|
|
||||||
let key = path_to_key(&path, index)?;
|
|
||||||
|
|
||||||
debug!(
|
|
||||||
"Selected bucket: \"{}\" {:?}, selected key: \"{}\"",
|
|
||||||
bucket_name, bucket_id, key
|
|
||||||
);
|
|
||||||
|
|
||||||
let ret_doc = match *req.method() {
|
|
||||||
Method::OPTIONS => handle_options_for_bucket(req, &bucket),
|
|
||||||
Method::HEAD => handle_head(garage.clone(), req, bucket_id, &key, None).await,
|
|
||||||
Method::GET => handle_get(garage.clone(), req, bucket_id, &key, None).await,
|
|
||||||
_ => Err(ApiError::bad_request("HTTP method not supported")),
|
|
||||||
}
|
|
||||||
.map_err(Error::from);
|
|
||||||
|
|
||||||
match ret_doc {
|
|
||||||
Err(error) => {
|
|
||||||
// For a HEAD or OPTIONS method, and for non-4xx errors,
|
|
||||||
// we don't return the error document as content,
|
|
||||||
// we return above and just return the error message
|
|
||||||
// by relying on err_to_res that is called when we return an Err.
|
|
||||||
if *req.method() == Method::HEAD
|
|
||||||
|| *req.method() == Method::OPTIONS
|
|
||||||
|| !error.http_status_code().is_client_error()
|
|
||||||
{
|
|
||||||
return Err(error);
|
|
||||||
}
|
|
||||||
|
|
||||||
// If no error document is set: just return the error directly
|
|
||||||
let error_document = match &website_config.error_document {
|
|
||||||
Some(ed) => ed.trim_start_matches('/').to_owned(),
|
|
||||||
None => return Err(error),
|
|
||||||
};
|
|
||||||
|
|
||||||
// We want to return the error document
|
|
||||||
// Create a fake HTTP request with path = the error document
|
|
||||||
let req2 = Request::builder()
|
|
||||||
.uri(format!("http://{}/{}", host, &error_document))
|
|
||||||
.body(Body::empty())
|
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
match handle_get(garage, &req2, bucket_id, &error_document, None).await {
|
|
||||||
Ok(mut error_doc) => {
|
|
||||||
// The error won't be logged back in handle_request,
|
|
||||||
// so log it here
|
|
||||||
info!(
|
|
||||||
"{} {} {} {}",
|
|
||||||
req.method(),
|
|
||||||
req.uri(),
|
|
||||||
error.http_status_code(),
|
|
||||||
error
|
|
||||||
);
|
|
||||||
|
|
||||||
*error_doc.status_mut() = error.http_status_code();
|
|
||||||
error.add_headers(error_doc.headers_mut());
|
|
||||||
|
|
||||||
// Preserve error message in a special header
|
|
||||||
for error_line in error.to_string().split('\n') {
|
|
||||||
if let Ok(v) = HeaderValue::from_bytes(error_line.as_bytes()) {
|
|
||||||
error_doc.headers_mut().append("X-Garage-Error", v);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(error_doc)
|
|
||||||
}
|
|
||||||
Err(error_doc_error) => {
|
|
||||||
warn!(
|
|
||||||
"Couldn't get error document {} for bucket {:?}: {}",
|
|
||||||
error_document, bucket_id, error_doc_error
|
|
||||||
);
|
|
||||||
Err(error)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(mut resp) => {
|
|
||||||
// Maybe add CORS headers
|
|
||||||
if let Some(rule) = find_matching_cors_rule(&bucket, req)? {
|
|
||||||
add_cors_headers(&mut resp, rule)
|
|
||||||
.ok_or_internal_error("Invalid bucket CORS configuration")?;
|
|
||||||
}
|
|
||||||
Ok(resp)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Path to key
|
/// Path to key
|
||||||
///
|
///
|
||||||
/// Convert the provided path to the internal key
|
/// Convert the provided path to the internal key
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue