performance improvements #342
42 changed files with 1337 additions and 1072 deletions
316
Cargo.lock
generated
316
Cargo.lock
generated
|
|
@ -163,7 +163,7 @@ dependencies = [
|
|||
"aws-smithy-types",
|
||||
"aws-smithy-xml",
|
||||
"aws-types",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"http",
|
||||
"md5",
|
||||
"tokio-stream",
|
||||
|
|
@ -193,7 +193,7 @@ checksum = "51d371fb688d909e5b866ff1f297bbec4621eed4f9fcdac566fcc33541f0c6a6"
|
|||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-http",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
"hex",
|
||||
"http",
|
||||
|
|
@ -227,7 +227,7 @@ dependencies = [
|
|||
"aws-smithy-http",
|
||||
"aws-smithy-http-tower",
|
||||
"aws-smithy-types",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http",
|
||||
"http-body",
|
||||
|
|
@ -248,7 +248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "f972226c639e0dc1eca2cb0220c1b5799e2bfc62eda37845b662c5d0cb972371"
|
||||
dependencies = [
|
||||
"aws-smithy-types",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"crc32fast",
|
||||
]
|
||||
|
||||
|
|
@ -260,7 +260,7 @@ checksum = "12c787e24b757634453a60ff05948aa1b450f5b3a7a2094f22acff8a5022635b"
|
|||
dependencies = [
|
||||
"aws-smithy-eventstream",
|
||||
"aws-smithy-types",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"bytes-utils",
|
||||
"futures-core",
|
||||
"http",
|
||||
|
|
@ -280,7 +280,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "64f80a2c56fc09fc9a2da3c63f286ec2a89465433219f8165e14e522283a5eb8"
|
||||
dependencies = [
|
||||
"aws-smithy-http",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"http",
|
||||
"http-body",
|
||||
"pin-project 1.0.10",
|
||||
|
|
@ -392,12 +392,6 @@ version = "1.4.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.1.0"
|
||||
|
|
@ -410,7 +404,7 @@ version = "0.1.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1934a3ef9cac8efde4966a92781e77713e1ba329f1d42e446c7d7eba340d8ef1"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"either",
|
||||
]
|
||||
|
||||
|
|
@ -783,20 +777,6 @@ dependencies = [
|
|||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "err-derive"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4"
|
||||
dependencies = [
|
||||
"proc-macro-error",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"rustversion",
|
||||
"syn",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "err-derive"
|
||||
version = "0.3.1"
|
||||
|
|
@ -985,13 +965,13 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "garage"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"assert-json-diff",
|
||||
"async-trait",
|
||||
"aws-sdk-s3",
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"bytesize",
|
||||
"chrono",
|
||||
"futures",
|
||||
|
|
@ -999,23 +979,23 @@ dependencies = [
|
|||
"garage_api",
|
||||
"garage_block",
|
||||
"garage_db",
|
||||
"garage_model 0.7.0",
|
||||
"garage_rpc 0.7.0",
|
||||
"garage_table 0.7.0",
|
||||
"garage_util 0.7.0",
|
||||
"garage_model",
|
||||
"garage_rpc",
|
||||
"garage_table",
|
||||
"garage_util",
|
||||
"garage_web",
|
||||
"hex",
|
||||
"hmac 0.12.1",
|
||||
"http",
|
||||
"hyper",
|
||||
"kuska-sodiumoxide",
|
||||
"netapp 0.4.5",
|
||||
"netapp",
|
||||
"opentelemetry",
|
||||
"opentelemetry-otlp",
|
||||
"opentelemetry-prometheus",
|
||||
"prometheus",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"serde_json",
|
||||
|
|
@ -1031,22 +1011,22 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "garage_api"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"crypto-common",
|
||||
"err-derive 0.3.1",
|
||||
"err-derive",
|
||||
"form_urlencoded",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_block",
|
||||
"garage_model 0.7.0",
|
||||
"garage_rpc 0.7.0",
|
||||
"garage_table 0.7.0",
|
||||
"garage_util 0.7.0",
|
||||
"garage_model",
|
||||
"garage_rpc",
|
||||
"garage_table",
|
||||
"garage_util",
|
||||
"hex",
|
||||
"hmac 0.12.1",
|
||||
"http",
|
||||
|
|
@ -1058,7 +1038,6 @@ dependencies = [
|
|||
"multer",
|
||||
"nom",
|
||||
"opentelemetry",
|
||||
"opentelemetry-otlp",
|
||||
"opentelemetry-prometheus",
|
||||
"percent-encoding",
|
||||
"pin-project 1.0.10",
|
||||
|
|
@ -1076,21 +1055,21 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "garage_block"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_db",
|
||||
"garage_rpc 0.7.0",
|
||||
"garage_table 0.7.0",
|
||||
"garage_util 0.7.0",
|
||||
"garage_rpc",
|
||||
"garage_table",
|
||||
"garage_util",
|
||||
"hex",
|
||||
"opentelemetry",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"tokio",
|
||||
|
|
@ -1103,7 +1082,7 @@ name = "garage_db"
|
|||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"clap 3.1.18",
|
||||
"err-derive 0.3.1",
|
||||
"err-derive",
|
||||
"heed",
|
||||
"hexdump",
|
||||
"mktemp",
|
||||
|
|
@ -1115,51 +1094,25 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "garage_model"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "584619e8999713d73761775591ad6f01ff8c9d724f3b20984f5932f1fc7f9988"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_rpc 0.5.1",
|
||||
"garage_table 0.5.1",
|
||||
"garage_util 0.5.1",
|
||||
"hex",
|
||||
"log",
|
||||
"netapp 0.3.1",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"sled",
|
||||
"tokio",
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_model"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"blake2",
|
||||
"err-derive 0.3.1",
|
||||
"err-derive",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_block",
|
||||
"garage_db",
|
||||
"garage_model 0.5.1",
|
||||
"garage_rpc 0.7.0",
|
||||
"garage_table 0.7.0",
|
||||
"garage_util 0.7.0",
|
||||
"garage_rpc",
|
||||
"garage_table",
|
||||
"garage_util",
|
||||
"hex",
|
||||
"netapp 0.4.5",
|
||||
"netapp",
|
||||
"opentelemetry",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"tokio",
|
||||
|
|
@ -1169,53 +1122,26 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "garage_rpc"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81e693aa4582cfe7a7ce70c07880e3662544b5d0cd68bc4b59c53febfbb8d1ec"
|
||||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_util 0.5.1",
|
||||
"gethostname",
|
||||
"hex",
|
||||
"hyper",
|
||||
"kuska-sodiumoxide",
|
||||
"log",
|
||||
"netapp 0.3.1",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_rpc"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_util 0.7.0",
|
||||
"garage_util",
|
||||
"gethostname",
|
||||
"hex",
|
||||
"hyper",
|
||||
"k8s-openapi",
|
||||
"kube",
|
||||
"kuska-sodiumoxide",
|
||||
"netapp 0.4.5",
|
||||
"netapp",
|
||||
"openssl",
|
||||
"opentelemetry",
|
||||
"pnet_datalink",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"rmp-serde",
|
||||
"schemars",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
|
|
@ -1227,41 +1153,19 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "garage_table"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c3557f3757e2acd29eaee86804d4e6c38d2abda81b4b349d8a0d2277044265c"
|
||||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_rpc 0.5.1",
|
||||
"garage_util 0.5.1",
|
||||
"hexdump",
|
||||
"log",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"sled",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_table"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_db",
|
||||
"garage_rpc 0.7.0",
|
||||
"garage_util 0.7.0",
|
||||
"garage_rpc",
|
||||
"garage_util",
|
||||
"hexdump",
|
||||
"opentelemetry",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_bytes",
|
||||
"tokio",
|
||||
|
|
@ -1270,50 +1174,26 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "garage_util"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e096994382447431e2f3c70e3685eb8b24c00eceff8667bb22a2a27ff17832f"
|
||||
dependencies = [
|
||||
"blake2",
|
||||
"chrono",
|
||||
"err-derive 0.3.1",
|
||||
"futures",
|
||||
"hex",
|
||||
"http",
|
||||
"hyper",
|
||||
"log",
|
||||
"netapp 0.3.1",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.9.9",
|
||||
"sled",
|
||||
"tokio",
|
||||
"toml",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_util"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"blake2",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"digest 0.10.3",
|
||||
"err-derive 0.3.1",
|
||||
"err-derive",
|
||||
"futures",
|
||||
"garage_db",
|
||||
"git-version",
|
||||
"hex",
|
||||
"http",
|
||||
"hyper",
|
||||
"netapp 0.4.5",
|
||||
"lazy_static",
|
||||
"netapp",
|
||||
"opentelemetry",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.2",
|
||||
|
|
@ -1325,14 +1205,14 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "garage_web"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
dependencies = [
|
||||
"err-derive 0.3.1",
|
||||
"err-derive",
|
||||
"futures",
|
||||
"garage_api",
|
||||
"garage_model 0.7.0",
|
||||
"garage_table 0.7.0",
|
||||
"garage_util 0.7.0",
|
||||
"garage_model",
|
||||
"garage_table",
|
||||
"garage_util",
|
||||
"http",
|
||||
"hyper",
|
||||
"opentelemetry",
|
||||
|
|
@ -1399,7 +1279,7 @@ version = "0.3.12"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62eeb471aa3e3c9197aa4bfeabfe02982f6dc96f750486c0bb0009ac58b26d2b"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
|
|
@ -1459,7 +1339,6 @@ dependencies = [
|
|||
"lmdb-rkv-sys",
|
||||
"once_cell",
|
||||
"page_size",
|
||||
"serde",
|
||||
"synchronoise",
|
||||
"url",
|
||||
]
|
||||
|
|
@ -1534,7 +1413,7 @@ version = "0.2.6"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"fnv",
|
||||
"itoa",
|
||||
]
|
||||
|
|
@ -1545,7 +1424,7 @@ version = "0.4.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"http",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
|
@ -1589,7 +1468,7 @@ version = "0.14.18"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
|
|
@ -1642,7 +1521,7 @@ version = "0.5.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"hyper",
|
||||
"native-tls",
|
||||
"tokio",
|
||||
|
|
@ -1771,7 +1650,7 @@ version = "0.0.1"
|
|||
dependencies = [
|
||||
"base64",
|
||||
"clap 3.1.18",
|
||||
"garage_util 0.7.0",
|
||||
"garage_util",
|
||||
"http",
|
||||
"log",
|
||||
"rusoto_core",
|
||||
|
|
@ -1790,7 +1669,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"http",
|
||||
"percent-encoding",
|
||||
|
|
@ -1819,7 +1698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "8cd12d68768b54bbd50547f4c7b57b73cff680ef8da3ba409463ee995cf0d707"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"dirs-next",
|
||||
"either",
|
||||
|
|
@ -2099,7 +1978,7 @@ version = "2.0.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5f8f35e687561d5c1667590911e6698a8cb714a134a7505718a182e7bc9d3836"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"encoding_rs",
|
||||
"futures-util",
|
||||
"http",
|
||||
|
|
@ -2135,28 +2014,6 @@ dependencies = [
|
|||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "netapp"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac7dcd2c6c5a9d5ea88ffc17d3339d49d308e68c4d8190c494b55ddbf78d80ad"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"bytes 0.6.0",
|
||||
"err-derive 0.2.4",
|
||||
"futures",
|
||||
"hex",
|
||||
"kuska-handshake",
|
||||
"kuska-sodiumoxide",
|
||||
"log",
|
||||
"rmp-serde 0.14.4",
|
||||
"serde",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util 0.6.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "netapp"
|
||||
version = "0.4.5"
|
||||
|
|
@ -2165,9 +2022,9 @@ checksum = "38e390e559aff9e3865c7cd326c691cbf25c580a96f259d2debc35bd4e14018e"
|
|||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"cfg-if 1.0.0",
|
||||
"err-derive 0.3.1",
|
||||
"err-derive",
|
||||
"futures",
|
||||
"hex",
|
||||
"kuska-handshake",
|
||||
|
|
@ -2176,7 +2033,7 @@ dependencies = [
|
|||
"opentelemetry",
|
||||
"opentelemetry-contrib",
|
||||
"rand 0.8.5",
|
||||
"rmp-serde 0.15.5",
|
||||
"rmp-serde",
|
||||
"serde",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
|
|
@ -2658,7 +2515,7 @@ version = "0.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"prost-derive",
|
||||
]
|
||||
|
||||
|
|
@ -2668,7 +2525,7 @@ version = "0.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"heck 0.3.3",
|
||||
"itertools 0.10.3",
|
||||
"lazy_static",
|
||||
|
|
@ -2701,7 +2558,7 @@ version = "0.9.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"prost",
|
||||
]
|
||||
|
||||
|
|
@ -2961,17 +2818,6 @@ dependencies = [
|
|||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmp-serde"
|
||||
version = "0.14.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ce7d70c926fe472aed493b902010bccc17fa9f7284145cb8772fd22fdb052d8"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"rmp",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rmp-serde"
|
||||
version = "0.15.5"
|
||||
|
|
@ -3000,7 +2846,7 @@ checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"
|
|||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"crc32fast",
|
||||
"futures",
|
||||
"http",
|
||||
|
|
@ -3042,7 +2888,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"digest 0.9.0",
|
||||
"futures",
|
||||
|
|
@ -3626,7 +3472,7 @@ version = "1.17.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"libc",
|
||||
"memchr",
|
||||
"mio",
|
||||
|
|
@ -3699,9 +3545,8 @@ version = "0.6.9"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"log",
|
||||
"pin-project-lite",
|
||||
|
|
@ -3715,7 +3560,7 @@ version = "0.7.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "64910e1b9c1901aaf5375561e35b9c057d95ff41a44ede043a03e09279eabaf1"
|
||||
dependencies = [
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
|
|
@ -3742,7 +3587,7 @@ dependencies = [
|
|||
"async-stream",
|
||||
"async-trait",
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"h2",
|
||||
|
|
@ -3803,7 +3648,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "81eca72647e58054bbfa41e6f297c23436f1c60aff6e5eb38455a0f9ca420bb5"
|
||||
dependencies = [
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http",
|
||||
|
|
@ -4250,4 +4095,5 @@ checksum = "2daf2f248d9ea44454bfcb2516534e8b8ad2fc91bf818a1885495fc42bc8ac9f"
|
|||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
[workspace]
|
||||
resolver = "2"
|
||||
members = [
|
||||
"src/db",
|
||||
"src/util",
|
||||
|
|
|
|||
|
|
@ -20,6 +20,24 @@ sudo apt-get update
|
|||
sudo apt-get install build-essential
|
||||
```
|
||||
|
||||
## Using source from the Gitea repository (recommended)
|
||||
|
||||
The primary location for Garage's source code is the
|
||||
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage).
|
||||
|
||||
Clone the repository and build Garage with the following commands:
|
||||
|
||||
```bash
|
||||
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage.git
|
||||
cd garage
|
||||
cargo build
|
||||
```
|
||||
|
||||
Be careful, as this will make a debug build of Garage, which will be extremely slow!
|
||||
To make a release build, invoke `cargo build --release` (this takes much longer).
|
||||
|
||||
The binaries built this way are found in `target/{debug,release}/garage`.
|
||||
|
||||
## Using source from `crates.io`
|
||||
|
||||
Garage's source code is published on `crates.io`, Rust's official package repository.
|
||||
|
|
@ -39,21 +57,20 @@ sudo cp $HOME/.cargo/bin/garage /usr/local/bin/garage
|
|||
```
|
||||
|
||||
|
||||
## Using source from the Gitea repository
|
||||
## Selecting features to activate in your build
|
||||
|
||||
The primary location for Garage's source code is the
|
||||
[Gitea repository](https://git.deuxfleurs.fr/Deuxfleurs/garage).
|
||||
|
||||
Clone the repository and build Garage with the following commands:
|
||||
|
||||
```bash
|
||||
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage.git
|
||||
cd garage
|
||||
cargo build
|
||||
```
|
||||
|
||||
Be careful, as this will make a debug build of Garage, which will be extremely slow!
|
||||
To make a release build, invoke `cargo build --release` (this takes much longer).
|
||||
|
||||
The binaries built this way are found in `target/{debug,release}/garage`.
|
||||
Garage supports a number of compilation options in the form of Cargo features,
|
||||
which can be used to provide builds adapted to your system and your use case.
|
||||
The following features are available:
|
||||
|
||||
| Feature | Enabled | Description |
|
||||
| ------- | ------- | ----------- |
|
||||
| `bundled-libs` | BY DEFAULT | Use bundled version of sqlite3, zstd, lmdb and libsodium |
|
||||
| `system-libs` | optional | Use system version of sqlite3, zstd, lmdb and libsodium if available (exclusive with `bundled-libs`, build using `cargo build --no-default-features --features system-libs`) |
|
||||
| `k2v` | optional | Enable the experimental K2V API (if used, all nodes on your Garage cluster must have it enabled as well) |
|
||||
| `kubernetes-discovery` | optional | Enable automatic registration and discovery of cluster nodes through the Kubernetes API |
|
||||
| `metrics` | BY DEFAULT | Enable collection of metrics in Prometheus format on the admin API |
|
||||
| `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry |
|
||||
| `sled` | BY DEFAULT | Enable using Sled to store Garage's metadata |
|
||||
| `lmdb` | optional | Enable using LMDB to store Garage's metadata |
|
||||
| `sqlite` | optional | Enable using Sqlite3 to store Garage's metadata |
|
||||
|
|
|
|||
|
|
@ -117,22 +117,34 @@ let
|
|||
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
|
||||
But we want to ship these additional features when we release Garage.
|
||||
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
|
||||
Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate.
|
||||
|
||||
[5] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library.
|
||||
However the features to do so get activated for some reason (due to a bug in cargo2nix?),
|
||||
so disable them manually here.
|
||||
*/
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage";
|
||||
overrideAttrs = drv: {
|
||||
overrideAttrs = drv:
|
||||
(if git_version != null then {
|
||||
/* [3] */ preConfigure = ''
|
||||
${drv.preConfigure or ""}
|
||||
export GIT_VERSION="${git_version}"
|
||||
'';
|
||||
} else {})
|
||||
//
|
||||
{
|
||||
/* [1] */ setBuildEnv = (buildEnv drv);
|
||||
/* [2] */ hardeningDisable = [ "pie" ];
|
||||
};
|
||||
overrideArgs = old: {
|
||||
/* [4] */ features = [ "bundled-libs" "sled" ]
|
||||
++ (if release then [ "kubernetes-discovery" "telemetry-otlp" "metrics" "lmdb" "sqlite" ] else []);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_rpc";
|
||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
overrideArgs = old: {
|
||||
/* [4] */ features = if release then [ "kubernetes-discovery" ] else [];
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
|
|
@ -142,15 +154,7 @@ let
|
|||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_util";
|
||||
overrideAttrs = drv:
|
||||
(if git_version != null then {
|
||||
/* [3] */ preConfigure = ''
|
||||
${drv.preConfigure or ""}
|
||||
export GIT_VERSION="${git_version}"
|
||||
'';
|
||||
} else {})
|
||||
//
|
||||
{ /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
|
|
@ -182,6 +186,20 @@ let
|
|||
name = "k2v-client";
|
||||
overrideAttrs = drv: { /* [1] */ setBuildEnv = (buildEnv drv); };
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "libsodium-sys";
|
||||
overrideArgs = old: {
|
||||
features = [ ]; /* [5] */
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "zstd-sys";
|
||||
overrideArgs = old: {
|
||||
features = [ ]; /* [5] */
|
||||
};
|
||||
})
|
||||
];
|
||||
|
||||
packageFun = import ../Cargo.nix;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "garage_api"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
|
@ -14,11 +14,11 @@ path = "lib.rs"
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_model = { version = "0.7.0", path = "../model" }
|
||||
garage_table = { version = "0.7.0", path = "../table" }
|
||||
garage_block = { version = "0.7.0", path = "../block" }
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
||||
garage_model = { version = "0.8.0", path = "../model" }
|
||||
garage_table = { version = "0.8.0", path = "../table" }
|
||||
garage_block = { version = "0.8.0", path = "../block" }
|
||||
garage_util = { version = "0.8.0", path = "../util" }
|
||||
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
base64 = "0.13"
|
||||
|
|
@ -54,9 +54,9 @@ quick-xml = { version = "0.21", features = [ "serialize" ] }
|
|||
url = "2.1"
|
||||
|
||||
opentelemetry = "0.17"
|
||||
opentelemetry-prometheus = "0.10"
|
||||
opentelemetry-otlp = "0.10"
|
||||
prometheus = "0.13"
|
||||
opentelemetry-prometheus = { version = "0.10", optional = true }
|
||||
prometheus = { version = "0.13", optional = true }
|
||||
|
||||
[features]
|
||||
k2v = [ "garage_util/k2v", "garage_model/k2v" ]
|
||||
metrics = [ "opentelemetry-prometheus", "prometheus" ]
|
||||
|
|
|
|||
|
|
@ -1,15 +1,17 @@
|
|||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use futures::future::Future;
|
||||
use http::header::{
|
||||
ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW, CONTENT_TYPE,
|
||||
};
|
||||
use http::header::{ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW};
|
||||
use hyper::{Body, Request, Response};
|
||||
|
||||
use opentelemetry::trace::{SpanRef, Tracer};
|
||||
use opentelemetry::trace::SpanRef;
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
use opentelemetry_prometheus::PrometheusExporter;
|
||||
#[cfg(feature = "metrics")]
|
||||
use prometheus::{Encoder, TextEncoder};
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
|
|
@ -25,6 +27,7 @@ use crate::admin::router::{Authorization, Endpoint};
|
|||
|
||||
pub struct AdminApiServer {
|
||||
garage: Arc<Garage>,
|
||||
#[cfg(feature = "metrics")]
|
||||
exporter: PrometheusExporter,
|
||||
metrics_token: Option<String>,
|
||||
admin_token: Option<String>,
|
||||
|
|
@ -32,7 +35,6 @@ pub struct AdminApiServer {
|
|||
|
||||
impl AdminApiServer {
|
||||
pub fn new(garage: Arc<Garage>) -> Self {
|
||||
let exporter = opentelemetry_prometheus::exporter().init();
|
||||
let cfg = &garage.config.admin;
|
||||
let metrics_token = cfg
|
||||
.metrics_token
|
||||
|
|
@ -44,21 +46,22 @@ impl AdminApiServer {
|
|||
.map(|tok| format!("Bearer {}", tok));
|
||||
Self {
|
||||
garage,
|
||||
exporter,
|
||||
#[cfg(feature = "metrics")]
|
||||
exporter: opentelemetry_prometheus::exporter().init(),
|
||||
metrics_token,
|
||||
admin_token,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(self, shutdown_signal: impl Future<Output = ()>) -> Result<(), GarageError> {
|
||||
if let Some(bind_addr) = self.garage.config.admin.api_bind_addr {
|
||||
pub async fn run(
|
||||
self,
|
||||
bind_addr: SocketAddr,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
let region = self.garage.config.s3_api.s3_region.clone();
|
||||
ApiServer::new(region, self)
|
||||
.run_server(bind_addr, shutdown_signal)
|
||||
.await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_options(&self, _req: &Request<Body>) -> Result<Response<Body>, Error> {
|
||||
|
|
@ -71,6 +74,10 @@ impl AdminApiServer {
|
|||
}
|
||||
|
||||
fn handle_metrics(&self) -> Result<Response<Body>, Error> {
|
||||
#[cfg(feature = "metrics")]
|
||||
{
|
||||
use opentelemetry::trace::Tracer;
|
||||
|
||||
let mut buffer = vec![];
|
||||
let encoder = TextEncoder::new();
|
||||
|
||||
|
|
@ -85,9 +92,14 @@ impl AdminApiServer {
|
|||
|
||||
Ok(Response::builder()
|
||||
.status(200)
|
||||
.header(CONTENT_TYPE, encoder.format_type())
|
||||
.header(http::header::CONTENT_TYPE, encoder.format_type())
|
||||
.body(Body::from(buffer))?)
|
||||
}
|
||||
#[cfg(not(feature = "metrics"))]
|
||||
Err(Error::bad_request(
|
||||
"Garage was built without the metrics feature".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@ use crate::helpers::{json_ok_response, parse_json_body};
|
|||
pub async fn handle_get_cluster_status(garage: &Arc<Garage>) -> Result<Response<Body>, Error> {
|
||||
let res = GetClusterStatusResponse {
|
||||
node: hex::encode(garage.system.id),
|
||||
garage_version: garage.system.garage_version(),
|
||||
garage_version: garage_util::version::garage_version(),
|
||||
garage_features: garage_util::version::garage_features(),
|
||||
db_engine: garage.db.engine(),
|
||||
known_nodes: garage
|
||||
.system
|
||||
|
|
@ -99,6 +100,7 @@ fn get_cluster_layout(garage: &Arc<Garage>) -> GetClusterLayoutResponse {
|
|||
struct GetClusterStatusResponse {
|
||||
node: String,
|
||||
garage_version: &'static str,
|
||||
garage_features: Option<&'static [&'static str]>,
|
||||
db_engine: String,
|
||||
known_nodes: HashMap<String, KnownNodeResp>,
|
||||
layout: GetClusterLayoutResponse,
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
|
@ -36,20 +37,13 @@ pub(crate) struct K2VApiEndpoint {
|
|||
impl K2VApiServer {
|
||||
pub async fn run(
|
||||
garage: Arc<Garage>,
|
||||
bind_addr: SocketAddr,
|
||||
s3_region: String,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
if let Some(cfg) = &garage.config.k2v_api {
|
||||
let bind_addr = cfg.api_bind_addr;
|
||||
|
||||
ApiServer::new(
|
||||
garage.config.s3_api.s3_region.clone(),
|
||||
K2VApiServer { garage },
|
||||
)
|
||||
ApiServer::new(s3_region, K2VApiServer { garage })
|
||||
.run_server(bind_addr, shutdown_signal)
|
||||
.await
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
|
@ -43,14 +44,11 @@ pub(crate) struct S3ApiEndpoint {
|
|||
impl S3ApiServer {
|
||||
pub async fn run(
|
||||
garage: Arc<Garage>,
|
||||
addr: SocketAddr,
|
||||
s3_region: String,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
let addr = garage.config.s3_api.api_bind_addr;
|
||||
|
||||
ApiServer::new(
|
||||
garage.config.s3_api.s3_region.clone(),
|
||||
S3ApiServer { garage },
|
||||
)
|
||||
ApiServer::new(s3_region, S3ApiServer { garage })
|
||||
.run_server(addr, shutdown_signal)
|
||||
.await
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "garage_block"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
|
@ -15,9 +15,9 @@ path = "lib.rs"
|
|||
|
||||
[dependencies]
|
||||
garage_db = { version = "0.8.0", path = "../db" }
|
||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
garage_table = { version = "0.7.0", path = "../table" }
|
||||
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||
garage_util = { version = "0.8.0", path = "../util" }
|
||||
garage_table = { version = "0.8.0", path = "../table" }
|
||||
|
||||
opentelemetry = "0.17"
|
||||
|
||||
|
|
@ -36,3 +36,7 @@ serde_bytes = "0.11"
|
|||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||
|
||||
|
||||
[features]
|
||||
system-libs = [ "zstd/pkg-config" ]
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ err-derive = "0.3"
|
|||
hexdump = "0.1"
|
||||
tracing = "0.1.30"
|
||||
|
||||
heed = "0.11"
|
||||
rusqlite = { version = "0.27", features = ["bundled"] }
|
||||
sled = "0.34"
|
||||
heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true }
|
||||
rusqlite = { version = "0.27", optional = true }
|
||||
sled = { version = "0.34", optional = true }
|
||||
|
||||
# cli deps
|
||||
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
||||
|
|
@ -33,4 +33,7 @@ pretty_env_logger = { version = "0.4", optional = true }
|
|||
mktemp = "0.4"
|
||||
|
||||
[features]
|
||||
bundled-libs = [ "rusqlite/bundled" ]
|
||||
cli = ["clap", "pretty_env_logger"]
|
||||
lmdb = [ "heed" ]
|
||||
sqlite = [ "rusqlite" ]
|
||||
|
|
|
|||
|
|
@ -1,8 +1,15 @@
|
|||
#[macro_use]
|
||||
#[cfg(feature = "sqlite")]
|
||||
extern crate tracing;
|
||||
|
||||
#[cfg(not(any(feature = "lmdb", feature = "sled", feature = "sqlite")))]
|
||||
compile_error!("Must activate the Cargo feature for at least one DB engine: lmdb, sled or sqlite.");
|
||||
|
||||
#[cfg(feature = "lmdb")]
|
||||
pub mod lmdb_adapter;
|
||||
#[cfg(feature = "sled")]
|
||||
pub mod sled_adapter;
|
||||
#[cfg(feature = "sqlite")]
|
||||
pub mod sqlite_adapter;
|
||||
|
||||
pub mod counted_tree_hack;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "garage"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
|
@ -22,13 +22,13 @@ path = "tests/lib.rs"
|
|||
|
||||
[dependencies]
|
||||
garage_db = { version = "0.8.0", path = "../db" }
|
||||
garage_api = { version = "0.7.0", path = "../api" }
|
||||
garage_block = { version = "0.7.0", path = "../block" }
|
||||
garage_model = { version = "0.7.0", path = "../model" }
|
||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
||||
garage_table = { version = "0.7.0", path = "../table" }
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
garage_web = { version = "0.7.0", path = "../web" }
|
||||
garage_api = { version = "0.8.0", path = "../api" }
|
||||
garage_block = { version = "0.8.0", path = "../block" }
|
||||
garage_model = { version = "0.8.0", path = "../model" }
|
||||
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||
garage_table = { version = "0.8.0", path = "../table" }
|
||||
garage_util = { version = "0.8.0", path = "../util" }
|
||||
garage_web = { version = "0.8.0", path = "../web" }
|
||||
|
||||
bytes = "1.0"
|
||||
bytesize = "1.1"
|
||||
|
|
@ -55,9 +55,9 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
|
|||
netapp = "0.4"
|
||||
|
||||
opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
|
||||
opentelemetry-prometheus = "0.10"
|
||||
opentelemetry-otlp = "0.10"
|
||||
prometheus = "0.13"
|
||||
opentelemetry-prometheus = { version = "0.10", optional = true }
|
||||
opentelemetry-otlp = { version = "0.10", optional = true }
|
||||
prometheus = { version = "0.13", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
aws-sdk-s3 = "0.8"
|
||||
|
|
@ -74,5 +74,26 @@ base64 = "0.13"
|
|||
|
||||
|
||||
[features]
|
||||
kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
|
||||
default = [ "bundled-libs", "metrics", "sled" ]
|
||||
|
||||
k2v = [ "garage_util/k2v", "garage_api/k2v" ]
|
||||
|
||||
# Database engines, Sled is still our default even though we don't like it
|
||||
sled = [ "garage_model/sled" ]
|
||||
lmdb = [ "garage_model/lmdb" ]
|
||||
sqlite = [ "garage_model/sqlite" ]
|
||||
|
||||
# Automatic registration and discovery via Kubernetes API
|
||||
kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
|
||||
# Prometheus exporter (/metrics endpoint).
|
||||
metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ]
|
||||
# Exporter for the OpenTelemetry Collector.
|
||||
telemetry-otlp = [ "opentelemetry-otlp" ]
|
||||
|
||||
# NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
|
||||
# exactly one of them should be enabled.
|
||||
|
||||
# Use bundled libsqlite instead of linking against system-provided.
|
||||
bundled-libs = [ "garage_db/bundled-libs" ]
|
||||
# Link against system-provided libsodium and libzstd.
|
||||
system-libs = [ "garage_block/system-libs", "garage_rpc/system-libs", "sodiumoxide/use-pkg-config" ]
|
||||
|
|
|
|||
|
|
@ -741,8 +741,11 @@ impl AdminRpcHandler {
|
|||
let mut ret = String::new();
|
||||
writeln!(
|
||||
&mut ret,
|
||||
"\nGarage version: {}",
|
||||
self.garage.system.garage_version(),
|
||||
"\nGarage version: {} [features: {}]",
|
||||
garage_util::version::garage_version(),
|
||||
garage_util::version::garage_features()
|
||||
.map(|list| list.join(", "))
|
||||
.unwrap_or_else(|| "(unknown)".into()),
|
||||
)
|
||||
.unwrap();
|
||||
writeln!(&mut ret, "\nDatabase engine: {}", self.garage.db.engine()).unwrap();
|
||||
|
|
|
|||
|
|
@ -1,65 +1,65 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_util::version;
|
||||
use structopt::StructOpt;
|
||||
|
||||
use garage_util::version::garage_version;
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub enum Command {
|
||||
/// Run Garage server
|
||||
#[structopt(name = "server", version = version::garage())]
|
||||
#[structopt(name = "server", version = garage_version())]
|
||||
Server,
|
||||
|
||||
/// Get network status
|
||||
#[structopt(name = "status", version = version::garage())]
|
||||
#[structopt(name = "status", version = garage_version())]
|
||||
Status,
|
||||
|
||||
/// Operations on individual Garage nodes
|
||||
#[structopt(name = "node", version = version::garage())]
|
||||
#[structopt(name = "node", version = garage_version())]
|
||||
Node(NodeOperation),
|
||||
|
||||
/// Operations on the assignation of node roles in the cluster layout
|
||||
#[structopt(name = "layout", version = version::garage())]
|
||||
#[structopt(name = "layout", version = garage_version())]
|
||||
Layout(LayoutOperation),
|
||||
|
||||
/// Operations on buckets
|
||||
#[structopt(name = "bucket", version = version::garage())]
|
||||
#[structopt(name = "bucket", version = garage_version())]
|
||||
Bucket(BucketOperation),
|
||||
|
||||
/// Operations on S3 access keys
|
||||
#[structopt(name = "key", version = version::garage())]
|
||||
#[structopt(name = "key", version = garage_version())]
|
||||
Key(KeyOperation),
|
||||
|
||||
/// Run migrations from previous Garage version
|
||||
/// (DO NOT USE WITHOUT READING FULL DOCUMENTATION)
|
||||
#[structopt(name = "migrate", version = version::garage())]
|
||||
#[structopt(name = "migrate", version = garage_version())]
|
||||
Migrate(MigrateOpt),
|
||||
|
||||
/// Start repair of node data on remote node
|
||||
#[structopt(name = "repair", version = version::garage())]
|
||||
#[structopt(name = "repair", version = garage_version())]
|
||||
Repair(RepairOpt),
|
||||
|
||||
/// Offline reparation of node data (these repairs must be run offline
|
||||
/// directly on the server node)
|
||||
#[structopt(name = "offline-repair", version = version::garage())]
|
||||
#[structopt(name = "offline-repair", version = garage_version())]
|
||||
OfflineRepair(OfflineRepairOpt),
|
||||
|
||||
/// Gather node statistics
|
||||
#[structopt(name = "stats", version = version::garage())]
|
||||
#[structopt(name = "stats", version = garage_version())]
|
||||
Stats(StatsOpt),
|
||||
|
||||
/// Manage background workers
|
||||
#[structopt(name = "worker", version = version::garage())]
|
||||
#[structopt(name = "worker", version = garage_version())]
|
||||
Worker(WorkerOpt),
|
||||
}
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub enum NodeOperation {
|
||||
/// Print identifier (public key) of this Garage node
|
||||
#[structopt(name = "id", version = version::garage())]
|
||||
#[structopt(name = "id", version = garage_version())]
|
||||
NodeId(NodeIdOpt),
|
||||
|
||||
/// Connect to Garage node that is currently isolated from the system
|
||||
#[structopt(name = "connect", version = version::garage())]
|
||||
#[structopt(name = "connect", version = garage_version())]
|
||||
Connect(ConnectNodeOpt),
|
||||
}
|
||||
|
||||
|
|
@ -80,23 +80,23 @@ pub struct ConnectNodeOpt {
|
|||
#[derive(StructOpt, Debug)]
|
||||
pub enum LayoutOperation {
|
||||
/// Assign role to Garage node
|
||||
#[structopt(name = "assign", version = version::garage())]
|
||||
#[structopt(name = "assign", version = garage_version())]
|
||||
Assign(AssignRoleOpt),
|
||||
|
||||
/// Remove role from Garage cluster node
|
||||
#[structopt(name = "remove", version = version::garage())]
|
||||
#[structopt(name = "remove", version = garage_version())]
|
||||
Remove(RemoveRoleOpt),
|
||||
|
||||
/// Show roles currently assigned to nodes and changes staged for commit
|
||||
#[structopt(name = "show", version = version::garage())]
|
||||
#[structopt(name = "show", version = garage_version())]
|
||||
Show,
|
||||
|
||||
/// Apply staged changes to cluster layout
|
||||
#[structopt(name = "apply", version = version::garage())]
|
||||
#[structopt(name = "apply", version = garage_version())]
|
||||
Apply(ApplyLayoutOpt),
|
||||
|
||||
/// Revert staged changes to cluster layout
|
||||
#[structopt(name = "revert", version = version::garage())]
|
||||
#[structopt(name = "revert", version = garage_version())]
|
||||
Revert(RevertLayoutOpt),
|
||||
}
|
||||
|
||||
|
|
@ -151,43 +151,43 @@ pub struct RevertLayoutOpt {
|
|||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||
pub enum BucketOperation {
|
||||
/// List buckets
|
||||
#[structopt(name = "list", version = version::garage())]
|
||||
#[structopt(name = "list", version = garage_version())]
|
||||
List,
|
||||
|
||||
/// Get bucket info
|
||||
#[structopt(name = "info", version = version::garage())]
|
||||
#[structopt(name = "info", version = garage_version())]
|
||||
Info(BucketOpt),
|
||||
|
||||
/// Create bucket
|
||||
#[structopt(name = "create", version = version::garage())]
|
||||
#[structopt(name = "create", version = garage_version())]
|
||||
Create(BucketOpt),
|
||||
|
||||
/// Delete bucket
|
||||
#[structopt(name = "delete", version = version::garage())]
|
||||
#[structopt(name = "delete", version = garage_version())]
|
||||
Delete(DeleteBucketOpt),
|
||||
|
||||
/// Alias bucket under new name
|
||||
#[structopt(name = "alias", version = version::garage())]
|
||||
#[structopt(name = "alias", version = garage_version())]
|
||||
Alias(AliasBucketOpt),
|
||||
|
||||
/// Remove bucket alias
|
||||
#[structopt(name = "unalias", version = version::garage())]
|
||||
#[structopt(name = "unalias", version = garage_version())]
|
||||
Unalias(UnaliasBucketOpt),
|
||||
|
||||
/// Allow key to read or write to bucket
|
||||
#[structopt(name = "allow", version = version::garage())]
|
||||
#[structopt(name = "allow", version = garage_version())]
|
||||
Allow(PermBucketOpt),
|
||||
|
||||
/// Deny key from reading or writing to bucket
|
||||
#[structopt(name = "deny", version = version::garage())]
|
||||
#[structopt(name = "deny", version = garage_version())]
|
||||
Deny(PermBucketOpt),
|
||||
|
||||
/// Expose as website or not
|
||||
#[structopt(name = "website", version = version::garage())]
|
||||
#[structopt(name = "website", version = garage_version())]
|
||||
Website(WebsiteOpt),
|
||||
|
||||
/// Set the quotas for this bucket
|
||||
#[structopt(name = "set-quotas", version = version::garage())]
|
||||
#[structopt(name = "set-quotas", version = garage_version())]
|
||||
SetQuotas(SetQuotasOpt),
|
||||
}
|
||||
|
||||
|
|
@ -293,35 +293,35 @@ pub struct SetQuotasOpt {
|
|||
#[derive(Serialize, Deserialize, StructOpt, Debug)]
|
||||
pub enum KeyOperation {
|
||||
/// List keys
|
||||
#[structopt(name = "list", version = version::garage())]
|
||||
#[structopt(name = "list", version = garage_version())]
|
||||
List,
|
||||
|
||||
/// Get key info
|
||||
#[structopt(name = "info", version = version::garage())]
|
||||
#[structopt(name = "info", version = garage_version())]
|
||||
Info(KeyOpt),
|
||||
|
||||
/// Create new key
|
||||
#[structopt(name = "new", version = version::garage())]
|
||||
#[structopt(name = "new", version = garage_version())]
|
||||
New(KeyNewOpt),
|
||||
|
||||
/// Rename key
|
||||
#[structopt(name = "rename", version = version::garage())]
|
||||
#[structopt(name = "rename", version = garage_version())]
|
||||
Rename(KeyRenameOpt),
|
||||
|
||||
/// Delete key
|
||||
#[structopt(name = "delete", version = version::garage())]
|
||||
#[structopt(name = "delete", version = garage_version())]
|
||||
Delete(KeyDeleteOpt),
|
||||
|
||||
/// Set permission flags for key
|
||||
#[structopt(name = "allow", version = version::garage())]
|
||||
#[structopt(name = "allow", version = garage_version())]
|
||||
Allow(KeyPermOpt),
|
||||
|
||||
/// Unset permission flags for key
|
||||
#[structopt(name = "deny", version = version::garage())]
|
||||
#[structopt(name = "deny", version = garage_version())]
|
||||
Deny(KeyPermOpt),
|
||||
|
||||
/// Import key
|
||||
#[structopt(name = "import", version = version::garage())]
|
||||
#[structopt(name = "import", version = garage_version())]
|
||||
Import(KeyImportOpt),
|
||||
}
|
||||
|
||||
|
|
@ -393,7 +393,7 @@ pub struct MigrateOpt {
|
|||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum MigrateWhat {
|
||||
/// Migrate buckets and permissions from v0.5.0
|
||||
#[structopt(name = "buckets050", version = version::garage())]
|
||||
#[structopt(name = "buckets050", version = garage_version())]
|
||||
Buckets050,
|
||||
}
|
||||
|
||||
|
|
@ -414,19 +414,19 @@ pub struct RepairOpt {
|
|||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum RepairWhat {
|
||||
/// Only do a full sync of metadata tables
|
||||
#[structopt(name = "tables", version = version::garage())]
|
||||
#[structopt(name = "tables", version = garage_version())]
|
||||
Tables,
|
||||
/// Only repair (resync/rebalance) the set of stored blocks
|
||||
#[structopt(name = "blocks", version = version::garage())]
|
||||
#[structopt(name = "blocks", version = garage_version())]
|
||||
Blocks,
|
||||
/// Only redo the propagation of object deletions to the version table (slow)
|
||||
#[structopt(name = "versions", version = version::garage())]
|
||||
#[structopt(name = "versions", version = garage_version())]
|
||||
Versions,
|
||||
/// Only redo the propagation of version deletions to the block ref table (extremely slow)
|
||||
#[structopt(name = "block_refs", version = version::garage())]
|
||||
#[structopt(name = "block_refs", version = garage_version())]
|
||||
BlockRefs,
|
||||
/// Verify integrity of all blocks on disc (extremely slow, i/o intensive)
|
||||
#[structopt(name = "scrub", version = version::garage())]
|
||||
#[structopt(name = "scrub", version = garage_version())]
|
||||
Scrub {
|
||||
#[structopt(subcommand)]
|
||||
cmd: ScrubCmd,
|
||||
|
|
@ -436,19 +436,19 @@ pub enum RepairWhat {
|
|||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum ScrubCmd {
|
||||
/// Start scrub
|
||||
#[structopt(name = "start", version = version::garage())]
|
||||
#[structopt(name = "start", version = garage_version())]
|
||||
Start,
|
||||
/// Pause scrub (it will resume automatically after 24 hours)
|
||||
#[structopt(name = "pause", version = version::garage())]
|
||||
#[structopt(name = "pause", version = garage_version())]
|
||||
Pause,
|
||||
/// Resume paused scrub
|
||||
#[structopt(name = "resume", version = version::garage())]
|
||||
#[structopt(name = "resume", version = garage_version())]
|
||||
Resume,
|
||||
/// Cancel scrub in progress
|
||||
#[structopt(name = "cancel", version = version::garage())]
|
||||
#[structopt(name = "cancel", version = garage_version())]
|
||||
Cancel,
|
||||
/// Set tranquility level for in-progress and future scrubs
|
||||
#[structopt(name = "set-tranquility", version = version::garage())]
|
||||
#[structopt(name = "set-tranquility", version = garage_version())]
|
||||
SetTranquility {
|
||||
#[structopt()]
|
||||
tranquility: u32,
|
||||
|
|
@ -469,10 +469,10 @@ pub struct OfflineRepairOpt {
|
|||
pub enum OfflineRepairWhat {
|
||||
/// Repair K2V item counters
|
||||
#[cfg(feature = "k2v")]
|
||||
#[structopt(name = "k2v_item_counters", version = version::garage())]
|
||||
#[structopt(name = "k2v_item_counters", version = garage_version())]
|
||||
K2VItemCounters,
|
||||
/// Repair object counters
|
||||
#[structopt(name = "object_counters", version = version::garage())]
|
||||
#[structopt(name = "object_counters", version = garage_version())]
|
||||
ObjectCounters,
|
||||
}
|
||||
|
||||
|
|
@ -496,13 +496,13 @@ pub struct WorkerOpt {
|
|||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum WorkerCmd {
|
||||
/// List all workers on Garage node
|
||||
#[structopt(name = "list", version = version::garage())]
|
||||
#[structopt(name = "list", version = garage_version())]
|
||||
List {
|
||||
#[structopt(flatten)]
|
||||
opt: WorkerListOpt,
|
||||
},
|
||||
/// Set worker parameter
|
||||
#[structopt(name = "set", version = version::garage())]
|
||||
#[structopt(name = "set", version = garage_version())]
|
||||
Set {
|
||||
#[structopt(subcommand)]
|
||||
opt: WorkerSetCmd,
|
||||
|
|
@ -522,12 +522,12 @@ pub struct WorkerListOpt {
|
|||
#[derive(Serialize, Deserialize, StructOpt, Debug, Eq, PartialEq, Clone)]
|
||||
pub enum WorkerSetCmd {
|
||||
/// Set tranquility of scrub operations
|
||||
#[structopt(name = "scrub-tranquility", version = version::garage())]
|
||||
#[structopt(name = "scrub-tranquility", version = garage_version())]
|
||||
ScrubTranquility { tranquility: u32 },
|
||||
/// Set number of concurrent block resync workers
|
||||
#[structopt(name = "resync-n-workers", version = version::garage())]
|
||||
#[structopt(name = "resync-n-workers", version = garage_version())]
|
||||
ResyncNWorkers { n_workers: usize },
|
||||
/// Set tranquility of block resync operations
|
||||
#[structopt(name = "resync-tranquility", version = version::garage())]
|
||||
#[structopt(name = "resync-tranquility", version = garage_version())]
|
||||
ResyncTranquility { tranquility: u32 },
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,8 +8,15 @@ mod admin;
|
|||
mod cli;
|
||||
mod repair;
|
||||
mod server;
|
||||
#[cfg(feature = "telemetry-otlp")]
|
||||
mod tracing_setup;
|
||||
|
||||
#[cfg(not(any(feature = "bundled-libs", feature = "system-libs")))]
|
||||
compile_error!("Either bundled-libs or system-libs Cargo feature must be enabled");
|
||||
|
||||
#[cfg(all(feature = "bundled-libs", feature = "system-libs"))]
|
||||
compile_error!("Only one of bundled-libs and system-libs Cargo features must be enabled");
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
|
||||
|
|
@ -22,7 +29,6 @@ use garage_util::error::*;
|
|||
|
||||
use garage_rpc::system::*;
|
||||
use garage_rpc::*;
|
||||
use garage_util::version;
|
||||
|
||||
use garage_model::helper::error::Error as HelperError;
|
||||
|
||||
|
|
@ -30,7 +36,10 @@ use admin::*;
|
|||
use cli::*;
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
#[structopt(name = "garage", version = version::garage(), about = "S3-compatible object store for self-hosted geo-distributed deployments")]
|
||||
#[structopt(
|
||||
name = "garage",
|
||||
about = "S3-compatible object store for self-hosted geo-distributed deployments"
|
||||
)]
|
||||
struct Opt {
|
||||
/// Host to connect to for admin operations, in the format:
|
||||
/// <public-key>@<ip>:<port>
|
||||
|
|
@ -71,7 +80,40 @@ async fn main() {
|
|||
std::process::abort();
|
||||
}));
|
||||
|
||||
let opt = Opt::from_args();
|
||||
// Initialize version and features info
|
||||
let features = &[
|
||||
#[cfg(feature = "k2v")]
|
||||
"k2v",
|
||||
#[cfg(feature = "sled")]
|
||||
"sled",
|
||||
#[cfg(feature = "lmdb")]
|
||||
"lmdb",
|
||||
#[cfg(feature = "sqlite")]
|
||||
"sqlite",
|
||||
#[cfg(feature = "kubernetes-discovery")]
|
||||
"kubernetes-discovery",
|
||||
#[cfg(feature = "metrics")]
|
||||
"metrics",
|
||||
#[cfg(feature = "telemetry-otlp")]
|
||||
"telemetry-otlp",
|
||||
#[cfg(feature = "bundled-libs")]
|
||||
"bundled-libs",
|
||||
#[cfg(feature = "system-libs")]
|
||||
"system-libs",
|
||||
][..];
|
||||
if let Some(git_version) = option_env!("GIT_VERSION") {
|
||||
garage_util::version::init_version(git_version);
|
||||
}
|
||||
garage_util::version::init_features(features);
|
||||
|
||||
// Parse arguments
|
||||
let version = format!(
|
||||
"{} [features: {}]",
|
||||
garage_util::version::garage_version(),
|
||||
features.join(", ")
|
||||
);
|
||||
let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches());
|
||||
|
||||
let res = match opt.cmd {
|
||||
Command::Server => server::run_server(opt.config_file).await,
|
||||
Command::OfflineRepair(repair_opt) => {
|
||||
|
|
|
|||
|
|
@ -9,12 +9,13 @@ use garage_util::error::Error;
|
|||
use garage_api::admin::api_server::AdminApiServer;
|
||||
use garage_api::s3::api_server::S3ApiServer;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_web::run_web_server;
|
||||
use garage_web::WebServer;
|
||||
|
||||
#[cfg(feature = "k2v")]
|
||||
use garage_api::k2v::api_server::K2VApiServer;
|
||||
|
||||
use crate::admin::*;
|
||||
#[cfg(feature = "telemetry-otlp")]
|
||||
use crate::tracing_setup::*;
|
||||
|
||||
async fn wait_from(mut chan: watch::Receiver<bool>) {
|
||||
|
|
@ -29,6 +30,8 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||
info!("Loading configuration...");
|
||||
let config = read_config(config_file)?;
|
||||
|
||||
// ---- Initialize Garage internals ----
|
||||
|
||||
info!("Initializing background runner...");
|
||||
let watch_cancel = netapp::util::watch_ctrl_c();
|
||||
let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
|
||||
|
|
@ -36,9 +39,14 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||
info!("Initializing Garage main data store...");
|
||||
let garage = Garage::new(config.clone(), background)?;
|
||||
|
||||
if config.admin.trace_sink.is_some() {
|
||||
info!("Initialize tracing...");
|
||||
if let Some(export_to) = config.admin.trace_sink {
|
||||
init_tracing(&export_to, garage.system.id)?;
|
||||
|
||||
#[cfg(feature = "telemetry-otlp")]
|
||||
init_tracing(config.admin.trace_sink.as_ref().unwrap(), garage.system.id)?;
|
||||
|
||||
#[cfg(not(feature = "telemetry-otlp"))]
|
||||
error!("Garage was built without OTLP exporter, admin.trace_sink is ignored.");
|
||||
}
|
||||
|
||||
info!("Initialize Admin API server and metrics collector...");
|
||||
|
|
@ -50,53 +58,78 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||
info!("Create admin RPC handler...");
|
||||
AdminRpcHandler::new(garage.clone());
|
||||
|
||||
info!("Initializing S3 API server...");
|
||||
let s3_api_server = tokio::spawn(S3ApiServer::run(
|
||||
garage.clone(),
|
||||
wait_from(watch_cancel.clone()),
|
||||
));
|
||||
// ---- Launch public-facing API servers ----
|
||||
|
||||
let mut servers = vec![];
|
||||
|
||||
if let Some(s3_bind_addr) = &config.s3_api.api_bind_addr {
|
||||
info!("Initializing S3 API server...");
|
||||
servers.push((
|
||||
"S3 API",
|
||||
tokio::spawn(S3ApiServer::run(
|
||||
garage.clone(),
|
||||
*s3_bind_addr,
|
||||
config.s3_api.s3_region.clone(),
|
||||
wait_from(watch_cancel.clone()),
|
||||
)),
|
||||
));
|
||||
}
|
||||
|
||||
if config.k2v_api.is_some() {
|
||||
#[cfg(feature = "k2v")]
|
||||
let k2v_api_server = {
|
||||
{
|
||||
info!("Initializing K2V API server...");
|
||||
servers.push((
|
||||
"K2V API",
|
||||
tokio::spawn(K2VApiServer::run(
|
||||
garage.clone(),
|
||||
config.k2v_api.as_ref().unwrap().api_bind_addr,
|
||||
config.s3_api.s3_region.clone(),
|
||||
wait_from(watch_cancel.clone()),
|
||||
))
|
||||
};
|
||||
|
||||
info!("Initializing web server...");
|
||||
let web_server = tokio::spawn(run_web_server(
|
||||
garage.clone(),
|
||||
wait_from(watch_cancel.clone()),
|
||||
)),
|
||||
));
|
||||
}
|
||||
#[cfg(not(feature = "k2v"))]
|
||||
error!("K2V is not enabled in this build, cannot start K2V API server");
|
||||
}
|
||||
|
||||
if let Some(web_config) = &config.s3_web {
|
||||
info!("Initializing web server...");
|
||||
servers.push((
|
||||
"Web",
|
||||
tokio::spawn(WebServer::run(
|
||||
garage.clone(),
|
||||
web_config.bind_addr,
|
||||
web_config.root_domain.clone(),
|
||||
wait_from(watch_cancel.clone()),
|
||||
)),
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(admin_bind_addr) = &config.admin.api_bind_addr {
|
||||
info!("Launching Admin API server...");
|
||||
let admin_server = tokio::spawn(admin_server.run(wait_from(watch_cancel.clone())));
|
||||
servers.push((
|
||||
"Admin",
|
||||
tokio::spawn(admin_server.run(*admin_bind_addr, wait_from(watch_cancel.clone()))),
|
||||
));
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "metrics"))]
|
||||
if config.admin.metrics_token.is_some() {
|
||||
warn!("This Garage version is built without the metrics feature");
|
||||
}
|
||||
|
||||
// Stuff runs
|
||||
|
||||
// When a cancel signal is sent, stuff stops
|
||||
if let Err(e) = s3_api_server.await? {
|
||||
warn!("S3 API server exited with error: {}", e);
|
||||
|
||||
// Collect stuff
|
||||
for (desc, join_handle) in servers {
|
||||
if let Err(e) = join_handle.await? {
|
||||
error!("{} server exited with error: {}", desc, e);
|
||||
} else {
|
||||
info!("S3 API server exited without error.");
|
||||
info!("{} server exited without error.", desc);
|
||||
}
|
||||
#[cfg(feature = "k2v")]
|
||||
if let Err(e) = k2v_api_server.await? {
|
||||
warn!("K2V API server exited with error: {}", e);
|
||||
} else {
|
||||
info!("K2V API server exited without error.");
|
||||
}
|
||||
if let Err(e) = web_server.await? {
|
||||
warn!("Web server exited with error: {}", e);
|
||||
} else {
|
||||
info!("Web server exited without error.");
|
||||
}
|
||||
if let Err(e) = admin_server.await? {
|
||||
warn!("Admin web server exited with error: {}", e);
|
||||
} else {
|
||||
info!("Admin API server exited without error.");
|
||||
}
|
||||
|
||||
// Remove RPC handlers for system to break reference cycles
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ mod common;
|
|||
mod admin;
|
||||
mod bucket;
|
||||
|
||||
mod s3;
|
||||
|
||||
#[cfg(feature = "k2v")]
|
||||
mod k2v;
|
||||
|
||||
mod s3;
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ tokio = "1.17.0"
|
|||
|
||||
# cli deps
|
||||
clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
|
||||
garage_util = { version = "0.7.0", path = "../util", optional = true }
|
||||
garage_util = { version = "0.8.0", path = "../util", optional = true }
|
||||
|
||||
|
||||
[features]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "garage_model"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
|
@ -15,11 +15,10 @@ path = "lib.rs"
|
|||
|
||||
[dependencies]
|
||||
garage_db = { version = "0.8.0", path = "../db" }
|
||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
||||
garage_table = { version = "0.7.0", path = "../table" }
|
||||
garage_block = { version = "0.7.0", path = "../block" }
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
garage_model_050 = { package = "garage_model", version = "0.5.1" }
|
||||
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||
garage_table = { version = "0.8.0", path = "../table" }
|
||||
garage_block = { version = "0.8.0", path = "../block" }
|
||||
garage_util = { version = "0.8.0", path = "../util" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
arc-swap = "1.0"
|
||||
|
|
@ -46,3 +45,6 @@ netapp = "0.4"
|
|||
|
||||
[features]
|
||||
k2v = [ "garage_util/k2v" ]
|
||||
lmdb = [ "garage_db/lmdb" ]
|
||||
sled = [ "garage_db/sled" ]
|
||||
sqlite = [ "garage_db/sqlite" ]
|
||||
|
|
|
|||
|
|
@ -80,6 +80,8 @@ impl Garage {
|
|||
let mut db_path = config.metadata_dir.clone();
|
||||
std::fs::create_dir_all(&db_path).expect("Unable to create Garage meta data directory");
|
||||
let db = match config.db_engine.as_str() {
|
||||
// ---- Sled DB ----
|
||||
#[cfg(feature = "sled")]
|
||||
"sled" => {
|
||||
db_path.push("db");
|
||||
info!("Opening Sled database at: {}", db_path.display());
|
||||
|
|
@ -91,6 +93,10 @@ impl Garage {
|
|||
.expect("Unable to open sled DB");
|
||||
db::sled_adapter::SledDb::init(db)
|
||||
}
|
||||
#[cfg(not(feature = "sled"))]
|
||||
"sled" => return Err(Error::Message("sled db not available in this build".into())),
|
||||
// ---- Sqlite DB ----
|
||||
#[cfg(feature = "sqlite")]
|
||||
"sqlite" | "sqlite3" | "rusqlite" => {
|
||||
db_path.push("db.sqlite");
|
||||
info!("Opening Sqlite database at: {}", db_path.display());
|
||||
|
|
@ -98,6 +104,14 @@ impl Garage {
|
|||
.expect("Unable to open sqlite DB");
|
||||
db::sqlite_adapter::SqliteDb::init(db)
|
||||
}
|
||||
#[cfg(not(feature = "sqlite"))]
|
||||
"sqlite" | "sqlite3" | "rusqlite" => {
|
||||
return Err(Error::Message(
|
||||
"sqlite db not available in this build".into(),
|
||||
))
|
||||
}
|
||||
// ---- LMDB DB ----
|
||||
#[cfg(feature = "lmdb")]
|
||||
"lmdb" | "heed" => {
|
||||
db_path.push("db.lmdb");
|
||||
info!("Opening LMDB database at: {}", db_path.display());
|
||||
|
|
@ -116,10 +130,22 @@ impl Garage {
|
|||
let db = env_builder.open(&db_path).expect("Unable to open LMDB DB");
|
||||
db::lmdb_adapter::LmdbDb::init(db)
|
||||
}
|
||||
#[cfg(not(feature = "lmdb"))]
|
||||
"lmdb" | "heed" => return Err(Error::Message("lmdb db not available in this build".into())),
|
||||
// ---- Unavailable DB engine ----
|
||||
e => {
|
||||
return Err(Error::Message(format!(
|
||||
"Unsupported DB engine: {} (options: sled, sqlite, lmdb)",
|
||||
e
|
||||
"Unsupported DB engine: {} (options: {})",
|
||||
e,
|
||||
vec![
|
||||
#[cfg(feature = "sled")]
|
||||
"sled",
|
||||
#[cfg(feature = "sqlite")]
|
||||
"sqlite",
|
||||
#[cfg(feature = "lmdb")]
|
||||
"lmdb",
|
||||
]
|
||||
.join(", ")
|
||||
)));
|
||||
}
|
||||
};
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use garage_util::data::*;
|
|||
|
||||
use crate::permission::BucketKeyPerm;
|
||||
|
||||
use garage_model_050::key_table as old;
|
||||
use crate::prev::v051::key_table as old;
|
||||
|
||||
/// An api key
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
#[macro_use]
|
||||
extern crate tracing;
|
||||
|
||||
// For migration from previous versions
|
||||
pub(crate) mod prev;
|
||||
|
||||
pub mod permission;
|
||||
|
||||
pub mod index_counter;
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use garage_util::data::*;
|
|||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::time::*;
|
||||
|
||||
use garage_model_050::bucket_table as old_bucket;
|
||||
use crate::prev::v051::bucket_table as old_bucket;
|
||||
|
||||
use crate::bucket_alias_table::*;
|
||||
use crate::bucket_table::*;
|
||||
|
|
|
|||
1
src/model/prev/mod.rs
Normal file
1
src/model/prev/mod.rs
Normal file
|
|
@ -0,0 +1 @@
|
|||
pub(crate) mod v051;
|
||||
63
src/model/prev/v051/bucket_table.rs
Normal file
63
src/model/prev/v051/bucket_table.rs
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_table::crdt::Crdt;
|
||||
use garage_table::*;
|
||||
|
||||
use super::key_table::PermissionSet;
|
||||
|
||||
/// A bucket is a collection of objects
|
||||
///
|
||||
/// Its parameters are not directly accessible as:
|
||||
/// - It must be possible to merge paramaters, hence the use of a LWW CRDT.
|
||||
/// - A bucket has 2 states, Present or Deleted and parameters make sense only if present.
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct Bucket {
|
||||
/// Name of the bucket
|
||||
pub name: String,
|
||||
/// State, and configuration if not deleted, of the bucket
|
||||
pub state: crdt::Lww<BucketState>,
|
||||
}
|
||||
|
||||
/// State of a bucket
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub enum BucketState {
|
||||
/// The bucket is deleted
|
||||
Deleted,
|
||||
/// The bucket exists
|
||||
Present(BucketParams),
|
||||
}
|
||||
|
||||
impl Crdt for BucketState {
|
||||
fn merge(&mut self, o: &Self) {
|
||||
match o {
|
||||
BucketState::Deleted => *self = BucketState::Deleted,
|
||||
BucketState::Present(other_params) => {
|
||||
if let BucketState::Present(params) = self {
|
||||
params.merge(other_params);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Configuration for a bucket
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct BucketParams {
|
||||
/// Map of key with access to the bucket, and what kind of access they give
|
||||
pub authorized_keys: crdt::LwwMap<String, PermissionSet>,
|
||||
/// Is the bucket served as http
|
||||
pub website: crdt::Lww<bool>,
|
||||
}
|
||||
|
||||
impl Crdt for BucketParams {
|
||||
fn merge(&mut self, o: &Self) {
|
||||
self.authorized_keys.merge(&o.authorized_keys);
|
||||
self.website.merge(&o.website);
|
||||
}
|
||||
}
|
||||
|
||||
impl Crdt for Bucket {
|
||||
fn merge(&mut self, other: &Self) {
|
||||
self.state.merge(&other.state);
|
||||
}
|
||||
}
|
||||
50
src/model/prev/v051/key_table.rs
Normal file
50
src/model/prev/v051/key_table.rs
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_table::crdt::*;
|
||||
use garage_table::*;
|
||||
|
||||
/// An api key
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct Key {
|
||||
/// The id of the key (immutable), used as partition key
|
||||
pub key_id: String,
|
||||
|
||||
/// The secret_key associated
|
||||
pub secret_key: String,
|
||||
|
||||
/// Name for the key
|
||||
pub name: crdt::Lww<String>,
|
||||
|
||||
/// Is the key deleted
|
||||
pub deleted: crdt::Bool,
|
||||
|
||||
/// Buckets in which the key is authorized. Empty if `Key` is deleted
|
||||
// CRDT interaction: deleted implies authorized_buckets is empty
|
||||
pub authorized_buckets: crdt::LwwMap<String, PermissionSet>,
|
||||
}
|
||||
|
||||
/// Permission given to a key in a bucket
|
||||
#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct PermissionSet {
|
||||
/// The key can be used to read the bucket
|
||||
pub allow_read: bool,
|
||||
/// The key can be used to write in the bucket
|
||||
pub allow_write: bool,
|
||||
}
|
||||
|
||||
impl AutoCrdt for PermissionSet {
|
||||
const WARN_IF_DIFFERENT: bool = true;
|
||||
}
|
||||
|
||||
impl Crdt for Key {
|
||||
fn merge(&mut self, other: &Self) {
|
||||
self.name.merge(&other.name);
|
||||
self.deleted.merge(&other.deleted);
|
||||
|
||||
if self.deleted.get() {
|
||||
self.authorized_buckets.clear();
|
||||
} else {
|
||||
self.authorized_buckets.merge(&other.authorized_buckets);
|
||||
}
|
||||
}
|
||||
}
|
||||
4
src/model/prev/v051/mod.rs
Normal file
4
src/model/prev/v051/mod.rs
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
pub(crate) mod bucket_table;
|
||||
pub(crate) mod key_table;
|
||||
pub(crate) mod object_table;
|
||||
pub(crate) mod version_table;
|
||||
149
src/model/prev/v051/object_table.rs
Normal file
149
src/model/prev/v051/object_table.rs
Normal file
|
|
@ -0,0 +1,149 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use garage_util::data::*;
|
||||
|
||||
use garage_table::crdt::*;
|
||||
|
||||
/// An object
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct Object {
|
||||
/// The bucket in which the object is stored, used as partition key
|
||||
pub bucket: String,
|
||||
|
||||
/// The key at which the object is stored in its bucket, used as sorting key
|
||||
pub key: String,
|
||||
|
||||
/// The list of currenty stored versions of the object
|
||||
versions: Vec<ObjectVersion>,
|
||||
}
|
||||
|
||||
impl Object {
|
||||
/// Get a list of currently stored versions of `Object`
|
||||
pub fn versions(&self) -> &[ObjectVersion] {
|
||||
&self.versions[..]
|
||||
}
|
||||
}
|
||||
|
||||
/// Informations about a version of an object
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ObjectVersion {
|
||||
/// Id of the version
|
||||
pub uuid: Uuid,
|
||||
/// Timestamp of when the object was created
|
||||
pub timestamp: u64,
|
||||
/// State of the version
|
||||
pub state: ObjectVersionState,
|
||||
}
|
||||
|
||||
/// State of an object version
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub enum ObjectVersionState {
|
||||
/// The version is being received
|
||||
Uploading(ObjectVersionHeaders),
|
||||
/// The version is fully received
|
||||
Complete(ObjectVersionData),
|
||||
/// The version uploaded containded errors or the upload was explicitly aborted
|
||||
Aborted,
|
||||
}
|
||||
|
||||
impl Crdt for ObjectVersionState {
|
||||
fn merge(&mut self, other: &Self) {
|
||||
use ObjectVersionState::*;
|
||||
match other {
|
||||
Aborted => {
|
||||
*self = Aborted;
|
||||
}
|
||||
Complete(b) => match self {
|
||||
Aborted => {}
|
||||
Complete(a) => {
|
||||
a.merge(b);
|
||||
}
|
||||
Uploading(_) => {
|
||||
*self = Complete(b.clone());
|
||||
}
|
||||
},
|
||||
Uploading(_) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Data stored in object version
|
||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||
pub enum ObjectVersionData {
|
||||
/// The object was deleted, this Version is a tombstone to mark it as such
|
||||
DeleteMarker,
|
||||
/// The object is short, it's stored inlined
|
||||
Inline(ObjectVersionMeta, #[serde(with = "serde_bytes")] Vec<u8>),
|
||||
/// The object is not short, Hash of first block is stored here, next segments hashes are
|
||||
/// stored in the version table
|
||||
FirstBlock(ObjectVersionMeta, Hash),
|
||||
}
|
||||
|
||||
impl AutoCrdt for ObjectVersionData {
|
||||
const WARN_IF_DIFFERENT: bool = true;
|
||||
}
|
||||
|
||||
/// Metadata about the object version
|
||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ObjectVersionMeta {
|
||||
/// Headers to send to the client
|
||||
pub headers: ObjectVersionHeaders,
|
||||
/// Size of the object
|
||||
pub size: u64,
|
||||
/// etag of the object
|
||||
pub etag: String,
|
||||
}
|
||||
|
||||
/// Additional headers for an object
|
||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ObjectVersionHeaders {
|
||||
/// Content type of the object
|
||||
pub content_type: String,
|
||||
/// Any other http headers to send
|
||||
pub other: BTreeMap<String, String>,
|
||||
}
|
||||
|
||||
impl ObjectVersion {
|
||||
fn cmp_key(&self) -> (u64, Uuid) {
|
||||
(self.timestamp, self.uuid)
|
||||
}
|
||||
|
||||
/// Is the object version completely received
|
||||
pub fn is_complete(&self) -> bool {
|
||||
matches!(self.state, ObjectVersionState::Complete(_))
|
||||
}
|
||||
}
|
||||
|
||||
impl Crdt for Object {
|
||||
fn merge(&mut self, other: &Self) {
|
||||
// Merge versions from other into here
|
||||
for other_v in other.versions.iter() {
|
||||
match self
|
||||
.versions
|
||||
.binary_search_by(|v| v.cmp_key().cmp(&other_v.cmp_key()))
|
||||
{
|
||||
Ok(i) => {
|
||||
self.versions[i].state.merge(&other_v.state);
|
||||
}
|
||||
Err(i) => {
|
||||
self.versions.insert(i, other_v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remove versions which are obsolete, i.e. those that come
|
||||
// before the last version which .is_complete().
|
||||
let last_complete = self
|
||||
.versions
|
||||
.iter()
|
||||
.enumerate()
|
||||
.rev()
|
||||
.find(|(_, v)| v.is_complete())
|
||||
.map(|(vi, _)| vi);
|
||||
|
||||
if let Some(last_vi) = last_complete {
|
||||
self.versions = self.versions.drain(last_vi..).collect::<Vec<_>>();
|
||||
}
|
||||
}
|
||||
}
|
||||
79
src/model/prev/v051/version_table.rs
Normal file
79
src/model/prev/v051/version_table.rs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use garage_util::data::*;
|
||||
|
||||
use garage_table::crdt::*;
|
||||
use garage_table::*;
|
||||
|
||||
/// A version of an object
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct Version {
|
||||
/// UUID of the version, used as partition key
|
||||
pub uuid: Uuid,
|
||||
|
||||
// Actual data: the blocks for this version
|
||||
// In the case of a multipart upload, also store the etags
|
||||
// of individual parts and check them when doing CompleteMultipartUpload
|
||||
/// Is this version deleted
|
||||
pub deleted: crdt::Bool,
|
||||
/// list of blocks of data composing the version
|
||||
pub blocks: crdt::Map<VersionBlockKey, VersionBlock>,
|
||||
/// Etag of each part in case of a multipart upload, empty otherwise
|
||||
pub parts_etags: crdt::Map<u64, String>,
|
||||
|
||||
// Back link to bucket+key so that we can figure if
|
||||
// this was deleted later on
|
||||
/// Bucket in which the related object is stored
|
||||
pub bucket: String,
|
||||
/// Key in which the related object is stored
|
||||
pub key: String,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||
pub struct VersionBlockKey {
|
||||
/// Number of the part
|
||||
pub part_number: u64,
|
||||
/// Offset of this sub-segment in its part
|
||||
pub offset: u64,
|
||||
}
|
||||
|
||||
impl Ord for VersionBlockKey {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
self.part_number
|
||||
.cmp(&other.part_number)
|
||||
.then(self.offset.cmp(&other.offset))
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for VersionBlockKey {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
/// Informations about a single block
|
||||
#[derive(PartialEq, Eq, Ord, PartialOrd, Clone, Copy, Debug, Serialize, Deserialize)]
|
||||
pub struct VersionBlock {
|
||||
/// Blake2 sum of the block
|
||||
pub hash: Hash,
|
||||
/// Size of the block
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
impl AutoCrdt for VersionBlock {
|
||||
const WARN_IF_DIFFERENT: bool = true;
|
||||
}
|
||||
|
||||
impl Crdt for Version {
|
||||
fn merge(&mut self, other: &Self) {
|
||||
self.deleted.merge(&other.deleted);
|
||||
|
||||
if self.deleted.get() {
|
||||
self.blocks.clear();
|
||||
self.parts_etags.clear();
|
||||
} else {
|
||||
self.blocks.merge(&other.blocks);
|
||||
self.parts_etags.merge(&other.parts_etags);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -14,7 +14,7 @@ use garage_table::*;
|
|||
use crate::index_counter::*;
|
||||
use crate::s3::version_table::*;
|
||||
|
||||
use garage_model_050::object_table as old;
|
||||
use crate::prev::v051::object_table as old;
|
||||
|
||||
pub const OBJECTS: &str = "objects";
|
||||
pub const UNFINISHED_UPLOADS: &str = "unfinished_uploads";
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ use garage_table::*;
|
|||
|
||||
use crate::s3::block_ref_table::*;
|
||||
|
||||
use garage_model_050::version_table as old;
|
||||
use crate::prev::v051::version_table as old;
|
||||
|
||||
/// A version of an object
|
||||
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "garage_rpc"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
|
@ -14,7 +14,7 @@ path = "lib.rs"
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
garage_util = { version = "0.8.0", path = "../util" }
|
||||
|
||||
arc-swap = "1.0"
|
||||
bytes = "1.0"
|
||||
|
|
@ -54,3 +54,4 @@ hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] }
|
|||
|
||||
[features]
|
||||
kubernetes-discovery = [ "kube", "k8s-openapi", "openssl", "schemars" ]
|
||||
system-libs = [ "sodiumoxide/use-pkg-config" ]
|
||||
|
|
|
|||
|
|
@ -27,7 +27,6 @@ use garage_util::data::*;
|
|||
use garage_util::error::*;
|
||||
use garage_util::persister::Persister;
|
||||
use garage_util::time::*;
|
||||
use garage_util::version;
|
||||
|
||||
use crate::consul::*;
|
||||
#[cfg(feature = "kubernetes-discovery")]
|
||||
|
|
@ -40,8 +39,10 @@ const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60);
|
|||
const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10);
|
||||
const PING_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Version tag used for version check upon Netapp connection
|
||||
pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650007; // garage 0x0007
|
||||
/// Version tag used for version check upon Netapp connection.
|
||||
/// Cluster nodes with different version tags are deemed
|
||||
/// incompatible and will refuse to connect.
|
||||
pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650008; // garage 0x0008
|
||||
|
||||
/// RPC endpoint used for calls related to membership
|
||||
pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc";
|
||||
|
|
@ -320,10 +321,6 @@ impl System {
|
|||
// ---- Administrative operations (directly available and
|
||||
// also available through RPC) ----
|
||||
|
||||
pub fn garage_version(&self) -> &'static str {
|
||||
version::garage()
|
||||
}
|
||||
|
||||
pub fn get_known_nodes(&self) -> Vec<KnownNodeInfo> {
|
||||
let node_status = self.node_status.read().unwrap();
|
||||
let known_nodes = self
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "garage_table"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
|
@ -15,8 +15,8 @@ path = "lib.rs"
|
|||
|
||||
[dependencies]
|
||||
garage_db = { version = "0.8.0", path = "../db" }
|
||||
garage_rpc = { version = "0.7.0", path = "../rpc" }
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
garage_rpc = { version = "0.8.0", path = "../rpc" }
|
||||
garage_util = { version = "0.8.0", path = "../util" }
|
||||
|
||||
opentelemetry = "0.17"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "garage_util"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
|
@ -16,17 +16,19 @@ path = "lib.rs"
|
|||
[dependencies]
|
||||
garage_db = { version = "0.8.0", path = "../db" }
|
||||
|
||||
arc-swap = "1.0"
|
||||
async-trait = "0.1"
|
||||
blake2 = "0.9"
|
||||
bytes = "1.0"
|
||||
digest = "0.10"
|
||||
err-derive = "0.3"
|
||||
git-version = "0.3.4"
|
||||
xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
|
||||
hex = "0.4"
|
||||
lazy_static = "1.4"
|
||||
tracing = "0.1.30"
|
||||
rand = "0.8"
|
||||
sha2 = "0.10"
|
||||
git-version = "0.3.4"
|
||||
|
||||
chrono = "0.4"
|
||||
rmp-serde = "0.15"
|
||||
|
|
|
|||
|
|
@ -77,11 +77,10 @@ pub struct Config {
|
|||
pub s3_api: S3ApiConfig,
|
||||
|
||||
/// Configuration for K2V api
|
||||
#[cfg(feature = "k2v")]
|
||||
pub k2v_api: Option<K2VApiConfig>,
|
||||
|
||||
/// Configuration for serving files as normal web server
|
||||
pub s3_web: WebConfig,
|
||||
pub s3_web: Option<WebConfig>,
|
||||
|
||||
/// Configuration for the admin API endpoint
|
||||
#[serde(default = "Default::default")]
|
||||
|
|
@ -92,7 +91,7 @@ pub struct Config {
|
|||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct S3ApiConfig {
|
||||
/// Address and port to bind for api serving
|
||||
pub api_bind_addr: SocketAddr,
|
||||
pub api_bind_addr: Option<SocketAddr>,
|
||||
/// S3 region to use
|
||||
pub s3_region: String,
|
||||
/// Suffix to remove from domain name to find bucket. If None,
|
||||
|
|
@ -101,7 +100,6 @@ pub struct S3ApiConfig {
|
|||
}
|
||||
|
||||
/// Configuration for K2V api
|
||||
#[cfg(feature = "k2v")]
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct K2VApiConfig {
|
||||
/// Address and port to bind for api serving
|
||||
|
|
|
|||
|
|
@ -1,7 +1,28 @@
|
|||
pub fn garage() -> &'static str {
|
||||
option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::{ArcSwap, ArcSwapOption};
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref VERSION: ArcSwap<&'static str> = ArcSwap::new(Arc::new(git_version::git_version!(
|
||||
prefix = "git:",
|
||||
cargo_prefix = "cargo:",
|
||||
fallback = "unknown"
|
||||
))
|
||||
)));
|
||||
static ref FEATURES: ArcSwapOption<&'static [&'static str]> = ArcSwapOption::new(None);
|
||||
}
|
||||
|
||||
pub fn garage_version() -> &'static str {
|
||||
&VERSION.load()
|
||||
}
|
||||
|
||||
pub fn garage_features() -> Option<&'static [&'static str]> {
|
||||
FEATURES.load().as_ref().map(|f| &f[..])
|
||||
}
|
||||
|
||||
pub fn init_version(version: &'static str) {
|
||||
VERSION.store(Arc::new(version));
|
||||
}
|
||||
|
||||
pub fn init_features(features: &'static [&'static str]) {
|
||||
FEATURES.store(Some(Arc::new(features)));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "garage_web"
|
||||
version = "0.7.0"
|
||||
version = "0.8.0"
|
||||
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
|
|
@ -14,10 +14,10 @@ path = "lib.rs"
|
|||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_api = { version = "0.7.0", path = "../api" }
|
||||
garage_model = { version = "0.7.0", path = "../model" }
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
garage_table = { version = "0.7.0", path = "../table" }
|
||||
garage_api = { version = "0.8.0", path = "../api" }
|
||||
garage_model = { version = "0.8.0", path = "../model" }
|
||||
garage_util = { version = "0.8.0", path = "../util" }
|
||||
garage_table = { version = "0.8.0", path = "../table" }
|
||||
|
||||
err-derive = "0.3"
|
||||
tracing = "0.1.30"
|
||||
|
|
|
|||
|
|
@ -6,4 +6,4 @@ mod error;
|
|||
pub use error::Error;
|
||||
|
||||
mod web_server;
|
||||
pub use web_server::run_web_server;
|
||||
pub use web_server::WebServer;
|
||||
|
|
|
|||
|
|
@ -57,44 +57,53 @@ impl WebMetrics {
|
|||
}
|
||||
}
|
||||
|
||||
/// Run a web server
|
||||
pub async fn run_web_server(
|
||||
pub struct WebServer {
|
||||
garage: Arc<Garage>,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
let addr = &garage.config.s3_web.bind_addr;
|
||||
metrics: Arc<WebMetrics>,
|
||||
root_domain: String,
|
||||
}
|
||||
|
||||
impl WebServer {
|
||||
/// Run a web server
|
||||
pub async fn run(
|
||||
garage: Arc<Garage>,
|
||||
addr: SocketAddr,
|
||||
root_domain: String,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
let metrics = Arc::new(WebMetrics::new());
|
||||
let web_server = Arc::new(WebServer {
|
||||
garage,
|
||||
metrics,
|
||||
root_domain,
|
||||
});
|
||||
|
||||
let service = make_service_fn(|conn: &AddrStream| {
|
||||
let garage = garage.clone();
|
||||
let metrics = metrics.clone();
|
||||
let web_server = web_server.clone();
|
||||
|
||||
let client_addr = conn.remote_addr();
|
||||
async move {
|
||||
Ok::<_, Error>(service_fn(move |req: Request<Body>| {
|
||||
let garage = garage.clone();
|
||||
let metrics = metrics.clone();
|
||||
let web_server = web_server.clone();
|
||||
|
||||
handle_request(garage, metrics, req, client_addr)
|
||||
web_server.handle_request(req, client_addr)
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
let server = Server::bind(addr).serve(service);
|
||||
let server = Server::bind(&addr).serve(service);
|
||||
let graceful = server.with_graceful_shutdown(shutdown_signal);
|
||||
info!("Web server listening on http://{}", addr);
|
||||
|
||||
graceful.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_request(
|
||||
garage: Arc<Garage>,
|
||||
metrics: Arc<WebMetrics>,
|
||||
async fn handle_request(
|
||||
self: Arc<Self>,
|
||||
req: Request<Body>,
|
||||
addr: SocketAddr,
|
||||
) -> Result<Response<Body>, Infallible> {
|
||||
) -> Result<Response<Body>, Infallible> {
|
||||
info!("{} {} {}", addr, req.method(), req.uri());
|
||||
|
||||
// Lots of instrumentation
|
||||
|
|
@ -111,13 +120,14 @@ async fn handle_request(
|
|||
let metrics_tags = &[KeyValue::new("method", req.method().to_string())];
|
||||
|
||||
// The actual handler
|
||||
let res = serve_file(garage, &req)
|
||||
let res = self
|
||||
.serve_file(&req)
|
||||
.with_context(Context::current_with_span(span))
|
||||
.record_duration(&metrics.request_duration, &metrics_tags[..])
|
||||
.record_duration(&self.metrics.request_duration, &metrics_tags[..])
|
||||
.await;
|
||||
|
||||
// More instrumentation
|
||||
metrics.request_counter.add(1, &metrics_tags[..]);
|
||||
self.metrics.request_counter.add(1, &metrics_tags[..]);
|
||||
|
||||
// Returning the result
|
||||
match res {
|
||||
|
|
@ -133,7 +143,7 @@ async fn handle_request(
|
|||
req.uri(),
|
||||
error
|
||||
);
|
||||
metrics.error_counter.add(
|
||||
self.metrics.error_counter.add(
|
||||
1,
|
||||
&[
|
||||
metrics_tags[0].clone(),
|
||||
|
|
@ -143,24 +153,9 @@ async fn handle_request(
|
|||
Ok(error_to_res(error))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn error_to_res(e: Error) -> Response<Body> {
|
||||
// If we are here, it is either that:
|
||||
// - there was an error before trying to get the requested URL
|
||||
// from the bucket (e.g. bucket not found)
|
||||
// - there was an error processing the request and (the request
|
||||
// was a HEAD request or we couldn't get the error document)
|
||||
// We do NOT enter this code path when returning the bucket's
|
||||
// error document (this is handled in serve_file)
|
||||
let body = Body::from(format!("{}\n", e));
|
||||
let mut http_error = Response::new(body);
|
||||
*http_error.status_mut() = e.http_status_code();
|
||||
e.add_headers(http_error.headers_mut());
|
||||
http_error
|
||||
}
|
||||
|
||||
async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response<Body>, Error> {
|
||||
async fn serve_file(self: &Arc<Self>, req: &Request<Body>) -> Result<Response<Body>, Error> {
|
||||
// Get http authority string (eg. [::1]:3902 or garage.tld:80)
|
||||
let authority = req
|
||||
.headers()
|
||||
|
|
@ -170,10 +165,10 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
|
|||
|
||||
// Get bucket
|
||||
let host = authority_to_host(authority)?;
|
||||
let root = &garage.config.s3_web.root_domain;
|
||||
|
||||
let bucket_name = host_to_bucket(&host, root).unwrap_or(&host);
|
||||
let bucket_id = garage
|
||||
let bucket_name = host_to_bucket(&host, &self.root_domain).unwrap_or(&host);
|
||||
let bucket_id = self
|
||||
.garage
|
||||
.bucket_alias_table
|
||||
.get(&EmptyKey, &bucket_name.to_string())
|
||||
.await?
|
||||
|
|
@ -181,7 +176,8 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
|
|||
.ok_or(Error::NotFound)?;
|
||||
|
||||
// Check bucket isn't deleted and has website access enabled
|
||||
let bucket = garage
|
||||
let bucket = self
|
||||
.garage
|
||||
.bucket_table
|
||||
.get(&EmptyKey, &bucket_id)
|
||||
.await?
|
||||
|
|
@ -207,8 +203,8 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
|
|||
|
||||
let ret_doc = match *req.method() {
|
||||
Method::OPTIONS => handle_options_for_bucket(req, &bucket),
|
||||
Method::HEAD => handle_head(garage.clone(), req, bucket_id, &key, None).await,
|
||||
Method::GET => handle_get(garage.clone(), req, bucket_id, &key, None).await,
|
||||
Method::HEAD => handle_head(self.garage.clone(), req, bucket_id, &key, None).await,
|
||||
Method::GET => handle_get(self.garage.clone(), req, bucket_id, &key, None).await,
|
||||
_ => Err(ApiError::bad_request("HTTP method not supported")),
|
||||
}
|
||||
.map_err(Error::from);
|
||||
|
|
@ -239,7 +235,8 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
|
|||
.body(Body::empty())
|
||||
.unwrap();
|
||||
|
||||
match handle_get(garage, &req2, bucket_id, &error_document, None).await {
|
||||
match handle_get(self.garage.clone(), &req2, bucket_id, &error_document, None).await
|
||||
{
|
||||
Ok(mut error_doc) => {
|
||||
// The error won't be logged back in handle_request,
|
||||
// so log it here
|
||||
|
|
@ -281,6 +278,22 @@ async fn serve_file(garage: Arc<Garage>, req: &Request<Body>) -> Result<Response
|
|||
Ok(resp)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn error_to_res(e: Error) -> Response<Body> {
|
||||
// If we are here, it is either that:
|
||||
// - there was an error before trying to get the requested URL
|
||||
// from the bucket (e.g. bucket not found)
|
||||
// - there was an error processing the request and (the request
|
||||
// was a HEAD request or we couldn't get the error document)
|
||||
// We do NOT enter this code path when returning the bucket's
|
||||
// error document (this is handled in serve_file)
|
||||
let body = Body::from(format!("{}\n", e));
|
||||
let mut http_error = Response::new(body);
|
||||
*http_error.status_mut() = e.http_status_code();
|
||||
e.add_headers(http_error.headers_mut());
|
||||
http_error
|
||||
}
|
||||
|
||||
/// Path to key
|
||||
|
|
|
|||
Loading…
Reference in a new issue