Compare commits

..

8 commits

Author SHA1 Message Date
84613e66a2
garage(tests): Remove RNG stuff
All checks were successful
continuous-integration/drone/pr Build is passing
continuous-integration/drone/push Build is passing
2022-02-11 10:50:55 +01:00
c8b30ebc79
garage(tests): Remove superfluous test 2022-02-10 17:55:51 +01:00
d7decda3f4
garage(tests): Add random suffix to created buckets. 2022-02-10 17:55:50 +01:00
cd13ea461b
garage(tests): Add some unsafe-usage doc and tweaks 2022-02-10 17:55:50 +01:00
5d19f3d2d7
Add integration tests to Drone 2022-02-10 17:55:50 +01:00
084dcdbd3a
Upgrade cargo2nix 2022-02-10 17:55:50 +01:00
3baa841d6f
tests: Fix garage integration test 2022-02-10 17:55:49 +01:00
dd407e7041
tests: Add garage integration tests (base) 2022-02-10 17:55:49 +01:00
15 changed files with 2000 additions and 82 deletions

View file

@ -46,10 +46,12 @@ steps:
- name: nix_config
path: /etc/nix
commands:
- nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
- nix-build --no-build-output --option log-lines 100 --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
- name: unit tests
- name: unit + func tests
image: nixpkgs/nix:nixos-21.05
environment:
GARAGE_TEST_INTEGRATION_EXE: result/bin/garage
volumes:
- name: nix_store
path: /nix
@ -59,15 +61,17 @@ steps:
- |
nix-build \
--no-build-output \
--option log-lines 100 \
--argstr target x86_64-unknown-linux-musl \
--argstr compileMode test
- ./result*/bin/garage_api*
- ./result*/bin/garage_model*
- ./result*/bin/garage_rpc*
- ./result*/bin/garage_table*
- ./result*/bin/garage_util*
- ./result*/bin/garage_web*
- ./result*/bin/garage*
- ./result/bin/garage_api-*
- ./result/bin/garage_model-*
- ./result/bin/garage_rpc-*
- ./result/bin/garage_table-*
- ./result/bin/garage_util-*
- ./result/bin/garage_web-*
- ./result/bin/garage-*
- ./result/bin/integration-*
- name: smoke-test
image: nixpkgs/nix:nixos-21.05
@ -469,6 +473,6 @@ node:
---
kind: signature
hmac: 928ea1bb59f3ac19b5ddd2a184f17b7c728cc355877c34e61b3d1b421544d4c3
hmac: 3fc19d6f9a3555519c8405e3281b2e74289bb802f644740d5481d53df3a01fa4
...

656
Cargo.lock generated
View file

@ -51,6 +51,212 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "aws-endpoint"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06d059b181b25940b751e8efecc173ceb4fe65f45d8975f56b02e98db5c42fd6"
dependencies = [
"aws-smithy-http",
"aws-types",
"http",
"regex",
"tracing",
]
[[package]]
name = "aws-http"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3049066e3282c98bbf01e90459a1772ccf6c0b96cd1483c3dd5aa34bef9b9de1"
dependencies = [
"aws-smithy-http",
"aws-smithy-types",
"aws-types",
"http",
"lazy_static",
"percent-encoding",
"tracing",
]
[[package]]
name = "aws-sdk-s3"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d70be50ac07c3c2b5f37056271856ac00190e80c19c76c58bcbee5be0b63ec9"
dependencies = [
"aws-endpoint",
"aws-http",
"aws-sig-auth",
"aws-sigv4",
"aws-smithy-async",
"aws-smithy-client",
"aws-smithy-eventstream",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-types",
"aws-smithy-xml",
"aws-types",
"bytes 1.1.0",
"http",
"md5",
"tokio-stream",
"tower",
]
[[package]]
name = "aws-sig-auth"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4012b5192350b5403aba19a01a5a3b1768158dab936c4269d89760970d4812bc"
dependencies = [
"aws-sigv4",
"aws-smithy-eventstream",
"aws-smithy-http",
"aws-types",
"http",
"thiserror",
"tracing",
]
[[package]]
name = "aws-sigv4"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41f4b9c0c3a34e5152a0cd5e43b8f2cfd780e3bd7a245948d8787e051095ac4c"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-http",
"bytes 1.1.0",
"form_urlencoded",
"hex",
"http",
"once_cell",
"percent-encoding",
"regex",
"ring",
"time 0.3.7",
"tracing",
]
[[package]]
name = "aws-smithy-async"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b69dad0aefb1b64e63e0d3a1310dc50191608d8c9e226f2f241f344a7173642e"
dependencies = [
"futures-util",
"pin-project-lite",
"tokio",
"tokio-stream",
]
[[package]]
name = "aws-smithy-client"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93e47a8aca2194672518d6630936507d3b54598c482f13ffe53f9b7932724bbb"
dependencies = [
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-http-tower",
"aws-smithy-types",
"bytes 1.1.0",
"fastrand",
"http",
"http-body",
"hyper",
"hyper-rustls",
"lazy_static",
"pin-project",
"pin-project-lite",
"tokio",
"tower",
"tracing",
]
[[package]]
name = "aws-smithy-eventstream"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f98bcfcb063d29c7cc7bb0a64830afe606090de75533c10a11a05460d814e8d9"
dependencies = [
"aws-smithy-types",
"bytes 1.1.0",
"crc32fast",
]
[[package]]
name = "aws-smithy-http"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c8bbe92ecdc4e39a612359b09994c45d000591d4951aa7343443f44b47e6696"
dependencies = [
"aws-smithy-eventstream",
"aws-smithy-types",
"bytes 1.1.0",
"bytes-utils",
"futures-core",
"http",
"http-body",
"hyper",
"percent-encoding",
"pin-project",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "aws-smithy-http-tower"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f23fdf1253855af3bb4abb25e42ad3152a71241af89014eebf27c14c7a59b81d"
dependencies = [
"aws-smithy-http",
"bytes 1.1.0",
"http",
"http-body",
"pin-project",
"tower",
"tracing",
]
[[package]]
name = "aws-smithy-types"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde96306a54777ec8781aa510830e242de614aa5746274713f5ecac0779f644f"
dependencies = [
"itoa 1.0.1",
"num-integer",
"ryu",
"time 0.3.7",
]
[[package]]
name = "aws-smithy-xml"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3b0466594a86074a6e96b11284f9a9ddc90c5c5b7d6144ab357a90be49d28c4"
dependencies = [
"thiserror",
"xmlparser",
]
[[package]]
name = "aws-types"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "433fd128ea727e9b83b34c72c6d4db1b900f067760fa27b387694fe896633142"
dependencies = [
"aws-smithy-async",
"aws-smithy-types",
"rustc_version",
"tracing",
"zeroize",
]
[[package]]
name = "base64"
version = "0.13.0"
@ -83,6 +289,12 @@ dependencies = [
"generic-array",
]
[[package]]
name = "bumpalo"
version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
[[package]]
name = "byteorder"
version = "1.4.3"
@ -101,6 +313,16 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
[[package]]
name = "bytes-utils"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e314712951c43123e5920a446464929adc667a5eade7f8fb3997776c9df6e54"
dependencies = [
"bytes 1.1.0",
"either",
]
[[package]]
name = "cc"
version = "1.0.71"
@ -116,6 +338,12 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "cfg_aliases"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e"
[[package]]
name = "chrono"
version = "0.4.19"
@ -125,7 +353,7 @@ dependencies = [
"libc",
"num-integer",
"num-traits",
"time",
"time 0.1.43",
"winapi",
]
@ -140,6 +368,22 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "core-foundation"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6888e10551bb93e424d8df1d07f1a8b4fceb0001a3a4b048bfc47554946f47b3"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "cpufeatures"
version = "0.2.1"
@ -201,6 +445,15 @@ dependencies = [
"subtle",
]
[[package]]
name = "ct-logs"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1a816186fa68d9e426e3cb4ae4dff1fcd8e4a2c34b781bf7a822574a0d0aac8"
dependencies = [
"sct",
]
[[package]]
name = "digest"
version = "0.9.0"
@ -210,6 +463,12 @@ dependencies = [
"generic-array",
]
[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
[[package]]
name = "env_logger"
version = "0.7.1"
@ -251,6 +510,15 @@ dependencies = [
"synstructure",
]
[[package]]
name = "fastrand"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3fcf0cee53519c866c09b5de1f6c56ff9d647101f81c1964fa632e148896cdf"
dependencies = [
"instant",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -385,6 +653,7 @@ name = "garage"
version = "0.6.0"
dependencies = [
"async-trait",
"aws-sdk-s3",
"bytes 1.1.0",
"futures",
"futures-util",
@ -396,6 +665,7 @@ dependencies = [
"garage_web",
"git-version",
"hex",
"http",
"kuska-sodiumoxide",
"log",
"netapp",
@ -405,6 +675,7 @@ dependencies = [
"serde",
"serde_bytes",
"sled",
"static_init",
"structopt",
"tokio",
"toml",
@ -708,6 +979,31 @@ dependencies = [
"syn",
]
[[package]]
name = "h2"
version = "0.3.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9f1f717ddc7b2ba36df7e871fd88db79326551d3d6f1fc406fbfd28b582ff8e"
dependencies = [
"bytes 1.1.0",
"fnv",
"futures-core",
"futures-sink",
"futures-util",
"http",
"indexmap",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "hashbrown"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
[[package]]
name = "heck"
version = "0.3.3"
@ -760,14 +1056,14 @@ checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b"
dependencies = [
"bytes 1.1.0",
"fnv",
"itoa",
"itoa 0.4.8",
]
[[package]]
name = "http-body"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5"
checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
dependencies = [
"bytes 1.1.0",
"http",
@ -817,11 +1113,12 @@ dependencies = [
"futures-channel",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"httparse",
"httpdate 1.0.1",
"itoa",
"itoa 0.4.8",
"pin-project-lite",
"socket2",
"tokio",
@ -830,6 +1127,23 @@ dependencies = [
"want",
]
[[package]]
name = "hyper-rustls"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
dependencies = [
"ct-logs",
"futures-util",
"hyper",
"log",
"rustls",
"rustls-native-certs",
"tokio",
"tokio-rustls",
"webpki",
]
[[package]]
name = "idna"
version = "0.2.3"
@ -841,6 +1155,16 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "indexmap"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
dependencies = [
"autocfg",
"hashbrown",
]
[[package]]
name = "instant"
version = "0.1.11"
@ -862,6 +1186,12 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
[[package]]
name = "itoa"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
[[package]]
name = "jobserver"
version = "0.1.24"
@ -871,6 +1201,15 @@ dependencies = [
"libc",
]
[[package]]
name = "js-sys"
version = "0.3.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04"
dependencies = [
"wasm-bindgen",
]
[[package]]
name = "kuska-handshake"
version = "0.2.0"
@ -903,9 +1242,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "libc"
version = "0.2.103"
version = "0.2.115"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8f7255a17a627354f321ef0055d63b898c6fb27eff628af4d1b66b7331edf6"
checksum = "0a8d982fa7a96a000f6ec4cfe966de9703eccde29750df2bb8949da91b0e818d"
[[package]]
name = "libsodium-sys"
@ -954,6 +1293,12 @@ dependencies = [
"opaque-debug",
]
[[package]]
name = "md5"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
[[package]]
name = "memchr"
version = "2.4.1"
@ -1068,6 +1413,15 @@ dependencies = [
"libc",
]
[[package]]
name = "num_threads"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97ba99ba6393e2c3734791401b66902d981cb03bf190af674ca69949b6d5fb15"
dependencies = [
"libc",
]
[[package]]
name = "once_cell"
version = "1.8.0"
@ -1080,6 +1434,12 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openssl-probe"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]]
name = "parking_lot"
version = "0.11.2"
@ -1301,6 +1661,21 @@ version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "ring"
version = "0.16.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc"
dependencies = [
"cc",
"libc",
"once_cell",
"spin",
"untrusted",
"web-sys",
"winapi",
]
[[package]]
name = "rmp"
version = "0.8.10"
@ -1342,6 +1717,40 @@ dependencies = [
"xmlparser",
]
[[package]]
name = "rustc_version"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
dependencies = [
"semver",
]
[[package]]
name = "rustls"
version = "0.19.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35edb675feee39aec9c99fa5ff985081995a06d594114ae14cbe797ad7b7a6d7"
dependencies = [
"base64",
"log",
"ring",
"sct",
"webpki",
]
[[package]]
name = "rustls-native-certs"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a07b7c1885bd8ed3831c289b7870b13ef46fe0e856d288c30d9cc17d75a2092"
dependencies = [
"openssl-probe",
"rustls",
"schannel",
"security-framework",
]
[[package]]
name = "rustversion"
version = "1.0.5"
@ -1363,12 +1772,61 @@ dependencies = [
"winapi-util",
]
[[package]]
name = "schannel"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
dependencies = [
"lazy_static",
"winapi",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sct"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b362b83898e0e69f38515b82ee15aa80636befe47c3b6d3d89a911e78fc228ce"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "security-framework"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d09d3c15d814eda1d6a836f2f2b56a6abc1446c8a34351cb3180d3db92ffe4ce"
dependencies = [
"bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e90dd10c41c6bfc633da6e0c659bd25d31e0791e5974ac42970267d59eba87f7"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "semver"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "568a8e6258aa33c13358f81fd834adb854c6f7c9468520910a9b1e8fac068012"
[[package]]
name = "serde"
version = "1.0.130"
@ -1404,7 +1862,7 @@ version = "1.0.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f690853975602e1bfe1ccbf50504d67174e3bcf340f23b5ea9992e0587a52d8"
dependencies = [
"itoa",
"itoa 0.4.8",
"ryu",
"serde",
]
@ -1469,6 +1927,40 @@ dependencies = [
"winapi",
]
[[package]]
name = "spin"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "static_init"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "208e44bfab7faad5dee24112ea8af2f76aa0d501ea3370b5d4b81729a528f119"
dependencies = [
"bitflags",
"cfg_aliases",
"libc",
"parking_lot",
"parking_lot_core",
"static_init_macro",
"winapi",
]
[[package]]
name = "static_init_macro"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70a2595fc3aa78f2d0e45dd425b22282dd863273761cc77780914b2cf3003acf"
dependencies = [
"cfg_aliases",
"memchr",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "structopt"
version = "0.3.23"
@ -1570,6 +2062,16 @@ dependencies = [
"winapi",
]
[[package]]
name = "time"
version = "0.3.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "004cbc98f30fa233c61a38bc77e96a9106e65c88f2d3bef182ae952027e5753d"
dependencies = [
"libc",
"num_threads",
]
[[package]]
name = "tinyvec"
version = "1.5.0"
@ -1616,10 +2118,21 @@ dependencies = [
]
[[package]]
name = "tokio-stream"
version = "0.1.7"
name = "tokio-rustls"
version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f"
checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
dependencies = [
"rustls",
"tokio",
"webpki",
]
[[package]]
name = "tokio-stream"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"
dependencies = [
"futures-core",
"pin-project-lite",
@ -1650,6 +2163,28 @@ dependencies = [
"serde",
]
[[package]]
name = "tower"
version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5651b5f6860a99bd1adb59dbfe1db8beb433e73709d9032b413a77e2fb7c066a"
dependencies = [
"futures-core",
"futures-util",
"pin-project",
"pin-project-lite",
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "tower-layer"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "343bc9466d3fe6b0f960ef45960509f84480bf4fd96f92901afe7ff3df9d3a62"
[[package]]
name = "tower-service"
version = "0.3.1"
@ -1663,10 +2198,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105"
dependencies = [
"cfg-if",
"log",
"pin-project-lite",
"tracing-attributes",
"tracing-core",
]
[[package]]
name = "tracing-attributes"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4f480b8f81512e825f337ad51e94c1eb5d3bbdf2b363dcd01e2b19a9ffe3f8e"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tracing-core"
version = "0.1.21"
@ -1721,6 +2269,12 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3"
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "url"
version = "2.2.2"
@ -1766,6 +2320,80 @@ version = "0.10.2+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
[[package]]
name = "wasm-bindgen"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06"
dependencies = [
"cfg-if",
"wasm-bindgen-macro",
]
[[package]]
name = "wasm-bindgen-backend"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca"
dependencies = [
"bumpalo",
"lazy_static",
"log",
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01"
dependencies = [
"quote",
"wasm-bindgen-macro-support",
]
[[package]]
name = "wasm-bindgen-macro-support"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc"
dependencies = [
"proc-macro2",
"quote",
"syn",
"wasm-bindgen-backend",
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-shared"
version = "0.2.79"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2"
[[package]]
name = "web-sys"
version = "0.3.56"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "winapi"
version = "0.3.9"
@ -1809,6 +2437,12 @@ version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e575e15bedf6e57b5c2d763ffc6c3c760143466cbd09d762d539680ab5992ded"
[[package]]
name = "zeroize"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4062c749be08d90be727e9c5895371c3a0e49b90ba2b9592dc7afda95cc2b719"
[[package]]
name = "zstd"
version = "0.9.0+zstd.1.5.0"

936
Cargo.nix

File diff suppressed because it is too large Load diff

View file

@ -17,13 +17,25 @@ in let
};
/*
The following complexity should be abstracted by makePackageSet' (note the final quote).
However its code uses deprecated features of rust-overlay that can lead to bug.
Instead, we build our own rustChannel object with the recommended API of rust-overlay.
Cargo2nix is built for rustOverlay which installs Rust from Mozilla releases.
We want our own Rust to avoir incompatibilities, like we had with musl 1.2.0.
rustc was built with musl < 1.2.0 and nix shipped musl >= 1.2.0 which lead to compilation breakage.
So we want a Rust release that is bound to our Nix repository to avoid these problems.
See here for more info: https://musl.libc.org/time64.html
Because Cargo2nix does not support the Rust environment shipped by NixOS,
we emulate the structure of the Rust object created by rustOverlay.
In practise, rustOverlay ships rustc+cargo in a single derivation while
NixOS ships them in separate ones. We reunite them with symlinkJoin.
*/
rustChannel = pkgs.rustPlatform.rust;
rustChannel = pkgs.symlinkJoin {
name ="rust-channel";
paths = [
pkgs.rustPlatform.rust.rustc
pkgs.rustPlatform.rust.cargo
];
};
overrides = pkgs.buildPackages.rustBuilder.overrides.all ++ [
overrides = pkgs.rustBuilder.overrides.all ++ [
/*
We want to inject the git version while keeping the build deterministic.
As we do not want to consider the .git folder as part of the input source,
@ -57,30 +69,27 @@ in let
packageFun = import ./Cargo.nix;
/*
The following definition is not elegant as we use a low level function of Cargo2nix
that enables us to pass our custom rustChannel object
*/
rustPkgs = pkgs.rustBuilder.makePackageSet {
inherit packageFun rustChannel release;
packageOverrides = overrides;
target = null; /* we set target to null because we want that cargo2nix computes it automatically */
buildRustPackages = pkgs.buildPackages.rustBuilder.makePackageSet {
inherit rustChannel packageFun;
packageOverrides = overrides;
target = null; /* we set target to null because we want that cargo2nix computes it automatically */
};
localPatterns = [
/*
The way the default rules are written make think we match recursively, on full path, but the rules are misleading.
In fact, the regex is only called on root elements of the crate (and not recursively).
This behavior does not work well with our nested modules.
We tried to build a "deny list" but negative lookup ahead are not supported on Nix.
As a workaround, we have to register all our submodules in this allow list...
*/
''^(src|tests)'' # fixed default
''.*\.(rs|toml)$'' # fixed default
''^(crdt|replication|cli|helper|signature)'' # our crate submodules
];
};
in
if compileMode == "test"
then builtins.mapAttrs (name: value: rustPkgs.workspace.${name} { inherit compileMode; }) rustPkgs.workspace
then pkgs.symlinkJoin {
name ="garage-tests";
paths = builtins.map (key: rustPkgs.workspace.${key} { inherit compileMode; }) (builtins.attrNames rustPkgs.workspace);
}
else rustPkgs.workspace.garage { inherit compileMode; }

View file

@ -8,11 +8,14 @@ rec {
sha256 = "1xy9zpypqfxs5gcq5dcla4bfkhxmh5nzn9dyqkr03lqycm9wg5cr";
};
cargo2nixSrc = fetchGit {
# As of 2021-10-06
# As of 2022-02-03
url = "https://github.com/superboum/cargo2nix";
rev = "1364752cd784764db2ef5b1e1248727cebfae2ce";
ref = "backward-compat";
rev = "08d963f32a774353ee8acf3f61749915875c1ec4";
};
/*
* Shared objects
*/

View file

@ -76,7 +76,7 @@ function refresh_toolchain {
pkgs.rustPlatform.rust.cargo
pkgs.clippy
pkgs.rustfmt
/*(pkgs.callPackage cargo2nix {}).package*/
cargo2nix.packages.x86_64-linux.cargo2nix
] else [])
++
(if integration then [

View file

@ -8,10 +8,16 @@ description = "Garage, an S3-compatible distributed object store for self-hosted
repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
readme = "../../README.md"
autotests = false
[[bin]]
name = "garage"
path = "main.rs"
[[test]]
name = "integration"
path = "tests/lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
@ -45,3 +51,9 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" }
netapp = "0.3.0"
[dev-dependencies]
aws-sdk-s3 = "0.6"
http = "0.2"
static_init = "1.0"

View file

@ -0,0 +1,22 @@
use aws_sdk_s3::{Client, Config, Credentials, Endpoint};
use super::garage::Instance;
pub fn build_client(instance: &Instance) -> Client {
let credentials = Credentials::new(
&instance.key.id,
&instance.key.secret,
None,
None,
"garage-integ-test",
);
let endpoint = Endpoint::immutable(instance.uri());
let config = Config::builder()
.region(super::REGION)
.credentials_provider(credentials)
.endpoint_resolver(endpoint)
.build();
Client::from_conf(config)
}

View file

@ -0,0 +1,3 @@
pub use process::*;
mod process;

View file

@ -0,0 +1,55 @@
use std::process;
pub trait CommandExt {
fn quiet(&mut self) -> &mut Self;
fn expect_success_status(&mut self, msg: &str) -> process::ExitStatus;
fn expect_success_output(&mut self, msg: &str) -> process::Output;
}
impl CommandExt for process::Command {
fn quiet(&mut self) -> &mut Self {
self.stdout(process::Stdio::null())
.stderr(process::Stdio::null())
}
fn expect_success_status(&mut self, msg: &str) -> process::ExitStatus {
let status = self.status().expect(msg);
status.expect_success(msg);
status
}
fn expect_success_output(&mut self, msg: &str) -> process::Output {
let output = self.output().expect(msg);
output.expect_success(msg);
output
}
}
pub trait OutputExt {
fn expect_success(&self, msg: &str);
}
impl OutputExt for process::Output {
fn expect_success(&self, msg: &str) {
self.status.expect_success(msg)
}
}
pub trait ExitStatusExt {
fn expect_success(&self, msg: &str);
}
impl ExitStatusExt for process::ExitStatus {
fn expect_success(&self, msg: &str) {
if !self.success() {
match self.code() {
Some(code) => panic!(
"Command exited with code {code}: {msg}",
code = code,
msg = msg
),
None => panic!("Command exited with signal: {msg}", msg = msg),
}
}
}
}

View file

@ -0,0 +1,220 @@
use std::mem::MaybeUninit;
use std::path::{Path, PathBuf};
use std::process;
use std::sync::Once;
use super::ext::*;
// https://xkcd.com/221/
const DEFAULT_PORT: u16 = 49995;
static GARAGE_TEST_SECRET: &str =
"c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44";
#[derive(Debug, Default)]
pub struct Key {
pub name: String,
pub id: String,
pub secret: String,
}
pub struct Instance {
process: process::Child,
pub path: PathBuf,
pub key: Key,
pub api_port: u16,
}
impl Instance {
fn new() -> Instance {
use std::{env, fs};
let port = env::var("GARAGE_TEST_INTEGRATION_PORT")
.map(|value| value.parse().expect("Invalid port provided"))
.ok()
.unwrap_or(DEFAULT_PORT);
let path = env::var("GARAGE_TEST_INTEGRATION_PATH")
.map(PathBuf::from)
.ok()
.unwrap_or_else(|| env::temp_dir().join(format!("garage-integ-test-{}", port)));
// Clean test runtime directory
if path.exists() {
fs::remove_dir_all(&path).expect("Could not clean test runtime directory");
}
fs::create_dir(&path).expect("Could not create test runtime directory");
let config = format!(
r#"
metadata_dir = "{path}/meta"
data_dir = "{path}/data"
replication_mode = "1"
rpc_bind_addr = "127.0.0.1:{rpc_port}"
rpc_public_addr = "127.0.0.1:{rpc_port}"
rpc_secret = "{secret}"
[s3_api]
s3_region = "{region}"
api_bind_addr = "127.0.0.1:{api_port}"
root_domain = ".s3.garage"
[s3_web]
bind_addr = "127.0.0.1:{web_port}"
root_domain = ".web.garage"
index = "index.html"
"#,
path = path.display(),
secret = GARAGE_TEST_SECRET,
region = super::REGION,
api_port = port,
rpc_port = port + 1,
web_port = port + 2,
);
fs::write(path.join("config.toml"), config).expect("Could not write garage config file");
let stdout =
fs::File::create(path.join("stdout.log")).expect("Could not create stdout logfile");
let stderr =
fs::File::create(path.join("stderr.log")).expect("Could not create stderr logfile");
let child = command(&path.join("config.toml"))
.arg("server")
.stdout(stdout)
.stderr(stderr)
.env("RUST_LOG", "garage=info,garage_api=debug")
.spawn()
.expect("Could not start garage");
Instance {
process: child,
path,
key: Key::default(),
api_port: port,
}
}
fn setup(&mut self) {
use std::{thread, time::Duration};
// Wait for node to be ready
thread::sleep(Duration::from_secs(2));
self.setup_layout();
self.key = self.new_key("garage_test");
}
fn setup_layout(&self) {
let node_id = self.node_id();
let node_short_id = &node_id[..64];
self.command()
.args(["layout", "assign"])
.arg(node_short_id)
.args(["-c", "1", "-z", "unzonned"])
.quiet()
.expect_success_status("Could not assign garage node layout");
self.command()
.args(["layout", "apply"])
.args(["--version", "1"])
.quiet()
.expect_success_status("Could not apply garage node layout");
}
fn terminate(&mut self) {
// TODO: Terminate "gracefully" the process with SIGTERM instead of directly SIGKILL it.
self.process
.kill()
.expect("Could not terminate garage process");
}
pub fn command(&self) -> process::Command {
command(&self.path.join("config.toml"))
}
pub fn node_id(&self) -> String {
let output = self
.command()
.args(["node", "id"])
.expect_success_output("Could not get node ID");
String::from_utf8(output.stdout).unwrap()
}
pub fn uri(&self) -> http::Uri {
format!("http://127.0.0.1:{api_port}", api_port = self.api_port)
.parse()
.expect("Could not build garage endpoint URI")
}
pub fn new_key(&self, name: &str) -> Key {
let mut key = Key::default();
let output = self
.command()
.args(["key", "new"])
.args(["--name", name])
.expect_success_output("Could not create key");
let stdout = String::from_utf8(output.stdout).unwrap();
for line in stdout.lines() {
if let Some(key_id) = line.strip_prefix("Key ID: ") {
key.id = key_id.to_owned();
continue;
}
if let Some(key_secret) = line.strip_prefix("Secret key: ") {
key.secret = key_secret.to_owned();
continue;
}
}
assert!(!key.id.is_empty(), "Invalid key: Key ID is empty");
assert!(!key.secret.is_empty(), "Invalid key: Key secret is empty");
Key {
name: name.to_owned(),
..key
}
}
}
static mut INSTANCE: MaybeUninit<Instance> = MaybeUninit::uninit();
static INSTANCE_INIT: Once = Once::new();
#[static_init::destructor]
extern "C" fn terminate_instance() {
if INSTANCE_INIT.is_completed() {
// This block is sound as it depends on `INSTANCE_INIT` being completed, meaning `INSTANCE`
// is actually initialized.
unsafe {
INSTANCE.assume_init_mut().terminate();
}
}
}
pub fn instance() -> &'static Instance {
INSTANCE_INIT.call_once(|| unsafe {
let mut instance = Instance::new();
instance.setup();
INSTANCE.write(instance);
});
// This block is sound as it depends on `INSTANCE_INIT` being completed by calling `call_once` (blocking),
// meaning `INSTANCE` is actually initialized.
unsafe { INSTANCE.assume_init_ref() }
}
pub fn command(config_path: &Path) -> process::Command {
use std::env;
let mut command = process::Command::new(
env::var("GARAGE_TEST_INTEGRATION_EXE")
.unwrap_or_else(|_| env!("CARGO_BIN_EXE_garage").to_owned()),
);
command.arg("-c").arg(config_path);
command
}

View file

@ -0,0 +1,11 @@
macro_rules! assert_bytes_eq {
($stream:expr, $bytes:expr) => {
let data = $stream
.collect()
.await
.expect("Error reading data")
.into_bytes();
assert_eq!(data.as_ref(), $bytes);
};
}

View file

@ -0,0 +1,52 @@
use aws_sdk_s3::{Client, Region};
use ext::*;
#[macro_use]
pub mod macros;
pub mod client;
pub mod ext;
pub mod garage;
const REGION: Region = Region::from_static("garage-integ-test");
pub struct Context {
pub garage: &'static garage::Instance,
pub client: Client,
}
impl Context {
fn new() -> Self {
let garage = garage::instance();
let client = client::build_client(garage);
Context { garage, client }
}
/// Create an unique bucket with a random suffix.
///
/// Return the created bucket full name.
pub fn create_bucket(&self, name: &str) -> String {
let bucket_name = name.to_owned();
self.garage
.command()
.args(["bucket", "create", &bucket_name])
.quiet()
.expect_success_status("Could not create bucket");
self.garage
.command()
.args(["bucket", "allow"])
.args(["--owner", "--read", "--write"])
.arg(&bucket_name)
.args(["--key", &self.garage.key.name])
.quiet()
.expect_success_status("Could not allow key for bucket");
bucket_name
}
}
pub fn context() -> Context {
Context::new()
}

4
src/garage/tests/lib.rs Normal file
View file

@ -0,0 +1,4 @@
#[macro_use]
mod common;
mod simple;

View file

@ -0,0 +1,31 @@
use crate::common;
#[tokio::test]
async fn test_simple() {
use aws_sdk_s3::ByteStream;
let ctx = common::context();
let bucket = ctx.create_bucket("test-simple");
let data = ByteStream::from_static(b"Hello world!");
ctx.client
.put_object()
.bucket(&bucket)
.key("test")
.body(data)
.send()
.await
.unwrap();
let res = ctx
.client
.get_object()
.bucket(&bucket)
.key("test")
.send()
.await
.unwrap();
assert_bytes_eq!(res.body, b"Hello world!");
}