diff --git a/Cargo.lock b/Cargo.lock index 477e4456..558dde1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,9 +238,9 @@ checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" [[package]] name = "aws-credential-types" -version = "1.1.4" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33cc49dcdd31c8b6e79850a179af4c367669150c7ac0135f176c61bec81a70f7" +checksum = "60e8f6b615cb5fc60a98132268508ad104310f0cfb25a1c22eee76efdf9154da" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -250,15 +250,16 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.1.4" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb031bff99877c26c28895766f7bb8484a05e24547e370768d6cc9db514662aa" +checksum = "76dd04d39cc12844c0994f2c9c5a6f5184c22e9188ec1ff723de41910a21dcad" dependencies = [ "aws-credential-types", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", @@ -266,6 +267,7 @@ dependencies = [ "fastrand", "http 0.2.12", "http-body 0.4.6", + "once_cell", "percent-encoding", "pin-project-lite", "tracing", @@ -274,9 +276,9 @@ dependencies = [ [[package]] name = "aws-sdk-config" -version = "1.13.0" +version = "1.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4af4f5b0f64563ada272e009cc95027effb546110ed85d014611420ac0d97858" +checksum = "0f94d79b8eef608af51b5415d13f5c670dec177880c6f78cd27bea968e5c9b76" dependencies = [ "aws-credential-types", "aws-runtime", @@ -296,9 +298,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.14.0" +version = "1.68.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "951f7730f51a2155c711c85c79f337fbc02a577fa99d2a0a8059acfce5392113" +checksum = "bc5ddf1dc70287dc9a2f953766a1fe15e3e74aef02fd1335f2afa475c9b4f4fc" dependencies = [ "aws-credential-types", "aws-runtime", @@ -314,20 +316,25 @@ dependencies = [ "aws-smithy-xml", "aws-types", "bytes", + "fastrand", + "hex", + "hmac", "http 0.2.12", "http-body 0.4.6", + "lru", "once_cell", "percent-encoding", "regex-lite", + "sha2", "tracing", "url", ] [[package]] name = "aws-sigv4" -version = "1.1.4" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c371c6b0ac54d4605eb6f016624fb5c7c2925d315fdf600ac1bf21b19d5f1742" +checksum = "9bfe75fad52793ce6dec0dc3d4b1f388f038b5eb866c8d4d7f3a8e21b5ea5051" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -354,9 +361,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.1.4" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72ee2d09cce0ef3ae526679b522835d63e75fb427aca5413cd371e490d52dcc6" +checksum = "fa59d1327d8b5053c54bf2eaae63bf629ba9e904434d0835a28ed3c0ed0a614e" dependencies = [ "futures-util", "pin-project-lite", @@ -365,9 +372,9 @@ dependencies = [ [[package]] name = "aws-smithy-checksums" -version = "0.60.4" +version = "0.60.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be2acd1b9c6ae5859999250ed5a62423aedc5cf69045b844432de15fa2f31f2b" +checksum = "ba1a71073fca26775c8b5189175ea8863afb1c9ea2cceb02a5de5ad9dfbaa795" dependencies = [ "aws-smithy-http", "aws-smithy-types", @@ -386,9 +393,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.4" +version = "0.60.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6363078f927f612b970edf9d1903ef5cef9a64d1e8423525ebb1f0a1633c858" +checksum = "8b18559a41e0c909b77625adf2b8c50de480a8041e5e4a3f5f7d177db70abc5a" dependencies = [ "aws-smithy-types", "bytes", @@ -397,9 +404,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.4" +version = "0.60.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab56aea3cd9e1101a0a999447fb346afb680ab1406cebc44b32346e25b4117d" +checksum = "7809c27ad8da6a6a68c454e651d4962479e81472aa19ae99e59f9aba1f9713cc" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -418,18 +425,18 @@ dependencies = [ [[package]] name = "aws-smithy-json" -version = "0.60.4" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3898ca6518f9215f62678870064398f00031912390efd03f1f6ef56d83aa8e" +checksum = "623a51127f24c30776c8b374295f2df78d92517386f77ba30773f15a30ce1422" dependencies = [ "aws-smithy-types", ] [[package]] name = "aws-smithy-runtime" -version = "1.1.4" +version = "1.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fafdab38f40ad7816e7da5dec279400dd505160780083759f01441af1bbb10ea" +checksum = "d526a12d9ed61fadefda24abe2e682892ba288c2018bcb38b1b4c111d13f6d92" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -440,6 +447,8 @@ dependencies = [ "h2 0.3.24", "http 0.2.12", "http-body 0.4.6", + "http-body 1.0.1", + "httparse", "hyper 0.14.32", "hyper-rustls 0.24.2", "once_cell", @@ -452,31 +461,36 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.1.4" +version = "1.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18276dd28852f34b3bf501f4f3719781f4999a51c7bff1a5c6dc8c4529adc29" +checksum = "92165296a47a812b267b4f41032ff8069ab7ff783696d217f0994a0d7ab585cd" dependencies = [ "aws-smithy-async", "aws-smithy-types", "bytes", "http 0.2.12", + "http 1.2.0", "pin-project-lite", "tokio", "tracing", + "zeroize", ] [[package]] name = "aws-smithy-types" -version = "1.1.4" +version = "1.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb3e134004170d3303718baa2a4eb4ca64ee0a1c0a7041dca31b38be0fb414f3" +checksum = "c7b8a53819e42f10d0821f56da995e1470b199686a1809168db6ca485665f042" dependencies = [ "base64-simd", "bytes", "bytes-utils", "futures-core", "http 0.2.12", + "http 1.2.0", "http-body 0.4.6", + "http-body 1.0.1", + "http-body-util", "itoa", "num-integer", "pin-project-lite", @@ -490,24 +504,23 @@ dependencies = [ [[package]] name = "aws-smithy-xml" -version = "0.60.4" +version = "0.60.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8604a11b25e9ecaf32f9aa56b9fe253c5e2f606a3477f0071e96d3155a5ed218" +checksum = "ab0b0166827aa700d3dc519f72f8b3a91c35d0b8d042dc5d643a91e6f80648fc" dependencies = [ "xmlparser", ] [[package]] name = "aws-types" -version = "1.1.4" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "789bbe008e65636fe1b6dbbb374c40c8960d1232b96af5ff4aec349f9c4accf4" +checksum = "dfbd0a668309ec1f66c0f6bda4840dd6d4796ae26d699ebc266d7cc95c6d040f" dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "http 0.2.12", "rustc_version", "tracing", ] @@ -1116,6 +1129,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0d2fde1f7b3d48b8395d5f2de76c18a528bd6a9cdde438df747bfcba3e05d6f" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1744,6 +1763,11 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "hashlink" @@ -2600,6 +2624,15 @@ version = "0.4.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbf5b083de1c7e0222a7a51dbfdba1cbe1c6ab0b15e29fff3f6c077fd9cd9f" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "matchers" version = "0.1.0" @@ -4915,7 +4948,7 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fa8f0be0..0e156358 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -141,8 +141,8 @@ thiserror = "1.0" assert-json-diff = "2.0" rustc_version = "0.4.0" static_init = "1.0" -aws-sdk-config = "1.13" -aws-sdk-s3 = "1.14" +aws-sdk-config = "1.62" +aws-sdk-s3 = "=1.68" [profile.dev] #lto = "thin" # disabled for now, adds 2-4 min to each CI build diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index 15929cd1..bcb72cc3 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -340,7 +340,12 @@ pub async fn handle_get_without_ctx( enc, &headers, pn, - checksum_mode, + ChecksumMode { + // TODO: for multipart uploads, checksums of each part should be stored + // so that we can return the corresponding checksum here + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + enabled: false, + }, ) .await } @@ -354,7 +359,12 @@ pub async fn handle_get_without_ctx( &headers, range.start, range.start + range.length, - checksum_mode, + ChecksumMode { + // TODO: for range queries that align with part boundaries, + // we should return the saved checksum of the part + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html + enabled: false, + }, ) .await } diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 5860cf97..966c8ac5 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -71,6 +71,7 @@ hyper-util.workspace = true mktemp.workspace = true sha2.workspace = true + static_init.workspace = true assert-json-diff.workspace = true serde_json.workspace = true diff --git a/src/garage/tests/common/client.rs b/src/garage/tests/common/client.rs index ffa4cae8..7a6612cb 100644 --- a/src/garage/tests/common/client.rs +++ b/src/garage/tests/common/client.rs @@ -12,7 +12,7 @@ pub fn build_client(key: &Key) -> Client { .endpoint_url(format!("http://127.0.0.1:{}", DEFAULT_PORT)) .region(super::REGION) .credentials_provider(credentials) - .behavior_version(BehaviorVersion::v2023_11_09()) + .behavior_version(BehaviorVersion::v2024_03_28()) .build(); Client::from_conf(config) diff --git a/src/garage/tests/s3/objects.rs b/src/garage/tests/s3/objects.rs index 77eca2b1..dfc5253d 100644 --- a/src/garage/tests/s3/objects.rs +++ b/src/garage/tests/s3/objects.rs @@ -189,12 +189,14 @@ async fn test_getobject() { #[tokio::test] async fn test_metadata() { + use aws_sdk_s3::primitives::{DateTime, DateTimeFormat}; + let ctx = common::context(); let bucket = ctx.create_bucket("testmetadata"); let etag = "\"46cf18a9b447991b450cad3facf5937e\""; - let exp = aws_sdk_s3::primitives::DateTime::from_secs(10000000000); - let exp2 = aws_sdk_s3::primitives::DateTime::from_secs(10000500000); + let exp = DateTime::from_secs(10000000000); + let exp2 = DateTime::from_secs(10000500000); { // Note. The AWS client SDK adds a Content-Type header @@ -227,7 +229,7 @@ async fn test_metadata() { assert_eq!(o.content_disposition, None); assert_eq!(o.content_encoding, None); assert_eq!(o.content_language, None); - assert_eq!(o.expires, None); + assert_eq!(o.expires_string, None); assert_eq!(o.metadata.unwrap_or_default().len(), 0); let o = ctx @@ -250,7 +252,10 @@ async fn test_metadata() { assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy"); assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy"); assert_eq!(o.content_language.unwrap().as_str(), "cldummy"); - assert_eq!(o.expires.unwrap(), exp); + assert_eq!( + o.expires_string.unwrap(), + exp.fmt(DateTimeFormat::HttpDate).unwrap() + ); } { @@ -288,7 +293,10 @@ async fn test_metadata() { assert_eq!(o.content_disposition.unwrap().as_str(), "cdtest"); assert_eq!(o.content_encoding.unwrap().as_str(), "cetest"); assert_eq!(o.content_language.unwrap().as_str(), "cltest"); - assert_eq!(o.expires.unwrap(), exp2); + assert_eq!( + o.expires_string.unwrap(), + exp2.fmt(DateTimeFormat::HttpDate).unwrap() + ); let mut meta = o.metadata.unwrap(); assert_eq!(meta.remove("testmeta").unwrap(), "hello people"); assert_eq!(meta.remove("nice-unicode-meta").unwrap(), "宅配便"); @@ -314,7 +322,10 @@ async fn test_metadata() { assert_eq!(o.content_disposition.unwrap().as_str(), "cddummy"); assert_eq!(o.content_encoding.unwrap().as_str(), "cedummy"); assert_eq!(o.content_language.unwrap().as_str(), "cldummy"); - assert_eq!(o.expires.unwrap(), exp); + assert_eq!( + o.expires_string.unwrap(), + exp.fmt(DateTimeFormat::HttpDate).unwrap() + ); } }