diff --git a/Cargo.lock b/Cargo.lock index b23da311..51f75afd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,6 +197,17 @@ dependencies = [ "zstd-safe", ] +[[package]] +name = "async-recursion" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -1365,6 +1376,7 @@ dependencies = [ "aes-gcm", "argon2", "async-compression", + "async-recursion", "async-trait", "base64 0.21.7", "bytes", diff --git a/Cargo.nix b/Cargo.nix index 124d3196..e63b92a2 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -34,7 +34,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "c0aa85d369b22875a652356862a5810c22838970be9fbec558dd108d5232881d"; + nixifiedLockHash = "fe0e77c1af963cc04accebb4ddd1607df5bbbaa48ca0b7a325f5e4497e21790e"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -349,6 +349,18 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".async-recursion."1.1.1" = overridableMkRustCrate (profileName: rec { + name = "async-recursion"; + version = "1.1.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "3b43422f69d8ff38f95f1b2bb76517c91589a924d1559a0e935d7c8ce0274c11"; }; + dependencies = { + proc_macro2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.78" { inherit profileName; }).out; + quote = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.35" { inherit profileName; }).out; + syn = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."2.0.48" { inherit profileName; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.5" = overridableMkRustCrate (profileName: rec { name = "async-stream"; version = "0.3.5"; @@ -2003,6 +2015,7 @@ in aes_gcm = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".aes-gcm."0.10.3" { inherit profileName; }).out; argon2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".argon2."0.5.3" { inherit profileName; }).out; async_compression = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-compression."0.4.6" { inherit profileName; }).out; + async_recursion = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-recursion."1.1.1" { profileName = "__noProfile"; }).out; async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.77" { profileName = "__noProfile"; }).out; base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.7" { inherit profileName; }).out; bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.5.0" { inherit profileName; }).out; diff --git a/flake.lock b/flake.lock index a8ebe3c2..0b835517 100644 --- a/flake.lock +++ b/flake.lock @@ -12,17 +12,17 @@ "rust-overlay": "rust-overlay" }, "locked": { - "lastModified": 1666087781, - "narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=", + "lastModified": 1725369207, + "narHash": "sha256-nIMmEOHOSSkyEZzIwXdKRmilV2FboLIa4ceipJc3oNo=", "owner": "Alexis211", "repo": "cargo2nix", - "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36", + "rev": "0f857db8af0f9e2d9223b1605e5318c1cf8b7235", "type": "github" }, "original": { "owner": "Alexis211", "repo": "cargo2nix", - "rev": "a7a61179b66054904ef6a195d8da736eaaa06c36", + "rev": "0f857db8af0f9e2d9223b1605e5318c1cf8b7235", "type": "github" } }, @@ -58,11 +58,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1724395761, - "narHash": "sha256-zRkDV/nbrnp3Y8oCADf5ETl1sDrdmAW6/bBVJ8EbIdQ=", + "lastModified": 1725194671, + "narHash": "sha256-tLGCFEFTB5TaOKkpfw3iYT9dnk4awTP/q4w+ROpMfuw=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "ae815cee91b417be55d43781eb4b73ae1ecc396c", + "rev": "b833ff01a0d694b910daca6e2ff4a3f26dee478c", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index e690aef1..c313135d 100644 --- a/flake.nix +++ b/flake.nix @@ -10,7 +10,8 @@ inputs.cargo2nix = { # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection - url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36"; + # with non deprecated URL literals. + url = "github:Alexis211/cargo2nix/0f857db8af0f9e2d9223b1605e5318c1cf8b7235"; # As of 2023-04-25: # - my two patches were merged into unstable (one for clippy and one to "fix" feature detection) @@ -71,6 +72,7 @@ rustfmt clang mold + cargo2nix.packages.${system}.default ]); # import the full shell using `nix develop .#full` @@ -79,6 +81,7 @@ rust-analyzer clang mold + cargo2nix.packages.${system}.default # ---- extra packages for dev tasks ---- cargo-audit cargo-outdated diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index a5645c26..cccb722f 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -68,6 +68,7 @@ quick-xml.workspace = true opentelemetry.workspace = true opentelemetry-prometheus = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } +async-recursion = "1.1.1" [features] k2v = [ "garage_util/k2v", "garage_model/k2v" ] diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 1737af33..e89d9a5e 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -201,7 +201,10 @@ impl ApiHandler for S3ApiServer { response_content_type, response_expires, }; - handle_get(ctx, &req, &key, part_number, overrides).await + // Redirects are flattened over the S3 API as per: + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/how-to-page-redirect.html + // > REST endpoint – Amazon S3 doesn't redirect the page request. It returns the requested object. + handle_get(ctx, &req, &key, part_number, overrides, true).await } Endpoint::UploadPart { key, diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index f5d3cf11..0d8a2dd6 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -280,22 +280,25 @@ pub async fn handle_head_without_ctx( /// Handle GET request pub async fn handle_get( ctx: ReqCtx, - req: &Request, + req: &Request, key: &str, part_number: Option, overrides: GetObjectOverrides, + flatten_redirect: bool ) -> Result, Error> { - handle_get_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number, overrides).await + handle_get_without_ctx(ctx.garage, req, ctx.bucket_id, key, part_number, overrides, flatten_redirect).await } /// Handle GET request +#[async_recursion::async_recursion] pub async fn handle_get_without_ctx( garage: Arc, - req: &Request, + req: &Request, bucket_id: Uuid, key: &str, part_number: Option, overrides: GetObjectOverrides, + flatten_redirect: bool ) -> Result, Error> { let object = garage .object_table @@ -329,6 +332,30 @@ pub async fn handle_get_without_ctx( let checksum_mode = checksum_mode(&req); + if let Some(redirect_hdr) = headers + .headers + .iter() + .find(|(k, _)| k == "x-amz-website-redirect-location") + .map(|(_, v)| v) + { + if flatten_redirect { + return handle_get_without_ctx(garage, + req, + bucket_id, + redirect_hdr, + part_number, + overrides, + flatten_redirect + ).await; + } else { + return Ok(Response::builder() + .status(StatusCode::FOUND) + .header("Location", redirect_hdr) + .body(empty_body()) + .unwrap()); + } + } + match (part_number, parse_range_header(req, last_v_meta.size)?) { (Some(_), Some(_)) => Err(Error::bad_request( "Cannot specify both partNumber and Range header", diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 1e3b1b44..956db2e0 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -618,9 +618,11 @@ pub(crate) fn get_headers(headers: &HeaderMap) -> Result { + // Redirects are not flattened in the web API, as per: + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/how-to-page-redirect.html + // > Region-specific website endpoint – Amazon S3 redirects the page request according to the value of the x-amz-website-redirect-location property. handle_get_without_ctx( self.garage.clone(), req, @@ -256,6 +259,7 @@ impl WebServer { &key, None, Default::default(), + false ) .await } @@ -309,6 +313,7 @@ impl WebServer { &error_document, None, Default::default(), + false ) .await {