From c5447a9c6f5e618593d879f916d9d86ff760bde3 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sun, 30 Jan 2022 15:19:40 +0100 Subject: [PATCH 01/14] add routing for Post Object --- src/api/api_server.rs | 16 +++++++++++----- src/api/s3_router.rs | 8 ++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/api/api_server.rs b/src/api/api_server.rs index 315116c8..a0c7655c 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -92,11 +92,6 @@ async fn handler( } async fn handler_inner(garage: Arc, req: Request) -> Result, Error> { - let (api_key, content_sha256) = check_payload_signature(&garage, &req).await?; - let api_key = api_key.ok_or_else(|| { - Error::Forbidden("Garage does not support anonymous access yet".to_string()) - })?; - let authority = req .headers() .get(header::HOST) @@ -115,6 +110,17 @@ async fn handler_inner(garage: Arc, req: Request) -> Result return handle_request_without_bucket(garage, req, api_key, endpoint).await, Some(bucket) => bucket.to_string(), diff --git a/src/api/s3_router.rs b/src/api/s3_router.rs index 51020a81..2a68d79e 100644 --- a/src/api/s3_router.rs +++ b/src/api/s3_router.rs @@ -410,6 +410,12 @@ pub enum Endpoint { part_number: u64, upload_id: String, }, + // This endpoint is not documented with others because it has special use case : + // It's intended to be used with HTML forms, using a multipart/form-data body. + // It works a lot like presigned requests, but everything is in the form instead + // of being query parameters of the URL, so authenticating it is a bit different. + PostObject { + }, }} impl Endpoint { @@ -543,6 +549,7 @@ impl Endpoint { UPLOADS => CreateMultipartUpload, ], no_key: [ + EMPTY => PostObject, DELETE => DeleteObjects, ] } @@ -1165,6 +1172,7 @@ mod tests { POST "/{Key+}?restore&versionId=VersionId" => RestoreObject PUT "/my-movie.m2ts?partNumber=1&uploadId=VCVsb2FkIElEIGZvciBlbZZpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZR" => UploadPart PUT "/Key+?partNumber=2&uploadId=UploadId" => UploadPart + POST "/" => PostObject ); // no bucket, won't work with the rest of the test suite assert!(matches!( -- 2.43.4 From 1e9d7dc0879cf849e164fac083863162240aa492 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Tue, 1 Feb 2022 23:41:02 +0100 Subject: [PATCH 02/14] add basic parsing for PostObject --- Cargo.lock | 42 +++++++++++++- src/api/Cargo.toml | 1 + src/api/api_server.rs | 5 +- src/api/lib.rs | 1 + src/api/s3_post_object.rs | 119 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 164 insertions(+), 4 deletions(-) create mode 100644 src/api/s3_post_object.rs diff --git a/Cargo.lock b/Cargo.lock index 880a1462..d4f5ec58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -469,6 +469,15 @@ version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encoding_rs" +version = "0.8.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df" +dependencies = [ + "cfg-if", +] + [[package]] name = "env_logger" version = "0.7.1" @@ -704,6 +713,7 @@ dependencies = [ "idna", "log", "md-5", + "multer", "nom", "percent-encoding", "pin-project", @@ -1314,6 +1324,12 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mime" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1342,6 +1358,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "multer" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f8f35e687561d5c1667590911e6698a8cb714a134a7505718a182e7bc9d3836" +dependencies = [ + "bytes 1.1.0", + "encoding_rs", + "futures-util", + "http", + "httparse", + "log", + "memchr", + "mime", + "spin 0.9.2", + "version_check", +] + [[package]] name = "netapp" version = "0.3.0" @@ -1670,7 +1704,7 @@ dependencies = [ "cc", "libc", "once_cell", - "spin", + "spin 0.5.2", "untrusted", "web-sys", "winapi", @@ -1933,6 +1967,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "spin" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" + [[package]] name = "static_init" version = "1.0.2" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index e93e5ec5..ad7bdc65 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -40,6 +40,7 @@ http = "0.2" httpdate = "0.3" http-range = "0.1" hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } +multer = "2.0" percent-encoding = "2.1.0" roxmltree = "0.14" serde = { version = "1.0", features = ["derive"] } diff --git a/src/api/api_server.rs b/src/api/api_server.rs index a0c7655c..77587de8 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -25,6 +25,7 @@ use crate::s3_cors::*; use crate::s3_delete::*; use crate::s3_get::*; use crate::s3_list::*; +use crate::s3_post_object::handle_post_object; use crate::s3_put::*; use crate::s3_router::{Authorization, Endpoint}; use crate::s3_website::*; @@ -111,9 +112,7 @@ async fn handler_inner(garage: Arc, req: Request) -> Result, + req: Request, + bucket: String, +) -> Result, Error> { + let boundary = req + .headers() + .get(header::CONTENT_TYPE) + .and_then(|ct| ct.to_str().ok()) + .and_then(|ct| multer::parse_boundary(ct).ok()) + .ok_or_else(|| Error::BadRequest("Counld not get multipart boundary".to_owned()))?; + + // these limits are rather arbitrary + let constraints = Constraints::new().size_limit( + SizeLimit::new() + .per_field(32 * 1024) + .for_field("file", 5 * 1024 * 1024 * 1024), + ); + + let mut multipart = Multipart::with_constraints(req.into_body(), boundary, constraints); + + let mut headers = HashMap::new(); + let mut key_id = None; + let mut key = None; + let mut policy = None; + let mut redirect = Err(204); + while let Some(mut field) = multipart.next_field().await.unwrap() { + let name = if let Some(name) = field.name() { + name.to_owned() + } else { + continue; + }; + + if name != "file" { + let content = field.text().await.unwrap(); + match name.as_str() { + // main fields + "AWSAccessKeyId" => { + key_id = Some(content); + } + "key" => { + key = Some(content); + } + "policy" => { + policy = Some(content); + } + // special handling + "success_action_redirect" | "redirect" => { + // TODO should verify it's a valid looking URI + redirect = Ok(content); + } + "success_action_status" => { + let code = name.parse::().unwrap_or(204); + redirect = Err(code); + } + "tagging" => { + // TODO Garage does not support tagging so this can be left empty. It's essentially + // a header except it must be parsed from xml to x-www-form-urlencoded + continue; + } + // headers to PutObject + "acl" | "Cache-Control" | "Content-Type" | "Content-Encoding" | "Expires" => { + headers.insert(name, content); + } + _ if name.starts_with("x-amz-") => { + headers.insert(name, content); + } + _ => { + // TODO should we ignore or error? + } + } + continue; + } + + let _file_name = field.file_name(); + let _content_type = field.content_type(); + while let Some(_chunk) = field.chunk().await.unwrap() {} + + let resp = match redirect { + Err(200) => Response::builder() + .status(StatusCode::OK) + .body(Body::empty())?, + Err(201) => { + // body should be an XML document, not sure which yet + Response::builder() + .status(StatusCode::CREATED) + .body(todo!())? + } + // invalid codes are handled as 204 + Err(_) => Response::builder() + .status(StatusCode::NO_CONTENT) + .body(Body::empty())?, + Ok(uri) => { + // TODO maybe body should contain a link to the ressource? + Response::builder() + .status(StatusCode::SEE_OTHER) + .header(header::LOCATION, uri) + .body(Body::empty())? + } + }; + + return Ok(resp); + } + + return Err(Error::BadRequest( + "Request did not contain a file".to_owned(), + )); +} -- 2.43.4 From efee345178c9f1a35286ddd201e4f632519d2ac0 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sat, 5 Feb 2022 13:38:39 +0100 Subject: [PATCH 03/14] working PostObject without policy support --- Cargo.lock | 1 + src/api/Cargo.toml | 1 + src/api/error.rs | 6 ++ src/api/s3_post_object.rs | 171 +++++++++++++++++++++++++++++------ src/api/s3_put.rs | 29 +++++- src/api/signature/payload.rs | 2 +- 6 files changed, 178 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d4f5ec58..d713012d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -721,6 +721,7 @@ dependencies = [ "roxmltree", "serde", "serde_bytes", + "serde_json", "sha2", "tokio", "url", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index ad7bdc65..51fd7ae0 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -45,5 +45,6 @@ percent-encoding = "2.1.0" roxmltree = "0.14" serde = { version = "1.0", features = ["derive"] } serde_bytes = "0.11" +serde_json = "1.0" quick-xml = { version = "0.21", features = [ "serialize" ] } url = "2.1" diff --git a/src/api/error.rs b/src/api/error.rs index d945295a..f53ed1fd 100644 --- a/src/api/error.rs +++ b/src/api/error.rs @@ -126,6 +126,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: multer::Error) -> Self { + Self::BadRequest(err.to_string()) + } +} + impl Error { /// Get the HTTP status code that best represents the meaning of the error for the client pub fn http_status_code(&self) -> StatusCode { diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index 788e387c..27c150b3 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -1,13 +1,19 @@ -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; +use chrono::{DateTime, NaiveDateTime, Utc}; +use hmac::Mac; use hyper::{header, Body, Request, Response, StatusCode}; +use multer::{Constraints, Multipart, SizeLimit}; +use serde::Deserialize; use garage_model::garage::Garage; -use crate::error::Error; - -use multer::{Constraints, Multipart, SizeLimit}; +use crate::api_server::resolve_bucket; +use crate::error::*; +use crate::s3_put::save_stream; +use crate::signature::payload::parse_credential; +use crate::signature::{signing_hmac, LONG_DATETIME}; pub async fn handle_post_object( garage: Arc, @@ -19,7 +25,7 @@ pub async fn handle_post_object( .get(header::CONTENT_TYPE) .and_then(|ct| ct.to_str().ok()) .and_then(|ct| multer::parse_boundary(ct).ok()) - .ok_or_else(|| Error::BadRequest("Counld not get multipart boundary".to_owned()))?; + .ok_or_bad_request("Counld not get multipart boundary")?; // these limits are rather arbitrary let constraints = Constraints::new().size_limit( @@ -30,12 +36,14 @@ pub async fn handle_post_object( let mut multipart = Multipart::with_constraints(req.into_body(), boundary, constraints); - let mut headers = HashMap::new(); - let mut key_id = None; + let mut headers = BTreeMap::new(); + let mut credential = None; let mut key = None; let mut policy = None; + let mut signature = None; + let mut date = None; let mut redirect = Err(204); - while let Some(mut field) = multipart.next_field().await.unwrap() { + while let Some(field) = multipart.next_field().await? { let name = if let Some(name) = field.name() { name.to_owned() } else { @@ -43,18 +51,25 @@ pub async fn handle_post_object( }; if name != "file" { - let content = field.text().await.unwrap(); - match name.as_str() { + let content = field.text().await?; + // TODO wouldn't a header map be better? + match name.to_ascii_lowercase().as_str() { // main fields - "AWSAccessKeyId" => { - key_id = Some(content); - } "key" => { key = Some(content); } "policy" => { policy = Some(content); } + "x-amz-credential" => { + credential = Some(content); + } + "x-amz-signature" => { + signature = Some(content); + } + "x-amz-date" => { + date = Some(content); + } // special handling "success_action_redirect" | "redirect" => { // TODO should verify it's a valid looking URI @@ -70,44 +85,118 @@ pub async fn handle_post_object( continue; } // headers to PutObject - "acl" | "Cache-Control" | "Content-Type" | "Content-Encoding" | "Expires" => { + "cache-control" | "content-type" | "content-encoding" | "expires" => { headers.insert(name, content); } + "acl" => { + headers.insert("x-amz-acl".to_owned(), content); + } _ if name.starts_with("x-amz-") => { headers.insert(name, content); } _ => { - // TODO should we ignore or error? + // TODO should we ignore, error or process? } } continue; } - let _file_name = field.file_name(); - let _content_type = field.content_type(); - while let Some(_chunk) = field.chunk().await.unwrap() {} + // Current part is file. Do some checks before handling to PutObject code + let credential = credential.ok_or_else(|| { + Error::Forbidden("Garage does not support anonymous access yet".to_string()) + })?; + let policy = policy.ok_or_bad_request("No policy was provided")?; + let signature = signature.ok_or_bad_request("No signature was provided")?; + let date = date.ok_or_bad_request("No date was provided")?; + let key = key.ok_or_bad_request("No key was provided")?; + + let key = if key.contains("${filename}") { + let filename = field.file_name(); + // is this correct? Maybe we should error instead of default? + key.replace("${filename}", &filename.unwrap_or_default()) + } else { + key + }; + + // TODO verify scope against bucket&date? + let (key_id, scope) = parse_credential(&credential)?; + // TODO duplicated from signature/* + let date: NaiveDateTime = NaiveDateTime::parse_from_str(&date, LONG_DATETIME) + .ok_or_bad_request("invalid date")?; + let date: DateTime = DateTime::from_utc(date, Utc); + + // TODO counldn't this be a garage.get_key? + let api_key = garage + .key_table + .get(&garage_table::EmptyKey, &key_id) + .await? + .filter(|k| !k.state.is_deleted()) + .ok_or_else(|| Error::Forbidden(format!("No such key: {}", key_id)))?; + + // TODO duplicated from signature/* + let key_p = api_key.params().unwrap(); + let secret_key = &key_p.secret_key; + + let mut hmac = signing_hmac(&date, secret_key, &garage.config.s3_api.s3_region, "s3") + .ok_or_internal_error("Unable to build signing HMAC")?; + hmac.update(policy.as_bytes()); + let our_signature = hex::encode(hmac.finalize().into_bytes()); + if signature != our_signature { + return Err(Error::Forbidden("Invalid signature".to_string())); + } + + let bucket_id = resolve_bucket(&garage, &bucket, &api_key).await?; + + if !api_key.allow_write(&bucket_id) { + return Err(Error::Forbidden( + "Operation is not allowed for this key.".to_string(), + )); + } + + let decoded_policy = base64::decode(&policy)?; + let _decoded_policy: Policy = serde_json::from_slice(&decoded_policy).unwrap(); + // TODO validate policy against request + // unsafe to merge until implemented + + let content_type = field + .content_type() + .map(ToString::to_string) + .unwrap_or_else(|| "blob".to_owned()); + let headers = garage_model::object_table::ObjectVersionHeaders { + content_type, + other: headers, + }; + + use futures::StreamExt; + let res = save_stream( + garage, + headers, + field.map(|r| r.map_err(Into::into)), + bucket_id, + &key, + None, + None, + ) + .await?; let resp = match redirect { Err(200) => Response::builder() .status(StatusCode::OK) .body(Body::empty())?, Err(201) => { - // body should be an XML document, not sure which yet + // TODO body should be an XML document, not sure which yet Response::builder() .status(StatusCode::CREATED) - .body(todo!())? + .body(res.into_body())? } // invalid codes are handled as 204 Err(_) => Response::builder() .status(StatusCode::NO_CONTENT) .body(Body::empty())?, - Ok(uri) => { - // TODO maybe body should contain a link to the ressource? - Response::builder() - .status(StatusCode::SEE_OTHER) - .header(header::LOCATION, uri) - .body(Body::empty())? - } + Ok(uri) => Response::builder() + .status(StatusCode::SEE_OTHER) + .header(header::LOCATION, uri.clone()) + .body(uri.into())?, }; return Ok(resp); @@ -117,3 +206,31 @@ pub async fn handle_post_object( "Request did not contain a file".to_owned(), )); } + +// TODO remove allow(dead_code) when policy is verified + +#[allow(dead_code)] +#[derive(Deserialize)] +struct Policy { + expiration: String, + conditions: Vec, +} + +/// A single condition from a policy +#[derive(Deserialize)] +#[serde(untagged)] +enum PolicyCondition { + // will contain a single key-value pair + Equal(HashMap), + OtherOp([String; 3]), + SizeRange(String, u64, u64), +} + +#[allow(dead_code)] +#[derive(PartialEq, Eq)] +enum Operation { + Equal, + StartsWith, + StartsWithCT, + SizeRange, +} diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index a6863cd3..86d71683 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -34,10 +34,6 @@ pub async fn handle_put( api_key: &Key, mut content_sha256: Option, ) -> Result, Error> { - // Generate identity of new version - let version_uuid = gen_uuid(); - let version_timestamp = now_msec(); - // Retrieve interesting headers from request let headers = get_headers(&req)?; debug!("Object headers: {:?}", headers); @@ -92,6 +88,31 @@ pub async fn handle_put( body.boxed() }; + save_stream( + garage, + headers, + body, + bucket_id, + key, + content_md5, + content_sha256, + ) + .await +} + +pub(crate) async fn save_stream> + Unpin>( + garage: Arc, + headers: ObjectVersionHeaders, + body: S, + bucket_id: Uuid, + key: &str, + content_md5: Option, + content_sha256: Option, +) -> Result, Error> { + // Generate identity of new version + let version_uuid = gen_uuid(); + let version_timestamp = now_msec(); + let mut chunker = StreamChunker::new(body, garage.config.block_size); let first_block = chunker.next().await?.unwrap_or_default(); diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index fe6120d3..eff4c89e 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -234,7 +234,7 @@ fn parse_query_authorization( }) } -fn parse_credential(cred: &str) -> Result<(String, String), Error> { +pub(crate) fn parse_credential(cred: &str) -> Result<(String, String), Error> { let first_slash = cred .find('/') .ok_or_bad_request("Credentials does not contain / in authorization field")?; -- 2.43.4 From 621bc849571b34dcc415310e94512a6002c7d108 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Sat, 5 Feb 2022 13:51:15 +0100 Subject: [PATCH 04/14] update Cargo.nix and fix clippy --- Cargo.nix | 57 ++++++++++++++++++++++++++++++++++++++- src/api/s3_post_object.rs | 8 +++--- 2 files changed, 60 insertions(+), 5 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index c01222fe..fef82575 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -560,7 +560,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; }; dependencies = { - ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.115" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; }; }; }); @@ -672,6 +672,18 @@ in [ "default" ] [ "use_std" ] ]; + "registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.30" = overridableMkRustCrate (profileName: rec { + name = "encoding_rs"; + version = "0.8.30"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7896dc8abb250ffdda33912550faa54c88ec8b998dec0b2c55ab224921ce11df"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "default" ] + ]; + dependencies = { + cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; + }; }); "registry+https://github.com/rust-lang/crates.io-index".env_logger."0.7.1" = overridableMkRustCrate (profileName: rec { @@ -1014,6 +1026,7 @@ in idna = rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.2.3" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.9.1" { inherit profileName; }; + multer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".multer."2.0.2" { inherit profileName; }; nom = rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.0" { inherit profileName; }; percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.8" { inherit profileName; }; @@ -1021,6 +1034,7 @@ in roxmltree = rustPackages."registry+https://github.com/rust-lang/crates.io-index".roxmltree."0.14.1" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.130" { inherit profileName; }; serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; + serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.68" { inherit profileName; }; sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.8" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.12.0" { inherit profileName; }; url = rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.2.2" { inherit profileName; }; @@ -1768,6 +1782,13 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".mime."0.3.16" = overridableMkRustCrate (profileName: rec { + name = "mime"; + version = "0.3.16"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".minimal-lexical."0.2.1" = overridableMkRustCrate (profileName: rec { name = "minimal-lexical"; version = "0.2.1"; @@ -1812,6 +1833,30 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".multer."2.0.2" = overridableMkRustCrate (profileName: rec { + name = "multer"; + version = "2.0.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5f8f35e687561d5c1667590911e6698a8cb714a134a7505718a182e7bc9d3836"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + encoding_rs = rustPackages."registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.30" { inherit profileName; }; + futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; + http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.5" { inherit profileName; }; + httparse = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.5.1" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + memchr = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; + mime = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mime."0.3.16" { inherit profileName; }; + spin = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.9.2" { inherit profileName; }; + }; + buildDependencies = { + version_check = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".version_check."0.9.3" { profileName = "__noProfile"; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.0" = overridableMkRustCrate (profileName: rec { name = "netapp"; version = "0.3.0"; @@ -2670,6 +2715,16 @@ in buildDependencies = { cfg_aliases = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg_aliases."0.1.1" { profileName = "__noProfile"; }; }; + + "registry+https://github.com/rust-lang/crates.io-index".spin."0.9.2" = overridableMkRustCrate (profileName: rec { + name = "spin"; + version = "0.9.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"; }; + features = builtins.concatLists [ + [ "mutex" ] + [ "spin_mutex" ] + ]; }); "registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.23" = overridableMkRustCrate (profileName: rec { diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index 27c150b3..4f9003c8 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -113,13 +113,13 @@ pub async fn handle_post_object( let key = if key.contains("${filename}") { let filename = field.file_name(); // is this correct? Maybe we should error instead of default? - key.replace("${filename}", &filename.unwrap_or_default()) + key.replace("${filename}", filename.unwrap_or_default()) } else { key }; // TODO verify scope against bucket&date? - let (key_id, scope) = parse_credential(&credential)?; + let (key_id, _scope) = parse_credential(&credential)?; // TODO duplicated from signature/* let date: NaiveDateTime = NaiveDateTime::parse_from_str(&date, LONG_DATETIME) .ok_or_bad_request("invalid date")?; @@ -202,9 +202,9 @@ pub async fn handle_post_object( return Ok(resp); } - return Err(Error::BadRequest( + Err(Error::BadRequest( "Request did not contain a file".to_owned(), - )); + )) } // TODO remove allow(dead_code) when policy is verified -- 2.43.4 From c629a9f4e21129577986dd5859805f93c2be5708 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sat, 5 Feb 2022 20:36:03 +0100 Subject: [PATCH 05/14] refactore signature verification to avoir duplication --- src/api/s3_post_object.rs | 33 +---------- src/api/signature/payload.rs | 111 ++++++++++++++++++++--------------- 2 files changed, 66 insertions(+), 78 deletions(-) diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index 4f9003c8..a95a3e9e 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -1,8 +1,6 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use chrono::{DateTime, NaiveDateTime, Utc}; -use hmac::Mac; use hyper::{header, Body, Request, Response, StatusCode}; use multer::{Constraints, Multipart, SizeLimit}; use serde::Deserialize; @@ -12,8 +10,7 @@ use garage_model::garage::Garage; use crate::api_server::resolve_bucket; use crate::error::*; use crate::s3_put::save_stream; -use crate::signature::payload::parse_credential; -use crate::signature::{signing_hmac, LONG_DATETIME}; +use crate::signature::payload::{parse_date, verify_v4}; pub async fn handle_post_object( garage: Arc, @@ -118,32 +115,8 @@ pub async fn handle_post_object( key }; - // TODO verify scope against bucket&date? - let (key_id, _scope) = parse_credential(&credential)?; - // TODO duplicated from signature/* - let date: NaiveDateTime = NaiveDateTime::parse_from_str(&date, LONG_DATETIME) - .ok_or_bad_request("invalid date")?; - let date: DateTime = DateTime::from_utc(date, Utc); - - // TODO counldn't this be a garage.get_key? - let api_key = garage - .key_table - .get(&garage_table::EmptyKey, &key_id) - .await? - .filter(|k| !k.state.is_deleted()) - .ok_or_else(|| Error::Forbidden(format!("No such key: {}", key_id)))?; - - // TODO duplicated from signature/* - let key_p = api_key.params().unwrap(); - let secret_key = &key_p.secret_key; - - let mut hmac = signing_hmac(&date, secret_key, &garage.config.s3_api.s3_region, "s3") - .ok_or_internal_error("Unable to build signing HMAC")?; - hmac.update(policy.as_bytes()); - let our_signature = hex::encode(hmac.finalize().into_bytes()); - if signature != our_signature { - return Err(Error::Forbidden("Invalid signature".to_string())); - } + let date = parse_date(&date)?; + let api_key = verify_v4(&garage, &credential, &date, &signature, policy.as_bytes()).await?; let bucket_id = resolve_bucket(&garage, &bucket, &api_key).await?; diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index eff4c89e..a6c32e41 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -49,23 +49,6 @@ pub async fn check_payload_signature( } }; - let scope = format!( - "{}/{}/s3/aws4_request", - authorization.date.format(SHORT_DATE), - garage.config.s3_api.s3_region - ); - if authorization.scope != scope { - return Err(Error::AuthorizationHeaderMalformed(scope.to_string())); - } - - let key = garage - .key_table - .get(&EmptyKey, &authorization.key_id) - .await? - .filter(|k| !k.state.is_deleted()) - .ok_or_else(|| Error::Forbidden(format!("No such key: {}", authorization.key_id)))?; - let key_p = key.params().unwrap(); - let canonical_request = canonical_request( request.method(), &request.uri().path().to_string(), @@ -74,24 +57,17 @@ pub async fn check_payload_signature( &authorization.signed_headers, &authorization.content_sha256, ); + let (_, scope) = parse_credential(&authorization.credential)?; let string_to_sign = string_to_sign(&authorization.date, &scope, &canonical_request); - let mut hmac = signing_hmac( + let key = verify_v4( + garage, + &authorization.credential, &authorization.date, - &key_p.secret_key, - &garage.config.s3_api.s3_region, - "s3", + &authorization.signature, + string_to_sign.as_bytes(), ) - .ok_or_internal_error("Unable to build signing HMAC")?; - hmac.update(string_to_sign.as_bytes()); - let signature = hex::encode(hmac.finalize().into_bytes()); - - if authorization.signature != signature { - trace!("Canonical request: ``{}``", canonical_request); - trace!("String to sign: ``{}``", string_to_sign); - trace!("Expected: {}, got: {}", signature, authorization.signature); - return Err(Error::Forbidden("Invalid signature".to_string())); - } + .await?; let content_sha256 = if authorization.content_sha256 == "UNSIGNED-PAYLOAD" { None @@ -108,8 +84,7 @@ pub async fn check_payload_signature( } struct Authorization { - key_id: String, - scope: String, + credential: String, signed_headers: String, signature: String, content_sha256: String, @@ -142,7 +117,6 @@ fn parse_authorization( let cred = auth_params .get("Credential") .ok_or_bad_request("Could not find Credential in Authorization field")?; - let (key_id, scope) = parse_credential(cred)?; let content_sha256 = headers .get("x-amz-content-sha256") @@ -150,18 +124,15 @@ fn parse_authorization( let date = headers .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field")?; - let date: NaiveDateTime = - NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; - let date: DateTime = DateTime::from_utc(date, Utc); + .ok_or_bad_request("Missing X-Amz-Date field") + .and_then(|d| parse_date(d))?; if Utc::now() - date > Duration::hours(24) { return Err(Error::BadRequest("Date is too old".to_string())); } let auth = Authorization { - key_id, - scope, + credential: cred.to_string(), signed_headers: auth_params .get("SignedHeaders") .ok_or_bad_request("Could not find SignedHeaders in Authorization field")? @@ -189,7 +160,6 @@ fn parse_query_authorization( let cred = headers .get("x-amz-credential") .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; - let (key_id, scope) = parse_credential(cred)?; let signed_headers = headers .get("x-amz-signedheaders") .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; @@ -215,18 +185,15 @@ fn parse_query_authorization( let date = headers .get("x-amz-date") - .ok_or_bad_request("Missing X-Amz-Date field")?; - let date: NaiveDateTime = - NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; - let date: DateTime = DateTime::from_utc(date, Utc); + .ok_or_bad_request("Missing X-Amz-Date field") + .and_then(|d| parse_date(d))?; if Utc::now() - date > Duration::seconds(duration) { return Err(Error::BadRequest("Date is too old".to_string())); } Ok(Authorization { - key_id, - scope, + credential: cred.to_string(), signed_headers: signed_headers.to_string(), signature: signature.to_string(), content_sha256: content_sha256.to_string(), @@ -234,7 +201,7 @@ fn parse_query_authorization( }) } -pub(crate) fn parse_credential(cred: &str) -> Result<(String, String), Error> { +fn parse_credential(cred: &str) -> Result<(String, String), Error> { let first_slash = cred .find('/') .ok_or_bad_request("Credentials does not contain / in authorization field")?; @@ -304,3 +271,51 @@ fn canonical_query_string(uri: &hyper::Uri) -> String { "".to_string() } } + +pub fn parse_date(date: &str) -> Result, Error> { + let date: NaiveDateTime = + NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; + Ok(DateTime::from_utc(date, Utc)) +} + +pub async fn verify_v4( + garage: &Garage, + credential: &str, + date: &DateTime, + signature: &str, + payload: &[u8], +) -> Result { + let (key_id, scope) = parse_credential(credential)?; + + let scope_expected = format!( + "{}/{}/s3/aws4_request", + date.format(SHORT_DATE), + garage.config.s3_api.s3_region + ); + if scope != scope_expected { + return Err(Error::AuthorizationHeaderMalformed(scope.to_string())); + } + + let key = garage + .key_table + .get(&EmptyKey, &key_id) + .await? + .filter(|k| !k.state.is_deleted()) + .ok_or_else(|| Error::Forbidden(format!("No such key: {}", &key_id)))?; + let key_p = key.params().unwrap(); + + let mut hmac = signing_hmac( + date, + &key_p.secret_key, + &garage.config.s3_api.s3_region, + "s3", + ) + .ok_or_internal_error("Unable to build signing HMAC")?; + hmac.update(payload); + let our_signature = hex::encode(hmac.finalize().into_bytes()); + if signature != our_signature { + return Err(Error::Forbidden("Invalid signature".to_string())); + } + + Ok(key) +} -- 2.43.4 From abb19a04435d439daa7353ada37510eb2ff27a43 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sun, 6 Feb 2022 00:09:40 +0100 Subject: [PATCH 06/14] refactore params parsing --- src/api/s3_copy.rs | 2 +- src/api/s3_post_object.rs | 177 ++++++++++++++++++-------------------- src/api/s3_put.rs | 18 ++-- 3 files changed, 93 insertions(+), 104 deletions(-) diff --git a/src/api/s3_copy.rs b/src/api/s3_copy.rs index 93947b78..2d050ff6 100644 --- a/src/api/s3_copy.rs +++ b/src/api/s3_copy.rs @@ -46,7 +46,7 @@ pub async fn handle_copy( // Implement x-amz-metadata-directive: REPLACE let new_meta = match req.headers().get("x-amz-metadata-directive") { Some(v) if v == hyper::header::HeaderValue::from_static("REPLACE") => ObjectVersionMeta { - headers: get_headers(req)?, + headers: get_headers(req.headers())?, size: source_version_meta.size, etag: source_version_meta.etag.clone(), }, diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index a95a3e9e..6d602b5d 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -1,7 +1,10 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::HashMap; +use std::convert::TryInto; use std::sync::Arc; -use hyper::{header, Body, Request, Response, StatusCode}; +use futures::StreamExt; +use hyper::header::{self, HeaderMap, HeaderName, HeaderValue}; +use hyper::{Body, Request, Response, StatusCode}; use multer::{Constraints, Multipart, SizeLimit}; use serde::Deserialize; @@ -9,7 +12,7 @@ use garage_model::garage::Garage; use crate::api_server::resolve_bucket; use crate::error::*; -use crate::s3_put::save_stream; +use crate::s3_put::{get_headers, save_stream}; use crate::signature::payload::{parse_date, verify_v4}; pub async fn handle_post_object( @@ -24,99 +27,72 @@ pub async fn handle_post_object( .and_then(|ct| multer::parse_boundary(ct).ok()) .ok_or_bad_request("Counld not get multipart boundary")?; - // these limits are rather arbitrary + // 16k seems plenty for a header. 5G is the max size of a single part, so it seemrs reasonable + // for a PostObject let constraints = Constraints::new().size_limit( SizeLimit::new() - .per_field(32 * 1024) + .per_field(16 * 1024) .for_field("file", 5 * 1024 * 1024 * 1024), ); let mut multipart = Multipart::with_constraints(req.into_body(), boundary, constraints); - let mut headers = BTreeMap::new(); - let mut credential = None; - let mut key = None; - let mut policy = None; - let mut signature = None; - let mut date = None; - let mut redirect = Err(204); + let mut params = HeaderMap::new(); while let Some(field) = multipart.next_field().await? { - let name = if let Some(name) = field.name() { - name.to_owned() + let name: HeaderName = if let Some(Ok(name)) = field.name().map(TryInto::try_into) { + name } else { continue; }; - if name != "file" { - let content = field.text().await?; - // TODO wouldn't a header map be better? - match name.to_ascii_lowercase().as_str() { - // main fields - "key" => { - key = Some(content); - } - "policy" => { - policy = Some(content); - } - "x-amz-credential" => { - credential = Some(content); - } - "x-amz-signature" => { - signature = Some(content); - } - "x-amz-date" => { - date = Some(content); - } - // special handling - "success_action_redirect" | "redirect" => { - // TODO should verify it's a valid looking URI - redirect = Ok(content); - } - "success_action_status" => { - let code = name.parse::().unwrap_or(204); - redirect = Err(code); - } - "tagging" => { - // TODO Garage does not support tagging so this can be left empty. It's essentially - // a header except it must be parsed from xml to x-www-form-urlencoded - continue; - } - // headers to PutObject - "cache-control" | "content-type" | "content-encoding" | "expires" => { - headers.insert(name, content); - } - "acl" => { - headers.insert("x-amz-acl".to_owned(), content); - } - _ if name.starts_with("x-amz-") => { - headers.insert(name, content); - } - _ => { - // TODO should we ignore, error or process? + if let Ok(content) = HeaderValue::from_str(&field.text().await?) { + match name.as_str() { + "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */), + "acl" => { + params.append("x-amz-acl", content); + } + _ => { + params.append(name, content); + } } } continue; } // Current part is file. Do some checks before handling to PutObject code - let credential = credential.ok_or_else(|| { - Error::Forbidden("Garage does not support anonymous access yet".to_string()) - })?; - let policy = policy.ok_or_bad_request("No policy was provided")?; - let signature = signature.ok_or_bad_request("No signature was provided")?; - let date = date.ok_or_bad_request("No date was provided")?; - let key = key.ok_or_bad_request("No key was provided")?; + let key = params + .get("key") + .ok_or_bad_request("No key was provided")? + .to_str()?; + let credential = params + .get("x-amz-credential") + .ok_or_else(|| { + Error::Forbidden("Garage does not support anonymous access yet".to_string()) + })? + .to_str()?; + let policy = params + .get("policy") + .ok_or_bad_request("No policy was provided")? + .to_str()?; + let signature = params + .get("x-amz-signature") + .ok_or_bad_request("No signature was provided")? + .to_str()?; + let date = params + .get("x-amz-date") + .ok_or_bad_request("No date was provided")? + .to_str()?; let key = if key.contains("${filename}") { let filename = field.file_name(); // is this correct? Maybe we should error instead of default? key.replace("${filename}", filename.unwrap_or_default()) } else { - key + key.to_owned() }; - let date = parse_date(&date)?; - let api_key = verify_v4(&garage, &credential, &date, &signature, policy.as_bytes()).await?; + let date = parse_date(date)?; + let api_key = verify_v4(&garage, credential, &date, signature, policy.as_bytes()).await?; let bucket_id = resolve_bucket(&garage, &bucket, &api_key).await?; @@ -128,19 +104,21 @@ pub async fn handle_post_object( let decoded_policy = base64::decode(&policy)?; let _decoded_policy: Policy = serde_json::from_slice(&decoded_policy).unwrap(); + // TODO validate policy against request // unsafe to merge until implemented let content_type = field .content_type() - .map(ToString::to_string) - .unwrap_or_else(|| "blob".to_owned()); - let headers = garage_model::object_table::ObjectVersionHeaders { - content_type, - other: headers, - }; + .map(AsRef::as_ref) + .map(HeaderValue::from_str) + .transpose() + .ok_or_bad_request("Invalid content type")? + .unwrap_or_else(|| HeaderValue::from_static("blob")); + + params.append(header::CONTENT_TYPE, content_type); + let headers = get_headers(¶ms)?; - use futures::StreamExt; let res = save_stream( garage, headers, @@ -152,24 +130,35 @@ pub async fn handle_post_object( ) .await?; - let resp = match redirect { - Err(200) => Response::builder() - .status(StatusCode::OK) - .body(Body::empty())?, - Err(201) => { - // TODO body should be an XML document, not sure which yet - Response::builder() - .status(StatusCode::CREATED) - .body(res.into_body())? - } - // invalid codes are handled as 204 - Err(_) => Response::builder() - .status(StatusCode::NO_CONTENT) - .body(Body::empty())?, - Ok(uri) => Response::builder() + let resp = if let Some(target) = params + .get("success_action_redirect") + .and_then(|h| h.to_str().ok()) + { + // TODO should validate it's a valid url + let target = target.to_owned(); + Response::builder() .status(StatusCode::SEE_OTHER) - .header(header::LOCATION, uri.clone()) - .body(uri.into())?, + .header(header::LOCATION, target.clone()) + .body(target.into())? + } else { + let action = params + .get("success_action_status") + .and_then(|h| h.to_str().ok()) + .unwrap_or("204"); + match action { + "200" => Response::builder() + .status(StatusCode::OK) + .body(Body::empty())?, + "201" => { + // TODO body should be an XML document, not sure which yet + Response::builder() + .status(StatusCode::CREATED) + .body(res.into_body())? + } + _ => Response::builder() + .status(StatusCode::NO_CONTENT) + .body(Body::empty())?, + } }; return Ok(resp); diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index 86d71683..5af29b79 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use chrono::{DateTime, NaiveDateTime, Utc}; use futures::{prelude::*, TryFutureExt}; use hyper::body::{Body, Bytes}; +use hyper::header::{HeaderMap, HeaderValue}; use hyper::{Request, Response}; use md5::{digest::generic_array::*, Digest as Md5Digest, Md5}; use sha2::Sha256; @@ -35,7 +36,7 @@ pub async fn handle_put( mut content_sha256: Option, ) -> Result, Error> { // Retrieve interesting headers from request - let headers = get_headers(&req)?; + let headers = get_headers(req.headers())?; debug!("Object headers: {:?}", headers); let content_md5 = match req.headers().get("content-md5") { @@ -394,7 +395,7 @@ pub async fn handle_create_multipart_upload( key: &str, ) -> Result, Error> { let version_uuid = gen_uuid(); - let headers = get_headers(req)?; + let headers = get_headers(req.headers())?; // Create object in object table let object_version = ObjectVersion { @@ -693,17 +694,16 @@ pub async fn handle_abort_multipart_upload( Ok(Response::new(Body::from(vec![]))) } -fn get_mime_type(req: &Request) -> Result { - Ok(req - .headers() +fn get_mime_type(headers: &HeaderMap) -> Result { + Ok(headers .get(hyper::header::CONTENT_TYPE) .map(|x| x.to_str()) .unwrap_or(Ok("blob"))? .to_string()) } -pub(crate) fn get_headers(req: &Request) -> Result { - let content_type = get_mime_type(req)?; +pub(crate) fn get_headers(headers: &HeaderMap) -> Result { + let content_type = get_mime_type(headers)?; let mut other = BTreeMap::new(); // Preserve standard headers @@ -715,7 +715,7 @@ pub(crate) fn get_headers(req: &Request) -> Result { other.insert(h.to_string(), v_str.to_string()); @@ -728,7 +728,7 @@ pub(crate) fn get_headers(req: &Request) -> Result { -- 2.43.4 From 217b214ca95c4b37fc9d1ab6313d10d0c5f71a64 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sun, 6 Feb 2022 16:03:43 +0100 Subject: [PATCH 07/14] add post object policy support except for body size --- src/api/s3_post_object.rs | 131 +++++++++++++++++++++++++++++++++++--- 1 file changed, 122 insertions(+), 9 deletions(-) diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index 6d602b5d..52229303 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -1,7 +1,9 @@ use std::collections::HashMap; use std::convert::TryInto; +use std::ops::RangeInclusive; use std::sync::Arc; +use chrono::{DateTime, Duration, Utc}; use futures::StreamExt; use hyper::header::{self, HeaderMap, HeaderName, HeaderValue}; use hyper::{Body, Request, Response, StatusCode}; @@ -103,7 +105,55 @@ pub async fn handle_post_object( } let decoded_policy = base64::decode(&policy)?; - let _decoded_policy: Policy = serde_json::from_slice(&decoded_policy).unwrap(); + let decoded_policy: Policy = + serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?; + + let expiration: DateTime = DateTime::parse_from_rfc3339(&decoded_policy.expiration) + .ok_or_bad_request("Invalid expiration date")? + .into(); + if Utc::now() - expiration > Duration::zero() { + return Err(Error::BadRequest( + "Expiration date is in the paste".to_string(), + )); + } + + let conditions = decoded_policy.into_conditions()?; + + for (key, value) in params.iter() { + let key = key.as_str(); + if key.eq_ignore_ascii_case("content-type") { + for cond in &conditions.content_type { + let ok = match cond { + Operation::Equal(s) => value == s, + Operation::StartsWith(s) => { + value.to_str()?.split(',').all(|v| v.starts_with(s)) + } + }; + if !ok { + return Err(Error::BadRequest(format!( + "Key '{}' has value not allowed in policy", + key + ))); + } + } + } else { + let conds = conditions.params.get(key).ok_or_else(|| { + Error::BadRequest(format!("Key '{}' is not allowed in policy", key)) + })?; + for cond in conds { + let ok = match cond { + Operation::Equal(s) => s == value, + Operation::StartsWith(s) => value.to_str()?.starts_with(s), + }; + if !ok { + return Err(Error::BadRequest(format!( + "Key '{}' has value not allowed in policy", + key + ))); + } + } + } + } // TODO validate policy against request // unsafe to merge until implemented @@ -169,15 +219,74 @@ pub async fn handle_post_object( )) } -// TODO remove allow(dead_code) when policy is verified - -#[allow(dead_code)] #[derive(Deserialize)] struct Policy { expiration: String, conditions: Vec, } +impl Policy { + fn into_conditions(self) -> Result { + let mut params = HashMap::<_, Vec<_>>::new(); + let mut content_type = Vec::new(); + + let mut length = (0, u64::MAX); + for condition in self.conditions { + match condition { + PolicyCondition::Equal(map) => { + if map.len() != 1 { + return Err(Error::BadRequest("Invalid policy item".to_owned())); + } + let (k, v) = map.into_iter().next().expect("size was verified"); + if k.eq_ignore_ascii_case("content-type") { + content_type.push(Operation::Equal(v)); + } else { + params.entry(k).or_default().push(Operation::Equal(v)); + } + } + PolicyCondition::OtherOp([cond, mut key, value]) => { + if key.remove(0) != '$' { + return Err(Error::BadRequest("Invalid policy item".to_owned())); + } + match cond.as_str() { + "eq" => { + if key.eq_ignore_ascii_case("content-type") { + content_type.push(Operation::Equal(value)); + } else { + params.entry(key).or_default().push(Operation::Equal(value)); + } + } + "starts-with" => { + if key.eq_ignore_ascii_case("content-type") { + content_type.push(Operation::StartsWith(value)); + } else { + params + .entry(key) + .or_default() + .push(Operation::StartsWith(value)); + } + } + _ => return Err(Error::BadRequest("Invalid policy item".to_owned())), + } + } + PolicyCondition::SizeRange(key, min, max) => { + if key == "content-length-range" { + length.0 = length.0.max(min); + length.1 = length.1.min(max); + } else { + return Err(Error::BadRequest("Invalid policy item".to_owned())); + } + } + } + } + Ok(Conditions { + params, + content_type, + content_length: RangeInclusive::new(length.0, length.1), + }) + } +} + /// A single condition from a policy #[derive(Deserialize)] #[serde(untagged)] @@ -188,11 +297,15 @@ enum PolicyCondition { SizeRange(String, u64, u64), } -#[allow(dead_code)] +struct Conditions { + params: HashMap>, + content_type: Vec, + #[allow(dead_code)] + content_length: RangeInclusive, +} + #[derive(PartialEq, Eq)] enum Operation { - Equal, - StartsWith, - StartsWithCT, - SizeRange, + Equal(String), + StartsWith(String), } -- 2.43.4 From 27485f94b9cdda84f8330d09e4e2fe1c73a52ae4 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Mon, 7 Feb 2022 19:40:16 +0100 Subject: [PATCH 08/14] add length support to post policy --- src/api/s3_post_object.rs | 96 +++++++++++++++++++++++++++++++++------ 1 file changed, 81 insertions(+), 15 deletions(-) diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index 52229303..20b2e13c 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -2,9 +2,11 @@ use std::collections::HashMap; use std::convert::TryInto; use std::ops::RangeInclusive; use std::sync::Arc; +use std::task::{Context, Poll}; +use bytes::Bytes; use chrono::{DateTime, Duration, Utc}; -use futures::StreamExt; +use futures::{Stream, StreamExt}; use hyper::header::{self, HeaderMap, HeaderName, HeaderValue}; use hyper::{Body, Request, Response, StatusCode}; use multer::{Constraints, Multipart, SizeLimit}; @@ -29,7 +31,7 @@ pub async fn handle_post_object( .and_then(|ct| multer::parse_boundary(ct).ok()) .ok_or_bad_request("Counld not get multipart boundary")?; - // 16k seems plenty for a header. 5G is the max size of a single part, so it seemrs reasonable + // 16k seems plenty for a header. 5G is the max size of a single part, so it seems reasonable // for a PostObject let constraints = Constraints::new().size_limit( SizeLimit::new() @@ -119,9 +121,9 @@ pub async fn handle_post_object( let conditions = decoded_policy.into_conditions()?; - for (key, value) in params.iter() { - let key = key.as_str(); - if key.eq_ignore_ascii_case("content-type") { + for (param_key, value) in params.iter() { + let param_key = param_key.as_str(); + if param_key.eq_ignore_ascii_case("content-type") { for cond in &conditions.content_type { let ok = match cond { Operation::Equal(s) => value == s, @@ -132,13 +134,29 @@ pub async fn handle_post_object( if !ok { return Err(Error::BadRequest(format!( "Key '{}' has value not allowed in policy", - key + param_key + ))); + } + } + } else if param_key == "key" { + let conds = conditions.params.get("key").ok_or_else(|| { + Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) + })?; + for cond in conds { + let ok = match cond { + Operation::Equal(s) => s == &key, + Operation::StartsWith(s) => key.starts_with(s), + }; + if !ok { + return Err(Error::BadRequest(format!( + "Key '{}' has value not allowed in policy", + param_key ))); } } } else { - let conds = conditions.params.get(key).ok_or_else(|| { - Error::BadRequest(format!("Key '{}' is not allowed in policy", key)) + let conds = conditions.params.get(param_key).ok_or_else(|| { + Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) })?; for cond in conds { let ok = match cond { @@ -148,16 +166,13 @@ pub async fn handle_post_object( if !ok { return Err(Error::BadRequest(format!( "Key '{}' has value not allowed in policy", - key + param_key ))); } } } } - // TODO validate policy against request - // unsafe to merge until implemented - let content_type = field .content_type() .map(AsRef::as_ref) @@ -169,10 +184,11 @@ pub async fn handle_post_object( params.append(header::CONTENT_TYPE, content_type); let headers = get_headers(¶ms)?; + let stream = field.map(|r| r.map_err(Into::into)); let res = save_stream( garage, headers, - field.map(|r| r.map_err(Into::into)), + StreamLimiter::new(stream, conditions.content_length), bucket_id, &key, None, @@ -183,9 +199,10 @@ pub async fn handle_post_object( let resp = if let Some(target) = params .get("success_action_redirect") .and_then(|h| h.to_str().ok()) + .and_then(|u| url::Url::parse(u).ok()) + .filter(|u| u.scheme() == "https" || u.scheme() == "http") { - // TODO should validate it's a valid url - let target = target.to_owned(); + let target = target.to_string(); Response::builder() .status(StatusCode::SEE_OTHER) .header(header::LOCATION, target.clone()) @@ -309,3 +326,52 @@ enum Operation { Equal(String), StartsWith(String), } + +struct StreamLimiter { + inner: T, + length: RangeInclusive, + read: u64, +} + +impl StreamLimiter { + fn new(stream: T, length: RangeInclusive) -> Self { + StreamLimiter { + inner: stream, + length, + read: 0, + } + } +} + +impl Stream for StreamLimiter +where + T: Stream> + Unpin, +{ + type Item = Result; + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + ctx: &mut Context<'_>, + ) -> Poll> { + let res = std::pin::Pin::new(&mut self.inner).poll_next(ctx); + match &res { + Poll::Ready(Some(Ok(bytes))) => { + self.read += bytes.len() as u64; + // optimization to fail early when we know before the end it's too long + if self.length.end() < &self.read { + return Poll::Ready(Some(Err(Error::BadRequest( + "File size does not match policy".to_owned(), + )))); + } + } + Poll::Ready(None) => { + if !self.length.contains(&self.read) { + return Poll::Ready(Some(Err(Error::BadRequest( + "File size does not match policy".to_owned(), + )))); + } + } + _ => {} + } + res + } +} -- 2.43.4 From 19ac5ce20fd1a0613aacc03966699f6ad33f4e73 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Tue, 8 Feb 2022 22:38:09 +0100 Subject: [PATCH 09/14] answer a bit more like aws and add todos for missing validation and location header --- src/api/s3_post_object.rs | 136 ++++++++++++++++++++++---------------- src/api/s3_put.rs | 9 +-- 2 files changed, 84 insertions(+), 61 deletions(-) diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index 20b2e13c..e5df3021 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -39,7 +39,8 @@ pub async fn handle_post_object( .for_field("file", 5 * 1024 * 1024 * 1024), ); - let mut multipart = Multipart::with_constraints(req.into_body(), boundary, constraints); + let (_head, body) = req.into_parts(); + let mut multipart = Multipart::with_constraints(body, boundary, constraints); let mut params = HeaderMap::new(); while let Some(field) = multipart.next_field().await? { @@ -56,6 +57,7 @@ pub async fn handle_post_object( params.append("x-amz-acl", content); } _ => { + // TODO actually that's illegal to have the same param multiple times params.append(name, content); } } @@ -122,57 +124,65 @@ pub async fn handle_post_object( let conditions = decoded_policy.into_conditions()?; for (param_key, value) in params.iter() { - let param_key = param_key.as_str(); - if param_key.eq_ignore_ascii_case("content-type") { - for cond in &conditions.content_type { - let ok = match cond { - Operation::Equal(s) => value == s, - Operation::StartsWith(s) => { - value.to_str()?.split(',').all(|v| v.starts_with(s)) + let mut param_key = param_key.to_string(); + param_key.make_ascii_lowercase(); + match param_key.as_str() { + "policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields + "content-type" => { + for cond in &conditions.content_type { + let ok = match cond { + Operation::Equal(s) => value == s, + Operation::StartsWith(s) => { + value.to_str()?.split(',').all(|v| v.starts_with(s)) + } + }; + if !ok { + return Err(Error::BadRequest(format!( + "Key '{}' has value not allowed in policy", + param_key + ))); } - }; - if !ok { - return Err(Error::BadRequest(format!( - "Key '{}' has value not allowed in policy", - param_key - ))); } } - } else if param_key == "key" { - let conds = conditions.params.get("key").ok_or_else(|| { - Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) - })?; - for cond in conds { - let ok = match cond { - Operation::Equal(s) => s == &key, - Operation::StartsWith(s) => key.starts_with(s), - }; - if !ok { - return Err(Error::BadRequest(format!( - "Key '{}' has value not allowed in policy", - param_key - ))); + "key" => { + let conds = conditions.params.get("key").ok_or_else(|| { + Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) + })?; + for cond in conds { + let ok = match cond { + Operation::Equal(s) => s == &key, + Operation::StartsWith(s) => key.starts_with(s), + }; + if !ok { + return Err(Error::BadRequest(format!( + "Key '{}' has value not allowed in policy", + param_key + ))); + } } } - } else { - let conds = conditions.params.get(param_key).ok_or_else(|| { - Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) - })?; - for cond in conds { - let ok = match cond { - Operation::Equal(s) => s == value, - Operation::StartsWith(s) => value.to_str()?.starts_with(s), - }; - if !ok { - return Err(Error::BadRequest(format!( - "Key '{}' has value not allowed in policy", - param_key - ))); + _ => { + let conds = conditions.params.get(¶m_key).ok_or_else(|| { + Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) + })?; + for cond in conds { + let ok = match cond { + Operation::Equal(s) => s == value, + Operation::StartsWith(s) => value.to_str()?.starts_with(s), + }; + if !ok { + return Err(Error::BadRequest(format!( + "Key '{}' has value not allowed in policy", + param_key + ))); + } } } } } + // TODO check that each policy item is used + let content_type = field .content_type() .map(AsRef::as_ref) @@ -185,7 +195,7 @@ pub async fn handle_post_object( let headers = get_headers(¶ms)?; let stream = field.map(|r| r.map_err(Into::into)); - let res = save_stream( + let (_, md5) = save_stream( garage, headers, StreamLimiter::new(stream, conditions.content_length), @@ -196,35 +206,45 @@ pub async fn handle_post_object( ) .await?; - let resp = if let Some(target) = params + let etag = format!("\"{}\"", md5); + // TODO get uri + // get Host + // append www-form-urlencoded key + let location = "todo"; + + let resp = if let Some(mut target) = params .get("success_action_redirect") .and_then(|h| h.to_str().ok()) .and_then(|u| url::Url::parse(u).ok()) .filter(|u| u.scheme() == "https" || u.scheme() == "http") { + target + .query_pairs_mut() + .append_pair("bucket", &bucket) + .append_pair("key", &key) + .append_pair("etag", &etag); let target = target.to_string(); Response::builder() .status(StatusCode::SEE_OTHER) .header(header::LOCATION, target.clone()) + .header(header::ETAG, etag) .body(target.into())? } else { let action = params .get("success_action_status") .and_then(|h| h.to_str().ok()) .unwrap_or("204"); + let builder = Response::builder() + .status(StatusCode::OK) + .header(header::LOCATION, location) + .header(header::ETAG, etag); match action { - "200" => Response::builder() - .status(StatusCode::OK) - .body(Body::empty())?, + "200" => builder.status(StatusCode::OK).body(Body::empty())?, "201" => { // TODO body should be an XML document, not sure which yet - Response::builder() - .status(StatusCode::CREATED) - .body(res.into_body())? + builder.status(StatusCode::CREATED).body(Body::from(""))? } - _ => Response::builder() - .status(StatusCode::NO_CONTENT) - .body(Body::empty())?, + _ => builder.status(StatusCode::NO_CONTENT).body(Body::empty())?, } }; @@ -254,8 +274,9 @@ impl Policy { if map.len() != 1 { return Err(Error::BadRequest("Invalid policy item".to_owned())); } - let (k, v) = map.into_iter().next().expect("size was verified"); - if k.eq_ignore_ascii_case("content-type") { + let (mut k, v) = map.into_iter().next().expect("size was verified"); + k.make_ascii_lowercase(); + if k == "content-type" { content_type.push(Operation::Equal(v)); } else { params.entry(k).or_default().push(Operation::Equal(v)); @@ -265,16 +286,17 @@ impl Policy { if key.remove(0) != '$' { return Err(Error::BadRequest("Invalid policy item".to_owned())); } + key.make_ascii_lowercase(); match cond.as_str() { "eq" => { - if key.eq_ignore_ascii_case("content-type") { + if key == "content-type" { content_type.push(Operation::Equal(value)); } else { params.entry(key).or_default().push(Operation::Equal(value)); } } "starts-with" => { - if key.eq_ignore_ascii_case("content-type") { + if key == "content-type" { content_type.push(Operation::StartsWith(value)); } else { params diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index 5af29b79..5735fd10 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -99,6 +99,7 @@ pub async fn handle_put( content_sha256, ) .await + .map(|(uuid, md5)| put_response(uuid, md5)) } pub(crate) async fn save_stream> + Unpin>( @@ -109,7 +110,7 @@ pub(crate) async fn save_stream> + Unpin>( key: &str, content_md5: Option, content_sha256: Option, -) -> Result, Error> { +) -> Result<(Uuid, String), Error> { // Generate identity of new version let version_uuid = gen_uuid(); let version_timestamp = now_msec(); @@ -150,7 +151,7 @@ pub(crate) async fn save_stream> + Unpin>( let object = Object::new(bucket_id, key.into(), vec![object_version]); garage.object_table.insert(&object).await?; - return Ok(put_response(version_uuid, data_md5sum_hex)); + return Ok((version_uuid, data_md5sum_hex)); } // Write version identifier in object table so that we have a trace @@ -216,7 +217,7 @@ pub(crate) async fn save_stream> + Unpin>( let object = Object::new(bucket_id, key.into(), vec![object_version]); garage.object_table.insert(&object).await?; - Ok(put_response(version_uuid, md5sum_hex)) + Ok((version_uuid, md5sum_hex)) } /// Validate MD5 sum against content-md5 header @@ -512,7 +513,7 @@ pub async fn handle_put_part( let response = Response::builder() .header("ETag", format!("\"{}\"", data_md5sum_hex)) - .body(Body::from(vec![])) + .body(Body::empty()) .unwrap(); Ok(response) } -- 2.43.4 From e0aee72a9c8824725e961636d942d37368ddcde0 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Wed, 9 Feb 2022 12:19:04 +0100 Subject: [PATCH 10/14] fix review comments --- src/api/s3_post_object.rs | 371 +++++++++++++++++++------------------- 1 file changed, 187 insertions(+), 184 deletions(-) diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index e5df3021..508cfc31 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -43,7 +43,14 @@ pub async fn handle_post_object( let mut multipart = Multipart::with_constraints(body, boundary, constraints); let mut params = HeaderMap::new(); - while let Some(field) = multipart.next_field().await? { + let field = loop { + let field = if let Some(field) = multipart.next_field().await? { + field + } else { + return Err(Error::BadRequest( + "Request did not contain a file".to_owned(), + )); + }; let name: HeaderName = if let Some(Ok(name)) = field.name().map(TryInto::try_into) { name } else { @@ -62,198 +69,194 @@ pub async fn handle_post_object( } } } - continue; - } - - // Current part is file. Do some checks before handling to PutObject code - let key = params - .get("key") - .ok_or_bad_request("No key was provided")? - .to_str()?; - let credential = params - .get("x-amz-credential") - .ok_or_else(|| { - Error::Forbidden("Garage does not support anonymous access yet".to_string()) - })? - .to_str()?; - let policy = params - .get("policy") - .ok_or_bad_request("No policy was provided")? - .to_str()?; - let signature = params - .get("x-amz-signature") - .ok_or_bad_request("No signature was provided")? - .to_str()?; - let date = params - .get("x-amz-date") - .ok_or_bad_request("No date was provided")? - .to_str()?; - - let key = if key.contains("${filename}") { - let filename = field.file_name(); - // is this correct? Maybe we should error instead of default? - key.replace("${filename}", filename.unwrap_or_default()) } else { - key.to_owned() - }; - - let date = parse_date(date)?; - let api_key = verify_v4(&garage, credential, &date, signature, policy.as_bytes()).await?; - - let bucket_id = resolve_bucket(&garage, &bucket, &api_key).await?; - - if !api_key.allow_write(&bucket_id) { - return Err(Error::Forbidden( - "Operation is not allowed for this key.".to_string(), - )); + break field; } + }; - let decoded_policy = base64::decode(&policy)?; - let decoded_policy: Policy = - serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?; + // Current part is file. Do some checks before handling to PutObject code + let key = params + .get("key") + .ok_or_bad_request("No key was provided")? + .to_str()?; + let credential = params + .get("x-amz-credential") + .ok_or_else(|| { + Error::Forbidden("Garage does not support anonymous access yet".to_string()) + })? + .to_str()?; + let policy = params + .get("policy") + .ok_or_bad_request("No policy was provided")? + .to_str()?; + let signature = params + .get("x-amz-signature") + .ok_or_bad_request("No signature was provided")? + .to_str()?; + let date = params + .get("x-amz-date") + .ok_or_bad_request("No date was provided")? + .to_str()?; - let expiration: DateTime = DateTime::parse_from_rfc3339(&decoded_policy.expiration) - .ok_or_bad_request("Invalid expiration date")? - .into(); - if Utc::now() - expiration > Duration::zero() { - return Err(Error::BadRequest( - "Expiration date is in the paste".to_string(), - )); - } + let key = if key.contains("${filename}") { + let filename = field.file_name(); + // is this correct? Maybe we should error instead of default? + key.replace("${filename}", filename.unwrap_or_default()) + } else { + key.to_owned() + }; - let conditions = decoded_policy.into_conditions()?; + let date = parse_date(date)?; + let api_key = verify_v4(&garage, credential, &date, signature, policy.as_bytes()).await?; - for (param_key, value) in params.iter() { - let mut param_key = param_key.to_string(); - param_key.make_ascii_lowercase(); - match param_key.as_str() { - "policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields - "content-type" => { - for cond in &conditions.content_type { - let ok = match cond { - Operation::Equal(s) => value == s, - Operation::StartsWith(s) => { - value.to_str()?.split(',').all(|v| v.starts_with(s)) - } - }; - if !ok { - return Err(Error::BadRequest(format!( - "Key '{}' has value not allowed in policy", - param_key - ))); - } - } - } - "key" => { - let conds = conditions.params.get("key").ok_or_else(|| { - Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) - })?; - for cond in conds { - let ok = match cond { - Operation::Equal(s) => s == &key, - Operation::StartsWith(s) => key.starts_with(s), - }; - if !ok { - return Err(Error::BadRequest(format!( - "Key '{}' has value not allowed in policy", - param_key - ))); - } - } - } - _ => { - let conds = conditions.params.get(¶m_key).ok_or_else(|| { - Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) - })?; - for cond in conds { - let ok = match cond { - Operation::Equal(s) => s == value, - Operation::StartsWith(s) => value.to_str()?.starts_with(s), - }; - if !ok { - return Err(Error::BadRequest(format!( - "Key '{}' has value not allowed in policy", - param_key - ))); - } - } - } - } - } + let bucket_id = resolve_bucket(&garage, &bucket, &api_key).await?; - // TODO check that each policy item is used - - let content_type = field - .content_type() - .map(AsRef::as_ref) - .map(HeaderValue::from_str) - .transpose() - .ok_or_bad_request("Invalid content type")? - .unwrap_or_else(|| HeaderValue::from_static("blob")); - - params.append(header::CONTENT_TYPE, content_type); - let headers = get_headers(¶ms)?; - - let stream = field.map(|r| r.map_err(Into::into)); - let (_, md5) = save_stream( - garage, - headers, - StreamLimiter::new(stream, conditions.content_length), - bucket_id, - &key, - None, - None, - ) - .await?; - - let etag = format!("\"{}\"", md5); - // TODO get uri - // get Host - // append www-form-urlencoded key - let location = "todo"; - - let resp = if let Some(mut target) = params - .get("success_action_redirect") - .and_then(|h| h.to_str().ok()) - .and_then(|u| url::Url::parse(u).ok()) - .filter(|u| u.scheme() == "https" || u.scheme() == "http") - { - target - .query_pairs_mut() - .append_pair("bucket", &bucket) - .append_pair("key", &key) - .append_pair("etag", &etag); - let target = target.to_string(); - Response::builder() - .status(StatusCode::SEE_OTHER) - .header(header::LOCATION, target.clone()) - .header(header::ETAG, etag) - .body(target.into())? - } else { - let action = params - .get("success_action_status") - .and_then(|h| h.to_str().ok()) - .unwrap_or("204"); - let builder = Response::builder() - .status(StatusCode::OK) - .header(header::LOCATION, location) - .header(header::ETAG, etag); - match action { - "200" => builder.status(StatusCode::OK).body(Body::empty())?, - "201" => { - // TODO body should be an XML document, not sure which yet - builder.status(StatusCode::CREATED).body(Body::from(""))? - } - _ => builder.status(StatusCode::NO_CONTENT).body(Body::empty())?, - } - }; - - return Ok(resp); + if !api_key.allow_write(&bucket_id) { + return Err(Error::Forbidden( + "Operation is not allowed for this key.".to_string(), + )); } - Err(Error::BadRequest( - "Request did not contain a file".to_owned(), - )) + let decoded_policy = base64::decode(&policy)?; + let decoded_policy: Policy = + serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?; + + let expiration: DateTime = DateTime::parse_from_rfc3339(&decoded_policy.expiration) + .ok_or_bad_request("Invalid expiration date")? + .into(); + if Utc::now() - expiration > Duration::zero() { + return Err(Error::BadRequest( + "Expiration date is in the paste".to_string(), + )); + } + + let conditions = decoded_policy.into_conditions()?; + + for (param_key, value) in params.iter() { + let mut param_key = param_key.to_string(); + param_key.make_ascii_lowercase(); + match param_key.as_str() { + "policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields + "content-type" => { + for cond in &conditions.content_type { + let ok = match cond { + Operation::Equal(s) => value == s, + Operation::StartsWith(s) => { + value.to_str()?.split(',').all(|v| v.starts_with(s)) + } + }; + if !ok { + return Err(Error::BadRequest(format!( + "Key '{}' has value not allowed in policy", + param_key + ))); + } + } + } + "key" => { + let conds = conditions.params.get("key").ok_or_else(|| { + Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) + })?; + for cond in conds { + let ok = match cond { + Operation::Equal(s) => s == &key, + Operation::StartsWith(s) => key.starts_with(s), + }; + if !ok { + return Err(Error::BadRequest(format!( + "Key '{}' has value not allowed in policy", + param_key + ))); + } + } + } + _ => { + let conds = conditions.params.get(¶m_key).ok_or_else(|| { + Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) + })?; + for cond in conds { + let ok = match cond { + Operation::Equal(s) => s == value, + Operation::StartsWith(s) => value.to_str()?.starts_with(s), + }; + if !ok { + return Err(Error::BadRequest(format!( + "Key '{}' has value not allowed in policy", + param_key + ))); + } + } + } + } + } + + // TODO check that each policy item is used + + let content_type = field + .content_type() + .map(AsRef::as_ref) + .map(HeaderValue::from_str) + .transpose() + .ok_or_bad_request("Invalid content type")? + .unwrap_or_else(|| HeaderValue::from_static("blob")); + + params.append(header::CONTENT_TYPE, content_type); + let headers = get_headers(¶ms)?; + + let stream = field.map(|r| r.map_err(Into::into)); + let (_, md5) = save_stream( + garage, + headers, + StreamLimiter::new(stream, conditions.content_length), + bucket_id, + &key, + None, + None, + ) + .await?; + + let etag = format!("\"{}\"", md5); + // TODO get uri + // get Host + // append www-form-urlencoded key + let location = "todo"; + + let resp = if let Some(mut target) = params + .get("success_action_redirect") + .and_then(|h| h.to_str().ok()) + .and_then(|u| url::Url::parse(u).ok()) + .filter(|u| u.scheme() == "https" || u.scheme() == "http") + { + target + .query_pairs_mut() + .append_pair("bucket", &bucket) + .append_pair("key", &key) + .append_pair("etag", &etag); + let target = target.to_string(); + Response::builder() + .status(StatusCode::SEE_OTHER) + .header(header::LOCATION, target.clone()) + .header(header::ETAG, etag) + .body(target.into())? + } else { + let action = params + .get("success_action_status") + .and_then(|h| h.to_str().ok()) + .unwrap_or("204"); + let builder = Response::builder() + .header(header::LOCATION, location) + .header(header::ETAG, etag); + match action { + "200" => builder.status(StatusCode::OK).body(Body::empty())?, + "201" => { + // TODO body should be an XML document, not sure which yet + builder.status(StatusCode::CREATED).body(Body::from(""))? + } + _ => builder.status(StatusCode::NO_CONTENT).body(Body::empty())?, + } + }; + + Ok(resp) } #[derive(Deserialize)] -- 2.43.4 From 63948190e4d26de95892a8fae31ba14d745644a1 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Wed, 9 Feb 2022 12:41:13 +0100 Subject: [PATCH 11/14] stricter policy validation --- src/api/s3_post_object.rs | 53 ++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 20 deletions(-) diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index 508cfc31..1bbc1bf0 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -56,21 +56,29 @@ pub async fn handle_post_object( } else { continue; }; - if name != "file" { - if let Ok(content) = HeaderValue::from_str(&field.text().await?) { - match name.as_str() { - "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */), - "acl" => { - params.append("x-amz-acl", content); + if name == "file" { + break field; + } + + if let Ok(content) = HeaderValue::from_str(&field.text().await?) { + match name.as_str() { + "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */), + "acl" => { + if params.insert("x-amz-acl", content).is_some() { + return Err(Error::BadRequest( + "Field 'acl' provided more than one time".to_string(), + )); } - _ => { - // TODO actually that's illegal to have the same param multiple times - params.append(name, content); + } + _ => { + if params.insert(&name, content).is_some() { + return Err(Error::BadRequest(format!( + "Field '{}' provided more than one time", + name + ))); } } } - } else { - break field; } }; @@ -100,7 +108,7 @@ pub async fn handle_post_object( let key = if key.contains("${filename}") { let filename = field.file_name(); - // is this correct? Maybe we should error instead of default? + // TODO is this correct? Maybe we should error instead of default? key.replace("${filename}", filename.unwrap_or_default()) } else { key.to_owned() @@ -130,7 +138,7 @@ pub async fn handle_post_object( )); } - let conditions = decoded_policy.into_conditions()?; + let mut conditions = decoded_policy.into_conditions()?; for (param_key, value) in params.iter() { let mut param_key = param_key.to_string(); @@ -154,13 +162,13 @@ pub async fn handle_post_object( } } "key" => { - let conds = conditions.params.get("key").ok_or_else(|| { + let conds = conditions.params.remove("key").ok_or_else(|| { Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) })?; for cond in conds { let ok = match cond { - Operation::Equal(s) => s == &key, - Operation::StartsWith(s) => key.starts_with(s), + Operation::Equal(s) => s == key, + Operation::StartsWith(s) => key.starts_with(&s), }; if !ok { return Err(Error::BadRequest(format!( @@ -171,13 +179,13 @@ pub async fn handle_post_object( } } _ => { - let conds = conditions.params.get(¶m_key).ok_or_else(|| { + let conds = conditions.params.remove(¶m_key).ok_or_else(|| { Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) })?; for cond in conds { let ok = match cond { - Operation::Equal(s) => s == value, - Operation::StartsWith(s) => value.to_str()?.starts_with(s), + Operation::Equal(s) => s.as_str() == value, + Operation::StartsWith(s) => value.to_str()?.starts_with(s.as_str()), }; if !ok { return Err(Error::BadRequest(format!( @@ -190,7 +198,12 @@ pub async fn handle_post_object( } } - // TODO check that each policy item is used + if let Some((param_key, _)) = conditions.params.iter().next() { + return Err(Error::BadRequest(format!( + "Key '{}' is required in policy, but no value was provided", + param_key + ))); + } let content_type = field .content_type() -- 2.43.4 From 93637d40ecbc6017720d658764cf8801adbad799 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Sat, 12 Feb 2022 00:09:19 +0100 Subject: [PATCH 12/14] correct location, response xml and behavior on missing filename --- Cargo.lock | 1 + src/api/Cargo.toml | 1 + src/api/s3_post_object.rs | 54 ++++++++++++++++++++++++++++++--------- src/api/s3_xml.rs | 14 ++++++++++ 4 files changed, 58 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d713012d..84ab24ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -699,6 +699,7 @@ dependencies = [ "chrono", "crypto-mac 0.10.1", "err-derive 0.3.0", + "form_urlencoded", "futures", "futures-util", "garage_model 0.6.0", diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 51fd7ae0..cc9635bb 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -36,6 +36,7 @@ futures-util = "0.3" pin-project = "1.0" tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +form_urlencoded = "1.0.0" http = "0.2" httpdate = "0.3" http-range = "0.1" diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index 1bbc1bf0..93965685 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -17,6 +17,7 @@ use garage_model::garage::Garage; use crate::api_server::resolve_bucket; use crate::error::*; use crate::s3_put::{get_headers, save_stream}; +use crate::s3_xml; use crate::signature::payload::{parse_date, verify_v4}; pub async fn handle_post_object( @@ -39,7 +40,7 @@ pub async fn handle_post_object( .for_field("file", 5 * 1024 * 1024 * 1024), ); - let (_head, body) = req.into_parts(); + let (head, body) = req.into_parts(); let mut multipart = Multipart::with_constraints(body, boundary, constraints); let mut params = HeaderMap::new(); @@ -107,9 +108,11 @@ pub async fn handle_post_object( .to_str()?; let key = if key.contains("${filename}") { - let filename = field.file_name(); - // TODO is this correct? Maybe we should error instead of default? - key.replace("${filename}", filename.unwrap_or_default()) + if let Some(filename) = field.file_name() { + key.replace("${filename}", filename) + } else { + key.to_owned() + } } else { key.to_owned() }; @@ -229,10 +232,6 @@ pub async fn handle_post_object( .await?; let etag = format!("\"{}\"", md5); - // TODO get uri - // get Host - // append www-form-urlencoded key - let location = "todo"; let resp = if let Some(mut target) = params .get("success_action_redirect") @@ -252,18 +251,49 @@ pub async fn handle_post_object( .header(header::ETAG, etag) .body(target.into())? } else { + let path = head + .uri + .into_parts() + .path_and_query + .map(|paq| paq.path().to_string()) + .unwrap_or_else(|| "/".to_string()); + let authority = head + .headers + .get(header::HOST) + .and_then(|h| h.to_str().ok()) + .unwrap_or_default(); + let proto = if !authority.is_empty() { + "https://" + } else { + "" + }; + + let url_key: String = form_urlencoded::byte_serialize(key.as_bytes()) + .flat_map(str::chars) + .collect(); + let location = format!("{}{}{}{}", proto, authority, path, url_key); + let action = params .get("success_action_status") .and_then(|h| h.to_str().ok()) .unwrap_or("204"); let builder = Response::builder() - .header(header::LOCATION, location) - .header(header::ETAG, etag); + .header(header::LOCATION, location.clone()) + .header(header::ETAG, etag.clone()); match action { "200" => builder.status(StatusCode::OK).body(Body::empty())?, "201" => { - // TODO body should be an XML document, not sure which yet - builder.status(StatusCode::CREATED).body(Body::from(""))? + let xml = s3_xml::PostObject { + xmlns: (), + location: s3_xml::Value(location), + bucket: s3_xml::Value(bucket), + key: s3_xml::Value(key), + etag: s3_xml::Value(etag), + }; + let body = s3_xml::to_xml_with_header(&xml)?; + builder + .status(StatusCode::CREATED) + .body(Body::from(body.into_bytes()))? } _ => builder.status(StatusCode::NO_CONTENT).body(Body::empty())?, } diff --git a/src/api/s3_xml.rs b/src/api/s3_xml.rs index 8a0dcee0..75ec4559 100644 --- a/src/api/s3_xml.rs +++ b/src/api/s3_xml.rs @@ -289,6 +289,20 @@ pub struct VersioningConfiguration { pub status: Option, } +#[derive(Debug, Serialize, PartialEq)] +pub struct PostObject { + #[serde(serialize_with = "xmlns_tag")] + pub xmlns: (), + #[serde(rename = "Location")] + pub location: Value, + #[serde(rename = "Bucket")] + pub bucket: Value, + #[serde(rename = "Key")] + pub key: Value, + #[serde(rename = "ETag")] + pub etag: Value, +} + #[cfg(test)] mod tests { use super::*; -- 2.43.4 From fe9210f07173913d8f5b4270e7e030a3186e1c61 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Wed, 16 Feb 2022 23:27:30 +0100 Subject: [PATCH 13/14] address review comments --- src/api/s3_post_object.rs | 126 +++++++++++++++++++++++++++----------- 1 file changed, 90 insertions(+), 36 deletions(-) diff --git a/src/api/s3_post_object.rs b/src/api/s3_post_object.rs index 93965685..585e0304 100644 --- a/src/api/s3_post_object.rs +++ b/src/api/s3_post_object.rs @@ -108,6 +108,7 @@ pub async fn handle_post_object( .to_str()?; let key = if key.contains("${filename}") { + // if no filename is provided, don't replace. This matches the behavior of AWS. if let Some(filename) = field.file_name() { key.replace("${filename}", filename) } else { @@ -149,11 +150,14 @@ pub async fn handle_post_object( match param_key.as_str() { "policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields "content-type" => { - for cond in &conditions.content_type { + let conds = conditions.params.remove("content-type").ok_or_else(|| { + Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) + })?; + for cond in conds { let ok = match cond { - Operation::Equal(s) => value == s, + Operation::Equal(s) => s.as_str() == value, Operation::StartsWith(s) => { - value.to_str()?.split(',').all(|v| v.starts_with(s)) + value.to_str()?.split(',').all(|v| v.starts_with(&s)) } }; if !ok { @@ -182,6 +186,12 @@ pub async fn handle_post_object( } } _ => { + if param_key.starts_with("x-ignore-") { + // if a x-ignore is provided in policy, it's not removed here, so it will be + // rejected as provided in policy but not in the request. As odd as it is, it's + // how aws seems to behave. + continue; + } let conds = conditions.params.remove(¶m_key).ok_or_else(|| { Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) })?; @@ -208,15 +218,6 @@ pub async fn handle_post_object( ))); } - let content_type = field - .content_type() - .map(AsRef::as_ref) - .map(HeaderValue::from_str) - .transpose() - .ok_or_bad_request("Invalid content type")? - .unwrap_or_else(|| HeaderValue::from_static("blob")); - - params.append(header::CONTENT_TYPE, content_type); let headers = get_headers(¶ms)?; let stream = field.map(|r| r.map_err(Into::into)); @@ -311,7 +312,6 @@ struct Policy { impl Policy { fn into_conditions(self) -> Result { let mut params = HashMap::<_, Vec<_>>::new(); - let mut content_type = Vec::new(); let mut length = (0, u64::MAX); for condition in self.conditions { @@ -322,11 +322,7 @@ impl Policy { } let (mut k, v) = map.into_iter().next().expect("size was verified"); k.make_ascii_lowercase(); - if k == "content-type" { - content_type.push(Operation::Equal(v)); - } else { - params.entry(k).or_default().push(Operation::Equal(v)); - } + params.entry(k).or_default().push(Operation::Equal(v)); } PolicyCondition::OtherOp([cond, mut key, value]) => { if key.remove(0) != '$' { @@ -335,21 +331,13 @@ impl Policy { key.make_ascii_lowercase(); match cond.as_str() { "eq" => { - if key == "content-type" { - content_type.push(Operation::Equal(value)); - } else { - params.entry(key).or_default().push(Operation::Equal(value)); - } + params.entry(key).or_default().push(Operation::Equal(value)); } "starts-with" => { - if key == "content-type" { - content_type.push(Operation::StartsWith(value)); - } else { - params - .entry(key) - .or_default() - .push(Operation::StartsWith(value)); - } + params + .entry(key) + .or_default() + .push(Operation::StartsWith(value)); } _ => return Err(Error::BadRequest("Invalid policy item".to_owned())), } @@ -366,14 +354,13 @@ impl Policy { } Ok(Conditions { params, - content_type, content_length: RangeInclusive::new(length.0, length.1), }) } } /// A single condition from a policy -#[derive(Deserialize)] +#[derive(Debug, Deserialize)] #[serde(untagged)] enum PolicyCondition { // will contain a single key-value pair @@ -382,14 +369,13 @@ enum PolicyCondition { SizeRange(String, u64, u64), } +#[derive(Debug)] struct Conditions { params: HashMap>, - content_type: Vec, - #[allow(dead_code)] content_length: RangeInclusive, } -#[derive(PartialEq, Eq)] +#[derive(Debug, PartialEq, Eq)] enum Operation { Equal(String), StartsWith(String), @@ -443,3 +429,71 @@ where res } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_policy_1() { + let policy_json = br#" +{ "expiration": "2007-12-01T12:00:00.000Z", + "conditions": [ + {"acl": "public-read" }, + {"bucket": "johnsmith" }, + ["starts-with", "$key", "user/eric/"] + ] +} + "#; + let policy_2: Policy = serde_json::from_slice(&policy_json[..]).unwrap(); + let mut conditions = policy_2.into_conditions().unwrap(); + + assert_eq!( + conditions.params.remove(&"acl".to_string()), + Some(vec![Operation::Equal("public-read".into())]) + ); + assert_eq!( + conditions.params.remove(&"bucket".to_string()), + Some(vec![Operation::Equal("johnsmith".into())]) + ); + assert_eq!( + conditions.params.remove(&"key".to_string()), + Some(vec![Operation::StartsWith("user/eric/".into())]) + ); + assert!(conditions.params.is_empty()); + assert_eq!(conditions.content_length, 0..=u64::MAX); + } + + #[test] + fn test_policy_2() { + let policy_json = br#" +{ "expiration": "2007-12-01T12:00:00.000Z", + "conditions": [ + [ "eq", "$acl", "public-read" ], + ["starts-with", "$Content-Type", "image/"], + ["starts-with", "$success_action_redirect", ""], + ["content-length-range", 1048576, 10485760] + ] +} + "#; + let policy_2: Policy = serde_json::from_slice(&policy_json[..]).unwrap(); + let mut conditions = policy_2.into_conditions().unwrap(); + + assert_eq!( + conditions.params.remove(&"acl".to_string()), + Some(vec![Operation::Equal("public-read".into())]) + ); + assert_eq!( + conditions.params.remove("content-type").unwrap(), + vec![Operation::StartsWith("image/".into())] + ); + assert_eq!( + conditions + .params + .remove(&"success_action_redirect".to_string()), + Some(vec![Operation::StartsWith("".into())]) + ); + assert!(conditions.params.is_empty()); + assert_eq!(conditions.content_length, 1048576..=10485760); + } +} -- 2.43.4 From 32864bb8f3a62bad4bc8b14249b43839cd7f2e08 Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Wed, 16 Feb 2022 23:42:48 +0100 Subject: [PATCH 14/14] re-update cargo.nix --- Cargo.nix | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index fef82575..8855e196 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -560,7 +560,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; }; dependencies = { - ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.103" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.115" { inherit profileName; }; }; }); @@ -672,6 +672,8 @@ in [ "default" ] [ "use_std" ] ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".encoding_rs."0.8.30" = overridableMkRustCrate (profileName: rec { name = "encoding_rs"; version = "0.8.30"; @@ -1012,6 +1014,7 @@ in chrono = rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.19" { inherit profileName; }; crypto_mac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crypto-mac."0.10.1" { inherit profileName; }; err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.0" { profileName = "__noProfile"; }; + form_urlencoded = rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.17" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.17" { inherit profileName; }; garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; @@ -2680,6 +2683,17 @@ in src = fetchCratesIo { inherit name version; sha256 = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"; }; }); + "registry+https://github.com/rust-lang/crates.io-index".spin."0.9.2" = overridableMkRustCrate (profileName: rec { + name = "spin"; + version = "0.9.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"; }; + features = builtins.concatLists [ + [ "mutex" ] + [ "spin_mutex" ] + ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.2" = overridableMkRustCrate (profileName: rec { name = "static_init"; version = "1.0.2"; @@ -2715,16 +2729,6 @@ in buildDependencies = { cfg_aliases = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg_aliases."0.1.1" { profileName = "__noProfile"; }; }; - - "registry+https://github.com/rust-lang/crates.io-index".spin."0.9.2" = overridableMkRustCrate (profileName: rec { - name = "spin"; - version = "0.9.2"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"; }; - features = builtins.concatLists [ - [ "mutex" ] - [ "spin_mutex" ] - ]; }); "registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.23" = overridableMkRustCrate (profileName: rec { -- 2.43.4