Deuxfleurs/garage
46
56
Fork 73
Code Issues 100 Pull requests 14 Releases 30 Wiki Activity

Support for PostObject #222

Merged
lx merged 14 commits from trinity-1686a/garage:post-object into main 2022-02-21 22:02:31 +00:00
Conversation 1 Commits 14 Files changed 12 +84 -61
2 changed files with 84 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 19ac5ce20f - Show all commits

64
src/api/s3_post_object.rs
View file

@ -39,7 +39,8 @@ pub async fn handle_post_object(
.for_field("file", 5 * 1024 * 1024 * 1024), .for_field("file", 5 * 1024 * 1024 * 1024),
); );
let mut multipart = Multipart::with_constraints(req.into_body(), boundary, constraints); let (_head, body) = req.into_parts();
let mut multipart = Multipart::with_constraints(body, boundary, constraints);
let mut params = HeaderMap::new(); let mut params = HeaderMap::new();
while let Some(field) = multipart.next_field().await? { while let Some(field) = multipart.next_field().await? {
trinity-1686a marked this conversation as resolved Outdated
lx commented 2022-02-08 22:05:42 +00:00
Outdated
Review
Copy link

I think we can avoid putting almost the entire code of this function in the while loop (and remove 1 indentation level almost everywhere) by doing something like this:

let file = loop {
    let field = match multipart.next_field().await? {
        None => return Err(no file field found),
        Some(x) => x,
    };
    let name = ...;
    if name == "file" {
         break field;
    }
    // here handle header field adding it to the headermap
};
// here do all of the rest of the processing once we have all headers and are now reading the file body

This looks much nicer to me, especially as in the current version we have a for inside the while, which looks a bit like a nested loop but is in fact not at all.

I think we can avoid putting almost the entire code of this function in the `while` loop (and remove 1 indentation level almost everywhere) by doing something like this: ```rust let file = loop { let field = match multipart.next_field().await? { None => return Err(no file field found), Some(x) => x, }; let name = ...; if name == "file" { break field; } // here handle header field adding it to the headermap }; // here do all of the rest of the processing once we have all headers and are now reading the file body ``` This looks much nicer to me, especially as in the current version we have a `for` inside the `while`, which looks a bit like a nested loop but is in fact not at all.
@ -56,6 +57,7 @@ pub async fn handle_post_object(
params.append("x-amz-acl", content); params.append("x-amz-acl", content);
} }
_ => { _ => {
// TODO actually that's illegal to have the same param multiple times
params.append(name, content); params.append(name, content);
} }
} }
@ -122,8 +124,11 @@ pub async fn handle_post_object(
let conditions = decoded_policy.into_conditions()?; let conditions = decoded_policy.into_conditions()?;
for (param_key, value) in params.iter() { for (param_key, value) in params.iter() {
let param_key = param_key.as_str(); let mut param_key = param_key.to_string();
if param_key.eq_ignore_ascii_case("content-type") { param_key.make_ascii_lowercase();
match param_key.as_str() {
"policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields
"content-type" => {
for cond in &conditions.content_type { for cond in &conditions.content_type {
let ok = match cond { let ok = match cond {
Operation::Equal(s) => value == s, Operation::Equal(s) => value == s,
@ -138,7 +143,8 @@ pub async fn handle_post_object(
))); )));
} }
lx marked this conversation as resolved Outdated
lx commented 2022-02-14 16:31:39 +00:00
Outdated
Review
Copy link

I feel like we should add unit tests for the policy decoding logic (not just for into_conditions but end-to-end starting with JSON)

I feel like we should add unit tests for the policy decoding logic (not just for into_conditions but end-to-end starting with JSON)
lx commented 2022-02-14 16:54:03 +00:00
Outdated
Review
Copy link

I'll write a test and post it in the comments of the PR so that you can copy and paste

I'll write a test and post it in the comments of the PR so that you can copy and paste
} }
} else if param_key == "key" { }
"key" => {
let conds = conditions.params.get("key").ok_or_else(|| { let conds = conditions.params.get("key").ok_or_else(|| {
Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key))
})?; })?;
@ -154,8 +160,9 @@ pub async fn handle_post_object(
))); )));
} }
} }
} else { }
let conds = conditions.params.get(param_key).ok_or_else(|| { _ => {
let conds = conditions.params.get(&param_key).ok_or_else(|| {
Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key))
})?; })?;
for cond in conds { for cond in conds {
@ -172,6 +179,9 @@ pub async fn handle_post_object(
} }
} }
} }
}
// TODO check that each policy item is used
let content_type = field let content_type = field
.content_type() .content_type()
trinity-1686a marked this conversation as resolved Outdated
lx commented 2022-02-14 16:45:36 +00:00
Outdated
Review
Copy link

Same here, can you point me to the doc which says a policy must be given for all fields?

Same here, can you point me to the doc which says a policy must be given for all fields?
trinity-1686a commented 2022-02-15 17:18:40 +00:00
Outdated
Review
Copy link

see comment on key

see comment on `key`
@ -185,7 +195,7 @@ pub async fn handle_post_object(
let headers = get_headers(&params)?; let headers = get_headers(&params)?;
let stream = field.map(|r| r.map_err(Into::into)); let stream = field.map(|r| r.map_err(Into::into));
let res = save_stream( let (_, md5) = save_stream(
garage, garage,
headers, headers,
StreamLimiter::new(stream, conditions.content_length), StreamLimiter::new(stream, conditions.content_length),
@ -196,35 +206,45 @@ pub async fn handle_post_object(
) )
.await?; .await?;
let resp = if let Some(target) = params let etag = format!("\"{}\"", md5);
// TODO get uri
// get Host
// append www-form-urlencoded key
let location = "todo";
let resp = if let Some(mut target) = params
.get("success_action_redirect") .get("success_action_redirect")
.and_then(|h| h.to_str().ok()) .and_then(|h| h.to_str().ok())
.and_then(|u| url::Url::parse(u).ok()) .and_then(|u| url::Url::parse(u).ok())
.filter(|u| u.scheme() == "https" || u.scheme() == "http") .filter(|u| u.scheme() == "https" || u.scheme() == "http")
lx marked this conversation as resolved Outdated
lx commented 2022-02-14 16:38:46 +00:00
Outdated
Review
Copy link

This definitely looks like it should have been done before the policy check

This definitely looks like it should have been done before the policy check
👍 1
trinity-1686a commented 2022-02-16 22:25:27 +00:00
Outdated
Review
Copy link

turns out AWS ignore this, and only consider content type set in what I called param, not in the field metadata

turns out AWS ignore this, and only consider content type set in what I called param, not in the field metadata
{ {
trinity-1686a commented 2022-02-07 18:51:55 +00:00
Outdated
Review
Copy link

the actual response should be

<PostResponse>
    <Location>https://bucketname.garage.tld/key</Location>
    <Bucket>bucketname</Bucket>
    <Key>key</Key>
    <ETag>"0123456789abcdef0123456789abcdef"</ETag>
</PostResponse>

with corresponding etag and location http headers (these headers are also here for 200 and 204, but not the body)

When using success_action_redirect, etag is set as usual, and location is set to ${success_action_redirect}?bucket=bucketname&key=key&etag=%220123456789abcdef0123456789abcdef%22

the actual response should be ```xml <PostResponse> <Location>https://bucketname.garage.tld/key</Location> <Bucket>bucketname</Bucket> <Key>key</Key> <ETag>"0123456789abcdef0123456789abcdef"</ETag> </PostResponse> ``` with corresponding `etag` and `location` http headers (these headers are also here for 200 and 204, but not the body) When using `success_action_redirect`, `etag` is set as usual, and location is set to `${success_action_redirect}?bucket=bucketname&key=key&etag=%220123456789abcdef0123456789abcdef%22`
target
.query_pairs_mut()
.append_pair("bucket", &bucket)
.append_pair("key", &key)
.append_pair("etag", &etag);
let target = target.to_string(); let target = target.to_string();
Response::builder() Response::builder()
.status(StatusCode::SEE_OTHER) .status(StatusCode::SEE_OTHER)
.header(header::LOCATION, target.clone()) .header(header::LOCATION, target.clone())
.header(header::ETAG, etag)
.body(target.into())? .body(target.into())?
} else { } else {
let action = params let action = params
.get("success_action_status") .get("success_action_status")
.and_then(|h| h.to_str().ok()) .and_then(|h| h.to_str().ok())
.unwrap_or("204"); .unwrap_or("204");
match action { let builder = Response::builder()
"200" => Response::builder()
.status(StatusCode::OK) .status(StatusCode::OK)
trinity-1686a marked this conversation as resolved Outdated
lx commented 2022-02-08 22:09:52 +00:00
Outdated
Review
Copy link

Looks like we don't need this .status() as we are calling it in all branches below

Looks like we don't need this `.status()` as we are calling it in all branches below
.body(Body::empty())?, .header(header::LOCATION, location)
.header(header::ETAG, etag);
match action {
"200" => builder.status(StatusCode::OK).body(Body::empty())?,
"201" => { "201" => {
// TODO body should be an XML document, not sure which yet // TODO body should be an XML document, not sure which yet
Response::builder() builder.status(StatusCode::CREATED).body(Body::from(""))?
.status(StatusCode::CREATED)
.body(res.into_body())?
} }
_ => Response::builder() _ => builder.status(StatusCode::NO_CONTENT).body(Body::empty())?,
.status(StatusCode::NO_CONTENT)
.body(Body::empty())?,
} }
}; };
@ -254,8 +274,9 @@ impl Policy {
if map.len() != 1 { if map.len() != 1 {
return Err(Error::BadRequest("Invalid policy item".to_owned())); return Err(Error::BadRequest("Invalid policy item".to_owned()));
} }
let (k, v) = map.into_iter().next().expect("size was verified"); let (mut k, v) = map.into_iter().next().expect("size was verified");
if k.eq_ignore_ascii_case("content-type") { k.make_ascii_lowercase();
if k == "content-type" {
content_type.push(Operation::Equal(v)); content_type.push(Operation::Equal(v));
} else { } else {
params.entry(k).or_default().push(Operation::Equal(v)); params.entry(k).or_default().push(Operation::Equal(v));
@ -265,16 +286,17 @@ impl Policy {
if key.remove(0) != '$' { if key.remove(0) != '$' {
return Err(Error::BadRequest("Invalid policy item".to_owned())); return Err(Error::BadRequest("Invalid policy item".to_owned()));
} }
key.make_ascii_lowercase();
match cond.as_str() { match cond.as_str() {
"eq" => { "eq" => {
if key.eq_ignore_ascii_case("content-type") { if key == "content-type" {
content_type.push(Operation::Equal(value)); content_type.push(Operation::Equal(value));
} else { } else {
params.entry(key).or_default().push(Operation::Equal(value)); params.entry(key).or_default().push(Operation::Equal(value));
} }
} }
"starts-with" => { "starts-with" => {
if key.eq_ignore_ascii_case("content-type") { if key == "content-type" {
content_type.push(Operation::StartsWith(value)); content_type.push(Operation::StartsWith(value));
} else { } else {
params params

9
src/api/s3_put.rs
View file

@ -99,6 +99,7 @@ pub async fn handle_put(
content_sha256, content_sha256,
) )
.await .await
.map(|(uuid, md5)| put_response(uuid, md5))
} }
pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>( pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
@ -109,7 +110,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
key: &str, key: &str,
content_md5: Option<String>, content_md5: Option<String>,
content_sha256: Option<FixedBytes32>, content_sha256: Option<FixedBytes32>,
) -> Result<Response<Body>, Error> { ) -> Result<(Uuid, String), Error> {
// Generate identity of new version // Generate identity of new version
let version_uuid = gen_uuid(); let version_uuid = gen_uuid();
let version_timestamp = now_msec(); let version_timestamp = now_msec();
@ -150,7 +151,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
let object = Object::new(bucket_id, key.into(), vec![object_version]); let object = Object::new(bucket_id, key.into(), vec![object_version]);
garage.object_table.insert(&object).await?; garage.object_table.insert(&object).await?;
return Ok(put_response(version_uuid, data_md5sum_hex)); return Ok((version_uuid, data_md5sum_hex));
} }
// Write version identifier in object table so that we have a trace // Write version identifier in object table so that we have a trace
@ -216,7 +217,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
let object = Object::new(bucket_id, key.into(), vec![object_version]); let object = Object::new(bucket_id, key.into(), vec![object_version]);
garage.object_table.insert(&object).await?; garage.object_table.insert(&object).await?;
Ok(put_response(version_uuid, md5sum_hex)) Ok((version_uuid, md5sum_hex))
} }
/// Validate MD5 sum against content-md5 header /// Validate MD5 sum against content-md5 header
@ -512,7 +513,7 @@ pub async fn handle_put_part(
let response = Response::builder() let response = Response::builder()
.header("ETag", format!("\"{}\"", data_md5sum_hex)) .header("ETag", format!("\"{}\"", data_md5sum_hex))
.body(Body::from(vec![])) .body(Body::empty())
.unwrap(); .unwrap();
Ok(response) Ok(response)
} }