From d867bbcfb513761c6709347509b8438cfdf408c7 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 1 May 2020 15:52:35 +0000 Subject: [PATCH] Implement DeleteObjects --- Cargo.lock | 17 +++++++ src/api/Cargo.toml | 2 +- src/api/api_server.rs | 28 +++++++---- src/api/s3_delete.rs | 108 ++++++++++++++++++++++++++++++++++++++---- src/api/s3_list.rs | 19 +++++--- src/util/Cargo.toml | 1 + src/util/error.rs | 12 +++++ 7 files changed, 162 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7c86808..97e05222 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -375,6 +375,7 @@ dependencies = [ "hyper 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "url 2.1.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -462,6 +463,7 @@ dependencies = [ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", + "roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1014,6 +1016,14 @@ dependencies = [ "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "roxmltree" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "xmlparser 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "rustls" version = "0.17.0" @@ -1489,6 +1499,11 @@ dependencies = [ "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "xmlparser" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [metadata] "checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" "checksum arc-swap 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62" @@ -1594,6 +1609,7 @@ dependencies = [ "checksum ring 0.16.12 (registry+https://github.com/rust-lang/crates.io-index)" = "1ba5a8ec64ee89a76c98c549af81ff14813df09c3e6dc4766c3856da48597a0c" "checksum rmp 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)" = "0f10b46df14cf1ee1ac7baa4d2fbc2c52c0622a4b82fa8740e37bc452ac0184f" "checksum rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)" = "4c1ee98f14fe8b8e9c5ea13d25da7b2a1796169202c57a09d7288de90d56222b" +"checksum roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d5001f134077069d87f77c8b9452b690df2445f7a43f1c7ca4a1af8dd505789d" "checksum rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" "checksum rustversion 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b3bba175698996010c4f6dce5e7f173b6eb781fce25d2cfc45e27091ce0b79f6" "checksum ryu 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "535622e6be132bccd223f4bb2b8ac8d53cda3c7a6394944d3b2b33fb974f9d76" @@ -1651,3 +1667,4 @@ dependencies = [ "checksum winapi-util 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" "checksum winapi-x86_64-pc-windows-gnu 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" "checksum ws2_32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +"checksum xmlparser 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)" = "ccb4240203dadf40be2de9369e5c6dec1bf427528115b030baca3334c18362d7" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 11a42be9..b8674fba 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -31,4 +31,4 @@ hyper = "0.13" url = "2.1" httpdate = "0.3" percent-encoding = "2.1.0" - +roxmltree = "0.11" diff --git a/src/api/api_server.rs b/src/api/api_server.rs index 642697da..913ff0af 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -74,8 +74,7 @@ async fn handler_inner( ) -> Result, Error> { let path = req.uri().path().to_string(); let path = percent_encoding::percent_decode_str(&path) - .decode_utf8() - .map_err(|e| Error::BadRequest(format!("Invalid utf8 key ({})", e)))?; + .decode_utf8()?; let (bucket, key) = parse_bucket_key(&path)?; if bucket.len() == 0 { @@ -125,10 +124,7 @@ async fn handler_inner( // CopyObject query let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?; let copy_source = percent_encoding::percent_decode_str(©_source) - .decode_utf8() - .map_err(|e| { - Error::BadRequest(format!("Invalid utf8 copy_source ({})", e)) - })?; + .decode_utf8()?; let (source_bucket, source_key) = parse_bucket_key(©_source)?; if !api_key.allow_read(&source_bucket) { return Err(Error::Forbidden(format!( @@ -153,8 +149,7 @@ async fn handler_inner( Ok(handle_abort_multipart_upload(garage, &bucket, &key, upload_id).await?) } else { // DeleteObject query - let version_uuid = handle_delete(garage, &bucket, &key).await?; - Ok(put_response(version_uuid)) + Ok(handle_delete(garage, &bucket, &key).await?) } } &Method::POST => { @@ -233,7 +228,22 @@ async fn handler_inner( ) .await?) } - _ => Err(Error::BadRequest(format!("Invalid method"))), + &Method::POST => { + if params.contains_key(&"delete".to_string()) { + // DeleteObjects + Ok(handle_delete_objects(garage, bucket, req).await?) + } else { + println!( + "Body: {}", + std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?) + .unwrap_or("") + ); + Err(Error::BadRequest(format!("Unsupported call"))) + } + } + _ => { + Err(Error::BadRequest(format!("Invalid method"))) + } } } } diff --git a/src/api/s3_delete.rs b/src/api/s3_delete.rs index 4d6805fb..001eb162 100644 --- a/src/api/s3_delete.rs +++ b/src/api/s3_delete.rs @@ -1,4 +1,7 @@ use std::sync::Arc; +use std::fmt::Write; + +use hyper::{Body, Request, Response}; use garage_util::data::*; use garage_util::error::Error; @@ -6,7 +9,10 @@ use garage_util::error::Error; use garage_core::garage::Garage; use garage_core::object_table::*; -pub async fn handle_delete(garage: Arc, bucket: &str, key: &str) -> Result { +use crate::http_util::*; +use crate::encoding::*; + +async fn handle_delete_internal(garage: &Garage, bucket: &str, key: &str) -> Result<(UUID, UUID), Error> { let object = match garage .object_table .get(&bucket.to_string(), &key.to_string()) @@ -14,7 +20,7 @@ pub async fn handle_delete(garage: Arc, bucket: &str, key: &str) -> Resu { None => { // No need to delete - return Ok([0u8; 32].into()); + return Err(Error::NotFound); } Some(o) => o, }; @@ -23,16 +29,19 @@ pub async fn handle_delete(garage: Arc, bucket: &str, key: &str) -> Resu v.data != ObjectVersionData::DeleteMarker && v.state != ObjectVersionState::Aborted }); - let mut must_delete = false; + let mut must_delete = None; let mut timestamp = now_msec(); for v in interesting_versions { - must_delete = true; + if v.timestamp + 1 > timestamp || must_delete.is_none() { + must_delete = Some(v.uuid); + } timestamp = std::cmp::max(timestamp, v.timestamp + 1); } - if !must_delete { - return Ok([0u8; 32].into()); - } + let deleted_version = match must_delete { + None => return Err(Error::NotFound), + Some(v) => v, + }; let version_uuid = gen_uuid(); @@ -50,5 +59,88 @@ pub async fn handle_delete(garage: Arc, bucket: &str, key: &str) -> Resu ); garage.object_table.insert(&object).await?; - return Ok(version_uuid); + return Ok((deleted_version, version_uuid)); } + +pub async fn handle_delete(garage: Arc, bucket: &str, key: &str) -> Result, Error> { + let (_deleted_version, delete_marker_version) = handle_delete_internal(&garage, bucket, key).await?; + + Ok(Response::builder() + .header("x-amz-version-id", hex::encode(delete_marker_version)) + .body(empty_body()) + .unwrap()) +} + +pub async fn handle_delete_objects(garage: Arc, bucket: &str, req: Request) -> Result, Error> { + let body = hyper::body::to_bytes(req.into_body()).await?; + let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?; + let cmd = parse_delete_objects_xml(&cmd_xml) + .map_err(|e| Error::BadRequest(format!("Invald delete XML query: {}", e)))?; + + let mut retxml = String::new(); + writeln!(&mut retxml, r#""#).unwrap(); + writeln!(&mut retxml, "").unwrap(); + + for obj in cmd.objects.iter() { + match handle_delete_internal(&garage, bucket, &obj.key).await { + Ok((deleted_version, delete_marker_version)) => { + writeln!(&mut retxml, "\t").unwrap(); + writeln!(&mut retxml, "\t\t{}", obj.key).unwrap(); + writeln!(&mut retxml, "\t\t{}", hex::encode(deleted_version)).unwrap(); + writeln!(&mut retxml, "\t\t{}", hex::encode(delete_marker_version)).unwrap(); + writeln!(&mut retxml, "\t").unwrap(); + } + Err(e) => { + writeln!(&mut retxml, "\t").unwrap(); + writeln!(&mut retxml, "\t\t{}", e.http_status_code()).unwrap(); + writeln!(&mut retxml, "\t\t{}", obj.key).unwrap(); + writeln!(&mut retxml, "\t\t{}", xml_escape(&format!("{}", e))).unwrap(); + writeln!(&mut retxml, "\t").unwrap(); + } + } + } + + writeln!(&mut retxml, "").unwrap(); + + Ok(Response::new(Box::new(BytesBody::from(retxml.into_bytes())))) +} + +struct DeleteRequest { + objects: Vec, +} + +struct DeleteObject { + key: String, +} + +fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Result { + let mut ret = DeleteRequest{objects: vec![]}; + + let root = xml.root(); + let delete = match root.first_child() { + Some(del) => del, + None => return Err(format!("Delete tag not found")), + }; + if !delete.has_tag_name("Delete") { + return Err(format!("Invalid root tag: {:?}", root)); + } + + for item in delete.children() { + if item.has_tag_name("Object") { + if let Some(key) = item.children().find(|e| e.has_tag_name("Key")) { + if let Some(key_str) = key.text() { + ret.objects.push(DeleteObject{key: key_str.to_string()}); + } else { + return Err(format!("No text for key: {:?}", key)); + } + } else { + return Err(format!("No delete key for item: {:?}", item)); + } + } else { + return Err(format!("Invalid delete item: {:?}", item)); + } + } + + Ok(ret) +} + diff --git a/src/api/s3_list.rs b/src/api/s3_list.rs index df792da2..d4d8161e 100644 --- a/src/api/s3_list.rs +++ b/src/api/s3_list.rs @@ -30,34 +30,38 @@ pub async fn handle_list( let mut result_keys = BTreeMap::::new(); let mut result_common_prefixes = BTreeSet::::new(); - let mut truncated = true; let mut next_chunk_start = marker.unwrap_or(prefix).to_string(); debug!("List request: `{}` {} `{}`", delimiter, max_keys, prefix); - while result_keys.len() + result_common_prefixes.len() < max_keys && truncated { + let truncated; + 'query_loop: loop { let objects = garage .object_table .get_range( &bucket.to_string(), Some(next_chunk_start.clone()), Some(()), - max_keys, + max_keys + 1, ) .await?; debug!( "List: get range {} (max {}), results: {}", next_chunk_start, - max_keys, + max_keys + 1, objects.len() ); for object in objects.iter() { if !object.key.starts_with(prefix) { truncated = false; - break; + break 'query_loop; } if let Some(version) = object.versions().iter().find(|x| x.is_data()) { + if result_keys.len() + result_common_prefixes.len() >= max_keys { + truncated = true; + break 'query_loop; + } let common_prefix = if delimiter.len() > 0 { let relative_key = &object.key[prefix.len()..]; match relative_key.find(delimiter) { @@ -83,8 +87,9 @@ pub async fn handle_list( }; } } - if objects.len() < max_keys { - truncated = false; + if objects.len() < max_keys + 1 { + truncated = false; + break 'query_loop; } if objects.len() > 0 { next_chunk_start = objects[objects.len() - 1].key.clone(); diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 6f61a586..d8c39c62 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -32,4 +32,5 @@ hyper = "0.13" rustls = "0.17" webpki = "0.21" +roxmltree = "0.11" diff --git a/src/util/error.rs b/src/util/error.rs index cb99cbbc..d2ed1ccf 100644 --- a/src/util/error.rs +++ b/src/util/error.rs @@ -114,3 +114,15 @@ impl From> for Error { Error::Message(format!("MPSC send error")) } } + +impl From for Error { + fn from(e: std::str::Utf8Error) -> Error { + Error::BadRequest(format!("Invalid UTF-8: {}", e)) + } +} + +impl From for Error { + fn from(e: roxmltree::Error) -> Error { + Error::BadRequest(format!("Invalid XML: {}", e)) + } +}