garage/src/api/s3_delete.rs

use std::fmt::Write;
use std::sync::Arc;

use hyper::{Body, Request, Response};

use garage_util::data::*;
use garage_util::time::*;

use garage_model::garage::Garage;
use garage_model::object_table::*;

use crate::encoding::*;
use crate::error::*;
use crate::signature::verify_signed_content;

async fn handle_delete_internal(
	garage: &Garage,
	bucket: &str,
	key: &str,
) -> Result<(Uuid, Uuid), Error> {
	let object = garage
		.object_table
		.get(&bucket.to_string(), &key.to_string())
		.await?
		.ok_or(Error::NotFound)?; // No need to delete

	let interesting_versions = object.versions().iter().filter(|v| {
		!matches!(
			v.state,
			ObjectVersionState::Aborted
				| ObjectVersionState::Complete(ObjectVersionData::DeleteMarker)
		)
	});

	let mut version_to_delete = None;
	let mut timestamp = now_msec();
	for v in interesting_versions {
		if v.timestamp + 1 > timestamp || version_to_delete.is_none() {
			version_to_delete = Some(v.uuid);
		}
		timestamp = std::cmp::max(timestamp, v.timestamp + 1);
	}

	let deleted_version = version_to_delete.ok_or(Error::NotFound)?;

	let version_uuid = gen_uuid();

	let object = Object::new(
		bucket.into(),
		key.into(),
		vec![ObjectVersion {
			uuid: version_uuid,
			timestamp,
			state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker),
		}],
	);

	garage.object_table.insert(&object).await?;
	return Ok((deleted_version, version_uuid));
}

pub async fn handle_delete(
	garage: Arc<Garage>,
	bucket: &str,
	key: &str,
) -> Result<Response<Body>, Error> {
	let (_deleted_version, delete_marker_version) =
		handle_delete_internal(&garage, bucket, key).await?;

	Ok(Response::builder()
		.header("x-amz-version-id", hex::encode(delete_marker_version))
		.body(Body::from(vec![]))
		.unwrap())
}

pub async fn handle_delete_objects(
	garage: Arc<Garage>,
	bucket: &str,
	req: Request<Body>,
	content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> {
	let body = hyper::body::to_bytes(req.into_body()).await?;
	verify_signed_content(content_sha256, &body[..])?;

	let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;
	let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;

	let mut retxml = String::new();
	writeln!(&mut retxml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
	writeln!(
		&mut retxml,
		r#"<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#
	)
	.unwrap();

	for obj in cmd.objects.iter() {
		match handle_delete_internal(&garage, bucket, &obj.key).await {
			Ok((deleted_version, delete_marker_version)) => {
				if cmd.quiet {
					continue;
				}
				writeln!(&mut retxml, "\t<Deleted>").unwrap();
				writeln!(&mut retxml, "\t\t<Key>{}</Key>", xml_escape(&obj.key)).unwrap();
				writeln!(
					&mut retxml,
					"\t\t<VersionId>{}</VersionId>",
					hex::encode(deleted_version)
				)
				.unwrap();
				writeln!(
					&mut retxml,
					"\t\t<DeleteMarkerVersionId>{}</DeleteMarkerVersionId>",
					hex::encode(delete_marker_version)
				)
				.unwrap();
				writeln!(&mut retxml, "\t</Deleted>").unwrap();
			}
			Err(e) => {
				writeln!(&mut retxml, "\t<Error>").unwrap();
				writeln!(&mut retxml, "\t\t<Code>{}</Code>", e.http_status_code()).unwrap();
				writeln!(&mut retxml, "\t\t<Key>{}</Key>", xml_escape(&obj.key)).unwrap();
				writeln!(
					&mut retxml,
					"\t\t<Message>{}</Message>",
					xml_escape(&format!("{}", e))
				)
				.unwrap();
				writeln!(&mut retxml, "\t</Error>").unwrap();
			}
		}
	}

	writeln!(&mut retxml, "</DeleteResult>").unwrap();

	Ok(Response::builder()
		.header("Content-Type", "application/xml")
		.body(Body::from(retxml.into_bytes()))?)
}

struct DeleteRequest {
	quiet: bool,
	objects: Vec<DeleteObject>,
}

struct DeleteObject {
	key: String,
}

fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Option<DeleteRequest> {
	let mut ret = DeleteRequest {
		quiet: false,
		objects: vec![],
	};

	let root = xml.root();
	let delete = root.first_child()?;

	if !delete.has_tag_name("Delete") {
		return None;
	}

	for item in delete.children() {
		if item.has_tag_name("Object") {
			let key = item.children().find(|e| e.has_tag_name("Key"))?;
			let key_str = key.text()?;
			ret.objects.push(DeleteObject {
				key: key_str.to_string(),
			});
		} else if item.has_tag_name("Quiet") {
			if item.text()? == "true" {
				ret.quiet = true;
			} else {
				ret.quiet = false;
			}
		} else {
			return None;
		}
	}

	Some(ret)
}
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`use std::fmt::Write;`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`use std::sync::Arc;`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00
			`use hyper::{Body, Request, Response};`
Work on API 2020-04-28 10:18:14 +00:00
			`use garage_util::data::*;`
Time and metadata improvements 2021-03-15 15:21:41 +00:00			`use garage_util::time::*;`
Work on API 2020-04-28 10:18:14 +00:00
Rename garage_core to garage_model 2020-07-07 11:59:22 +00:00			`use garage_model::garage::Garage;`
			`use garage_model::object_table::*;`
Work on API 2020-04-28 10:18:14 +00:00
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`use crate::encoding::*;`
Refactor error management in API part 2020-11-08 14:04:30 +00:00			`use crate::error::*;`
Add verification of part numbers in CompleteMultipartUpload (WIP #30) 2021-02-19 23:13:07 +00:00			`use crate::signature::verify_signed_content;`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`async fn handle_delete_internal(`
			`garage: &Garage,`
			`bucket: &str,`
			`key: &str,`
rename types to CamelCase 2021-05-02 21:13:08 +00:00			`) -> Result<(Uuid, Uuid), Error> {`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`let object = garage`
Work on API 2020-04-28 10:18:14 +00:00			`.object_table`
			`.get(&bucket.to_string(), &key.to_string())`
			`.await?`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`.ok_or(Error::NotFound)?; // No need to delete`
Work on API 2020-04-28 10:18:14 +00:00
fix clippy warnings on api 2021-04-23 20:18:00 +00:00			`let interesting_versions = object.versions().iter().filter(\|v\| {`
			`!matches!(`
			`v.state,`
			`ObjectVersionState::Aborted`
			`\| ObjectVersionState::Complete(ObjectVersionData::DeleteMarker)`
			`)`
Work on API 2020-04-28 10:18:14 +00:00			`});`

Fix race condition 2021-03-15 14:26:29 +00:00			`let mut version_to_delete = None;`
Work on API 2020-04-28 10:18:14 +00:00			`let mut timestamp = now_msec();`
			`for v in interesting_versions {`
Fix race condition 2021-03-15 14:26:29 +00:00			`if v.timestamp + 1 > timestamp \|\| version_to_delete.is_none() {`
			`version_to_delete = Some(v.uuid);`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`}`
Work on API 2020-04-28 10:18:14 +00:00			`timestamp = std::cmp::max(timestamp, v.timestamp + 1);`
			`}`

Fix race condition 2021-03-15 14:26:29 +00:00			`let deleted_version = version_to_delete.ok_or(Error::NotFound)?;`
Work on API 2020-04-28 10:18:14 +00:00
			`let version_uuid = gen_uuid();`

			`let object = Object::new(`
			`bucket.into(),`
			`key.into(),`
			`vec![ObjectVersion {`
			`uuid: version_uuid,`
Small fixes 2021-03-15 18:51:16 +00:00			`timestamp,`
Migrate S3 api to use new model 2020-07-08 15:33:24 +00:00			`state: ObjectVersionState::Complete(ObjectVersionData::DeleteMarker),`
Work on API 2020-04-28 10:18:14 +00:00			`}],`
			`);`

			`garage.object_table.insert(&object).await?;`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`return Ok((deleted_version, version_uuid));`
			`}`

Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`pub async fn handle_delete(`
			`garage: Arc<Garage>,`
			`bucket: &str,`
			`key: &str,`
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`) -> Result<Response<Body>, Error> {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`let (_deleted_version, delete_marker_version) =`
			`handle_delete_internal(&garage, bucket, key).await?;`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00
			`Ok(Response::builder()`
			`.header("x-amz-version-id", hex::encode(delete_marker_version))`
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`.body(Body::from(vec![]))`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`.unwrap())`
			`}`

Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`pub async fn handle_delete_objects(`
			`garage: Arc<Garage>,`
			`bucket: &str,`
			`req: Request<Body>,`
Add verification of part numbers in CompleteMultipartUpload (WIP #30) 2021-02-19 23:13:07 +00:00			`content_sha256: Option<Hash>,`
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`) -> Result<Response<Body>, Error> {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`let body = hyper::body::to_bytes(req.into_body()).await?;`
Add verification of part numbers in CompleteMultipartUpload (WIP #30) 2021-02-19 23:13:07 +00:00			`verify_signed_content(content_sha256, &body[..])?;`

Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`let cmd_xml = roxmltree::Document::parse(&std::str::from_utf8(&body)?)?;`
Refactor error management in API part 2020-11-08 14:04:30 +00:00			`let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00
			`let mut retxml = String::new();`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`writeln!(&mut retxml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();`
Many S3 compatibility improvements: - return XML errors - implement AuthorizationHeaderMalformed error to redirect clients to correct location (used by minio client) - implement GetBucketLocation - fix DeleteObjects XML parsing and response 2021-04-27 23:05:40 +00:00			`writeln!(`
			`&mut retxml,`
			`r#"<DeleteResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#`
			`)`
			`.unwrap();`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`for obj in cmd.objects.iter() {`
			`match handle_delete_internal(&garage, bucket, &obj.key).await {`
			`Ok((deleted_version, delete_marker_version)) => {`
Many S3 compatibility improvements: - return XML errors - implement AuthorizationHeaderMalformed error to redirect clients to correct location (used by minio client) - implement GetBucketLocation - fix DeleteObjects XML parsing and response 2021-04-27 23:05:40 +00:00			`if cmd.quiet {`
			`continue;`
			`}`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`writeln!(&mut retxml, "\t<Deleted>").unwrap();`
add application/xml header and missing xml escapes 2021-02-19 22:40:18 +00:00			`writeln!(&mut retxml, "\t\t<Key>{}</Key>", xml_escape(&obj.key)).unwrap();`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`writeln!(`
			`&mut retxml,`
			`"\t\t<VersionId>{}</VersionId>",`
			`hex::encode(deleted_version)`
			`)`
			`.unwrap();`
			`writeln!(`
			`&mut retxml,`
			`"\t\t<DeleteMarkerVersionId>{}</DeleteMarkerVersionId>",`
			`hex::encode(delete_marker_version)`
			`)`
			`.unwrap();`
			`writeln!(&mut retxml, "\t</Deleted>").unwrap();`
			`}`
			`Err(e) => {`
			`writeln!(&mut retxml, "\t<Error>").unwrap();`
			`writeln!(&mut retxml, "\t\t<Code>{}</Code>", e.http_status_code()).unwrap();`
add application/xml header and missing xml escapes 2021-02-19 22:40:18 +00:00			`writeln!(&mut retxml, "\t\t<Key>{}</Key>", xml_escape(&obj.key)).unwrap();`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`writeln!(`
			`&mut retxml,`
			`"\t\t<Message>{}</Message>",`
			`xml_escape(&format!("{}", e))`
			`)`
			`.unwrap();`
			`writeln!(&mut retxml, "\t</Error>").unwrap();`
			`}`
			`}`
			`}`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00
Many S3 compatibility improvements: - return XML errors - implement AuthorizationHeaderMalformed error to redirect clients to correct location (used by minio client) - implement GetBucketLocation - fix DeleteObjects XML parsing and response 2021-04-27 23:05:40 +00:00			`writeln!(&mut retxml, "</DeleteResult>").unwrap();`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00
add application/xml header and missing xml escapes 2021-02-19 22:40:18 +00:00			`Ok(Response::builder()`
Cargo fmt 2021-02-23 17:46:25 +00:00			`.header("Content-Type", "application/xml")`
			`.body(Body::from(retxml.into_bytes()))?)`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`}`

			`struct DeleteRequest {`
Many S3 compatibility improvements: - return XML errors - implement AuthorizationHeaderMalformed error to redirect clients to correct location (used by minio client) - implement GetBucketLocation - fix DeleteObjects XML parsing and response 2021-04-27 23:05:40 +00:00			`quiet: bool,`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`objects: Vec<DeleteObject>,`
Work on API 2020-04-28 10:18:14 +00:00			`}`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00
			`struct DeleteObject {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`key: String,`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`}`

Add verification of part numbers in CompleteMultipartUpload (WIP #30) 2021-02-19 23:13:07 +00:00			`fn parse_delete_objects_xml(xml: &roxmltree::Document) -> Option<DeleteRequest> {`
Many S3 compatibility improvements: - return XML errors - implement AuthorizationHeaderMalformed error to redirect clients to correct location (used by minio client) - implement GetBucketLocation - fix DeleteObjects XML parsing and response 2021-04-27 23:05:40 +00:00			`let mut ret = DeleteRequest {`
			`quiet: false,`
			`objects: vec![],`
			`};`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00
			`let root = xml.root();`
Add verification of part numbers in CompleteMultipartUpload (WIP #30) 2021-02-19 23:13:07 +00:00			`let delete = root.first_child()?;`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`if !delete.has_tag_name("Delete") {`
Add verification of part numbers in CompleteMultipartUpload (WIP #30) 2021-02-19 23:13:07 +00:00			`return None;`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`}`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`for item in delete.children() {`
			`if item.has_tag_name("Object") {`
Add verification of part numbers in CompleteMultipartUpload (WIP #30) 2021-02-19 23:13:07 +00:00			`let key = item.children().find(\|e\| e.has_tag_name("Key"))?;`
			`let key_str = key.text()?;`
			`ret.objects.push(DeleteObject {`
			`key: key_str.to_string(),`
			`});`
Many S3 compatibility improvements: - return XML errors - implement AuthorizationHeaderMalformed error to redirect clients to correct location (used by minio client) - implement GetBucketLocation - fix DeleteObjects XML parsing and response 2021-04-27 23:05:40 +00:00			`} else if item.has_tag_name("Quiet") {`
			`if item.text()? == "true" {`
			`ret.quiet = true;`
			`} else {`
			`ret.quiet = false;`
			`}`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`} else {`
Add verification of part numbers in CompleteMultipartUpload (WIP #30) 2021-02-19 23:13:07 +00:00			`return None;`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`}`
			`}`

Add verification of part numbers in CompleteMultipartUpload (WIP #30) 2021-02-19 23:13:07 +00:00			`Some(ret)`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`}`