garage/src/api/s3_list.rs

use std::collections::{BTreeMap, BTreeSet};
use std::fmt::Write;
use std::sync::Arc;

use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};
use hyper::{Body, Response};

use garage_util::error::Error;

use garage_model::garage::Garage;
use garage_model::object_table::*;

use crate::encoding::*;

#[derive(Debug)]
struct ListResultInfo {
	last_modified: u64,
	size: u64,
}

pub async fn handle_list(
	garage: Arc<Garage>,
	bucket: &str,
	delimiter: &str,
	max_keys: usize,
	prefix: &str,
	marker: Option<&str>,
	urlencode_resp: bool,
) -> Result<Response<Body>, Error> {
	let mut result_keys = BTreeMap::<String, ListResultInfo>::new();
	let mut result_common_prefixes = BTreeSet::<String>::new();

	let mut next_chunk_start = marker.unwrap_or(prefix).to_string();

	debug!("List request: `{}` {} `{}`", delimiter, max_keys, prefix);

	let truncated;
	'query_loop: loop {
		let objects = garage
			.object_table
			.get_range(
				&bucket.to_string(),
				Some(next_chunk_start.clone()),
				Some(()),
				max_keys + 1,
			)
			.await?;
		debug!(
			"List: get range {} (max {}), results: {}",
			next_chunk_start,
			max_keys + 1,
			objects.len()
		);

		for object in objects.iter() {
			if !object.key.starts_with(prefix) {
				truncated = false;
				break 'query_loop;
			}
			if let Some(version) = object.versions().iter().find(|x| x.is_data()) {
				if result_keys.len() + result_common_prefixes.len() >= max_keys {
					truncated = true;
					break 'query_loop;
				}
				let common_prefix = if delimiter.len() > 0 {
					let relative_key = &object.key[prefix.len()..];
					match relative_key.find(delimiter) {
						Some(i) => Some(&object.key[..prefix.len() + i + delimiter.len()]),
						None => None,
					}
				} else {
					None
				};
				if let Some(pfx) = common_prefix {
					result_common_prefixes.insert(pfx.to_string());
				} else {
                    let size = match &version.state {
                        ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta.size,
                        ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => meta.size,
                        _ => unreachable!(),
                    };
					let info = match result_keys.get(&object.key) {
						None => ListResultInfo {
							last_modified: version.timestamp,
							size,
						},
						Some(_lri) => {
							return Err(Error::Message(format!("Duplicate key?? {}", object.key)))
						}
					};
					result_keys.insert(object.key.clone(), info);
				};
			}
		}
		if objects.len() < max_keys + 1 {
			truncated = false;
			break 'query_loop;
		}
		if objects.len() > 0 {
			next_chunk_start = objects[objects.len() - 1].key.clone();
		}
	}

	let mut xml = String::new();
	writeln!(&mut xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();
	writeln!(
		&mut xml,
		r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#
	)
	.unwrap();
	writeln!(&mut xml, "\t<Bucket>{}</Bucket>", bucket).unwrap();
	writeln!(&mut xml, "\t<Prefix>{}</Prefix>", prefix).unwrap();
	writeln!(&mut xml, "\t<KeyCount>{}</KeyCount>", result_keys.len()).unwrap();
	writeln!(&mut xml, "\t<MaxKeys>{}</MaxKeys>", max_keys).unwrap();
	writeln!(&mut xml, "\t<IsTruncated>{}</IsTruncated>", truncated).unwrap();
	for (key, info) in result_keys.iter() {
		let last_modif = NaiveDateTime::from_timestamp(info.last_modified as i64 / 1000, 0);
		let last_modif = DateTime::<Utc>::from_utc(last_modif, Utc);
		let last_modif = last_modif.to_rfc3339_opts(SecondsFormat::Millis, true);
		writeln!(&mut xml, "\t<Contents>").unwrap();
		writeln!(
			&mut xml,
			"\t\t<Key>{}</Key>",
			xml_escape(key),
			//xml_encode_key(key, urlencode_resp)       // doesn't work with nextcloud, wtf
		)
		.unwrap();
		writeln!(&mut xml, "\t\t<LastModified>{}</LastModified>", last_modif).unwrap();
		writeln!(&mut xml, "\t\t<Size>{}</Size>", info.size).unwrap();
		writeln!(&mut xml, "\t\t<StorageClass>STANDARD</StorageClass>").unwrap();
		writeln!(&mut xml, "\t</Contents>").unwrap();
	}
	if result_common_prefixes.len() > 0 {
		writeln!(&mut xml, "\t<CommonPrefixes>").unwrap();
		for pfx in result_common_prefixes.iter() {
			writeln!(
				&mut xml,
				"\t\t<Prefix>{}</Prefix>",
				xml_escape(pfx),
				//xml_encode_key(pfx, urlencode_resp)
			)
			.unwrap();
		}
		writeln!(&mut xml, "\t</CommonPrefixes>").unwrap();
	}
	writeln!(&mut xml, "</ListBucketResult>").unwrap();
	println!("{}", xml);

	Ok(Response::new(Body::from(xml.into_bytes())))
}
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`use std::collections::{BTreeMap, BTreeSet};`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`use std::fmt::Write;`
			`use std::sync::Arc;`

xml escape 2020-04-24 18:56:00 +00:00			`use chrono::{DateTime, NaiveDateTime, SecondsFormat, Utc};`
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`use hyper::{Body, Response};`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
			`use garage_util::error::Error;`

Rename garage_core to garage_model 2020-07-07 11:59:22 +00:00			`use garage_model::garage::Garage;`
Migrate S3 api to use new model 2020-07-08 15:33:24 +00:00			`use garage_model::object_table::*;`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Work on API 2020-04-28 10:18:14 +00:00			`use crate::encoding::*;`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
			`#[derive(Debug)]`
			`struct ListResultInfo {`
			`last_modified: u64,`
			`size: u64,`
			`}`

			`pub async fn handle_list(`
			`garage: Arc<Garage>,`
			`bucket: &str,`
			`delimiter: &str,`
			`max_keys: usize,`
			`prefix: &str,`
Work on API 2020-04-28 10:18:14 +00:00			`marker: Option<&str>,`
			`urlencode_resp: bool,`
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`) -> Result<Response<Body>, Error> {`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`let mut result_keys = BTreeMap::<String, ListResultInfo>::new();`
			`let mut result_common_prefixes = BTreeSet::<String>::new();`
Slightly improved S3 compatibility - ListBucket does not require any of the parameters (delimiter, prefix, max-keys, etc) - URLs are properly percent_decoded - PutObject and DeleteObject calls now answer correctly (empty body, version id in the x-amz-version-id: header) 2020-05-01 14:30:50 +00:00
Work on API 2020-04-28 10:18:14 +00:00			`let mut next_chunk_start = marker.unwrap_or(prefix).to_string();`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Less verbosity 2020-04-24 19:27:27 +00:00			debug!("List request: `{}` {} `{}`", delimiter, max_keys, prefix);
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`let truncated;`
			`'query_loop: loop {`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`let objects = garage`
			`.object_table`
			`.get_range(`
			`&bucket.to_string(),`
			`Some(next_chunk_start.clone()),`
			`Some(()),`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`max_keys + 1,`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`)`
			`.await?;`
Slightly improved S3 compatibility - ListBucket does not require any of the parameters (delimiter, prefix, max-keys, etc) - URLs are properly percent_decoded - PutObject and DeleteObject calls now answer correctly (empty body, version id in the x-amz-version-id: header) 2020-05-01 14:30:50 +00:00			`debug!(`
			`"List: get range {} (max {}), results: {}",`
			`next_chunk_start,`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`max_keys + 1,`
Slightly improved S3 compatibility - ListBucket does not require any of the parameters (delimiter, prefix, max-keys, etc) - URLs are properly percent_decoded - PutObject and DeleteObject calls now answer correctly (empty body, version id in the x-amz-version-id: header) 2020-05-01 14:30:50 +00:00			`objects.len()`
			`);`

Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`for object in objects.iter() {`
Slightly improved S3 compatibility - ListBucket does not require any of the parameters (delimiter, prefix, max-keys, etc) - URLs are properly percent_decoded - PutObject and DeleteObject calls now answer correctly (empty body, version id in the x-amz-version-id: header) 2020-05-01 14:30:50 +00:00			`if !object.key.starts_with(prefix) {`
			`truncated = false;`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`break 'query_loop;`
Slightly improved S3 compatibility - ListBucket does not require any of the parameters (delimiter, prefix, max-keys, etc) - URLs are properly percent_decoded - PutObject and DeleteObject calls now answer correctly (empty body, version id in the x-amz-version-id: header) 2020-05-01 14:30:50 +00:00			`}`
Prepare for multipart uploads 2020-04-26 18:55:13 +00:00			`if let Some(version) = object.versions().iter().find(\|x\| x.is_data()) {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`if result_keys.len() + result_common_prefixes.len() >= max_keys {`
			`truncated = true;`
			`break 'query_loop;`
			`}`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`let common_prefix = if delimiter.len() > 0 {`
			`let relative_key = &object.key[prefix.len()..];`
Remove leading / in keys; better delimiter handling 2020-04-24 20:28:15 +00:00			`match relative_key.find(delimiter) {`
Prepare for multipart uploads 2020-04-26 18:55:13 +00:00			`Some(i) => Some(&object.key[..prefix.len() + i + delimiter.len()]),`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`None => None,`
Remove leading / in keys; better delimiter handling 2020-04-24 20:28:15 +00:00			`}`
			`} else {`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`None`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`};`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`if let Some(pfx) = common_prefix {`
			`result_common_prefixes.insert(pfx.to_string());`
			`} else {`
Migrate S3 api to use new model 2020-07-08 15:33:24 +00:00			`let size = match &version.state {`
			`ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta.size,`
			`ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => meta.size,`
			`_ => unreachable!(),`
			`};`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`let info = match result_keys.get(&object.key) {`
			`None => ListResultInfo {`
			`last_modified: version.timestamp,`
Migrate S3 api to use new model 2020-07-08 15:33:24 +00:00			`size,`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`},`
Prepare for multipart uploads 2020-04-26 18:55:13 +00:00			`Some(_lri) => {`
			`return Err(Error::Message(format!("Duplicate key?? {}", object.key)))`
			`}`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`};`
			`result_keys.insert(object.key.clone(), info);`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`};`
			`}`
			`}`
Implement DeleteObjects 2020-05-01 15:52:35 +00:00			`if objects.len() < max_keys + 1 {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`truncated = false;`
			`break 'query_loop;`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`}`
			`if objects.len() > 0 {`
			`next_chunk_start = objects[objects.len() - 1].key.clone();`
			`}`
			`}`

			`let mut xml = String::new();`
			`writeln!(&mut xml, r#"<?xml version="1.0" encoding="UTF-8"?>"#).unwrap();`
			`writeln!(`
			`&mut xml,`
			`r#"<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">"#`
			`)`
			`.unwrap();`
			`writeln!(&mut xml, "\t<Bucket>{}</Bucket>", bucket).unwrap();`
			`writeln!(&mut xml, "\t<Prefix>{}</Prefix>", prefix).unwrap();`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`writeln!(&mut xml, "\t<KeyCount>{}</KeyCount>", result_keys.len()).unwrap();`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`writeln!(&mut xml, "\t<MaxKeys>{}</MaxKeys>", max_keys).unwrap();`
			`writeln!(&mut xml, "\t<IsTruncated>{}</IsTruncated>", truncated).unwrap();`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`for (key, info) in result_keys.iter() {`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`let last_modif = NaiveDateTime::from_timestamp(info.last_modified as i64 / 1000, 0);`
			`let last_modif = DateTime::<Utc>::from_utc(last_modif, Utc);`
			`let last_modif = last_modif.to_rfc3339_opts(SecondsFormat::Millis, true);`
			`writeln!(&mut xml, "\t<Contents>").unwrap();`
Work on API 2020-04-28 10:18:14 +00:00			`writeln!(`
			`&mut xml,`
			`"\t\t<Key>{}</Key>",`
Compatibility fixes 2020-04-28 10:35:04 +00:00			`xml_escape(key),`
			`//xml_encode_key(key, urlencode_resp) // doesn't work with nextcloud, wtf`
Work on API 2020-04-28 10:18:14 +00:00			`)`
			`.unwrap();`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`writeln!(&mut xml, "\t\t<LastModified>{}</LastModified>", last_modif).unwrap();`
			`writeln!(&mut xml, "\t\t<Size>{}</Size>", info.size).unwrap();`
			`writeln!(&mut xml, "\t\t<StorageClass>STANDARD</StorageClass>").unwrap();`
			`writeln!(&mut xml, "\t</Contents>").unwrap();`
			`}`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`if result_common_prefixes.len() > 0 {`
			`writeln!(&mut xml, "\t<CommonPrefixes>").unwrap();`
			`for pfx in result_common_prefixes.iter() {`
Work on API 2020-04-28 10:18:14 +00:00			`writeln!(`
			`&mut xml,`
Slightly improved S3 compatibility - ListBucket does not require any of the parameters (delimiter, prefix, max-keys, etc) - URLs are properly percent_decoded - PutObject and DeleteObject calls now answer correctly (empty body, version id in the x-amz-version-id: header) 2020-05-01 14:30:50 +00:00			`"\t\t<Prefix>{}</Prefix>",`
Compatibility fixes 2020-04-28 10:35:04 +00:00			`xml_escape(pfx),`
			`//xml_encode_key(pfx, urlencode_resp)`
Work on API 2020-04-28 10:18:14 +00:00			`)`
			`.unwrap();`
S3 compatibility: fix bucket listing and HEAD and PUT on bucket 2020-04-26 16:22:33 +00:00			`}`
			`writeln!(&mut xml, "\t</CommonPrefixes>").unwrap();`
			`}`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`writeln!(&mut xml, "</ListBucketResult>").unwrap();`
Slightly improved S3 compatibility - ListBucket does not require any of the parameters (delimiter, prefix, max-keys, etc) - URLs are properly percent_decoded - PutObject and DeleteObject calls now answer correctly (empty body, version id in the x-amz-version-id: header) 2020-05-01 14:30:50 +00:00			`println!("{}", xml);`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`Ok(Response::new(Body::from(xml.into_bytes())))`
Prepare for multipart uploads 2020-04-26 18:55:13 +00:00			`}`