garage/src/api/s3_get.rs

//! Function related to GET and HEAD requests
use std::sync::Arc;
use std::time::{Duration, UNIX_EPOCH};

use futures::stream::*;
use hyper::body::Bytes;
use hyper::{Body, Request, Response, StatusCode};

use garage_table::EmptyKey;
use garage_util::data::*;

use garage_model::garage::Garage;
use garage_model::object_table::*;

use crate::error::*;

fn object_headers(
	version: &ObjectVersion,
	version_meta: &ObjectVersionMeta,
) -> http::response::Builder {
	debug!("Version meta: {:?}", version_meta);

	let date = UNIX_EPOCH + Duration::from_millis(version.timestamp);
	let date_str = httpdate::fmt_http_date(date);

	let mut resp = Response::builder()
		.header(
			"Content-Type",
			version_meta.headers.content_type.to_string(),
		)
		.header("Last-Modified", date_str)
		.header("Accept-Ranges", "bytes".to_string());

	if !version_meta.etag.is_empty() {
		resp = resp.header("ETag", format!("\"{}\"", version_meta.etag));
	}

	for (k, v) in version_meta.headers.other.iter() {
		resp = resp.header(k, v.to_string());
	}

	resp
}

fn try_answer_cached(
	version: &ObjectVersion,
	version_meta: &ObjectVersionMeta,
	req: &Request<Body>,
) -> Option<Response<Body>> {
	// <trinity> It is possible, and is even usually the case, [that both If-None-Match and
	// If-Modified-Since] are present in a request. In this situation If-None-Match takes
	// precedence and If-Modified-Since is ignored (as per 6.Precedence from rfc7232). The rational
	// being that etag based matching is more accurate, it has no issue with sub-second precision
	// for instance (in case of very fast updates)
	let cached = if let Some(none_match) = req.headers().get(http::header::IF_NONE_MATCH) {
		let none_match = none_match.to_str().ok()?;
		let expected = format!("\"{}\"", version_meta.etag);
		let found = none_match
			.split(',')
			.map(str::trim)
			.any(|etag| etag == expected || etag == "\"*\"");
		found
	} else if let Some(modified_since) = req.headers().get(http::header::IF_MODIFIED_SINCE) {
		let modified_since = modified_since.to_str().ok()?;
		let client_date = httpdate::parse_http_date(modified_since).ok()?;
		let server_date = UNIX_EPOCH + Duration::from_millis(version.timestamp);
		client_date >= server_date
	} else {
		false
	};

	if cached {
		Some(
			Response::builder()
				.status(StatusCode::NOT_MODIFIED)
				.body(Body::empty())
				.unwrap(),
		)
	} else {
		None
	}
}

/// Handle HEAD request
pub async fn handle_head(
	garage: Arc<Garage>,
	req: &Request<Body>,
	bucket_id: Uuid,
	key: &str,
) -> Result<Response<Body>, Error> {
	let object = garage
		.object_table
		.get(&bucket_id, &key.to_string())
		.await?
		.ok_or(Error::NotFound)?;

	let version = object
		.versions()
		.iter()
		.rev()
		.find(|v| v.is_data())
		.ok_or(Error::NotFound)?;

	let version_meta = match &version.state {
		ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta,
		ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => meta,
		_ => unreachable!(),
	};

	if let Some(cached) = try_answer_cached(version, version_meta, req) {
		return Ok(cached);
	}

	let body: Body = Body::empty();
	let response = object_headers(version, version_meta)
		.header("Content-Length", format!("{}", version_meta.size))
		.status(StatusCode::OK)
		.body(body)
		.unwrap();
	Ok(response)
}

/// Handle GET request
pub async fn handle_get(
	garage: Arc<Garage>,
	req: &Request<Body>,
	bucket_id: Uuid,
	key: &str,
) -> Result<Response<Body>, Error> {
	let object = garage
		.object_table
		.get(&bucket_id, &key.to_string())
		.await?
		.ok_or(Error::NotFound)?;

	let last_v = object
		.versions()
		.iter()
		.rev()
		.find(|v| v.is_complete())
		.ok_or(Error::NotFound)?;

	let last_v_data = match &last_v.state {
		ObjectVersionState::Complete(x) => x,
		_ => unreachable!(),
	};
	let last_v_meta = match last_v_data {
		ObjectVersionData::DeleteMarker => return Err(Error::NotFound),
		ObjectVersionData::Inline(meta, _) => meta,
		ObjectVersionData::FirstBlock(meta, _) => meta,
	};

	if let Some(cached) = try_answer_cached(last_v, last_v_meta, req) {
		return Ok(cached);
	}

	let range = match req.headers().get("range") {
		Some(range) => {
			let range_str = range.to_str()?;
			let mut ranges = http_range::HttpRange::parse(range_str, last_v_meta.size)
				.map_err(|e| (e, last_v_meta.size))?;
			if ranges.len() > 1 {
				// garage does not support multi-range requests yet, so we respond with the entire
				// object when multiple ranges are requested
				None
			} else {
				ranges.pop()
			}
		}
		None => None,
	};
	if let Some(range) = range {
		return handle_get_range(
			garage,
			last_v,
			last_v_data,
			last_v_meta,
			range.start,
			range.start + range.length,
		)
		.await;
	}

	let resp_builder = object_headers(last_v, last_v_meta)
		.header("Content-Length", format!("{}", last_v_meta.size))
		.status(StatusCode::OK);

	match &last_v_data {
		ObjectVersionData::DeleteMarker => unreachable!(),
		ObjectVersionData::Inline(_, bytes) => {
			let body: Body = Body::from(bytes.to_vec());
			Ok(resp_builder.body(body)?)
		}
		ObjectVersionData::FirstBlock(_, first_block_hash) => {
			let read_first_block = garage.block_manager.rpc_get_block(first_block_hash);
			let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);

			let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;
			let version = version.ok_or(Error::NotFound)?;

			let mut blocks = version
				.blocks
				.items()
				.iter()
				.map(|(_, vb)| (vb.hash, None))
				.collect::<Vec<_>>();
			blocks[0].1 = Some(first_block);

			let body_stream = futures::stream::iter(blocks)
				.map(move |(hash, data_opt)| {
					let garage = garage.clone();
					async move {
						if let Some(data) = data_opt {
							Ok(Bytes::from(data))
						} else {
							garage
								.block_manager
								.rpc_get_block(&hash)
								.await
								.map(Bytes::from)
						}
					}
				})
				.buffered(2);

			let body = hyper::body::Body::wrap_stream(body_stream);
			Ok(resp_builder.body(body)?)
		}
	}
}

async fn handle_get_range(
	garage: Arc<Garage>,
	version: &ObjectVersion,
	version_data: &ObjectVersionData,
	version_meta: &ObjectVersionMeta,
	begin: u64,
	end: u64,
) -> Result<Response<Body>, Error> {
	let resp_builder = object_headers(version, version_meta)
		.header("Content-Length", format!("{}", end - begin))
		.header(
			"Content-Range",
			format!("bytes {}-{}/{}", begin, end - 1, version_meta.size),
		)
		.status(StatusCode::PARTIAL_CONTENT);

	match &version_data {
		ObjectVersionData::DeleteMarker => unreachable!(),
		ObjectVersionData::Inline(_meta, bytes) => {
			if end as usize <= bytes.len() {
				let body: Body = Body::from(bytes[begin as usize..end as usize].to_vec());
				Ok(resp_builder.body(body)?)
			} else {
				None.ok_or_internal_error(
					"Requested range not present in inline bytes when it should have been",
				)
			}
		}
		ObjectVersionData::FirstBlock(_meta, _first_block_hash) => {
			let version = garage.version_table.get(&version.uuid, &EmptyKey).await?;
			let version = match version {
				Some(v) => v,
				None => return Err(Error::NotFound),
			};

			// We will store here the list of blocks that have an intersection with the requested
			// range, as well as their "true offset", which is their actual offset in the complete
			// file (whereas block.offset designates the offset of the block WITHIN THE PART
			// block.part_number, which is not the same in the case of a multipart upload)
			let mut blocks = Vec::with_capacity(std::cmp::min(
				version.blocks.len(),
				4 + ((end - begin) / std::cmp::max(version.blocks.items()[0].1.size as u64, 1024))
					as usize,
			));
			let mut true_offset = 0;
			for (_, b) in version.blocks.items().iter() {
				if true_offset >= end {
					break;
				}
				// Keep only blocks that have an intersection with the requested range
				if true_offset < end && true_offset + b.size > begin {
					blocks.push((*b, true_offset));
				}
				true_offset += b.size;
			}

			let body_stream = futures::stream::iter(blocks)
				.map(move |(block, true_offset)| {
					let garage = garage.clone();
					async move {
						let data = garage.block_manager.rpc_get_block(&block.hash).await?;
						let data = Bytes::from(data);
						let start_in_block = if true_offset > begin {
							0
						} else {
							begin - true_offset
						};
						let end_in_block = if true_offset + block.size < end {
							block.size
						} else {
							end - true_offset
						};
						Result::<Bytes, Error>::Ok(
							data.slice(start_in_block as usize..end_in_block as usize),
						)
					}
				})
				.buffered(2);

			let body = hyper::body::Body::wrap_stream(body_stream);
			Ok(resp_builder.body(body)?)
		}
	}
}
document api crate 2021-03-26 21:32:09 +00:00			`//! Function related to GET and HEAD requests`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`use std::sync::Arc;`
			`use std::time::{Duration, UNIX_EPOCH};`

			`use futures::stream::*;`
			`use hyper::body::Bytes;`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`use hyper::{Body, Request, Response, StatusCode};`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
			`use garage_table::EmptyKey;`
New model for buckets 2021-12-14 12:55:11 +00:00			`use garage_util::data::*;`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Rename garage_core to garage_model 2020-07-07 11:59:22 +00:00			`use garage_model::garage::Garage;`
			`use garage_model::object_table::*;`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Refactor error management in API part 2020-11-08 14:04:30 +00:00			`use crate::error::*;`

cargo fmt 2020-07-08 15:34:37 +00:00			`fn object_headers(`
			`version: &ObjectVersion,`
			`version_meta: &ObjectVersionMeta,`
			`) -> http::response::Builder {`
Fix #28, extra headers being ignored (because of profound stupidity) 2021-02-19 11:38:22 +00:00			`debug!("Version meta: {:?}", version_meta);`

Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`let date = UNIX_EPOCH + Duration::from_millis(version.timestamp);`
			`let date_str = httpdate::fmt_http_date(date);`

More headers taken into account 2020-07-09 15:04:43 +00:00			`let mut resp = Response::builder()`
cargo fmt 2020-07-08 15:34:37 +00:00			`.header(`
			`"Content-Type",`
			`version_meta.headers.content_type.to_string(),`
			`)`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.header("Last-Modified", date_str)`
fix clippy warnings on api 2021-04-23 20:18:00 +00:00			`.header("Accept-Ranges", "bytes".to_string());`
More headers taken into account 2020-07-09 15:04:43 +00:00
Propose ETag fix 2020-12-05 18:23:46 +00:00			`if !version_meta.etag.is_empty() {`
			`resp = resp.header("ETag", format!("\"{}\"", version_meta.etag));`
			`}`

Validate content MD5 and SHA256 sums for PutObject and UploadPart 2020-07-15 13:31:13 +00:00			`for (k, v) in version_meta.headers.other.iter() {`
			`resp = resp.header(k, v.to_string());`
			`}`
More headers taken into account 2020-07-09 15:04:43 +00:00
Validate content MD5 and SHA256 sums for PutObject and UploadPart 2020-07-15 13:31:13 +00:00			`resp`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`}`

add support for caching headers 2021-03-18 14:46:33 +00:00			`fn try_answer_cached(`
			`version: &ObjectVersion,`
			`version_meta: &ObjectVersionMeta,`
			`req: &Request<Body>,`
			`) -> Option<Response<Body>> {`
Add trinity's comment in the code 2021-03-18 18:46:43 +00:00			`// <trinity> It is possible, and is even usually the case, [that both If-None-Match and`
			`// If-Modified-Since] are present in a request. In this situation If-None-Match takes`
			`// precedence and If-Modified-Since is ignored (as per 6.Precedence from rfc7232). The rational`
			`// being that etag based matching is more accurate, it has no issue with sub-second precision`
			`// for instance (in case of very fast updates)`
add support for caching headers 2021-03-18 14:46:33 +00:00			`let cached = if let Some(none_match) = req.headers().get(http::header::IF_NONE_MATCH) {`
			`let none_match = none_match.to_str().ok()?;`
			`let expected = format!("\"{}\"", version_meta.etag);`
			`let found = none_match`
			`.split(',')`
			`.map(str::trim)`
			`.any(\|etag\| etag == expected \|\| etag == "\"*\"");`
			`found`
			`} else if let Some(modified_since) = req.headers().get(http::header::IF_MODIFIED_SINCE) {`
			`let modified_since = modified_since.to_str().ok()?;`
			`let client_date = httpdate::parse_http_date(modified_since).ok()?;`
			`let server_date = UNIX_EPOCH + Duration::from_millis(version.timestamp);`
			`client_date >= server_date`
			`} else {`
			`false`
			`};`

			`if cached {`
			`Some(`
			`Response::builder()`
			`.status(StatusCode::NOT_MODIFIED)`
			`.body(Body::empty())`
			`.unwrap(),`
			`)`
			`} else {`
			`None`
			`}`
			`}`

document api crate 2021-03-26 21:32:09 +00:00			`/// Handle HEAD request`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`pub async fn handle_head(`
			`garage: Arc<Garage>,`
add support for caching headers 2021-03-18 14:46:33 +00:00			`req: &Request<Body>,`
New model for buckets 2021-12-14 12:55:11 +00:00			`bucket_id: Uuid,`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`key: &str,`
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`) -> Result<Response<Body>, Error> {`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`let object = garage`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.object_table`
New model for buckets 2021-12-14 12:55:11 +00:00			`.get(&bucket_id, &key.to_string())`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.await?`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`.ok_or(Error::NotFound)?;`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`let version = object`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.versions()`
			`.iter()`
			`.rev()`
fix clippy warnings on api 2021-04-23 20:18:00 +00:00			`.find(\|v\| v.is_data())`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`.ok_or(Error::NotFound)?;`

cargo fmt 2020-07-08 15:34:37 +00:00			`let version_meta = match &version.state {`
			`ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta,`
			`ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => meta,`
			`_ => unreachable!(),`
			`};`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Fix clippy lints (fix #121) 2021-10-26 08:20:05 +00:00			`if let Some(cached) = try_answer_cached(version, version_meta, req) {`
add support for caching headers 2021-03-18 14:46:33 +00:00			`return Ok(cached);`
			`}`

			`let body: Body = Body::empty();`
Fix clippy lints (fix #121) 2021-10-26 08:20:05 +00:00			`let response = object_headers(version, version_meta)`
Reformulate patch 2020-12-05 18:20:07 +00:00			`.header("Content-Length", format!("{}", version_meta.size))`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.status(StatusCode::OK)`
			`.body(body)`
			`.unwrap();`
			`Ok(response)`
			`}`

document api crate 2021-03-26 21:32:09 +00:00			`/// Handle GET request`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`pub async fn handle_get(`
			`garage: Arc<Garage>,`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`req: &Request<Body>,`
New model for buckets 2021-12-14 12:55:11 +00:00			`bucket_id: Uuid,`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`key: &str,`
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`) -> Result<Response<Body>, Error> {`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`let object = garage`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.object_table`
New model for buckets 2021-12-14 12:55:11 +00:00			`.get(&bucket_id, &key.to_string())`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.await?`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`.ok_or(Error::NotFound)?;`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`let last_v = object`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.versions()`
			`.iter()`
			`.rev()`
fix clippy warnings on api 2021-04-23 20:18:00 +00:00			`.find(\|v\| v.is_complete())`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`.ok_or(Error::NotFound)?;`

cargo fmt 2020-07-08 15:34:37 +00:00			`let last_v_data = match &last_v.state {`
			`ObjectVersionState::Complete(x) => x,`
			`_ => unreachable!(),`
			`};`
			`let last_v_meta = match last_v_data {`
			`ObjectVersionData::DeleteMarker => return Err(Error::NotFound),`
			`ObjectVersionData::Inline(meta, _) => meta,`
			`ObjectVersionData::FirstBlock(meta, _) => meta,`
			`};`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Fix clippy lints (fix #121) 2021-10-26 08:20:05 +00:00			`if let Some(cached) = try_answer_cached(last_v, last_v_meta, req) {`
add support for caching headers 2021-03-18 14:46:33 +00:00			`return Ok(cached);`
			`}`

Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`let range = match req.headers().get("range") {`
			`Some(range) => {`
Refactor error management in API part 2020-11-08 14:04:30 +00:00			`let range_str = range.to_str()?;`
Improved handling of HTTP ranges - correct HTTP code when range syntax is invalid (fix #140) - when multiple ranges are given, simply ignore and send whole file Co-authored-by: Trinity Pointard <trinity.pointard@gmail.com> Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/157 Reviewed-by: Alex <alex@adnab.me> Co-authored-by: trinity-1686a <trinity.pointard@gmail.com> Co-committed-by: trinity-1686a <trinity.pointard@gmail.com> 2021-11-29 10:52:42 +00:00			`let mut ranges = http_range::HttpRange::parse(range_str, last_v_meta.size)`
			`.map_err(\|e\| (e, last_v_meta.size))?;`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`if ranges.len() > 1 {`
Improved handling of HTTP ranges - correct HTTP code when range syntax is invalid (fix #140) - when multiple ranges are given, simply ignore and send whole file Co-authored-by: Trinity Pointard <trinity.pointard@gmail.com> Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/157 Reviewed-by: Alex <alex@adnab.me> Co-authored-by: trinity-1686a <trinity.pointard@gmail.com> Co-committed-by: trinity-1686a <trinity.pointard@gmail.com> 2021-11-29 10:52:42 +00:00			`// garage does not support multi-range requests yet, so we respond with the entire`
			`// object when multiple ranges are requested`
			`None`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`} else {`
			`ranges.pop()`
			`}`
			`}`
			`None => None,`
			`};`
			`if let Some(range) = range {`
cargo fmt 2020-07-08 15:34:37 +00:00			`return handle_get_range(`
			`garage,`
			`last_v,`
			`last_v_data,`
			`last_v_meta,`
			`range.start,`
			`range.start + range.length,`
			`)`
			`.await;`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`}`

Fix clippy lints (fix #121) 2021-10-26 08:20:05 +00:00			`let resp_builder = object_headers(last_v, last_v_meta)`
Reformulate patch 2020-12-05 18:20:07 +00:00			`.header("Content-Length", format!("{}", last_v_meta.size))`
			`.status(StatusCode::OK);`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
Migrate S3 api to use new model 2020-07-08 15:33:24 +00:00			`match &last_v_data {`
			`ObjectVersionData::DeleteMarker => unreachable!(),`
			`ObjectVersionData::Inline(_, bytes) => {`
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`let body: Body = Body::from(bytes.to_vec());`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`Ok(resp_builder.body(body)?)`
			`}`
Migrate S3 api to use new model 2020-07-08 15:33:24 +00:00			`ObjectVersionData::FirstBlock(_, first_block_hash) => {`
Fix clippy lints (fix #121) 2021-10-26 08:20:05 +00:00			`let read_first_block = garage.block_manager.rpc_get_block(first_block_hash);`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);`

			`let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;`
Replace with option syntaxic sugar 2020-11-11 15:12:42 +00:00			`let version = version.ok_or(Error::NotFound)?;`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00
			`let mut blocks = version`
Refactor model stuff, including cleaner CRDTs 2021-03-10 15:21:56 +00:00			`.blocks`
			`.items()`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.iter()`
Refactor model stuff, including cleaner CRDTs 2021-03-10 15:21:56 +00:00			`.map(\|(_, vb)\| (vb.hash, None))`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`.collect::<Vec<_>>();`
			`blocks[0].1 = Some(first_block);`

			`let body_stream = futures::stream::iter(blocks)`
			`.map(move \|(hash, data_opt)\| {`
			`let garage = garage.clone();`
			`async move {`
			`if let Some(data) = data_opt {`
			`Ok(Bytes::from(data))`
			`} else {`
			`garage`
			`.block_manager`
			`.rpc_get_block(&hash)`
			`.await`
			`.map(Bytes::from)`
			`}`
			`}`
			`})`
			`.buffered(2);`
Content-range fix 2020-12-05 15:37:59 +00:00
Apply cargo fmt; add trace output when request signature is bad 2020-07-08 11:33:02 +00:00			`let body = hyper::body::Body::wrap_stream(body_stream);`
Some basic S3 functionnality 2020-04-24 18:47:11 +00:00			`Ok(resp_builder.body(body)?)`
			`}`
			`}`
			`}`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00
document api crate 2021-03-26 21:32:09 +00:00			`async fn handle_get_range(`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`garage: Arc<Garage>,`
			`version: &ObjectVersion,`
cargo fmt 2020-07-08 15:34:37 +00:00			`version_data: &ObjectVersionData,`
			`version_meta: &ObjectVersionMeta,`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`begin: u64,`
			`end: u64,`
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io 2020-07-07 15:15:53 +00:00			`) -> Result<Response<Body>, Error> {`
Reformulate patch 2020-12-05 18:20:07 +00:00			`let resp_builder = object_headers(version, version_meta)`
			`.header("Content-Length", format!("{}", end - begin))`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`.header(`
			`"Content-Range",`
Content-range fix 2020-12-05 15:37:59 +00:00			`format!("bytes {}-{}/{}", begin, end - 1, version_meta.size),`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`)`
			`.status(StatusCode::PARTIAL_CONTENT);`

Migrate S3 api to use new model 2020-07-08 15:33:24 +00:00			`match &version_data {`
			`ObjectVersionData::DeleteMarker => unreachable!(),`
			`ObjectVersionData::Inline(_meta, bytes) => {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`if end as usize <= bytes.len() {`
Apply cargo fmt; add trace output when request signature is bad 2020-07-08 11:33:02 +00:00			`let body: Body = Body::from(bytes[begin as usize..end as usize].to_vec());`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`Ok(resp_builder.body(body)?)`
			`} else {`
Refactor error management in API part 2020-11-08 14:04:30 +00:00			`None.ok_or_internal_error(`
			`"Requested range not present in inline bytes when it should have been",`
			`)`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`}`
			`}`
Migrate S3 api to use new model 2020-07-08 15:33:24 +00:00			`ObjectVersionData::FirstBlock(_meta, _first_block_hash) => {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`let version = garage.version_table.get(&version.uuid, &EmptyKey).await?;`
			`let version = match version {`
			`Some(v) => v,`
			`None => return Err(Error::NotFound),`
			`};`

Content-range fix 2020-12-05 15:37:59 +00:00			`// We will store here the list of blocks that have an intersection with the requested`
			`// range, as well as their "true offset", which is their actual offset in the complete`
			`// file (whereas block.offset designates the offset of the block WITHIN THE PART`
			`// block.part_number, which is not the same in the case of a multipart upload)`
			`let mut blocks = Vec::with_capacity(std::cmp::min(`
Refactor model stuff, including cleaner CRDTs 2021-03-10 15:21:56 +00:00			`version.blocks.len(),`
			`4 + ((end - begin) / std::cmp::max(version.blocks.items()[0].1.size as u64, 1024))`
			`as usize,`
Content-range fix 2020-12-05 15:37:59 +00:00			`));`
			`let mut true_offset = 0;`
Refactor model stuff, including cleaner CRDTs 2021-03-10 15:21:56 +00:00			`for (_, b) in version.blocks.items().iter() {`
Content-range fix 2020-12-05 15:37:59 +00:00			`if true_offset >= end {`
			`break;`
			`}`
			`// Keep only blocks that have an intersection with the requested range`
			`if true_offset < end && true_offset + b.size > begin {`
fix clippy warnings on api 2021-04-23 20:18:00 +00:00			`blocks.push((*b, true_offset));`
Content-range fix 2020-12-05 15:37:59 +00:00			`}`
			`true_offset += b.size;`
			`}`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00
			`let body_stream = futures::stream::iter(blocks)`
Content-range fix 2020-12-05 15:37:59 +00:00			`.map(move \|(block, true_offset)\| {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`let garage = garage.clone();`
			`async move {`
			`let data = garage.block_manager.rpc_get_block(&block.hash).await?;`
Content-range fix 2020-12-05 15:37:59 +00:00			`let data = Bytes::from(data);`
			`let start_in_block = if true_offset > begin {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`0`
			`} else {`
Content-range fix 2020-12-05 15:37:59 +00:00			`begin - true_offset`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`};`
Content-range fix 2020-12-05 15:37:59 +00:00			`let end_in_block = if true_offset + block.size < end {`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`block.size`
			`} else {`
Content-range fix 2020-12-05 15:37:59 +00:00			`end - true_offset`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`};`
fix clippy warnings on api 2021-04-23 20:18:00 +00:00			`Result::<Bytes, Error>::Ok(`
Content-range fix 2020-12-05 15:37:59 +00:00			`data.slice(start_in_block as usize..end_in_block as usize),`
fix clippy warnings on api 2021-04-23 20:18:00 +00:00			`)`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`}`
			`})`
			`.buffered(2);`
Content-range fix 2020-12-05 15:37:59 +00:00
Apply cargo fmt; add trace output when request signature is bad 2020-07-08 11:33:02 +00:00			`let body = hyper::body::Body::wrap_stream(body_stream);`
Implement HTTP ranges in get 2020-05-04 13:09:23 +00:00			`Ok(resp_builder.body(body)?)`
			`}`
			`}`
			`}`