rune/garage
1
0
Fork 0
forked from Deuxfleurs/garage
Code Issues Pull requests Projects Releases Wiki Activity
garage/src/api/s3_get.rs

228 lines
5.9 KiB
Rust
Raw Normal View History

Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
use std::sync::Arc;
use std::time::{Duration, UNIX_EPOCH};
use futures::stream::*;
use hyper::body::Bytes;
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
use hyper::{Body, Request, Response, StatusCode};
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
use garage_util::error::Error;
use garage_table::EmptyKey;
Rename garage_core to garage_model
2020-07-07 11:59:22 +00:00
use garage_model::garage::Garage;
use garage_model::object_table::*;
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
fn object_headers(version: &ObjectVersion) -> http::response::Builder {
let date = UNIX_EPOCH + Duration::from_millis(version.timestamp);
let date_str = httpdate::fmt_http_date(date);
Response::builder()
.header("Content-Type", version.mime_type.to_string())
.header("Content-Length", format!("{}", version.size))
.header("Last-Modified", date_str)
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
.header("Accept-Ranges", format!("bytes"))
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
}
pub async fn handle_head(
garage: Arc<Garage>,
bucket: &str,
key: &str,
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
) -> Result<Response<Body>, Error> {
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
let object = match garage
.object_table
.get(&bucket.to_string(), &key.to_string())
.await?
{
None => return Err(Error::NotFound),
Some(o) => o,
};
let version = match object
.versions()
.iter()
.rev()
Prepare for multipart uploads
2020-04-26 18:55:13 +00:00
.filter(|v| v.is_data())
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
.next()
{
Some(v) => v,
None => return Err(Error::NotFound),
};
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
let body: Body = Body::from(vec![]);
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
let response = object_headers(&version)
.status(StatusCode::OK)
.body(body)
.unwrap();
Ok(response)
}
pub async fn handle_get(
garage: Arc<Garage>,
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
req: &Request<Body>,
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
bucket: &str,
key: &str,
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
) -> Result<Response<Body>, Error> {
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
let object = match garage
.object_table
.get(&bucket.to_string(), &key.to_string())
.await?
{
None => return Err(Error::NotFound),
Some(o) => o,
};
let last_v = match object
.versions()
.iter()
.rev()
Prepare for multipart uploads
2020-04-26 18:55:13 +00:00
.filter(|v| v.is_complete())
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
.next()
{
Some(v) => v,
None => return Err(Error::NotFound),
};
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
let range = match req.headers().get("range") {
Some(range) => {
let range_str = range
.to_str()
.map_err(|e| Error::BadRequest(format!("Invalid range header: {}", e)))?;
let mut ranges = http_range::HttpRange::parse(range_str, last_v.size)
.map_err(|_e| Error::BadRequest(format!("Invalid range")))?;
if ranges.len() > 1 {
return Err(Error::BadRequest(format!("Multiple ranges not supported")));
} else {
ranges.pop()
}
}
None => None,
};
if let Some(range) = range {
return handle_get_range(garage, last_v, range.start, range.start + range.length).await;
}
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
let resp_builder = object_headers(&last_v).status(StatusCode::OK);
match &last_v.data {
Implement multipart uploads
2020-04-26 20:39:32 +00:00
ObjectVersionData::Uploading => Err(Error::Message(format!(
"Version is_complete() but data is stil Uploading (internal error)"
))),
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
ObjectVersionData::DeleteMarker => Err(Error::NotFound),
ObjectVersionData::Inline(bytes) => {
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
let body: Body = Body::from(bytes.to_vec());
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
Ok(resp_builder.body(body)?)
}
ObjectVersionData::FirstBlock(first_block_hash) => {
let read_first_block = garage.block_manager.rpc_get_block(&first_block_hash);
let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);
let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;
let version = match version {
Some(v) => v,
None => return Err(Error::NotFound),
};
let mut blocks = version
.blocks()
.iter()
.map(|vb| (vb.hash, None))
.collect::<Vec<_>>();
blocks[0].1 = Some(first_block);
let body_stream = futures::stream::iter(blocks)
.map(move |(hash, data_opt)| {
let garage = garage.clone();
async move {
if let Some(data) = data_opt {
Ok(Bytes::from(data))
} else {
garage
.block_manager
.rpc_get_block(&hash)
.await
.map(Bytes::from)
}
}
})
.buffered(2);
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
//let body: Body = Box::new(StreamBody::new(Box::pin(body_stream)));
let body = hyper::body::Body::wrap_stream(body_stream);
Some basic S3 functionnality
2020-04-24 18:47:11 +00:00
Ok(resp_builder.body(body)?)
}
}
}
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
pub async fn handle_get_range(
garage: Arc<Garage>,
version: &ObjectVersion,
begin: u64,
end: u64,
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
) -> Result<Response<Body>, Error> {
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
if end > version.size {
return Err(Error::BadRequest(format!("Range not included in file")));
}
let resp_builder = object_headers(&version)
.header(
"Content-Range",
format!("bytes {}-{}/{}", begin, end, version.size),
)
.status(StatusCode::PARTIAL_CONTENT);
match &version.data {
ObjectVersionData::Uploading => Err(Error::Message(format!(
"Version is_complete() but data is stil Uploading (internal error)"
))),
ObjectVersionData::DeleteMarker => Err(Error::NotFound),
ObjectVersionData::Inline(bytes) => {
if end as usize <= bytes.len() {
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
let body: Body = Body::from(
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
bytes[begin as usize..end as usize].to_vec(),
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
);
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
Ok(resp_builder.body(body)?)
} else {
Err(Error::Message(format!("Internal error: requested range not present in inline bytes when it should have been")))
}
}
ObjectVersionData::FirstBlock(_first_block_hash) => {
let version = garage.version_table.get(&version.uuid, &EmptyKey).await?;
let version = match version {
Some(v) => v,
None => return Err(Error::NotFound),
};
let blocks = version
.blocks()
.iter()
.cloned()
.filter(|block| block.offset + block.size > begin && block.offset < end)
.collect::<Vec<_>>();
let body_stream = futures::stream::iter(blocks)
.map(move |block| {
let garage = garage.clone();
async move {
let data = garage.block_manager.rpc_get_block(&block.hash).await?;
let start_in_block = if block.offset > begin {
0
} else {
begin - block.offset
};
let end_in_block = if block.offset + block.size < end {
block.size
} else {
end - block.offset
};
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
Result::<Bytes,Error>::Ok(Bytes::from(
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
data[start_in_block as usize..end_in_block as usize].to_vec(),
))
}
})
.buffered(2);
Update to Hyper 0.13.6 that accepts non-Sync streams in wrap_stream. Simplifies code and makes it possible to publish on crates.io
2020-07-07 15:15:53 +00:00
//let body: Body = Box::new(StreamBody::new(Box::pin(body_stream)));
let body = hyper::body::Body::wrap_stream(body_stream);
Implement HTTP ranges in get
2020-05-04 13:09:23 +00:00
Ok(resp_builder.body(body)?)
}
}
}
Reference in a new issue Copy permalink