Deuxfleurs/garage
44
56
Fork 74
Code Issues 100 Pull requests 14 Releases 30 Wiki Activity

add proper request router for s3 api #163

Merged
lx merged 5 commits from trinity-1686a/garage:s3-router into main 2021-12-06 14:17:47 +00:00
Conversation 2 Commits 5 Files changed 7 +1476 -185
7 changed files with 1476 additions and 185 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

292
src/api/api_server.rs
View file

@ -1,4 +1,3 @@
use std::collections::HashMap;
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::Arc; use std::sync::Arc;
@ -6,7 +5,7 @@ use futures::future::Future;
use hyper::header; use hyper::header;
use hyper::server::conn::AddrStream; use hyper::server::conn::AddrStream;
use hyper::service::{make_service_fn, service_fn}; use hyper::service::{make_service_fn, service_fn};
use hyper::{Body, Method, Request, Response, Server}; use hyper::{Body, Request, Response, Server};
use garage_util::error::Error as GarageError; use garage_util::error::Error as GarageError;
@ -22,6 +21,7 @@ use crate::s3_delete::*;
use crate::s3_get::*; use crate::s3_get::*;
use crate::s3_list::*; use crate::s3_list::*;
use crate::s3_put::*; use crate::s3_put::*;
use crate::s3_router::{Authorization, Endpoint};
/// Run the S3 API server /// Run the S3 API server
pub async fn run_api_server( pub async fn run_api_server(
@ -86,8 +86,6 @@ async fn handler(
} }
async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> { async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<Body>, Error> {
let path = req.uri().path().to_string();
let path = percent_encoding::percent_decode_str(&path).decode_utf8()?;
let (api_key, content_sha256) = check_signature(&garage, &req).await?; let (api_key, content_sha256) = check_signature(&garage, &req).await?;
let authority = req let authority = req
@ -105,166 +103,158 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
.as_ref() .as_ref()
.and_then(|root_domain| host_to_bucket(&host, root_domain)); .and_then(|root_domain| host_to_bucket(&host, root_domain));
if path == "/" && bucket.is_none() { let endpoint = Endpoint::from_request(&req, bucket.map(ToOwned::to_owned))?;
return handle_list_buckets(&api_key); let allowed = match endpoint.authorization_type() {
} Authorization::None => true,
Authorization::Read(bucket) => api_key.allow_read(bucket),
let (bucket, key) = parse_bucket_key(&path, bucket)?; Authorization::Write(bucket) => api_key.allow_write(bucket),
let allowed = match req.method() {
&Method::HEAD | &Method::GET => api_key.allow_read(bucket),
_ => api_key.allow_write(bucket),
}; };
if !allowed { if !allowed {
return Err(Error::Forbidden( return Err(Error::Forbidden(
"Operation is not allowed for this key.".to_string(), "Operation is not allowed for this key.".to_string(),
)); ));
} }
let mut params = HashMap::new(); match endpoint {
if let Some(query) = req.uri().query() { Endpoint::ListBuckets => handle_list_buckets(&api_key),
let query_pairs = url::form_urlencoded::parse(query.as_bytes()); Endpoint::HeadObject { bucket, key, .. } => handle_head(garage, &req, &bucket, &key).await,
for (key, val) in query_pairs { Endpoint::GetObject { bucket, key, .. } => handle_get(garage, &req, &bucket, &key).await,
params.insert(key.to_lowercase(), val.to_string()); Endpoint::UploadPart {
bucket,
key,
part_number,
upload_id,
} => {
handle_put_part(
garage,
req,
&bucket,
&key,
part_number,
&upload_id,
content_sha256,
)
.await
} }
} Endpoint::CopyObject { bucket, key } => {
let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?;
if let Some(key) = key { let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?;
match *req.method() { let (source_bucket, source_key) = parse_bucket_key(&copy_source, None)?;
Method::HEAD => { if !api_key.allow_read(source_bucket) {
// HeadObject query return Err(Error::Forbidden(format!(
Ok(handle_head(garage, &req, bucket, key).await?) "Reading from bucket {} not allowed for this key",
source_bucket
)));
} }
Method::GET => { let source_key = source_key.ok_or_bad_request("No source key specified")?;
// GetObject query handle_copy(garage, &req, &bucket, &key, source_bucket, source_key).await
Ok(handle_get(garage, &req, bucket, key).await?) }
} Endpoint::PutObject { bucket, key } => {
Method::PUT => { handle_put(garage, req, &bucket, &key, content_sha256).await
if params.contains_key(&"partnumber".to_string()) }
&& params.contains_key(&"uploadid".to_string()) Endpoint::AbortMultipartUpload {
{ bucket,
// UploadPart query key,
let part_number = params.get("partnumber").unwrap(); upload_id,
let upload_id = params.get("uploadid").unwrap(); } => handle_abort_multipart_upload(garage, &bucket, &key, &upload_id).await,
Ok(handle_put_part( Endpoint::DeleteObject { bucket, key, .. } => handle_delete(garage, &bucket, &key).await,
garage, Endpoint::CreateMultipartUpload { bucket, key } => {
req, handle_create_multipart_upload(garage, &req, &bucket, &key).await
}
Endpoint::CompleteMultipartUpload {
bucket,
key,
upload_id,
} => {
handle_complete_multipart_upload(garage, req, &bucket, &key, &upload_id, content_sha256)
.await
}
Endpoint::CreateBucket { bucket } => {
debug!(
"Body: {}",
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
.unwrap_or("<invalid utf8>")
);
let empty_body: Body = Body::from(vec![]);
let response = Response::builder()
.header("Location", format!("/{}", bucket))
.body(empty_body)
.unwrap();
Ok(response)
}
Endpoint::HeadBucket { .. } => {
let empty_body: Body = Body::from(vec![]);
let response = Response::builder().body(empty_body).unwrap();
Ok(response)
}
Endpoint::DeleteBucket { .. } => Err(Error::Forbidden(
"Cannot delete buckets using S3 api, please talk to Garage directly".into(),
)),
Endpoint::GetBucketLocation { .. } => handle_get_bucket_location(garage),
Endpoint::GetBucketVersioning { .. } => handle_get_bucket_versioning(),
Endpoint::ListObjects {
bucket,
delimiter,
encoding_type,
marker,
max_keys,
prefix,
} => {
handle_list(
garage,
&ListObjectsQuery {
is_v2: false,
bucket,
delimiter: delimiter.map(|d| d.to_string()),
max_keys: max_keys.unwrap_or(1000),
prefix: prefix.unwrap_or_default(),
marker,
continuation_token: None,
start_after: None,
urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
},
)
.await
}
Endpoint::ListObjectsV2 {
bucket,
delimiter,
encoding_type,
max_keys,
prefix,
continuation_token,
start_after,
list_type,
..
} => {
if list_type == "2" {
handle_list(
garage,
&ListObjectsQuery {
lx commented 2021-11-30 14:01:23 +00:00
Outdated
Review
Copy link

At this point we should probably return 400 bad request if list_type is not equal to "v2". It doesn't make much sense calling handle_list with is_v2 = false here because the arguments we are giving are those of ListObjectsV2: continuation_token, start_after, etc.

(question: what happens on AWS S3 if we give a list_type value that is not "v2" ?)

At this point we should probably return 400 bad request if `list_type` is not equal to `"v2"`. It doesn't make much sense calling `handle_list` with `is_v2 = false` here because the arguments we are giving are those of ListObjectsV2: continuation_token, start_after, etc. (question: what happens on AWS S3 if we give a list_type value that is not "v2" ?)
is_v2: true,
bucket, bucket,
key, delimiter: delimiter.map(|d| d.to_string()),
part_number, max_keys: max_keys.unwrap_or(1000),
upload_id, prefix: prefix.unwrap_or_default(),
content_sha256, marker: None,
) continuation_token,
.await?) start_after,
} else if req.headers().contains_key("x-amz-copy-source") { urlencode_resp: encoding_type.map(|e| e == "url").unwrap_or(false),
// CopyObject query },
let copy_source = req.headers().get("x-amz-copy-source").unwrap().to_str()?; )
let copy_source = .await
percent_encoding::percent_decode_str(copy_source).decode_utf8()?; } else {
let (source_bucket, source_key) = parse_bucket_key(&copy_source, None)?; Err(Error::BadRequest(format!(
if !api_key.allow_read(source_bucket) { "Invalid endpoint: list-type={}",
return Err(Error::Forbidden(format!( list_type
lx commented 2021-11-30 13:56:13 +00:00
Outdated
Review
Copy link

TODO (not necessarily in this PR): add a new error type so that we return HTTP 501

TODO (not necessarily in this PR): add a new error type so that we return HTTP 501
👍 1
"Reading from bucket {} not allowed for this key", )))
source_bucket
)));
}
let source_key = source_key.ok_or_bad_request("No source key specified")?;
Ok(handle_copy(garage, &req, bucket, key, source_bucket, source_key).await?)
} else {
// PutObject query
Ok(handle_put(garage, req, bucket, key, content_sha256).await?)
}
} }
Method::DELETE => {
if params.contains_key(&"uploadid".to_string()) {
// AbortMultipartUpload query
let upload_id = params.get("uploadid").unwrap();
Ok(handle_abort_multipart_upload(garage, bucket, key, upload_id).await?)
} else {
// DeleteObject query
Ok(handle_delete(garage, bucket, key).await?)
}
}
Method::POST => {
if params.contains_key(&"uploads".to_string()) {
// CreateMultipartUpload call
Ok(handle_create_multipart_upload(garage, &req, bucket, key).await?)
} else if params.contains_key(&"uploadid".to_string()) {
// CompleteMultipartUpload call
let upload_id = params.get("uploadid").unwrap();
Ok(handle_complete_multipart_upload(
garage,
req,
bucket,
key,
upload_id,
content_sha256,
)
.await?)
} else {
Err(Error::BadRequest(
"Not a CreateMultipartUpload call, what is it?".to_string(),
))
}
}
_ => Err(Error::BadRequest("Invalid method".to_string())),
} }
} else { Endpoint::DeleteObjects { bucket } => {
match *req.method() { handle_delete_objects(garage, &bucket, req, content_sha256).await
Method::PUT => {
// CreateBucket
// If we're here, the bucket already exists, so just answer ok
debug!(
"Body: {}",
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
.unwrap_or("<invalid utf8>")
);
let empty_body: Body = Body::from(vec![]);
let response = Response::builder()
.header("Location", format!("/{}", bucket))
.body(empty_body)
.unwrap();
Ok(response)
}
Method::HEAD => {
// HeadBucket
let empty_body: Body = Body::from(vec![]);
let response = Response::builder().body(empty_body).unwrap();
Ok(response)
}
Method::DELETE => {
// DeleteBucket query
Err(Error::Forbidden(
"Cannot delete buckets using S3 api, please talk to Garage directly".into(),
))
}
Method::GET => {
if params.contains_key("location") {
// GetBucketLocation call
Ok(handle_get_bucket_location(garage)?)
} else if params.contains_key("versioning") {
// GetBucketVersioning
Ok(handle_get_bucket_versioning()?)
} else {
// ListObjects or ListObjectsV2 query
let q = parse_list_objects_query(bucket, &params)?;
Ok(handle_list(garage, &q).await?)
}
}
Method::POST => {
if params.contains_key(&"delete".to_string()) {
// DeleteObjects
Ok(handle_delete_objects(garage, bucket, req, content_sha256).await?)
} else {
debug!(
"Body: {}",
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
.unwrap_or("<invalid utf8>")
);
Err(Error::BadRequest("Unsupported call".to_string()))
}
}
_ => Err(Error::BadRequest("Invalid method".to_string())),
} }
endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
} }
} }

5
src/api/error.rs
View file

@ -65,6 +65,10 @@ pub enum Error {
/// The client sent an invalid request /// The client sent an invalid request
#[error(display = "Bad request: {}", _0)] #[error(display = "Bad request: {}", _0)]
BadRequest(String), BadRequest(String),
/// The client sent a request for an action not supported by garage
#[error(display = "Unimplemented action: {}", _0)]
NotImplemented(String),
} }
impl From<roxmltree::Error> for Error { impl From<roxmltree::Error> for Error {
@ -94,6 +98,7 @@ impl Error {
StatusCode::INTERNAL_SERVER_ERROR StatusCode::INTERNAL_SERVER_ERROR
} }
Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE, Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE,
Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED,
_ => StatusCode::BAD_REQUEST, _ => StatusCode::BAD_REQUEST,
} }
} }

2
src/api/helpers.rs
View file

@ -26,7 +26,7 @@ pub fn host_to_bucket<'a>(host: &'a str, root: &str) -> Option<&'a str> {
/// The HTTP host contains both a host and a port. /// The HTTP host contains both a host and a port.
/// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6 /// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6
/// We do not use the collect pattern as there is no way in std rust to collect over a stack allocated value /// We do not use the collect pattern as there is no way in std rust to collect over a stack allocated value
/// check here: https://docs.rs/collect_slice/1.2.0/collect_slice/ /// check here: <https://docs.rs/collect_slice/1.2.0/collect_slice/>
pub fn authority_to_host(authority: &str) -> Result<String, Error> { pub fn authority_to_host(authority: &str) -> Result<String, Error> {
let mut iter = authority.chars().enumerate(); let mut iter = authority.chars().enumerate();
let (_, first_char) = iter let (_, first_char) = iter

1
src/api/lib.rs
View file

@ -19,4 +19,5 @@ mod s3_delete;
pub mod s3_get; pub mod s3_get;
mod s3_list; mod s3_list;
mod s3_put; mod s3_put;
mod s3_router;
mod s3_xml; mod s3_xml;

28
src/api/s3_list.rs
View file

@ -1,4 +1,4 @@
use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::collections::{BTreeMap, BTreeSet};
use std::sync::Arc; use std::sync::Arc;
use hyper::{Body, Response}; use hyper::{Body, Response};
@ -35,32 +35,6 @@ struct ListResultInfo {
etag: String, etag: String,
} }
pub fn parse_list_objects_query(
bucket: &str,
params: &HashMap<String, String>,
) -> Result<ListObjectsQuery, Error> {
Ok(ListObjectsQuery {
is_v2: params.get("list-type").map(|x| x == "2").unwrap_or(false),
bucket: bucket.to_string(),
delimiter: params.get("delimiter").filter(|x| !x.is_empty()).cloned(),
max_keys: params
.get("max-keys")
.map(|x| {
x.parse::<usize>()
.ok_or_bad_request("Invalid value for max-keys")
})
.unwrap_or(Ok(1000))?,
prefix: params.get("prefix").cloned().unwrap_or_default(),
marker: params.get("marker").cloned(),
continuation_token: params.get("continuation-token").cloned(),
start_after: params.get("start-after").cloned(),
urlencode_resp: params
.get("encoding-type")
.map(|x| x == "url")
.unwrap_or(false),
})
}
pub async fn handle_list( pub async fn handle_list(
garage: Arc<Garage>, garage: Arc<Garage>,
query: &ListObjectsQuery, query: &ListObjectsQuery,

7
src/api/s3_put.rs
View file

@ -354,15 +354,10 @@ pub async fn handle_put_part(
req: Request<Body>, req: Request<Body>,
bucket: &str, bucket: &str,
key: &str, key: &str,
part_number_str: &str, part_number: u64,
upload_id: &str, upload_id: &str,
content_sha256: Option<Hash>, content_sha256: Option<Hash>,
) -> Result<Response<Body>, Error> { ) -> Result<Response<Body>, Error> {
// Check parameters
let part_number = part_number_str
.parse::<u64>()
.ok_or_bad_request("Invalid part number")?;
let version_uuid = decode_upload_id(upload_id)?; let version_uuid = decode_upload_id(upload_id)?;
let content_md5 = match req.headers().get("content-md5") { let content_md5 = match req.headers().get("content-md5") {

1326
src/api/s3_router.rs Normal file
View file

File diff suppressed because it is too large Load diff