Deuxfleurs/garage
44
56
Fork 74
Code Issues 100 Pull requests 14 Releases 30 Wiki Activity

CreateBuckets and DeleteBuckets #181

Merged
lx merged 9 commits from create-delete-buckets into main 2022-01-10 13:16:02 +00:00
Conversation 0 Commits 9 Files changed 16 +421 -73
16 changed files with 421 additions and 73 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
doc/book/src/reference_manual/s3_compatibility.md
View file

@ -28,9 +28,9 @@ All APIs that are not mentionned are not implemented and will return a 400 bad r
| AbortMultipartUpload | Implemented | | AbortMultipartUpload | Implemented |
| CompleteMultipartUpload | Implemented | | CompleteMultipartUpload | Implemented |
| CopyObject | Implemented | | CopyObject | Implemented |
| CreateBucket | Unsupported, stub (see below) | | CreateBucket | Implemented |
| CreateMultipartUpload | Implemented | | CreateMultipartUpload | Implemented |
| DeleteBucket | Unsupported (see below) | | DeleteBucket | Implemented |
| DeleteBucketWebsite | Implemented | | DeleteBucketWebsite | Implemented |
| DeleteObject | Implemented | | DeleteObject | Implemented |
| DeleteObjects | Implemented | | DeleteObjects | Implemented |
@ -48,11 +48,6 @@ All APIs that are not mentionned are not implemented and will return a 400 bad r
| UploadPart | Implemented | | UploadPart | Implemented |
- **CreateBucket:** Garage does not yet accept creating buckets or giving access using API calls, it has to be done using the CLI tools. CreateBucket will return a 200 if the bucket exists and user has write access, and a 403 Forbidden in all other cases.
- **DeleteBucket:** Garage does not yet accept deleting buckets using API calls, it has to be done using the CLI tools. This request will return a 403 Forbidden.
- **GetBucketVersioning:** Stub implementation (Garage does not yet support versionning so this always returns - **GetBucketVersioning:** Stub implementation (Garage does not yet support versionning so this always returns
"versionning not enabled"). "versionning not enabled").

4
doc/book/src/working_documents/compatibility_target.md
View file

@ -9,8 +9,8 @@ your motivations for doing so in the PR message.
| **S-tier** (high priority) | | | **S-tier** (high priority) | |
| | HeadBucket | | | HeadBucket |
| | GetBucketLocation | | | GetBucketLocation |
| | [*CreateBucket*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/97) | | | CreateBucket |
| | [*DeleteBucket*](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/97) | | | DeleteBucket |
| | ListBuckets | | | ListBuckets |
| | ListObjects | | | ListObjects |
| | ListObjectsV2 | | | ListObjectsV2 |

31
src/api/api_server.rs
View file

@ -107,6 +107,12 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
.and_then(|root_domain| host_to_bucket(&host, root_domain)); .and_then(|root_domain| host_to_bucket(&host, root_domain));
let endpoint = Endpoint::from_request(&req, bucket.map(ToOwned::to_owned))?; let endpoint = Endpoint::from_request(&req, bucket.map(ToOwned::to_owned))?;
debug!("Endpoint: {:?}", endpoint);
// Special code path for CreateBucket API endpoint
if let Endpoint::CreateBucket { bucket } = endpoint {
return handle_create_bucket(&garage, req, content_sha256, api_key, bucket).await;
}
let bucket_name = match endpoint.get_bucket() { let bucket_name = match endpoint.get_bucket() {
None => return handle_request_without_bucket(garage, req, api_key, endpoint).await, None => return handle_request_without_bucket(garage, req, api_key, endpoint).await,
@ -118,6 +124,7 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
let allowed = match endpoint.authorization_type() { let allowed = match endpoint.authorization_type() {
Authorization::Read(_) => api_key.allow_read(&bucket_id), Authorization::Read(_) => api_key.allow_read(&bucket_id),
Authorization::Write(_) => api_key.allow_write(&bucket_id), Authorization::Write(_) => api_key.allow_write(&bucket_id),
Authorization::Owner(_) => api_key.allow_owner(&bucket_id),
_ => unreachable!(), _ => unreachable!(),
}; };
@ -188,27 +195,15 @@ async fn handler_inner(garage: Arc<Garage>, req: Request<Body>) -> Result<Respon
) )
.await .await
} }
Endpoint::CreateBucket { bucket } => { Endpoint::CreateBucket { .. } => unreachable!(),
debug!(
"Body: {}",
std::str::from_utf8(&hyper::body::to_bytes(req.into_body()).await?)
.unwrap_or("<invalid utf8>")
);
let empty_body: Body = Body::from(vec![]);
let response = Response::builder()
.header("Location", format!("/{}", bucket))
.body(empty_body)
.unwrap();
Ok(response)
}
Endpoint::HeadBucket { .. } => { Endpoint::HeadBucket { .. } => {
let empty_body: Body = Body::from(vec![]); let empty_body: Body = Body::from(vec![]);
let response = Response::builder().body(empty_body).unwrap(); let response = Response::builder().body(empty_body).unwrap();
Ok(response) Ok(response)
} }
Endpoint::DeleteBucket { .. } => Err(Error::Forbidden( Endpoint::DeleteBucket { .. } => {
"Cannot delete buckets using S3 api, please talk to Garage directly".into(), handle_delete_bucket(&garage, bucket_id, bucket_name, api_key).await
)), }
Endpoint::GetBucketLocation { .. } => handle_get_bucket_location(garage), Endpoint::GetBucketLocation { .. } => handle_get_bucket_location(garage),
Endpoint::GetBucketVersioning { .. } => handle_get_bucket_versioning(), Endpoint::GetBucketVersioning { .. } => handle_get_bucket_versioning(),
Endpoint::ListObjects { Endpoint::ListObjects {
@ -303,7 +298,7 @@ async fn resolve_bucket(
let api_key_params = api_key let api_key_params = api_key
.state .state
.as_option() .as_option()
.ok_or_else(|| Error::Forbidden("Operation is not allowed for this key.".to_string()))?; .ok_or_internal_error("Key should not be deleted at this point")?;
if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) { if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) {
Ok(*bucket_id) Ok(*bucket_id)
@ -312,7 +307,7 @@ async fn resolve_bucket(
.bucket_helper() .bucket_helper()
.resolve_global_bucket_name(bucket_name) .resolve_global_bucket_name(bucket_name)
.await? .await?
.ok_or(Error::NotFound)?) .ok_or(Error::NoSuchBucket)?)
} }
} }

30
src/api/error.rs
View file

@ -35,8 +35,24 @@ pub enum Error {
AuthorizationHeaderMalformed(String), AuthorizationHeaderMalformed(String),
/// The object requested don't exists /// The object requested don't exists
#[error(display = "Not found")] #[error(display = "Key not found")]
NotFound, NoSuchKey,
/// The bucket requested don't exists
#[error(display = "Bucket not found")]
NoSuchBucket,
/// The multipart upload requested don't exists
#[error(display = "Upload not found")]
NoSuchUpload,
/// Tried to create a bucket that already exist
#[error(display = "Bucket already exists")]
BucketAlreadyExists,
/// Tried to delete a non-empty bucket
#[error(display = "Tried to delete a non-empty bucket")]
BucketNotEmpty,
// Category: bad request // Category: bad request
/// The request contained an invalid UTF-8 sequence in its path or in other parameters /// The request contained an invalid UTF-8 sequence in its path or in other parameters
@ -97,7 +113,8 @@ impl Error {
/// Get the HTTP status code that best represents the meaning of the error for the client /// Get the HTTP status code that best represents the meaning of the error for the client
pub fn http_status_code(&self) -> StatusCode { pub fn http_status_code(&self) -> StatusCode {
match self { match self {
Error::NotFound => StatusCode::NOT_FOUND, Error::NoSuchKey | Error::NoSuchBucket | Error::NoSuchUpload => StatusCode::NOT_FOUND,
Error::BucketNotEmpty | Error::BucketAlreadyExists => StatusCode::CONFLICT,
Error::Forbidden(_) => StatusCode::FORBIDDEN, Error::Forbidden(_) => StatusCode::FORBIDDEN,
Error::InternalError( Error::InternalError(
GarageError::Timeout GarageError::Timeout
@ -115,9 +132,14 @@ impl Error {
pub fn aws_code(&self) -> &'static str { pub fn aws_code(&self) -> &'static str {
match self { match self {
Error::NotFound => "NoSuchKey", Error::NoSuchKey => "NoSuchKey",
Error::NoSuchBucket => "NoSuchBucket",
Error::NoSuchUpload => "NoSuchUpload",
Error::BucketAlreadyExists => "BucketAlreadyExists",
Error::BucketNotEmpty => "BucketNotEmpty",
Error::Forbidden(_) => "AccessDenied", Error::Forbidden(_) => "AccessDenied",
Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed", Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed",
Error::NotImplemented(_) => "NotImplemented",
Error::InternalError( Error::InternalError(
GarageError::Timeout GarageError::Timeout
| GarageError::RemoteError(_) | GarageError::RemoteError(_)

247
src/api/s3_bucket.rs
View file

@ -1,16 +1,21 @@
use std::collections::HashMap; use std::collections::HashMap;
use std::sync::Arc; use std::sync::Arc;
use hyper::{Body, Response}; use hyper::{Body, Request, Response, StatusCode};
use garage_model::bucket_alias_table::*;
use garage_model::bucket_table::Bucket;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_model::key_table::Key; use garage_model::key_table::Key;
use garage_table::util::EmptyKey; use garage_model::permission::BucketKeyPerm;
use garage_table::util::*;
use garage_util::crdt::*; use garage_util::crdt::*;
use garage_util::data::*;
use garage_util::time::*; use garage_util::time::*;
use crate::error::*; use crate::error::*;
use crate::s3_xml; use crate::s3_xml;
use crate::signature::verify_signed_content;
pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>, Error> { pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>, Error> {
let loc = s3_xml::LocationConstraint { let loc = s3_xml::LocationConstraint {
@ -50,7 +55,7 @@ pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result<Respo
.authorized_buckets .authorized_buckets
.items() .items()
.iter() .iter()
.filter(|(_, perms)| perms.allow_read || perms.allow_write || perms.allow_owner) .filter(|(_, perms)| perms.is_any())
.map(|(id, _)| *id) .map(|(id, _)| *id)
.collect::<Vec<_>>(); .collect::<Vec<_>>();
@ -105,3 +110,239 @@ pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result<Respo
.header("Content-Type", "application/xml") .header("Content-Type", "application/xml")
.body(Body::from(xml))?) .body(Body::from(xml))?)
} }
pub async fn handle_create_bucket(
garage: &Garage,
req: Request<Body>,
content_sha256: Option<Hash>,
api_key: Key,
bucket_name: String,
) -> Result<Response<Body>, Error> {
let body = hyper::body::to_bytes(req.into_body()).await?;
verify_signed_content(content_sha256, &body[..])?;
let cmd =
parse_create_bucket_xml(&body[..]).ok_or_bad_request("Invalid create bucket XML query")?;
if let Some(location_constraint) = cmd {
if location_constraint != garage.config.s3_api.s3_region {
return Err(Error::BadRequest(format!(
"Cannot satisfy location constraint `{}`: buckets can only be created in region `{}`",
location_constraint,
garage.config.s3_api.s3_region
)));
}
}
let key_params = api_key
.params()
.ok_or_internal_error("Key should not be deleted at this point")?;
let existing_bucket = if let Some(Some(bucket_id)) = key_params.local_aliases.get(&bucket_name)
{
Some(*bucket_id)
} else {
garage
.bucket_helper()
.resolve_global_bucket_name(&bucket_name)
.await?
};
if let Some(bucket_id) = existing_bucket {
// Check we have write or owner permission on the bucket,
// in that case it's fine, return 200 OK, bucket exists;
// otherwise return a forbidden error.
let kp = api_key.bucket_permissions(&bucket_id);
if !(kp.allow_write || kp.allow_owner) {
return Err(Error::BucketAlreadyExists);
}
} else {
// Create the bucket!
if !is_valid_bucket_name(&bucket_name) {
return Err(Error::BadRequest(format!(
"{}: {}",
bucket_name, INVALID_BUCKET_NAME_MESSAGE
)));
}
let bucket = Bucket::new();
garage.bucket_table.insert(&bucket).await?;
garage
.bucket_helper()
.set_bucket_key_permissions(bucket.id, &api_key.key_id, BucketKeyPerm::ALL_PERMISSIONS)
.await?;
garage
.bucket_helper()
.set_local_bucket_alias(bucket.id, &api_key.key_id, &bucket_name)
.await?;
}
Ok(Response::builder()
.header("Location", format!("/{}", bucket_name))
.body(Body::empty())
.unwrap())
}
pub async fn handle_delete_bucket(
garage: &Garage,
bucket_id: Uuid,
bucket_name: String,
api_key: Key,
) -> Result<Response<Body>, Error> {
let key_params = api_key
.params()
.ok_or_internal_error("Key should not be deleted at this point")?;
let is_local_alias = matches!(key_params.local_aliases.get(&bucket_name), Some(Some(_)));
let mut bucket = garage
.bucket_helper()
.get_existing_bucket(bucket_id)
.await?;
let bucket_state = bucket.state.as_option().unwrap();
// If the bucket has no other aliases, this is a true deletion.
// Otherwise, it is just an alias removal.
let has_other_global_aliases = bucket_state
.aliases
.items()
.iter()
.filter(|(_, _, active)| *active)
.any(|(n, _, _)| is_local_alias || (*n != bucket_name));
let has_other_local_aliases = bucket_state
.local_aliases
.items()
.iter()
.filter(|(_, _, active)| *active)
.any(|((k, n), _, _)| !is_local_alias || *n != bucket_name || *k != api_key.key_id);
if !has_other_global_aliases && !has_other_local_aliases {
// Delete bucket
// Check bucket is empty
let objects = garage
.object_table
.get_range(&bucket_id, None, Some(DeletedFilter::NotDeleted), 10)
.await?;
if !objects.is_empty() {
return Err(Error::BucketNotEmpty);
}
// --- done checking, now commit ---
// 1. delete bucket alias
if is_local_alias {
garage
.bucket_helper()
.unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name)
.await?;
} else {
garage
.bucket_helper()
.unset_global_bucket_alias(bucket_id, &bucket_name)
.await?;
}
// 2. delete authorization from keys that had access
for (key_id, _) in bucket.authorized_keys() {
garage
.bucket_helper()
.set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS)
.await?;
}
// 3. delete bucket
bucket.state = Deletable::delete();
garage.bucket_table.insert(&bucket).await?;
} else if is_local_alias {
// Just unalias
garage
.bucket_helper()
.unset_local_bucket_alias(bucket_id, &api_key.key_id, &bucket_name)
.await?;
} else {
// Just unalias (but from global namespace)
garage
.bucket_helper()
.unset_global_bucket_alias(bucket_id, &bucket_name)
.await?;
}
Ok(Response::builder()
.status(StatusCode::NO_CONTENT)
.body(Body::empty())?)
}
fn parse_create_bucket_xml(xml_bytes: &[u8]) -> Option<Option<String>> {
// Returns None if invalid data
// Returns Some(None) if no location constraint is given
// Returns Some(Some("xxxx")) where xxxx is the given location constraint
let xml_str = std::str::from_utf8(xml_bytes).ok()?;
if xml_str.trim_matches(char::is_whitespace).is_empty() {
return Some(None);
}
let xml = roxmltree::Document::parse(xml_str).ok()?;
let cbc = xml.root().first_child()?;
if !cbc.has_tag_name("CreateBucketConfiguration") {
return None;
}
let mut ret = None;
for item in cbc.children() {
println!("{:?}", item);
if item.has_tag_name("LocationConstraint") {
if ret != None {
return None;
}
ret = Some(item.text()?.to_string());
} else if !item.is_text() {
return None;
}
}
Some(ret)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn create_bucket() {
assert_eq!(parse_create_bucket_xml(br#""#), Some(None));
assert_eq!(
parse_create_bucket_xml(
br#"
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
</CreateBucketConfiguration >
"#
),
Some(None)
);
assert_eq!(
parse_create_bucket_xml(
br#"
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<LocationConstraint>Europe</LocationConstraint>
</CreateBucketConfiguration >
"#
),
Some(Some("Europe".into()))
);
assert_eq!(
parse_create_bucket_xml(
br#"
<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
</Crea >
"#
),
None
);
}
}

8
src/api/s3_copy.rs
View file

@ -27,14 +27,14 @@ pub async fn handle_copy(
.object_table .object_table
.get(&source_bucket_id, &source_key.to_string()) .get(&source_bucket_id, &source_key.to_string())
.await? .await?
.ok_or(Error::NotFound)?; .ok_or(Error::NoSuchKey)?;
let source_last_v = source_object let source_last_v = source_object
.versions() .versions()
.iter() .iter()
.rev() .rev()
.find(|v| v.is_complete()) .find(|v| v.is_complete())
.ok_or(Error::NotFound)?; .ok_or(Error::NoSuchKey)?;
let source_last_state = match &source_last_v.state { let source_last_state = match &source_last_v.state {
ObjectVersionState::Complete(x) => x, ObjectVersionState::Complete(x) => x,
@ -47,7 +47,7 @@ pub async fn handle_copy(
// Implement x-amz-metadata-directive: REPLACE // Implement x-amz-metadata-directive: REPLACE
let old_meta = match source_last_state { let old_meta = match source_last_state {
ObjectVersionData::DeleteMarker => { ObjectVersionData::DeleteMarker => {
return Err(Error::NotFound); return Err(Error::NoSuchKey);
} }
ObjectVersionData::Inline(meta, _bytes) => meta, ObjectVersionData::Inline(meta, _bytes) => meta,
ObjectVersionData::FirstBlock(meta, _fbh) => meta, ObjectVersionData::FirstBlock(meta, _fbh) => meta,
@ -88,7 +88,7 @@ pub async fn handle_copy(
.version_table .version_table
.get(&source_last_v.uuid, &EmptyKey) .get(&source_last_v.uuid, &EmptyKey)
.await?; .await?;
let source_version = source_version.ok_or(Error::NotFound)?; let source_version = source_version.ok_or(Error::NoSuchKey)?;
// Write an "uploading" marker in Object table // Write an "uploading" marker in Object table
// This holds a reference to the object in the Version table // This holds a reference to the object in the Version table

4
src/api/s3_delete.rs
View file

@ -21,7 +21,7 @@ async fn handle_delete_internal(
.object_table .object_table
.get(&bucket_id, &key.to_string()) .get(&bucket_id, &key.to_string())
.await? .await?
.ok_or(Error::NotFound)?; // No need to delete .ok_or(Error::NoSuchKey)?; // No need to delete
let interesting_versions = object.versions().iter().filter(|v| { let interesting_versions = object.versions().iter().filter(|v| {
!matches!( !matches!(
@ -40,7 +40,7 @@ async fn handle_delete_internal(
timestamp = std::cmp::max(timestamp, v.timestamp + 1); timestamp = std::cmp::max(timestamp, v.timestamp + 1);
} }
let deleted_version = version_to_delete.ok_or(Error::NotFound)?; let deleted_version = version_to_delete.ok_or(Error::NoSuchKey)?;
let version_uuid = gen_uuid(); let version_uuid = gen_uuid();

14
src/api/s3_get.rs
View file

@ -92,14 +92,14 @@ pub async fn handle_head(
.object_table .object_table
.get(&bucket_id, &key.to_string()) .get(&bucket_id, &key.to_string())
.await? .await?
.ok_or(Error::NotFound)?; .ok_or(Error::NoSuchKey)?;
let version = object let version = object
.versions() .versions()
.iter() .iter()
.rev() .rev()
.find(|v| v.is_data()) .find(|v| v.is_data())
.ok_or(Error::NotFound)?; .ok_or(Error::NoSuchKey)?;
let version_meta = match &version.state { let version_meta = match &version.state {
ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta, ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta,
@ -131,21 +131,21 @@ pub async fn handle_get(
.object_table .object_table
.get(&bucket_id, &key.to_string()) .get(&bucket_id, &key.to_string())
.await? .await?
.ok_or(Error::NotFound)?; .ok_or(Error::NoSuchKey)?;
let last_v = object let last_v = object
.versions() .versions()
.iter() .iter()
.rev() .rev()
.find(|v| v.is_complete()) .find(|v| v.is_complete())
.ok_or(Error::NotFound)?; .ok_or(Error::NoSuchKey)?;
let last_v_data = match &last_v.state { let last_v_data = match &last_v.state {
ObjectVersionState::Complete(x) => x, ObjectVersionState::Complete(x) => x,
_ => unreachable!(), _ => unreachable!(),
}; };
let last_v_meta = match last_v_data { let last_v_meta = match last_v_data {
ObjectVersionData::DeleteMarker => return Err(Error::NotFound), ObjectVersionData::DeleteMarker => return Err(Error::NoSuchKey),
ObjectVersionData::Inline(meta, _) => meta, ObjectVersionData::Inline(meta, _) => meta,
ObjectVersionData::FirstBlock(meta, _) => meta, ObjectVersionData::FirstBlock(meta, _) => meta,
}; };
@ -196,7 +196,7 @@ pub async fn handle_get(
let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey); let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey);
let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?; let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?;
let version = version.ok_or(Error::NotFound)?; let version = version.ok_or(Error::NoSuchKey)?;
let mut blocks = version let mut blocks = version
.blocks .blocks
@ -261,7 +261,7 @@ async fn handle_get_range(
let version = garage.version_table.get(&version.uuid, &EmptyKey).await?; let version = garage.version_table.get(&version.uuid, &EmptyKey).await?;
let version = match version { let version = match version {
Some(v) => v, Some(v) => v,
None => return Err(Error::NotFound), None => return Err(Error::NoSuchKey),
}; };
// We will store here the list of blocks that have an intersection with the requested // We will store here the list of blocks that have an intersection with the requested

16
src/api/s3_put.rs
View file

@ -382,7 +382,7 @@ pub async fn handle_put_part(
.iter() .iter()
.any(|v| v.uuid == version_uuid && v.is_uploading()) .any(|v| v.uuid == version_uuid && v.is_uploading())
{ {
return Err(Error::NotFound); return Err(Error::NoSuchUpload);
} }
// Copy block to store // Copy block to store
@ -449,15 +449,15 @@ pub async fn handle_complete_multipart_upload(
garage.version_table.get(&version_uuid, &EmptyKey), garage.version_table.get(&version_uuid, &EmptyKey),
)?; )?;
let object = object.ok_or_else(|| Error::BadRequest("Object not found".to_string()))?; let object = object.ok_or(Error::NoSuchKey)?;
let mut object_version = object let mut object_version = object
.versions() .versions()
.iter() .iter()
.find(|v| v.uuid == version_uuid && v.is_uploading()) .find(|v| v.uuid == version_uuid && v.is_uploading())
.cloned() .cloned()
.ok_or_else(|| Error::BadRequest("Version not found".to_string()))?; .ok_or(Error::NoSuchUpload)?;
let version = version.ok_or_else(|| Error::BadRequest("Version not found".to_string()))?; let version = version.ok_or(Error::NoSuchKey)?;
if version.blocks.is_empty() { if version.blocks.is_empty() {
return Err(Error::BadRequest("No data was uploaded".to_string())); return Err(Error::BadRequest("No data was uploaded".to_string()));
} }
@ -538,14 +538,14 @@ pub async fn handle_abort_multipart_upload(
.object_table .object_table
.get(&bucket_id, &key.to_string()) .get(&bucket_id, &key.to_string())
.await?; .await?;
let object = object.ok_or_else(|| Error::BadRequest("Object not found".to_string()))?; let object = object.ok_or(Error::NoSuchKey)?;
let object_version = object let object_version = object
.versions() .versions()
.iter() .iter()
.find(|v| v.uuid == version_uuid && v.is_uploading()); .find(|v| v.uuid == version_uuid && v.is_uploading());
let mut object_version = match object_version { let mut object_version = match object_version {
None => return Err(Error::NotFound), None => return Err(Error::NoSuchUpload),
Some(x) => x.clone(), Some(x) => x.clone(),
}; };
@ -611,9 +611,9 @@ pub(crate) fn get_headers(req: &Request<Body>) -> Result<ObjectVersionHeaders, E
} }
fn decode_upload_id(id: &str) -> Result<Uuid, Error> { fn decode_upload_id(id: &str) -> Result<Uuid, Error> {
let id_bin = hex::decode(id).ok_or_bad_request("Invalid upload ID")?; let id_bin = hex::decode(id).map_err(|_| Error::NoSuchUpload)?;
if id_bin.len() != 32 { if id_bin.len() != 32 {
return None.ok_or_bad_request("Invalid upload ID"); return Err(Error::NoSuchUpload);
} }
let mut uuid = [0u8; 32]; let mut uuid = [0u8; 32];
uuid.copy_from_slice(&id_bin[..]); uuid.copy_from_slice(&id_bin[..]);

51
src/api/s3_router.rs
View file

@ -789,7 +789,6 @@ impl Endpoint {
GetBucketRequestPayment, GetBucketRequestPayment,
GetBucketTagging, GetBucketTagging,
GetBucketVersioning, GetBucketVersioning,
GetBucketWebsite,
GetObject, GetObject,
GetObjectAcl, GetObjectAcl,
GetObjectLegalHold, GetObjectLegalHold,
@ -813,8 +812,22 @@ impl Endpoint {
] ]
} }
.is_some(); .is_some();
let owner = s3_match! {
@extract
self,
bucket,
[
DeleteBucket,
GetBucketWebsite,
PutBucketWebsite,
DeleteBucketWebsite,
]
}
.is_some();
if readonly { if readonly {
Authorization::Read(bucket) Authorization::Read(bucket)
} else if owner {
Authorization::Owner(bucket)
} else { } else {
Authorization::Write(bucket) Authorization::Write(bucket)
} }
@ -830,6 +843,8 @@ pub enum Authorization<'a> {
Read(&'a str), Read(&'a str),
/// Having Write permission on bucket .0 is required /// Having Write permission on bucket .0 is required
Write(&'a str), Write(&'a str),
/// Having Owner permission on bucket .0 is required
Owner(&'a str),
} }
/// This macro is used to generate part of the code in this module. It must be called only one, and /// This macro is used to generate part of the code in this module. It must be called only one, and
@ -985,13 +1000,13 @@ mod tests {
$( $(
assert!( assert!(
matches!( matches!(
parse(stringify!($method), $uri, Some("my_bucket".to_owned()), None), parse(test_cases!{@actual_method $method}, $uri, Some("my_bucket".to_owned()), None),
Endpoint::$variant { .. } Endpoint::$variant { .. }
) )
); );
assert!( assert!(
matches!( matches!(
parse(stringify!($method), concat!("/my_bucket", $uri), None, None), parse(test_cases!{@actual_method $method}, concat!("/my_bucket", $uri), None, None),
Endpoint::$variant { .. } Endpoint::$variant { .. }
) )
); );
@ -999,6 +1014,16 @@ mod tests {
test_cases!{@auth $method $uri} test_cases!{@auth $method $uri}
)* )*
}}; }};
(@actual_method HEAD) => {{ "HEAD" }};
(@actual_method GET) => {{ "GET" }};
(@actual_method OWNER_GET) => {{ "GET" }};
(@actual_method PUT) => {{ "PUT" }};
(@actual_method OWNER_PUT) => {{ "PUT" }};
(@actual_method POST) => {{ "POST" }};
(@actual_method DELETE) => {{ "DELETE" }};
(@actual_method OWNER_DELETE) => {{ "DELETE" }};
(@auth HEAD $uri:expr) => {{ (@auth HEAD $uri:expr) => {{
assert_eq!(parse("HEAD", concat!("/my_bucket", $uri), None, None).authorization_type(), assert_eq!(parse("HEAD", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Read("my_bucket")) Authorization::Read("my_bucket"))
@ -1007,10 +1032,18 @@ mod tests {
assert_eq!(parse("GET", concat!("/my_bucket", $uri), None, None).authorization_type(), assert_eq!(parse("GET", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Read("my_bucket")) Authorization::Read("my_bucket"))
}}; }};
(@auth OWNER_GET $uri:expr) => {{
assert_eq!(parse("GET", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Owner("my_bucket"))
}};
(@auth PUT $uri:expr) => {{ (@auth PUT $uri:expr) => {{
assert_eq!(parse("PUT", concat!("/my_bucket", $uri), None, None).authorization_type(), assert_eq!(parse("PUT", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Write("my_bucket")) Authorization::Write("my_bucket"))
}}; }};
(@auth OWNER_PUT $uri:expr) => {{
assert_eq!(parse("PUT", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Owner("my_bucket"))
}};
(@auth POST $uri:expr) => {{ (@auth POST $uri:expr) => {{
assert_eq!(parse("POST", concat!("/my_bucket", $uri), None, None).authorization_type(), assert_eq!(parse("POST", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Write("my_bucket")) Authorization::Write("my_bucket"))
@ -1019,6 +1052,10 @@ mod tests {
assert_eq!(parse("DELETE", concat!("/my_bucket", $uri), None, None).authorization_type(), assert_eq!(parse("DELETE", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Write("my_bucket")) Authorization::Write("my_bucket"))
}}; }};
(@auth OWNER_DELETE $uri:expr) => {{
assert_eq!(parse("DELETE", concat!("/my_bucket", $uri), None, None).authorization_type(),
Authorization::Owner("my_bucket"))
}};
} }
#[test] #[test]
@ -1094,7 +1131,7 @@ mod tests {
PUT "/" => CreateBucket PUT "/" => CreateBucket
POST "/example-object?uploads" => CreateMultipartUpload POST "/example-object?uploads" => CreateMultipartUpload
POST "/{Key+}?uploads" => CreateMultipartUpload POST "/{Key+}?uploads" => CreateMultipartUpload
DELETE "/" => DeleteBucket OWNER_DELETE "/" => DeleteBucket
DELETE "/?analytics&id=list1" => DeleteBucketAnalyticsConfiguration DELETE "/?analytics&id=list1" => DeleteBucketAnalyticsConfiguration
DELETE "/?analytics&id=Id" => DeleteBucketAnalyticsConfiguration DELETE "/?analytics&id=Id" => DeleteBucketAnalyticsConfiguration
DELETE "/?cors" => DeleteBucketCors DELETE "/?cors" => DeleteBucketCors
@ -1109,7 +1146,7 @@ mod tests {
DELETE "/?policy" => DeleteBucketPolicy DELETE "/?policy" => DeleteBucketPolicy
DELETE "/?replication" => DeleteBucketReplication DELETE "/?replication" => DeleteBucketReplication
DELETE "/?tagging" => DeleteBucketTagging DELETE "/?tagging" => DeleteBucketTagging
DELETE "/?website" => DeleteBucketWebsite OWNER_DELETE "/?website" => DeleteBucketWebsite
DELETE "/my-second-image.jpg" => DeleteObject DELETE "/my-second-image.jpg" => DeleteObject
DELETE "/my-third-image.jpg?versionId=UIORUnfndfiufdisojhr398493jfdkjFJjkndnqUifhnw89493jJFJ" => DeleteObject DELETE "/my-third-image.jpg?versionId=UIORUnfndfiufdisojhr398493jfdkjFJjkndnqUifhnw89493jJFJ" => DeleteObject
DELETE "/Key+?versionId=VersionId" => DeleteObject DELETE "/Key+?versionId=VersionId" => DeleteObject
@ -1138,7 +1175,7 @@ mod tests {
GET "/?requestPayment" => GetBucketRequestPayment GET "/?requestPayment" => GetBucketRequestPayment
GET "/?tagging" => GetBucketTagging GET "/?tagging" => GetBucketTagging
GET "/?versioning" => GetBucketVersioning GET "/?versioning" => GetBucketVersioning
GET "/?website" => GetBucketWebsite OWNER_GET "/?website" => GetBucketWebsite
GET "/my-image.jpg" => GetObject GET "/my-image.jpg" => GetObject
GET "/myObject?versionId=3/L4kqtJlcpXroDTDmpUMLUo" => GetObject GET "/myObject?versionId=3/L4kqtJlcpXroDTDmpUMLUo" => GetObject
GET "/Junk3.txt?response-cache-control=No-cache&response-content-disposition=attachment%3B%20filename%3Dtesting.txt&response-content-encoding=x-gzip&response-content-language=mi%2C%20en&response-expires=Thu%2C%2001%20Dec%201994%2016:00:00%20GMT" => GetObject GET "/Junk3.txt?response-cache-control=No-cache&response-content-disposition=attachment%3B%20filename%3Dtesting.txt&response-content-encoding=x-gzip&response-content-language=mi%2C%20en&response-expires=Thu%2C%2001%20Dec%201994%2016:00:00%20GMT" => GetObject
@ -1212,7 +1249,7 @@ mod tests {
PUT "/?requestPayment" => PutBucketRequestPayment PUT "/?requestPayment" => PutBucketRequestPayment
PUT "/?tagging" => PutBucketTagging PUT "/?tagging" => PutBucketTagging
PUT "/?versioning" => PutBucketVersioning PUT "/?versioning" => PutBucketVersioning
PUT "/?website" => PutBucketWebsite OWNER_PUT "/?website" => PutBucketWebsite
PUT "/my-image.jpg" => PutObject PUT "/my-image.jpg" => PutObject
PUT "/Key+" => PutObject PUT "/Key+" => PutObject
PUT "/my-image.jpg?acl" => PutObjectAcl PUT "/my-image.jpg?acl" => PutObjectAcl

4
src/api/s3_website.rs
View file

@ -22,7 +22,7 @@ pub async fn handle_delete_website(
.bucket_table .bucket_table
.get(&EmptyKey, &bucket_id) .get(&EmptyKey, &bucket_id)
.await? .await?
.ok_or(Error::NotFound)?; .ok_or(Error::NoSuchBucket)?;
if let crdt::Deletable::Present(param) = &mut bucket.state { if let crdt::Deletable::Present(param) = &mut bucket.state {
param.website_config.update(None); param.website_config.update(None);
@ -50,7 +50,7 @@ pub async fn handle_put_website(
.bucket_table .bucket_table
.get(&EmptyKey, &bucket_id) .get(&EmptyKey, &bucket_id)
.await? .await?
.ok_or(Error::NotFound)?; .ok_or(Error::NoSuchBucket)?;
let conf: WebsiteConfiguration = from_reader(&body as &[u8])?; let conf: WebsiteConfiguration = from_reader(&body as &[u8])?;
conf.validate()?; conf.validate()?;

28
src/garage/admin.rs
View file

@ -429,6 +429,8 @@ impl AdminRpcHandler {
KeyOperation::New(query) => self.handle_create_key(query).await, KeyOperation::New(query) => self.handle_create_key(query).await,
KeyOperation::Rename(query) => self.handle_rename_key(query).await, KeyOperation::Rename(query) => self.handle_rename_key(query).await,
KeyOperation::Delete(query) => self.handle_delete_key(query).await, KeyOperation::Delete(query) => self.handle_delete_key(query).await,
KeyOperation::Allow(query) => self.handle_allow_key(query).await,
KeyOperation::Deny(query) => self.handle_deny_key(query).await,
KeyOperation::Import(query) => self.handle_import_key(query).await, KeyOperation::Import(query) => self.handle_import_key(query).await,
} }
} }
@ -523,6 +525,32 @@ impl AdminRpcHandler {
))) )))
} }
async fn handle_allow_key(&self, query: &KeyPermOpt) -> Result<AdminRpc, Error> {
let mut key = self
.garage
.bucket_helper()
.get_existing_matching_key(&query.key_pattern)
.await?;
if query.create_bucket {
key.params_mut().unwrap().allow_create_bucket.update(true);
}
self.garage.key_table.insert(&key).await?;
self.key_info_result(key).await
}
async fn handle_deny_key(&self, query: &KeyPermOpt) -> Result<AdminRpc, Error> {
let mut key = self
.garage
.bucket_helper()
.get_existing_matching_key(&query.key_pattern)
.await?;
if query.create_bucket {
key.params_mut().unwrap().allow_create_bucket.update(false);
}
self.garage.key_table.insert(&key).await?;
self.key_info_result(key).await
}
async fn handle_import_key(&self, query: &KeyImportOpt) -> Result<AdminRpc, Error> { async fn handle_import_key(&self, query: &KeyImportOpt) -> Result<AdminRpc, Error> {
let prev_key = self.garage.key_table.get(&EmptyKey, &query.key_id).await?; let prev_key = self.garage.key_table.get(&EmptyKey, &query.key_id).await?;
if prev_key.is_some() { if prev_key.is_some() {

18
src/garage/cli/structs.rs
View file

@ -274,6 +274,14 @@ pub enum KeyOperation {
#[structopt(name = "delete")] #[structopt(name = "delete")]
Delete(KeyDeleteOpt), Delete(KeyDeleteOpt),
/// Set permission flags for key
#[structopt(name = "allow")]
Allow(KeyPermOpt),
/// Unset permission flags for key
#[structopt(name = "deny")]
Deny(KeyPermOpt),
/// Import key /// Import key
#[structopt(name = "import")] #[structopt(name = "import")]
Import(KeyImportOpt), Import(KeyImportOpt),
@ -311,6 +319,16 @@ pub struct KeyDeleteOpt {
pub yes: bool, pub yes: bool,
} }
#[derive(Serialize, Deserialize, StructOpt, Debug)]
pub struct KeyPermOpt {
/// ID or name of the key
pub key_pattern: String,
/// Flag that allows key to create buckets using S3's CreateBucket call
#[structopt(long = "create-bucket")]
pub create_bucket: bool,
}
#[derive(Serialize, Deserialize, StructOpt, Debug)] #[derive(Serialize, Deserialize, StructOpt, Debug)]
pub struct KeyImportOpt { pub struct KeyImportOpt {
/// Access key ID /// Access key ID

13
src/garage/cli/util.rs
View file

@ -18,15 +18,15 @@ pub fn print_bucket_list(bl: Vec<Bucket>) {
.filter(|(_, _, active)| *active) .filter(|(_, _, active)| *active)
.map(|(name, _, _)| name.to_string()) .map(|(name, _, _)| name.to_string())
.collect::<Vec<_>>(); .collect::<Vec<_>>();
let local_aliases_n = match bucket let local_aliases_n = match &bucket
.local_aliases() .local_aliases()
.iter() .iter()
.filter(|(_, _, active)| *active) .filter(|(_, _, active)| *active)
.count() .collect::<Vec<_>>()[..]
{ {
0 => "".into(), [] => "".into(),
1 => "1 local alias".into(), [((k, n), _, _)] => format!("{}:{}", k, n),
n => format!("{} local aliases", n), s => format!("[{} local aliases]", s.len()),
}; };
table.push(format!( table.push(format!(
"\t{}\t{}\t{}", "\t{}\t{}\t{}",
@ -88,6 +88,9 @@ pub fn print_key_info(key: &Key, relevant_buckets: &HashMap<Uuid, Bucket>) {
println!("\nAuthorized buckets:"); println!("\nAuthorized buckets:");
let mut table = vec![]; let mut table = vec![];
for (bucket_id, perm) in p.authorized_buckets.items().iter() { for (bucket_id, perm) in p.authorized_buckets.items().iter() {
if !perm.is_any() {
continue;
}
let rflag = if perm.allow_read { "R" } else { " " }; let rflag = if perm.allow_read { "R" } else { " " };
let wflag = if perm.allow_write { "W" } else { " " }; let wflag = if perm.allow_write { "W" } else { " " };
let oflag = if perm.allow_owner { "O" } else { " " }; let oflag = if perm.allow_owner { "O" } else { " " };

6
src/model/helper/bucket.rs
View file

@ -433,13 +433,11 @@ impl<'a> BucketHelper<'a> {
let mut bucket = self.get_internal_bucket(bucket_id).await?; let mut bucket = self.get_internal_bucket(bucket_id).await?;
let mut key = self.get_internal_key(key_id).await?; let mut key = self.get_internal_key(key_id).await?;
let allow_any = perm.allow_read || perm.allow_write || perm.allow_owner;
if let Some(bstate) = bucket.state.as_option() { if let Some(bstate) = bucket.state.as_option() {
if let Some(kp) = bstate.authorized_keys.get(key_id) { if let Some(kp) = bstate.authorized_keys.get(key_id) {
perm.timestamp = increment_logical_clock_2(perm.timestamp, kp.timestamp); perm.timestamp = increment_logical_clock_2(perm.timestamp, kp.timestamp);
} }
} else if allow_any { } else if perm.is_any() {
return Err(Error::BadRequest( return Err(Error::BadRequest(
"Trying to give permissions on a deleted bucket".into(), "Trying to give permissions on a deleted bucket".into(),
)); ));
@ -449,7 +447,7 @@ impl<'a> BucketHelper<'a> {
if let Some(bp) = kstate.authorized_buckets.get(&bucket_id) { if let Some(bp) = kstate.authorized_buckets.get(&bucket_id) {
perm.timestamp = increment_logical_clock_2(perm.timestamp, bp.timestamp); perm.timestamp = increment_logical_clock_2(perm.timestamp, bp.timestamp);
} }
} else if allow_any { } else if perm.is_any() {
return Err(Error::BadRequest( return Err(Error::BadRequest(
"Trying to give permissions to a deleted key".into(), "Trying to give permissions to a deleted key".into(),
)); ));

11
src/model/permission.rs
View file

@ -27,6 +27,17 @@ impl BucketKeyPerm {
allow_write: false, allow_write: false,
allow_owner: false, allow_owner: false,
}; };
pub const ALL_PERMISSIONS: Self = Self {
timestamp: 0,
allow_read: true,
allow_write: true,
allow_owner: true,
};
pub fn is_any(&self) -> bool {
self.allow_read || self.allow_write || self.allow_owner
}
} }
impl Crdt for BucketKeyPerm { impl Crdt for BucketKeyPerm {