From 2b93a01d2bead391e15f529dc5b4db80dcbeeba8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 12 May 2022 10:20:34 +0200 Subject: [PATCH] ListBucket and GetBucketInfo --- doc/drafts/admin-api.md | 114 ++++++++++++++++++++ src/api/admin/api_server.rs | 8 +- src/api/admin/bucket.rs | 208 ++++++++++++++++++++++++++++++++++++ src/api/admin/key.rs | 16 ++- src/api/admin/mod.rs | 1 + src/api/admin/router.rs | 21 +++- 6 files changed, 355 insertions(+), 13 deletions(-) create mode 100644 src/api/admin/bucket.rs diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 840dd4f7..edfdeae7 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -339,3 +339,117 @@ All fields (`name`, `allow` and `deny`) are optionnal. If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed. The possible flags in `allow` and `deny` are: `createBucket`. + +## Bucket operations + +### ListBuckets `GET /bucket` + +Returns all storage buckets in the cluster. + +Example response: + +```json +[ + { + "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033", + "globalAliases": [ + "test2" + ], + "localAliases": [] + }, + { + "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95", + "globalAliases": [ + "alex" + ], + "localAliases": [] + }, + { + "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995", + "globalAliases": [ + "test3" + ], + "localAliases": [] + }, + { + "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b", + "globalAliases": [], + "localAliases": [ + { + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "alias": "test" + } + ] + } +] +``` + +### GetBucketInfo `GET /bucket?id=` +### GetBucketInfo `GET /bucket?globalAlias=` + +Returns information about the requested storage bucket. + +If `id` is set, the bucket is looked up using its exact identifier. +If `globalAlias` is set, the bucket is looked up using its global alias. +(both are fast) + +Example response: + +```json +{ + "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b", + "globalAliases": [ + "alex" + ], + "keys": [ + { + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "name": "alex", + "permissions": { + "read": true, + "write": true, + "owner": true + }, + "bucketLocalAliases": [ + "test" + ] + } + ] +} +``` + +### CreateBucket `POST /bucket` + +Creates a new storage bucket. + +Request body format: + +```json +{ + "globalAlias": "NameOfMyBucket" +} +``` + +OR + +```json +{ + "localAlias": { + "key": "GK31c2f218a2e44f485b94239e", + "alias": "NameOfMyBucket" + } +} +``` + +OR + +```json +{} +``` + +Creates a new bucket, either with a global alias, a local one, +or no alias at all. + +### DeleteBucket `DELETE /bucket?id=` + +Deletes a storage bucket. A bucket cannot be deleted if it is not empty. diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 952f6a73..8e5a8c6c 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -18,6 +18,7 @@ use garage_util::error::Error as GarageError; use crate::error::*; use crate::generic_server::*; +use crate::admin::bucket::*; use crate::admin::cluster::*; use crate::admin::key::*; use crate::admin::router::{Authorization, Endpoint}; @@ -139,12 +140,15 @@ impl ApiHandler for AdminApiServer { Endpoint::CreateKey => handle_create_key(&self.garage, req).await, Endpoint::UpdateKey { id } => handle_update_key(&self.garage, id, req).await, Endpoint::DeleteKey { id } => handle_delete_key(&self.garage, id).await, - /* + // Buckets + Endpoint::ListBuckets => handle_list_buckets(&self.garage).await, + Endpoint::GetBucketInfo { id, global_alias } => { + handle_get_bucket_info(&self.garage, id, global_alias).await + } _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() ))), - */ } } } diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs new file mode 100644 index 00000000..003203c1 --- /dev/null +++ b/src/api/admin/bucket.rs @@ -0,0 +1,208 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use hyper::{Body, Request, Response, StatusCode}; +use serde::{Deserialize, Serialize}; + +use garage_util::data::*; +use garage_util::error::Error as GarageError; + +use garage_table::*; + +use garage_model::bucket_table::*; +use garage_model::garage::Garage; +use garage_model::key_table::*; + +use crate::admin::key::KeyBucketPermResult; +use crate::error::*; +use crate::helpers::*; + +pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { + let buckets = garage + .bucket_table + .get_range( + &EmptyKey, + None, + Some(DeletedFilter::NotDeleted), + 10000, + EnumerationOrder::Forward, + ) + .await?; + + let res = buckets + .into_iter() + .map(|b| { + let state = b.state.as_option().unwrap(); + ListBucketResultItem { + id: hex::encode(b.id), + global_aliases: state + .aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + local_aliases: state + .local_aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|((k, n), _, _)| ListBucketLocalAlias { + access_key_id: k.to_string(), + alias: n.to_string(), + }) + .collect::>(), + } + }) + .collect::>(); + + let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(resp_json))?) +} + +#[derive(Serialize)] +struct ListBucketResultItem { + id: String, + #[serde(rename = "globalAliases")] + global_aliases: Vec, + #[serde(rename = "localAliases")] + local_aliases: Vec, +} + +#[derive(Serialize)] +struct ListBucketLocalAlias { + #[serde(rename = "accessKeyId")] + access_key_id: String, + alias: String, +} + +pub async fn handle_get_bucket_info( + garage: &Arc, + id: Option, + global_alias: Option, +) -> Result, Error> { + let bucket_id = match (id, global_alias) { + (Some(id), None) => { + let id_hex = hex::decode(&id).ok_or_bad_request("Invalid bucket id")?; + Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")? + } + (None, Some(ga)) => garage + .bucket_helper() + .resolve_global_bucket_name(&ga) + .await? + .ok_or_bad_request("Bucket not found")?, + _ => { + return Err(Error::BadRequest( + "Either id or globalAlias must be provided (but not both)".into(), + )) + } + }; + + let bucket = garage + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; + + let mut relevant_keys = HashMap::new(); + for (k, _) in bucket + .state + .as_option() + .unwrap() + .authorized_keys + .items() + .iter() + { + if let Some(key) = garage + .key_table + .get(&EmptyKey, k) + .await? + .filter(|k| !k.is_deleted()) + { + if !key.state.is_deleted() { + relevant_keys.insert(k.clone(), key); + } + } + } + for ((k, _), _, _) in bucket + .state + .as_option() + .unwrap() + .local_aliases + .items() + .iter() + { + if relevant_keys.contains_key(k) { + continue; + } + if let Some(key) = garage.key_table.get(&EmptyKey, k).await? { + if !key.state.is_deleted() { + relevant_keys.insert(k.clone(), key); + } + } + } + + let state = bucket.state.as_option().unwrap(); + + let res = GetBucketInfoResult { + id: hex::encode(&bucket.id), + global_aliases: state + .aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + keys: relevant_keys + .into_iter() + .map(|(_, key)| { + let p = key.state.as_option().unwrap(); + GetBucketInfoKey { + access_key_id: key.key_id, + name: p.name.get().to_string(), + permissions: p + .authorized_buckets + .get(&bucket.id) + .map(|p| KeyBucketPermResult { + read: p.allow_read, + write: p.allow_write, + owner: p.allow_owner, + }) + .unwrap_or_default(), + bucket_local_aliases: p + .local_aliases + .items() + .iter() + .filter(|(_, _, b)| *b == Some(bucket.id)) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + } + }) + .collect::>(), + }; + + let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(resp_json))?) +} + +#[derive(Serialize)] +struct GetBucketInfoResult { + id: String, + #[serde(rename = "globalAliases")] + global_aliases: Vec, + keys: Vec, +} + +#[derive(Serialize)] +struct GetBucketInfoKey { + #[serde(rename = "accessKeyId")] + access_key_id: String, + #[serde(rename = "name")] + name: String, + permissions: KeyBucketPermResult, + #[serde(rename = "bucketLocalAliases")] + bucket_local_aliases: Vec, +} diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 7cfe3fce..1252d2c8 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -203,11 +203,7 @@ async fn key_info_results(garage: &Arc, key: Key) -> Result>(), @@ -246,9 +242,9 @@ struct KeyInfoBucketResult { permissions: KeyBucketPermResult, } -#[derive(Serialize)] -struct KeyBucketPermResult { - read: bool, - write: bool, - owner: bool, +#[derive(Serialize, Default)] +pub(crate) struct KeyBucketPermResult { + pub(crate) read: bool, + pub(crate) write: bool, + pub(crate) owner: bool, } diff --git a/src/api/admin/mod.rs b/src/api/admin/mod.rs index f6c3b2ee..05097c8b 100644 --- a/src/api/admin/mod.rs +++ b/src/api/admin/mod.rs @@ -1,5 +1,6 @@ pub mod api_server; mod router; +mod bucket; mod cluster; mod key; diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index 626cced1..a6e1c848 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -18,10 +18,12 @@ pub enum Endpoint { Options, Metrics, GetClusterStatus, + // Layout GetClusterLayout, UpdateClusterLayout, ApplyClusterLayout, RevertClusterLayout, + // Keys ListKeys, CreateKey, GetKeyInfo { @@ -34,6 +36,16 @@ pub enum Endpoint { UpdateKey { id: String, }, + // Buckets + ListBuckets, + CreateBucket, + GetBucketInfo { + id: Option, + global_alias: Option, + }, + DeleteBucket { + id: String, + }, }} impl Endpoint { @@ -63,6 +75,12 @@ impl Endpoint { POST "/key" => CreateKey, DELETE "/key" if id => DeleteKey (query::id), GET "/key" => ListKeys, + // Bucket endpoints + GET "/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/bucket" => ListBuckets, + POST "/bucket" => CreateBucket, + DELETE "/bucket" if id => DeleteBucket (query::id), ]); if let Some(message) = query.nonempty_message() { @@ -82,5 +100,6 @@ impl Endpoint { generateQueryParameters! { "id" => id, - "search" => search + "search" => search, + "globalAlias" => global_alias }