From df35feba18787cac06b2a87e3426752fb78da2d5 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 3 Jan 2022 19:06:04 +0100 Subject: [PATCH] New buckets for 0.6.0: make bucket id a SK and not a HK, CLI updates --- src/api/s3_bucket.rs | 20 +++++++---------- src/api/s3_website.rs | 4 ++-- src/garage/admin.rs | 10 ++++----- src/garage/cli/cmd.rs | 15 ++----------- src/garage/cli/util.rs | 44 ++++++++++++++++++++++++++++++++++++-- src/garage/main.rs | 4 ++-- src/model/bucket_table.rs | 28 ++++++++++++++++++------ src/model/helper/bucket.rs | 6 +++--- src/web/web_server.rs | 2 +- 9 files changed, 86 insertions(+), 47 deletions(-) diff --git a/src/api/s3_bucket.rs b/src/api/s3_bucket.rs index 24ec6b98..50aeb173 100644 --- a/src/api/s3_bucket.rs +++ b/src/api/s3_bucket.rs @@ -58,21 +58,17 @@ pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result Result, Error> { let mut bucket = garage .bucket_table - .get(&bucket_id, &EmptyKey) + .get(&EmptyKey, &bucket_id) .await? .ok_or(Error::NotFound)?; @@ -48,7 +48,7 @@ pub async fn handle_put_website( let mut bucket = garage .bucket_table - .get(&bucket_id, &EmptyKey) + .get(&EmptyKey, &bucket_id) .await? .ok_or(Error::NotFound)?; diff --git a/src/garage/admin.rs b/src/garage/admin.rs index bca1bc5a..b1eb6915 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -38,7 +38,7 @@ pub enum AdminRpc { // Replies Ok(String), - BucketList(Vec), + BucketList(Vec), BucketInfo(Bucket, HashMap), KeyList(Vec<(String, String)>), KeyInfo(Key, HashMap), @@ -76,12 +76,12 @@ impl AdminRpcHandler { } async fn handle_list_buckets(&self) -> Result { - let bucket_aliases = self + let buckets = self .garage - .bucket_alias_table + .bucket_table .get_range(&EmptyKey, None, Some(DeletedFilter::NotDeleted), 10000) .await?; - Ok(AdminRpc::BucketList(bucket_aliases)) + Ok(AdminRpc::BucketList(buckets)) } async fn handle_bucket_info(&self, query: &BucketOpt) -> Result { @@ -536,7 +536,7 @@ impl AdminRpcHandler { .items() .iter() { - if let Some(b) = self.garage.bucket_table.get(id, &EmptyKey).await? { + if let Some(b) = self.garage.bucket_table.get(&EmptyKey, id).await? { relevant_buckets.insert(*id, b); } } diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs index 515f2143..a90277a0 100644 --- a/src/garage/cli/cmd.rs +++ b/src/garage/cli/cmd.rs @@ -165,24 +165,13 @@ pub async fn cmd_admin( println!("{}", msg); } AdminRpc::BucketList(bl) => { - println!("List of buckets:"); - let mut table = vec![]; - for alias in bl { - if let Some(alias_bucket) = alias.state.get() { - table.push(format!("\t{}\t{:?}", alias.name(), alias_bucket)); - } - } - format_table(table); - println!("Buckets that don't have a global alias (i.e. that only exist in the namespace of an access key) are not shown."); + print_bucket_list(bl); } AdminRpc::BucketInfo(bucket, rk) => { print_bucket_info(&bucket, &rk); } AdminRpc::KeyList(kl) => { - println!("List of keys:"); - for key in kl { - println!("{}\t{}", key.0, key.1); - } + print_key_list(kl); } AdminRpc::KeyInfo(key, rb) => { print_key_info(&key, &rb); diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs index 8d31a4c5..7a7d0e9b 100644 --- a/src/garage/cli/util.rs +++ b/src/garage/cli/util.rs @@ -7,6 +7,46 @@ use garage_util::error::*; use garage_model::bucket_table::*; use garage_model::key_table::*; +pub fn print_bucket_list(bl: Vec) { + println!("List of buckets:"); + + let mut table = vec![]; + for bucket in bl { + let aliases = bucket + .aliases() + .iter() + .filter(|(_, _, active)| *active) + .map(|(name, _, _)| name.to_string()) + .collect::>(); + let local_aliases_n = match bucket + .local_aliases() + .iter() + .filter(|(_, _, active)| *active) + .count() + { + 0 => "".into(), + 1 => "1 local alias".into(), + n => format!("{} local aliases", n), + }; + table.push(format!( + "\t{}\t{}\t{}", + aliases.join(","), + local_aliases_n, + hex::encode(bucket.id) + )); + } + format_table(table); +} + +pub fn print_key_list(kl: Vec<(String, String)>) { + println!("List of keys:"); + let mut table = vec![]; + for key in kl { + table.push(format!("\t{}\t{}", key.0, key.1)); + } + format_table(table); +} + pub fn print_key_info(key: &Key, relevant_buckets: &HashMap) { let bucket_global_aliases = |b: &Uuid| { if let Some(bucket) = relevant_buckets.get(b) { @@ -99,7 +139,7 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap) .get(key_id) .map(|k| k.name.get().as_str()) .unwrap_or(""); - table.push(format!("\t{}\t{} ({})", alias, key_id, key_name)); + table.push(format!("\t{} ({})\t{}", key_id, key_name, alias)); } } format_table(table); @@ -115,7 +155,7 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap) .map(|k| k.name.get().as_str()) .unwrap_or(""); table.push(format!( - "\t{}{}{}\t{} ({})", + "\t{}{}{}\t{}\t{}", rflag, wflag, oflag, k, key_name )); } diff --git a/src/garage/main.rs b/src/garage/main.rs index 60a13ac7..870455e1 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -139,8 +139,8 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { let admin_rpc_endpoint = netapp.endpoint::(ADMIN_RPC_PATH.into()); match cli_command_dispatch(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await { - Err(HelperError::Internal(i)) => Err(i), - Err(HelperError::BadRequest(b)) => Err(Error::Message(format!("bad request: {}", b))), + Err(HelperError::Internal(i)) => Err(Error::Message(format!("Internal error: {}", i))), + Err(HelperError::BadRequest(b)) => Err(Error::Message(b)), Ok(x) => Ok(x), } } diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index 52c2316c..d687e774 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -105,15 +105,29 @@ impl Bucket { crdt::Deletable::Present(state) => state.authorized_keys.items(), } } + + pub fn aliases(&self) -> &[(String, u64, bool)] { + match &self.state { + crdt::Deletable::Deleted => &[], + crdt::Deletable::Present(state) => state.aliases.items(), + } + } + + pub fn local_aliases(&self) -> &[((String, String), u64, bool)] { + match &self.state { + crdt::Deletable::Deleted => &[], + crdt::Deletable::Present(state) => state.local_aliases.items(), + } + } } -impl Entry for Bucket { - fn partition_key(&self) -> &Uuid { - &self.id - } - fn sort_key(&self) -> &EmptyKey { +impl Entry for Bucket { + fn partition_key(&self) -> &EmptyKey { &EmptyKey } + fn sort_key(&self) -> &Uuid { + &self.id + } } impl Crdt for Bucket { @@ -127,8 +141,8 @@ pub struct BucketTable; impl TableSchema for BucketTable { const TABLE_NAME: &'static str = "bucket_v2"; - type P = Uuid; - type S = EmptyKey; + type P = EmptyKey; + type S = Uuid; type E = Bucket; type Filter = DeletedFilter; diff --git a/src/model/helper/bucket.rs b/src/model/helper/bucket.rs index 52cedb12..6f171c8b 100644 --- a/src/model/helper/bucket.rs +++ b/src/model/helper/bucket.rs @@ -36,7 +36,7 @@ impl<'a> BucketHelper<'a> { Ok(self .0 .bucket_table - .get(&bucket_id, &EmptyKey) + .get(&EmptyKey, &bucket_id) .await? .filter(|x| !x.state.is_deleted()) .map(|_| bucket_id)) @@ -58,7 +58,7 @@ impl<'a> BucketHelper<'a> { Ok(self .0 .bucket_table - .get(&bucket_id, &EmptyKey) + .get(&EmptyKey, &bucket_id) .await? .ok_or_message(format!("Bucket {:?} does not exist", bucket_id))?) } @@ -70,7 +70,7 @@ impl<'a> BucketHelper<'a> { pub async fn get_existing_bucket(&self, bucket_id: Uuid) -> Result { self.0 .bucket_table - .get(&bucket_id, &EmptyKey) + .get(&EmptyKey, &bucket_id) .await? .filter(|b| !b.is_deleted()) .ok_or_bad_request(format!( diff --git a/src/web/web_server.rs b/src/web/web_server.rs index f13f289e..49e5f21b 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -93,7 +93,7 @@ async fn serve_file(garage: Arc, req: Request) -> Result