New buckets for 0.6.0: make bucket id a SK and not a HK, CLI updates

2022-01-03 19:06:04 +01:00 · 2022-01-03 19:06:04 +01:00 · df35feba18
parent 1bcd6fabbd
commit df35feba18
9 changed files with 86 additions and 47 deletions
--- a/src/api/s3_bucket.rs
+++ b/src/api/s3_bucket.rs
@ -58,21 +58,17 @@ pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result<Respo
 	let mut aliases = HashMap::new();

 	for bucket_id in ids.iter() {
-		let bucket = garage.bucket_table.get(bucket_id, &EmptyKey).await?;
+		let bucket = garage.bucket_table.get(&EmptyKey, bucket_id).await?;
 		if let Some(bucket) = bucket {
-			if let Deletable::Present(param) = bucket.state {
-				for (alias, _, active) in param.aliases.items() {
-					if *active {
-						let alias_ent = garage.bucket_alias_table.get(&EmptyKey, alias).await?;
-						if let Some(alias_ent) = alias_ent {
-							if let Some(alias_bucket) = alias_ent.state.get() {
-								if alias_bucket == bucket_id {
-									aliases.insert(alias_ent.name().to_string(), *bucket_id);
-								}
-							}
-						}
+			for (alias, _, _active) in bucket.aliases().iter().filter(|(_, _, active)| *active) {
+				let alias_opt = garage.bucket_alias_table.get(&EmptyKey, alias).await?;
+				if let Some(alias_ent) = alias_opt {
+					if *alias_ent.state.get() == Some(*bucket_id) {
+						aliases.insert(alias_ent.name().to_string(), *bucket_id);
 					}
 				}
+			}
+			if let Deletable::Present(param) = bucket.state {
 				buckets_by_id.insert(bucket_id, param);
 			}
 		}
--- a/src/api/s3_website.rs
+++ b/src/api/s3_website.rs
@ -20,7 +20,7 @@ pub async fn handle_delete_website(
 ) -> Result<Response<Body>, Error> {
 	let mut bucket = garage
 		.bucket_table
-		.get(&bucket_id, &EmptyKey)
+		.get(&EmptyKey, &bucket_id)
 		.await?
 		.ok_or(Error::NotFound)?;

@ -48,7 +48,7 @@ pub async fn handle_put_website(

 	let mut bucket = garage
 		.bucket_table
-		.get(&bucket_id, &EmptyKey)
+		.get(&EmptyKey, &bucket_id)
 		.await?
 		.ok_or(Error::NotFound)?;

--- a/src/garage/admin.rs
+++ b/src/garage/admin.rs
@ -38,7 +38,7 @@ pub enum AdminRpc {

 	// Replies
 	Ok(String),
-	BucketList(Vec<BucketAlias>),
+	BucketList(Vec<Bucket>),
 	BucketInfo(Bucket, HashMap<String, Key>),
 	KeyList(Vec<(String, String)>),
 	KeyInfo(Key, HashMap<Uuid, Bucket>),
@ -76,12 +76,12 @@ impl AdminRpcHandler {
 	}

 	async fn handle_list_buckets(&self) -> Result<AdminRpc, Error> {
-		let bucket_aliases = self
+		let buckets = self
 			.garage
-			.bucket_alias_table
+			.bucket_table
 			.get_range(&EmptyKey, None, Some(DeletedFilter::NotDeleted), 10000)
 			.await?;
-		Ok(AdminRpc::BucketList(bucket_aliases))
+		Ok(AdminRpc::BucketList(buckets))
 	}

 	async fn handle_bucket_info(&self, query: &BucketOpt) -> Result<AdminRpc, Error> {
@ -536,7 +536,7 @@ impl AdminRpcHandler {
 			.items()
 			.iter()
 		{
-			if let Some(b) = self.garage.bucket_table.get(id, &EmptyKey).await? {
+			if let Some(b) = self.garage.bucket_table.get(&EmptyKey, id).await? {
 				relevant_buckets.insert(*id, b);
 			}
 		}
--- a/src/garage/cli/cmd.rs
+++ b/src/garage/cli/cmd.rs
@ -165,24 +165,13 @@ pub async fn cmd_admin(
 			println!("{}", msg);
 		}
 		AdminRpc::BucketList(bl) => {
-			println!("List of buckets:");
-			let mut table = vec![];
-			for alias in bl {
-				if let Some(alias_bucket) = alias.state.get() {
-					table.push(format!("\t{}\t{:?}", alias.name(), alias_bucket));
-				}
-			}
-			format_table(table);
-			println!("Buckets that don't have a global alias (i.e. that only exist in the namespace of an access key) are not shown.");
+			print_bucket_list(bl);
 		}
 		AdminRpc::BucketInfo(bucket, rk) => {
 			print_bucket_info(&bucket, &rk);
 		}
 		AdminRpc::KeyList(kl) => {
-			println!("List of keys:");
-			for key in kl {
-				println!("{}\t{}", key.0, key.1);
-			}
+			print_key_list(kl);
 		}
 		AdminRpc::KeyInfo(key, rb) => {
 			print_key_info(&key, &rb);
--- a/src/garage/cli/util.rs
+++ b/src/garage/cli/util.rs
@ -7,6 +7,46 @@ use garage_util::error::*;
 use garage_model::bucket_table::*;
 use garage_model::key_table::*;

+pub fn print_bucket_list(bl: Vec<Bucket>) {
+	println!("List of buckets:");
+
+	let mut table = vec![];
+	for bucket in bl {
+		let aliases = bucket
+			.aliases()
+			.iter()
+			.filter(|(_, _, active)| *active)
+			.map(|(name, _, _)| name.to_string())
+			.collect::<Vec<_>>();
+		let local_aliases_n = match bucket
+			.local_aliases()
+			.iter()
+			.filter(|(_, _, active)| *active)
+			.count()
+		{
+			0 => "".into(),
+			1 => "1 local alias".into(),
+			n => format!("{} local aliases", n),
+		};
+		table.push(format!(
+			"\t{}\t{}\t{}",
+			aliases.join(","),
+			local_aliases_n,
+			hex::encode(bucket.id)
+		));
+	}
+	format_table(table);
+}
+
+pub fn print_key_list(kl: Vec<(String, String)>) {
+	println!("List of keys:");
+	let mut table = vec![];
+	for key in kl {
+		table.push(format!("\t{}\t{}", key.0, key.1));
+	}
+	format_table(table);
+}
+
 pub fn print_key_info(key: &Key, relevant_buckets: &HashMap<Uuid, Bucket>) {
 	let bucket_global_aliases = |b: &Uuid| {
 		if let Some(bucket) = relevant_buckets.get(b) {
@ -99,7 +139,7 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap<String, Key>)
 						.get(key_id)
 						.map(|k| k.name.get().as_str())
 						.unwrap_or("");
-					table.push(format!("\t{}\t{} ({})", alias, key_id, key_name));
+					table.push(format!("\t{} ({})\t{}", key_id, key_name, alias));
 				}
 			}
 			format_table(table);
@ -115,7 +155,7 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap<String, Key>)
 					.map(|k| k.name.get().as_str())
 					.unwrap_or("");
 				table.push(format!(
-					"\t{}{}{}\t{} ({})",
+					"\t{}{}{}\t{}\t{}",
 					rflag, wflag, oflag, k, key_name
 				));
 			}
--- a/src/garage/main.rs
+++ b/src/garage/main.rs
@ -139,8 +139,8 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 	let admin_rpc_endpoint = netapp.endpoint::<AdminRpc, ()>(ADMIN_RPC_PATH.into());

 	match cli_command_dispatch(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await {
-		Err(HelperError::Internal(i)) => Err(i),
-		Err(HelperError::BadRequest(b)) => Err(Error::Message(format!("bad request: {}", b))),
+		Err(HelperError::Internal(i)) => Err(Error::Message(format!("Internal error: {}", i))),
+		Err(HelperError::BadRequest(b)) => Err(Error::Message(b)),
 		Ok(x) => Ok(x),
 	}
 }
--- a/src/model/bucket_table.rs
+++ b/src/model/bucket_table.rs
@ -105,15 +105,29 @@ impl Bucket {
 			crdt::Deletable::Present(state) => state.authorized_keys.items(),
 		}
 	}
+
+	pub fn aliases(&self) -> &[(String, u64, bool)] {
+		match &self.state {
+			crdt::Deletable::Deleted => &[],
+			crdt::Deletable::Present(state) => state.aliases.items(),
+		}
+	}
+
+	pub fn local_aliases(&self) -> &[((String, String), u64, bool)] {
+		match &self.state {
+			crdt::Deletable::Deleted => &[],
+			crdt::Deletable::Present(state) => state.local_aliases.items(),
+		}
+	}
 }

-impl Entry<Uuid, EmptyKey> for Bucket {
-	fn partition_key(&self) -> &Uuid {
-		&self.id
-	}
-	fn sort_key(&self) -> &EmptyKey {
+impl Entry<EmptyKey, Uuid> for Bucket {
+	fn partition_key(&self) -> &EmptyKey {
 		&EmptyKey
 	}
+	fn sort_key(&self) -> &Uuid {
+		&self.id
+	}
 }

 impl Crdt for Bucket {
@ -127,8 +141,8 @@ pub struct BucketTable;
 impl TableSchema for BucketTable {
 	const TABLE_NAME: &'static str = "bucket_v2";

-	type P = Uuid;
-	type S = EmptyKey;
+	type P = EmptyKey;
+	type S = Uuid;
 	type E = Bucket;
 	type Filter = DeletedFilter;

--- a/src/model/helper/bucket.rs
+++ b/src/model/helper/bucket.rs
@ -36,7 +36,7 @@ impl<'a> BucketHelper<'a> {
 			Ok(self
 				.0
 				.bucket_table
-				.get(&bucket_id, &EmptyKey)
+				.get(&EmptyKey, &bucket_id)
 				.await?
 				.filter(|x| !x.state.is_deleted())
 				.map(|_| bucket_id))
@ -58,7 +58,7 @@ impl<'a> BucketHelper<'a> {
 		Ok(self
 			.0
 			.bucket_table
-			.get(&bucket_id, &EmptyKey)
+			.get(&EmptyKey, &bucket_id)
 			.await?
 			.ok_or_message(format!("Bucket {:?} does not exist", bucket_id))?)
 	}
@ -70,7 +70,7 @@ impl<'a> BucketHelper<'a> {
 	pub async fn get_existing_bucket(&self, bucket_id: Uuid) -> Result<Bucket, Error> {
 		self.0
 			.bucket_table
-			.get(&bucket_id, &EmptyKey)
+			.get(&EmptyKey, &bucket_id)
 			.await?
 			.filter(|b| !b.is_deleted())
 			.ok_or_bad_request(format!(
--- a/src/web/web_server.rs
+++ b/src/web/web_server.rs
@ -93,7 +93,7 @@ async fn serve_file(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<
 	// Check bucket isn't deleted and has website access enabled
 	let _: Bucket = garage
 		.bucket_table
-		.get(&bucket_id, &EmptyKey)
+		.get(&EmptyKey, &bucket_id)
 		.await?
 		.filter(|b| {
 			b.state