Deuxfleurs/garage
41
56
Fork 73
Code Issues 99 Pull requests 14 Releases 30 Wiki Activity

New model for buckets #172

Merged
lx merged 19 commits from new-buckets into main 2022-01-10 11:32:42 +00:00
Conversation 0 Commits 19 Files changed 58 +86 -47
9 changed files with 86 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit df35feba18 - Show all commits

16
src/api/s3_bucket.rs
View file

@ -58,21 +58,17 @@ pub async fn handle_list_buckets(garage: &Garage, api_key: &Key) -> Result<Respo
let mut aliases = HashMap::new();
for bucket_id in ids.iter() {
let bucket = garage.bucket_table.get(bucket_id, &EmptyKey).await?;
let bucket = garage.bucket_table.get(&EmptyKey, bucket_id).await?;
if let Some(bucket) = bucket {
if let Deletable::Present(param) = bucket.state {
for (alias, _, active) in param.aliases.items() {
if *active {
let alias_ent = garage.bucket_alias_table.get(&EmptyKey, alias).await?;
if let Some(alias_ent) = alias_ent {
if let Some(alias_bucket) = alias_ent.state.get() {
if alias_bucket == bucket_id {
for (alias, _, _active) in bucket.aliases().iter().filter(|(_, _, active)| *active) {
let alias_opt = garage.bucket_alias_table.get(&EmptyKey, alias).await?;
if let Some(alias_ent) = alias_opt {
if *alias_ent.state.get() == Some(*bucket_id) {
aliases.insert(alias_ent.name().to_string(), *bucket_id);
}
}
}
lx marked this conversation as resolved Outdated
trinity-1686a commented 2021-12-18 16:42:19 +00:00
Outdated
Review
Copy link

this is a lot of indentation. It may be possible to remove a few layers by using Option::and_then, and doing param.aliases.items().filter(|(_,_,active)| active)

this is a lot of indentation. It may be possible to remove a few layers by using Option::and_then, and doing `param.aliases.items().filter(|(_,_,active)| active)`
}
}
if let Deletable::Present(param) = bucket.state {
buckets_by_id.insert(bucket_id, param);
}
}

4
src/api/s3_website.rs
View file

@ -20,7 +20,7 @@ pub async fn handle_delete_website(
) -> Result<Response<Body>, Error> {
let mut bucket = garage
.bucket_table
.get(&bucket_id, &EmptyKey)
.get(&EmptyKey, &bucket_id)
.await?
.ok_or(Error::NotFound)?;
@ -48,7 +48,7 @@ pub async fn handle_put_website(
let mut bucket = garage
.bucket_table
.get(&bucket_id, &EmptyKey)
.get(&EmptyKey, &bucket_id)
.await?
.ok_or(Error::NotFound)?;

10
src/garage/admin.rs
View file

@ -38,7 +38,7 @@ pub enum AdminRpc {
// Replies
Ok(String),
BucketList(Vec<BucketAlias>),
BucketList(Vec<Bucket>),
BucketInfo(Bucket, HashMap<String, Key>),
KeyList(Vec<(String, String)>),
KeyInfo(Key, HashMap<Uuid, Bucket>),
@ -76,12 +76,12 @@ impl AdminRpcHandler {
}
async fn handle_list_buckets(&self) -> Result<AdminRpc, Error> {
let bucket_aliases = self
let buckets = self
.garage
.bucket_alias_table
.bucket_table
.get_range(&EmptyKey, None, Some(DeletedFilter::NotDeleted), 10000)
.await?;
Ok(AdminRpc::BucketList(bucket_aliases))
Ok(AdminRpc::BucketList(buckets))
}
async fn handle_bucket_info(&self, query: &BucketOpt) -> Result<AdminRpc, Error> {
@ -536,7 +536,7 @@ impl AdminRpcHandler {
.items()
.iter()
{
if let Some(b) = self.garage.bucket_table.get(id, &EmptyKey).await? {
if let Some(b) = self.garage.bucket_table.get(&EmptyKey, id).await? {
relevant_buckets.insert(*id, b);
}
}

15
src/garage/cli/cmd.rs
View file

@ -165,24 +165,13 @@ pub async fn cmd_admin(
println!("{}", msg);
}
AdminRpc::BucketList(bl) => {
println!("List of buckets:");
let mut table = vec![];
for alias in bl {
if let Some(alias_bucket) = alias.state.get() {
table.push(format!("\t{}\t{:?}", alias.name(), alias_bucket));
}
}
format_table(table);
println!("Buckets that don't have a global alias (i.e. that only exist in the namespace of an access key) are not shown.");
print_bucket_list(bl);
}
AdminRpc::BucketInfo(bucket, rk) => {
print_bucket_info(&bucket, &rk);
}
AdminRpc::KeyList(kl) => {
println!("List of keys:");
for key in kl {
println!("{}\t{}", key.0, key.1);
}
print_key_list(kl);
}
AdminRpc::KeyInfo(key, rb) => {
print_key_info(&key, &rb);

44
src/garage/cli/util.rs
View file

@ -7,6 +7,46 @@ use garage_util::error::*;
use garage_model::bucket_table::*;
use garage_model::key_table::*;
pub fn print_bucket_list(bl: Vec<Bucket>) {
println!("List of buckets:");
let mut table = vec![];
for bucket in bl {
let aliases = bucket
.aliases()
.iter()
.filter(|(_, _, active)| *active)
.map(|(name, _, _)| name.to_string())
.collect::<Vec<_>>();
let local_aliases_n = match bucket
.local_aliases()
.iter()
.filter(|(_, _, active)| *active)
.count()
{
0 => "".into(),
1 => "1 local alias".into(),
n => format!("{} local aliases", n),
};
table.push(format!(
"\t{}\t{}\t{}",
aliases.join(","),
local_aliases_n,
hex::encode(bucket.id)
));
}
format_table(table);
}
pub fn print_key_list(kl: Vec<(String, String)>) {
println!("List of keys:");
let mut table = vec![];
for key in kl {
table.push(format!("\t{}\t{}", key.0, key.1));
}
format_table(table);
}
pub fn print_key_info(key: &Key, relevant_buckets: &HashMap<Uuid, Bucket>) {
let bucket_global_aliases = |b: &Uuid| {
if let Some(bucket) = relevant_buckets.get(b) {
@ -99,7 +139,7 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap<String, Key>)
.get(key_id)
.map(|k| k.name.get().as_str())
.unwrap_or("");
table.push(format!("\t{}\t{} ({})", alias, key_id, key_name));
table.push(format!("\t{} ({})\t{}", key_id, key_name, alias));
}
}
format_table(table);
@ -115,7 +155,7 @@ pub fn print_bucket_info(bucket: &Bucket, relevant_keys: &HashMap<String, Key>)
.map(|k| k.name.get().as_str())
.unwrap_or("");
table.push(format!(
"\t{}{}{}\t{} ({})",
"\t{}{}{}\t{}\t{}",
rflag, wflag, oflag, k, key_name
));
}

4
src/garage/main.rs
View file

@ -139,8 +139,8 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
let admin_rpc_endpoint = netapp.endpoint::<AdminRpc, ()>(ADMIN_RPC_PATH.into());
match cli_command_dispatch(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await {
Err(HelperError::Internal(i)) => Err(i),
Err(HelperError::BadRequest(b)) => Err(Error::Message(format!("bad request: {}", b))),
Err(HelperError::Internal(i)) => Err(Error::Message(format!("Internal error: {}", i))),
Err(HelperError::BadRequest(b)) => Err(Error::Message(b)),
Ok(x) => Ok(x),
}
}

26
src/model/bucket_table.rs
View file

@ -105,15 +105,29 @@ impl Bucket {
crdt::Deletable::Present(state) => state.authorized_keys.items(),
}
}
pub fn aliases(&self) -> &[(String, u64, bool)] {
match &self.state {
crdt::Deletable::Deleted => &[],
crdt::Deletable::Present(state) => state.aliases.items(),
}
}
impl Entry<Uuid, EmptyKey> for Bucket {
fn partition_key(&self) -> &Uuid {
&self.id
pub fn local_aliases(&self) -> &[((String, String), u64, bool)] {
match &self.state {
crdt::Deletable::Deleted => &[],
crdt::Deletable::Present(state) => state.local_aliases.items(),
}
fn sort_key(&self) -> &EmptyKey {
}
}
impl Entry<EmptyKey, Uuid> for Bucket {
fn partition_key(&self) -> &EmptyKey {
&EmptyKey
}
fn sort_key(&self) -> &Uuid {
&self.id
}
}
impl Crdt for Bucket {
@ -127,8 +141,8 @@ pub struct BucketTable;
impl TableSchema for BucketTable {
const TABLE_NAME: &'static str = "bucket_v2";
type P = Uuid;
type S = EmptyKey;
type P = EmptyKey;
type S = Uuid;
type E = Bucket;
type Filter = DeletedFilter;

6
src/model/helper/bucket.rs
View file

@ -36,7 +36,7 @@ impl<'a> BucketHelper<'a> {
Ok(self
.0
.bucket_table
.get(&bucket_id, &EmptyKey)
.get(&EmptyKey, &bucket_id)
.await?
.filter(|x| !x.state.is_deleted())
.map(|_| bucket_id))
@ -58,7 +58,7 @@ impl<'a> BucketHelper<'a> {
Ok(self
.0
.bucket_table
.get(&bucket_id, &EmptyKey)
.get(&EmptyKey, &bucket_id)
.await?
.ok_or_message(format!("Bucket {:?} does not exist", bucket_id))?)
}
@ -70,7 +70,7 @@ impl<'a> BucketHelper<'a> {
pub async fn get_existing_bucket(&self, bucket_id: Uuid) -> Result<Bucket, Error> {
self.0
.bucket_table
.get(&bucket_id, &EmptyKey)
.get(&EmptyKey, &bucket_id)
.await?
.filter(|b| !b.is_deleted())
.ok_or_bad_request(format!(

2
src/web/web_server.rs
View file

@ -93,7 +93,7 @@ async fn serve_file(garage: Arc<Garage>, req: Request<Body>) -> Result<Response<
// Check bucket isn't deleted and has website access enabled
let _: Bucket = garage
.bucket_table
.get(&bucket_id, &EmptyKey)
.get(&EmptyKey, &bucket_id)
.await?
.filter(|b| {
b.state