Deuxfleurs/garage
41
56
Fork 73
Code Issues 98 Pull requests 14 Releases 30 Wiki Activity

improve internal item counter mechanisms and implement bucket quotas #326

Merged
lx merged 23 commits from counters into main 2022-06-15 18:20:31 +00:00
Conversation 1 Commits 23 Files changed 32 +18 -69
6 changed files with 18 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 180e7fef0a - Show all commits

26
src/api/admin/bucket.rs
View file

@ -33,28 +33,10 @@ pub async fn handle_list_buckets(garage: &Arc<Garage>) -> Result<Response<Body>,
)
.await?;
let ring = garage.system.ring.borrow().clone();
let counters = garage
.object_counter_table
.table
.get_range(
&EmptyKey,
None,
Some((DeletedFilter::NotDeleted, ring.layout.node_id_vec.clone())),
15000,
EnumerationOrder::Forward,
)
.await?
.iter()
.map(|x| (x.sk, x.filtered_values(&ring)))
.collect::<HashMap<_, _>>();
let res = buckets
.into_iter()
.map(|b| {
let state = b.state.as_option().unwrap();
let empty_cnts = HashMap::new();
let cnts = counters.get(&b.id).unwrap_or(&empty_cnts);
ListBucketResultItem {
id: hex::encode(b.id),
global_aliases: state
@ -74,9 +56,6 @@ pub async fn handle_list_buckets(garage: &Arc<Garage>) -> Result<Response<Body>,
alias: n.to_string(),
})
.collect::<Vec<_>>(),
objects: cnts.get(OBJECTS).cloned().unwrap_or_default(),
bytes: cnts.get(BYTES).cloned().unwrap_or_default(),
unfinshed_uploads: cnts.get(UNFINISHED_UPLOADS).cloned().unwrap_or_default(),
}
})
.collect::<Vec<_>>();
@ -90,9 +69,6 @@ struct ListBucketResultItem {
id: String,
global_aliases: Vec<String>,
local_aliases: Vec<BucketLocalAlias>,
objects: i64,
bytes: i64,
unfinshed_uploads: i64,
}
#[derive(Serialize)]
@ -143,7 +119,7 @@ async fn bucket_info_results(
let counters = garage
.object_counter_table
.table
.get(&EmptyKey, &bucket_id)
.get(&bucket_id, &EmptyKey)
.await?
.map(|x| x.filtered_values(&garage.system.ring.borrow()))
.unwrap_or_default();

2
src/api/s3/put.rs
View file

@ -226,7 +226,7 @@ async fn check_quotas(
let key = key.to_string();
let (prev_object, counters) = futures::try_join!(
garage.object_table.get(&bucket.id, &key),
garage.object_counter_table.table.get(&EmptyKey, &bucket.id),
garage.object_counter_table.table.get(&bucket.id, &EmptyKey),
)?;
let counters = counters

26
src/garage/admin.rs
View file

@ -39,10 +39,7 @@ pub enum AdminRpc {
// Replies
Ok(String),
BucketList {
buckets: Vec<Bucket>,
counters: HashMap<Uuid, HashMap<String, i64>>,
},
BucketList(Vec<Bucket>),
BucketInfo {
bucket: Bucket,
relevant_keys: HashMap<String, Key>,
@ -97,24 +94,7 @@ impl AdminRpcHandler {
)
.await?;
let ring = self.garage.system.ring.borrow().clone();
let counters = self
.garage
.object_counter_table
.table
.get_range(
&EmptyKey,
None,
Some((DeletedFilter::NotDeleted, ring.layout.node_id_vec.clone())),
15000,
EnumerationOrder::Forward,
)
.await?
.iter()
.map(|x| (x.sk, x.filtered_values(&ring)))
.collect::<HashMap<_, _>>();
Ok(AdminRpc::BucketList { buckets, counters })
Ok(AdminRpc::BucketList(buckets))
}
async fn handle_bucket_info(&self, query: &BucketOpt) -> Result<AdminRpc, Error> {
@ -135,7 +115,7 @@ impl AdminRpcHandler {
.garage
.object_counter_table
.table
.get(&EmptyKey, &bucket_id)
.get(&bucket_id, &EmptyKey)
.await?
.map(|x| x.filtered_values(&self.garage.system.ring.borrow()))
.unwrap_or_default();

4
src/garage/cli/cmd.rs
View file

@ -166,8 +166,8 @@ pub async fn cmd_admin(
AdminRpc::Ok(msg) => {
println!("{}", msg);
}
AdminRpc::BucketList { buckets, counters } => {
print_bucket_list(buckets, counters);
AdminRpc::BucketList(bl) => {
print_bucket_list(bl);
}
AdminRpc::BucketInfo {
bucket,

13
src/garage/cli/util.rs
View file

@ -9,11 +9,11 @@ use garage_model::bucket_table::*;
use garage_model::key_table::*;
use garage_model::s3::object_table::{BYTES, OBJECTS, UNFINISHED_UPLOADS};
pub fn print_bucket_list(buckets: Vec<Bucket>, counters: HashMap<Uuid, HashMap<String, i64>>) {
pub fn print_bucket_list(bl: Vec<Bucket>) {
println!("List of buckets:");
let mut table = vec![];
for bucket in buckets {
for bucket in bl {
let aliases = bucket
.aliases()
.iter()
@ -31,18 +31,11 @@ pub fn print_bucket_list(buckets: Vec<Bucket>, counters: HashMap<Uuid, HashMap<S
s => format!("[{} local aliases]", s.len()),
};
let empty_counters = HashMap::new();
let cnt = counters.get(&bucket.id).unwrap_or(&empty_counters);
table.push(format!(
"\t{}\t{}\t{}\t{}\t{}\t{}",
"\t{}\t{}\t{}",
aliases.join(","),
local_aliases_n,
hex::encode(bucket.id),
bytesize::ByteSize::b(cnt.get(BYTES).cloned().unwrap_or_default() as u64)
.to_string_as(true),
cnt.get(OBJECTS).cloned().unwrap_or_default(),
cnt.get(UNFINISHED_UPLOADS).cloned().unwrap_or_default(),
));
}
format_table(table);

16
src/model/s3/object_table.rs
View file

@ -302,17 +302,17 @@ impl TableSchema for ObjectTable {
impl CountedItem for Object {
const COUNTER_TABLE_NAME: &'static str = "bucket_object_counter";
// Partition key = nothing
type CP = EmptyKey;
// Sort key = bucket id
type CS = Uuid;
// Partition key = bucket id
type CP = Uuid;
// Sort key = nothing
type CS = EmptyKey;
fn counter_partition_key(&self) -> &EmptyKey {
&EmptyKey
}
fn counter_sort_key(&self) -> &Uuid {
fn counter_partition_key(&self) -> &Uuid {
&self.bucket_id
}
fn counter_sort_key(&self) -> &EmptyKey {
&EmptyKey
}
fn counts(&self) -> Vec<(&'static str, i64)> {
let versions = self.versions();