Deuxfleurs/garage
41
56
Fork 73
Code Issues 99 Pull requests 14 Releases 30 Wiki Activity

Fix #204 (full Multipart Uploads semantics) #553

Merged
lx merged 20 commits from nlnet-task1 into next 2023-06-09 15:34:10 +00:00
Conversation 0 Commits 20 Files changed 33 +94 -81
1 changed files with 94 additions and 81 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 058518c22b - Show all commits

171
src/garage/repair/online.rs
View file

@ -5,11 +5,15 @@ use async_trait::async_trait;
use tokio::sync::watch; use tokio::sync::watch;
use garage_block::repair::ScrubWorkerCommand; use garage_block::repair::ScrubWorkerCommand;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_model::s3::block_ref_table::*; use garage_model::s3::block_ref_table::*;
use garage_model::s3::object_table::*; use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*; use garage_model::s3::version_table::*;
use garage_table::replication::*;
use garage_table::*; use garage_table::*;
use garage_util::background::*; use garage_util::background::*;
use garage_util::error::Error; use garage_util::error::Error;
use garage_util::migrate::Migrate; use garage_util::migrate::Migrate;
@ -32,11 +36,11 @@ pub async fn launch_online_repair(
} }
RepairWhat::Versions => { RepairWhat::Versions => {
info!("Repairing the versions table"); info!("Repairing the versions table");
bg.spawn_worker(RepairVersionsWorker::new(garage.clone())); bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairVersions));
} }
RepairWhat::BlockRefs => { RepairWhat::BlockRefs => {
info!("Repairing the block refs table"); info!("Repairing the block refs table");
bg.spawn_worker(RepairBlockrefsWorker::new(garage.clone())); bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs));
} }
RepairWhat::Blocks => { RepairWhat::Blocks => {
info!("Repairing the stored blocks"); info!("Repairing the stored blocks");
@ -67,71 +71,70 @@ pub async fn launch_online_repair(
// ---- // ----
struct RepairVersionsWorker { #[async_trait]
trait TableRepair: Send + Sync + 'static {
type T: TableSchema;
fn table(garage: &Garage) -> &Table<Self::T, TableShardedReplication>;
async fn process(
&mut self,
garage: &Garage,
entry: <<Self as TableRepair>::T as TableSchema>::E,
) -> Result<bool, Error>;
}
struct TableRepairWorker<T: TableRepair> {
garage: Arc<Garage>, garage: Arc<Garage>,
pos: Vec<u8>, pos: Vec<u8>,
counter: usize, counter: usize,
repairs: usize,
inner: T,
} }
impl RepairVersionsWorker { impl<R: TableRepair> TableRepairWorker<R> {
fn new(garage: Arc<Garage>) -> Self { fn new(garage: Arc<Garage>, inner: R) -> Self {
Self { Self {
garage, garage,
inner,
pos: vec![], pos: vec![],
counter: 0, counter: 0,
repairs: 0,
} }
} }
} }
#[async_trait] #[async_trait]
impl Worker for RepairVersionsWorker { impl<R: TableRepair> Worker for TableRepairWorker<R> {
fn name(&self) -> String { fn name(&self) -> String {
"Version repair worker".into() format!("{} repair worker", R::T::TABLE_NAME)
} }
fn status(&self) -> WorkerStatus { fn status(&self) -> WorkerStatus {
WorkerStatus { WorkerStatus {
progress: Some(self.counter.to_string()), progress: Some(format!("{} ({})", self.counter, self.repairs)),
..Default::default() ..Default::default()
} }
} }
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> { async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> {
let (item_bytes, next_pos) = match self.garage.version_table.data.store.get_gt(&self.pos)? { let (item_bytes, next_pos) = match R::table(&self.garage).data.store.get_gt(&self.pos)? {
Some((k, v)) => (v, k), Some((k, v)) => (v, k),
None => { None => {
info!("repair_versions: finished, done {}", self.counter); info!(
"{}: finished, done {}, fixed {}",
self.name(),
self.counter,
self.repairs
);
return Ok(WorkerState::Done); return Ok(WorkerState::Done);
} }
}; };
let version = Version::decode(&item_bytes).ok_or_message("Cannot decode Version")?; let entry = <R::T as TableSchema>::E::decode(&item_bytes)
if !version.deleted.get() { .ok_or_message("Cannot decode table entry")?;
let version_exists = match &version.backlink { if self.inner.process(&self.garage, entry).await? {
VersionBacklink::Object { bucket_id, key } => { self.repairs += 1;
let object = self.garage.object_table.get(&bucket_id, &key).await?;
match object {
Some(o) => o.versions().iter().any(|x| {
x.uuid == version.uuid && x.state != ObjectVersionState::Aborted
}),
None => false,
}
}
VersionBacklink::MultipartUpload { upload_id } => {
let mpu = self.garage.mpu_table.get(&upload_id, &EmptyKey).await?;
match mpu {
Some(u) => !u.deleted.get(),
None => false,
}
}
};
if !version_exists {
info!("Repair versions: marking version as deleted: {:?}", version);
self.garage
.version_table
.insert(&Version::new(version.uuid, version.backlink, true))
.await?;
}
} }
self.counter += 1; self.counter += 1;
@ -147,49 +150,65 @@ impl Worker for RepairVersionsWorker {
// ---- // ----
struct RepairBlockrefsWorker { struct RepairVersions;
garage: Arc<Garage>,
pos: Vec<u8>,
counter: usize,
}
impl RepairBlockrefsWorker {
fn new(garage: Arc<Garage>) -> Self {
Self {
garage,
pos: vec![],
counter: 0,
}
}
}
#[async_trait] #[async_trait]
impl Worker for RepairBlockrefsWorker { impl TableRepair for RepairVersions {
fn name(&self) -> String { type T = VersionTable;
"Block refs repair worker".into()
fn table(garage: &Garage) -> &Table<Self::T, TableShardedReplication> {
&garage.version_table
} }
fn status(&self) -> WorkerStatus { async fn process(&mut self, garage: &Garage, version: Version) -> Result<bool, Error> {
WorkerStatus { if !version.deleted.get() {
progress: Some(self.counter.to_string()), let version_exists = match &version.backlink {
..Default::default() VersionBacklink::Object { bucket_id, key } => {
let object = garage.object_table.get(&bucket_id, &key).await?;
match object {
Some(o) => o.versions().iter().any(|x| {
x.uuid == version.uuid && x.state != ObjectVersionState::Aborted
}),
None => false,
} }
} }
VersionBacklink::MultipartUpload { upload_id } => {
async fn work(&mut self, _must_exit: &mut watch::Receiver<bool>) -> Result<WorkerState, Error> { let mpu = garage.mpu_table.get(&upload_id, &EmptyKey).await?;
let (item_bytes, next_pos) = match mpu {
match self.garage.block_ref_table.data.store.get_gt(&self.pos)? { Some(u) => !u.deleted.get(),
Some((k, v)) => (v, k), None => false,
None => { }
info!("repair_block_ref: finished, done {}", self.counter);
return Ok(WorkerState::Done);
} }
}; };
if !version_exists {
info!("Repair versions: marking version as deleted: {:?}", version);
garage
.version_table
.insert(&Version::new(version.uuid, version.backlink, true))
.await?;
return Ok(true);
}
}
let block_ref = BlockRef::decode(&item_bytes).ok_or_message("Cannot decode BlockRef")?; Ok(false)
}
}
// ----
struct RepairBlockRefs;
#[async_trait]
impl TableRepair for RepairBlockRefs {
type T = BlockRefTable;
fn table(garage: &Garage) -> &Table<Self::T, TableShardedReplication> {
&garage.block_ref_table
}
async fn process(&mut self, garage: &Garage, block_ref: BlockRef) -> Result<bool, Error> {
if !block_ref.deleted.get() { if !block_ref.deleted.get() {
let version = self let version = garage
.garage
.version_table .version_table
.get(&block_ref.version, &EmptyKey) .get(&block_ref.version, &EmptyKey)
.await?; .await?;
@ -200,7 +219,7 @@ impl Worker for RepairBlockrefsWorker {
"Repair block ref: marking block_ref as deleted: {:?}", "Repair block ref: marking block_ref as deleted: {:?}",
block_ref block_ref
); );
self.garage garage
.block_ref_table .block_ref_table
.insert(&BlockRef { .insert(&BlockRef {
block: block_ref.block, block: block_ref.block,
@ -208,16 +227,10 @@ impl Worker for RepairBlockrefsWorker {
deleted: true.into(), deleted: true.into(),
}) })
.await?; .await?;
return Ok(true);
} }
} }
self.counter += 1; Ok(false)
self.pos = next_pos;
Ok(WorkerState::Busy)
}
async fn wait_for_work(&mut self) -> WorkerState {
unreachable!()
} }
} }