Add multipart upload repair

This commit is contained in:
Alex 2023-05-04 10:45:44 +02:00
parent 058518c22b
commit 4ea53dc759
2 changed files with 75 additions and 30 deletions
src/garage

View file

@ -452,6 +452,9 @@ pub enum RepairWhat {
/// Only redo the propagation of object deletions to the version table (slow) /// Only redo the propagation of object deletions to the version table (slow)
#[structopt(name = "versions", version = garage_version())] #[structopt(name = "versions", version = garage_version())]
Versions, Versions,
/// Only redo the propagation of object deletions to the multipart upload table (slow)
#[structopt(name = "mpu", version = garage_version())]
MultipartUploads,
/// Only redo the propagation of version deletions to the block ref table (extremely slow) /// Only redo the propagation of version deletions to the block ref table (extremely slow)
#[structopt(name = "block_refs", version = garage_version())] #[structopt(name = "block_refs", version = garage_version())]
BlockRefs, BlockRefs,

View file

@ -8,6 +8,7 @@ use garage_block::repair::ScrubWorkerCommand;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_model::s3::block_ref_table::*; use garage_model::s3::block_ref_table::*;
use garage_model::s3::mpu_table::*;
use garage_model::s3::object_table::*; use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*; use garage_model::s3::version_table::*;
@ -38,6 +39,10 @@ pub async fn launch_online_repair(
info!("Repairing the versions table"); info!("Repairing the versions table");
bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairVersions)); bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairVersions));
} }
RepairWhat::MultipartUploads => {
info!("Repairing the multipart uploads table");
bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairMpu));
}
RepairWhat::BlockRefs => { RepairWhat::BlockRefs => {
info!("Repairing the block refs table"); info!("Repairing the block refs table");
bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs)); bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs));
@ -162,25 +167,26 @@ impl TableRepair for RepairVersions {
async fn process(&mut self, garage: &Garage, version: Version) -> Result<bool, Error> { async fn process(&mut self, garage: &Garage, version: Version) -> Result<bool, Error> {
if !version.deleted.get() { if !version.deleted.get() {
let version_exists = match &version.backlink { let ref_exists = match &version.backlink {
VersionBacklink::Object { bucket_id, key } => { VersionBacklink::Object { bucket_id, key } => garage
let object = garage.object_table.get(&bucket_id, &key).await?; .object_table
match object { .get(&bucket_id, &key)
Some(o) => o.versions().iter().any(|x| { .await?
.map(|o| {
o.versions().iter().any(|x| {
x.uuid == version.uuid && x.state != ObjectVersionState::Aborted x.uuid == version.uuid && x.state != ObjectVersionState::Aborted
}), })
None => false, })
} .unwrap_or(false),
} VersionBacklink::MultipartUpload { upload_id } => garage
VersionBacklink::MultipartUpload { upload_id } => { .mpu_table
let mpu = garage.mpu_table.get(&upload_id, &EmptyKey).await?; .get(&upload_id, &EmptyKey)
match mpu { .await?
Some(u) => !u.deleted.get(), .map(|u| !u.deleted.get())
None => false, .unwrap_or(false),
}
}
}; };
if !version_exists {
if !ref_exists {
info!("Repair versions: marking version as deleted: {:?}", version); info!("Repair versions: marking version as deleted: {:?}", version);
garage garage
.version_table .version_table
@ -206,27 +212,63 @@ impl TableRepair for RepairBlockRefs {
&garage.block_ref_table &garage.block_ref_table
} }
async fn process(&mut self, garage: &Garage, block_ref: BlockRef) -> Result<bool, Error> { async fn process(&mut self, garage: &Garage, mut block_ref: BlockRef) -> Result<bool, Error> {
if !block_ref.deleted.get() { if !block_ref.deleted.get() {
let version = garage let ref_exists = garage
.version_table .version_table
.get(&block_ref.version, &EmptyKey) .get(&block_ref.version, &EmptyKey)
.await?; .await?
// The version might not exist if it has been GC'ed .map(|v| !v.deleted.get())
let ref_exists = version.map(|v| !v.deleted.get()).unwrap_or(false); .unwrap_or(false);
if !ref_exists { if !ref_exists {
info!( info!(
"Repair block ref: marking block_ref as deleted: {:?}", "Repair block ref: marking block_ref as deleted: {:?}",
block_ref block_ref
); );
garage block_ref.deleted.set();
.block_ref_table garage.block_ref_table.insert(&block_ref).await?;
.insert(&BlockRef { return Ok(true);
block: block_ref.block, }
version: block_ref.version, }
deleted: true.into(),
}) Ok(false)
.await?; }
}
// ----
struct RepairMpu;
#[async_trait]
impl TableRepair for RepairMpu {
type T = MultipartUploadTable;
fn table(garage: &Garage) -> &Table<Self::T, TableShardedReplication> {
&garage.mpu_table
}
async fn process(&mut self, garage: &Garage, mut mpu: MultipartUpload) -> Result<bool, Error> {
if !mpu.deleted.get() {
let ref_exists = garage
.object_table
.get(&mpu.bucket_id, &mpu.key)
.await?
.map(|o| {
o.versions()
.iter()
.any(|x| x.uuid == mpu.upload_id && x.is_uploading(Some(true)))
})
.unwrap_or(false);
if !ref_exists {
info!(
"Repair multipart uploads: marking mpu as deleted: {:?}",
mpu
);
mpu.parts.clear();
mpu.deleted.set();
garage.mpu_table.insert(&mpu).await?;
return Ok(true); return Ok(true);
} }
} }