Add multipart upload repair
This commit is contained in:
parent
058518c22b
commit
4ea53dc759
2 changed files with 75 additions and 30 deletions
src/garage
|
@ -452,6 +452,9 @@ pub enum RepairWhat {
|
||||||
/// Only redo the propagation of object deletions to the version table (slow)
|
/// Only redo the propagation of object deletions to the version table (slow)
|
||||||
#[structopt(name = "versions", version = garage_version())]
|
#[structopt(name = "versions", version = garage_version())]
|
||||||
Versions,
|
Versions,
|
||||||
|
/// Only redo the propagation of object deletions to the multipart upload table (slow)
|
||||||
|
#[structopt(name = "mpu", version = garage_version())]
|
||||||
|
MultipartUploads,
|
||||||
/// Only redo the propagation of version deletions to the block ref table (extremely slow)
|
/// Only redo the propagation of version deletions to the block ref table (extremely slow)
|
||||||
#[structopt(name = "block_refs", version = garage_version())]
|
#[structopt(name = "block_refs", version = garage_version())]
|
||||||
BlockRefs,
|
BlockRefs,
|
||||||
|
|
|
@ -8,6 +8,7 @@ use garage_block::repair::ScrubWorkerCommand;
|
||||||
|
|
||||||
use garage_model::garage::Garage;
|
use garage_model::garage::Garage;
|
||||||
use garage_model::s3::block_ref_table::*;
|
use garage_model::s3::block_ref_table::*;
|
||||||
|
use garage_model::s3::mpu_table::*;
|
||||||
use garage_model::s3::object_table::*;
|
use garage_model::s3::object_table::*;
|
||||||
use garage_model::s3::version_table::*;
|
use garage_model::s3::version_table::*;
|
||||||
|
|
||||||
|
@ -38,6 +39,10 @@ pub async fn launch_online_repair(
|
||||||
info!("Repairing the versions table");
|
info!("Repairing the versions table");
|
||||||
bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairVersions));
|
bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairVersions));
|
||||||
}
|
}
|
||||||
|
RepairWhat::MultipartUploads => {
|
||||||
|
info!("Repairing the multipart uploads table");
|
||||||
|
bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairMpu));
|
||||||
|
}
|
||||||
RepairWhat::BlockRefs => {
|
RepairWhat::BlockRefs => {
|
||||||
info!("Repairing the block refs table");
|
info!("Repairing the block refs table");
|
||||||
bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs));
|
bg.spawn_worker(TableRepairWorker::new(garage.clone(), RepairBlockRefs));
|
||||||
|
@ -162,25 +167,26 @@ impl TableRepair for RepairVersions {
|
||||||
|
|
||||||
async fn process(&mut self, garage: &Garage, version: Version) -> Result<bool, Error> {
|
async fn process(&mut self, garage: &Garage, version: Version) -> Result<bool, Error> {
|
||||||
if !version.deleted.get() {
|
if !version.deleted.get() {
|
||||||
let version_exists = match &version.backlink {
|
let ref_exists = match &version.backlink {
|
||||||
VersionBacklink::Object { bucket_id, key } => {
|
VersionBacklink::Object { bucket_id, key } => garage
|
||||||
let object = garage.object_table.get(&bucket_id, &key).await?;
|
.object_table
|
||||||
match object {
|
.get(&bucket_id, &key)
|
||||||
Some(o) => o.versions().iter().any(|x| {
|
.await?
|
||||||
|
.map(|o| {
|
||||||
|
o.versions().iter().any(|x| {
|
||||||
x.uuid == version.uuid && x.state != ObjectVersionState::Aborted
|
x.uuid == version.uuid && x.state != ObjectVersionState::Aborted
|
||||||
}),
|
})
|
||||||
None => false,
|
})
|
||||||
}
|
.unwrap_or(false),
|
||||||
}
|
VersionBacklink::MultipartUpload { upload_id } => garage
|
||||||
VersionBacklink::MultipartUpload { upload_id } => {
|
.mpu_table
|
||||||
let mpu = garage.mpu_table.get(&upload_id, &EmptyKey).await?;
|
.get(&upload_id, &EmptyKey)
|
||||||
match mpu {
|
.await?
|
||||||
Some(u) => !u.deleted.get(),
|
.map(|u| !u.deleted.get())
|
||||||
None => false,
|
.unwrap_or(false),
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
if !version_exists {
|
|
||||||
|
if !ref_exists {
|
||||||
info!("Repair versions: marking version as deleted: {:?}", version);
|
info!("Repair versions: marking version as deleted: {:?}", version);
|
||||||
garage
|
garage
|
||||||
.version_table
|
.version_table
|
||||||
|
@ -206,27 +212,63 @@ impl TableRepair for RepairBlockRefs {
|
||||||
&garage.block_ref_table
|
&garage.block_ref_table
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn process(&mut self, garage: &Garage, block_ref: BlockRef) -> Result<bool, Error> {
|
async fn process(&mut self, garage: &Garage, mut block_ref: BlockRef) -> Result<bool, Error> {
|
||||||
if !block_ref.deleted.get() {
|
if !block_ref.deleted.get() {
|
||||||
let version = garage
|
let ref_exists = garage
|
||||||
.version_table
|
.version_table
|
||||||
.get(&block_ref.version, &EmptyKey)
|
.get(&block_ref.version, &EmptyKey)
|
||||||
.await?;
|
.await?
|
||||||
// The version might not exist if it has been GC'ed
|
.map(|v| !v.deleted.get())
|
||||||
let ref_exists = version.map(|v| !v.deleted.get()).unwrap_or(false);
|
.unwrap_or(false);
|
||||||
|
|
||||||
if !ref_exists {
|
if !ref_exists {
|
||||||
info!(
|
info!(
|
||||||
"Repair block ref: marking block_ref as deleted: {:?}",
|
"Repair block ref: marking block_ref as deleted: {:?}",
|
||||||
block_ref
|
block_ref
|
||||||
);
|
);
|
||||||
garage
|
block_ref.deleted.set();
|
||||||
.block_ref_table
|
garage.block_ref_table.insert(&block_ref).await?;
|
||||||
.insert(&BlockRef {
|
return Ok(true);
|
||||||
block: block_ref.block,
|
}
|
||||||
version: block_ref.version,
|
}
|
||||||
deleted: true.into(),
|
|
||||||
})
|
Ok(false)
|
||||||
.await?;
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ----
|
||||||
|
|
||||||
|
struct RepairMpu;
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl TableRepair for RepairMpu {
|
||||||
|
type T = MultipartUploadTable;
|
||||||
|
|
||||||
|
fn table(garage: &Garage) -> &Table<Self::T, TableShardedReplication> {
|
||||||
|
&garage.mpu_table
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn process(&mut self, garage: &Garage, mut mpu: MultipartUpload) -> Result<bool, Error> {
|
||||||
|
if !mpu.deleted.get() {
|
||||||
|
let ref_exists = garage
|
||||||
|
.object_table
|
||||||
|
.get(&mpu.bucket_id, &mpu.key)
|
||||||
|
.await?
|
||||||
|
.map(|o| {
|
||||||
|
o.versions()
|
||||||
|
.iter()
|
||||||
|
.any(|x| x.uuid == mpu.upload_id && x.is_uploading(Some(true)))
|
||||||
|
})
|
||||||
|
.unwrap_or(false);
|
||||||
|
|
||||||
|
if !ref_exists {
|
||||||
|
info!(
|
||||||
|
"Repair multipart uploads: marking mpu as deleted: {:?}",
|
||||||
|
mpu
|
||||||
|
);
|
||||||
|
mpu.parts.clear();
|
||||||
|
mpu.deleted.set();
|
||||||
|
garage.mpu_table.insert(&mpu).await?;
|
||||||
return Ok(true);
|
return Ok(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue