Deuxfleurs/garage
48
55
Fork 72
Code Issues 99 Pull requests 14 Releases 30 Wiki Activity

Online repair new workers, except blocks and scrub
Some checks failed
continuous-integration/drone/pr Build is failing
Details
continuous-integration/drone/push Build is failing
Details

Browse source
This commit is contained in:
Alex 2022-06-21 15:27:58 +02:00
parent b8338dea56
commit a855c54bdb
Signed by: lx
GPG key ID: 0E496D15096376BE
2 changed files with 142 additions and 96 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
src/garage/admin.rs
View file

@ -24,7 +24,7 @@ use garage_model::migrate::Migrate;
use garage_model::permission::*;
use crate::cli::*;
use crate::repair::online::OnlineRepair;
use crate::repair::online::launch_online_repair;
pub const ADMIN_RPC_PATH: &str = "garage/admin_rpc.rs/Rpc";
@ -693,15 +693,7 @@ impl AdminRpcHandler {
)))
}
} else {
let repair = OnlineRepair {
garage: self.garage.clone(),
};
self.garage
.system
.background
.spawn_worker("Repair worker".into(), move |must_exit| async move {
repair.repair_worker(opt, must_exit).await
});
launch_online_repair(self.garage.clone(), opt)?;
Ok(AdminRpc::Ok(format!(
"Repair launched on {:?}",
self.garage.system.id

226
src/garage/repair/online.rs
View file

@ -1,5 +1,6 @@
use std::sync::Arc;
use async_trait::async_trait;
use tokio::sync::watch;
use garage_model::garage::Garage;
@ -7,83 +8,103 @@ use garage_model::s3::block_ref_table::*;
use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*;
use garage_table::*;
use garage_util::background::*;
use garage_util::error::Error;
use crate::*;
pub struct OnlineRepair {
pub garage: Arc<Garage>,
pub fn launch_online_repair(garage: Arc<Garage>, opt: RepairOpt) -> Result<(), Error> {
match opt.what {
RepairWhat::Tables => {
info!("Launching a full sync of tables");
garage.bucket_table.syncer.add_full_sync();
garage.object_table.syncer.add_full_sync();
garage.version_table.syncer.add_full_sync();
garage.block_ref_table.syncer.add_full_sync();
garage.key_table.syncer.add_full_sync();
}
RepairWhat::Versions => {
info!("Repairing the versions table");
garage
.background
.spawn_worker(RepairVersionsWorker::new(garage.clone()));
}
RepairWhat::BlockRefs => {
info!("Repairing the block refs table");
garage
.background
.spawn_worker(RepairBlockrefsWorker::new(garage.clone()));
}
RepairWhat::Blocks => {
unimplemented!()
/*
info!("Repairing the stored blocks");
self.garage
.block_manager
.repair_data_store(&must_exit)
.await?;
*/
}
RepairWhat::Scrub { tranquility } => {
unimplemented!()
/*
info!("Verifying integrity of stored blocks");
self.garage
.block_manager
.scrub_data_store(&must_exit, tranquility)
.await?;
*/
}
}
Ok(())
}
impl OnlineRepair {
pub async fn repair_worker(&self, opt: RepairOpt, must_exit: watch::Receiver<bool>) {
if let Err(e) = self.repair_worker_aux(opt, must_exit).await {
warn!("Repair worker failed with error: {}", e);
// ----
struct RepairVersionsWorker {
garage: Arc<Garage>,
pos: Vec<u8>,
iter: usize,
}
impl RepairVersionsWorker {
fn new(garage: Arc<Garage>) -> Self {
Self {
garage,
pos: vec![],
iter: 0,
}
}
}
async fn repair_worker_aux(
&self,
opt: RepairOpt,
must_exit: watch::Receiver<bool>,
) -> Result<(), Error> {
match opt.what {
RepairWhat::Tables => {
info!("Launching a full sync of tables");
self.garage.bucket_table.syncer.add_full_sync();
self.garage.object_table.syncer.add_full_sync();
self.garage.version_table.syncer.add_full_sync();
self.garage.block_ref_table.syncer.add_full_sync();
self.garage.key_table.syncer.add_full_sync();
}
RepairWhat::Versions => {
info!("Repairing the versions table");
self.repair_versions(&must_exit).await?;
}
RepairWhat::BlockRefs => {
info!("Repairing the block refs table");
self.repair_block_ref(&must_exit).await?;
}
RepairWhat::Blocks => {
info!("Repairing the stored blocks");
self.garage
.block_manager
.repair_data_store(&must_exit)
.await?;
}
RepairWhat::Scrub { tranquility } => {
info!("Verifying integrity of stored blocks");
self.garage
.block_manager
.scrub_data_store(&must_exit, tranquility)
.await?;
}
}
Ok(())
#[async_trait]
impl Worker for RepairVersionsWorker {
fn name(&self) -> String {
"Version repair worker".into()
}
async fn repair_versions(&self, must_exit: &watch::Receiver<bool>) -> Result<(), Error> {
let mut pos = vec![];
let mut i = 0;
while !*must_exit.borrow() {
let item_bytes = match self.garage.version_table.data.store.get_gt(pos)? {
Some((k, v)) => {
pos = k;
v
}
None => break,
};
i += 1;
if i % 1000 == 0 {
info!("repair_versions: {}", i);
async fn work(
&mut self,
_must_exit: &mut watch::Receiver<bool>,
) -> Result<WorkerStatus, Error> {
let item_bytes = match self.garage.version_table.data.store.get_gt(&self.pos)? {
Some((k, v)) => {
self.pos = k;
v
}
let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
if version.deleted.get() {
continue;
None => {
info!("repair_versions: finished, done {}", self.iter);
return Ok(WorkerStatus::Done);
}
};
self.iter += 1;
if self.iter % 1000 == 0 {
info!("repair_versions: {}", self.iter);
}
let version = rmp_serde::decode::from_read_ref::<_, Version>(&item_bytes)?;
if !version.deleted.get() {
let object = self
.garage
.object_table
@ -109,32 +130,61 @@ impl OnlineRepair {
.await?;
}
}
info!("repair_versions: finished, done {}", i);
Ok(())
Ok(WorkerStatus::Busy)
}
async fn repair_block_ref(&self, must_exit: &watch::Receiver<bool>) -> Result<(), Error> {
let mut pos = vec![];
let mut i = 0;
async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerStatus {
unreachable!()
}
}
while !*must_exit.borrow() {
let item_bytes = match self.garage.block_ref_table.data.store.get_gt(pos)? {
Some((k, v)) => {
pos = k;
v
}
None => break,
};
// ----
i += 1;
if i % 1000 == 0 {
info!("repair_block_ref: {}", i);
struct RepairBlockrefsWorker {
garage: Arc<Garage>,
pos: Vec<u8>,
iter: usize,
}
impl RepairBlockrefsWorker {
fn new(garage: Arc<Garage>) -> Self {
Self {
garage,
pos: vec![],
iter: 0,
}
}
}
#[async_trait]
impl Worker for RepairBlockrefsWorker {
fn name(&self) -> String {
"Block refs repair worker".into()
}
async fn work(
&mut self,
_must_exit: &mut watch::Receiver<bool>,
) -> Result<WorkerStatus, Error> {
let item_bytes = match self.garage.block_ref_table.data.store.get_gt(&self.pos)? {
Some((k, v)) => {
self.pos = k;
v
}
let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
if block_ref.deleted.get() {
continue;
None => {
info!("repair_block_ref: finished, done {}", self.iter);
return Ok(WorkerStatus::Done);
}
};
self.iter += 1;
if self.iter % 1000 == 0 {
info!("repair_block_ref: {}", self.iter);
}
let block_ref = rmp_serde::decode::from_read_ref::<_, BlockRef>(&item_bytes)?;
if !block_ref.deleted.get() {
let version = self
.garage
.version_table
@ -157,7 +207,11 @@ impl OnlineRepair {
.await?;
}
}
info!("repair_block_ref: finished, done {}", i);
Ok(())
Ok(WorkerStatus::Busy)
}
async fn wait_for_work(&mut self, _must_exit: &watch::Receiver<bool>) -> WorkerStatus {
unreachable!()
}
}