forked from Deuxfleurs/garage
Implement sending blocks to nodes that need them
This commit is contained in:
parent
db1c4222ce
commit
4abfb75509
5 changed files with 69 additions and 6 deletions
2
Makefile
2
Makefile
|
@ -1,3 +1,3 @@
|
|||
all:
|
||||
cargo fmt
|
||||
cargo fmt || true
|
||||
cargo build
|
||||
|
|
5
TODO
5
TODO
|
@ -1,9 +1,7 @@
|
|||
Replication
|
||||
-----------
|
||||
|
||||
- for each interval of tokens, we know the list of nodes that are responsible
|
||||
- every node watches the current ring and state of the network
|
||||
- and thus determines the interval of tokens for which they are responsible
|
||||
Finish the thing that sends blocks to other nodes if needed before deleting them locally.
|
||||
|
||||
How are we going to test that our replication method works correctly?
|
||||
We will have to introduce lots of dummy data and then add/remove nodes many times.
|
||||
|
@ -12,7 +10,6 @@ We will have to introduce lots of dummy data and then add/remove nodes many time
|
|||
To do list
|
||||
----------
|
||||
|
||||
- important: check block values on read and repare corrupted block contents
|
||||
- less a priority: hinted handoff
|
||||
- FIXME in rpc_server when garage shuts down and futures can be interrupted
|
||||
(tokio::spawn should be replaced by a new function background::spawn_joinable)
|
||||
|
|
61
src/block.rs
61
src/block.rs
|
@ -3,6 +3,7 @@ use std::sync::Arc;
|
|||
use std::time::Duration;
|
||||
|
||||
use arc_swap::ArcSwapOption;
|
||||
use futures::future::*;
|
||||
use futures::stream::*;
|
||||
use tokio::fs;
|
||||
use tokio::prelude::*;
|
||||
|
@ -16,6 +17,8 @@ use crate::proto::*;
|
|||
use crate::rpc_client::*;
|
||||
use crate::server::Garage;
|
||||
|
||||
const NEED_BLOCK_QUERY_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
pub struct BlockManager {
|
||||
pub data_dir: PathBuf,
|
||||
pub rc: sled::Tree,
|
||||
|
@ -102,6 +105,22 @@ impl BlockManager {
|
|||
}))
|
||||
}
|
||||
|
||||
pub async fn need_block(&self, hash: &Hash) -> Result<bool, Error> {
|
||||
let needed = self
|
||||
.rc
|
||||
.get(hash.as_ref())?
|
||||
.map(|x| u64_from_bytes(x.as_ref()) > 0)
|
||||
.unwrap_or(false);
|
||||
if needed {
|
||||
let mut path = self.data_dir.clone();
|
||||
path.push(hex::encode(hash.as_ref()));
|
||||
let exists = fs::metadata(&path).await.is_ok();
|
||||
Ok(!exists)
|
||||
} else {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
fn block_dir(&self, hash: &Hash) -> PathBuf {
|
||||
let mut path = self.data_dir.clone();
|
||||
path.push(hex::encode(&hash.as_slice()[0..1]));
|
||||
|
@ -191,7 +210,47 @@ impl BlockManager {
|
|||
.await?;
|
||||
let needed_by_others = !active_refs.is_empty();
|
||||
if needed_by_others {
|
||||
// TODO check they have it and send it if not
|
||||
let ring = garage.system.ring.borrow().clone();
|
||||
let who = ring.walk_ring(&hash, garage.system.config.data_replication_factor);
|
||||
let msg = Message::NeedBlockQuery(hash.clone());
|
||||
let who_needs_fut = who
|
||||
.iter()
|
||||
.map(|to| rpc_call(garage.system.clone(), to, &msg, NEED_BLOCK_QUERY_TIMEOUT));
|
||||
let who_needs = join_all(who_needs_fut).await;
|
||||
|
||||
let mut need_nodes = vec![];
|
||||
let mut errors = 0;
|
||||
for (node, needed) in who.into_iter().zip(who_needs.iter()) {
|
||||
match needed {
|
||||
Ok(Message::NeedBlockReply(true)) => {
|
||||
need_nodes.push(node);
|
||||
}
|
||||
Err(_) => {
|
||||
errors += 1;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
if errors > (garage.system.config.data_replication_factor - 1) / 2 {
|
||||
return Err(Error::Message(format!(
|
||||
"Should delete block, but not enough nodes confirm that they have it."
|
||||
)));
|
||||
}
|
||||
|
||||
if need_nodes.len() > 0 {
|
||||
let put_block_message = self.read_block(hash).await?;
|
||||
for resp in rpc_call_many(
|
||||
garage.system.clone(),
|
||||
&need_nodes[..],
|
||||
put_block_message,
|
||||
BLOCK_RW_TIMEOUT,
|
||||
)
|
||||
.await
|
||||
{
|
||||
resp?;
|
||||
}
|
||||
}
|
||||
}
|
||||
fs::remove_file(path).await?;
|
||||
self.resync_queue.remove(&hash)?;
|
||||
|
|
|
@ -20,6 +20,8 @@ pub enum Message {
|
|||
|
||||
GetBlock(Hash),
|
||||
PutBlock(PutBlockMessage),
|
||||
NeedBlockQuery(Hash),
|
||||
NeedBlockReply(bool),
|
||||
|
||||
TableRPC(String, #[serde(with = "serde_bytes")] Vec<u8>),
|
||||
}
|
||||
|
|
|
@ -66,6 +66,11 @@ async fn handler(
|
|||
tokio::spawn(write_fut).await?
|
||||
}
|
||||
Message::GetBlock(h) => garage.block_manager.read_block(&h).await,
|
||||
Message::NeedBlockQuery(h) => garage
|
||||
.block_manager
|
||||
.need_block(&h)
|
||||
.await
|
||||
.map(Message::NeedBlockReply),
|
||||
|
||||
Message::TableRPC(table, msg) => {
|
||||
// Same trick for table RPCs than for PutBlock
|
||||
|
|
Loading…
Reference in a new issue