WIP add content defined chunking #42

Closed
trinity-1686a wants to merge 42 commits from content-defined-chunking into master
Showing only changes of commit f7c2cd1cd7 - Show all commits

View file

@ -440,9 +440,13 @@ where
warn!("Hashes differ between stored value and Merkle tree, key: {:?} (if your server is very busy, don't worry, this happens when the Merkle tree can't be updated fast enough)", ik); warn!("Hashes differ between stored value and Merkle tree, key: {:?} (if your server is very busy, don't worry, this happens when the Merkle tree can't be updated fast enough)", ik);
} }
todo_items.push(val.to_vec()); todo_items.push(val.to_vec());
} else {
warn!("Item from Merkle tree not found in store: {:?} (if your server is very busy, don't worry, this happens when the Merkle tree can't be updated fast enough)", ik);
} }
} }
MerkleNode::Intermediate(l) => { MerkleNode::Intermediate(l) => {
// Get Merkle node for this tree position at remote node
// and compare it with local node
let remote_node = match self let remote_node = match self
.aux .aux
.rpc_client .rpc_client
@ -462,7 +466,11 @@ where
} }
}; };
let int_l2 = match remote_node { let int_l2 = match remote_node {
// If they have an intermediate node at this tree position,
// we can compare them to find differences
MerkleNode::Intermediate(l2) => l2, MerkleNode::Intermediate(l2) => l2,
// Otherwise, treat it as if they have nothing for this subtree,
// which will have the consequence of sending them everything
_ => vec![], _ => vec![],
}; };
@ -493,20 +501,18 @@ where
Ok(()) Ok(())
} }
async fn send_items(&self, who: UUID, item_list: Vec<Vec<u8>>) -> Result<(), Error> { async fn send_items(&self, who: UUID, item_value_list: Vec<Vec<u8>>) -> Result<(), Error> {
info!( info!(
"({}) Sending {} items to {:?}", "({}) Sending {} items to {:?}",
self.data.name, self.data.name,
item_list.len(), item_value_list.len(),
who who
); );
let mut values = vec![]; let values = item_value_list.into_iter()
for item in item_list.iter() { .map(|x| Arc::new(ByteBuf::from(x)))
if let Some(v) = self.data.store.get(&item[..])? { .collect::<Vec<_>>();
values.push(Arc::new(ByteBuf::from(v.as_ref())));
}
}
let rpc_resp = self let rpc_resp = self
.aux .aux
.rpc_client .rpc_client