change crate used for cdc

previous one seemed to output incorrect results
This commit is contained in:
Trinity Pointard 2021-03-17 01:31:35 +01:00 committed by Alex Auvolat
parent 71a13f366e
commit 9c9471c64f
No known key found for this signature in database
GPG key ID: EDABF9711E244EB1
3 changed files with 19 additions and 24 deletions

17
Cargo.lock generated
View file

@ -222,10 +222,10 @@ dependencies = [
] ]
[[package]] [[package]]
name = "fmt-extra" name = "fastcdc"
version = "0.2.1" version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07f11f71b1f9be830047fbb1899d90601c3b21a471dc99fe1057303eee37f2b9" checksum = "5afa29be46b12c8c380b997def8d1ac77c2665da93eb0a768fab0bf4db79333f"
[[package]] [[package]]
name = "fnv" name = "fnv"
@ -389,12 +389,12 @@ dependencies = [
"chrono", "chrono",
"crypto-mac 0.10.0", "crypto-mac 0.10.0",
"err-derive", "err-derive",
"fastcdc",
"futures", "futures",
"futures-util", "futures-util",
"garage_model", "garage_model",
"garage_table", "garage_table",
"garage_util", "garage_util",
"hash-roll",
"hex", "hex",
"hmac", "hmac",
"http", "http",
@ -600,15 +600,6 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "hash-roll"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9e27803a4b526df90ed2a3f60523eeec6b5ace6ba7530f9920fbee82027fa11"
dependencies = [
"fmt-extra",
]
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.9.1" version = "0.9.1"

View file

@ -22,7 +22,7 @@ bytes = "1.0"
chrono = "0.4" chrono = "0.4"
crypto-mac = "0.10" crypto-mac = "0.10"
err-derive = "0.3" err-derive = "0.3"
hash-roll = "0.3.0" fastcdc = "1.0.5"
hex = "0.4" hex = "0.4"
hmac = "0.10" hmac = "0.10"
log = "0.4" log = "0.4"

View file

@ -2,8 +2,8 @@ use std::collections::{BTreeMap, VecDeque};
use std::fmt::Write; use std::fmt::Write;
use std::sync::Arc; use std::sync::Arc;
use fastcdc::{Chunk, FastCDC};
use futures::stream::*; use futures::stream::*;
use hash_roll::{ChunkIncr, fastcdc::{FastCdc, FastCdcIncr}, gear_table::GEAR_64};
use hyper::{Body, Request, Response}; use hyper::{Body, Request, Response};
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5}; use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
use sha2::Sha256; use sha2::Sha256;
@ -269,22 +269,24 @@ async fn put_block_meta(
struct BodyChunker { struct BodyChunker {
body: Body, body: Body,
read_all: bool, read_all: bool,
min_block_size: usize,
avg_block_size: usize,
max_block_size: usize, max_block_size: usize,
buf: VecDeque<u8>, buf: VecDeque<u8>,
chunker: FastCdcIncr<'static>,
} }
impl BodyChunker { impl BodyChunker {
fn new(body: Body, block_size: usize) -> Self { fn new(body: Body, block_size: usize) -> Self {
let min_block_size = block_size / 4 * 3;
let avg_block_size = block_size;
let max_block_size = block_size * 2; let max_block_size = block_size * 2;
let chunker = FastCdc::new(&GEAR_64, block_size as u64 / 2, block_size as u64, max_block_size as u64);
let chunker = (&chunker).into();
Self { Self {
body, body,
read_all: false, read_all: false,
min_block_size,
avg_block_size,
max_block_size, max_block_size,
buf: VecDeque::with_capacity(2 * max_block_size), buf: VecDeque::with_capacity(2 * max_block_size),
chunker,
} }
} }
async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> { async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> {
@ -299,12 +301,14 @@ impl BodyChunker {
} }
if self.buf.len() == 0 { if self.buf.len() == 0 {
Ok(None) Ok(None)
} else if let Some(index) = self.chunker.push(self.buf.make_contiguous()) {
let block = self.buf.drain(..index).collect::<Vec<u8>>();
Ok(Some(block))
} else { } else {
let block = self.buf.drain(..).collect::<Vec<u8>>(); let mut iter = FastCDC::with_eof(self.buf.make_contiguous(), self.min_block_size, self.avg_block_size, self.max_block_size, self.read_all);
Ok(Some(block)) if let Some(Chunk {length, ..}) = iter.next() {
let block = self.buf.drain(..length).collect::<Vec<u8>>();
Ok(Some(block))
} else {
Ok(None)
}
} }
} }
} }