forked from Deuxfleurs/garage
change crate used for cdc
previous one seemed to output incorrect results
This commit is contained in:
parent
71a13f366e
commit
9c9471c64f
3 changed files with 19 additions and 24 deletions
17
Cargo.lock
generated
17
Cargo.lock
generated
|
@ -222,10 +222,10 @@ dependencies = [
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fmt-extra"
|
name = "fastcdc"
|
||||||
version = "0.2.1"
|
version = "1.0.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "07f11f71b1f9be830047fbb1899d90601c3b21a471dc99fe1057303eee37f2b9"
|
checksum = "5afa29be46b12c8c380b997def8d1ac77c2665da93eb0a768fab0bf4db79333f"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fnv"
|
name = "fnv"
|
||||||
|
@ -389,12 +389,12 @@ dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
"crypto-mac 0.10.0",
|
"crypto-mac 0.10.0",
|
||||||
"err-derive",
|
"err-derive",
|
||||||
|
"fastcdc",
|
||||||
"futures",
|
"futures",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"garage_model",
|
"garage_model",
|
||||||
"garage_table",
|
"garage_table",
|
||||||
"garage_util",
|
"garage_util",
|
||||||
"hash-roll",
|
|
||||||
"hex",
|
"hex",
|
||||||
"hmac",
|
"hmac",
|
||||||
"http",
|
"http",
|
||||||
|
@ -600,15 +600,6 @@ dependencies = [
|
||||||
"tracing",
|
"tracing",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "hash-roll"
|
|
||||||
version = "0.3.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "a9e27803a4b526df90ed2a3f60523eeec6b5ace6ba7530f9920fbee82027fa11"
|
|
||||||
dependencies = [
|
|
||||||
"fmt-extra",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hashbrown"
|
name = "hashbrown"
|
||||||
version = "0.9.1"
|
version = "0.9.1"
|
||||||
|
|
|
@ -22,7 +22,7 @@ bytes = "1.0"
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
crypto-mac = "0.10"
|
crypto-mac = "0.10"
|
||||||
err-derive = "0.3"
|
err-derive = "0.3"
|
||||||
hash-roll = "0.3.0"
|
fastcdc = "1.0.5"
|
||||||
hex = "0.4"
|
hex = "0.4"
|
||||||
hmac = "0.10"
|
hmac = "0.10"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
|
|
|
@ -2,8 +2,8 @@ use std::collections::{BTreeMap, VecDeque};
|
||||||
use std::fmt::Write;
|
use std::fmt::Write;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use fastcdc::{Chunk, FastCDC};
|
||||||
use futures::stream::*;
|
use futures::stream::*;
|
||||||
use hash_roll::{ChunkIncr, fastcdc::{FastCdc, FastCdcIncr}, gear_table::GEAR_64};
|
|
||||||
use hyper::{Body, Request, Response};
|
use hyper::{Body, Request, Response};
|
||||||
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
|
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
|
||||||
use sha2::Sha256;
|
use sha2::Sha256;
|
||||||
|
@ -269,22 +269,24 @@ async fn put_block_meta(
|
||||||
struct BodyChunker {
|
struct BodyChunker {
|
||||||
body: Body,
|
body: Body,
|
||||||
read_all: bool,
|
read_all: bool,
|
||||||
|
min_block_size: usize,
|
||||||
|
avg_block_size: usize,
|
||||||
max_block_size: usize,
|
max_block_size: usize,
|
||||||
buf: VecDeque<u8>,
|
buf: VecDeque<u8>,
|
||||||
chunker: FastCdcIncr<'static>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl BodyChunker {
|
impl BodyChunker {
|
||||||
fn new(body: Body, block_size: usize) -> Self {
|
fn new(body: Body, block_size: usize) -> Self {
|
||||||
|
let min_block_size = block_size / 4 * 3;
|
||||||
|
let avg_block_size = block_size;
|
||||||
let max_block_size = block_size * 2;
|
let max_block_size = block_size * 2;
|
||||||
let chunker = FastCdc::new(&GEAR_64, block_size as u64 / 2, block_size as u64, max_block_size as u64);
|
|
||||||
let chunker = (&chunker).into();
|
|
||||||
Self {
|
Self {
|
||||||
body,
|
body,
|
||||||
read_all: false,
|
read_all: false,
|
||||||
|
min_block_size,
|
||||||
|
avg_block_size,
|
||||||
max_block_size,
|
max_block_size,
|
||||||
buf: VecDeque::with_capacity(2 * max_block_size),
|
buf: VecDeque::with_capacity(2 * max_block_size),
|
||||||
chunker,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> {
|
async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> {
|
||||||
|
@ -299,12 +301,14 @@ impl BodyChunker {
|
||||||
}
|
}
|
||||||
if self.buf.len() == 0 {
|
if self.buf.len() == 0 {
|
||||||
Ok(None)
|
Ok(None)
|
||||||
} else if let Some(index) = self.chunker.push(self.buf.make_contiguous()) {
|
|
||||||
let block = self.buf.drain(..index).collect::<Vec<u8>>();
|
|
||||||
Ok(Some(block))
|
|
||||||
} else {
|
} else {
|
||||||
let block = self.buf.drain(..).collect::<Vec<u8>>();
|
let mut iter = FastCDC::with_eof(self.buf.make_contiguous(), self.min_block_size, self.avg_block_size, self.max_block_size, self.read_all);
|
||||||
Ok(Some(block))
|
if let Some(Chunk {length, ..}) = iter.next() {
|
||||||
|
let block = self.buf.drain(..length).collect::<Vec<u8>>();
|
||||||
|
Ok(Some(block))
|
||||||
|
} else {
|
||||||
|
Ok(None)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue