Deuxfleurs/garage
41
56
Fork 73
Code Issues 102 Pull requests 14 Releases 30 Wiki Activity

change crate used for cdc

Browse source 
previous one seemed to output incorrect results
This commit is contained in:
Trinity Pointard 2021-03-17 01:31:35 +01:00 committed by Alex Auvolat
parent 71a13f366e
commit 9c9471c64f
No known key found for this signature in database
GPG key ID: EDABF9711E244EB1
3 changed files with 19 additions and 24 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
Cargo.lock generated
View file

@ -222,10 +222,10 @@ dependencies = [
]
[[package]]
name = "fmt-extra"
version = "0.2.1"
name = "fastcdc"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "07f11f71b1f9be830047fbb1899d90601c3b21a471dc99fe1057303eee37f2b9"
checksum = "5afa29be46b12c8c380b997def8d1ac77c2665da93eb0a768fab0bf4db79333f"
[[package]]
name = "fnv"
@ -389,12 +389,12 @@ dependencies = [
"chrono",
"crypto-mac 0.10.0",
"err-derive",
"fastcdc",
"futures",
"futures-util",
"garage_model",
"garage_table",
"garage_util",
"hash-roll",
"hex",
"hmac",
"http",
@ -600,15 +600,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "hash-roll"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9e27803a4b526df90ed2a3f60523eeec6b5ace6ba7530f9920fbee82027fa11"
dependencies = [
"fmt-extra",
]
[[package]]
name = "hashbrown"
version = "0.9.1"

2
src/api/Cargo.toml
View file

@ -22,7 +22,7 @@ bytes = "1.0"
chrono = "0.4"
crypto-mac = "0.10"
err-derive = "0.3"
hash-roll = "0.3.0"
fastcdc = "1.0.5"
hex = "0.4"
hmac = "0.10"
log = "0.4"

22
src/api/s3_put.rs
View file

@ -2,8 +2,8 @@ use std::collections::{BTreeMap, VecDeque};
use std::fmt::Write;
use std::sync::Arc;
use fastcdc::{Chunk, FastCDC};
use futures::stream::*;
use hash_roll::{ChunkIncr, fastcdc::{FastCdc, FastCdcIncr}, gear_table::GEAR_64};
use hyper::{Body, Request, Response};
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
use sha2::Sha256;
@ -269,22 +269,24 @@ async fn put_block_meta(
struct BodyChunker {
body: Body,
read_all: bool,
min_block_size: usize,
avg_block_size: usize,
max_block_size: usize,
buf: VecDeque<u8>,
chunker: FastCdcIncr<'static>,
}
impl BodyChunker {
fn new(body: Body, block_size: usize) -> Self {
let min_block_size = block_size / 4 * 3;
let avg_block_size = block_size;
let max_block_size = block_size * 2;
let chunker = FastCdc::new(&GEAR_64, block_size as u64 / 2, block_size as u64, max_block_size as u64);
let chunker = (&chunker).into();
Self {
body,
read_all: false,
min_block_size,
avg_block_size,
max_block_size,
buf: VecDeque::with_capacity(2 * max_block_size),
chunker,
}
}
async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> {
@ -299,12 +301,14 @@ impl BodyChunker {
}
if self.buf.len() == 0 {
Ok(None)
} else if let Some(index) = self.chunker.push(self.buf.make_contiguous()) {
let block = self.buf.drain(..index).collect::<Vec<u8>>();
} else {
let mut iter = FastCDC::with_eof(self.buf.make_contiguous(), self.min_block_size, self.avg_block_size, self.max_block_size, self.read_all);
if let Some(Chunk {length, ..}) = iter.next() {
let block = self.buf.drain(..length).collect::<Vec<u8>>();
Ok(Some(block))
} else {
let block = self.buf.drain(..).collect::<Vec<u8>>();
Ok(Some(block))
Ok(None)
}
}
}
}