From a32c0bac50ba8de81fc9563b1a7cd8e78aff7b18 Mon Sep 17 00:00:00 2001 From: Trinity Pointard Date: Wed, 17 Mar 2021 01:31:35 +0100 Subject: [PATCH] change crate used for cdc previous one seemed to output incorrect results --- Cargo.lock | 23 +++++++---------------- src/api/Cargo.toml | 2 +- src/api/s3_put.rs | 30 ++++++++++++++++++++---------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a696b479..23093482 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -233,6 +233,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "fastcdc" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5afa29be46b12c8c380b997def8d1ac77c2665da93eb0a768fab0bf4db79333f" + [[package]] name = "fasthash" version = "0.4.0" @@ -255,12 +261,6 @@ dependencies = [ "gcc", ] -[[package]] -name = "fmt-extra" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07f11f71b1f9be830047fbb1899d90601c3b21a471dc99fe1057303eee37f2b9" - [[package]] name = "fnv" version = "1.0.7" @@ -432,12 +432,12 @@ dependencies = [ "chrono", "crypto-mac 0.10.0", "err-derive", + "fastcdc", "futures", "futures-util", "garage_model", "garage_table", "garage_util", - "hash-roll", "hex", "hmac", "http", @@ -662,15 +662,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hash-roll" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9e27803a4b526df90ed2a3f60523eeec6b5ace6ba7530f9920fbee82027fa11" -dependencies = [ - "fmt-extra", -] - [[package]] name = "hashbrown" version = "0.9.1" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index f29bea31..a7b1703b 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -22,7 +22,7 @@ bytes = "1.0" chrono = "0.4" crypto-mac = "0.10" err-derive = "0.3" -hash-roll = "0.3.0" +fastcdc = "1.0.5" hex = "0.4" hmac = "0.10" log = "0.4" diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index e2a1b54d..b7b4e998 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -2,8 +2,8 @@ use std::collections::{BTreeMap, VecDeque}; use std::fmt::Write; use std::sync::Arc; +use fastcdc::{Chunk, FastCDC}; use futures::stream::*; -use hash_roll::{ChunkIncr, fastcdc::{FastCdc, FastCdcIncr}, gear_table::GEAR_64}; use hyper::{Body, Request, Response}; use md5::{digest::generic_array::*, Digest as Md5Digest, Md5}; use sha2::Sha256; @@ -269,25 +269,33 @@ async fn put_block_meta( struct BodyChunker { body: Body, read_all: bool, + min_block_size: usize, + avg_block_size: usize, max_block_size: usize, buf: VecDeque, - chunker: FastCdcIncr<'static>, + chunk_buf: VecDeque, } impl BodyChunker { fn new(body: Body, block_size: usize) -> Self { + let min_block_size = block_size / 4 * 3; + let avg_block_size = block_size; let max_block_size = block_size * 2; - let chunker = FastCdc::new(&GEAR_64, block_size as u64 / 2, block_size as u64, max_block_size as u64); - let chunker = (&chunker).into(); Self { body, read_all: false, + min_block_size, + avg_block_size, max_block_size, buf: VecDeque::with_capacity(2 * max_block_size), - chunker, + chunk_buf: VecDeque::new(), } } async fn next(&mut self) -> Result>, GarageError> { + if let Some(Chunk {length, ..}) = self.chunk_buf.pop_front() { + let block = self.buf.drain(..length).collect::>(); + return Ok(Some(block)); + } while !self.read_all && self.buf.len() < self.max_block_size { if let Some(block) = self.body.next().await { let bytes = block?; @@ -299,12 +307,14 @@ impl BodyChunker { } if self.buf.len() == 0 { Ok(None) - } else if let Some(index) = self.chunker.push(self.buf.make_contiguous()) { - let block = self.buf.drain(..index).collect::>(); - Ok(Some(block)) } else { - let block = self.buf.drain(..).collect::>(); - Ok(Some(block)) + let mut iter = FastCDC::with_eof(self.buf.make_contiguous(), self.min_block_size, self.avg_block_size, self.max_block_size, self.read_all); + if let Some(Chunk {length, ..}) = iter.next() { + let block = self.buf.drain(..length).collect::>(); + Ok(Some(block)) + } else { + Ok(None) + } } } }