diff options
author | Trinity Pointard <trinity.pointard@gmail.com> | 2021-03-16 21:08:39 +0100 |
---|---|---|
committer | Alex Auvolat <alex@adnab.me> | 2021-04-27 16:53:47 +0200 |
commit | 71a13f366ea2b5c779bd2ea74385af1b03a2455b (patch) | |
tree | ed5c979f2de8d75e591bfff308f668d253434a46 /src | |
parent | 119217f9f6f904efda33d80f744404c6935ed827 (diff) | |
download | garage-71a13f366ea2b5c779bd2ea74385af1b03a2455b.tar.gz garage-71a13f366ea2b5c779bd2ea74385af1b03a2455b.zip |
add content defined chuking
Diffstat (limited to 'src')
-rw-r--r-- | src/api/Cargo.toml | 2 | ||||
-rw-r--r-- | src/api/s3_put.rs | 20 |
2 files changed, 15 insertions, 7 deletions
diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 0b824ca3..5bc170a3 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -22,10 +22,12 @@ bytes = "1.0" chrono = "0.4" crypto-mac = "0.10" err-derive = "0.3" +hash-roll = "0.3.0" hex = "0.4" hmac = "0.10" log = "0.4" md-5 = "0.9" +rand = "0.7" sha2 = "0.9" futures = "0.3" diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index c4e3b818..e2a1b54d 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -3,6 +3,7 @@ use std::fmt::Write; use std::sync::Arc; use futures::stream::*; +use hash_roll::{ChunkIncr, fastcdc::{FastCdc, FastCdcIncr}, gear_table::GEAR_64}; use hyper::{Body, Request, Response}; use md5::{digest::generic_array::*, Digest as Md5Digest, Md5}; use sha2::Sha256; @@ -268,21 +269,26 @@ async fn put_block_meta( struct BodyChunker { body: Body, read_all: bool, - block_size: usize, + max_block_size: usize, buf: VecDeque<u8>, + chunker: FastCdcIncr<'static>, } impl BodyChunker { fn new(body: Body, block_size: usize) -> Self { + let max_block_size = block_size * 2; + let chunker = FastCdc::new(&GEAR_64, block_size as u64 / 2, block_size as u64, max_block_size as u64); + let chunker = (&chunker).into(); Self { body, read_all: false, - block_size, - buf: VecDeque::with_capacity(2 * block_size), + max_block_size, + buf: VecDeque::with_capacity(2 * max_block_size), + chunker, } } async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> { - while !self.read_all && self.buf.len() < self.block_size { + while !self.read_all && self.buf.len() < self.max_block_size { if let Some(block) = self.body.next().await { let bytes = block?; trace!("Body next: {} bytes", bytes.len()); @@ -293,11 +299,11 @@ impl BodyChunker { } if self.buf.len() == 0 { Ok(None) - } else if self.buf.len() <= self.block_size { - let block = self.buf.drain(..).collect::<Vec<u8>>(); + } else if let Some(index) = self.chunker.push(self.buf.make_contiguous()) { + let block = self.buf.drain(..index).collect::<Vec<u8>>(); Ok(Some(block)) } else { - let block = self.buf.drain(..self.block_size).collect::<Vec<u8>>(); + let block = self.buf.drain(..).collect::<Vec<u8>>(); Ok(Some(block)) } } |