aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTrinity Pointard <trinity.pointard@gmail.com>2021-03-17 01:31:35 +0100
committerAlex Auvolat <alex@adnab.me>2021-04-27 16:53:48 +0200
commit9c9471c64f977a18e4253a4396eff595e2917edd (patch)
tree44132e0ddcd96f6b626ff8a990540a9c788fdd39
parent71a13f366ea2b5c779bd2ea74385af1b03a2455b (diff)
downloadgarage-9c9471c64f977a18e4253a4396eff595e2917edd.tar.gz
garage-9c9471c64f977a18e4253a4396eff595e2917edd.zip
change crate used for cdc
previous one seemed to output incorrect results
-rw-r--r--Cargo.lock17
-rw-r--r--src/api/Cargo.toml2
-rw-r--r--src/api/s3_put.rs24
3 files changed, 19 insertions, 24 deletions
diff --git a/Cargo.lock b/Cargo.lock
index e71152e4..c97968ef 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -222,10 +222,10 @@ dependencies = [
]
[[package]]
-name = "fmt-extra"
-version = "0.2.1"
+name = "fastcdc"
+version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07f11f71b1f9be830047fbb1899d90601c3b21a471dc99fe1057303eee37f2b9"
+checksum = "5afa29be46b12c8c380b997def8d1ac77c2665da93eb0a768fab0bf4db79333f"
[[package]]
name = "fnv"
@@ -389,12 +389,12 @@ dependencies = [
"chrono",
"crypto-mac 0.10.0",
"err-derive",
+ "fastcdc",
"futures",
"futures-util",
"garage_model",
"garage_table",
"garage_util",
- "hash-roll",
"hex",
"hmac",
"http",
@@ -601,15 +601,6 @@ dependencies = [
]
[[package]]
-name = "hash-roll"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9e27803a4b526df90ed2a3f60523eeec6b5ace6ba7530f9920fbee82027fa11"
-dependencies = [
- "fmt-extra",
-]
-
-[[package]]
name = "hashbrown"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml
index 5bc170a3..b328f671 100644
--- a/src/api/Cargo.toml
+++ b/src/api/Cargo.toml
@@ -22,7 +22,7 @@ bytes = "1.0"
chrono = "0.4"
crypto-mac = "0.10"
err-derive = "0.3"
-hash-roll = "0.3.0"
+fastcdc = "1.0.5"
hex = "0.4"
hmac = "0.10"
log = "0.4"
diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs
index e2a1b54d..f5607c9f 100644
--- a/src/api/s3_put.rs
+++ b/src/api/s3_put.rs
@@ -2,8 +2,8 @@ use std::collections::{BTreeMap, VecDeque};
use std::fmt::Write;
use std::sync::Arc;
+use fastcdc::{Chunk, FastCDC};
use futures::stream::*;
-use hash_roll::{ChunkIncr, fastcdc::{FastCdc, FastCdcIncr}, gear_table::GEAR_64};
use hyper::{Body, Request, Response};
use md5::{digest::generic_array::*, Digest as Md5Digest, Md5};
use sha2::Sha256;
@@ -269,22 +269,24 @@ async fn put_block_meta(
struct BodyChunker {
body: Body,
read_all: bool,
+ min_block_size: usize,
+ avg_block_size: usize,
max_block_size: usize,
buf: VecDeque<u8>,
- chunker: FastCdcIncr<'static>,
}
impl BodyChunker {
fn new(body: Body, block_size: usize) -> Self {
+ let min_block_size = block_size / 4 * 3;
+ let avg_block_size = block_size;
let max_block_size = block_size * 2;
- let chunker = FastCdc::new(&GEAR_64, block_size as u64 / 2, block_size as u64, max_block_size as u64);
- let chunker = (&chunker).into();
Self {
body,
read_all: false,
+ min_block_size,
+ avg_block_size,
max_block_size,
buf: VecDeque::with_capacity(2 * max_block_size),
- chunker,
}
}
async fn next(&mut self) -> Result<Option<Vec<u8>>, GarageError> {
@@ -299,12 +301,14 @@ impl BodyChunker {
}
if self.buf.len() == 0 {
Ok(None)
- } else if let Some(index) = self.chunker.push(self.buf.make_contiguous()) {
- let block = self.buf.drain(..index).collect::<Vec<u8>>();
- Ok(Some(block))
} else {
- let block = self.buf.drain(..).collect::<Vec<u8>>();
- Ok(Some(block))
+ let mut iter = FastCDC::with_eof(self.buf.make_contiguous(), self.min_block_size, self.avg_block_size, self.max_block_size, self.read_all);
+ if let Some(Chunk {length, ..}) = iter.next() {
+ let block = self.buf.drain(..length).collect::<Vec<u8>>();
+ Ok(Some(block))
+ } else {
+ Ok(None)
+ }
}
}
}