diff options
| -rw-r--r-- | Cargo.lock | 46 | ||||
| -rw-r--r-- | src/api/Cargo.toml | 1 | ||||
| -rw-r--r-- | src/api/s3_put.rs | 28 |
3 files changed, 67 insertions, 8 deletions
@@ -60,6 +60,14 @@ dependencies = [ ] [[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "block-padding" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -187,6 +195,14 @@ dependencies = [ ] [[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "generic-array 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "env_logger" version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -382,6 +398,7 @@ dependencies = [ "httpdate 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "md-5 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", "percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -603,6 +620,15 @@ dependencies = [ ] [[package]] +name = "generic-array" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typenum 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "version_check 0.9.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "gethostname" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -844,6 +870,16 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] +name = "md-5" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "block-buffer 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "digest 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "opaque-debug 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "memchr" version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -942,6 +978,11 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] +name = "opaque-debug" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] name = "parking_lot" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1695,6 +1736,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +"checksum block-buffer 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" "checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" "checksum bumpalo 3.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" "checksum byte-tools 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" @@ -1711,6 +1753,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" "checksum crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4434400df11d95d556bac068ddfedd482915eb18fe8bea89bc80b6e4b1c179e5" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +"checksum digest 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" "checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" "checksum err-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" @@ -1734,6 +1777,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum garage_util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77de76a4167c041094f3f3415c6d3d773373e0326668fbce70dfd3b024788800" "checksum generator 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "add72f17bb81521258fcc8a7a3245b1e184e916bfbe34f0ea89558f440df5c68" "checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" +"checksum generic-array 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)" = "60fb4bb6bba52f78a471264d9a3b7d026cc0af47b22cd2cffbc0b787ca003e63" "checksum gethostname 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028" "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" "checksum h2 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "79b7246d7e4b979c03fa093da39cfb3617a96bbeee6310af63991668d7e843ff" @@ -1762,6 +1806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum loom 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4ecc775857611e1df29abba5c41355cdf540e7e9d4acfdf0f355eefee82330b7" "checksum matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +"checksum md-5 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" "checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" "checksum memoffset 0.5.5 (registry+https://github.com/rust-lang/crates.io-index)" = "c198b026e1bbf08a937e94c6c60f9ec4a2267f5b0d2eec9c1b21b061ce2be55f" "checksum mio 0.6.22 (registry+https://github.com/rust-lang/crates.io-index)" = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" @@ -1773,6 +1818,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" "checksum once_cell 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0b631f7e854af39a1739f401cf34a8a013dfe09eac4fa4dba91e9768bd28168d" "checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" +"checksum opaque-debug 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" "checksum parking_lot 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" "checksum parking_lot_core 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" "checksum percent-encoding 2.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index c8c5bf22..4e0599d5 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -21,6 +21,7 @@ bytes = "0.4" hex = "0.3" log = "0.4" chrono = "0.4" +md-5 = "0.9.1" sha2 = "0.8" hmac = "0.7" crypto-mac = "0.7" diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index 0a010a82..fdc188c9 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -2,6 +2,7 @@ use std::collections::{BTreeMap, VecDeque}; use std::fmt::Write; use std::sync::Arc; +use md5::{Md5, Digest}; use futures::stream::*; use hyper::{Body, Request, Response}; @@ -41,18 +42,22 @@ pub async fn handle_put( }; if first_block.len() < INLINE_THRESHOLD { + let mut md5sum = Md5::new(); + md5sum.update(&first_block[..]); + let etag = hex::encode(md5sum.finalize()); + object_version.state = ObjectVersionState::Complete(ObjectVersionData::Inline( ObjectVersionMeta { headers, size: first_block.len() as u64, - etag: "".to_string(), // TODO + etag: etag.clone(), }, first_block, )); let object = Object::new(bucket.into(), key.into(), vec![object_version]); garage.object_table.insert(&object).await?; - return Ok(put_response(version_uuid)); + return Ok(put_response(version_uuid, etag)); } let version = Version::new(version_uuid, bucket.into(), key.into(), false, vec![]); @@ -61,7 +66,7 @@ pub async fn handle_put( let object = Object::new(bucket.into(), key.into(), vec![object_version.clone()]); garage.object_table.insert(&object).await?; - let total_size = read_and_put_blocks( + let (total_size, etag) = read_and_put_blocks( &garage, version, 1, @@ -77,7 +82,7 @@ pub async fn handle_put( ObjectVersionMeta { headers, size: total_size, - etag: "".to_string(), // TODO + etag: etag.clone(), }, first_block_hash, )); @@ -85,7 +90,7 @@ pub async fn handle_put( let object = Object::new(bucket.into(), key.into(), vec![object_version]); garage.object_table.insert(&object).await?; - Ok(put_response(version_uuid)) + Ok(put_response(version_uuid, etag)) } async fn read_and_put_blocks( @@ -95,7 +100,10 @@ async fn read_and_put_blocks( first_block: Vec<u8>, first_block_hash: Hash, chunker: &mut BodyChunker, -) -> Result<u64, Error> { +) -> Result<(u64, String), Error> { + let mut md5sum = Md5::new(); + md5sum.update(&first_block[..]); + let mut next_offset = first_block.len(); let mut put_curr_version_block = put_block_meta( garage.clone(), @@ -113,6 +121,7 @@ async fn read_and_put_blocks( let (_, _, next_block) = futures::try_join!(put_curr_block, put_curr_version_block, chunker.next())?; if let Some(block) = next_block { + md5sum.update(&block[..]); let block_hash = hash(&block[..]); let block_len = block.len(); put_curr_version_block = put_block_meta( @@ -130,7 +139,9 @@ async fn read_and_put_blocks( } } - Ok(next_offset as u64) + let total_size = next_offset as u64; + let md5sum = hex::encode(md5sum.finalize()); + Ok((total_size, md5sum)) } async fn put_block_meta( @@ -203,9 +214,10 @@ impl BodyChunker { } } -pub fn put_response(version_uuid: UUID) -> Response<Body> { +pub fn put_response(version_uuid: UUID, etag: String) -> Response<Body> { Response::builder() .header("x-amz-version-id", hex::encode(version_uuid)) + .header("ETag", etag) // TODO ETag .body(Body::from(vec![])) .unwrap() |