From 4a5bbbb81088c9bd25bbe142f67daf4669b6538e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sat, 5 Dec 2020 19:23:46 +0100 Subject: Propose ETag fix --- src/api/Cargo.toml | 1 + src/api/s3_get.rs | 5 ++++- src/api/s3_put.rs | 17 ++++++++++++++++- 3 files changed, 21 insertions(+), 2 deletions(-) (limited to 'src/api') diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index a366f9b8..079993c3 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -27,6 +27,7 @@ md-5 = "0.9.1" sha2 = "0.8" hmac = "0.7" crypto-mac = "0.7" +rand = "0.7" futures = "0.3" futures-util = "0.3" diff --git a/src/api/s3_get.rs b/src/api/s3_get.rs index 43215923..1a23f476 100644 --- a/src/api/s3_get.rs +++ b/src/api/s3_get.rs @@ -24,10 +24,13 @@ fn object_headers( "Content-Type", version_meta.headers.content_type.to_string(), ) - .header("ETag", version_meta.etag.to_string()) .header("Last-Modified", date_str) .header("Accept-Ranges", format!("bytes")); + if !version_meta.etag.is_empty() { + resp = resp.header("ETag", format!("\"{}\"", version_meta.etag)); + } + for (k, v) in version_meta.headers.other.iter() { resp = resp.header(k, v.to_string()); } diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index 9c4d625c..c42309b2 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -428,6 +428,21 @@ pub async fn handle_complete_multipart_upload( _ => unreachable!(), }; + // ETag calculation: we produce ETags that have the same form as + // those of S3 multipart uploads, but we don't use their actual + // calculation for the first part (we use random bytes). This + // shouldn't impact compatibility as the S3 docs specify that + // the ETag is an opaque value in case of a multipart upload. + // See also: https://teppen.io/2018/06/23/aws_s3_etags/ + let num_parts = version.blocks().last().unwrap().part_number + - version.blocks().first().unwrap().part_number + + 1; + let etag = format!( + "{}-{}", + hex::encode(&rand::random::<[u8; 16]>()[..]), + num_parts + ); + // TODO: check that all the parts that they pretend they gave us are indeed there // TODO: when we read the XML from _req, remember to check the sha256 sum of the payload // against the signed x-amz-content-sha256 @@ -442,7 +457,7 @@ pub async fn handle_complete_multipart_upload( ObjectVersionMeta { headers, size: total_size, - etag: "".to_string(), // TODO + etag: etag, }, version.blocks()[0].hash, )); -- cgit v1.2.3 From 022b386a5085cad79d649a82846c41cad730920b Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Sun, 6 Dec 2020 15:39:03 +0100 Subject: Improved compatibility on list API call --- src/api/s3_list.rs | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) (limited to 'src/api') diff --git a/src/api/s3_list.rs b/src/api/s3_list.rs index 3b739a8a..599d0d11 100644 --- a/src/api/s3_list.rs +++ b/src/api/s3_list.rs @@ -18,6 +18,7 @@ use crate::encoding::*; struct ListResultInfo { last_modified: u64, size: u64, + etag: String, } pub async fn handle_list( @@ -56,12 +57,12 @@ pub async fn handle_list( for object in objects.iter() { if !object.key.starts_with(prefix) { - truncated = false; + truncated = None; break 'query_loop; } if let Some(version) = object.versions().iter().find(|x| x.is_data()) { if result_keys.len() + result_common_prefixes.len() >= max_keys { - truncated = true; + truncated = Some(object.key.to_string()); break 'query_loop; } let common_prefix = if delimiter.len() > 0 { @@ -75,19 +76,18 @@ pub async fn handle_list( if let Some(pfx) = common_prefix { result_common_prefixes.insert(pfx.to_string()); } else { - let size = match &version.state { - ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => { - meta.size - } + let meta = match &version.state { + ObjectVersionState::Complete(ObjectVersionData::Inline(meta, _)) => meta, ObjectVersionState::Complete(ObjectVersionData::FirstBlock(meta, _)) => { - meta.size + meta } _ => unreachable!(), }; let info = match result_keys.get(&object.key) { None => ListResultInfo { last_modified: version.timestamp, - size, + size: meta.size, + etag: meta.etag.to_string(), }, Some(_lri) => { return Err(Error::Message(format!("Duplicate key?? {}", object.key))) @@ -98,7 +98,7 @@ pub async fn handle_list( } } if objects.len() < max_keys + 1 { - truncated = false; + truncated = None; break 'query_loop; } if objects.len() > 0 { @@ -113,11 +113,22 @@ pub async fn handle_list( r#""# ) .unwrap(); - writeln!(&mut xml, "\t{}", bucket).unwrap(); + writeln!(&mut xml, "\t{}", bucket).unwrap(); writeln!(&mut xml, "\t{}", prefix).unwrap(); + if let Some(mkr) = marker { + writeln!(&mut xml, "\t{}", mkr).unwrap(); + } writeln!(&mut xml, "\t{}", result_keys.len()).unwrap(); writeln!(&mut xml, "\t{}", max_keys).unwrap(); - writeln!(&mut xml, "\t{}", truncated).unwrap(); + writeln!( + &mut xml, + "\t{}", + truncated.is_some() + ) + .unwrap(); + if let Some(next_marker) = truncated { + writeln!(&mut xml, "\t{}", next_marker).unwrap(); + } for (key, info) in result_keys.iter() { let last_modif = NaiveDateTime::from_timestamp(info.last_modified as i64 / 1000, 0); let last_modif = DateTime::::from_utc(last_modif, Utc); @@ -132,6 +143,9 @@ pub async fn handle_list( .unwrap(); writeln!(&mut xml, "\t\t{}", last_modif).unwrap(); writeln!(&mut xml, "\t\t{}", info.size).unwrap(); + if !info.etag.is_empty() { + writeln!(&mut xml, "\t\t\"{}\"", info.etag).unwrap(); + } writeln!(&mut xml, "\t\tSTANDARD").unwrap(); writeln!(&mut xml, "\t").unwrap(); } -- cgit v1.2.3