diff options
Diffstat (limited to 'src/garage/tests')
| -rw-r--r-- | src/garage/tests/common/custom_requester.rs | 280 | ||||
| -rw-r--r-- | src/garage/tests/common/garage.rs | 2 | ||||
| -rw-r--r-- | src/garage/tests/common/mod.rs | 11 | ||||
| -rw-r--r-- | src/garage/tests/lib.rs | 1 | ||||
| -rw-r--r-- | src/garage/tests/multipart.rs | 2 | ||||
| -rw-r--r-- | src/garage/tests/objects.rs | 2 | ||||
| -rw-r--r-- | src/garage/tests/simple.rs | 2 | ||||
| -rw-r--r-- | src/garage/tests/streaming_signature.rs | 101 | ||||
| -rw-r--r-- | src/garage/tests/website.rs | 2 |
9 files changed, 397 insertions, 6 deletions
diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs new file mode 100644 index 00000000..11dc10d6 --- /dev/null +++ b/src/garage/tests/common/custom_requester.rs @@ -0,0 +1,280 @@ +#![allow(dead_code)] + +use std::collections::HashMap; +use std::convert::TryFrom; + +use chrono::{offset::Utc, DateTime}; +use hmac::{Hmac, Mac}; +use hyper::client::HttpConnector; +use hyper::{Body, Client, Method, Request, Response, Uri}; + +use super::garage::{Instance, Key}; +use garage_api::signature; + +/// You should ever only use this to send requests AWS sdk won't send, +/// like to reproduce behavior of unusual implementations found to be +/// problematic. +pub struct CustomRequester { + key: Key, + uri: Uri, + client: Client<HttpConnector>, +} + +impl CustomRequester { + pub fn new(instance: &Instance) -> Self { + /* + let credentials = Credentials::new( + &instance.key.id, + &instance.key.secret, + None, + None, + "garage-integ-test", + ); + let endpoint = Endpoint::immutable(instance.uri()); + */ + CustomRequester { + key: instance.key.clone(), + uri: instance.uri(), + client: Client::new(), + } + } + + pub fn builder(&self, bucket: String) -> RequestBuilder<'_> { + RequestBuilder { + requester: self, + bucket, + method: Method::GET, + path: String::new(), + query_params: HashMap::new(), + signed_headers: HashMap::new(), + unsigned_headers: HashMap::new(), + body: Vec::new(), + body_signature: BodySignature::Classic, + vhost_style: false, + } + } + /* + pub async fn request(&self, method: &str, path: String, headers: &HashMap<String, (bool, String)>, body: &[u8], vhost_style: bool) -> hyper::Result<Response<Body>> { + let request = Request::builder() + .method( + self.client.request(todo!()).await + } + */ +} + +pub struct RequestBuilder<'a> { + requester: &'a CustomRequester, + bucket: String, + method: Method, + path: String, + query_params: HashMap<String, Option<String>>, + signed_headers: HashMap<String, String>, + unsigned_headers: HashMap<String, String>, + body: Vec<u8>, + body_signature: BodySignature, + vhost_style: bool, +} + +impl<'a> RequestBuilder<'a> { + pub fn method(&mut self, method: Method) -> &mut Self { + self.method = method; + self + } + + pub fn path(&mut self, path: String) -> &mut Self { + self.path = path; + self + } + + pub fn query_params(&mut self, query_params: HashMap<String, Option<String>>) -> &mut Self { + self.query_params = query_params; + self + } + + pub fn signed_headers(&mut self, signed_headers: HashMap<String, String>) -> &mut Self { + self.signed_headers = signed_headers; + self + } + + pub fn unsigned_headers(&mut self, unsigned_headers: HashMap<String, String>) -> &mut Self { + self.unsigned_headers = unsigned_headers; + self + } + + pub fn body(&mut self, body: Vec<u8>) -> &mut Self { + self.body = body; + self + } + + pub fn body_signature(&mut self, body_signature: BodySignature) -> &mut Self { + self.body_signature = body_signature; + self + } + + pub fn vhost_style(&mut self, vhost_style: bool) -> &mut Self { + self.vhost_style = vhost_style; + self + } + + pub async fn send(&mut self) -> hyper::Result<Response<Body>> { + // TODO this is a bit incorrect in that path and query params should be url-encoded and + // aren't, but this is good enought for now. + + let query = query_param_to_string(&self.query_params); + let (host, path) = if self.vhost_style { + ( + format!("{}.s3.garage", self.bucket), + format!("{}{}", self.path, query), + ) + } else { + ( + "s3.garage".to_owned(), + format!("{}/{}{}", self.bucket, self.path, query), + ) + }; + let uri = format!("{}{}", self.requester.uri, path); + + let now = Utc::now(); + let scope = signature::compute_scope(&now, super::REGION.as_ref()); + let mut signer = signature::signing_hmac( + &now, + &self.requester.key.secret, + super::REGION.as_ref(), + "s3", + ) + .unwrap(); + let streaming_signer = signer.clone(); + + let mut all_headers = self.signed_headers.clone(); + + let date = now.format(signature::LONG_DATETIME).to_string(); + all_headers.insert("x-amz-date".to_owned(), date); + all_headers.insert("host".to_owned(), host); + + let body_sha = match self.body_signature { + BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(), + BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)), + BodySignature::Streaming(size) => { + all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned()); + all_headers.insert( + "x-amz-decoded-content-length".to_owned(), + self.body.len().to_string(), + ); + // this is a pretty lazy and inefficient way to do it, but it's enought for + // test code. + all_headers.insert( + "content-length".to_owned(), + to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "") + .len() + .to_string(), + ); + + "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned() + } + }; + all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone()); + + let mut signed_headers = all_headers + .iter() + .map(|(k, _)| k.as_ref()) + .collect::<Vec<&str>>(); + signed_headers.sort(); + let signed_headers = signed_headers.join(";"); + + all_headers.extend(self.unsigned_headers.clone()); + + let canonical_request = signature::payload::canonical_request( + &self.method, + &Uri::try_from(&uri).unwrap(), + &all_headers, + &signed_headers, + &body_sha, + ); + + let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request); + + signer.update(string_to_sign.as_bytes()); + let signature = hex::encode(signer.finalize().into_bytes()); + let authorization = format!( + "AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}", + self.requester.key.id, scope, signed_headers, signature + ); + all_headers.insert("authorization".to_owned(), authorization); + + let mut request = Request::builder(); + for (k, v) in all_headers { + request = request.header(k, v); + } + + let body = if let BodySignature::Streaming(size) = self.body_signature { + to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope) + } else { + self.body.clone() + }; + let request = request + .uri(uri) + .method(self.method.clone()) + .body(Body::from(body)) + .unwrap(); + self.requester.client.request(request).await + } +} + +pub enum BodySignature { + Unsigned, + Classic, + Streaming(usize), +} + +fn query_param_to_string(params: &HashMap<String, Option<String>>) -> String { + if params.is_empty() { + return String::new(); + } + + "?".to_owned() + + ¶ms + .iter() + .map(|(k, v)| { + if let Some(v) = v { + format!("{}={}", k, v) + } else { + k.clone() + } + }) + .collect::<Vec<String>>() + .join("&") +} + +fn to_streaming_body( + body: &[u8], + chunk_size: usize, + mut seed: String, + hasher: Hmac<sha2::Sha256>, + now: DateTime<Utc>, + scope: &str, +) -> Vec<u8> { + const SHA_NULL: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + let now = now.format(signature::LONG_DATETIME).to_string(); + let mut res = Vec::with_capacity(body.len()); + for chunk in body.chunks(chunk_size).chain(std::iter::once(&[][..])) { + let to_sign = format!( + "AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}", + now, + scope, + seed, + SHA_NULL, + hex::encode(garage_util::data::sha256sum(chunk)) + ); + + let mut hasher = hasher.clone(); + hasher.update(to_sign.as_bytes()); + seed = hex::encode(hasher.finalize().into_bytes()); + + let header = format!("{:x};chunk-signature={}\r\n", chunk.len(), seed); + res.extend_from_slice(header.as_bytes()); + res.extend_from_slice(chunk); + res.extend_from_slice(b"\r\n"); + } + + res +} diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs index 36adb55e..88c51501 100644 --- a/src/garage/tests/common/garage.rs +++ b/src/garage/tests/common/garage.rs @@ -11,7 +11,7 @@ pub const DEFAULT_PORT: u16 = 49995; static GARAGE_TEST_SECRET: &str = "c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44"; -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct Key { pub name: String, pub id: String, diff --git a/src/garage/tests/common/mod.rs b/src/garage/tests/common/mod.rs index 32fa3848..8f88c731 100644 --- a/src/garage/tests/common/mod.rs +++ b/src/garage/tests/common/mod.rs @@ -5,22 +5,31 @@ use ext::*; pub mod macros; pub mod client; +pub mod custom_requester; pub mod ext; pub mod garage; +use custom_requester::CustomRequester; + const REGION: Region = Region::from_static("garage-integ-test"); pub struct Context { pub garage: &'static garage::Instance, pub client: Client, + pub custom_request: CustomRequester, } impl Context { fn new() -> Self { let garage = garage::instance(); let client = client::build_client(garage); + let custom_request = CustomRequester::new(garage); - Context { garage, client } + Context { + garage, + client, + custom_request, + } } /// Create an unique bucket with a random suffix. diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs index 9d7e4322..8799c395 100644 --- a/src/garage/tests/lib.rs +++ b/src/garage/tests/lib.rs @@ -7,4 +7,5 @@ mod list; mod multipart; mod objects; mod simple; +mod streaming_signature; mod website; diff --git a/src/garage/tests/multipart.rs b/src/garage/tests/multipart.rs index 7fec4de6..895a2993 100644 --- a/src/garage/tests/multipart.rs +++ b/src/garage/tests/multipart.rs @@ -1,6 +1,6 @@ use crate::common; use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart}; -use aws_sdk_s3::ByteStream; +use aws_sdk_s3::types::ByteStream; const SZ_5MB: usize = 5 * 1024 * 1024; const SZ_10MB: usize = 10 * 1024 * 1024; diff --git a/src/garage/tests/objects.rs b/src/garage/tests/objects.rs index 9086073e..e1175b81 100644 --- a/src/garage/tests/objects.rs +++ b/src/garage/tests/objects.rs @@ -1,6 +1,6 @@ use crate::common; use aws_sdk_s3::model::{Delete, ObjectIdentifier}; -use aws_sdk_s3::ByteStream; +use aws_sdk_s3::types::ByteStream; const STD_KEY: &str = "hello world"; const CTRL_KEY: &str = "\x00\x01\x02\x00"; diff --git a/src/garage/tests/simple.rs b/src/garage/tests/simple.rs index a15cbf80..f54ae9ac 100644 --- a/src/garage/tests/simple.rs +++ b/src/garage/tests/simple.rs @@ -2,7 +2,7 @@ use crate::common; #[tokio::test] async fn test_simple() { - use aws_sdk_s3::ByteStream; + use aws_sdk_s3::types::ByteStream; let ctx = common::context(); let bucket = ctx.create_bucket("test-simple"); diff --git a/src/garage/tests/streaming_signature.rs b/src/garage/tests/streaming_signature.rs new file mode 100644 index 00000000..66f985ac --- /dev/null +++ b/src/garage/tests/streaming_signature.rs @@ -0,0 +1,101 @@ +use std::collections::HashMap; + +use crate::common; +use common::custom_requester::BodySignature; +use hyper::Method; + +const STD_KEY: &str = "hello-world"; +//const CTRL_KEY: &str = "\x00\x01\x02\x00"; +const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +#[tokio::test] +async fn test_putobject_streaming() { + let ctx = common::context(); + let bucket = ctx.create_bucket("putobject-streaming"); + + { + // Send an empty object (can serve as a directory marker) + // with a content type + let etag = "\"d41d8cd98f00b204e9800998ecf8427e\""; + let content_type = "text/csv"; + let mut headers = HashMap::new(); + headers.insert("content-type".to_owned(), content_type.to_owned()); + let _ = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + .path(STD_KEY.to_owned()) + .unsigned_headers(headers) + .vhost_style(true) + .body(vec![]) + .body_signature(BodySignature::Streaming(10)) + .send() + .await + .unwrap(); + + // assert_eq!(r.e_tag.unwrap().as_str(), etag); + // We return a version ID here + // We should check if Amazon is returning one when versioning is not enabled + // assert!(r.version_id.is_some()); + + //let _version = r.version_id.unwrap(); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + .key(STD_KEY) + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, b""); + assert_eq!(o.e_tag.unwrap(), etag); + // We do not return version ID + // We should check if Amazon is returning one when versioning is not enabled + // assert_eq!(o.version_id.unwrap(), _version); + assert_eq!(o.content_type.unwrap(), content_type); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length, 0); + assert_eq!(o.parts_count, 0); + assert_eq!(o.tag_count, 0); + } + + { + let etag = "\"46cf18a9b447991b450cad3facf5937e\""; + + let _ = ctx + .custom_request + .builder(bucket.clone()) + .method(Method::PUT) + //.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this + //fail + .path("abc".to_owned()) + .vhost_style(true) + .body(BODY.to_vec()) + .body_signature(BodySignature::Streaming(16)) + .send() + .await + .unwrap(); + + // assert_eq!(r.e_tag.unwrap().as_str(), etag); + // assert!(r.version_id.is_some()); + + let o = ctx + .client + .get_object() + .bucket(&bucket) + //.key(CTRL_KEY) + .key("abc") + .send() + .await + .unwrap(); + + assert_bytes_eq!(o.body, BODY); + assert_eq!(o.e_tag.unwrap(), etag); + assert!(o.last_modified.is_some()); + assert_eq!(o.content_length, 62); + assert_eq!(o.parts_count, 0); + assert_eq!(o.tag_count, 0); + } +} diff --git a/src/garage/tests/website.rs b/src/garage/tests/website.rs index 34093a79..963d11ea 100644 --- a/src/garage/tests/website.rs +++ b/src/garage/tests/website.rs @@ -2,7 +2,7 @@ use crate::common; use crate::common::ext::*; use aws_sdk_s3::{ model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, - ByteStream, + types::ByteStream, }; use http::Request; use hyper::{ |