add test framework for arbitraty S3 requests

and implement some basic test with it

12 files changed, 409 insertions, 16 deletions

diff --git a/src/api/lib.rs b/src/api/lib.rs
index f865325e..de60ec53 100644
--- a/src/api/lib.rs
+++ b/src/api/lib.rs
@@ -10,7 +10,8 @@ mod encoding;
 mod api_server;
 pub use api_server::run_api_server;
 
-mod signature;
+/// This mode is public only to help testing. Don't expect stability here
+pub mod signature;
 
 pub mod helpers;
 mod s3_bucket;
diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs
index 88b58922..88ec1f00 100644
--- a/src/api/signature/payload.rs
+++ b/src/api/signature/payload.rs
@@ -51,8 +51,7 @@ pub async fn check_payload_signature(
 
 	let canonical_request = canonical_request(
 		request.method(),
-		request.uri().path(),
-		&canonical_query_string(request.uri()),
+		request.uri(),
 		&headers,
 		&authorization.signed_headers,
 		&authorization.content_sha256,
@@ -215,7 +214,7 @@ fn parse_credential(cred: &str) -> Result<(String, String), Error> {
 	))
 }
 
-fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String {
+pub fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &str) -> String {
 	let mut hasher = Sha256::default();
 	hasher.update(canonical_req.as_bytes());
 	[
@@ -227,18 +226,17 @@ fn string_to_sign(datetime: &DateTime<Utc>, scope_string: &str, canonical_req: &
 	.join("\n")
 }
 
-fn canonical_request(
+pub fn canonical_request(
 	method: &Method,
-	url_path: &str,
-	canonical_query_string: &str,
+	uri: &hyper::Uri,
 	headers: &HashMap<String, String>,
 	signed_headers: &str,
 	content_sha256: &str,
 ) -> String {
 	[
 		method.as_str(),
-		url_path,
-		canonical_query_string,
+		&uri.path().to_string(),
+		&canonical_query_string(uri),
 		&canonical_header_string(headers, signed_headers),
 		"",
 		signed_headers,
diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml
index 88eff20b..86f52495 100644
--- a/src/garage/Cargo.toml
+++ b/src/garage/Cargo.toml
@@ -55,8 +55,11 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
 netapp = "0.4.1"
 
 [dev-dependencies]
-aws-sdk-s3 = "0.6"
+aws-sdk-s3 = "0.8"
+chrono = "0.4"
 http = "0.2"
+hmac = "0.10"
 hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
+sha2 = "0.9"
 
 static_init = "1.0"
diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs
new file mode 100644
index 00000000..11dc10d6
--- /dev/null
+++ b/src/garage/tests/common/custom_requester.rs
@@ -0,0 +1,280 @@
+#![allow(dead_code)]
+
+use std::collections::HashMap;
+use std::convert::TryFrom;
+
+use chrono::{offset::Utc, DateTime};
+use hmac::{Hmac, Mac};
+use hyper::client::HttpConnector;
+use hyper::{Body, Client, Method, Request, Response, Uri};
+
+use super::garage::{Instance, Key};
+use garage_api::signature;
+
+/// You should ever only use this to send requests AWS sdk won't send,
+/// like to reproduce behavior of unusual implementations found to be
+/// problematic.
+pub struct CustomRequester {
+	key: Key,
+	uri: Uri,
+	client: Client<HttpConnector>,
+}
+
+impl CustomRequester {
+	pub fn new(instance: &Instance) -> Self {
+		/*
+		let credentials = Credentials::new(
+			&instance.key.id,
+			&instance.key.secret,
+			None,
+			None,
+			"garage-integ-test",
+		);
+		let endpoint = Endpoint::immutable(instance.uri());
+		*/
+		CustomRequester {
+			key: instance.key.clone(),
+			uri: instance.uri(),
+			client: Client::new(),
+		}
+	}
+
+	pub fn builder(&self, bucket: String) -> RequestBuilder<'_> {
+		RequestBuilder {
+			requester: self,
+			bucket,
+			method: Method::GET,
+			path: String::new(),
+			query_params: HashMap::new(),
+			signed_headers: HashMap::new(),
+			unsigned_headers: HashMap::new(),
+			body: Vec::new(),
+			body_signature: BodySignature::Classic,
+			vhost_style: false,
+		}
+	}
+	/*
+	pub async fn request(&self, method: &str, path: String, headers: &HashMap<String, (bool, String)>, body: &[u8], vhost_style: bool) -> hyper::Result<Response<Body>> {
+		let request = Request::builder()
+			.method(
+		self.client.request(todo!()).await
+	}
+	*/
+}
+
+pub struct RequestBuilder<'a> {
+	requester: &'a CustomRequester,
+	bucket: String,
+	method: Method,
+	path: String,
+	query_params: HashMap<String, Option<String>>,
+	signed_headers: HashMap<String, String>,
+	unsigned_headers: HashMap<String, String>,
+	body: Vec<u8>,
+	body_signature: BodySignature,
+	vhost_style: bool,
+}
+
+impl<'a> RequestBuilder<'a> {
+	pub fn method(&mut self, method: Method) -> &mut Self {
+		self.method = method;
+		self
+	}
+
+	pub fn path(&mut self, path: String) -> &mut Self {
+		self.path = path;
+		self
+	}
+
+	pub fn query_params(&mut self, query_params: HashMap<String, Option<String>>) -> &mut Self {
+		self.query_params = query_params;
+		self
+	}
+
+	pub fn signed_headers(&mut self, signed_headers: HashMap<String, String>) -> &mut Self {
+		self.signed_headers = signed_headers;
+		self
+	}
+
+	pub fn unsigned_headers(&mut self, unsigned_headers: HashMap<String, String>) -> &mut Self {
+		self.unsigned_headers = unsigned_headers;
+		self
+	}
+
+	pub fn body(&mut self, body: Vec<u8>) -> &mut Self {
+		self.body = body;
+		self
+	}
+
+	pub fn body_signature(&mut self, body_signature: BodySignature) -> &mut Self {
+		self.body_signature = body_signature;
+		self
+	}
+
+	pub fn vhost_style(&mut self, vhost_style: bool) -> &mut Self {
+		self.vhost_style = vhost_style;
+		self
+	}
+
+	pub async fn send(&mut self) -> hyper::Result<Response<Body>> {
+		// TODO this is a bit incorrect in that path and query params should be url-encoded and
+		// aren't, but this is good enought for now.
+
+		let query = query_param_to_string(&self.query_params);
+		let (host, path) = if self.vhost_style {
+			(
+				format!("{}.s3.garage", self.bucket),
+				format!("{}{}", self.path, query),
+			)
+		} else {
+			(
+				"s3.garage".to_owned(),
+				format!("{}/{}{}", self.bucket, self.path, query),
+			)
+		};
+		let uri = format!("{}{}", self.requester.uri, path);
+
+		let now = Utc::now();
+		let scope = signature::compute_scope(&now, super::REGION.as_ref());
+		let mut signer = signature::signing_hmac(
+			&now,
+			&self.requester.key.secret,
+			super::REGION.as_ref(),
+			"s3",
+		)
+		.unwrap();
+		let streaming_signer = signer.clone();
+
+		let mut all_headers = self.signed_headers.clone();
+
+		let date = now.format(signature::LONG_DATETIME).to_string();
+		all_headers.insert("x-amz-date".to_owned(), date);
+		all_headers.insert("host".to_owned(), host);
+
+		let body_sha = match self.body_signature {
+			BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(),
+			BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)),
+			BodySignature::Streaming(size) => {
+				all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned());
+				all_headers.insert(
+					"x-amz-decoded-content-length".to_owned(),
+					self.body.len().to_string(),
+				);
+				// this is a pretty lazy and inefficient way to do it, but it's enought for
+				// test code.
+				all_headers.insert(
+					"content-length".to_owned(),
+					to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "")
+						.len()
+						.to_string(),
+				);
+
+				"STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned()
+			}
+		};
+		all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone());
+
+		let mut signed_headers = all_headers
+			.iter()
+			.map(|(k, _)| k.as_ref())
+			.collect::<Vec<&str>>();
+		signed_headers.sort();
+		let signed_headers = signed_headers.join(";");
+
+		all_headers.extend(self.unsigned_headers.clone());
+
+		let canonical_request = signature::payload::canonical_request(
+			&self.method,
+			&Uri::try_from(&uri).unwrap(),
+			&all_headers,
+			&signed_headers,
+			&body_sha,
+		);
+
+		let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request);
+
+		signer.update(string_to_sign.as_bytes());
+		let signature = hex::encode(signer.finalize().into_bytes());
+		let authorization = format!(
+			"AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}",
+			self.requester.key.id, scope, signed_headers, signature
+		);
+		all_headers.insert("authorization".to_owned(), authorization);
+
+		let mut request = Request::builder();
+		for (k, v) in all_headers {
+			request = request.header(k, v);
+		}
+
+		let body = if let BodySignature::Streaming(size) = self.body_signature {
+			to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope)
+		} else {
+			self.body.clone()
+		};
+		let request = request
+			.uri(uri)
+			.method(self.method.clone())
+			.body(Body::from(body))
+			.unwrap();
+		self.requester.client.request(request).await
+	}
+}
+
+pub enum BodySignature {
+	Unsigned,
+	Classic,
+	Streaming(usize),
+}
+
+fn query_param_to_string(params: &HashMap<String, Option<String>>) -> String {
+	if params.is_empty() {
+		return String::new();
+	}
+
+	"?".to_owned()
+		+ &params
+			.iter()
+			.map(|(k, v)| {
+				if let Some(v) = v {
+					format!("{}={}", k, v)
+				} else {
+					k.clone()
+				}
+			})
+			.collect::<Vec<String>>()
+			.join("&")
+}
+
+fn to_streaming_body(
+	body: &[u8],
+	chunk_size: usize,
+	mut seed: String,
+	hasher: Hmac<sha2::Sha256>,
+	now: DateTime<Utc>,
+	scope: &str,
+) -> Vec<u8> {
+	const SHA_NULL: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+	let now = now.format(signature::LONG_DATETIME).to_string();
+	let mut res = Vec::with_capacity(body.len());
+	for chunk in body.chunks(chunk_size).chain(std::iter::once(&[][..])) {
+		let to_sign = format!(
+			"AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}",
+			now,
+			scope,
+			seed,
+			SHA_NULL,
+			hex::encode(garage_util::data::sha256sum(chunk))
+		);
+
+		let mut hasher = hasher.clone();
+		hasher.update(to_sign.as_bytes());
+		seed = hex::encode(hasher.finalize().into_bytes());
+
+		let header = format!("{:x};chunk-signature={}\r\n", chunk.len(), seed);
+		res.extend_from_slice(header.as_bytes());
+		res.extend_from_slice(chunk);
+		res.extend_from_slice(b"\r\n");
+	}
+
+	res
+}
diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs
index 36adb55e..88c51501 100644
--- a/src/garage/tests/common/garage.rs
+++ b/src/garage/tests/common/garage.rs
@@ -11,7 +11,7 @@ pub const DEFAULT_PORT: u16 = 49995;
 static GARAGE_TEST_SECRET: &str =
 	"c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44";
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 pub struct Key {
 	pub name: String,
 	pub id: String,
diff --git a/src/garage/tests/common/mod.rs b/src/garage/tests/common/mod.rs
index 32fa3848..8f88c731 100644
--- a/src/garage/tests/common/mod.rs
+++ b/src/garage/tests/common/mod.rs
@@ -5,22 +5,31 @@ use ext::*;
 pub mod macros;
 
 pub mod client;
+pub mod custom_requester;
 pub mod ext;
 pub mod garage;
 
+use custom_requester::CustomRequester;
+
 const REGION: Region = Region::from_static("garage-integ-test");
 
 pub struct Context {
 	pub garage: &'static garage::Instance,
 	pub client: Client,
+	pub custom_request: CustomRequester,
 }
 
 impl Context {
 	fn new() -> Self {
 		let garage = garage::instance();
 		let client = client::build_client(garage);
+		let custom_request = CustomRequester::new(garage);
 
-		Context { garage, client }
+		Context {
+			garage,
+			client,
+			custom_request,
+		}
 	}
 
 	/// Create an unique bucket with a random suffix.
diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs
index 9d7e4322..8799c395 100644
--- a/src/garage/tests/lib.rs
+++ b/src/garage/tests/lib.rs
@@ -7,4 +7,5 @@ mod list;
 mod multipart;
 mod objects;
 mod simple;
+mod streaming_signature;
 mod website;
diff --git a/src/garage/tests/multipart.rs b/src/garage/tests/multipart.rs
index 7fec4de6..895a2993 100644
--- a/src/garage/tests/multipart.rs
+++ b/src/garage/tests/multipart.rs
@@ -1,6 +1,6 @@
 use crate::common;
 use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
-use aws_sdk_s3::ByteStream;
+use aws_sdk_s3::types::ByteStream;
 
 const SZ_5MB: usize = 5 * 1024 * 1024;
 const SZ_10MB: usize = 10 * 1024 * 1024;
diff --git a/src/garage/tests/objects.rs b/src/garage/tests/objects.rs
index 9086073e..e1175b81 100644
--- a/src/garage/tests/objects.rs
+++ b/src/garage/tests/objects.rs
@@ -1,6 +1,6 @@
 use crate::common;
 use aws_sdk_s3::model::{Delete, ObjectIdentifier};
-use aws_sdk_s3::ByteStream;
+use aws_sdk_s3::types::ByteStream;
 
 const STD_KEY: &str = "hello world";
 const CTRL_KEY: &str = "\x00\x01\x02\x00";
diff --git a/src/garage/tests/simple.rs b/src/garage/tests/simple.rs
index a15cbf80..f54ae9ac 100644
--- a/src/garage/tests/simple.rs
+++ b/src/garage/tests/simple.rs
@@ -2,7 +2,7 @@ use crate::common;
 
 #[tokio::test]
 async fn test_simple() {
-	use aws_sdk_s3::ByteStream;
+	use aws_sdk_s3::types::ByteStream;
 
 	let ctx = common::context();
 	let bucket = ctx.create_bucket("test-simple");
diff --git a/src/garage/tests/streaming_signature.rs b/src/garage/tests/streaming_signature.rs
new file mode 100644
index 00000000..66f985ac
--- /dev/null
+++ b/src/garage/tests/streaming_signature.rs
@@ -0,0 +1,101 @@
+use std::collections::HashMap;
+
+use crate::common;
+use common::custom_requester::BodySignature;
+use hyper::Method;
+
+const STD_KEY: &str = "hello-world";
+//const CTRL_KEY: &str = "\x00\x01\x02\x00";
+const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+#[tokio::test]
+async fn test_putobject_streaming() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("putobject-streaming");
+
+	{
+		// Send an empty object (can serve as a directory marker)
+		// with a content type
+		let etag = "\"d41d8cd98f00b204e9800998ecf8427e\"";
+		let content_type = "text/csv";
+		let mut headers = HashMap::new();
+		headers.insert("content-type".to_owned(), content_type.to_owned());
+		let _ = ctx
+			.custom_request
+			.builder(bucket.clone())
+			.method(Method::PUT)
+			.path(STD_KEY.to_owned())
+			.unsigned_headers(headers)
+			.vhost_style(true)
+			.body(vec![])
+			.body_signature(BodySignature::Streaming(10))
+			.send()
+			.await
+			.unwrap();
+
+		// assert_eq!(r.e_tag.unwrap().as_str(), etag);
+		// We return a version ID here
+		// We should check if Amazon is returning one when versioning is not enabled
+		// assert!(r.version_id.is_some());
+
+		//let _version = r.version_id.unwrap();
+
+		let o = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			.key(STD_KEY)
+			.send()
+			.await
+			.unwrap();
+
+		assert_bytes_eq!(o.body, b"");
+		assert_eq!(o.e_tag.unwrap(), etag);
+		// We do not return version ID
+		// We should check if Amazon is returning one when versioning is not enabled
+		// assert_eq!(o.version_id.unwrap(), _version);
+		assert_eq!(o.content_type.unwrap(), content_type);
+		assert!(o.last_modified.is_some());
+		assert_eq!(o.content_length, 0);
+		assert_eq!(o.parts_count, 0);
+		assert_eq!(o.tag_count, 0);
+	}
+
+	{
+		let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
+
+		let _ = ctx
+			.custom_request
+			.builder(bucket.clone())
+			.method(Method::PUT)
+			//.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this
+			//fail
+			.path("abc".to_owned())
+			.vhost_style(true)
+			.body(BODY.to_vec())
+			.body_signature(BodySignature::Streaming(16))
+			.send()
+			.await
+			.unwrap();
+
+		// assert_eq!(r.e_tag.unwrap().as_str(), etag);
+		// assert!(r.version_id.is_some());
+
+		let o = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			//.key(CTRL_KEY)
+			.key("abc")
+			.send()
+			.await
+			.unwrap();
+
+		assert_bytes_eq!(o.body, BODY);
+		assert_eq!(o.e_tag.unwrap(), etag);
+		assert!(o.last_modified.is_some());
+		assert_eq!(o.content_length, 62);
+		assert_eq!(o.parts_count, 0);
+		assert_eq!(o.tag_count, 0);
+	}
+}
diff --git a/src/garage/tests/website.rs b/src/garage/tests/website.rs
index 34093a79..963d11ea 100644
--- a/src/garage/tests/website.rs
+++ b/src/garage/tests/website.rs
@@ -2,7 +2,7 @@ use crate::common;
 use crate::common::ext::*;
 use aws_sdk_s3::{
 	model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
-	ByteStream,
+	types::ByteStream,
 };
 use http::Request;
 use hyper::{