aboutsummaryrefslogtreecommitdiff
path: root/src/garage/tests
diff options
context:
space:
mode:
Diffstat (limited to 'src/garage/tests')
-rw-r--r--src/garage/tests/common/custom_requester.rs280
-rw-r--r--src/garage/tests/common/garage.rs2
-rw-r--r--src/garage/tests/common/mod.rs11
-rw-r--r--src/garage/tests/lib.rs1
-rw-r--r--src/garage/tests/multipart.rs2
-rw-r--r--src/garage/tests/objects.rs2
-rw-r--r--src/garage/tests/simple.rs2
-rw-r--r--src/garage/tests/streaming_signature.rs101
-rw-r--r--src/garage/tests/website.rs2
9 files changed, 397 insertions, 6 deletions
diff --git a/src/garage/tests/common/custom_requester.rs b/src/garage/tests/common/custom_requester.rs
new file mode 100644
index 00000000..11dc10d6
--- /dev/null
+++ b/src/garage/tests/common/custom_requester.rs
@@ -0,0 +1,280 @@
+#![allow(dead_code)]
+
+use std::collections::HashMap;
+use std::convert::TryFrom;
+
+use chrono::{offset::Utc, DateTime};
+use hmac::{Hmac, Mac};
+use hyper::client::HttpConnector;
+use hyper::{Body, Client, Method, Request, Response, Uri};
+
+use super::garage::{Instance, Key};
+use garage_api::signature;
+
+/// You should ever only use this to send requests AWS sdk won't send,
+/// like to reproduce behavior of unusual implementations found to be
+/// problematic.
+pub struct CustomRequester {
+ key: Key,
+ uri: Uri,
+ client: Client<HttpConnector>,
+}
+
+impl CustomRequester {
+ pub fn new(instance: &Instance) -> Self {
+ /*
+ let credentials = Credentials::new(
+ &instance.key.id,
+ &instance.key.secret,
+ None,
+ None,
+ "garage-integ-test",
+ );
+ let endpoint = Endpoint::immutable(instance.uri());
+ */
+ CustomRequester {
+ key: instance.key.clone(),
+ uri: instance.uri(),
+ client: Client::new(),
+ }
+ }
+
+ pub fn builder(&self, bucket: String) -> RequestBuilder<'_> {
+ RequestBuilder {
+ requester: self,
+ bucket,
+ method: Method::GET,
+ path: String::new(),
+ query_params: HashMap::new(),
+ signed_headers: HashMap::new(),
+ unsigned_headers: HashMap::new(),
+ body: Vec::new(),
+ body_signature: BodySignature::Classic,
+ vhost_style: false,
+ }
+ }
+ /*
+ pub async fn request(&self, method: &str, path: String, headers: &HashMap<String, (bool, String)>, body: &[u8], vhost_style: bool) -> hyper::Result<Response<Body>> {
+ let request = Request::builder()
+ .method(
+ self.client.request(todo!()).await
+ }
+ */
+}
+
+pub struct RequestBuilder<'a> {
+ requester: &'a CustomRequester,
+ bucket: String,
+ method: Method,
+ path: String,
+ query_params: HashMap<String, Option<String>>,
+ signed_headers: HashMap<String, String>,
+ unsigned_headers: HashMap<String, String>,
+ body: Vec<u8>,
+ body_signature: BodySignature,
+ vhost_style: bool,
+}
+
+impl<'a> RequestBuilder<'a> {
+ pub fn method(&mut self, method: Method) -> &mut Self {
+ self.method = method;
+ self
+ }
+
+ pub fn path(&mut self, path: String) -> &mut Self {
+ self.path = path;
+ self
+ }
+
+ pub fn query_params(&mut self, query_params: HashMap<String, Option<String>>) -> &mut Self {
+ self.query_params = query_params;
+ self
+ }
+
+ pub fn signed_headers(&mut self, signed_headers: HashMap<String, String>) -> &mut Self {
+ self.signed_headers = signed_headers;
+ self
+ }
+
+ pub fn unsigned_headers(&mut self, unsigned_headers: HashMap<String, String>) -> &mut Self {
+ self.unsigned_headers = unsigned_headers;
+ self
+ }
+
+ pub fn body(&mut self, body: Vec<u8>) -> &mut Self {
+ self.body = body;
+ self
+ }
+
+ pub fn body_signature(&mut self, body_signature: BodySignature) -> &mut Self {
+ self.body_signature = body_signature;
+ self
+ }
+
+ pub fn vhost_style(&mut self, vhost_style: bool) -> &mut Self {
+ self.vhost_style = vhost_style;
+ self
+ }
+
+ pub async fn send(&mut self) -> hyper::Result<Response<Body>> {
+ // TODO this is a bit incorrect in that path and query params should be url-encoded and
+ // aren't, but this is good enought for now.
+
+ let query = query_param_to_string(&self.query_params);
+ let (host, path) = if self.vhost_style {
+ (
+ format!("{}.s3.garage", self.bucket),
+ format!("{}{}", self.path, query),
+ )
+ } else {
+ (
+ "s3.garage".to_owned(),
+ format!("{}/{}{}", self.bucket, self.path, query),
+ )
+ };
+ let uri = format!("{}{}", self.requester.uri, path);
+
+ let now = Utc::now();
+ let scope = signature::compute_scope(&now, super::REGION.as_ref());
+ let mut signer = signature::signing_hmac(
+ &now,
+ &self.requester.key.secret,
+ super::REGION.as_ref(),
+ "s3",
+ )
+ .unwrap();
+ let streaming_signer = signer.clone();
+
+ let mut all_headers = self.signed_headers.clone();
+
+ let date = now.format(signature::LONG_DATETIME).to_string();
+ all_headers.insert("x-amz-date".to_owned(), date);
+ all_headers.insert("host".to_owned(), host);
+
+ let body_sha = match self.body_signature {
+ BodySignature::Unsigned => "UNSIGNED-PAYLOAD".to_owned(),
+ BodySignature::Classic => hex::encode(garage_util::data::sha256sum(&self.body)),
+ BodySignature::Streaming(size) => {
+ all_headers.insert("content-encoding".to_owned(), "aws-chunked".to_owned());
+ all_headers.insert(
+ "x-amz-decoded-content-length".to_owned(),
+ self.body.len().to_string(),
+ );
+ // this is a pretty lazy and inefficient way to do it, but it's enought for
+ // test code.
+ all_headers.insert(
+ "content-length".to_owned(),
+ to_streaming_body(&self.body, size, String::new(), signer.clone(), now, "")
+ .len()
+ .to_string(),
+ );
+
+ "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".to_owned()
+ }
+ };
+ all_headers.insert("x-amz-content-sha256".to_owned(), body_sha.clone());
+
+ let mut signed_headers = all_headers
+ .iter()
+ .map(|(k, _)| k.as_ref())
+ .collect::<Vec<&str>>();
+ signed_headers.sort();
+ let signed_headers = signed_headers.join(";");
+
+ all_headers.extend(self.unsigned_headers.clone());
+
+ let canonical_request = signature::payload::canonical_request(
+ &self.method,
+ &Uri::try_from(&uri).unwrap(),
+ &all_headers,
+ &signed_headers,
+ &body_sha,
+ );
+
+ let string_to_sign = signature::payload::string_to_sign(&now, &scope, &canonical_request);
+
+ signer.update(string_to_sign.as_bytes());
+ let signature = hex::encode(signer.finalize().into_bytes());
+ let authorization = format!(
+ "AWS4-HMAC-SHA256 Credential={}/{},SignedHeaders={},Signature={}",
+ self.requester.key.id, scope, signed_headers, signature
+ );
+ all_headers.insert("authorization".to_owned(), authorization);
+
+ let mut request = Request::builder();
+ for (k, v) in all_headers {
+ request = request.header(k, v);
+ }
+
+ let body = if let BodySignature::Streaming(size) = self.body_signature {
+ to_streaming_body(&self.body, size, signature, streaming_signer, now, &scope)
+ } else {
+ self.body.clone()
+ };
+ let request = request
+ .uri(uri)
+ .method(self.method.clone())
+ .body(Body::from(body))
+ .unwrap();
+ self.requester.client.request(request).await
+ }
+}
+
+pub enum BodySignature {
+ Unsigned,
+ Classic,
+ Streaming(usize),
+}
+
+fn query_param_to_string(params: &HashMap<String, Option<String>>) -> String {
+ if params.is_empty() {
+ return String::new();
+ }
+
+ "?".to_owned()
+ + &params
+ .iter()
+ .map(|(k, v)| {
+ if let Some(v) = v {
+ format!("{}={}", k, v)
+ } else {
+ k.clone()
+ }
+ })
+ .collect::<Vec<String>>()
+ .join("&")
+}
+
+fn to_streaming_body(
+ body: &[u8],
+ chunk_size: usize,
+ mut seed: String,
+ hasher: Hmac<sha2::Sha256>,
+ now: DateTime<Utc>,
+ scope: &str,
+) -> Vec<u8> {
+ const SHA_NULL: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+ let now = now.format(signature::LONG_DATETIME).to_string();
+ let mut res = Vec::with_capacity(body.len());
+ for chunk in body.chunks(chunk_size).chain(std::iter::once(&[][..])) {
+ let to_sign = format!(
+ "AWS4-HMAC-SHA256-PAYLOAD\n{}\n{}\n{}\n{}\n{}",
+ now,
+ scope,
+ seed,
+ SHA_NULL,
+ hex::encode(garage_util::data::sha256sum(chunk))
+ );
+
+ let mut hasher = hasher.clone();
+ hasher.update(to_sign.as_bytes());
+ seed = hex::encode(hasher.finalize().into_bytes());
+
+ let header = format!("{:x};chunk-signature={}\r\n", chunk.len(), seed);
+ res.extend_from_slice(header.as_bytes());
+ res.extend_from_slice(chunk);
+ res.extend_from_slice(b"\r\n");
+ }
+
+ res
+}
diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs
index 36adb55e..88c51501 100644
--- a/src/garage/tests/common/garage.rs
+++ b/src/garage/tests/common/garage.rs
@@ -11,7 +11,7 @@ pub const DEFAULT_PORT: u16 = 49995;
static GARAGE_TEST_SECRET: &str =
"c3ea8cb80333d04e208d136698b1a01ae370d463f0d435ab2177510b3478bf44";
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
pub struct Key {
pub name: String,
pub id: String,
diff --git a/src/garage/tests/common/mod.rs b/src/garage/tests/common/mod.rs
index 32fa3848..8f88c731 100644
--- a/src/garage/tests/common/mod.rs
+++ b/src/garage/tests/common/mod.rs
@@ -5,22 +5,31 @@ use ext::*;
pub mod macros;
pub mod client;
+pub mod custom_requester;
pub mod ext;
pub mod garage;
+use custom_requester::CustomRequester;
+
const REGION: Region = Region::from_static("garage-integ-test");
pub struct Context {
pub garage: &'static garage::Instance,
pub client: Client,
+ pub custom_request: CustomRequester,
}
impl Context {
fn new() -> Self {
let garage = garage::instance();
let client = client::build_client(garage);
+ let custom_request = CustomRequester::new(garage);
- Context { garage, client }
+ Context {
+ garage,
+ client,
+ custom_request,
+ }
}
/// Create an unique bucket with a random suffix.
diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs
index 9d7e4322..8799c395 100644
--- a/src/garage/tests/lib.rs
+++ b/src/garage/tests/lib.rs
@@ -7,4 +7,5 @@ mod list;
mod multipart;
mod objects;
mod simple;
+mod streaming_signature;
mod website;
diff --git a/src/garage/tests/multipart.rs b/src/garage/tests/multipart.rs
index 7fec4de6..895a2993 100644
--- a/src/garage/tests/multipart.rs
+++ b/src/garage/tests/multipart.rs
@@ -1,6 +1,6 @@
use crate::common;
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart};
-use aws_sdk_s3::ByteStream;
+use aws_sdk_s3::types::ByteStream;
const SZ_5MB: usize = 5 * 1024 * 1024;
const SZ_10MB: usize = 10 * 1024 * 1024;
diff --git a/src/garage/tests/objects.rs b/src/garage/tests/objects.rs
index 9086073e..e1175b81 100644
--- a/src/garage/tests/objects.rs
+++ b/src/garage/tests/objects.rs
@@ -1,6 +1,6 @@
use crate::common;
use aws_sdk_s3::model::{Delete, ObjectIdentifier};
-use aws_sdk_s3::ByteStream;
+use aws_sdk_s3::types::ByteStream;
const STD_KEY: &str = "hello world";
const CTRL_KEY: &str = "\x00\x01\x02\x00";
diff --git a/src/garage/tests/simple.rs b/src/garage/tests/simple.rs
index a15cbf80..f54ae9ac 100644
--- a/src/garage/tests/simple.rs
+++ b/src/garage/tests/simple.rs
@@ -2,7 +2,7 @@ use crate::common;
#[tokio::test]
async fn test_simple() {
- use aws_sdk_s3::ByteStream;
+ use aws_sdk_s3::types::ByteStream;
let ctx = common::context();
let bucket = ctx.create_bucket("test-simple");
diff --git a/src/garage/tests/streaming_signature.rs b/src/garage/tests/streaming_signature.rs
new file mode 100644
index 00000000..66f985ac
--- /dev/null
+++ b/src/garage/tests/streaming_signature.rs
@@ -0,0 +1,101 @@
+use std::collections::HashMap;
+
+use crate::common;
+use common::custom_requester::BodySignature;
+use hyper::Method;
+
+const STD_KEY: &str = "hello-world";
+//const CTRL_KEY: &str = "\x00\x01\x02\x00";
+const BODY: &[u8; 62] = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz";
+
+#[tokio::test]
+async fn test_putobject_streaming() {
+ let ctx = common::context();
+ let bucket = ctx.create_bucket("putobject-streaming");
+
+ {
+ // Send an empty object (can serve as a directory marker)
+ // with a content type
+ let etag = "\"d41d8cd98f00b204e9800998ecf8427e\"";
+ let content_type = "text/csv";
+ let mut headers = HashMap::new();
+ headers.insert("content-type".to_owned(), content_type.to_owned());
+ let _ = ctx
+ .custom_request
+ .builder(bucket.clone())
+ .method(Method::PUT)
+ .path(STD_KEY.to_owned())
+ .unsigned_headers(headers)
+ .vhost_style(true)
+ .body(vec![])
+ .body_signature(BodySignature::Streaming(10))
+ .send()
+ .await
+ .unwrap();
+
+ // assert_eq!(r.e_tag.unwrap().as_str(), etag);
+ // We return a version ID here
+ // We should check if Amazon is returning one when versioning is not enabled
+ // assert!(r.version_id.is_some());
+
+ //let _version = r.version_id.unwrap();
+
+ let o = ctx
+ .client
+ .get_object()
+ .bucket(&bucket)
+ .key(STD_KEY)
+ .send()
+ .await
+ .unwrap();
+
+ assert_bytes_eq!(o.body, b"");
+ assert_eq!(o.e_tag.unwrap(), etag);
+ // We do not return version ID
+ // We should check if Amazon is returning one when versioning is not enabled
+ // assert_eq!(o.version_id.unwrap(), _version);
+ assert_eq!(o.content_type.unwrap(), content_type);
+ assert!(o.last_modified.is_some());
+ assert_eq!(o.content_length, 0);
+ assert_eq!(o.parts_count, 0);
+ assert_eq!(o.tag_count, 0);
+ }
+
+ {
+ let etag = "\"46cf18a9b447991b450cad3facf5937e\"";
+
+ let _ = ctx
+ .custom_request
+ .builder(bucket.clone())
+ .method(Method::PUT)
+ //.path(CTRL_KEY.to_owned()) at the moment custom_request does not encode url so this
+ //fail
+ .path("abc".to_owned())
+ .vhost_style(true)
+ .body(BODY.to_vec())
+ .body_signature(BodySignature::Streaming(16))
+ .send()
+ .await
+ .unwrap();
+
+ // assert_eq!(r.e_tag.unwrap().as_str(), etag);
+ // assert!(r.version_id.is_some());
+
+ let o = ctx
+ .client
+ .get_object()
+ .bucket(&bucket)
+ //.key(CTRL_KEY)
+ .key("abc")
+ .send()
+ .await
+ .unwrap();
+
+ assert_bytes_eq!(o.body, BODY);
+ assert_eq!(o.e_tag.unwrap(), etag);
+ assert!(o.last_modified.is_some());
+ assert_eq!(o.content_length, 62);
+ assert_eq!(o.parts_count, 0);
+ assert_eq!(o.tag_count, 0);
+ }
+}
diff --git a/src/garage/tests/website.rs b/src/garage/tests/website.rs
index 34093a79..963d11ea 100644
--- a/src/garage/tests/website.rs
+++ b/src/garage/tests/website.rs
@@ -2,7 +2,7 @@ use crate::common;
use crate::common::ext::*;
use aws_sdk_s3::{
model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
- ByteStream,
+ types::ByteStream,
};
use http::Request;
use hyper::{
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-01 03:33:25 +0000