Merge pull request 'Garage v1.0' (#683) from next-0.10 into main

Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/683

5 files changed, 626 insertions, 36 deletions

diff --git a/src/garage/tests/common/ext/process.rs b/src/garage/tests/common/ext/process.rs
index ba533b6c..8e20bf7c 100644
--- a/src/garage/tests/common/ext/process.rs
+++ b/src/garage/tests/common/ext/process.rs
@@ -14,42 +14,20 @@ impl CommandExt for process::Command {
 	}
 
 	fn expect_success_status(&mut self, msg: &str) -> process::ExitStatus {
-		let status = self.status().expect(msg);
-		status.expect_success(msg);
-		status
+		self.expect_success_output(msg).status
 	}
 	fn expect_success_output(&mut self, msg: &str) -> process::Output {
 		let output = self.output().expect(msg);
-		output.expect_success(msg);
-		output
-	}
-}
-
-pub trait OutputExt {
-	fn expect_success(&self, msg: &str);
-}
-
-impl OutputExt for process::Output {
-	fn expect_success(&self, msg: &str) {
-		self.status.expect_success(msg)
-	}
-}
-
-pub trait ExitStatusExt {
-	fn expect_success(&self, msg: &str);
-}
-
-impl ExitStatusExt for process::ExitStatus {
-	fn expect_success(&self, msg: &str) {
-		if !self.success() {
-			match self.code() {
-				Some(code) => panic!(
-					"Command exited with code {code}: {msg}",
-					code = code,
-					msg = msg
-				),
-				None => panic!("Command exited with signal: {msg}", msg = msg),
-			}
+		if !output.status.success() {
+			panic!(
+				"{}: command {:?} exited with error {:?}\nSTDOUT: {}\nSTDERR: {}",
+				msg,
+				self,
+				output.status.code(),
+				String::from_utf8_lossy(&output.stdout),
+				String::from_utf8_lossy(&output.stderr)
+			);
 		}
+		output
 	}
 }
diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs
index 006337ee..db23d316 100644
--- a/src/garage/tests/common/garage.rs
+++ b/src/garage/tests/common/garage.rs
@@ -58,7 +58,7 @@ metadata_dir = "{path}/meta"
 data_dir = "{path}/data"
 db_engine = "{db_engine}"
 
-replication_mode = "1"
+replication_factor = 1
 
 rpc_bind_addr = "127.0.0.1:{rpc_port}"
 rpc_public_addr = "127.0.0.1:{rpc_port}"
@@ -100,7 +100,7 @@ api_bind_addr = "127.0.0.1:{admin_port}"
 			.arg("server")
 			.stdout(stdout)
 			.stderr(stderr)
-			.env("RUST_LOG", "garage=info,garage_api=trace")
+			.env("RUST_LOG", "garage=debug,garage_api=trace")
 			.spawn()
 			.expect("Could not start garage");
 
diff --git a/src/garage/tests/s3/mod.rs b/src/garage/tests/s3/mod.rs
index 4ebc4914..e75b1397 100644
--- a/src/garage/tests/s3/mod.rs
+++ b/src/garage/tests/s3/mod.rs
@@ -3,5 +3,6 @@ mod multipart;
 mod objects;
 mod presigned;
 mod simple;
+mod ssec;
 mod streaming_signature;
 mod website;
diff --git a/src/garage/tests/s3/multipart.rs b/src/garage/tests/s3/multipart.rs
index 51c9df74..cc424f59 100644
--- a/src/garage/tests/s3/multipart.rs
+++ b/src/garage/tests/s3/multipart.rs
@@ -1,6 +1,7 @@
 use crate::common;
 use aws_sdk_s3::primitives::ByteStream;
-use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+use aws_sdk_s3::types::{ChecksumAlgorithm, CompletedMultipartUpload, CompletedPart};
+use base64::prelude::*;
 
 const SZ_5MB: usize = 5 * 1024 * 1024;
 const SZ_10MB: usize = 10 * 1024 * 1024;
@@ -190,6 +191,153 @@ async fn test_multipart_upload() {
 }
 
 #[tokio::test]
+async fn test_multipart_with_checksum() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("testmpu-cksum");
+
+	let u1 = vec![0x11; SZ_5MB];
+	let u2 = vec![0x22; SZ_5MB];
+	let u3 = vec![0x33; SZ_5MB];
+
+	let ck1 = calculate_sha1(&u1);
+	let ck2 = calculate_sha1(&u2);
+	let ck3 = calculate_sha1(&u3);
+
+	let up = ctx
+		.client
+		.create_multipart_upload()
+		.bucket(&bucket)
+		.checksum_algorithm(ChecksumAlgorithm::Sha1)
+		.key("a")
+		.send()
+		.await
+		.unwrap();
+	assert!(up.upload_id.is_some());
+
+	let uid = up.upload_id.as_ref().unwrap();
+
+	let p1 = ctx
+		.client
+		.upload_part()
+		.bucket(&bucket)
+		.key("a")
+		.upload_id(uid)
+		.part_number(1)
+		.checksum_sha1(&ck1)
+		.body(ByteStream::from(u1.clone()))
+		.send()
+		.await
+		.unwrap();
+
+	// wrong checksum value should return an error
+	let err1 = ctx
+		.client
+		.upload_part()
+		.bucket(&bucket)
+		.key("a")
+		.upload_id(uid)
+		.part_number(2)
+		.checksum_sha1(&ck1)
+		.body(ByteStream::from(u2.clone()))
+		.send()
+		.await;
+	assert!(err1.is_err());
+
+	let p2 = ctx
+		.client
+		.upload_part()
+		.bucket(&bucket)
+		.key("a")
+		.upload_id(uid)
+		.part_number(2)
+		.checksum_sha1(&ck2)
+		.body(ByteStream::from(u2))
+		.send()
+		.await
+		.unwrap();
+
+	let p3 = ctx
+		.client
+		.upload_part()
+		.bucket(&bucket)
+		.key("a")
+		.upload_id(uid)
+		.part_number(3)
+		.checksum_sha1(&ck3)
+		.body(ByteStream::from(u3.clone()))
+		.send()
+		.await
+		.unwrap();
+
+	{
+		let r = ctx
+			.client
+			.list_parts()
+			.bucket(&bucket)
+			.key("a")
+			.upload_id(uid)
+			.send()
+			.await
+			.unwrap();
+		let parts = r.parts.unwrap();
+		assert_eq!(parts.len(), 3);
+		assert!(parts[0].checksum_crc32.is_none());
+		assert!(parts[0].checksum_crc32_c.is_none());
+		assert!(parts[0].checksum_sha256.is_none());
+		assert_eq!(parts[0].checksum_sha1.as_deref().unwrap(), ck1);
+		assert_eq!(parts[1].checksum_sha1.as_deref().unwrap(), ck2);
+		assert_eq!(parts[2].checksum_sha1.as_deref().unwrap(), ck3);
+	}
+
+	let cmp = CompletedMultipartUpload::builder()
+		.parts(
+			CompletedPart::builder()
+				.part_number(1)
+				.checksum_sha1(&ck1)
+				.e_tag(p1.e_tag.unwrap())
+				.build(),
+		)
+		.parts(
+			CompletedPart::builder()
+				.part_number(2)
+				.checksum_sha1(&ck2)
+				.e_tag(p2.e_tag.unwrap())
+				.build(),
+		)
+		.parts(
+			CompletedPart::builder()
+				.part_number(3)
+				.checksum_sha1(&ck3)
+				.e_tag(p3.e_tag.unwrap())
+				.build(),
+		)
+		.build();
+
+	let expected_checksum = calculate_sha1(
+		&vec![
+			BASE64_STANDARD.decode(&ck1).unwrap(),
+			BASE64_STANDARD.decode(&ck2).unwrap(),
+			BASE64_STANDARD.decode(&ck3).unwrap(),
+		]
+		.concat(),
+	);
+
+	let res = ctx
+		.client
+		.complete_multipart_upload()
+		.bucket(&bucket)
+		.key("a")
+		.upload_id(uid)
+		.checksum_sha1(expected_checksum.clone())
+		.multipart_upload(cmp)
+		.send()
+		.await
+		.unwrap();
+
+	assert_eq!(res.checksum_sha1, Some(expected_checksum));
+}
+
+#[tokio::test]
 async fn test_uploadlistpart() {
 	let ctx = common::context();
 	let bucket = ctx.create_bucket("uploadpart");
@@ -624,3 +772,11 @@ async fn test_uploadpartcopy() {
 	assert_eq!(real_obj.len(), exp_obj.len());
 	assert_eq!(real_obj, exp_obj);
 }
+
+fn calculate_sha1(bytes: &[u8]) -> String {
+	use sha1::{Digest, Sha1};
+
+	let mut hasher = Sha1::new();
+	hasher.update(bytes);
+	BASE64_STANDARD.encode(&hasher.finalize()[..])
+}
diff --git a/src/garage/tests/s3/ssec.rs b/src/garage/tests/s3/ssec.rs
new file mode 100644
index 00000000..d8f11950
--- /dev/null
+++ b/src/garage/tests/s3/ssec.rs
@@ -0,0 +1,455 @@
+use crate::common::{self, Context};
+use aws_sdk_s3::primitives::ByteStream;
+use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
+
+const SSEC_KEY: &str = "u8zCfnEyt5Imo/krN+sxA1DQXxLWtPJavU6T6gOVj1Y=";
+const SSEC_KEY_MD5: &str = "jMGbs3GyZkYjJUP6q5jA7g==";
+const SSEC_KEY2: &str = "XkYVk4Z3vVDO2yJaUqCAEZX6lL10voMxtV06d8my/eU=";
+const SSEC_KEY2_MD5: &str = "kedo2ab8J1MCjHwJuLTJHw==";
+
+const SZ_2MB: usize = 2 * 1024 * 1024;
+
+#[tokio::test]
+async fn test_ssec_object() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("sse-c");
+
+	let bytes1 = b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz".to_vec();
+	let bytes2 = (0..400000)
+		.map(|x| ((x * 3792) % 256) as u8)
+		.collect::<Vec<u8>>();
+
+	for data in vec![bytes1, bytes2] {
+		let stream = ByteStream::new(data.clone().into());
+
+		// Write encrypted object
+		let r = ctx
+			.client
+			.put_object()
+			.bucket(&bucket)
+			.key("testobj")
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY)
+			.sse_customer_key_md5(SSEC_KEY_MD5)
+			.body(stream)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(r.sse_customer_algorithm, Some("AES256".into()));
+		assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY_MD5.into()));
+
+		test_read_encrypted(
+			&ctx,
+			&bucket,
+			"testobj",
+			&data,
+			SSEC_KEY,
+			SSEC_KEY_MD5,
+			SSEC_KEY2,
+			SSEC_KEY2_MD5,
+		)
+		.await;
+
+		// Test copy from encrypted to non-encrypted
+		let r = ctx
+			.client
+			.copy_object()
+			.bucket(&bucket)
+			.key("test-copy-enc-dec")
+			.copy_source(format!("{}/{}", bucket, "testobj"))
+			.copy_source_sse_customer_algorithm("AES256")
+			.copy_source_sse_customer_key(SSEC_KEY)
+			.copy_source_sse_customer_key_md5(SSEC_KEY_MD5)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(r.sse_customer_algorithm, None);
+		assert_eq!(r.sse_customer_key_md5, None);
+
+		// Test read decrypted file
+		let r = ctx
+			.client
+			.get_object()
+			.bucket(&bucket)
+			.key("test-copy-enc-dec")
+			.send()
+			.await
+			.unwrap();
+		assert_bytes_eq!(r.body, &data);
+		assert_eq!(r.sse_customer_algorithm, None);
+		assert_eq!(r.sse_customer_key_md5, None);
+
+		// Test copy from non-encrypted to encrypted
+		let r = ctx
+			.client
+			.copy_object()
+			.bucket(&bucket)
+			.key("test-copy-enc-dec-enc")
+			.copy_source(format!("{}/test-copy-enc-dec", bucket))
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY2)
+			.sse_customer_key_md5(SSEC_KEY2_MD5)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(r.sse_customer_algorithm, Some("AES256".into()));
+		assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY2_MD5.into()));
+
+		test_read_encrypted(
+			&ctx,
+			&bucket,
+			"test-copy-enc-dec-enc",
+			&data,
+			SSEC_KEY2,
+			SSEC_KEY2_MD5,
+			SSEC_KEY,
+			SSEC_KEY_MD5,
+		)
+		.await;
+
+		// Test copy from encrypted to encrypted with different keys
+		let r = ctx
+			.client
+			.copy_object()
+			.bucket(&bucket)
+			.key("test-copy-enc-enc")
+			.copy_source(format!("{}/{}", bucket, "testobj"))
+			.copy_source_sse_customer_algorithm("AES256")
+			.copy_source_sse_customer_key(SSEC_KEY)
+			.copy_source_sse_customer_key_md5(SSEC_KEY_MD5)
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY2)
+			.sse_customer_key_md5(SSEC_KEY2_MD5)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(r.sse_customer_algorithm, Some("AES256".into()));
+		assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY2_MD5.into()));
+		test_read_encrypted(
+			&ctx,
+			&bucket,
+			"test-copy-enc-enc",
+			&data,
+			SSEC_KEY2,
+			SSEC_KEY2_MD5,
+			SSEC_KEY,
+			SSEC_KEY_MD5,
+		)
+		.await;
+
+		// Test copy from encrypted to encrypted with the same key
+		let r = ctx
+			.client
+			.copy_object()
+			.bucket(&bucket)
+			.key("test-copy-enc-enc-same")
+			.copy_source(format!("{}/{}", bucket, "testobj"))
+			.copy_source_sse_customer_algorithm("AES256")
+			.copy_source_sse_customer_key(SSEC_KEY)
+			.copy_source_sse_customer_key_md5(SSEC_KEY_MD5)
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY)
+			.sse_customer_key_md5(SSEC_KEY_MD5)
+			.send()
+			.await
+			.unwrap();
+		assert_eq!(r.sse_customer_algorithm, Some("AES256".into()));
+		assert_eq!(r.sse_customer_key_md5, Some(SSEC_KEY_MD5.into()));
+		test_read_encrypted(
+			&ctx,
+			&bucket,
+			"test-copy-enc-enc-same",
+			&data,
+			SSEC_KEY,
+			SSEC_KEY_MD5,
+			SSEC_KEY2,
+			SSEC_KEY2_MD5,
+		)
+		.await;
+	}
+}
+
+#[tokio::test]
+async fn test_multipart_upload() {
+	let ctx = common::context();
+	let bucket = ctx.create_bucket("test-ssec-mpu");
+
+	let u1 = vec![0x11; SZ_2MB];
+	let u2 = vec![0x22; SZ_2MB];
+	let u3 = vec![0x33; SZ_2MB];
+	let all = [&u1[..], &u2[..], &u3[..]].concat();
+
+	// Test simple encrypted mpu
+	{
+		let up = ctx
+			.client
+			.create_multipart_upload()
+			.bucket(&bucket)
+			.key("a")
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY)
+			.sse_customer_key_md5(SSEC_KEY_MD5)
+			.send()
+			.await
+			.unwrap();
+		assert!(up.upload_id.is_some());
+		assert_eq!(up.sse_customer_algorithm, Some("AES256".into()));
+		assert_eq!(up.sse_customer_key_md5, Some(SSEC_KEY_MD5.into()));
+
+		let uid = up.upload_id.as_ref().unwrap();
+
+		let mut etags = vec![];
+		for (i, part) in vec![&u1, &u2, &u3].into_iter().enumerate() {
+			let pu = ctx
+				.client
+				.upload_part()
+				.bucket(&bucket)
+				.key("a")
+				.upload_id(uid)
+				.part_number((i + 1) as i32)
+				.sse_customer_algorithm("AES256")
+				.sse_customer_key(SSEC_KEY)
+				.sse_customer_key_md5(SSEC_KEY_MD5)
+				.body(ByteStream::from(part.to_vec()))
+				.send()
+				.await
+				.unwrap();
+			etags.push(pu.e_tag.unwrap());
+		}
+
+		let mut cmp = CompletedMultipartUpload::builder();
+		for (i, etag) in etags.into_iter().enumerate() {
+			cmp = cmp.parts(
+				CompletedPart::builder()
+					.part_number((i + 1) as i32)
+					.e_tag(etag)
+					.build(),
+			);
+		}
+
+		ctx.client
+			.complete_multipart_upload()
+			.bucket(&bucket)
+			.key("a")
+			.upload_id(uid)
+			.multipart_upload(cmp.build())
+			.send()
+			.await
+			.unwrap();
+
+		test_read_encrypted(
+			&ctx,
+			&bucket,
+			"a",
+			&all,
+			SSEC_KEY,
+			SSEC_KEY_MD5,
+			SSEC_KEY2,
+			SSEC_KEY2_MD5,
+		)
+		.await;
+	}
+
+	// Test upload part copy from first object
+	{
+		// (setup) Upload a single part object
+		ctx.client
+			.put_object()
+			.bucket(&bucket)
+			.key("b")
+			.body(ByteStream::from(u1.clone()))
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY2)
+			.sse_customer_key_md5(SSEC_KEY2_MD5)
+			.send()
+			.await
+			.unwrap();
+
+		let up = ctx
+			.client
+			.create_multipart_upload()
+			.bucket(&bucket)
+			.key("target")
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY2)
+			.sse_customer_key_md5(SSEC_KEY2_MD5)
+			.send()
+			.await
+			.unwrap();
+		let uid = up.upload_id.as_ref().unwrap();
+
+		let p1 = ctx
+			.client
+			.upload_part()
+			.bucket(&bucket)
+			.key("target")
+			.upload_id(uid)
+			.part_number(1)
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY2)
+			.sse_customer_key_md5(SSEC_KEY2_MD5)
+			.body(ByteStream::from(u3.clone()))
+			.send()
+			.await
+			.unwrap();
+
+		let p2 = ctx
+			.client
+			.upload_part_copy()
+			.bucket(&bucket)
+			.key("target")
+			.upload_id(uid)
+			.part_number(2)
+			.copy_source(format!("{}/a", bucket))
+			.copy_source_range("bytes=500-550000")
+			.copy_source_sse_customer_algorithm("AES256")
+			.copy_source_sse_customer_key(SSEC_KEY)
+			.copy_source_sse_customer_key_md5(SSEC_KEY_MD5)
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY2)
+			.sse_customer_key_md5(SSEC_KEY2_MD5)
+			.send()
+			.await
+			.unwrap();
+
+		let p3 = ctx
+			.client
+			.upload_part()
+			.bucket(&bucket)
+			.key("target")
+			.upload_id(uid)
+			.part_number(3)
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY2)
+			.sse_customer_key_md5(SSEC_KEY2_MD5)
+			.body(ByteStream::from(u2.clone()))
+			.send()
+			.await
+			.unwrap();
+
+		let p4 = ctx
+			.client
+			.upload_part_copy()
+			.bucket(&bucket)
+			.key("target")
+			.upload_id(uid)
+			.part_number(4)
+			.copy_source(format!("{}/b", bucket))
+			.copy_source_range("bytes=1500-20500")
+			.copy_source_sse_customer_algorithm("AES256")
+			.copy_source_sse_customer_key(SSEC_KEY2)
+			.copy_source_sse_customer_key_md5(SSEC_KEY2_MD5)
+			.sse_customer_algorithm("AES256")
+			.sse_customer_key(SSEC_KEY2)
+			.sse_customer_key_md5(SSEC_KEY2_MD5)
+			.send()
+			.await
+			.unwrap();
+
+		let cmp = CompletedMultipartUpload::builder()
+			.parts(
+				CompletedPart::builder()
+					.part_number(1)
+					.e_tag(p1.e_tag.unwrap())
+					.build(),
+			)
+			.parts(
+				CompletedPart::builder()
+					.part_number(2)
+					.e_tag(p2.copy_part_result.unwrap().e_tag.unwrap())
+					.build(),
+			)
+			.parts(
+				CompletedPart::builder()
+					.part_number(3)
+					.e_tag(p3.e_tag.unwrap())
+					.build(),
+			)
+			.parts(
+				CompletedPart::builder()
+					.part_number(4)
+					.e_tag(p4.copy_part_result.unwrap().e_tag.unwrap())
+					.build(),
+			)
+			.build();
+
+		ctx.client
+			.complete_multipart_upload()
+			.bucket(&bucket)
+			.key("target")
+			.upload_id(uid)
+			.multipart_upload(cmp)
+			.send()
+			.await
+			.unwrap();
+
+		// (check) Get object
+		let expected = [&u3[..], &all[500..550001], &u2[..], &u1[1500..20501]].concat();
+		test_read_encrypted(
+			&ctx,
+			&bucket,
+			"target",
+			&expected,
+			SSEC_KEY2,
+			SSEC_KEY2_MD5,
+			SSEC_KEY,
+			SSEC_KEY_MD5,
+		)
+		.await;
+	}
+}
+
+async fn test_read_encrypted(
+	ctx: &Context,
+	bucket: &str,
+	obj_key: &str,
+	expected_data: &[u8],
+	enc_key: &str,
+	enc_key_md5: &str,
+	wrong_enc_key: &str,
+	wrong_enc_key_md5: &str,
+) {
+	// Test read encrypted without key
+	let o = ctx
+		.client
+		.get_object()
+		.bucket(bucket)
+		.key(obj_key)
+		.send()
+		.await;
+	assert!(
+		o.is_err(),
+		"encrypted file could be read without encryption key"
+	);
+
+	// Test read encrypted with wrong key
+	let o = ctx
+		.client
+		.get_object()
+		.bucket(bucket)
+		.key(obj_key)
+		.sse_customer_key(wrong_enc_key)
+		.sse_customer_key_md5(wrong_enc_key_md5)
+		.send()
+		.await;
+	assert!(
+		o.is_err(),
+		"encrypted file could be read with incorrect encryption key"
+	);
+
+	// Test read encrypted with correct key
+	let o = ctx
+		.client
+		.get_object()
+		.bucket(bucket)
+		.key(obj_key)
+		.sse_customer_algorithm("AES256")
+		.sse_customer_key(enc_key)
+		.sse_customer_key_md5(enc_key_md5)
+		.send()
+		.await
+		.unwrap();
+	assert_bytes_eq!(o.body, expected_data);
+	assert_eq!(o.sse_customer_algorithm, Some("AES256".into()));
+	assert_eq!(o.sse_customer_key_md5, Some(enc_key_md5.to_string()));
+}