api: refactor: move checksum algorithms to common

14 files changed, 215 insertions, 171 deletions

diff --git a/Cargo.lock b/Cargo.lock
index ad5d098d..26f6ea1d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1301,8 +1301,11 @@ dependencies = [
 name = "garage_api_common"
 version = "1.0.1"
 dependencies = [
+ "base64 0.21.7",
  "bytes",
  "chrono",
+ "crc32c",
+ "crc32fast",
  "crypto-common",
  "err-derive",
  "futures",
@@ -1316,11 +1319,13 @@ dependencies = [
  "hyper 1.6.0",
  "hyper-util",
  "idna 0.5.0",
+ "md-5",
  "nom",
  "opentelemetry",
  "pin-project",
  "serde",
  "serde_json",
+ "sha1",
  "sha2",
  "tokio",
  "tracing",
diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml
index 5b9cf479..c33d585d 100644
--- a/src/api/common/Cargo.toml
+++ b/src/api/common/Cargo.toml
@@ -18,16 +18,21 @@ garage_model.workspace = true
 garage_table.workspace = true
 garage_util.workspace = true
 
+base64.workspace = true
 bytes.workspace = true
 chrono.workspace = true
+crc32fast.workspace = true
+crc32c.workspace = true
 crypto-common.workspace = true
 err-derive.workspace = true
 hex.workspace = true
 hmac.workspace = true
+md-5.workspace = true
 idna.workspace = true
 tracing.workspace = true
 nom.workspace = true
 pin-project.workspace = true
+sha1.workspace = true
 sha2.workspace = true
 
 futures.workspace = true
diff --git a/src/api/common/signature/checksum.rs b/src/api/common/signature/checksum.rs
new file mode 100644
index 00000000..c6beb33f
--- /dev/null
+++ b/src/api/common/signature/checksum.rs
@@ -0,0 +1,181 @@
+use std::convert::{TryFrom, TryInto};
+use std::hash::Hasher;
+
+use base64::prelude::*;
+use crc32c::Crc32cHasher as Crc32c;
+use crc32fast::Hasher as Crc32;
+use md5::{Digest, Md5};
+use sha1::Sha1;
+use sha2::Sha256;
+
+use http::HeaderName;
+
+use garage_util::data::*;
+
+use garage_model::s3::object_table::*;
+
+use super::error::*;
+
+pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName =
+	HeaderName::from_static("x-amz-checksum-algorithm");
+pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode");
+pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32");
+pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c");
+pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1");
+pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256");
+
+pub type Crc32Checksum = [u8; 4];
+pub type Crc32cChecksum = [u8; 4];
+pub type Md5Checksum = [u8; 16];
+pub type Sha1Checksum = [u8; 20];
+pub type Sha256Checksum = [u8; 32];
+
+#[derive(Debug, Default)]
+pub struct ExpectedChecksums {
+	// base64-encoded md5 (content-md5 header)
+	pub md5: Option<String>,
+	// content_sha256 (as a Hash / FixedBytes32)
+	pub sha256: Option<Hash>,
+	// extra x-amz-checksum-* header
+	pub extra: Option<ChecksumValue>,
+}
+
+pub struct Checksummer {
+	pub crc32: Option<Crc32>,
+	pub crc32c: Option<Crc32c>,
+	pub md5: Option<Md5>,
+	pub sha1: Option<Sha1>,
+	pub sha256: Option<Sha256>,
+}
+
+#[derive(Default)]
+pub struct Checksums {
+	pub crc32: Option<Crc32Checksum>,
+	pub crc32c: Option<Crc32cChecksum>,
+	pub md5: Option<Md5Checksum>,
+	pub sha1: Option<Sha1Checksum>,
+	pub sha256: Option<Sha256Checksum>,
+}
+
+impl Checksummer {
+	pub fn init(expected: &ExpectedChecksums, require_md5: bool) -> Self {
+		let mut ret = Self {
+			crc32: None,
+			crc32c: None,
+			md5: None,
+			sha1: None,
+			sha256: None,
+		};
+
+		if expected.md5.is_some() || require_md5 {
+			ret.md5 = Some(Md5::new());
+		}
+		if expected.sha256.is_some() || matches!(&expected.extra, Some(ChecksumValue::Sha256(_))) {
+			ret.sha256 = Some(Sha256::new());
+		}
+		if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) {
+			ret.crc32 = Some(Crc32::new());
+		}
+		if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) {
+			ret.crc32c = Some(Crc32c::default());
+		}
+		if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) {
+			ret.sha1 = Some(Sha1::new());
+		}
+		ret
+	}
+
+	pub fn add(mut self, algo: Option<ChecksumAlgorithm>) -> Self {
+		match algo {
+			Some(ChecksumAlgorithm::Crc32) => {
+				self.crc32 = Some(Crc32::new());
+			}
+			Some(ChecksumAlgorithm::Crc32c) => {
+				self.crc32c = Some(Crc32c::default());
+			}
+			Some(ChecksumAlgorithm::Sha1) => {
+				self.sha1 = Some(Sha1::new());
+			}
+			Some(ChecksumAlgorithm::Sha256) => {
+				self.sha256 = Some(Sha256::new());
+			}
+			None => (),
+		}
+		self
+	}
+
+	pub fn update(&mut self, bytes: &[u8]) {
+		if let Some(crc32) = &mut self.crc32 {
+			crc32.update(bytes);
+		}
+		if let Some(crc32c) = &mut self.crc32c {
+			crc32c.write(bytes);
+		}
+		if let Some(md5) = &mut self.md5 {
+			md5.update(bytes);
+		}
+		if let Some(sha1) = &mut self.sha1 {
+			sha1.update(bytes);
+		}
+		if let Some(sha256) = &mut self.sha256 {
+			sha256.update(bytes);
+		}
+	}
+
+	pub fn finalize(self) -> Checksums {
+		Checksums {
+			crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize())),
+			crc32c: self
+				.crc32c
+				.map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())),
+			md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()),
+			sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()),
+			sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()),
+		}
+	}
+}
+
+impl Checksums {
+	pub fn verify(&self, expected: &ExpectedChecksums) -> Result<(), Error> {
+		if let Some(expected_md5) = &expected.md5 {
+			match self.md5 {
+				Some(md5) if BASE64_STANDARD.encode(&md5) == expected_md5.trim_matches('"') => (),
+				_ => {
+					return Err(Error::InvalidDigest(
+						"MD5 checksum verification failed (from content-md5)".into(),
+					))
+				}
+			}
+		}
+		if let Some(expected_sha256) = &expected.sha256 {
+			match self.sha256 {
+				Some(sha256) if &sha256[..] == expected_sha256.as_slice() => (),
+				_ => {
+					return Err(Error::InvalidDigest(
+						"SHA256 checksum verification failed (from x-amz-content-sha256)".into(),
+					))
+				}
+			}
+		}
+		if let Some(extra) = expected.extra {
+			let algo = extra.algorithm();
+			if self.extract(Some(algo)) != Some(extra) {
+				return Err(Error::InvalidDigest(format!(
+					"Failed to validate checksum for algorithm {:?}",
+					algo
+				)));
+			}
+		}
+		Ok(())
+	}
+
+	pub fn extract(&self, algo: Option<ChecksumAlgorithm>) -> Option<ChecksumValue> {
+		match algo {
+			None => None,
+			Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())),
+			Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())),
+			Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())),
+			Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())),
+		}
+	}
+}
diff --git a/src/api/common/signature/error.rs b/src/api/common/signature/error.rs
index 2d92a072..b2f396b5 100644
--- a/src/api/common/signature/error.rs
+++ b/src/api/common/signature/error.rs
@@ -18,6 +18,10 @@ pub enum Error {
 	/// The request contained an invalid UTF-8 sequence in its path or in other parameters
 	#[error(display = "Invalid UTF-8: {}", _0)]
 	InvalidUtf8Str(#[error(source)] std::str::Utf8Error),
+
+	/// The provided digest (checksum) value was invalid
+	#[error(display = "Invalid digest: {}", _0)]
+	InvalidDigest(String),
 }
 
 impl<T> From<T> for Error
diff --git a/src/api/common/signature/mod.rs b/src/api/common/signature/mod.rs
index 27082168..08b0aa7e 100644
--- a/src/api/common/signature/mod.rs
+++ b/src/api/common/signature/mod.rs
@@ -11,6 +11,7 @@ use garage_util::data::{sha256sum, Hash};
 
 use error::*;
 
+pub mod checksum;
 pub mod error;
 pub mod payload;
 pub mod streaming;
diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs
index 3cd0e6f7..b7ca5aa4 100644
--- a/src/api/k2v/error.rs
+++ b/src/api/k2v/error.rs
@@ -23,6 +23,10 @@ pub enum Error {
 	#[error(display = "Authorization header malformed, unexpected scope: {}", _0)]
 	AuthorizationHeaderMalformed(String),
 
+	/// The provided digest (checksum) value was invalid
+	#[error(display = "Invalid digest: {}", _0)]
+	InvalidDigest(String),
+
 	/// The object requested don't exists
 	#[error(display = "Key not found")]
 	NoSuchKey,
@@ -54,6 +58,7 @@ impl From<SignatureError> for Error {
 				Self::AuthorizationHeaderMalformed(c)
 			}
 			SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i),
+			SignatureError::InvalidDigest(d) => Self::InvalidDigest(d),
 		}
 	}
 }
@@ -71,6 +76,7 @@ impl Error {
 			Error::InvalidBase64(_) => "InvalidBase64",
 			Error::InvalidUtf8Str(_) => "InvalidUtf8String",
 			Error::InvalidCausalityToken => "CausalityToken",
+            Error::InvalidDigest(_) => "InvalidDigest",
 		}
 	}
 }
@@ -85,6 +91,7 @@ impl ApiError for Error {
 			Error::AuthorizationHeaderMalformed(_)
 			| Error::InvalidBase64(_)
 			| Error::InvalidUtf8Str(_)
+            | Error::InvalidDigest(_)
 			| Error::InvalidCausalityToken => StatusCode::BAD_REQUEST,
 		}
 	}
diff --git a/src/api/s3/checksum.rs b/src/api/s3/checksum.rs
index 02fb55ec..a720a82f 100644
--- a/src/api/s3/checksum.rs
+++ b/src/api/s3/checksum.rs
@@ -8,180 +8,15 @@ use md5::{Digest, Md5};
 use sha1::Sha1;
 use sha2::Sha256;
 
-use http::{HeaderMap, HeaderName, HeaderValue};
+use http::{HeaderMap, HeaderValue};
 
-use garage_util::data::*;
 use garage_util::error::OkOrMessage;
 
 use garage_model::s3::object_table::*;
 
-use crate::error::*;
-
-pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName =
-	HeaderName::from_static("x-amz-checksum-algorithm");
-pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode");
-pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32");
-pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c");
-pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1");
-pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256");
-
-pub type Crc32Checksum = [u8; 4];
-pub type Crc32cChecksum = [u8; 4];
-pub type Md5Checksum = [u8; 16];
-pub type Sha1Checksum = [u8; 20];
-pub type Sha256Checksum = [u8; 32];
-
-#[derive(Debug, Default)]
-pub(crate) struct ExpectedChecksums {
-	// base64-encoded md5 (content-md5 header)
-	pub md5: Option<String>,
-	// content_sha256 (as a Hash / FixedBytes32)
-	pub sha256: Option<Hash>,
-	// extra x-amz-checksum-* header
-	pub extra: Option<ChecksumValue>,
-}
-
-pub(crate) struct Checksummer {
-	pub crc32: Option<Crc32>,
-	pub crc32c: Option<Crc32c>,
-	pub md5: Option<Md5>,
-	pub sha1: Option<Sha1>,
-	pub sha256: Option<Sha256>,
-}
-
-#[derive(Default)]
-pub(crate) struct Checksums {
-	pub crc32: Option<Crc32Checksum>,
-	pub crc32c: Option<Crc32cChecksum>,
-	pub md5: Option<Md5Checksum>,
-	pub sha1: Option<Sha1Checksum>,
-	pub sha256: Option<Sha256Checksum>,
-}
-
-impl Checksummer {
-	pub(crate) fn init(expected: &ExpectedChecksums, require_md5: bool) -> Self {
-		let mut ret = Self {
-			crc32: None,
-			crc32c: None,
-			md5: None,
-			sha1: None,
-			sha256: None,
-		};
-
-		if expected.md5.is_some() || require_md5 {
-			ret.md5 = Some(Md5::new());
-		}
-		if expected.sha256.is_some() || matches!(&expected.extra, Some(ChecksumValue::Sha256(_))) {
-			ret.sha256 = Some(Sha256::new());
-		}
-		if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) {
-			ret.crc32 = Some(Crc32::new());
-		}
-		if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) {
-			ret.crc32c = Some(Crc32c::default());
-		}
-		if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) {
-			ret.sha1 = Some(Sha1::new());
-		}
-		ret
-	}
-
-	pub(crate) fn add(mut self, algo: Option<ChecksumAlgorithm>) -> Self {
-		match algo {
-			Some(ChecksumAlgorithm::Crc32) => {
-				self.crc32 = Some(Crc32::new());
-			}
-			Some(ChecksumAlgorithm::Crc32c) => {
-				self.crc32c = Some(Crc32c::default());
-			}
-			Some(ChecksumAlgorithm::Sha1) => {
-				self.sha1 = Some(Sha1::new());
-			}
-			Some(ChecksumAlgorithm::Sha256) => {
-				self.sha256 = Some(Sha256::new());
-			}
-			None => (),
-		}
-		self
-	}
+use garage_api_common::signature::checksum::*;
 
-	pub(crate) fn update(&mut self, bytes: &[u8]) {
-		if let Some(crc32) = &mut self.crc32 {
-			crc32.update(bytes);
-		}
-		if let Some(crc32c) = &mut self.crc32c {
-			crc32c.write(bytes);
-		}
-		if let Some(md5) = &mut self.md5 {
-			md5.update(bytes);
-		}
-		if let Some(sha1) = &mut self.sha1 {
-			sha1.update(bytes);
-		}
-		if let Some(sha256) = &mut self.sha256 {
-			sha256.update(bytes);
-		}
-	}
-
-	pub(crate) fn finalize(self) -> Checksums {
-		Checksums {
-			crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize())),
-			crc32c: self
-				.crc32c
-				.map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())),
-			md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()),
-			sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()),
-			sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()),
-		}
-	}
-}
-
-impl Checksums {
-	pub fn verify(&self, expected: &ExpectedChecksums) -> Result<(), Error> {
-		if let Some(expected_md5) = &expected.md5 {
-			match self.md5 {
-				Some(md5) if BASE64_STANDARD.encode(&md5) == expected_md5.trim_matches('"') => (),
-				_ => {
-					return Err(Error::InvalidDigest(
-						"MD5 checksum verification failed (from content-md5)".into(),
-					))
-				}
-			}
-		}
-		if let Some(expected_sha256) = &expected.sha256 {
-			match self.sha256 {
-				Some(sha256) if &sha256[..] == expected_sha256.as_slice() => (),
-				_ => {
-					return Err(Error::InvalidDigest(
-						"SHA256 checksum verification failed (from x-amz-content-sha256)".into(),
-					))
-				}
-			}
-		}
-		if let Some(extra) = expected.extra {
-			let algo = extra.algorithm();
-			if self.extract(Some(algo)) != Some(extra) {
-				return Err(Error::InvalidDigest(format!(
-					"Failed to validate checksum for algorithm {:?}",
-					algo
-				)));
-			}
-		}
-		Ok(())
-	}
-
-	pub fn extract(&self, algo: Option<ChecksumAlgorithm>) -> Option<ChecksumValue> {
-		match algo {
-			None => None,
-			Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())),
-			Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())),
-			Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())),
-			Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())),
-		}
-	}
-}
-
-// ----
+use crate::error::*;
 
 #[derive(Default)]
 pub(crate) struct MultipartChecksummer {
diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs
index 07d50ea5..4bf68406 100644
--- a/src/api/s3/copy.rs
+++ b/src/api/s3/copy.rs
@@ -21,6 +21,7 @@ use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 
 use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::*;
 
 use crate::api_server::{ReqBody, ResBody};
 use crate::checksum::*;
diff --git a/src/api/s3/encryption.rs b/src/api/s3/encryption.rs
index b38d7792..fa7285ca 100644
--- a/src/api/s3/encryption.rs
+++ b/src/api/s3/encryption.rs
@@ -29,8 +29,8 @@ use garage_model::garage::Garage;
 use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner};
 
 use garage_api_common::common_error::*;
+use garage_api_common::signature::checksum::Md5Checksum;
 
-use crate::checksum::Md5Checksum;
 use crate::error::Error;
 
 const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName =
diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs
index 1bb8909c..6d4b7a11 100644
--- a/src/api/s3/error.rs
+++ b/src/api/s3/error.rs
@@ -80,7 +80,7 @@ pub enum Error {
 	#[error(display = "Invalid encryption algorithm: {:?}, should be AES256", _0)]
 	InvalidEncryptionAlgorithm(String),
 
-	/// The client sent invalid XML data
+	/// The provided digest (checksum) value was invalid
 	#[error(display = "Invalid digest: {}", _0)]
 	InvalidDigest(String),
 
@@ -119,6 +119,7 @@ impl From<SignatureError> for Error {
 				Self::AuthorizationHeaderMalformed(c)
 			}
 			SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i),
+			SignatureError::InvalidDigest(d) => Self::InvalidDigest(d),
 		}
 	}
 }
diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs
index c2393a51..6627cf4a 100644
--- a/src/api/s3/get.rs
+++ b/src/api/s3/get.rs
@@ -26,9 +26,10 @@ use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 
 use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::X_AMZ_CHECKSUM_MODE;
 
 use crate::api_server::ResBody;
-use crate::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE};
+use crate::checksum::add_checksum_response_headers;
 use crate::encryption::EncryptionParams;
 use crate::error::*;
 
diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs
index fa053df2..7f8d6440 100644
--- a/src/api/s3/multipart.rs
+++ b/src/api/s3/multipart.rs
@@ -16,6 +16,7 @@ use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 
 use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::*;
 use garage_api_common::signature::verify_signed_content;
 
 use crate::api_server::{ReqBody, ResBody};
diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs
index 6c0e73d4..908ee9f3 100644
--- a/src/api/s3/post_object.rs
+++ b/src/api/s3/post_object.rs
@@ -18,6 +18,7 @@ use garage_model::s3::object_table::*;
 
 use garage_api_common::cors::*;
 use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::*;
 use garage_api_common::signature::payload::{verify_v4, Authorization};
 
 use crate::api_server::ResBody;
diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs
index 530b4e7b..834be6f1 100644
--- a/src/api/s3/put.rs
+++ b/src/api/s3/put.rs
@@ -31,6 +31,7 @@ use garage_model::s3::object_table::*;
 use garage_model::s3::version_table::*;
 
 use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::*;
 
 use crate::api_server::{ReqBody, ResBody};
 use crate::checksum::*;