aboutsummaryrefslogtreecommitdiff
path: root/src/model
diff options
context:
space:
mode:
authorAlex Auvolat <alex@adnab.me>2024-03-21 14:06:59 +0100
committerAlex Auvolat <alex@adnab.me>2024-03-26 15:01:34 +0100
commit74949c69cbf1a8222b6d10a02fcf5fe139ccb560 (patch)
tree54128ccf24e8598c52c2a5366ad0b509bf1c296e /src/model
parent7e0107c47db71e8da13990c9111ebde8cbf60d8f (diff)
downloadgarage-74949c69cbf1a8222b6d10a02fcf5fe139ccb560.tar.gz
garage-74949c69cbf1a8222b6d10a02fcf5fe139ccb560.zip
[s3-checksum] implement x-amz-checksum-* headers
Diffstat (limited to 'src/model')
-rw-r--r--src/model/s3/mpu_table.rs9
-rw-r--r--src/model/s3/object_table.rs53
2 files changed, 55 insertions, 7 deletions
diff --git a/src/model/s3/mpu_table.rs b/src/model/s3/mpu_table.rs
index 238cbf11..c9f79caf 100644
--- a/src/model/s3/mpu_table.rs
+++ b/src/model/s3/mpu_table.rs
@@ -17,6 +17,7 @@ pub const PARTS: &str = "parts";
pub const BYTES: &str = "bytes";
mod v09 {
+ use crate::s3::object_table::ChecksumValue;
use garage_util::crdt;
use garage_util::data::Uuid;
use serde::{Deserialize, Serialize};
@@ -61,6 +62,9 @@ mod v09 {
pub version: Uuid,
/// ETag of the content of this part (known only once done uploading)
pub etag: Option<String>,
+ /// Checksum requested by x-amz-checksum-algorithm
+ #[serde(default)]
+ pub checksum: Option<ChecksumValue>,
/// Size of this part (known only once done uploading)
pub size: Option<u64>,
}
@@ -155,6 +159,11 @@ impl Crdt for MpuPart {
(Some(x), Some(y)) if x < y => other.size,
(x, _) => x,
};
+ self.checksum = match (self.checksum.take(), &other.checksum) {
+ (None, Some(_)) => other.checksum.clone(),
+ (Some(x), Some(y)) if x < *y => other.checksum.clone(),
+ (x, _) => x,
+ };
}
}
diff --git a/src/model/s3/object_table.rs b/src/model/s3/object_table.rs
index eedb9615..b2f25803 100644
--- a/src/model/s3/object_table.rs
+++ b/src/model/s3/object_table.rs
@@ -208,6 +208,8 @@ mod v010 {
Uploading {
/// Indicates whether this is a multipart upload
multipart: bool,
+ /// Checksum algorithm to use
+ checksum_algorithm: Option<ChecksumAlgorithm>,
/// Encryption params + headers to be included in the final object
encryption: ObjectVersionEncryption,
},
@@ -247,10 +249,10 @@ mod v010 {
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
pub enum ObjectVersionEncryption {
SseC {
- /// Encrypted serialized ObjectVersionHeaders struct.
+ /// Encrypted serialized ObjectVersionInner struct.
/// This is never compressed, just encrypted using AES256-GCM.
#[serde(with = "serde_bytes")]
- headers: Vec<u8>,
+ inner: Vec<u8>,
/// Whether data blocks are compressed in addition to being encrypted
/// (compression happens before encryption, whereas for non-encrypted
/// objects, compression is handled at the level of the block manager)
@@ -258,13 +260,35 @@ mod v010 {
},
Plaintext {
/// Plain-text headers
- headers: ObjectVersionHeaders,
+ inner: ObjectVersionMetaInner,
},
}
/// Vector of headers, as tuples of the format (header name, header value)
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)]
- pub struct ObjectVersionHeaders(pub Vec<(String, String)>);
+ pub struct ObjectVersionMetaInner {
+ pub headers: HeaderList,
+ pub checksum: Option<ChecksumValue>,
+ }
+
+ pub type HeaderList = Vec<(String, String)>;
+
+ #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)]
+ pub enum ChecksumAlgorithm {
+ Crc32,
+ Crc32c,
+ Sha1,
+ Sha256,
+ }
+
+ /// Checksum value for x-amz-checksum-algorithm
+ #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Serialize, Deserialize)]
+ pub enum ChecksumValue {
+ Crc32(#[serde(with = "serde_bytes")] [u8; 4]),
+ Crc32c(#[serde(with = "serde_bytes")] [u8; 4]),
+ Sha1(#[serde(with = "serde_bytes")] [u8; 20]),
+ Sha256(#[serde(with = "serde_bytes")] [u8; 32]),
+ }
impl garage_util::migrate::Migrate for Object {
const VERSION_MARKER: &'static [u8] = b"G010s3ob";
@@ -288,6 +312,7 @@ mod v010 {
v09::ObjectVersionState::Uploading { multipart, headers } => {
ObjectVersionState::Uploading {
multipart,
+ checksum_algorithm: None,
encryption: migrate_headers(headers),
}
}
@@ -331,15 +356,18 @@ mod v010 {
}
ObjectVersionEncryption::Plaintext {
- headers: ObjectVersionHeaders(new_headers),
+ inner: ObjectVersionMetaInner {
+ headers: new_headers,
+ checksum: None,
+ },
}
}
// Since ObjectVersionHeaders can now be serialized independently, for the
// purpose of being encrypted, we need it to support migrations on its own
// as well.
- impl garage_util::migrate::InitialFormat for ObjectVersionHeaders {
- const VERSION_MARKER: &'static [u8] = b"G010s3oh";
+ impl garage_util::migrate::InitialFormat for ObjectVersionMetaInner {
+ const VERSION_MARKER: &'static [u8] = b"G010s3om";
}
}
@@ -454,6 +482,17 @@ impl Entry<Uuid, String> for Object {
}
}
+impl ChecksumValue {
+ pub fn algorithm(&self) -> ChecksumAlgorithm {
+ match self {
+ ChecksumValue::Crc32(_) => ChecksumAlgorithm::Crc32,
+ ChecksumValue::Crc32c(_) => ChecksumAlgorithm::Crc32c,
+ ChecksumValue::Sha1(_) => ChecksumAlgorithm::Sha1,
+ ChecksumValue::Sha256(_) => ChecksumAlgorithm::Sha256,
+ }
+ }
+}
+
impl Crdt for Object {
fn merge(&mut self, other: &Self) {
// Merge versions from other into here