aboutsummaryrefslogtreecommitdiff
path: root/src/api/s3
diff options
context:
space:
mode:
Diffstat (limited to 'src/api/s3')
-rw-r--r--src/api/s3/Cargo.toml60
-rw-r--r--src/api/s3/api_server.rs82
-rw-r--r--src/api/s3/bucket.rs21
-rw-r--r--src/api/s3/checksum.rs406
-rw-r--r--src/api/s3/copy.rs26
-rw-r--r--src/api/s3/cors.rs182
-rw-r--r--src/api/s3/delete.rs20
-rw-r--r--src/api/s3/encryption.rs7
-rw-r--r--src/api/s3/error.rs43
-rw-r--r--src/api/s3/get.rs54
-rw-r--r--src/api/s3/lib.rs (renamed from src/api/s3/mod.rs)6
-rw-r--r--src/api/s3/lifecycle.rs19
-rw-r--r--src/api/s3/list.rs18
-rw-r--r--src/api/s3/multipart.rs177
-rw-r--r--src/api/s3/post_object.rs29
-rw-r--r--src/api/s3/put.rs100
-rw-r--r--src/api/s3/router.rs22
-rw-r--r--src/api/s3/website.rs26
-rw-r--r--src/api/s3/xml.rs2
19 files changed, 489 insertions, 811 deletions
diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml
new file mode 100644
index 00000000..387e45db
--- /dev/null
+++ b/src/api/s3/Cargo.toml
@@ -0,0 +1,60 @@
+[package]
+name = "garage_api_s3"
+version = "1.0.1"
+authors = ["Alex Auvolat <alex@adnab.me>"]
+edition = "2018"
+license = "AGPL-3.0"
+description = "S3 API server crate for the Garage object store"
+repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
+readme = "../../README.md"
+
+[lib]
+path = "lib.rs"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+garage_model.workspace = true
+garage_table.workspace = true
+garage_block.workspace = true
+garage_net.workspace = true
+garage_util.workspace = true
+garage_rpc.workspace = true
+garage_api_common.workspace = true
+
+aes-gcm.workspace = true
+async-compression.workspace = true
+base64.workspace = true
+bytes.workspace = true
+chrono.workspace = true
+crc32fast.workspace = true
+crc32c.workspace = true
+err-derive.workspace = true
+hex.workspace = true
+tracing.workspace = true
+md-5.workspace = true
+pin-project.workspace = true
+sha1.workspace = true
+sha2.workspace = true
+
+futures.workspace = true
+tokio.workspace = true
+tokio-stream.workspace = true
+tokio-util.workspace = true
+
+form_urlencoded.workspace = true
+http.workspace = true
+httpdate.workspace = true
+http-range.workspace = true
+http-body-util.workspace = true
+hyper = { workspace = true, default-features = false, features = ["server", "http1"] }
+multer.workspace = true
+percent-encoding.workspace = true
+roxmltree.workspace = true
+url.workspace = true
+
+serde.workspace = true
+serde_json.workspace = true
+quick-xml.workspace = true
+
+opentelemetry.workspace = true
diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs
index 1737af33..e26c2b65 100644
--- a/src/api/s3/api_server.rs
+++ b/src/api/s3/api_server.rs
@@ -1,7 +1,5 @@
use std::sync::Arc;
-use async_trait::async_trait;
-
use hyper::header;
use hyper::{body::Incoming as IncomingBody, Request, Response};
use tokio::sync::watch;
@@ -14,33 +12,33 @@ use garage_util::socket_address::UnixOrTCPSocketAddress;
use garage_model::garage::Garage;
use garage_model::key_table::Key;
-use crate::generic_server::*;
-use crate::s3::error::*;
-
-use crate::signature::verify_request;
-
-use crate::helpers::*;
-use crate::s3::bucket::*;
-use crate::s3::copy::*;
-use crate::s3::cors::*;
-use crate::s3::delete::*;
-use crate::s3::get::*;
-use crate::s3::lifecycle::*;
-use crate::s3::list::*;
-use crate::s3::multipart::*;
-use crate::s3::post_object::handle_post_object;
-use crate::s3::put::*;
-use crate::s3::router::Endpoint;
-use crate::s3::website::*;
-
-pub use crate::signature::streaming::ReqBody;
+use garage_api_common::cors::*;
+use garage_api_common::generic_server::*;
+use garage_api_common::helpers::*;
+use garage_api_common::signature::verify_request;
+
+use crate::bucket::*;
+use crate::copy::*;
+use crate::cors::*;
+use crate::delete::*;
+use crate::error::*;
+use crate::get::*;
+use crate::lifecycle::*;
+use crate::list::*;
+use crate::multipart::*;
+use crate::post_object::handle_post_object;
+use crate::put::*;
+use crate::router::Endpoint;
+use crate::website::*;
+
+pub use garage_api_common::signature::streaming::ReqBody;
pub type ResBody = BoxBody<Error>;
pub struct S3ApiServer {
garage: Arc<Garage>,
}
-pub(crate) struct S3ApiEndpoint {
+pub struct S3ApiEndpoint {
bucket_name: Option<String>,
endpoint: Endpoint,
}
@@ -70,7 +68,6 @@ impl S3ApiServer {
}
}
-#[async_trait]
impl ApiHandler for S3ApiServer {
const API_NAME: &'static str = "s3";
const API_NAME_DISPLAY: &'static str = "S3";
@@ -124,7 +121,9 @@ impl ApiHandler for S3ApiServer {
return Ok(options_res.map(|_empty_body: EmptyBody| empty_body()));
}
- let (req, api_key, content_sha256) = verify_request(&garage, req, "s3").await?;
+ let verified_request = verify_request(&garage, req, "s3").await?;
+ let req = verified_request.request;
+ let api_key = verified_request.access_key;
let bucket_name = match bucket_name {
None => {
@@ -137,20 +136,14 @@ impl ApiHandler for S3ApiServer {
// Special code path for CreateBucket API endpoint
if let Endpoint::CreateBucket {} = endpoint {
- return handle_create_bucket(
- &garage,
- req,
- content_sha256,
- &api_key.key_id,
- bucket_name,
- )
- .await;
+ return handle_create_bucket(&garage, req, &api_key.key_id, bucket_name).await;
}
let bucket_id = garage
.bucket_helper()
.resolve_bucket(&bucket_name, &api_key)
- .await?;
+ .await
+ .map_err(pass_helper_error)?;
let bucket = garage
.bucket_helper()
.get_existing_bucket(bucket_id)
@@ -181,7 +174,7 @@ impl ApiHandler for S3ApiServer {
let resp = match endpoint {
Endpoint::HeadObject {
key, part_number, ..
- } => handle_head(ctx, &req, &key, part_number).await,
+ } => handle_head(ctx, &req.map(|_| ()), &key, part_number).await,
Endpoint::GetObject {
key,
part_number,
@@ -201,20 +194,20 @@ impl ApiHandler for S3ApiServer {
response_content_type,
response_expires,
};
- handle_get(ctx, &req, &key, part_number, overrides).await
+ handle_get(ctx, &req.map(|_| ()), &key, part_number, overrides).await
}
Endpoint::UploadPart {
key,
part_number,
upload_id,
- } => handle_put_part(ctx, req, &key, part_number, &upload_id, content_sha256).await,
+ } => handle_put_part(ctx, req, &key, part_number, &upload_id).await,
Endpoint::CopyObject { key } => handle_copy(ctx, &req, &key).await,
Endpoint::UploadPartCopy {
key,
part_number,
upload_id,
} => handle_upload_part_copy(ctx, &req, &key, part_number, &upload_id).await,
- Endpoint::PutObject { key } => handle_put(ctx, req, &key, content_sha256).await,
+ Endpoint::PutObject { key } => handle_put(ctx, req, &key).await,
Endpoint::AbortMultipartUpload { key, upload_id } => {
handle_abort_multipart_upload(ctx, &key, &upload_id).await
}
@@ -223,7 +216,7 @@ impl ApiHandler for S3ApiServer {
handle_create_multipart_upload(ctx, &req, &key).await
}
Endpoint::CompleteMultipartUpload { key, upload_id } => {
- handle_complete_multipart_upload(ctx, req, &key, &upload_id, content_sha256).await
+ handle_complete_multipart_upload(ctx, req, &key, &upload_id).await
}
Endpoint::CreateBucket {} => unreachable!(),
Endpoint::HeadBucket {} => {
@@ -319,7 +312,6 @@ impl ApiHandler for S3ApiServer {
} => {
let query = ListPartsQuery {
bucket_name: ctx.bucket_name.clone(),
- bucket_id,
key,
upload_id,
part_number_marker: part_number_marker.map(|p| p.min(10000)),
@@ -327,17 +319,15 @@ impl ApiHandler for S3ApiServer {
};
handle_list_parts(ctx, req, &query).await
}
- Endpoint::DeleteObjects {} => handle_delete_objects(ctx, req, content_sha256).await,
+ Endpoint::DeleteObjects {} => handle_delete_objects(ctx, req).await,
Endpoint::GetBucketWebsite {} => handle_get_website(ctx).await,
- Endpoint::PutBucketWebsite {} => handle_put_website(ctx, req, content_sha256).await,
+ Endpoint::PutBucketWebsite {} => handle_put_website(ctx, req).await,
Endpoint::DeleteBucketWebsite {} => handle_delete_website(ctx).await,
Endpoint::GetBucketCors {} => handle_get_cors(ctx).await,
- Endpoint::PutBucketCors {} => handle_put_cors(ctx, req, content_sha256).await,
+ Endpoint::PutBucketCors {} => handle_put_cors(ctx, req).await,
Endpoint::DeleteBucketCors {} => handle_delete_cors(ctx).await,
Endpoint::GetBucketLifecycleConfiguration {} => handle_get_lifecycle(ctx).await,
- Endpoint::PutBucketLifecycleConfiguration {} => {
- handle_put_lifecycle(ctx, req, content_sha256).await
- }
+ Endpoint::PutBucketLifecycleConfiguration {} => handle_put_lifecycle(ctx, req).await,
Endpoint::DeleteBucketLifecycle {} => handle_delete_lifecycle(ctx).await,
endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
};
diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs
index 6a12aa9c..3a09e769 100644
--- a/src/api/s3/bucket.rs
+++ b/src/api/s3/bucket.rs
@@ -1,6 +1,5 @@
use std::collections::HashMap;
-use http_body_util::BodyExt;
use hyper::{Request, Response, StatusCode};
use garage_model::bucket_alias_table::*;
@@ -10,15 +9,14 @@ use garage_model::key_table::Key;
use garage_model::permission::BucketKeyPerm;
use garage_table::util::*;
use garage_util::crdt::*;
-use garage_util::data::*;
use garage_util::time::*;
-use crate::common_error::CommonError;
-use crate::helpers::*;
-use crate::s3::api_server::{ReqBody, ResBody};
-use crate::s3::error::*;
-use crate::s3::xml as s3_xml;
-use crate::signature::verify_signed_content;
+use garage_api_common::common_error::CommonError;
+use garage_api_common::helpers::*;
+
+use crate::api_server::{ReqBody, ResBody};
+use crate::error::*;
+use crate::xml as s3_xml;
pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
let ReqCtx { garage, .. } = ctx;
@@ -121,15 +119,10 @@ pub async fn handle_list_buckets(
pub async fn handle_create_bucket(
garage: &Garage,
req: Request<ReqBody>,
- content_sha256: Option<Hash>,
api_key_id: &String,
bucket_name: String,
) -> Result<Response<ResBody>, Error> {
- let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
- if let Some(content_sha256) = content_sha256 {
- verify_signed_content(content_sha256, &body[..])?;
- }
+ let body = req.into_body().collect().await?;
let cmd =
parse_create_bucket_xml(&body[..]).ok_or_bad_request("Invalid create bucket XML query")?;
diff --git a/src/api/s3/checksum.rs b/src/api/s3/checksum.rs
deleted file mode 100644
index c7527163..00000000
--- a/src/api/s3/checksum.rs
+++ /dev/null
@@ -1,406 +0,0 @@
-use std::convert::{TryFrom, TryInto};
-use std::hash::Hasher;
-
-use base64::prelude::*;
-use crc32c::Crc32cHasher as Crc32c;
-use crc32fast::Hasher as Crc32;
-use md5::{Digest, Md5};
-use sha1::Sha1;
-use sha2::Sha256;
-
-use http::{HeaderMap, HeaderName, HeaderValue};
-
-use garage_util::data::*;
-use garage_util::error::OkOrMessage;
-
-use garage_model::s3::object_table::*;
-
-use crate::s3::error::*;
-
-pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName =
- HeaderName::from_static("x-amz-checksum-algorithm");
-pub const X_AMZ_CHECKSUM_MODE: HeaderName = HeaderName::from_static("x-amz-checksum-mode");
-pub const X_AMZ_CHECKSUM_CRC32: HeaderName = HeaderName::from_static("x-amz-checksum-crc32");
-pub const X_AMZ_CHECKSUM_CRC32C: HeaderName = HeaderName::from_static("x-amz-checksum-crc32c");
-pub const X_AMZ_CHECKSUM_SHA1: HeaderName = HeaderName::from_static("x-amz-checksum-sha1");
-pub const X_AMZ_CHECKSUM_SHA256: HeaderName = HeaderName::from_static("x-amz-checksum-sha256");
-
-pub type Crc32Checksum = [u8; 4];
-pub type Crc32cChecksum = [u8; 4];
-pub type Md5Checksum = [u8; 16];
-pub type Sha1Checksum = [u8; 20];
-pub type Sha256Checksum = [u8; 32];
-
-#[derive(Debug, Default)]
-pub(crate) struct ExpectedChecksums {
- // base64-encoded md5 (content-md5 header)
- pub md5: Option<String>,
- // content_sha256 (as a Hash / FixedBytes32)
- pub sha256: Option<Hash>,
- // extra x-amz-checksum-* header
- pub extra: Option<ChecksumValue>,
-}
-
-pub(crate) struct Checksummer {
- pub crc32: Option<Crc32>,
- pub crc32c: Option<Crc32c>,
- pub md5: Option<Md5>,
- pub sha1: Option<Sha1>,
- pub sha256: Option<Sha256>,
-}
-
-#[derive(Default)]
-pub(crate) struct Checksums {
- pub crc32: Option<Crc32Checksum>,
- pub crc32c: Option<Crc32cChecksum>,
- pub md5: Option<Md5Checksum>,
- pub sha1: Option<Sha1Checksum>,
- pub sha256: Option<Sha256Checksum>,
-}
-
-impl Checksummer {
- pub(crate) fn init(expected: &ExpectedChecksums, require_md5: bool) -> Self {
- let mut ret = Self {
- crc32: None,
- crc32c: None,
- md5: None,
- sha1: None,
- sha256: None,
- };
-
- if expected.md5.is_some() || require_md5 {
- ret.md5 = Some(Md5::new());
- }
- if expected.sha256.is_some() || matches!(&expected.extra, Some(ChecksumValue::Sha256(_))) {
- ret.sha256 = Some(Sha256::new());
- }
- if matches!(&expected.extra, Some(ChecksumValue::Crc32(_))) {
- ret.crc32 = Some(Crc32::new());
- }
- if matches!(&expected.extra, Some(ChecksumValue::Crc32c(_))) {
- ret.crc32c = Some(Crc32c::default());
- }
- if matches!(&expected.extra, Some(ChecksumValue::Sha1(_))) {
- ret.sha1 = Some(Sha1::new());
- }
- ret
- }
-
- pub(crate) fn add(mut self, algo: Option<ChecksumAlgorithm>) -> Self {
- match algo {
- Some(ChecksumAlgorithm::Crc32) => {
- self.crc32 = Some(Crc32::new());
- }
- Some(ChecksumAlgorithm::Crc32c) => {
- self.crc32c = Some(Crc32c::default());
- }
- Some(ChecksumAlgorithm::Sha1) => {
- self.sha1 = Some(Sha1::new());
- }
- Some(ChecksumAlgorithm::Sha256) => {
- self.sha256 = Some(Sha256::new());
- }
- None => (),
- }
- self
- }
-
- pub(crate) fn update(&mut self, bytes: &[u8]) {
- if let Some(crc32) = &mut self.crc32 {
- crc32.update(bytes);
- }
- if let Some(crc32c) = &mut self.crc32c {
- crc32c.write(bytes);
- }
- if let Some(md5) = &mut self.md5 {
- md5.update(bytes);
- }
- if let Some(sha1) = &mut self.sha1 {
- sha1.update(bytes);
- }
- if let Some(sha256) = &mut self.sha256 {
- sha256.update(bytes);
- }
- }
-
- pub(crate) fn finalize(self) -> Checksums {
- Checksums {
- crc32: self.crc32.map(|x| u32::to_be_bytes(x.finalize())),
- crc32c: self
- .crc32c
- .map(|x| u32::to_be_bytes(u32::try_from(x.finish()).unwrap())),
- md5: self.md5.map(|x| x.finalize()[..].try_into().unwrap()),
- sha1: self.sha1.map(|x| x.finalize()[..].try_into().unwrap()),
- sha256: self.sha256.map(|x| x.finalize()[..].try_into().unwrap()),
- }
- }
-}
-
-impl Checksums {
- pub fn verify(&self, expected: &ExpectedChecksums) -> Result<(), Error> {
- if let Some(expected_md5) = &expected.md5 {
- match self.md5 {
- Some(md5) if BASE64_STANDARD.encode(&md5) == expected_md5.trim_matches('"') => (),
- _ => {
- return Err(Error::InvalidDigest(
- "MD5 checksum verification failed (from content-md5)".into(),
- ))
- }
- }
- }
- if let Some(expected_sha256) = &expected.sha256 {
- match self.sha256 {
- Some(sha256) if &sha256[..] == expected_sha256.as_slice() => (),
- _ => {
- return Err(Error::InvalidDigest(
- "SHA256 checksum verification failed (from x-amz-content-sha256)".into(),
- ))
- }
- }
- }
- if let Some(extra) = expected.extra {
- let algo = extra.algorithm();
- if self.extract(Some(algo)) != Some(extra) {
- return Err(Error::InvalidDigest(format!(
- "Failed to validate checksum for algorithm {:?}",
- algo
- )));
- }
- }
- Ok(())
- }
-
- pub fn extract(&self, algo: Option<ChecksumAlgorithm>) -> Option<ChecksumValue> {
- match algo {
- None => None,
- Some(ChecksumAlgorithm::Crc32) => Some(ChecksumValue::Crc32(self.crc32.unwrap())),
- Some(ChecksumAlgorithm::Crc32c) => Some(ChecksumValue::Crc32c(self.crc32c.unwrap())),
- Some(ChecksumAlgorithm::Sha1) => Some(ChecksumValue::Sha1(self.sha1.unwrap())),
- Some(ChecksumAlgorithm::Sha256) => Some(ChecksumValue::Sha256(self.sha256.unwrap())),
- }
- }
-}
-
-// ----
-
-#[derive(Default)]
-pub(crate) struct MultipartChecksummer {
- pub md5: Md5,
- pub extra: Option<MultipartExtraChecksummer>,
-}
-
-pub(crate) enum MultipartExtraChecksummer {
- Crc32(Crc32),
- Crc32c(Crc32c),
- Sha1(Sha1),
- Sha256(Sha256),
-}
-
-impl MultipartChecksummer {
- pub(crate) fn init(algo: Option<ChecksumAlgorithm>) -> Self {
- Self {
- md5: Md5::new(),
- extra: match algo {
- None => None,
- Some(ChecksumAlgorithm::Crc32) => {
- Some(MultipartExtraChecksummer::Crc32(Crc32::new()))
- }
- Some(ChecksumAlgorithm::Crc32c) => {
- Some(MultipartExtraChecksummer::Crc32c(Crc32c::default()))
- }
- Some(ChecksumAlgorithm::Sha1) => Some(MultipartExtraChecksummer::Sha1(Sha1::new())),
- Some(ChecksumAlgorithm::Sha256) => {
- Some(MultipartExtraChecksummer::Sha256(Sha256::new()))
- }
- },
- }
- }
-
- pub(crate) fn update(
- &mut self,
- etag: &str,
- checksum: Option<ChecksumValue>,
- ) -> Result<(), Error> {
- self.md5
- .update(&hex::decode(&etag).ok_or_message("invalid etag hex")?);
- match (&mut self.extra, checksum) {
- (None, _) => (),
- (
- Some(MultipartExtraChecksummer::Crc32(ref mut crc32)),
- Some(ChecksumValue::Crc32(x)),
- ) => {
- crc32.update(&x);
- }
- (
- Some(MultipartExtraChecksummer::Crc32c(ref mut crc32c)),
- Some(ChecksumValue::Crc32c(x)),
- ) => {
- crc32c.write(&x);
- }
- (Some(MultipartExtraChecksummer::Sha1(ref mut sha1)), Some(ChecksumValue::Sha1(x))) => {
- sha1.update(&x);
- }
- (
- Some(MultipartExtraChecksummer::Sha256(ref mut sha256)),
- Some(ChecksumValue::Sha256(x)),
- ) => {
- sha256.update(&x);
- }
- (Some(_), b) => {
- return Err(Error::internal_error(format!(
- "part checksum was not computed correctly, got: {:?}",
- b
- )))
- }
- }
- Ok(())
- }
-
- pub(crate) fn finalize(self) -> (Md5Checksum, Option<ChecksumValue>) {
- let md5 = self.md5.finalize()[..].try_into().unwrap();
- let extra = match self.extra {
- None => None,
- Some(MultipartExtraChecksummer::Crc32(crc32)) => {
- Some(ChecksumValue::Crc32(u32::to_be_bytes(crc32.finalize())))
- }
- Some(MultipartExtraChecksummer::Crc32c(crc32c)) => Some(ChecksumValue::Crc32c(
- u32::to_be_bytes(u32::try_from(crc32c.finish()).unwrap()),
- )),
- Some(MultipartExtraChecksummer::Sha1(sha1)) => {
- Some(ChecksumValue::Sha1(sha1.finalize()[..].try_into().unwrap()))
- }
- Some(MultipartExtraChecksummer::Sha256(sha256)) => Some(ChecksumValue::Sha256(
- sha256.finalize()[..].try_into().unwrap(),
- )),
- };
- (md5, extra)
- }
-}
-
-// ----
-
-/// Extract the value of the x-amz-checksum-algorithm header
-pub(crate) fn request_checksum_algorithm(
- headers: &HeaderMap<HeaderValue>,
-) -> Result<Option<ChecksumAlgorithm>, Error> {
- match headers.get(X_AMZ_CHECKSUM_ALGORITHM) {
- None => Ok(None),
- Some(x) if x == "CRC32" => Ok(Some(ChecksumAlgorithm::Crc32)),
- Some(x) if x == "CRC32C" => Ok(Some(ChecksumAlgorithm::Crc32c)),
- Some(x) if x == "SHA1" => Ok(Some(ChecksumAlgorithm::Sha1)),
- Some(x) if x == "SHA256" => Ok(Some(ChecksumAlgorithm::Sha256)),
- _ => Err(Error::bad_request("invalid checksum algorithm")),
- }
-}
-
-/// Extract the value of any of the x-amz-checksum-* headers
-pub(crate) fn request_checksum_value(
- headers: &HeaderMap<HeaderValue>,
-) -> Result<Option<ChecksumValue>, Error> {
- let mut ret = vec![];
-
- if let Some(crc32_str) = headers.get(X_AMZ_CHECKSUM_CRC32) {
- let crc32 = BASE64_STANDARD
- .decode(&crc32_str)
- .ok()
- .and_then(|x| x.try_into().ok())
- .ok_or_bad_request("invalid x-amz-checksum-crc32 header")?;
- ret.push(ChecksumValue::Crc32(crc32))
- }
- if let Some(crc32c_str) = headers.get(X_AMZ_CHECKSUM_CRC32C) {
- let crc32c = BASE64_STANDARD
- .decode(&crc32c_str)
- .ok()
- .and_then(|x| x.try_into().ok())
- .ok_or_bad_request("invalid x-amz-checksum-crc32c header")?;
- ret.push(ChecksumValue::Crc32c(crc32c))
- }
- if let Some(sha1_str) = headers.get(X_AMZ_CHECKSUM_SHA1) {
- let sha1 = BASE64_STANDARD
- .decode(&sha1_str)
- .ok()
- .and_then(|x| x.try_into().ok())
- .ok_or_bad_request("invalid x-amz-checksum-sha1 header")?;
- ret.push(ChecksumValue::Sha1(sha1))
- }
- if let Some(sha256_str) = headers.get(X_AMZ_CHECKSUM_SHA256) {
- let sha256 = BASE64_STANDARD
- .decode(&sha256_str)
- .ok()
- .and_then(|x| x.try_into().ok())
- .ok_or_bad_request("invalid x-amz-checksum-sha256 header")?;
- ret.push(ChecksumValue::Sha256(sha256))
- }
-
- if ret.len() > 1 {
- return Err(Error::bad_request(
- "multiple x-amz-checksum-* headers given",
- ));
- }
- Ok(ret.pop())
-}
-
-/// Checks for the presence of x-amz-checksum-algorithm
-/// if so extract the corresponding x-amz-checksum-* value
-pub(crate) fn request_checksum_algorithm_value(
- headers: &HeaderMap<HeaderValue>,
-) -> Result<Option<ChecksumValue>, Error> {
- match headers.get(X_AMZ_CHECKSUM_ALGORITHM) {
- Some(x) if x == "CRC32" => {
- let crc32 = headers
- .get(X_AMZ_CHECKSUM_CRC32)
- .and_then(|x| BASE64_STANDARD.decode(&x).ok())
- .and_then(|x| x.try_into().ok())
- .ok_or_bad_request("invalid x-amz-checksum-crc32 header")?;
- Ok(Some(ChecksumValue::Crc32(crc32)))
- }
- Some(x) if x == "CRC32C" => {
- let crc32c = headers
- .get(X_AMZ_CHECKSUM_CRC32C)
- .and_then(|x| BASE64_STANDARD.decode(&x).ok())
- .and_then(|x| x.try_into().ok())
- .ok_or_bad_request("invalid x-amz-checksum-crc32c header")?;
- Ok(Some(ChecksumValue::Crc32c(crc32c)))
- }
- Some(x) if x == "SHA1" => {
- let sha1 = headers
- .get(X_AMZ_CHECKSUM_SHA1)
- .and_then(|x| BASE64_STANDARD.decode(&x).ok())
- .and_then(|x| x.try_into().ok())
- .ok_or_bad_request("invalid x-amz-checksum-sha1 header")?;
- Ok(Some(ChecksumValue::Sha1(sha1)))
- }
- Some(x) if x == "SHA256" => {
- let sha256 = headers
- .get(X_AMZ_CHECKSUM_SHA256)
- .and_then(|x| BASE64_STANDARD.decode(&x).ok())
- .and_then(|x| x.try_into().ok())
- .ok_or_bad_request("invalid x-amz-checksum-sha256 header")?;
- Ok(Some(ChecksumValue::Sha256(sha256)))
- }
- Some(_) => Err(Error::bad_request("invalid x-amz-checksum-algorithm")),
- None => Ok(None),
- }
-}
-
-pub(crate) fn add_checksum_response_headers(
- checksum: &Option<ChecksumValue>,
- mut resp: http::response::Builder,
-) -> http::response::Builder {
- match checksum {
- Some(ChecksumValue::Crc32(crc32)) => {
- resp = resp.header(X_AMZ_CHECKSUM_CRC32, BASE64_STANDARD.encode(&crc32));
- }
- Some(ChecksumValue::Crc32c(crc32c)) => {
- resp = resp.header(X_AMZ_CHECKSUM_CRC32C, BASE64_STANDARD.encode(&crc32c));
- }
- Some(ChecksumValue::Sha1(sha1)) => {
- resp = resp.header(X_AMZ_CHECKSUM_SHA1, BASE64_STANDARD.encode(&sha1));
- }
- Some(ChecksumValue::Sha256(sha256)) => {
- resp = resp.header(X_AMZ_CHECKSUM_SHA256, BASE64_STANDARD.encode(&sha256));
- }
- None => (),
- }
- resp
-}
diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs
index e375a714..ff8019e6 100644
--- a/src/api/s3/copy.rs
+++ b/src/api/s3/copy.rs
@@ -20,15 +20,16 @@ use garage_model::s3::mpu_table::*;
use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*;
-use crate::helpers::*;
-use crate::s3::api_server::{ReqBody, ResBody};
-use crate::s3::checksum::*;
-use crate::s3::encryption::EncryptionParams;
-use crate::s3::error::*;
-use crate::s3::get::full_object_byte_stream;
-use crate::s3::multipart;
-use crate::s3::put::{get_headers, save_stream, ChecksumMode, SaveStreamResult};
-use crate::s3::xml::{self as s3_xml, xmlns_tag};
+use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::*;
+
+use crate::api_server::{ReqBody, ResBody};
+use crate::encryption::EncryptionParams;
+use crate::error::*;
+use crate::get::full_object_byte_stream;
+use crate::multipart;
+use crate::put::{extract_metadata_headers, save_stream, ChecksumMode, SaveStreamResult};
+use crate::xml::{self as s3_xml, xmlns_tag};
// -------- CopyObject ---------
@@ -72,7 +73,7 @@ pub async fn handle_copy(
let dest_object_meta = ObjectVersionMetaInner {
headers: match req.headers().get("x-amz-metadata-directive") {
Some(v) if v == hyper::header::HeaderValue::from_static("REPLACE") => {
- get_headers(req.headers())?
+ extract_metadata_headers(req.headers())?
}
_ => source_object_meta_inner.into_owned().headers,
},
@@ -655,7 +656,8 @@ async fn get_copy_source(ctx: &ReqCtx, req: &Request<ReqBody>) -> Result<Object,
let source_bucket_id = garage
.bucket_helper()
.resolve_bucket(&source_bucket.to_string(), api_key)
- .await?;
+ .await
+ .map_err(pass_helper_error)?;
if !api_key.allow_read(&source_bucket_id) {
return Err(Error::forbidden(format!(
@@ -861,7 +863,7 @@ pub struct CopyPartResult {
#[cfg(test)]
mod tests {
use super::*;
- use crate::s3::xml::to_xml_with_header;
+ use crate::xml::to_xml_with_header;
#[test]
fn copy_object_result() -> Result<(), Error> {
diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs
index 173b7ffe..fcfdb934 100644
--- a/src/api/s3/cors.rs
+++ b/src/api/s3/cors.rs
@@ -1,29 +1,16 @@
use quick_xml::de::from_reader;
-use std::sync::Arc;
-use http::header::{
- ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN,
- ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD,
-};
-use hyper::{
- body::Body, body::Incoming as IncomingBody, header::HeaderName, Method, Request, Response,
- StatusCode,
-};
-
-use http_body_util::BodyExt;
+use hyper::{header::HeaderName, Method, Request, Response, StatusCode};
use serde::{Deserialize, Serialize};
-use crate::common_error::CommonError;
-use crate::helpers::*;
-use crate::s3::api_server::{ReqBody, ResBody};
-use crate::s3::error::*;
-use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
-use crate::signature::verify_signed_content;
+use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule};
+
+use garage_api_common::helpers::*;
-use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule};
-use garage_model::garage::Garage;
-use garage_util::data::*;
+use crate::api_server::{ReqBody, ResBody};
+use crate::error::*;
+use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
pub async fn handle_get_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
let ReqCtx { bucket_params, .. } = ctx;
@@ -68,7 +55,6 @@ pub async fn handle_delete_cors(ctx: ReqCtx) -> Result<Response<ResBody>, Error>
pub async fn handle_put_cors(
ctx: ReqCtx,
req: Request<ReqBody>,
- content_sha256: Option<Hash>,
) -> Result<Response<ResBody>, Error> {
let ReqCtx {
garage,
@@ -77,11 +63,7 @@ pub async fn handle_put_cors(
..
} = ctx;
- let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
- if let Some(content_sha256) = content_sha256 {
- verify_signed_content(content_sha256, &body[..])?;
- }
+ let body = req.into_body().collect().await?;
let conf: CorsConfiguration = from_reader(&body as &[u8])?;
conf.validate()?;
@@ -99,154 +81,6 @@ pub async fn handle_put_cors(
.body(empty_body())?)
}
-pub async fn handle_options_api(
- garage: Arc<Garage>,
- req: &Request<IncomingBody>,
- bucket_name: Option<String>,
-) -> Result<Response<EmptyBody>, CommonError> {
- // FIXME: CORS rules of buckets with local aliases are
- // not taken into account.
-
- // If the bucket name is a global bucket name,
- // we try to apply the CORS rules of that bucket.
- // If a user has a local bucket name that has
- // the same name, its CORS rules won't be applied
- // and will be shadowed by the rules of the globally
- // existing bucket (but this is inevitable because
- // OPTIONS calls are not auhtenticated).
- if let Some(bn) = bucket_name {
- let helper = garage.bucket_helper();
- let bucket_id = helper.resolve_global_bucket_name(&bn).await?;
- if let Some(id) = bucket_id {
- let bucket = garage.bucket_helper().get_existing_bucket(id).await?;
- let bucket_params = bucket.state.into_option().unwrap();
- handle_options_for_bucket(req, &bucket_params)
- } else {
- // If there is a bucket name in the request, but that name
- // does not correspond to a global alias for a bucket,
- // then it's either a non-existing bucket or a local bucket.
- // We have no way of knowing, because the request is not
- // authenticated and thus we can't resolve local aliases.
- // We take the permissive approach of allowing everything,
- // because we don't want to prevent web apps that use
- // local bucket names from making API calls.
- Ok(Response::builder()
- .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")
- .header(ACCESS_CONTROL_ALLOW_METHODS, "*")
- .status(StatusCode::OK)
- .body(EmptyBody::new())?)
- }
- } else {
- // If there is no bucket name in the request,
- // we are doing a ListBuckets call, which we want to allow
- // for all origins.
- Ok(Response::builder()
- .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*")
- .header(ACCESS_CONTROL_ALLOW_METHODS, "GET")
- .status(StatusCode::OK)
- .body(EmptyBody::new())?)
- }
-}
-
-pub fn handle_options_for_bucket(
- req: &Request<IncomingBody>,
- bucket_params: &BucketParams,
-) -> Result<Response<EmptyBody>, CommonError> {
- let origin = req
- .headers()
- .get("Origin")
- .ok_or_bad_request("Missing Origin header")?
- .to_str()?;
- let request_method = req
- .headers()
- .get(ACCESS_CONTROL_REQUEST_METHOD)
- .ok_or_bad_request("Missing Access-Control-Request-Method header")?
- .to_str()?;
- let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) {
- Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::<Vec<_>>(),
- None => vec![],
- };
-
- if let Some(cors_config) = bucket_params.cors_config.get() {
- let matching_rule = cors_config
- .iter()
- .find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter()));
- if let Some(rule) = matching_rule {
- let mut resp = Response::builder()
- .status(StatusCode::OK)
- .body(EmptyBody::new())?;
- add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?;
- return Ok(resp);
- }
- }
-
- Err(CommonError::Forbidden(
- "This CORS request is not allowed.".into(),
- ))
-}
-
-pub fn find_matching_cors_rule<'a>(
- bucket_params: &'a BucketParams,
- req: &Request<impl Body>,
-) -> Result<Option<&'a GarageCorsRule>, Error> {
- if let Some(cors_config) = bucket_params.cors_config.get() {
- if let Some(origin) = req.headers().get("Origin") {
- let origin = origin.to_str()?;
- let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) {
- Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::<Vec<_>>(),
- None => vec![],
- };
- return Ok(cors_config.iter().find(|rule| {
- cors_rule_matches(rule, origin, req.method().as_ref(), request_headers.iter())
- }));
- }
- }
- Ok(None)
-}
-
-fn cors_rule_matches<'a, HI, S>(
- rule: &GarageCorsRule,
- origin: &'a str,
- method: &'a str,
- mut request_headers: HI,
-) -> bool
-where
- HI: Iterator<Item = S>,
- S: AsRef<str>,
-{
- rule.allow_origins.iter().any(|x| x == "*" || x == origin)
- && rule.allow_methods.iter().any(|x| x == "*" || x == method)
- && request_headers.all(|h| {
- rule.allow_headers
- .iter()
- .any(|x| x == "*" || x == h.as_ref())
- })
-}
-
-pub fn add_cors_headers(
- resp: &mut Response<impl Body>,
- rule: &GarageCorsRule,
-) -> Result<(), http::header::InvalidHeaderValue> {
- let h = resp.headers_mut();
- h.insert(
- ACCESS_CONTROL_ALLOW_ORIGIN,
- rule.allow_origins.join(", ").parse()?,
- );
- h.insert(
- ACCESS_CONTROL_ALLOW_METHODS,
- rule.allow_methods.join(", ").parse()?,
- );
- h.insert(
- ACCESS_CONTROL_ALLOW_HEADERS,
- rule.allow_headers.join(", ").parse()?,
- );
- h.insert(
- ACCESS_CONTROL_EXPOSE_HEADERS,
- rule.expose_headers.join(", ").parse()?,
- );
- Ok(())
-}
-
// ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ----
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
diff --git a/src/api/s3/delete.rs b/src/api/s3/delete.rs
index 57f6f948..d785b9d8 100644
--- a/src/api/s3/delete.rs
+++ b/src/api/s3/delete.rs
@@ -1,16 +1,15 @@
-use http_body_util::BodyExt;
use hyper::{Request, Response, StatusCode};
use garage_util::data::*;
use garage_model::s3::object_table::*;
-use crate::helpers::*;
-use crate::s3::api_server::{ReqBody, ResBody};
-use crate::s3::error::*;
-use crate::s3::put::next_timestamp;
-use crate::s3::xml as s3_xml;
-use crate::signature::verify_signed_content;
+use garage_api_common::helpers::*;
+
+use crate::api_server::{ReqBody, ResBody};
+use crate::error::*;
+use crate::put::next_timestamp;
+use crate::xml as s3_xml;
async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> {
let ReqCtx {
@@ -67,13 +66,8 @@ pub async fn handle_delete(ctx: ReqCtx, key: &str) -> Result<Response<ResBody>,
pub async fn handle_delete_objects(
ctx: ReqCtx,
req: Request<ReqBody>,
- content_sha256: Option<Hash>,
) -> Result<Response<ResBody>, Error> {
- let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
- if let Some(content_sha256) = content_sha256 {
- verify_signed_content(content_sha256, &body[..])?;
- }
+ let body = req.into_body().collect().await?;
let cmd_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
let cmd = parse_delete_objects_xml(&cmd_xml).ok_or_bad_request("Invalid delete XML query")?;
diff --git a/src/api/s3/encryption.rs b/src/api/s3/encryption.rs
index 2e6ed65c..fa7285ca 100644
--- a/src/api/s3/encryption.rs
+++ b/src/api/s3/encryption.rs
@@ -28,9 +28,10 @@ use garage_util::migrate::Migrate;
use garage_model::garage::Garage;
use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner};
-use crate::common_error::*;
-use crate::s3::checksum::Md5Checksum;
-use crate::s3::error::Error;
+use garage_api_common::common_error::*;
+use garage_api_common::signature::checksum::Md5Checksum;
+
+use crate::error::Error;
const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName =
HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm");
diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs
index 2855e0b3..6d4b7a11 100644
--- a/src/api/s3/error.rs
+++ b/src/api/s3/error.rs
@@ -4,19 +4,30 @@ use err_derive::Error;
use hyper::header::HeaderValue;
use hyper::{HeaderMap, StatusCode};
-use crate::common_error::CommonError;
-pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError};
-use crate::generic_server::ApiError;
-use crate::helpers::*;
-use crate::s3::xml as s3_xml;
-use crate::signature::error::Error as SignatureError;
+use garage_model::helper::error::Error as HelperError;
+
+pub(crate) use garage_api_common::common_error::pass_helper_error;
+
+use garage_api_common::common_error::{
+ commonErrorDerivative, helper_error_as_internal, CommonError,
+};
+
+pub use garage_api_common::common_error::{
+ CommonErrorDerivative, OkOrBadRequest, OkOrInternalError,
+};
+
+use garage_api_common::generic_server::ApiError;
+use garage_api_common::helpers::*;
+use garage_api_common::signature::error::Error as SignatureError;
+
+use crate::xml as s3_xml;
/// Errors of this crate
#[derive(Debug, Error)]
pub enum Error {
#[error(display = "{}", _0)]
/// Error from common error
- Common(CommonError),
+ Common(#[error(source)] CommonError),
// Category: cannot process
/// Authorization Header Malformed
@@ -69,7 +80,7 @@ pub enum Error {
#[error(display = "Invalid encryption algorithm: {:?}, should be AES256", _0)]
InvalidEncryptionAlgorithm(String),
- /// The client sent invalid XML data
+ /// The provided digest (checksum) value was invalid
#[error(display = "Invalid digest: {}", _0)]
InvalidDigest(String),
@@ -78,17 +89,16 @@ pub enum Error {
NotImplemented(String),
}
-impl<T> From<T> for Error
-where
- CommonError: From<T>,
-{
- fn from(err: T) -> Self {
- Error::Common(CommonError::from(err))
+commonErrorDerivative!(Error);
+
+// Helper errors are always passed as internal errors by default.
+// To pass the specific error code back to the client, use `pass_helper_error`.
+impl From<HelperError> for Error {
+ fn from(err: HelperError) -> Error {
+ Error::Common(helper_error_as_internal(err))
}
}
-impl CommonErrorDerivative for Error {}
-
impl From<roxmltree::Error> for Error {
fn from(err: roxmltree::Error) -> Self {
Self::InvalidXml(format!("{}", err))
@@ -109,6 +119,7 @@ impl From<SignatureError> for Error {
Self::AuthorizationHeaderMalformed(c)
}
SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i),
+ SignatureError::InvalidDigest(d) => Self::InvalidDigest(d),
}
}
}
diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs
index f5d3cf11..bcb72cc3 100644
--- a/src/api/s3/get.rs
+++ b/src/api/s3/get.rs
@@ -12,7 +12,7 @@ use http::header::{
CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE, ETAG, EXPIRES, IF_MODIFIED_SINCE, IF_NONE_MATCH,
LAST_MODIFIED, RANGE,
};
-use hyper::{body::Body, Request, Response, StatusCode};
+use hyper::{Request, Response, StatusCode};
use tokio::sync::mpsc;
use garage_net::stream::ByteStream;
@@ -25,11 +25,12 @@ use garage_model::garage::Garage;
use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*;
-use crate::helpers::*;
-use crate::s3::api_server::ResBody;
-use crate::s3::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE};
-use crate::s3::encryption::EncryptionParams;
-use crate::s3::error::*;
+use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE};
+
+use crate::api_server::ResBody;
+use crate::encryption::EncryptionParams;
+use crate::error::*;
const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count";
@@ -68,14 +69,11 @@ fn object_headers(
// See: https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html
let mut headers_by_name = BTreeMap::new();
for (name, value) in meta_inner.headers.iter() {
- match headers_by_name.get_mut(name) {
- None => {
- headers_by_name.insert(name, vec![value.as_str()]);
- }
- Some(headers) => {
- headers.push(value.as_str());
- }
- }
+ let name_lower = name.to_ascii_lowercase();
+ headers_by_name
+ .entry(name_lower)
+ .or_insert(vec![])
+ .push(value.as_str());
}
for (name, values) in headers_by_name {
@@ -120,7 +118,7 @@ fn getobject_override_headers(
fn try_answer_cached(
version: &ObjectVersion,
version_meta: &ObjectVersionMeta,
- req: &Request<impl Body>,
+ req: &Request<()>,
) -> Option<Response<ResBody>> {
// <trinity> It is possible, and is even usually the case, [that both If-None-Match and
// If-Modified-Since] are present in a request. In this situation If-None-Match takes
@@ -159,7 +157,7 @@ fn try_answer_cached(
/// Handle HEAD request
pub async fn handle_head(
ctx: ReqCtx,
- req: &Request<impl Body>,
+ req: &Request<()>,
key: &str,
part_number: Option<u64>,
) -> Result<Response<ResBody>, Error> {
@@ -169,7 +167,7 @@ pub async fn handle_head(
/// Handle HEAD request for website
pub async fn handle_head_without_ctx(
garage: Arc<Garage>,
- req: &Request<impl Body>,
+ req: &Request<()>,
bucket_id: Uuid,
key: &str,
part_number: Option<u64>,
@@ -280,7 +278,7 @@ pub async fn handle_head_without_ctx(
/// Handle GET request
pub async fn handle_get(
ctx: ReqCtx,
- req: &Request<impl Body>,
+ req: &Request<()>,
key: &str,
part_number: Option<u64>,
overrides: GetObjectOverrides,
@@ -291,7 +289,7 @@ pub async fn handle_get(
/// Handle GET request
pub async fn handle_get_without_ctx(
garage: Arc<Garage>,
- req: &Request<impl Body>,
+ req: &Request<()>,
bucket_id: Uuid,
key: &str,
part_number: Option<u64>,
@@ -342,7 +340,12 @@ pub async fn handle_get_without_ctx(
enc,
&headers,
pn,
- checksum_mode,
+ ChecksumMode {
+ // TODO: for multipart uploads, checksums of each part should be stored
+ // so that we can return the corresponding checksum here
+ // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+ enabled: false,
+ },
)
.await
}
@@ -356,7 +359,12 @@ pub async fn handle_get_without_ctx(
&headers,
range.start,
range.start + range.length,
- checksum_mode,
+ ChecksumMode {
+ // TODO: for range queries that align with part boundaries,
+ // we should return the saved checksum of the part
+ // https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html
+ enabled: false,
+ },
)
.await
}
@@ -579,7 +587,7 @@ async fn handle_get_part(
}
fn parse_range_header(
- req: &Request<impl Body>,
+ req: &Request<()>,
total_size: u64,
) -> Result<Option<http_range::HttpRange>, Error> {
let range = match req.headers().get(RANGE) {
@@ -620,7 +628,7 @@ struct ChecksumMode {
enabled: bool,
}
-fn checksum_mode(req: &Request<impl Body>) -> ChecksumMode {
+fn checksum_mode(req: &Request<()>) -> ChecksumMode {
ChecksumMode {
enabled: req
.headers()
diff --git a/src/api/s3/mod.rs b/src/api/s3/lib.rs
index b9bb1a6f..83f684f8 100644
--- a/src/api/s3/mod.rs
+++ b/src/api/s3/lib.rs
@@ -1,3 +1,6 @@
+#[macro_use]
+extern crate tracing;
+
pub mod api_server;
pub mod error;
@@ -11,9 +14,8 @@ mod list;
mod multipart;
mod post_object;
mod put;
-mod website;
+pub mod website;
-mod checksum;
mod encryption;
mod router;
pub mod xml;
diff --git a/src/api/s3/lifecycle.rs b/src/api/s3/lifecycle.rs
index 7eb1c2cb..c140494e 100644
--- a/src/api/s3/lifecycle.rs
+++ b/src/api/s3/lifecycle.rs
@@ -1,21 +1,19 @@
use quick_xml::de::from_reader;
-use http_body_util::BodyExt;
use hyper::{Request, Response, StatusCode};
use serde::{Deserialize, Serialize};
-use crate::helpers::*;
-use crate::s3::api_server::{ReqBody, ResBody};
-use crate::s3::error::*;
-use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
-use crate::signature::verify_signed_content;
+use garage_api_common::helpers::*;
+
+use crate::api_server::{ReqBody, ResBody};
+use crate::error::*;
+use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
use garage_model::bucket_table::{
parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration,
LifecycleFilter as GarageLifecycleFilter, LifecycleRule as GarageLifecycleRule,
};
-use garage_util::data::*;
pub async fn handle_get_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
let ReqCtx { bucket_params, .. } = ctx;
@@ -55,7 +53,6 @@ pub async fn handle_delete_lifecycle(ctx: ReqCtx) -> Result<Response<ResBody>, E
pub async fn handle_put_lifecycle(
ctx: ReqCtx,
req: Request<ReqBody>,
- content_sha256: Option<Hash>,
) -> Result<Response<ResBody>, Error> {
let ReqCtx {
garage,
@@ -64,11 +61,7 @@ pub async fn handle_put_lifecycle(
..
} = ctx;
- let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
- if let Some(content_sha256) = content_sha256 {
- verify_signed_content(content_sha256, &body[..])?;
- }
+ let body = req.into_body().collect().await?;
let conf: LifecycleConfiguration = from_reader(&body as &[u8])?;
let config = conf
diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs
index 68d6cbe6..94c2c895 100644
--- a/src/api/s3/list.rs
+++ b/src/api/s3/list.rs
@@ -13,13 +13,14 @@ use garage_model::s3::object_table::*;
use garage_table::EnumerationOrder;
-use crate::encoding::*;
-use crate::helpers::*;
-use crate::s3::api_server::{ReqBody, ResBody};
-use crate::s3::encryption::EncryptionParams;
-use crate::s3::error::*;
-use crate::s3::multipart as s3_multipart;
-use crate::s3::xml as s3_xml;
+use garage_api_common::encoding::*;
+use garage_api_common::helpers::*;
+
+use crate::api_server::{ReqBody, ResBody};
+use crate::encryption::EncryptionParams;
+use crate::error::*;
+use crate::multipart as s3_multipart;
+use crate::xml as s3_xml;
const DUMMY_NAME: &str = "Dummy Key";
const DUMMY_KEY: &str = "GKDummyKey";
@@ -53,7 +54,6 @@ pub struct ListMultipartUploadsQuery {
#[derive(Debug)]
pub struct ListPartsQuery {
pub bucket_name: String,
- pub bucket_id: Uuid,
pub key: String,
pub upload_id: String,
pub part_number_marker: Option<u64>,
@@ -1244,10 +1244,8 @@ mod tests {
#[test]
fn test_fetch_part_info() -> Result<(), Error> {
- let uuid = Uuid::from([0x08; 32]);
let mut query = ListPartsQuery {
bucket_name: "a".to_string(),
- bucket_id: uuid,
key: "a".to_string(),
upload_id: "xx".to_string(),
part_number_marker: None,
diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs
index 3db3e8aa..d6eb26cb 100644
--- a/src/api/s3/multipart.rs
+++ b/src/api/s3/multipart.rs
@@ -1,13 +1,20 @@
use std::collections::HashMap;
-use std::convert::TryInto;
+use std::convert::{TryFrom, TryInto};
+use std::hash::Hasher;
use std::sync::Arc;
use base64::prelude::*;
+use crc32c::Crc32cHasher as Crc32c;
+use crc32fast::Hasher as Crc32;
use futures::prelude::*;
use hyper::{Request, Response};
+use md5::{Digest, Md5};
+use sha1::Sha1;
+use sha2::Sha256;
use garage_table::*;
use garage_util::data::*;
+use garage_util::error::OkOrMessage;
use garage_model::garage::Garage;
use garage_model::s3::block_ref_table::*;
@@ -15,14 +22,14 @@ use garage_model::s3::mpu_table::*;
use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*;
-use crate::helpers::*;
-use crate::s3::api_server::{ReqBody, ResBody};
-use crate::s3::checksum::*;
-use crate::s3::encryption::EncryptionParams;
-use crate::s3::error::*;
-use crate::s3::put::*;
-use crate::s3::xml as s3_xml;
-use crate::signature::verify_signed_content;
+use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::*;
+
+use crate::api_server::{ReqBody, ResBody};
+use crate::encryption::EncryptionParams;
+use crate::error::*;
+use crate::put::*;
+use crate::xml as s3_xml;
// ----
@@ -42,7 +49,7 @@ pub async fn handle_create_multipart_upload(
let upload_id = gen_uuid();
let timestamp = next_timestamp(existing_object.as_ref());
- let headers = get_headers(req.headers())?;
+ let headers = extract_metadata_headers(req.headers())?;
let meta = ObjectVersionMetaInner {
headers,
checksum: None,
@@ -93,7 +100,6 @@ pub async fn handle_put_part(
key: &str,
part_number: u64,
upload_id: &str,
- content_sha256: Option<Hash>,
) -> Result<Response<ResBody>, Error> {
let ReqCtx { garage, .. } = &ctx;
@@ -104,17 +110,30 @@ pub async fn handle_put_part(
Some(x) => Some(x.to_str()?.to_string()),
None => None,
},
- sha256: content_sha256,
+ sha256: None,
extra: request_checksum_value(req.headers())?,
};
- // Read first chuck, and at the same time try to get object to see if it exists
let key = key.to_string();
- let (req_head, req_body) = req.into_parts();
- let stream = body_stream(req_body);
+ let (req_head, mut req_body) = req.into_parts();
+
+ // Before we stream the body, configure the needed checksums.
+ req_body.add_expected_checksums(expected_checksums.clone());
+ // TODO: avoid parsing encryption headers twice...
+ if !EncryptionParams::new_from_headers(&garage, &req_head.headers)?.is_encrypted() {
+ // For non-encrypted objects, we need to compute the md5sum in all cases
+ // (even if content-md5 is not set), because it is used as an etag of the
+ // part, which is in turn used in the etag computation of the whole object
+ req_body.add_md5();
+ }
+
+ let (stream, stream_checksums) = req_body.streaming_with_checksums();
+ let stream = stream.map_err(Error::from);
+
let mut chunker = StreamChunker::new(stream, garage.config.block_size);
+ // Read first chuck, and at the same time try to get object to see if it exists
let ((_, object_version, mut mpu), first_block) =
futures::try_join!(get_upload(&ctx, &key, &upload_id), chunker.next(),)?;
@@ -171,21 +190,21 @@ pub async fn handle_put_part(
garage.version_table.insert(&version).await?;
// Copy data to version
- let checksummer =
- Checksummer::init(&expected_checksums, !encryption.is_encrypted()).add(checksum_algorithm);
- let (total_size, checksums, _) = read_and_put_blocks(
+ let (total_size, _, _) = read_and_put_blocks(
&ctx,
&version,
encryption,
part_number,
first_block,
- &mut chunker,
- checksummer,
+ chunker,
+ Checksummer::new(),
)
.await?;
- // Verify that checksums map
- checksums.verify(&expected_checksums)?;
+ // Verify that checksums match
+ let checksums = stream_checksums
+ .await
+ .ok_or_internal_error("checksum calculation")??;
// Store part etag in version
let etag = encryption.etag_from_md5(&checksums.md5);
@@ -247,7 +266,6 @@ pub async fn handle_complete_multipart_upload(
req: Request<ReqBody>,
key: &str,
upload_id: &str,
- content_sha256: Option<Hash>,
) -> Result<Response<ResBody>, Error> {
let ReqCtx {
garage,
@@ -259,11 +277,7 @@ pub async fn handle_complete_multipart_upload(
let expected_checksum = request_checksum_value(&req_head.headers)?;
- let body = http_body_util::BodyExt::collect(req_body).await?.to_bytes();
-
- if let Some(content_sha256) = content_sha256 {
- verify_signed_content(content_sha256, &body[..])?;
- }
+ let body = req_body.collect().await?;
let body_xml = roxmltree::Document::parse(std::str::from_utf8(&body)?)?;
let body_list_of_parts = parse_complete_multipart_upload_body(&body_xml)
@@ -429,7 +443,16 @@ pub async fn handle_complete_multipart_upload(
// Send response saying ok we're done
let result = s3_xml::CompleteMultipartUploadResult {
xmlns: (),
- location: None,
+ // FIXME: the location returned is not always correct:
+ // - we always return https, but maybe some people do http
+ // - if root_domain is not specified, a full URL is not returned
+ location: garage
+ .config
+ .s3_api
+ .root_domain
+ .as_ref()
+ .map(|rd| s3_xml::Value(format!("https://{}.{}/{}", bucket_name, rd, key)))
+ .or(Some(s3_xml::Value(format!("/{}/{}", bucket_name, key)))),
bucket: s3_xml::Value(bucket_name.to_string()),
key: s3_xml::Value(key),
etag: s3_xml::Value(format!("\"{}\"", etag)),
@@ -592,3 +615,99 @@ fn parse_complete_multipart_upload_body(
Some(parts)
}
+
+// ====== checksummer ====
+
+#[derive(Default)]
+pub(crate) struct MultipartChecksummer {
+ pub md5: Md5,
+ pub extra: Option<MultipartExtraChecksummer>,
+}
+
+pub(crate) enum MultipartExtraChecksummer {
+ Crc32(Crc32),
+ Crc32c(Crc32c),
+ Sha1(Sha1),
+ Sha256(Sha256),
+}
+
+impl MultipartChecksummer {
+ pub(crate) fn init(algo: Option<ChecksumAlgorithm>) -> Self {
+ Self {
+ md5: Md5::new(),
+ extra: match algo {
+ None => None,
+ Some(ChecksumAlgorithm::Crc32) => {
+ Some(MultipartExtraChecksummer::Crc32(Crc32::new()))
+ }
+ Some(ChecksumAlgorithm::Crc32c) => {
+ Some(MultipartExtraChecksummer::Crc32c(Crc32c::default()))
+ }
+ Some(ChecksumAlgorithm::Sha1) => Some(MultipartExtraChecksummer::Sha1(Sha1::new())),
+ Some(ChecksumAlgorithm::Sha256) => {
+ Some(MultipartExtraChecksummer::Sha256(Sha256::new()))
+ }
+ },
+ }
+ }
+
+ pub(crate) fn update(
+ &mut self,
+ etag: &str,
+ checksum: Option<ChecksumValue>,
+ ) -> Result<(), Error> {
+ self.md5
+ .update(&hex::decode(&etag).ok_or_message("invalid etag hex")?);
+ match (&mut self.extra, checksum) {
+ (None, _) => (),
+ (
+ Some(MultipartExtraChecksummer::Crc32(ref mut crc32)),
+ Some(ChecksumValue::Crc32(x)),
+ ) => {
+ crc32.update(&x);
+ }
+ (
+ Some(MultipartExtraChecksummer::Crc32c(ref mut crc32c)),
+ Some(ChecksumValue::Crc32c(x)),
+ ) => {
+ crc32c.write(&x);
+ }
+ (Some(MultipartExtraChecksummer::Sha1(ref mut sha1)), Some(ChecksumValue::Sha1(x))) => {
+ sha1.update(&x);
+ }
+ (
+ Some(MultipartExtraChecksummer::Sha256(ref mut sha256)),
+ Some(ChecksumValue::Sha256(x)),
+ ) => {
+ sha256.update(&x);
+ }
+ (Some(_), b) => {
+ return Err(Error::internal_error(format!(
+ "part checksum was not computed correctly, got: {:?}",
+ b
+ )))
+ }
+ }
+ Ok(())
+ }
+
+ pub(crate) fn finalize(self) -> (Md5Checksum, Option<ChecksumValue>) {
+ let md5 = self.md5.finalize()[..].try_into().unwrap();
+ let extra = match self.extra {
+ None => None,
+ Some(MultipartExtraChecksummer::Crc32(crc32)) => {
+ Some(ChecksumValue::Crc32(u32::to_be_bytes(crc32.finalize())))
+ }
+ Some(MultipartExtraChecksummer::Crc32c(crc32c)) => Some(ChecksumValue::Crc32c(
+ u32::to_be_bytes(u32::try_from(crc32c.finish()).unwrap()),
+ )),
+ Some(MultipartExtraChecksummer::Sha1(sha1)) => {
+ Some(ChecksumValue::Sha1(sha1.finalize()[..].try_into().unwrap()))
+ }
+ Some(MultipartExtraChecksummer::Sha256(sha256)) => Some(ChecksumValue::Sha256(
+ sha256.finalize()[..].try_into().unwrap(),
+ )),
+ };
+ (md5, extra)
+ }
+}
diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs
index 725f3847..b9bccae6 100644
--- a/src/api/s3/post_object.rs
+++ b/src/api/s3/post_object.rs
@@ -16,15 +16,16 @@ use serde::Deserialize;
use garage_model::garage::Garage;
use garage_model::s3::object_table::*;
-use crate::helpers::*;
-use crate::s3::api_server::ResBody;
-use crate::s3::checksum::*;
-use crate::s3::cors::*;
-use crate::s3::encryption::EncryptionParams;
-use crate::s3::error::*;
-use crate::s3::put::{get_headers, save_stream, ChecksumMode};
-use crate::s3::xml as s3_xml;
-use crate::signature::payload::{verify_v4, Authorization};
+use garage_api_common::cors::*;
+use garage_api_common::helpers::*;
+use garage_api_common::signature::checksum::*;
+use garage_api_common::signature::payload::{verify_v4, Authorization};
+
+use crate::api_server::ResBody;
+use crate::encryption::EncryptionParams;
+use crate::error::*;
+use crate::put::{extract_metadata_headers, save_stream, ChecksumMode};
+use crate::xml as s3_xml;
pub async fn handle_post_object(
garage: Arc<Garage>,
@@ -107,7 +108,8 @@ pub async fn handle_post_object(
let bucket_id = garage
.bucket_helper()
.resolve_bucket(&bucket_name, &api_key)
- .await?;
+ .await
+ .map_err(pass_helper_error)?;
if !api_key.allow_write(&bucket_id) {
return Err(Error::forbidden("Operation is not allowed for this key."));
@@ -214,8 +216,9 @@ pub async fn handle_post_object(
// if we ever start supporting ACLs, we likely want to map "acl" to x-amz-acl" somewhere
// around here to make sure the rest of the machinery takes our acl into account.
- let headers = get_headers(&params)?;
+ let headers = extract_metadata_headers(&params)?;
+ let checksum_algorithm = request_checksum_algorithm(&params)?;
let expected_checksums = ExpectedChecksums {
md5: params
.get("content-md5")
@@ -223,7 +226,9 @@ pub async fn handle_post_object(
.transpose()?
.map(str::to_string),
sha256: None,
- extra: request_checksum_algorithm_value(&params)?,
+ extra: checksum_algorithm
+ .map(|algo| extract_checksum_value(&params, algo))
+ .transpose()?,
};
let meta = ObjectVersionMetaInner {
diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs
index 1e3b1b44..830a7998 100644
--- a/src/api/s3/put.rs
+++ b/src/api/s3/put.rs
@@ -30,11 +30,14 @@ use garage_model::s3::block_ref_table::*;
use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*;
-use crate::helpers::*;
-use crate::s3::api_server::{ReqBody, ResBody};
-use crate::s3::checksum::*;
-use crate::s3::encryption::EncryptionParams;
-use crate::s3::error::*;
+use garage_api_common::helpers::*;
+use garage_api_common::signature::body::StreamingChecksumReceiver;
+use garage_api_common::signature::checksum::*;
+
+use crate::api_server::{ReqBody, ResBody};
+use crate::encryption::EncryptionParams;
+use crate::error::*;
+use crate::website::X_AMZ_WEBSITE_REDIRECT_LOCATION;
const PUT_BLOCKS_MAX_PARALLEL: usize = 3;
@@ -47,6 +50,10 @@ pub(crate) struct SaveStreamResult {
pub(crate) enum ChecksumMode<'a> {
Verify(&'a ExpectedChecksums),
+ VerifyFrom {
+ checksummer: StreamingChecksumReceiver,
+ trailer_algo: Option<ChecksumAlgorithm>,
+ },
Calculate(Option<ChecksumAlgorithm>),
}
@@ -54,10 +61,9 @@ pub async fn handle_put(
ctx: ReqCtx,
req: Request<ReqBody>,
key: &String,
- content_sha256: Option<Hash>,
) -> Result<Response<ResBody>, Error> {
// Retrieve interesting headers from request
- let headers = get_headers(req.headers())?;
+ let headers = extract_metadata_headers(req.headers())?;
debug!("Object headers: {:?}", headers);
let expected_checksums = ExpectedChecksums {
@@ -65,9 +71,10 @@ pub async fn handle_put(
Some(x) => Some(x.to_str()?.to_string()),
None => None,
},
- sha256: content_sha256,
+ sha256: None,
extra: request_checksum_value(req.headers())?,
};
+ let trailer_checksum_algorithm = request_trailer_checksum_algorithm(req.headers())?;
let meta = ObjectVersionMetaInner {
headers,
@@ -77,7 +84,19 @@ pub async fn handle_put(
// Determine whether object should be encrypted, and if so the key
let encryption = EncryptionParams::new_from_headers(&ctx.garage, req.headers())?;
- let stream = body_stream(req.into_body());
+ // The request body is a special ReqBody object (see garage_api_common::signature::body)
+ // which supports calculating checksums while streaming the data.
+ // Before we start streaming, we configure it to calculate all the checksums we need.
+ let mut req_body = req.into_body();
+ req_body.add_expected_checksums(expected_checksums.clone());
+ if !encryption.is_encrypted() {
+ // For non-encrypted objects, we need to compute the md5sum in all cases
+ // (even if content-md5 is not set), because it is used as the object etag
+ req_body.add_md5();
+ }
+
+ let (stream, checksummer) = req_body.streaming_with_checksums();
+ let stream = stream.map_err(Error::from);
let res = save_stream(
&ctx,
@@ -85,7 +104,10 @@ pub async fn handle_put(
encryption,
stream,
key,
- ChecksumMode::Verify(&expected_checksums),
+ ChecksumMode::VerifyFrom {
+ checksummer,
+ trailer_algo: trailer_checksum_algorithm,
+ },
)
.await?;
@@ -121,10 +143,15 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
let version_uuid = gen_uuid();
let version_timestamp = next_timestamp(existing_object.as_ref());
- let mut checksummer = match checksum_mode {
+ let mut checksummer = match &checksum_mode {
ChecksumMode::Verify(expected) => Checksummer::init(expected, !encryption.is_encrypted()),
ChecksumMode::Calculate(algo) => {
- Checksummer::init(&Default::default(), !encryption.is_encrypted()).add(algo)
+ Checksummer::init(&Default::default(), !encryption.is_encrypted()).add(*algo)
+ }
+ ChecksumMode::VerifyFrom { .. } => {
+ // Checksums are calculated by the garage_api_common::signature module
+ // so here we can just have an empty checksummer that does nothing
+ Checksummer::new()
}
};
@@ -132,7 +159,7 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
// as "inline data". We can then return immediately.
if first_block.len() < INLINE_THRESHOLD {
checksummer.update(&first_block);
- let checksums = checksummer.finalize();
+ let mut checksums = checksummer.finalize();
match checksum_mode {
ChecksumMode::Verify(expected) => {
@@ -141,6 +168,18 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
ChecksumMode::Calculate(algo) => {
meta.checksum = checksums.extract(algo);
}
+ ChecksumMode::VerifyFrom {
+ checksummer,
+ trailer_algo,
+ } => {
+ drop(chunker);
+ checksums = checksummer
+ .await
+ .ok_or_internal_error("checksum calculation")??;
+ if let Some(algo) = trailer_algo {
+ meta.checksum = checksums.extract(Some(algo));
+ }
+ }
};
let size = first_block.len() as u64;
@@ -212,13 +251,13 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
garage.version_table.insert(&version).await?;
// Transfer data
- let (total_size, checksums, first_block_hash) = read_and_put_blocks(
+ let (total_size, mut checksums, first_block_hash) = read_and_put_blocks(
ctx,
&version,
encryption,
1,
first_block,
- &mut chunker,
+ chunker,
checksummer,
)
.await?;
@@ -231,6 +270,17 @@ pub(crate) async fn save_stream<S: Stream<Item = Result<Bytes, Error>> + Unpin>(
ChecksumMode::Calculate(algo) => {
meta.checksum = checksums.extract(algo);
}
+ ChecksumMode::VerifyFrom {
+ checksummer,
+ trailer_algo,
+ } => {
+ checksums = checksummer
+ .await
+ .ok_or_internal_error("checksum calculation")??;
+ if let Some(algo) = trailer_algo {
+ meta.checksum = checksums.extract(Some(algo));
+ }
+ }
};
// Verify quotas are respsected
@@ -331,7 +381,7 @@ pub(crate) async fn read_and_put_blocks<S: Stream<Item = Result<Bytes, Error>> +
encryption: EncryptionParams,
part_number: u64,
first_block: Bytes,
- chunker: &mut StreamChunker<S>,
+ mut chunker: StreamChunker<S>,
checksummer: Checksummer,
) -> Result<(u64, Checksums, Hash), Error> {
let tracer = opentelemetry::global::tracer("garage");
@@ -600,7 +650,9 @@ impl Drop for InterruptedCleanup {
// ============ helpers ============
-pub(crate) fn get_headers(headers: &HeaderMap<HeaderValue>) -> Result<HeaderList, Error> {
+pub(crate) fn extract_metadata_headers(
+ headers: &HeaderMap<HeaderValue>,
+) -> Result<HeaderList, Error> {
let mut ret = Vec::new();
// Preserve standard headers
@@ -622,10 +674,22 @@ pub(crate) fn get_headers(headers: &HeaderMap<HeaderValue>) -> Result<HeaderList
for (name, value) in headers.iter() {
if name.as_str().starts_with("x-amz-meta-") {
ret.push((
- name.to_string(),
+ name.as_str().to_ascii_lowercase(),
std::str::from_utf8(value.as_bytes())?.to_string(),
));
}
+ if name == X_AMZ_WEBSITE_REDIRECT_LOCATION {
+ let value = std::str::from_utf8(value.as_bytes())?.to_string();
+ if !(value.starts_with("/")
+ || value.starts_with("http://")
+ || value.starts_with("https://"))
+ {
+ return Err(Error::bad_request(format!(
+ "Invalid {X_AMZ_WEBSITE_REDIRECT_LOCATION} header",
+ )));
+ }
+ ret.push((X_AMZ_WEBSITE_REDIRECT_LOCATION.to_string(), value));
+ }
}
Ok(ret)
diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs
index e7ac1d77..e3f58490 100644
--- a/src/api/s3/router.rs
+++ b/src/api/s3/router.rs
@@ -3,9 +3,10 @@ use std::borrow::Cow;
use hyper::header::HeaderValue;
use hyper::{HeaderMap, Method, Request};
-use crate::helpers::Authorization;
-use crate::router_macros::{generateQueryParameters, router_match};
-use crate::s3::error::*;
+use garage_api_common::helpers::Authorization;
+use garage_api_common::router_macros::{generateQueryParameters, router_match};
+
+use crate::error::*;
router_match! {@func
@@ -351,6 +352,18 @@ impl Endpoint {
_ => return Err(Error::bad_request("Unknown method")),
};
+ if let Some(x_id) = query.x_id.take() {
+ if x_id != res.name() {
+ // I think AWS ignores the x-id parameter.
+ // Let's make this at least be a warnin to help debugging.
+ warn!(
+ "x-id ({}) does not match parsed endpoint ({})",
+ x_id,
+ res.name()
+ );
+ }
+ }
+
if let Some(message) = query.nonempty_message() {
debug!("Unused query parameter: {}", message)
}
@@ -695,7 +708,8 @@ generateQueryParameters! {
"uploadId" => upload_id,
"upload-id-marker" => upload_id_marker,
"versionId" => version_id,
- "version-id-marker" => version_id_marker
+ "version-id-marker" => version_id_marker,
+ "x-id" => x_id
]
}
diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs
index fa36bc32..03cc01d8 100644
--- a/src/api/s3/website.rs
+++ b/src/api/s3/website.rs
@@ -1,17 +1,18 @@
use quick_xml::de::from_reader;
-use http_body_util::BodyExt;
-use hyper::{Request, Response, StatusCode};
+use hyper::{header::HeaderName, Request, Response, StatusCode};
use serde::{Deserialize, Serialize};
-use crate::helpers::*;
-use crate::s3::api_server::{ReqBody, ResBody};
-use crate::s3::error::*;
-use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
-use crate::signature::verify_signed_content;
-
use garage_model::bucket_table::*;
-use garage_util::data::*;
+
+use garage_api_common::helpers::*;
+
+use crate::api_server::{ReqBody, ResBody};
+use crate::error::*;
+use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
+
+pub const X_AMZ_WEBSITE_REDIRECT_LOCATION: HeaderName =
+ HeaderName::from_static("x-amz-website-redirect-location");
pub async fn handle_get_website(ctx: ReqCtx) -> Result<Response<ResBody>, Error> {
let ReqCtx { bucket_params, .. } = ctx;
@@ -60,7 +61,6 @@ pub async fn handle_delete_website(ctx: ReqCtx) -> Result<Response<ResBody>, Err
pub async fn handle_put_website(
ctx: ReqCtx,
req: Request<ReqBody>,
- content_sha256: Option<Hash>,
) -> Result<Response<ResBody>, Error> {
let ReqCtx {
garage,
@@ -69,11 +69,7 @@ pub async fn handle_put_website(
..
} = ctx;
- let body = BodyExt::collect(req.into_body()).await?.to_bytes();
-
- if let Some(content_sha256) = content_sha256 {
- verify_signed_content(content_sha256, &body[..])?;
- }
+ let body = req.into_body().collect().await?;
let conf: WebsiteConfiguration = from_reader(&body as &[u8])?;
conf.validate()?;
diff --git a/src/api/s3/xml.rs b/src/api/s3/xml.rs
index 1e569ade..e8af3ec0 100644
--- a/src/api/s3/xml.rs
+++ b/src/api/s3/xml.rs
@@ -1,7 +1,7 @@
use quick_xml::se::to_string;
use serde::{Deserialize, Serialize, Serializer};
-use crate::s3::error::Error as ApiError;
+use crate::error::Error as ApiError;
pub fn to_xml_with_header<T: Serialize>(x: &T) -> Result<String, ApiError> {
let mut xml = r#"<?xml version="1.0" encoding="UTF-8"?>"#.to_string();
generated by cgit v1.2.3 (git 2.25.1) at 2025-02-24 00:03:28 +0000