From 9fa20d45bebab2a3f66b9721c3643dbd607d944d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 18:18:04 +0100 Subject: wip: split garage_api into garage_api_{common,s3,k2v,admin} --- src/api/s3/Cargo.toml | 71 +++++++++++++++++++++++++++++++++++++++++++++++ src/api/s3/api_server.rs | 40 +++++++++++++------------- src/api/s3/bucket.rs | 12 ++++---- src/api/s3/checksum.rs | 2 +- src/api/s3/copy.rs | 18 ++++++------ src/api/s3/cors.rs | 12 ++++---- src/api/s3/delete.rs | 12 ++++---- src/api/s3/encryption.rs | 6 ++-- src/api/s3/error.rs | 19 ++++++++----- src/api/s3/get.rs | 10 +++---- src/api/s3/lib.rs | 22 +++++++++++++++ src/api/s3/lifecycle.rs | 10 +++---- src/api/s3/list.rs | 14 +++++----- src/api/s3/mod.rs | 19 ------------- src/api/s3/multipart.rs | 16 +++++------ src/api/s3/post_object.rs | 18 ++++++------ src/api/s3/put.rs | 10 +++---- src/api/s3/router.rs | 6 ++-- src/api/s3/website.rs | 10 +++---- src/api/s3/xml.rs | 2 +- 20 files changed, 204 insertions(+), 125 deletions(-) create mode 100644 src/api/s3/Cargo.toml create mode 100644 src/api/s3/lib.rs delete mode 100644 src/api/s3/mod.rs (limited to 'src/api/s3') diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml new file mode 100644 index 00000000..c610b43a --- /dev/null +++ b/src/api/s3/Cargo.toml @@ -0,0 +1,71 @@ +[package] +name = "garage_api_s3" +version = "1.0.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "S3 API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_block.workspace = true +garage_net.workspace = true +garage_util.workspace = true +garage_rpc.workspace = true +garage_api_common.workspace = true + +aes-gcm.workspace = true +argon2.workspace = true +async-compression.workspace = true +async-trait.workspace = true +base64.workspace = true +bytes.workspace = true +chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true +crypto-common.workspace = true +err-derive.workspace = true +hex.workspace = true +hmac.workspace = true +idna.workspace = true +tracing.workspace = true +md-5.workspace = true +nom.workspace = true +pin-project.workspace = true +sha1.workspace = true +sha2.workspace = true + +futures.workspace = true +futures-util.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tokio-util.workspace = true + +form_urlencoded.workspace = true +http.workspace = true +httpdate.workspace = true +http-range.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +hyper-util.workspace = true +multer.workspace = true +percent-encoding.workspace = true +roxmltree.workspace = true +url.workspace = true + +serde.workspace = true +serde_bytes.workspace = true +serde_json.workspace = true +quick-xml.workspace = true + +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index f9dafa10..d24f6a0c 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -14,26 +14,26 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; use garage_model::key_table::Key; -use crate::generic_server::*; -use crate::s3::error::*; - -use crate::signature::verify_request; - -use crate::helpers::*; -use crate::s3::bucket::*; -use crate::s3::copy::*; -use crate::s3::cors::*; -use crate::s3::delete::*; -use crate::s3::get::*; -use crate::s3::lifecycle::*; -use crate::s3::list::*; -use crate::s3::multipart::*; -use crate::s3::post_object::handle_post_object; -use crate::s3::put::*; -use crate::s3::router::Endpoint; -use crate::s3::website::*; - -pub use crate::signature::streaming::ReqBody; +use crate::error::*; +use garage_api_common::generic_server::*; + +use garage_api_common::signature::verify_request; + +use crate::bucket::*; +use crate::copy::*; +use crate::cors::*; +use crate::delete::*; +use crate::get::*; +use crate::lifecycle::*; +use crate::list::*; +use crate::multipart::*; +use crate::post_object::handle_post_object; +use crate::put::*; +use crate::router::Endpoint; +use crate::website::*; +use garage_api_common::helpers::*; + +pub use garage_api_common::signature::streaming::ReqBody; pub type ResBody = BoxBody; pub struct S3ApiServer { diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 6a12aa9c..09c5742b 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -13,12 +13,12 @@ use garage_util::crdt::*; use garage_util::data::*; use garage_util::time::*; -use crate::common_error::CommonError; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml as s3_xml; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result, Error> { let ReqCtx { garage, .. } = ctx; diff --git a/src/api/s3/checksum.rs b/src/api/s3/checksum.rs index c7527163..02fb55ec 100644 --- a/src/api/s3/checksum.rs +++ b/src/api/s3/checksum.rs @@ -15,7 +15,7 @@ use garage_util::error::OkOrMessage; use garage_model::s3::object_table::*; -use crate::s3::error::*; +use crate::error::*; pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-checksum-algorithm"); diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index b67ace88..1a474fd0 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -20,15 +20,15 @@ use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::checksum::*; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; -use crate::s3::get::full_object_byte_stream; -use crate::s3::multipart; -use crate::s3::put::{get_headers, save_stream, ChecksumMode, SaveStreamResult}; -use crate::s3::xml::{self as s3_xml, xmlns_tag}; +use crate::api_server::{ReqBody, ResBody}; +use crate::checksum::*; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::get::full_object_byte_stream; +use crate::multipart; +use crate::put::{get_headers, save_stream, ChecksumMode, SaveStreamResult}; +use crate::xml::{self as s3_xml, xmlns_tag}; +use garage_api_common::helpers::*; // -------- CopyObject --------- diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index 32dcc0d5..ae8352c3 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -15,12 +15,12 @@ use http_body_util::BodyExt; use serde::{Deserialize, Serialize}; -use crate::common_error::{helper_error_as_internal, CommonError}; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_api_common::common_error::{helper_error_as_internal, CommonError}; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule}; use garage_model::garage::Garage; diff --git a/src/api/s3/delete.rs b/src/api/s3/delete.rs index 57f6f948..1711a9b4 100644 --- a/src/api/s3/delete.rs +++ b/src/api/s3/delete.rs @@ -5,12 +5,12 @@ use garage_util::data::*; use garage_model::s3::object_table::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::put::next_timestamp; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::put::next_timestamp; +use crate::xml as s3_xml; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> { let ReqCtx { diff --git a/src/api/s3/encryption.rs b/src/api/s3/encryption.rs index 2e6ed65c..c54d487b 100644 --- a/src/api/s3/encryption.rs +++ b/src/api/s3/encryption.rs @@ -28,9 +28,9 @@ use garage_util::migrate::Migrate; use garage_model::garage::Garage; use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner}; -use crate::common_error::*; -use crate::s3::checksum::Md5Checksum; -use crate::s3::error::Error; +use crate::checksum::Md5Checksum; +use crate::error::Error; +use garage_api_common::common_error::*; const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm"); diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 22d2fe14..77dc07c8 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -6,13 +6,18 @@ use hyper::{HeaderMap, StatusCode}; use garage_model::helper::error::Error as HelperError; -pub(crate) use crate::common_error::pass_helper_error; -use crate::common_error::{helper_error_as_internal, CommonError}; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::helpers::*; -use crate::s3::xml as s3_xml; -use crate::signature::error::Error as SignatureError; +pub(crate) use garage_api_common::common_error::pass_helper_error; + +use garage_api_common::common_error::{helper_error_as_internal, CommonError}; + +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; + +use crate::xml as s3_xml; +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; +use garage_api_common::signature::error::Error as SignatureError; /// Errors of this crate #[derive(Debug, Error)] diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index f61aae11..c4cd9d48 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -25,11 +25,11 @@ use garage_model::garage::Garage; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::*; -use crate::s3::api_server::ResBody; -use crate::s3::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; +use crate::api_server::ResBody; +use crate::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use garage_api_common::helpers::*; const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count"; diff --git a/src/api/s3/lib.rs b/src/api/s3/lib.rs new file mode 100644 index 00000000..fd99b443 --- /dev/null +++ b/src/api/s3/lib.rs @@ -0,0 +1,22 @@ +#[macro_use] +extern crate tracing; + +pub mod api_server; +pub mod error; + +mod bucket; +mod copy; +pub mod cors; +mod delete; +pub mod get; +mod lifecycle; +mod list; +mod multipart; +mod post_object; +mod put; +mod website; + +mod checksum; +mod encryption; +mod router; +pub mod xml; diff --git a/src/api/s3/lifecycle.rs b/src/api/s3/lifecycle.rs index 7eb1c2cb..da211585 100644 --- a/src/api/s3/lifecycle.rs +++ b/src/api/s3/lifecycle.rs @@ -5,11 +5,11 @@ use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; use garage_model::bucket_table::{ parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration, diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index 68d6cbe6..de808c32 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -13,13 +13,13 @@ use garage_model::s3::object_table::*; use garage_table::EnumerationOrder; -use crate::encoding::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; -use crate::s3::multipart as s3_multipart; -use crate::s3::xml as s3_xml; +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::multipart as s3_multipart; +use crate::xml as s3_xml; +use garage_api_common::encoding::*; +use garage_api_common::helpers::*; const DUMMY_NAME: &str = "Dummy Key"; const DUMMY_KEY: &str = "GKDummyKey"; diff --git a/src/api/s3/mod.rs b/src/api/s3/mod.rs deleted file mode 100644 index b9bb1a6f..00000000 --- a/src/api/s3/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -pub mod api_server; -pub mod error; - -mod bucket; -mod copy; -pub mod cors; -mod delete; -pub mod get; -mod lifecycle; -mod list; -mod multipart; -mod post_object; -mod put; -mod website; - -mod checksum; -mod encryption; -mod router; -pub mod xml; diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs index 3db3e8aa..047ed06a 100644 --- a/src/api/s3/multipart.rs +++ b/src/api/s3/multipart.rs @@ -15,14 +15,14 @@ use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::checksum::*; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; -use crate::s3::put::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::checksum::*; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::put::*; +use crate::xml as s3_xml; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; // ---- diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 5279ec6a..6416c523 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -16,15 +16,15 @@ use serde::Deserialize; use garage_model::garage::Garage; use garage_model::s3::object_table::*; -use crate::helpers::*; -use crate::s3::api_server::ResBody; -use crate::s3::checksum::*; -use crate::s3::cors::*; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; -use crate::s3::put::{get_headers, save_stream, ChecksumMode}; -use crate::s3::xml as s3_xml; -use crate::signature::payload::{verify_v4, Authorization}; +use crate::api_server::ResBody; +use crate::checksum::*; +use crate::cors::*; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::put::{get_headers, save_stream, ChecksumMode}; +use crate::xml as s3_xml; +use garage_api_common::helpers::*; +use garage_api_common::signature::payload::{verify_v4, Authorization}; pub async fn handle_post_object( garage: Arc, diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index bfb0dc9b..47dcb8f7 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -30,11 +30,11 @@ use garage_model::s3::block_ref_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::checksum::*; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; +use crate::api_server::{ReqBody, ResBody}; +use crate::checksum::*; +use crate::encryption::EncryptionParams; +use crate::error::*; +use garage_api_common::helpers::*; const PUT_BLOCKS_MAX_PARALLEL: usize = 3; diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index e7ac1d77..94951e80 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -3,9 +3,9 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; -use crate::helpers::Authorization; -use crate::router_macros::{generateQueryParameters, router_match}; -use crate::s3::error::*; +use crate::error::*; +use garage_api_common::helpers::Authorization; +use garage_api_common::router_macros::{generateQueryParameters, router_match}; router_match! {@func diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index fa36bc32..46decccf 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -4,11 +4,11 @@ use http_body_util::BodyExt; use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; use garage_model::bucket_table::*; use garage_util::data::*; diff --git a/src/api/s3/xml.rs b/src/api/s3/xml.rs index 1e569ade..e8af3ec0 100644 --- a/src/api/s3/xml.rs +++ b/src/api/s3/xml.rs @@ -1,7 +1,7 @@ use quick_xml::se::to_string; use serde::{Deserialize, Serialize, Serializer}; -use crate::s3::error::Error as ApiError; +use crate::error::Error as ApiError; pub fn to_xml_with_header(x: &T) -> Result { let mut xml = r#""#.to_string(); -- cgit v1.2.3 From 84f1db91c4e53a8d0c037fd01adb695fd9400ed5 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 18:34:57 +0100 Subject: fix things up --- src/api/s3/api_server.rs | 7 +++---- src/api/s3/bucket.rs | 7 ++++--- src/api/s3/copy.rs | 3 ++- src/api/s3/cors.rs | 13 +++++++------ src/api/s3/delete.rs | 5 +++-- src/api/s3/encryption.rs | 3 ++- src/api/s3/error.rs | 20 +++++++------------- src/api/s3/get.rs | 3 ++- src/api/s3/lifecycle.rs | 5 +++-- src/api/s3/list.rs | 5 +++-- src/api/s3/multipart.rs | 5 +++-- src/api/s3/post_object.rs | 5 +++-- src/api/s3/put.rs | 3 ++- src/api/s3/router.rs | 3 ++- src/api/s3/website.rs | 11 ++++++----- 15 files changed, 52 insertions(+), 46 deletions(-) (limited to 'src/api/s3') diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index d24f6a0c..a0dbf52c 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -14,15 +14,15 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; use garage_model::key_table::Key; -use crate::error::*; use garage_api_common::generic_server::*; - +use garage_api_common::helpers::*; use garage_api_common::signature::verify_request; use crate::bucket::*; use crate::copy::*; use crate::cors::*; use crate::delete::*; +use crate::error::*; use crate::get::*; use crate::lifecycle::*; use crate::list::*; @@ -31,7 +31,6 @@ use crate::post_object::handle_post_object; use crate::put::*; use crate::router::Endpoint; use crate::website::*; -use garage_api_common::helpers::*; pub use garage_api_common::signature::streaming::ReqBody; pub type ResBody = BoxBody; @@ -40,7 +39,7 @@ pub struct S3ApiServer { garage: Arc, } -pub(crate) struct S3ApiEndpoint { +pub struct S3ApiEndpoint { bucket_name: Option, endpoint: Endpoint, } diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 09c5742b..0a192ba6 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -13,13 +13,14 @@ use garage_util::crdt::*; use garage_util::data::*; use garage_util::time::*; -use crate::api_server::{ReqBody, ResBody}; -use crate::error::*; -use crate::xml as s3_xml; use garage_api_common::common_error::CommonError; use garage_api_common::helpers::*; use garage_api_common::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml as s3_xml; + pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result, Error> { let ReqCtx { garage, .. } = ctx; let loc = s3_xml::LocationConstraint { diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 1a474fd0..e4992a18 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -20,6 +20,8 @@ use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use garage_api_common::helpers::*; + use crate::api_server::{ReqBody, ResBody}; use crate::checksum::*; use crate::encryption::EncryptionParams; @@ -28,7 +30,6 @@ use crate::get::full_object_byte_stream; use crate::multipart; use crate::put::{get_headers, save_stream, ChecksumMode, SaveStreamResult}; use crate::xml::{self as s3_xml, xmlns_tag}; -use garage_api_common::helpers::*; // -------- CopyObject --------- diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index ae8352c3..4bd81e32 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -15,16 +15,17 @@ use http_body_util::BodyExt; use serde::{Deserialize, Serialize}; -use crate::api_server::{ReqBody, ResBody}; -use crate::error::*; -use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule}; +use garage_model::garage::Garage; +use garage_util::data::*; + use garage_api_common::common_error::{helper_error_as_internal, CommonError}; use garage_api_common::helpers::*; use garage_api_common::signature::verify_signed_content; -use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule}; -use garage_model::garage::Garage; -use garage_util::data::*; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; pub async fn handle_get_cors(ctx: ReqCtx) -> Result, Error> { let ReqCtx { bucket_params, .. } = ctx; diff --git a/src/api/s3/delete.rs b/src/api/s3/delete.rs index 1711a9b4..b799e67a 100644 --- a/src/api/s3/delete.rs +++ b/src/api/s3/delete.rs @@ -5,12 +5,13 @@ use garage_util::data::*; use garage_model::s3::object_table::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; + use crate::api_server::{ReqBody, ResBody}; use crate::error::*; use crate::put::next_timestamp; use crate::xml as s3_xml; -use garage_api_common::helpers::*; -use garage_api_common::signature::verify_signed_content; async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> { let ReqCtx { diff --git a/src/api/s3/encryption.rs b/src/api/s3/encryption.rs index c54d487b..b38d7792 100644 --- a/src/api/s3/encryption.rs +++ b/src/api/s3/encryption.rs @@ -28,9 +28,10 @@ use garage_util::migrate::Migrate; use garage_model::garage::Garage; use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner}; +use garage_api_common::common_error::*; + use crate::checksum::Md5Checksum; use crate::error::Error; -use garage_api_common::common_error::*; const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm"); diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 77dc07c8..1bb8909c 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -8,23 +8,26 @@ use garage_model::helper::error::Error as HelperError; pub(crate) use garage_api_common::common_error::pass_helper_error; -use garage_api_common::common_error::{helper_error_as_internal, CommonError}; +use garage_api_common::common_error::{ + commonErrorDerivative, helper_error_as_internal, CommonError, +}; pub use garage_api_common::common_error::{ CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, }; -use crate::xml as s3_xml; use garage_api_common::generic_server::ApiError; use garage_api_common::helpers::*; use garage_api_common::signature::error::Error as SignatureError; +use crate::xml as s3_xml; + /// Errors of this crate #[derive(Debug, Error)] pub enum Error { #[error(display = "{}", _0)] /// Error from common error - Common(CommonError), + Common(#[error(source)] CommonError), // Category: cannot process /// Authorization Header Malformed @@ -86,14 +89,7 @@ pub enum Error { NotImplemented(String), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) - } -} +commonErrorDerivative!(Error); // Helper errors are always passed as internal errors by default. // To pass the specific error code back to the client, use `pass_helper_error`. @@ -103,8 +99,6 @@ impl From for Error { } } -impl CommonErrorDerivative for Error {} - impl From for Error { fn from(err: roxmltree::Error) -> Self { Self::InvalidXml(format!("{}", err)) diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index c4cd9d48..c2393a51 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -25,11 +25,12 @@ use garage_model::garage::Garage; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use garage_api_common::helpers::*; + use crate::api_server::ResBody; use crate::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; use crate::encryption::EncryptionParams; use crate::error::*; -use garage_api_common::helpers::*; const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count"; diff --git a/src/api/s3/lifecycle.rs b/src/api/s3/lifecycle.rs index da211585..c35047ed 100644 --- a/src/api/s3/lifecycle.rs +++ b/src/api/s3/lifecycle.rs @@ -5,11 +5,12 @@ use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; + use crate::api_server::{ReqBody, ResBody}; use crate::error::*; use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use garage_api_common::helpers::*; -use garage_api_common::signature::verify_signed_content; use garage_model::bucket_table::{ parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration, diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index de808c32..a5cc03b0 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -13,13 +13,14 @@ use garage_model::s3::object_table::*; use garage_table::EnumerationOrder; +use garage_api_common::encoding::*; +use garage_api_common::helpers::*; + use crate::api_server::{ReqBody, ResBody}; use crate::encryption::EncryptionParams; use crate::error::*; use crate::multipart as s3_multipart; use crate::xml as s3_xml; -use garage_api_common::encoding::*; -use garage_api_common::helpers::*; const DUMMY_NAME: &str = "Dummy Key"; const DUMMY_KEY: &str = "GKDummyKey"; diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs index 047ed06a..fe39fc93 100644 --- a/src/api/s3/multipart.rs +++ b/src/api/s3/multipart.rs @@ -15,14 +15,15 @@ use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; + use crate::api_server::{ReqBody, ResBody}; use crate::checksum::*; use crate::encryption::EncryptionParams; use crate::error::*; use crate::put::*; use crate::xml as s3_xml; -use garage_api_common::helpers::*; -use garage_api_common::signature::verify_signed_content; // ---- diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 6416c523..2bcabf1d 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -16,6 +16,9 @@ use serde::Deserialize; use garage_model::garage::Garage; use garage_model::s3::object_table::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::payload::{verify_v4, Authorization}; + use crate::api_server::ResBody; use crate::checksum::*; use crate::cors::*; @@ -23,8 +26,6 @@ use crate::encryption::EncryptionParams; use crate::error::*; use crate::put::{get_headers, save_stream, ChecksumMode}; use crate::xml as s3_xml; -use garage_api_common::helpers::*; -use garage_api_common::signature::payload::{verify_v4, Authorization}; pub async fn handle_post_object( garage: Arc, diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 47dcb8f7..530b4e7b 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -30,11 +30,12 @@ use garage_model::s3::block_ref_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use garage_api_common::helpers::*; + use crate::api_server::{ReqBody, ResBody}; use crate::checksum::*; use crate::encryption::EncryptionParams; use crate::error::*; -use garage_api_common::helpers::*; const PUT_BLOCKS_MAX_PARALLEL: usize = 3; diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index 94951e80..9de84b2b 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -3,10 +3,11 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; -use crate::error::*; use garage_api_common::helpers::Authorization; use garage_api_common::router_macros::{generateQueryParameters, router_match}; +use crate::error::*; + router_match! {@func /// List of all S3 API endpoints. diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index 46decccf..b55bb345 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -4,14 +4,15 @@ use http_body_util::BodyExt; use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::api_server::{ReqBody, ResBody}; -use crate::error::*; -use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_model::bucket_table::*; +use garage_util::data::*; + use garage_api_common::helpers::*; use garage_api_common::signature::verify_signed_content; -use garage_model::bucket_table::*; -use garage_util::data::*; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; pub async fn handle_get_website(ctx: ReqCtx) -> Result, Error> { let ReqCtx { bucket_params, .. } = ctx; -- cgit v1.2.3 From afa28706e5566737376f8448bcc548f780f0f57f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 18:42:14 +0100 Subject: split s3/cors.rs into also common/cors.rs --- src/api/s3/api_server.rs | 1 + src/api/s3/cors.rs | 170 +--------------------------------------------- src/api/s3/post_object.rs | 2 +- 3 files changed, 4 insertions(+), 169 deletions(-) (limited to 'src/api/s3') diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index a0dbf52c..ed71b108 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -14,6 +14,7 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; use garage_model::key_table::Key; +use garage_api_common::cors::*; use garage_api_common::generic_server::*; use garage_api_common::helpers::*; use garage_api_common::signature::verify_request; diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index 4bd81e32..625b84db 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -1,25 +1,14 @@ -use std::sync::Arc; - use quick_xml::de::from_reader; -use http::header::{ - ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, - ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, -}; -use hyper::{ - body::Body, body::Incoming as IncomingBody, header::HeaderName, Method, Request, Response, - StatusCode, -}; +use hyper::{header::HeaderName, Method, Request, Response, StatusCode}; use http_body_util::BodyExt; use serde::{Deserialize, Serialize}; -use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule}; -use garage_model::garage::Garage; +use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule}; use garage_util::data::*; -use garage_api_common::common_error::{helper_error_as_internal, CommonError}; use garage_api_common::helpers::*; use garage_api_common::signature::verify_signed_content; @@ -101,161 +90,6 @@ pub async fn handle_put_cors( .body(empty_body())?) } -pub async fn handle_options_api( - garage: Arc, - req: &Request, - bucket_name: Option, -) -> Result, CommonError> { - // FIXME: CORS rules of buckets with local aliases are - // not taken into account. - - // If the bucket name is a global bucket name, - // we try to apply the CORS rules of that bucket. - // If a user has a local bucket name that has - // the same name, its CORS rules won't be applied - // and will be shadowed by the rules of the globally - // existing bucket (but this is inevitable because - // OPTIONS calls are not auhtenticated). - if let Some(bn) = bucket_name { - let helper = garage.bucket_helper(); - let bucket_id = helper - .resolve_global_bucket_name(&bn) - .await - .map_err(helper_error_as_internal)?; - if let Some(id) = bucket_id { - let bucket = garage - .bucket_helper() - .get_existing_bucket(id) - .await - .map_err(helper_error_as_internal)?; - let bucket_params = bucket.state.into_option().unwrap(); - handle_options_for_bucket(req, &bucket_params) - } else { - // If there is a bucket name in the request, but that name - // does not correspond to a global alias for a bucket, - // then it's either a non-existing bucket or a local bucket. - // We have no way of knowing, because the request is not - // authenticated and thus we can't resolve local aliases. - // We take the permissive approach of allowing everything, - // because we don't want to prevent web apps that use - // local bucket names from making API calls. - Ok(Response::builder() - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(ACCESS_CONTROL_ALLOW_METHODS, "*") - .status(StatusCode::OK) - .body(EmptyBody::new())?) - } - } else { - // If there is no bucket name in the request, - // we are doing a ListBuckets call, which we want to allow - // for all origins. - Ok(Response::builder() - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(ACCESS_CONTROL_ALLOW_METHODS, "GET") - .status(StatusCode::OK) - .body(EmptyBody::new())?) - } -} - -pub fn handle_options_for_bucket( - req: &Request, - bucket_params: &BucketParams, -) -> Result, CommonError> { - let origin = req - .headers() - .get("Origin") - .ok_or_bad_request("Missing Origin header")? - .to_str()?; - let request_method = req - .headers() - .get(ACCESS_CONTROL_REQUEST_METHOD) - .ok_or_bad_request("Missing Access-Control-Request-Method header")? - .to_str()?; - let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { - Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), - None => vec![], - }; - - if let Some(cors_config) = bucket_params.cors_config.get() { - let matching_rule = cors_config - .iter() - .find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter())); - if let Some(rule) = matching_rule { - let mut resp = Response::builder() - .status(StatusCode::OK) - .body(EmptyBody::new())?; - add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?; - return Ok(resp); - } - } - - Err(CommonError::Forbidden( - "This CORS request is not allowed.".into(), - )) -} - -pub fn find_matching_cors_rule<'a>( - bucket_params: &'a BucketParams, - req: &Request, -) -> Result, Error> { - if let Some(cors_config) = bucket_params.cors_config.get() { - if let Some(origin) = req.headers().get("Origin") { - let origin = origin.to_str()?; - let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { - Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), - None => vec![], - }; - return Ok(cors_config.iter().find(|rule| { - cors_rule_matches(rule, origin, req.method().as_ref(), request_headers.iter()) - })); - } - } - Ok(None) -} - -fn cors_rule_matches<'a, HI, S>( - rule: &GarageCorsRule, - origin: &'a str, - method: &'a str, - mut request_headers: HI, -) -> bool -where - HI: Iterator, - S: AsRef, -{ - rule.allow_origins.iter().any(|x| x == "*" || x == origin) - && rule.allow_methods.iter().any(|x| x == "*" || x == method) - && request_headers.all(|h| { - rule.allow_headers - .iter() - .any(|x| x == "*" || x == h.as_ref()) - }) -} - -pub fn add_cors_headers( - resp: &mut Response, - rule: &GarageCorsRule, -) -> Result<(), http::header::InvalidHeaderValue> { - let h = resp.headers_mut(); - h.insert( - ACCESS_CONTROL_ALLOW_ORIGIN, - rule.allow_origins.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_ALLOW_METHODS, - rule.allow_methods.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_ALLOW_HEADERS, - rule.allow_headers.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_EXPOSE_HEADERS, - rule.expose_headers.join(", ").parse()?, - ); - Ok(()) -} - // ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ---- #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 2bcabf1d..6c0e73d4 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -16,12 +16,12 @@ use serde::Deserialize; use garage_model::garage::Garage; use garage_model::s3::object_table::*; +use garage_api_common::cors::*; use garage_api_common::helpers::*; use garage_api_common::signature::payload::{verify_v4, Authorization}; use crate::api_server::ResBody; use crate::checksum::*; -use crate::cors::*; use crate::encryption::EncryptionParams; use crate::error::*; use crate::put::{get_headers, save_stream, ChecksumMode}; -- cgit v1.2.3 From 4563313f87af4b7be26152164a5ce09a451da0d9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 18:47:30 +0100 Subject: use cargo-shear to remove many unused dependencies between crates --- src/api/s3/Cargo.toml | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'src/api/s3') diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml index c610b43a..a1751c9f 100644 --- a/src/api/s3/Cargo.toml +++ b/src/api/s3/Cargo.toml @@ -23,7 +23,6 @@ garage_rpc.workspace = true garage_api_common.workspace = true aes-gcm.workspace = true -argon2.workspace = true async-compression.workspace = true async-trait.workspace = true base64.workspace = true @@ -31,20 +30,15 @@ bytes.workspace = true chrono.workspace = true crc32fast.workspace = true crc32c.workspace = true -crypto-common.workspace = true err-derive.workspace = true hex.workspace = true -hmac.workspace = true -idna.workspace = true tracing.workspace = true md-5.workspace = true -nom.workspace = true pin-project.workspace = true sha1.workspace = true sha2.workspace = true futures.workspace = true -futures-util.workspace = true tokio.workspace = true tokio-stream.workspace = true tokio-util.workspace = true @@ -55,17 +49,13 @@ httpdate.workspace = true http-range.workspace = true http-body-util.workspace = true hyper = { workspace = true, default-features = false, features = ["server", "http1"] } -hyper-util.workspace = true multer.workspace = true percent-encoding.workspace = true roxmltree.workspace = true url.workspace = true serde.workspace = true -serde_bytes.workspace = true serde_json.workspace = true quick-xml.workspace = true opentelemetry.workspace = true -opentelemetry-prometheus = { workspace = true, optional = true } -prometheus = { workspace = true, optional = true } -- cgit v1.2.3 From d18c5ad0ffc9ffe3ec38d234445ef1a826a57e67 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 19:12:05 +0100 Subject: fix tests --- src/api/s3/copy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/api/s3') diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index e4992a18..07d50ea5 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -863,7 +863,7 @@ pub struct CopyPartResult { #[cfg(test)] mod tests { use super::*; - use crate::s3::xml::to_xml_with_header; + use crate::xml::to_xml_with_header; #[test] fn copy_object_result() -> Result<(), Error> { -- cgit v1.2.3