From 9fa20d45bebab2a3f66b9721c3643dbd607d944d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 18:18:04 +0100 Subject: wip: split garage_api into garage_api_{common,s3,k2v,admin} --- src/api/Cargo.toml | 74 ----- src/api/admin/Cargo.toml | 71 +++++ src/api/admin/api_server.rs | 18 +- src/api/admin/bucket.rs | 10 +- src/api/admin/cluster.rs | 6 +- src/api/admin/error.rs | 10 +- src/api/admin/key.rs | 6 +- src/api/admin/lib.rs | 11 + src/api/admin/mod.rs | 8 - src/api/admin/router_v0.rs | 4 +- src/api/admin/router_v1.rs | 6 +- src/api/common/Cargo.toml | 73 +++++ src/api/common/common_error.rs | 217 +++++++++++++ src/api/common/encoding.rs | 22 ++ src/api/common/generic_server.rs | 378 +++++++++++++++++++++++ src/api/common/helpers.rs | 371 ++++++++++++++++++++++ src/api/common/lib.rs | 12 + src/api/common/router_macros.rs | 226 ++++++++++++++ src/api/common/signature/error.rs | 32 ++ src/api/common/signature/mod.rs | 78 +++++ src/api/common/signature/payload.rs | 562 ++++++++++++++++++++++++++++++++++ src/api/common/signature/streaming.rs | 373 ++++++++++++++++++++++ src/api/common_error.rs | 217 ------------- src/api/encoding.rs | 22 -- src/api/generic_server.rs | 378 ----------------------- src/api/helpers.rs | 371 ---------------------- src/api/k2v/Cargo.toml | 75 +++++ src/api/k2v/api_server.rs | 20 +- src/api/k2v/batch.rs | 2 +- src/api/k2v/error.rs | 14 +- src/api/k2v/index.rs | 8 +- src/api/k2v/item.rs | 6 +- src/api/k2v/lib.rs | 12 + src/api/k2v/mod.rs | 9 - src/api/k2v/range.rs | 4 +- src/api/k2v/router.rs | 6 +- src/api/lib.rs | 17 - src/api/router_macros.rs | 224 -------------- src/api/s3/Cargo.toml | 71 +++++ src/api/s3/api_server.rs | 40 +-- src/api/s3/bucket.rs | 12 +- src/api/s3/checksum.rs | 2 +- src/api/s3/copy.rs | 18 +- src/api/s3/cors.rs | 12 +- src/api/s3/delete.rs | 12 +- src/api/s3/encryption.rs | 6 +- src/api/s3/error.rs | 19 +- src/api/s3/get.rs | 10 +- src/api/s3/lib.rs | 22 ++ src/api/s3/lifecycle.rs | 10 +- src/api/s3/list.rs | 14 +- src/api/s3/mod.rs | 19 -- src/api/s3/multipart.rs | 16 +- src/api/s3/post_object.rs | 18 +- src/api/s3/put.rs | 10 +- src/api/s3/router.rs | 6 +- src/api/s3/website.rs | 10 +- src/api/s3/xml.rs | 2 +- src/api/signature/error.rs | 32 -- src/api/signature/mod.rs | 78 ----- src/api/signature/payload.rs | 562 ---------------------------------- src/api/signature/streaming.rs | 373 ---------------------- 62 files changed, 2779 insertions(+), 2548 deletions(-) delete mode 100644 src/api/Cargo.toml create mode 100644 src/api/admin/Cargo.toml create mode 100644 src/api/admin/lib.rs delete mode 100644 src/api/admin/mod.rs create mode 100644 src/api/common/Cargo.toml create mode 100644 src/api/common/common_error.rs create mode 100644 src/api/common/encoding.rs create mode 100644 src/api/common/generic_server.rs create mode 100644 src/api/common/helpers.rs create mode 100644 src/api/common/lib.rs create mode 100644 src/api/common/router_macros.rs create mode 100644 src/api/common/signature/error.rs create mode 100644 src/api/common/signature/mod.rs create mode 100644 src/api/common/signature/payload.rs create mode 100644 src/api/common/signature/streaming.rs delete mode 100644 src/api/common_error.rs delete mode 100644 src/api/encoding.rs delete mode 100644 src/api/generic_server.rs delete mode 100644 src/api/helpers.rs create mode 100644 src/api/k2v/Cargo.toml create mode 100644 src/api/k2v/lib.rs delete mode 100644 src/api/k2v/mod.rs delete mode 100644 src/api/lib.rs delete mode 100644 src/api/router_macros.rs create mode 100644 src/api/s3/Cargo.toml create mode 100644 src/api/s3/lib.rs delete mode 100644 src/api/s3/mod.rs delete mode 100644 src/api/signature/error.rs delete mode 100644 src/api/signature/mod.rs delete mode 100644 src/api/signature/payload.rs delete mode 100644 src/api/signature/streaming.rs (limited to 'src/api') diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml deleted file mode 100644 index 85b78a5b..00000000 --- a/src/api/Cargo.toml +++ /dev/null @@ -1,74 +0,0 @@ -[package] -name = "garage_api" -version = "1.0.1" -authors = ["Alex Auvolat "] -edition = "2018" -license = "AGPL-3.0" -description = "S3 API server crate for the Garage object store" -repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" -readme = "../../README.md" - -[lib] -path = "lib.rs" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -garage_model.workspace = true -garage_table.workspace = true -garage_block.workspace = true -garage_net.workspace = true -garage_util.workspace = true -garage_rpc.workspace = true - -aes-gcm.workspace = true -argon2.workspace = true -async-compression.workspace = true -async-trait.workspace = true -base64.workspace = true -bytes.workspace = true -chrono.workspace = true -crc32fast.workspace = true -crc32c.workspace = true -crypto-common.workspace = true -err-derive.workspace = true -hex.workspace = true -hmac.workspace = true -idna.workspace = true -tracing.workspace = true -md-5.workspace = true -nom.workspace = true -pin-project.workspace = true -sha1.workspace = true -sha2.workspace = true - -futures.workspace = true -futures-util.workspace = true -tokio.workspace = true -tokio-stream.workspace = true -tokio-util.workspace = true - -form_urlencoded.workspace = true -http.workspace = true -httpdate.workspace = true -http-range.workspace = true -http-body-util.workspace = true -hyper = { workspace = true, default-features = false, features = ["server", "http1"] } -hyper-util.workspace = true -multer.workspace = true -percent-encoding.workspace = true -roxmltree.workspace = true -url.workspace = true - -serde.workspace = true -serde_bytes.workspace = true -serde_json.workspace = true -quick-xml.workspace = true - -opentelemetry.workspace = true -opentelemetry-prometheus = { workspace = true, optional = true } -prometheus = { workspace = true, optional = true } - -[features] -k2v = [ "garage_util/k2v", "garage_model/k2v" ] -metrics = [ "opentelemetry-prometheus", "prometheus" ] diff --git a/src/api/admin/Cargo.toml b/src/api/admin/Cargo.toml new file mode 100644 index 00000000..02cbfc3d --- /dev/null +++ b/src/api/admin/Cargo.toml @@ -0,0 +1,71 @@ +[package] +name = "garage_api_admin" +version = "1.0.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "S3 API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_block.workspace = true +garage_net.workspace = true +garage_util.workspace = true +garage_rpc.workspace = true +garage_api_common.workspace = true + +aes-gcm.workspace = true +argon2.workspace = true +async-compression.workspace = true +async-trait.workspace = true +base64.workspace = true +bytes.workspace = true +chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true +crypto-common.workspace = true +err-derive.workspace = true +hex.workspace = true +hmac.workspace = true +idna.workspace = true +tracing.workspace = true +md-5.workspace = true +nom.workspace = true +pin-project.workspace = true +sha1.workspace = true +sha2.workspace = true + +futures.workspace = true +futures-util.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tokio-util.workspace = true + +form_urlencoded.workspace = true +http.workspace = true +httpdate.workspace = true +http-range.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +hyper-util.workspace = true +multer.workspace = true +percent-encoding.workspace = true +roxmltree.workspace = true +url.workspace = true + +serde.workspace = true +serde_bytes.workspace = true +serde_json.workspace = true +quick-xml.workspace = true + +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 0e4565bb..7f8a51a6 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -20,15 +20,15 @@ use garage_rpc::system::ClusterHealthStatus; use garage_util::error::Error as GarageError; use garage_util::socket_address::UnixOrTCPSocketAddress; -use crate::generic_server::*; - -use crate::admin::bucket::*; -use crate::admin::cluster::*; -use crate::admin::error::*; -use crate::admin::key::*; -use crate::admin::router_v0; -use crate::admin::router_v1::{Authorization, Endpoint}; -use crate::helpers::*; +use garage_api_common::generic_server::*; + +use crate::bucket::*; +use crate::cluster::*; +use crate::error::*; +use crate::key::*; +use crate::router_v0; +use crate::router_v1::{Authorization, Endpoint}; +use garage_api_common::helpers::*; pub type ResBody = BoxBody; diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index ac3cba00..3afed694 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -17,11 +17,11 @@ use garage_model::permission::*; use garage_model::s3::mpu_table; use garage_model::s3::object_table::*; -use crate::admin::api_server::ResBody; -use crate::admin::error::*; -use crate::admin::key::ApiBucketKeyPerm; -use crate::common_error::CommonError; -use crate::helpers::*; +use crate::api_server::ResBody; +use crate::error::*; +use crate::key::ApiBucketKeyPerm; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { let buckets = garage diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 357ac600..d4a645a2 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -12,9 +12,9 @@ use garage_rpc::layout; use garage_model::garage::Garage; -use crate::admin::api_server::ResBody; -use crate::admin::error::*; -use crate::helpers::{json_ok_response, parse_json_body}; +use crate::api_server::ResBody; +use crate::error::*; +use garage_api_common::helpers::{json_ok_response, parse_json_body}; pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { let layout = garage.system.cluster_layout(); diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 40d686e3..1c962776 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -6,10 +6,12 @@ use hyper::{HeaderMap, StatusCode}; pub use garage_model::helper::error::Error as HelperError; -use crate::common_error::CommonError; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::helpers::*; +use garage_api_common::common_error::CommonError; +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; /// Errors of this crate #[derive(Debug, Error)] diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 291b6d54..0c017a26 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -9,9 +9,9 @@ use garage_table::*; use garage_model::garage::Garage; use garage_model::key_table::*; -use crate::admin::api_server::ResBody; -use crate::admin::error::*; -use crate::helpers::*; +use crate::api_server::ResBody; +use crate::error::*; +use garage_api_common::helpers::*; pub async fn handle_list_keys(garage: &Arc) -> Result, Error> { let res = garage diff --git a/src/api/admin/lib.rs b/src/api/admin/lib.rs new file mode 100644 index 00000000..599e9b44 --- /dev/null +++ b/src/api/admin/lib.rs @@ -0,0 +1,11 @@ +#[macro_use] +extern crate tracing; + +pub mod api_server; +mod error; +mod router_v0; +mod router_v1; + +mod bucket; +mod cluster; +mod key; diff --git a/src/api/admin/mod.rs b/src/api/admin/mod.rs deleted file mode 100644 index 43a8c59c..00000000 --- a/src/api/admin/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod api_server; -mod error; -mod router_v0; -mod router_v1; - -mod bucket; -mod cluster; -mod key; diff --git a/src/api/admin/router_v0.rs b/src/api/admin/router_v0.rs index 68676445..0c832fe1 100644 --- a/src/api/admin/router_v0.rs +++ b/src/api/admin/router_v0.rs @@ -2,8 +2,8 @@ use std::borrow::Cow; use hyper::{Method, Request}; -use crate::admin::error::*; -use crate::router_macros::*; +use crate::error::*; +use garage_api_common::router_macros::*; router_match! {@func diff --git a/src/api/admin/router_v1.rs b/src/api/admin/router_v1.rs index cc5ff2ec..d9febd34 100644 --- a/src/api/admin/router_v1.rs +++ b/src/api/admin/router_v1.rs @@ -2,9 +2,9 @@ use std::borrow::Cow; use hyper::{Method, Request}; -use crate::admin::error::*; -use crate::admin::router_v0; -use crate::router_macros::*; +use crate::error::*; +use crate::router_v0; +use garage_api_common::router_macros::*; pub enum Authorization { None, diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml new file mode 100644 index 00000000..e5dc57d4 --- /dev/null +++ b/src/api/common/Cargo.toml @@ -0,0 +1,73 @@ +[package] +name = "garage_api_common" +version = "1.0.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "S3 API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_block.workspace = true +garage_net.workspace = true +garage_util.workspace = true +garage_rpc.workspace = true + +aes-gcm.workspace = true +argon2.workspace = true +async-compression.workspace = true +async-trait.workspace = true +base64.workspace = true +bytes.workspace = true +chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true +crypto-common.workspace = true +err-derive.workspace = true +hex.workspace = true +hmac.workspace = true +idna.workspace = true +tracing.workspace = true +md-5.workspace = true +nom.workspace = true +pin-project.workspace = true +sha1.workspace = true +sha2.workspace = true + +futures.workspace = true +futures-util.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tokio-util.workspace = true + +form_urlencoded.workspace = true +http.workspace = true +httpdate.workspace = true +http-range.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +hyper-util.workspace = true +multer.workspace = true +percent-encoding.workspace = true +roxmltree.workspace = true +url.workspace = true + +serde.workspace = true +serde_bytes.workspace = true +serde_json.workspace = true +quick-xml.workspace = true + +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } + +[features] +metrics = [ "opentelemetry-prometheus", "prometheus" ] diff --git a/src/api/common/common_error.rs b/src/api/common/common_error.rs new file mode 100644 index 00000000..1e3f9feb --- /dev/null +++ b/src/api/common/common_error.rs @@ -0,0 +1,217 @@ +use std::convert::TryFrom; + +use err_derive::Error; +use hyper::StatusCode; + +use garage_util::error::Error as GarageError; + +use garage_model::helper::error::Error as HelperError; + +/// Errors of this crate +#[derive(Debug, Error)] +pub enum CommonError { + // ---- INTERNAL ERRORS ---- + /// Error related to deeper parts of Garage + #[error(display = "Internal error: {}", _0)] + InternalError(#[error(source)] GarageError), + + /// Error related to Hyper + #[error(display = "Internal error (Hyper error): {}", _0)] + Hyper(#[error(source)] hyper::Error), + + /// Error related to HTTP + #[error(display = "Internal error (HTTP error): {}", _0)] + Http(#[error(source)] http::Error), + + // ---- GENERIC CLIENT ERRORS ---- + /// Proper authentication was not provided + #[error(display = "Forbidden: {}", _0)] + Forbidden(String), + + /// Generic bad request response with custom message + #[error(display = "Bad request: {}", _0)] + BadRequest(String), + + /// The client sent a header with invalid value + #[error(display = "Invalid header value: {}", _0)] + InvalidHeader(#[error(source)] hyper::header::ToStrError), + + // ---- SPECIFIC ERROR CONDITIONS ---- + // These have to be error codes referenced in the S3 spec here: + // https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList + /// The bucket requested don't exists + #[error(display = "Bucket not found: {}", _0)] + NoSuchBucket(String), + + /// Tried to create a bucket that already exist + #[error(display = "Bucket already exists")] + BucketAlreadyExists, + + /// Tried to delete a non-empty bucket + #[error(display = "Tried to delete a non-empty bucket")] + BucketNotEmpty, + + // Category: bad request + /// Bucket name is not valid according to AWS S3 specs + #[error(display = "Invalid bucket name: {}", _0)] + InvalidBucketName(String), +} + +impl CommonError { + pub fn http_status_code(&self) -> StatusCode { + match self { + CommonError::InternalError( + GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..), + ) => StatusCode::SERVICE_UNAVAILABLE, + CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } + CommonError::BadRequest(_) => StatusCode::BAD_REQUEST, + CommonError::Forbidden(_) => StatusCode::FORBIDDEN, + CommonError::NoSuchBucket(_) => StatusCode::NOT_FOUND, + CommonError::BucketNotEmpty | CommonError::BucketAlreadyExists => StatusCode::CONFLICT, + CommonError::InvalidBucketName(_) | CommonError::InvalidHeader(_) => { + StatusCode::BAD_REQUEST + } + } + } + + pub fn aws_code(&self) -> &'static str { + match self { + CommonError::Forbidden(_) => "AccessDenied", + CommonError::InternalError( + GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..), + ) => "ServiceUnavailable", + CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { + "InternalError" + } + CommonError::BadRequest(_) => "InvalidRequest", + CommonError::NoSuchBucket(_) => "NoSuchBucket", + CommonError::BucketAlreadyExists => "BucketAlreadyExists", + CommonError::BucketNotEmpty => "BucketNotEmpty", + CommonError::InvalidBucketName(_) => "InvalidBucketName", + CommonError::InvalidHeader(_) => "InvalidHeaderValue", + } + } + + pub fn bad_request(msg: M) -> Self { + CommonError::BadRequest(msg.to_string()) + } +} + +impl TryFrom for CommonError { + type Error = HelperError; + + fn try_from(err: HelperError) -> Result { + match err { + HelperError::Internal(i) => Ok(Self::InternalError(i)), + HelperError::BadRequest(b) => Ok(Self::BadRequest(b)), + HelperError::InvalidBucketName(n) => Ok(Self::InvalidBucketName(n)), + HelperError::NoSuchBucket(n) => Ok(Self::NoSuchBucket(n)), + e => Err(e), + } + } +} + +/// This function converts HelperErrors into CommonErrors, +/// for variants that exist in CommonError. +/// This is used for helper functions that might return InvalidBucketName +/// or NoSuchBucket for instance, and we want to pass that error +/// up to our caller. +pub fn pass_helper_error(err: HelperError) -> CommonError { + match CommonError::try_from(err) { + Ok(e) => e, + Err(e) => panic!("Helper error `{}` should hot have happenned here", e), + } +} + +pub fn helper_error_as_internal(err: HelperError) -> CommonError { + match err { + HelperError::Internal(e) => CommonError::InternalError(e), + e => CommonError::InternalError(GarageError::Message(e.to_string())), + } +} + +pub trait CommonErrorDerivative: From { + fn internal_error(msg: M) -> Self { + Self::from(CommonError::InternalError(GarageError::Message( + msg.to_string(), + ))) + } + + fn bad_request(msg: M) -> Self { + Self::from(CommonError::BadRequest(msg.to_string())) + } + + fn forbidden(msg: M) -> Self { + Self::from(CommonError::Forbidden(msg.to_string())) + } +} + +/// Trait to map error to the Bad Request error code +pub trait OkOrBadRequest { + type S; + fn ok_or_bad_request>(self, reason: M) -> Result; +} + +impl OkOrBadRequest for Result +where + E: std::fmt::Display, +{ + type S = T; + fn ok_or_bad_request>(self, reason: M) -> Result { + match self { + Ok(x) => Ok(x), + Err(e) => Err(CommonError::BadRequest(format!( + "{}: {}", + reason.as_ref(), + e + ))), + } + } +} + +impl OkOrBadRequest for Option { + type S = T; + fn ok_or_bad_request>(self, reason: M) -> Result { + match self { + Some(x) => Ok(x), + None => Err(CommonError::BadRequest(reason.as_ref().to_string())), + } + } +} + +/// Trait to map an error to an Internal Error code +pub trait OkOrInternalError { + type S; + fn ok_or_internal_error>(self, reason: M) -> Result; +} + +impl OkOrInternalError for Result +where + E: std::fmt::Display, +{ + type S = T; + fn ok_or_internal_error>(self, reason: M) -> Result { + match self { + Ok(x) => Ok(x), + Err(e) => Err(CommonError::InternalError(GarageError::Message(format!( + "{}: {}", + reason.as_ref(), + e + )))), + } + } +} + +impl OkOrInternalError for Option { + type S = T; + fn ok_or_internal_error>(self, reason: M) -> Result { + match self { + Some(x) => Ok(x), + None => Err(CommonError::InternalError(GarageError::Message( + reason.as_ref().to_string(), + ))), + } + } +} diff --git a/src/api/common/encoding.rs b/src/api/common/encoding.rs new file mode 100644 index 00000000..e286a784 --- /dev/null +++ b/src/api/common/encoding.rs @@ -0,0 +1,22 @@ +//! Module containing various helpers for encoding + +/// Encode &str for use in a URI +pub fn uri_encode(string: &str, encode_slash: bool) -> String { + let mut result = String::with_capacity(string.len() * 2); + for c in string.chars() { + match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' | '~' | '.' => result.push(c), + '/' if encode_slash => result.push_str("%2F"), + '/' if !encode_slash => result.push('/'), + _ => { + result.push_str( + &format!("{}", c) + .bytes() + .map(|b| format!("%{:02X}", b)) + .collect::(), + ); + } + } + } + result +} diff --git a/src/api/common/generic_server.rs b/src/api/common/generic_server.rs new file mode 100644 index 00000000..d92a3465 --- /dev/null +++ b/src/api/common/generic_server.rs @@ -0,0 +1,378 @@ +use std::convert::Infallible; +use std::fs::{self, Permissions}; +use std::os::unix::fs::PermissionsExt; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; + +use futures::future::Future; +use futures::stream::{futures_unordered::FuturesUnordered, StreamExt}; + +use http_body_util::BodyExt; +use hyper::header::HeaderValue; +use hyper::server::conn::http1; +use hyper::service::service_fn; +use hyper::{body::Incoming as IncomingBody, Request, Response}; +use hyper::{HeaderMap, StatusCode}; +use hyper_util::rt::TokioIo; + +use tokio::io::{AsyncRead, AsyncWrite}; +use tokio::net::{TcpListener, TcpStream, UnixListener, UnixStream}; +use tokio::sync::watch; +use tokio::time::{sleep_until, Instant}; + +use opentelemetry::{ + global, + metrics::{Counter, ValueRecorder}, + trace::{FutureExt, SpanRef, TraceContextExt, Tracer}, + Context, KeyValue, +}; + +use garage_util::error::Error as GarageError; +use garage_util::forwarded_headers; +use garage_util::metrics::{gen_trace_id, RecordDuration}; +use garage_util::socket_address::UnixOrTCPSocketAddress; + +use crate::helpers::{BoxBody, ErrorBody}; + +pub trait ApiEndpoint: Send + Sync + 'static { + fn name(&self) -> &'static str; + fn add_span_attributes(&self, span: SpanRef<'_>); +} + +pub trait ApiError: std::error::Error + Send + Sync + 'static { + fn http_status_code(&self) -> StatusCode; + fn add_http_headers(&self, header_map: &mut HeaderMap); + fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody; +} + +#[async_trait] +pub trait ApiHandler: Send + Sync + 'static { + const API_NAME: &'static str; + const API_NAME_DISPLAY: &'static str; + + type Endpoint: ApiEndpoint; + type Error: ApiError; + + fn parse_endpoint(&self, r: &Request) -> Result; + async fn handle( + &self, + req: Request, + endpoint: Self::Endpoint, + ) -> Result>, Self::Error>; +} + +pub struct ApiServer { + region: String, + api_handler: A, + + // Metrics + request_counter: Counter, + error_counter: Counter, + request_duration: ValueRecorder, +} + +impl ApiServer { + pub fn new(region: String, api_handler: A) -> Arc { + let meter = global::meter("garage/api"); + Arc::new(Self { + region, + api_handler, + request_counter: meter + .u64_counter(format!("api.{}.request_counter", A::API_NAME)) + .with_description(format!( + "Number of API calls to the various {} API endpoints", + A::API_NAME_DISPLAY + )) + .init(), + error_counter: meter + .u64_counter(format!("api.{}.error_counter", A::API_NAME)) + .with_description(format!( + "Number of API calls to the various {} API endpoints that resulted in errors", + A::API_NAME_DISPLAY + )) + .init(), + request_duration: meter + .f64_value_recorder(format!("api.{}.request_duration", A::API_NAME)) + .with_description(format!( + "Duration of API calls to the various {} API endpoints", + A::API_NAME_DISPLAY + )) + .init(), + }) + } + + pub async fn run_server( + self: Arc, + bind_addr: UnixOrTCPSocketAddress, + unix_bind_addr_mode: Option, + must_exit: watch::Receiver, + ) -> Result<(), GarageError> { + let server_name = format!("{} API", A::API_NAME_DISPLAY); + info!("{} server listening on {}", server_name, bind_addr); + + match bind_addr { + UnixOrTCPSocketAddress::TCPSocket(addr) => { + let listener = TcpListener::bind(addr).await?; + + let handler = move |request, socketaddr| self.clone().handler(request, socketaddr); + server_loop(server_name, listener, handler, must_exit).await + } + UnixOrTCPSocketAddress::UnixSocket(ref path) => { + if path.exists() { + fs::remove_file(path)? + } + + let listener = UnixListener::bind(path)?; + let listener = UnixListenerOn(listener, path.display().to_string()); + + fs::set_permissions( + path, + Permissions::from_mode(unix_bind_addr_mode.unwrap_or(0o222)), + )?; + + let handler = move |request, socketaddr| self.clone().handler(request, socketaddr); + server_loop(server_name, listener, handler, must_exit).await + } + } + } + + async fn handler( + self: Arc, + req: Request, + addr: String, + ) -> Result>, http::Error> { + let uri = req.uri().clone(); + + if let Ok(forwarded_for_ip_addr) = + forwarded_headers::handle_forwarded_for_headers(req.headers()) + { + info!( + "{} (via {}) {} {}", + forwarded_for_ip_addr, + addr, + req.method(), + uri + ); + } else { + info!("{} {} {}", addr, req.method(), uri); + } + debug!("{:?}", req); + + let tracer = opentelemetry::global::tracer("garage"); + let span = tracer + .span_builder(format!("{} API call (unknown)", A::API_NAME_DISPLAY)) + .with_trace_id(gen_trace_id()) + .with_attributes(vec![ + KeyValue::new("method", format!("{}", req.method())), + KeyValue::new("uri", req.uri().to_string()), + ]) + .start(&tracer); + + let res = self + .handler_stage2(req) + .with_context(Context::current_with_span(span)) + .await; + + match res { + Ok(x) => { + debug!("{} {:?}", x.status(), x.headers()); + Ok(x) + } + Err(e) => { + let body = e.http_body(&self.region, uri.path()); + let mut http_error_builder = Response::builder().status(e.http_status_code()); + + if let Some(header_map) = http_error_builder.headers_mut() { + e.add_http_headers(header_map) + } + + let http_error = http_error_builder.body(body)?; + + if e.http_status_code().is_server_error() { + warn!("Response: error {}, {}", e.http_status_code(), e); + } else { + info!("Response: error {}, {}", e.http_status_code(), e); + } + Ok(http_error + .map(|body| BoxBody::new(body.map_err(|_: Infallible| unreachable!())))) + } + } + } + + async fn handler_stage2( + &self, + req: Request, + ) -> Result>, A::Error> { + let endpoint = self.api_handler.parse_endpoint(&req)?; + debug!("Endpoint: {}", endpoint.name()); + + let current_context = Context::current(); + let current_span = current_context.span(); + current_span.update_name::(format!( + "{} API {}", + A::API_NAME_DISPLAY, + endpoint.name() + )); + current_span.set_attribute(KeyValue::new("endpoint", endpoint.name())); + endpoint.add_span_attributes(current_span); + + let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())]; + + let res = self + .api_handler + .handle(req, endpoint) + .record_duration(&self.request_duration, &metrics_tags[..]) + .await; + + self.request_counter.add(1, &metrics_tags[..]); + + let status_code = match &res { + Ok(r) => r.status(), + Err(e) => e.http_status_code(), + }; + if status_code.is_client_error() || status_code.is_server_error() { + self.error_counter.add( + 1, + &[ + metrics_tags[0].clone(), + KeyValue::new("status_code", status_code.as_str().to_string()), + ], + ); + } + + res + } +} + +// ==== helper functions ==== + +#[async_trait] +pub trait Accept: Send + Sync + 'static { + type Stream: AsyncRead + AsyncWrite + Send + Sync + 'static; + async fn accept(&self) -> std::io::Result<(Self::Stream, String)>; +} + +#[async_trait] +impl Accept for TcpListener { + type Stream = TcpStream; + async fn accept(&self) -> std::io::Result<(Self::Stream, String)> { + self.accept() + .await + .map(|(stream, addr)| (stream, addr.to_string())) + } +} + +pub struct UnixListenerOn(pub UnixListener, pub String); + +#[async_trait] +impl Accept for UnixListenerOn { + type Stream = UnixStream; + async fn accept(&self) -> std::io::Result<(Self::Stream, String)> { + self.0 + .accept() + .await + .map(|(stream, _addr)| (stream, self.1.clone())) + } +} + +pub async fn server_loop( + server_name: String, + listener: A, + handler: H, + mut must_exit: watch::Receiver, +) -> Result<(), GarageError> +where + A: Accept, + H: Fn(Request, String) -> F + Send + Sync + Clone + 'static, + F: Future>, http::Error>> + Send + 'static, + E: Send + Sync + std::error::Error + 'static, +{ + let (conn_in, mut conn_out) = tokio::sync::mpsc::unbounded_channel(); + let connection_collector = tokio::spawn({ + let server_name = server_name.clone(); + async move { + let mut connections = FuturesUnordered::>::new(); + loop { + let collect_next = async { + if connections.is_empty() { + futures::future::pending().await + } else { + connections.next().await + } + }; + tokio::select! { + result = collect_next => { + trace!("{} server: HTTP connection finished: {:?}", server_name, result); + } + new_fut = conn_out.recv() => { + match new_fut { + Some(f) => connections.push(f), + None => break, + } + } + } + } + let deadline = Instant::now() + Duration::from_secs(10); + while !connections.is_empty() { + info!( + "{} server: {} connections still open, deadline in {:.2}s", + server_name, + connections.len(), + (deadline - Instant::now()).as_secs_f32(), + ); + tokio::select! { + conn_res = connections.next() => { + trace!( + "{} server: HTTP connection finished: {:?}", + server_name, + conn_res.unwrap(), + ); + } + _ = sleep_until(deadline) => { + warn!("{} server: exit deadline reached with {} connections still open, killing them now", + server_name, + connections.len()); + for conn in connections.iter() { + conn.abort(); + } + for conn in connections { + assert!(conn.await.unwrap_err().is_cancelled()); + } + break; + } + } + } + } + }); + + while !*must_exit.borrow() { + let (stream, client_addr) = tokio::select! { + acc = listener.accept() => acc?, + _ = must_exit.changed() => continue, + }; + + let io = TokioIo::new(stream); + + let handler = handler.clone(); + let serve = move |req: Request| handler(req, client_addr.clone()); + + let fut = tokio::task::spawn(async move { + let io = Box::pin(io); + if let Err(e) = http1::Builder::new() + .serve_connection(io, service_fn(serve)) + .await + { + debug!("Error handling HTTP connection: {}", e); + } + }); + conn_in.send(fut)?; + } + + info!("{} server exiting", server_name); + drop(conn_in); + connection_collector.await?; + + Ok(()) +} diff --git a/src/api/common/helpers.rs b/src/api/common/helpers.rs new file mode 100644 index 00000000..c8586de4 --- /dev/null +++ b/src/api/common/helpers.rs @@ -0,0 +1,371 @@ +use std::convert::Infallible; +use std::sync::Arc; + +use futures::{Stream, StreamExt, TryStreamExt}; + +use http_body_util::{BodyExt, Full as FullBody}; +use hyper::{ + body::{Body, Bytes}, + Request, Response, +}; +use idna::domain_to_unicode; +use serde::{Deserialize, Serialize}; + +use garage_model::bucket_table::BucketParams; +use garage_model::garage::Garage; +use garage_model::key_table::Key; +use garage_util::data::Uuid; +use garage_util::error::Error as GarageError; + +use crate::common_error::{CommonError as Error, *}; + +/// What kind of authorization is required to perform a given action +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Authorization { + /// No authorization is required + None, + /// Having Read permission on bucket + Read, + /// Having Write permission on bucket + Write, + /// Having Owner permission on bucket + Owner, +} + +/// The values which are known for each request related to a bucket +pub struct ReqCtx { + pub garage: Arc, + pub bucket_id: Uuid, + pub bucket_name: String, + pub bucket_params: BucketParams, + pub api_key: Key, +} + +/// Host to bucket +/// +/// Convert a host, like "bucket.garage-site.tld" to the corresponding bucket "bucket", +/// considering that ".garage-site.tld" is the "root domain". For domains not matching +/// the provided root domain, no bucket is returned +/// This behavior has been chosen to follow AWS S3 semantic. +pub fn host_to_bucket<'a>(host: &'a str, root: &str) -> Option<&'a str> { + let root = root.trim_start_matches('.'); + let label_root = root.chars().filter(|c| c == &'.').count() + 1; + let root = root.rsplit('.'); + let mut host = host.rsplitn(label_root + 1, '.'); + for root_part in root { + let host_part = host.next()?; + if root_part != host_part { + return None; + } + } + host.next() +} + +/// Extract host from the authority section given by the HTTP host header +/// +/// The HTTP host contains both a host and a port. +/// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6 +/// We do not use the collect pattern as there is no way in std rust to collect over a stack allocated value +/// check here: +pub fn authority_to_host(authority: &str) -> Result { + let mut iter = authority.chars().enumerate(); + let (_, first_char) = iter + .next() + .ok_or_else(|| Error::bad_request("Authority is empty".to_string()))?; + + let split = match first_char { + '[' => { + let mut iter = iter.skip_while(|(_, c)| c != &']'); + match iter.next() { + Some((_, ']')) => iter.next(), + _ => { + return Err(Error::bad_request(format!( + "Authority {} has an illegal format", + authority + ))) + } + } + } + _ => iter.find(|(_, c)| *c == ':'), + }; + + let authority = match split { + Some((i, ':')) => Ok(&authority[..i]), + None => Ok(authority), + Some((_, _)) => Err(Error::bad_request(format!( + "Authority {} has an illegal format", + authority + ))), + }; + authority.map(|h| domain_to_unicode(h).0) +} + +/// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in +/// the host header of the request +/// +/// S3 internally manages only buckets and keys. This function splits +/// an HTTP path to get the corresponding bucket name and key. +pub fn parse_bucket_key<'a>( + path: &'a str, + host_bucket: Option<&'a str>, +) -> Result<(&'a str, Option<&'a str>), Error> { + let path = path.trim_start_matches('/'); + + if let Some(bucket) = host_bucket { + if !path.is_empty() { + return Ok((bucket, Some(path))); + } else { + return Ok((bucket, None)); + } + } + + let (bucket, key) = match path.find('/') { + Some(i) => { + let key = &path[i + 1..]; + if !key.is_empty() { + (&path[..i], Some(key)) + } else { + (&path[..i], None) + } + } + None => (path, None), + }; + if bucket.is_empty() { + return Err(Error::bad_request("No bucket specified")); + } + Ok((bucket, key)) +} + +const UTF8_BEFORE_LAST_CHAR: char = '\u{10FFFE}'; + +/// Compute the key after the prefix +pub fn key_after_prefix(pfx: &str) -> Option { + let mut next = pfx.to_string(); + while !next.is_empty() { + let tail = next.pop().unwrap(); + if tail >= char::MAX { + continue; + } + + // Circumvent a limitation of RangeFrom that overflow earlier than needed + // See: https://doc.rust-lang.org/core/ops/struct.RangeFrom.html + let new_tail = if tail == UTF8_BEFORE_LAST_CHAR { + char::MAX + } else { + (tail..).nth(1).unwrap() + }; + + next.push(new_tail); + return Some(next); + } + + None +} + +// =============== body helpers ================= + +pub type EmptyBody = http_body_util::Empty; +pub type ErrorBody = FullBody; +pub type BoxBody = http_body_util::combinators::BoxBody; + +pub fn string_body(s: String) -> BoxBody { + bytes_body(bytes::Bytes::from(s.into_bytes())) +} +pub fn bytes_body(b: bytes::Bytes) -> BoxBody { + BoxBody::new(FullBody::new(b).map_err(|_: Infallible| unreachable!())) +} +pub fn empty_body() -> BoxBody { + BoxBody::new(http_body_util::Empty::new().map_err(|_: Infallible| unreachable!())) +} +pub fn error_body(s: String) -> ErrorBody { + ErrorBody::from(bytes::Bytes::from(s.into_bytes())) +} + +pub async fn parse_json_body(req: Request) -> Result +where + T: for<'de> Deserialize<'de>, + B: Body, + E: From<::Error> + From, +{ + let body = req.into_body().collect().await?.to_bytes(); + let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; + Ok(resp) +} + +pub fn json_ok_response(res: &T) -> Result>, E> +where + E: From, +{ + let resp_json = serde_json::to_string_pretty(res) + .map_err(GarageError::from) + .map_err(Error::from)?; + Ok(Response::builder() + .status(hyper::StatusCode::OK) + .header(http::header::CONTENT_TYPE, "application/json") + .body(string_body(resp_json)) + .unwrap()) +} + +pub fn body_stream(body: B) -> impl Stream> +where + B: Body, + ::Error: Into, + E: From, +{ + let stream = http_body_util::BodyStream::new(body); + let stream = TryStreamExt::map_err(stream, Into::into); + stream.map(|x| { + x.and_then(|f| { + f.into_data() + .map_err(|_| E::from(Error::bad_request("non-data frame"))) + }) + }) +} + +pub fn is_default(v: &T) -> bool { + *v == T::default() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_bucket_containing_a_key() -> Result<(), Error> { + let (bucket, key) = parse_bucket_key("/my_bucket/a/super/file.jpg", None)?; + assert_eq!(bucket, "my_bucket"); + assert_eq!(key.expect("key must be set"), "a/super/file.jpg"); + Ok(()) + } + + #[test] + fn parse_bucket_containing_no_key() -> Result<(), Error> { + let (bucket, key) = parse_bucket_key("/my_bucket/", None)?; + assert_eq!(bucket, "my_bucket"); + assert!(key.is_none()); + let (bucket, key) = parse_bucket_key("/my_bucket", None)?; + assert_eq!(bucket, "my_bucket"); + assert!(key.is_none()); + Ok(()) + } + + #[test] + fn parse_bucket_containing_no_bucket() { + let parsed = parse_bucket_key("", None); + assert!(parsed.is_err()); + let parsed = parse_bucket_key("/", None); + assert!(parsed.is_err()); + let parsed = parse_bucket_key("////", None); + assert!(parsed.is_err()); + } + + #[test] + fn parse_bucket_with_vhost_and_key() -> Result<(), Error> { + let (bucket, key) = parse_bucket_key("/a/super/file.jpg", Some("my-bucket"))?; + assert_eq!(bucket, "my-bucket"); + assert_eq!(key.expect("key must be set"), "a/super/file.jpg"); + Ok(()) + } + + #[test] + fn parse_bucket_with_vhost_no_key() -> Result<(), Error> { + let (bucket, key) = parse_bucket_key("", Some("my-bucket"))?; + assert_eq!(bucket, "my-bucket"); + assert!(key.is_none()); + let (bucket, key) = parse_bucket_key("/", Some("my-bucket"))?; + assert_eq!(bucket, "my-bucket"); + assert!(key.is_none()); + Ok(()) + } + + #[test] + fn authority_to_host_with_port() -> Result<(), Error> { + let domain = authority_to_host("[::1]:3902")?; + assert_eq!(domain, "[::1]"); + let domain2 = authority_to_host("garage.tld:65200")?; + assert_eq!(domain2, "garage.tld"); + let domain3 = authority_to_host("127.0.0.1:80")?; + assert_eq!(domain3, "127.0.0.1"); + Ok(()) + } + + #[test] + fn authority_to_host_without_port() -> Result<(), Error> { + let domain = authority_to_host("[::1]")?; + assert_eq!(domain, "[::1]"); + let domain2 = authority_to_host("garage.tld")?; + assert_eq!(domain2, "garage.tld"); + let domain3 = authority_to_host("127.0.0.1")?; + assert_eq!(domain3, "127.0.0.1"); + assert!(authority_to_host("[").is_err()); + assert!(authority_to_host("[hello").is_err()); + Ok(()) + } + + #[test] + fn host_to_bucket_test() { + assert_eq!( + host_to_bucket("john.doe.garage.tld", ".garage.tld").unwrap(), + "john.doe" + ); + + assert_eq!( + host_to_bucket("john.doe.garage.tld", "garage.tld").unwrap(), + "john.doe" + ); + + assert_eq!(host_to_bucket("john.doe.com", "garage.tld"), None); + + assert_eq!(host_to_bucket("john.doe.com", ".garage.tld"), None); + + assert_eq!(host_to_bucket("garage.tld", "garage.tld"), None); + + assert_eq!(host_to_bucket("garage.tld", ".garage.tld"), None); + + assert_eq!(host_to_bucket("not-garage.tld", "garage.tld"), None); + assert_eq!(host_to_bucket("not-garage.tld", ".garage.tld"), None); + } + + #[test] + fn test_key_after_prefix() { + use std::iter::FromIterator; + + assert_eq!(UTF8_BEFORE_LAST_CHAR as u32, (char::MAX as u32) - 1); + assert_eq!(key_after_prefix("a/b/").unwrap().as_str(), "a/b0"); + assert_eq!(key_after_prefix("€").unwrap().as_str(), "₭"); + assert_eq!( + key_after_prefix("􏿽").unwrap().as_str(), + String::from(char::from_u32(0x10FFFE).unwrap()) + ); + + // When the last character is the biggest UTF8 char + let a = String::from_iter(['a', char::MAX].iter()); + assert_eq!(key_after_prefix(a.as_str()).unwrap().as_str(), "b"); + + // When all characters are the biggest UTF8 char + let b = String::from_iter([char::MAX; 3].iter()); + assert!(key_after_prefix(b.as_str()).is_none()); + + // Check utf8 surrogates + let c = String::from('\u{D7FF}'); + assert_eq!( + key_after_prefix(c.as_str()).unwrap().as_str(), + String::from('\u{E000}') + ); + + // Check the character before the biggest one + let d = String::from('\u{10FFFE}'); + assert_eq!( + key_after_prefix(d.as_str()).unwrap().as_str(), + String::from(char::MAX) + ); + } +} + +#[derive(Serialize)] +pub struct CustomApiErrorBody { + pub code: String, + pub message: String, + pub region: String, + pub path: String, +} diff --git a/src/api/common/lib.rs b/src/api/common/lib.rs new file mode 100644 index 00000000..49d463d7 --- /dev/null +++ b/src/api/common/lib.rs @@ -0,0 +1,12 @@ +//! Crate for serving a S3 compatible API +#[macro_use] +extern crate tracing; + +pub mod common_error; + +pub mod encoding; +pub mod generic_server; +pub mod helpers; +pub mod router_macros; +/// This mode is public only to help testing. Don't expect stability here +pub mod signature; diff --git a/src/api/common/router_macros.rs b/src/api/common/router_macros.rs new file mode 100644 index 00000000..d9fe86db --- /dev/null +++ b/src/api/common/router_macros.rs @@ -0,0 +1,226 @@ +/// This macro is used to generate very repetitive match {} blocks in this module +/// It is _not_ made to be used anywhere else +#[macro_export] +macro_rules! router_match { + (@match $enum:expr , [ $($endpoint:ident,)* ]) => {{ + // usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] } + // returns true if the variant was one of the listed variants, false otherwise. + match $enum { + $( + Endpoint::$endpoint { .. } => true, + )* + _ => false + } + }}; + (@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{ + // usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] } + // returns Some(field_value), or None if the variant was not one of the listed variants. + match $enum { + $( + Endpoint::$endpoint {$param, ..} => Some($param), + )* + _ => None + } + }}; + (@gen_path_parser ($method:expr, $reqpath:expr, $query:expr) + [ + $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)* + ]) => {{ + { + #[allow(unused_parens)] + match ($method, $reqpath) { + $( + (&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api { + $($( + $param: router_match!(@@parse_param $query, $conv, $param), + )*)? + }, + )* + (m, p) => { + return Err(Error::bad_request(format!( + "Unknown API endpoint: {} {}", + m, p + ))) + } + } + } + }}; + (@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr), + key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*], + no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{ + // usage: router_match {@gen_parser (keyword, key, query, header), + // key: [ + // SOME_KEYWORD => VariantWithKey, + // ... + // ], + // no_key: [ + // SOME_KEYWORD => VariantWithoutKey, + // ... + // ] + // } + // See in from_{method} for more detailed usage. + match ($keyword, !$key.is_empty()){ + $( + (Keyword::$kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok(Endpoint::$api_k { + $key, + $($( + $param_k: router_match!(@@parse_param $query, $conv_k, $param_k), + )*)? + }), + )* + $( + (Keyword::$kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok(Endpoint::$api_nk { + $($( + $param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk), + )*)? + }), + )* + (kw, _) => Err(Error::bad_request(format!("Invalid endpoint: {}", kw))) + } + }}; + + (@@parse_param $query:expr, query_opt, $param:ident) => {{ + // extract optional query parameter + $query.$param.take().map(|param| param.into_owned()) + }}; + (@@parse_param $query:expr, query, $param:ident) => {{ + // extract mendatory query parameter + $query.$param.take().ok_or_bad_request("Missing argument for endpoint")?.into_owned() + }}; + (@@parse_param $query:expr, opt_parse, $param:ident) => {{ + // extract and parse optional query parameter + // missing parameter is file, however parse error is reported as an error + $query.$param + .take() + .map(|param| param.parse()) + .transpose() + .map_err(|_| Error::bad_request("Failed to parse query parameter"))? + }}; + (@@parse_param $query:expr, parse, $param:ident) => {{ + // extract and parse mandatory query parameter + // both missing and un-parseable parameters are reported as errors + $query.$param.take().ok_or_bad_request("Missing argument for endpoint")? + .parse() + .map_err(|_| Error::bad_request("Failed to parse query parameter"))? + }}; + (@func + $(#[$doc:meta])* + pub enum Endpoint { + $( + $(#[$outer:meta])* + $variant:ident $({ + $($name:ident: $ty:ty,)* + })?, + )* + }) => { + $(#[$doc])* + pub enum Endpoint { + $( + $(#[$outer])* + $variant $({ + $($name: $ty, )* + })?, + )* + } + impl Endpoint { + pub fn name(&self) -> &'static str { + match self { + $(Endpoint::$variant $({ $($name: _,)* .. })? => stringify!($variant),)* + } + } + } + }; +} + +/// This macro is used to generate part of the code in this module. It must be called only one, and +/// is useless outside of this module. +#[macro_export] +macro_rules! generateQueryParameters { + ( + keywords: [ $($kw_param:expr => $kw_name: ident),* ], + fields: [ $($f_param:expr => $f_name:ident),* ] + ) => { + #[derive(Debug)] + #[allow(non_camel_case_types)] + #[allow(clippy::upper_case_acronyms)] + enum Keyword { + EMPTY, + $( $kw_name, )* + } + + impl std::fmt::Display for Keyword { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + Keyword::EMPTY => write!(f, "``"), + $( Keyword::$kw_name => write!(f, "`{}`", $kw_param), )* + } + } + } + + impl Default for Keyword { + fn default() -> Self { + Keyword::EMPTY + } + } + + /// Struct containing all query parameters used in endpoints. Think of it as an HashMap, + /// but with keys statically known. + #[derive(Debug, Default)] + struct QueryParameters<'a> { + keyword: Option, + $( + $f_name: Option>, + )* + } + + impl<'a> QueryParameters<'a> { + /// Build this struct from the query part of an URI. + fn from_query(query: &'a str) -> Result { + let mut res: Self = Default::default(); + for (k, v) in url::form_urlencoded::parse(query.as_bytes()) { + match k.as_ref() { + $( + $kw_param => if let Some(prev_kw) = res.keyword.replace(Keyword::$kw_name) { + return Err(Error::bad_request(format!( + "Multiple keywords: '{}' and '{}'", prev_kw, $kw_param + ))); + }, + )* + $( + $f_param => if !v.is_empty() { + if res.$f_name.replace(v).is_some() { + return Err(Error::bad_request(format!( + "Query parameter repeated: '{}'", k + ))); + } + }, + )* + _ => { + if !(k.starts_with("response-") || k.starts_with("X-Amz-")) { + debug!("Received an unknown query parameter: '{}'", k); + } + } + }; + } + Ok(res) + } + + /// Get an error message in case not all parameters where used when extracting them to + /// build an Endpoint variant + fn nonempty_message(&self) -> Option<&str> { + if self.keyword.is_some() { + Some("Keyword not used") + } $( + else if self.$f_name.is_some() { + Some(concat!("'", $f_param, "'")) + } + )* else { + None + } + } + } + } +} + +pub use generateQueryParameters; +pub use router_match; diff --git a/src/api/common/signature/error.rs b/src/api/common/signature/error.rs new file mode 100644 index 00000000..2d92a072 --- /dev/null +++ b/src/api/common/signature/error.rs @@ -0,0 +1,32 @@ +use err_derive::Error; + +use crate::common_error::CommonError; +pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; + +/// Errors of this crate +#[derive(Debug, Error)] +pub enum Error { + #[error(display = "{}", _0)] + /// Error from common error + Common(CommonError), + + /// Authorization Header Malformed + #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] + AuthorizationHeaderMalformed(String), + + // Category: bad request + /// The request contained an invalid UTF-8 sequence in its path or in other parameters + #[error(display = "Invalid UTF-8: {}", _0)] + InvalidUtf8Str(#[error(source)] std::str::Utf8Error), +} + +impl From for Error +where + CommonError: From, +{ + fn from(err: T) -> Self { + Error::Common(CommonError::from(err)) + } +} + +impl CommonErrorDerivative for Error {} diff --git a/src/api/common/signature/mod.rs b/src/api/common/signature/mod.rs new file mode 100644 index 00000000..6514da43 --- /dev/null +++ b/src/api/common/signature/mod.rs @@ -0,0 +1,78 @@ +use chrono::{DateTime, Utc}; +use hmac::{Hmac, Mac}; +use sha2::Sha256; + +use hyper::{body::Incoming as IncomingBody, Request}; + +use garage_model::garage::Garage; +use garage_model::key_table::Key; +use garage_util::data::{sha256sum, Hash}; + +use error::*; + +pub mod error; +pub mod payload; +pub mod streaming; + +pub const SHORT_DATE: &str = "%Y%m%d"; +pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ"; + +type HmacSha256 = Hmac; + +pub async fn verify_request( + garage: &Garage, + mut req: Request, + service: &'static str, +) -> Result<(Request, Key, Option), Error> { + let (api_key, mut content_sha256) = + payload::check_payload_signature(&garage, &mut req, service).await?; + let api_key = + api_key.ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; + + let req = streaming::parse_streaming_body( + &api_key, + req, + &mut content_sha256, + &garage.config.s3_api.s3_region, + service, + )?; + + Ok((req, api_key, content_sha256)) +} + +pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> { + if expected_sha256 != sha256sum(body) { + return Err(Error::bad_request( + "Request content hash does not match signed hash".to_string(), + )); + } + Ok(()) +} + +pub fn signing_hmac( + datetime: &DateTime, + secret_key: &str, + region: &str, + service: &str, +) -> Result { + let secret = String::from("AWS4") + secret_key; + let mut date_hmac = HmacSha256::new_from_slice(secret.as_bytes())?; + date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes()); + let mut region_hmac = HmacSha256::new_from_slice(&date_hmac.finalize().into_bytes())?; + region_hmac.update(region.as_bytes()); + let mut service_hmac = HmacSha256::new_from_slice(®ion_hmac.finalize().into_bytes())?; + service_hmac.update(service.as_bytes()); + let mut signing_hmac = HmacSha256::new_from_slice(&service_hmac.finalize().into_bytes())?; + signing_hmac.update(b"aws4_request"); + let hmac = HmacSha256::new_from_slice(&signing_hmac.finalize().into_bytes())?; + Ok(hmac) +} + +pub fn compute_scope(datetime: &DateTime, region: &str, service: &str) -> String { + format!( + "{}/{}/{}/aws4_request", + datetime.format(SHORT_DATE), + region, + service + ) +} diff --git a/src/api/common/signature/payload.rs b/src/api/common/signature/payload.rs new file mode 100644 index 00000000..81541e4a --- /dev/null +++ b/src/api/common/signature/payload.rs @@ -0,0 +1,562 @@ +use std::collections::HashMap; +use std::convert::TryFrom; + +use chrono::{DateTime, Duration, NaiveDateTime, TimeZone, Utc}; +use hmac::Mac; +use hyper::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, HOST}; +use hyper::{body::Incoming as IncomingBody, Method, Request}; +use sha2::{Digest, Sha256}; + +use garage_table::*; +use garage_util::data::Hash; + +use garage_model::garage::Garage; +use garage_model::key_table::*; + +use super::LONG_DATETIME; +use super::{compute_scope, signing_hmac}; + +use crate::encoding::uri_encode; +use crate::signature::error::*; + +pub const X_AMZ_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-algorithm"); +pub const X_AMZ_CREDENTIAL: HeaderName = HeaderName::from_static("x-amz-credential"); +pub const X_AMZ_DATE: HeaderName = HeaderName::from_static("x-amz-date"); +pub const X_AMZ_EXPIRES: HeaderName = HeaderName::from_static("x-amz-expires"); +pub const X_AMZ_SIGNEDHEADERS: HeaderName = HeaderName::from_static("x-amz-signedheaders"); +pub const X_AMZ_SIGNATURE: HeaderName = HeaderName::from_static("x-amz-signature"); +pub const X_AMZ_CONTENT_SH256: HeaderName = HeaderName::from_static("x-amz-content-sha256"); + +pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256"; +pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; +pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + +pub type QueryMap = HeaderMap; +pub struct QueryValue { + /// Original key with potential uppercase characters, + /// for use in signature calculation + key: String, + value: String, +} + +pub async fn check_payload_signature( + garage: &Garage, + request: &mut Request, + service: &'static str, +) -> Result<(Option, Option), Error> { + let query = parse_query_map(request.uri())?; + + if query.contains_key(&X_AMZ_ALGORITHM) { + // We check for presigned-URL-style authentication first, because + // the browser or something else could inject an Authorization header + // that is totally unrelated to AWS signatures. + check_presigned_signature(garage, service, request, query).await + } else if request.headers().contains_key(AUTHORIZATION) { + check_standard_signature(garage, service, request, query).await + } else { + // Unsigned (anonymous) request + let content_sha256 = request + .headers() + .get("x-amz-content-sha256") + .filter(|c| c.as_bytes() != UNSIGNED_PAYLOAD.as_bytes()); + if let Some(content_sha256) = content_sha256 { + let sha256 = hex::decode(content_sha256) + .ok() + .and_then(|bytes| Hash::try_from(&bytes)) + .ok_or_bad_request("Invalid content sha256 hash")?; + Ok((None, Some(sha256))) + } else { + Ok((None, None)) + } + } +} + +async fn check_standard_signature( + garage: &Garage, + service: &'static str, + request: &Request, + query: QueryMap, +) -> Result<(Option, Option), Error> { + let authorization = Authorization::parse_header(request.headers())?; + + // Verify that all necessary request headers are included in signed_headers + // The following must be included for all signatures: + // - the Host header (mandatory) + // - all x-amz-* headers used in the request + // AWS also indicates that the Content-Type header should be signed if + // it is used, but Minio client doesn't sign it so we don't check it for compatibility. + let signed_headers = split_signed_headers(&authorization)?; + verify_signed_headers(request.headers(), &signed_headers)?; + + let canonical_request = canonical_request( + service, + request.method(), + request.uri().path(), + &query, + request.headers(), + &signed_headers, + &authorization.content_sha256, + )?; + let string_to_sign = string_to_sign( + &authorization.date, + &authorization.scope, + &canonical_request, + ); + + trace!("canonical request:\n{}", canonical_request); + trace!("string to sign:\n{}", string_to_sign); + + let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; + + let content_sha256 = if authorization.content_sha256 == UNSIGNED_PAYLOAD { + None + } else if authorization.content_sha256 == STREAMING_AWS4_HMAC_SHA256_PAYLOAD { + let bytes = hex::decode(authorization.signature).ok_or_bad_request("Invalid signature")?; + Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid signature")?) + } else { + let bytes = hex::decode(authorization.content_sha256) + .ok_or_bad_request("Invalid content sha256 hash")?; + Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid content sha256 hash")?) + }; + + Ok((Some(key), content_sha256)) +} + +async fn check_presigned_signature( + garage: &Garage, + service: &'static str, + request: &mut Request, + mut query: QueryMap, +) -> Result<(Option, Option), Error> { + let algorithm = query.get(&X_AMZ_ALGORITHM).unwrap(); + let authorization = Authorization::parse_presigned(&algorithm.value, &query)?; + + // Verify that all necessary request headers are included in signed_headers + // For AWSv4 pre-signed URLs, the following must be included: + // - the Host header (mandatory) + // - all x-amz-* headers used in the request + let signed_headers = split_signed_headers(&authorization)?; + verify_signed_headers(request.headers(), &signed_headers)?; + + // The X-Amz-Signature value is passed as a query parameter, + // but the signature cannot be computed from a string that contains itself. + // AWS specifies that all query params except X-Amz-Signature are included + // in the canonical request. + query.remove(&X_AMZ_SIGNATURE); + let canonical_request = canonical_request( + service, + request.method(), + request.uri().path(), + &query, + request.headers(), + &signed_headers, + &authorization.content_sha256, + )?; + let string_to_sign = string_to_sign( + &authorization.date, + &authorization.scope, + &canonical_request, + ); + + trace!("canonical request (presigned url):\n{}", canonical_request); + trace!("string to sign (presigned url):\n{}", string_to_sign); + + let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; + + // In the page on presigned URLs, AWS specifies that if a signed query + // parameter and a signed header of the same name have different values, + // then an InvalidRequest error is raised. + let headers_mut = request.headers_mut(); + for (name, value) in query.iter() { + if let Some(existing) = headers_mut.get(name) { + if signed_headers.contains(&name) && existing.as_bytes() != value.value.as_bytes() { + return Err(Error::bad_request(format!( + "Conflicting values for `{}` in query parameters and request headers", + name + ))); + } + } + if name.as_str().starts_with("x-amz-") { + // Query parameters that start by x-amz- are actually intended to stand in for + // headers that can't be added at the time the request is made. + // What we do is just add them to the Request object as regular headers, + // that will be handled downstream as if they were included like in a normal request. + // (Here we allow such query parameters to override headers with the same name + // that are not signed, however there is not much reason that this would happen) + headers_mut.insert( + name, + HeaderValue::from_bytes(value.value.as_bytes()) + .ok_or_bad_request("invalid query parameter value")?, + ); + } + } + + // Presigned URLs always use UNSIGNED-PAYLOAD, + // so there is no sha256 hash to return. + Ok((Some(key), None)) +} + +pub fn parse_query_map(uri: &http::uri::Uri) -> Result { + let mut query = QueryMap::with_capacity(0); + if let Some(query_str) = uri.query() { + let query_pairs = url::form_urlencoded::parse(query_str.as_bytes()); + for (key, val) in query_pairs { + let name = + HeaderName::from_bytes(key.as_bytes()).ok_or_bad_request("Invalid header name")?; + + let value = QueryValue { + key: key.to_string(), + value: val.into_owned(), + }; + + if query.insert(name, value).is_some() { + return Err(Error::bad_request(format!( + "duplicate query parameter: `{}`", + key + ))); + } + } + } + Ok(query) +} + +fn parse_credential(cred: &str) -> Result<(String, String), Error> { + let first_slash = cred + .find('/') + .ok_or_bad_request("Credentials does not contain '/' in authorization field")?; + let (key_id, scope) = cred.split_at(first_slash); + Ok(( + key_id.to_string(), + scope.trim_start_matches('/').to_string(), + )) +} + +fn split_signed_headers(authorization: &Authorization) -> Result, Error> { + let mut signed_headers = authorization + .signed_headers + .split(';') + .map(HeaderName::try_from) + .collect::, _>>() + .ok_or_bad_request("invalid header name")?; + signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); + Ok(signed_headers) +} + +fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) -> Result<(), Error> { + if !signed_headers.contains(&HOST) { + return Err(Error::bad_request("Header `Host` should be signed")); + } + for (name, _) in headers.iter() { + if name.as_str().starts_with("x-amz-") { + if !signed_headers.contains(name) { + return Err(Error::bad_request(format!( + "Header `{}` should be signed", + name + ))); + } + } + } + Ok(()) +} + +pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { + let mut hasher = Sha256::default(); + hasher.update(canonical_req.as_bytes()); + [ + AWS4_HMAC_SHA256, + &datetime.format(LONG_DATETIME).to_string(), + scope_string, + &hex::encode(hasher.finalize().as_slice()), + ] + .join("\n") +} + +pub fn canonical_request( + service: &'static str, + method: &Method, + canonical_uri: &str, + query: &QueryMap, + headers: &HeaderMap, + signed_headers: &[HeaderName], + content_sha256: &str, +) -> Result { + // There seems to be evidence that in AWSv4 signatures, the path component is url-encoded + // a second time when building the canonical request, as specified in this documentation page: + // -> https://docs.aws.amazon.com/rolesanywhere/latest/userguide/authentication-sign-process.html + // However this documentation page is for a specific service ("roles anywhere"), and + // in the S3 service we know for a fact that there is no double-urlencoding, because all of + // the tests we made with external software work without it. + // + // The theory is that double-urlencoding occurs for all services except S3, + // which is what is implemented in rusoto_signature: + // -> https://docs.rs/rusoto_signature/latest/src/rusoto_signature/signature.rs.html#464 + // + // Digging into the code of the official AWS Rust SDK, we learn that double-URI-encoding can + // be set or unset on a per-request basis (the signature crates, aws-sigv4 and aws-sig-auth, + // are agnostic to this). Grepping the codebase confirms that S3 is the only API for which + // double_uri_encode is set to false, meaning it is true (its default value) for all other + // AWS services. We will therefore implement this behavior in Garage as well. + // + // Note that this documentation page, which is touted as the "authoritative reference" on + // AWSv4 signatures, makes no mention of either single- or double-urlencoding: + // -> https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html + // This page of the S3 documentation does also not mention anything specific: + // -> https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html + // + // Note that there is also the issue of path normalization, which I hope is unrelated to the + // one of URI-encoding. At least in aws-sigv4 both parameters can be set independently, + // and rusoto_signature does not seem to do any effective path normalization, even though + // it mentions it in the comments (same link to the source code as above). + // We make the explicit choice of NOT normalizing paths in the K2V API because doing so + // would make non-normalized paths invalid K2V partition keys, and we don't want that. + let canonical_uri: std::borrow::Cow = if service != "s3" { + uri_encode(canonical_uri, false).into() + } else { + canonical_uri.into() + }; + + // Canonical query string from passed HeaderMap + let canonical_query_string = { + let mut items = Vec::with_capacity(query.len()); + for (_, QueryValue { key, value }) in query.iter() { + items.push(uri_encode(&key, true) + "=" + &uri_encode(&value, true)); + } + items.sort(); + items.join("&") + }; + + // Canonical header string calculated from signed headers + let canonical_header_string = signed_headers + .iter() + .map(|name| { + let value = headers + .get(name) + .ok_or_bad_request(format!("signed header `{}` is not present", name))?; + let value = std::str::from_utf8(value.as_bytes())?; + Ok(format!("{}:{}", name.as_str(), value.trim())) + }) + .collect::, Error>>()? + .join("\n"); + let signed_headers = signed_headers.join(";"); + + let list = [ + method.as_str(), + &canonical_uri, + &canonical_query_string, + &canonical_header_string, + "", + &signed_headers, + content_sha256, + ]; + Ok(list.join("\n")) +} + +pub fn parse_date(date: &str) -> Result, Error> { + let date: NaiveDateTime = + NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; + Ok(Utc.from_utc_datetime(&date)) +} + +pub async fn verify_v4( + garage: &Garage, + service: &str, + auth: &Authorization, + payload: &[u8], +) -> Result { + let scope_expected = compute_scope(&auth.date, &garage.config.s3_api.s3_region, service); + if auth.scope != scope_expected { + return Err(Error::AuthorizationHeaderMalformed(auth.scope.to_string())); + } + + let key = garage + .key_table + .get(&EmptyKey, &auth.key_id) + .await? + .filter(|k| !k.state.is_deleted()) + .ok_or_else(|| Error::forbidden(format!("No such key: {}", &auth.key_id)))?; + let key_p = key.params().unwrap(); + + let mut hmac = signing_hmac( + &auth.date, + &key_p.secret_key, + &garage.config.s3_api.s3_region, + service, + ) + .ok_or_internal_error("Unable to build signing HMAC")?; + hmac.update(payload); + let signature = + hex::decode(&auth.signature).map_err(|_| Error::forbidden("Invalid signature"))?; + if hmac.verify_slice(&signature).is_err() { + return Err(Error::forbidden("Invalid signature")); + } + + Ok(key) +} + +// ============ Authorization header, or X-Amz-* query params ========= + +pub struct Authorization { + key_id: String, + scope: String, + signed_headers: String, + signature: String, + content_sha256: String, + date: DateTime, +} + +impl Authorization { + fn parse_header(headers: &HeaderMap) -> Result { + let authorization = headers + .get(AUTHORIZATION) + .ok_or_bad_request("Missing authorization header")? + .to_str()?; + + let (auth_kind, rest) = authorization + .split_once(' ') + .ok_or_bad_request("Authorization field to short")?; + + if auth_kind != AWS4_HMAC_SHA256 { + return Err(Error::bad_request("Unsupported authorization method")); + } + + let mut auth_params = HashMap::new(); + for auth_part in rest.split(',') { + let auth_part = auth_part.trim(); + let eq = auth_part + .find('=') + .ok_or_bad_request("Field without value in authorization header")?; + let (key, value) = auth_part.split_at(eq); + auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string()); + } + + let cred = auth_params + .get("Credential") + .ok_or_bad_request("Could not find Credential in Authorization field")?; + let signed_headers = auth_params + .get("SignedHeaders") + .ok_or_bad_request("Could not find SignedHeaders in Authorization field")? + .to_string(); + let signature = auth_params + .get("Signature") + .ok_or_bad_request("Could not find Signature in Authorization field")? + .to_string(); + + let content_sha256 = headers + .get(X_AMZ_CONTENT_SH256) + .ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?; + + let date = headers + .get(X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field") + .map_err(Error::from)? + .to_str()?; + let date = parse_date(date)?; + + if Utc::now() - date > Duration::hours(24) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(cred)?; + let auth = Authorization { + key_id, + scope, + signed_headers, + signature, + content_sha256: content_sha256.to_str()?.to_string(), + date, + }; + Ok(auth) + } + + fn parse_presigned(algorithm: &str, query: &QueryMap) -> Result { + if algorithm != AWS4_HMAC_SHA256 { + return Err(Error::bad_request( + "Unsupported authorization method".to_string(), + )); + } + + let cred = query + .get(&X_AMZ_CREDENTIAL) + .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; + let signed_headers = query + .get(&X_AMZ_SIGNEDHEADERS) + .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; + let signature = query + .get(&X_AMZ_SIGNATURE) + .ok_or_bad_request("X-Amz-Signature not found in query parameters")?; + + let duration = query + .get(&X_AMZ_EXPIRES) + .ok_or_bad_request("X-Amz-Expires not found in query parameters")? + .value + .parse() + .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; + + if duration > 7 * 24 * 3600 { + return Err(Error::bad_request( + "X-Amz-Expires may not exceed a week".to_string(), + )); + } + + let date = query + .get(&X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field")?; + let date = parse_date(&date.value)?; + + if Utc::now() - date > Duration::seconds(duration) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(&cred.value)?; + Ok(Authorization { + key_id, + scope, + signed_headers: signed_headers.value.clone(), + signature: signature.value.clone(), + content_sha256: UNSIGNED_PAYLOAD.to_string(), + date, + }) + } + + pub fn parse_form(params: &HeaderMap) -> Result { + let algorithm = params + .get(X_AMZ_ALGORITHM) + .ok_or_bad_request("Missing X-Amz-Algorithm header")? + .to_str()?; + if algorithm != AWS4_HMAC_SHA256 { + return Err(Error::bad_request( + "Unsupported authorization method".to_string(), + )); + } + + let credential = params + .get(X_AMZ_CREDENTIAL) + .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? + .to_str()?; + let signature = params + .get(X_AMZ_SIGNATURE) + .ok_or_bad_request("No signature was provided")? + .to_str()? + .to_string(); + let date = params + .get(X_AMZ_DATE) + .ok_or_bad_request("No date was provided")? + .to_str()?; + let date = parse_date(date)?; + + if Utc::now() - date > Duration::hours(24) { + return Err(Error::bad_request("Date is too old".to_string())); + } + + let (key_id, scope) = parse_credential(credential)?; + let auth = Authorization { + key_id, + scope, + signed_headers: "".to_string(), + signature, + content_sha256: UNSIGNED_PAYLOAD.to_string(), + date, + }; + Ok(auth) + } +} diff --git a/src/api/common/signature/streaming.rs b/src/api/common/signature/streaming.rs new file mode 100644 index 00000000..e223d1b1 --- /dev/null +++ b/src/api/common/signature/streaming.rs @@ -0,0 +1,373 @@ +use std::pin::Pin; + +use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; +use futures::prelude::*; +use futures::task; +use garage_model::key_table::Key; +use hmac::Mac; +use http_body_util::StreamBody; +use hyper::body::{Bytes, Incoming as IncomingBody}; +use hyper::Request; + +use garage_util::data::Hash; + +use super::{compute_scope, sha256sum, HmacSha256, LONG_DATETIME}; + +use crate::helpers::*; +use crate::signature::error::*; +use crate::signature::payload::{ + STREAMING_AWS4_HMAC_SHA256_PAYLOAD, X_AMZ_CONTENT_SH256, X_AMZ_DATE, +}; + +pub const AWS4_HMAC_SHA256_PAYLOAD: &str = "AWS4-HMAC-SHA256-PAYLOAD"; + +pub type ReqBody = BoxBody; + +pub fn parse_streaming_body( + api_key: &Key, + req: Request, + content_sha256: &mut Option, + region: &str, + service: &str, +) -> Result, Error> { + match req.headers().get(X_AMZ_CONTENT_SH256) { + Some(header) if header == STREAMING_AWS4_HMAC_SHA256_PAYLOAD => { + let signature = content_sha256 + .take() + .ok_or_bad_request("No signature provided")?; + + let secret_key = &api_key + .state + .as_option() + .ok_or_internal_error("Deleted key state")? + .secret_key; + + let date = req + .headers() + .get(X_AMZ_DATE) + .ok_or_bad_request("Missing X-Amz-Date field")? + .to_str()?; + let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME) + .ok_or_bad_request("Invalid date")?; + let date: DateTime = Utc.from_utc_datetime(&date); + + let scope = compute_scope(&date, region, service); + let signing_hmac = crate::signature::signing_hmac(&date, secret_key, region, service) + .ok_or_internal_error("Unable to build signing HMAC")?; + + Ok(req.map(move |body| { + let stream = body_stream::<_, Error>(body); + let signed_payload_stream = + SignedPayloadStream::new(stream, signing_hmac, date, &scope, signature) + .map(|x| x.map(hyper::body::Frame::data)) + .map_err(Error::from); + ReqBody::new(StreamBody::new(signed_payload_stream)) + })) + } + _ => Ok(req.map(|body| ReqBody::new(http_body_util::BodyExt::map_err(body, Error::from)))), + } +} + +/// Result of `sha256("")` +const EMPTY_STRING_HEX_DIGEST: &str = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +fn compute_streaming_payload_signature( + signing_hmac: &HmacSha256, + date: DateTime, + scope: &str, + previous_signature: Hash, + content_sha256: Hash, +) -> Result { + let string_to_sign = [ + AWS4_HMAC_SHA256_PAYLOAD, + &date.format(LONG_DATETIME).to_string(), + scope, + &hex::encode(previous_signature), + EMPTY_STRING_HEX_DIGEST, + &hex::encode(content_sha256), + ] + .join("\n"); + + let mut hmac = signing_hmac.clone(); + hmac.update(string_to_sign.as_bytes()); + + Ok(Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature")?) +} + +mod payload { + use garage_util::data::Hash; + + pub enum Error { + Parser(nom::error::Error), + BadSignature, + } + + impl Error { + pub fn description(&self) -> &str { + match *self { + Error::Parser(ref e) => e.code.description(), + Error::BadSignature => "Bad signature", + } + } + } + + #[derive(Debug, Clone)] + pub struct Header { + pub size: usize, + pub signature: Hash, + } + + impl Header { + pub fn parse(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { + use nom::bytes::streaming::tag; + use nom::character::streaming::hex_digit1; + use nom::combinator::map_res; + use nom::number::streaming::hex_u32; + + macro_rules! try_parse { + ($expr:expr) => { + $expr.map_err(|e| e.map(Error::Parser))? + }; + } + + let (input, size) = try_parse!(hex_u32(input)); + let (input, _) = try_parse!(tag(";")(input)); + + let (input, _) = try_parse!(tag("chunk-signature=")(input)); + let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input)); + let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?; + + let (input, _) = try_parse!(tag("\r\n")(input)); + + let header = Header { + size: size as usize, + signature, + }; + + Ok((input, header)) + } + } +} + +#[derive(Debug)] +pub enum SignedPayloadStreamError { + Stream(Error), + InvalidSignature, + Message(String), +} + +impl SignedPayloadStreamError { + fn message(msg: &str) -> Self { + SignedPayloadStreamError::Message(msg.into()) + } +} + +impl From for Error { + fn from(err: SignedPayloadStreamError) -> Self { + match err { + SignedPayloadStreamError::Stream(e) => e, + SignedPayloadStreamError::InvalidSignature => { + Error::bad_request("Invalid payload signature") + } + SignedPayloadStreamError::Message(e) => { + Error::bad_request(format!("Chunk format error: {}", e)) + } + } + } +} + +impl From> for SignedPayloadStreamError { + fn from(err: payload::Error) -> Self { + Self::message(err.description()) + } +} + +impl From> for SignedPayloadStreamError { + fn from(err: nom::error::Error) -> Self { + Self::message(err.code.description()) + } +} + +struct SignedPayload { + header: payload::Header, + data: Bytes, +} + +#[pin_project::pin_project] +pub struct SignedPayloadStream +where + S: Stream>, +{ + #[pin] + stream: S, + buf: bytes::BytesMut, + datetime: DateTime, + scope: String, + signing_hmac: HmacSha256, + previous_signature: Hash, +} + +impl SignedPayloadStream +where + S: Stream>, +{ + pub fn new( + stream: S, + signing_hmac: HmacSha256, + datetime: DateTime, + scope: &str, + seed_signature: Hash, + ) -> Self { + Self { + stream, + buf: bytes::BytesMut::new(), + datetime, + scope: scope.into(), + signing_hmac, + previous_signature: seed_signature, + } + } + + fn parse_next(input: &[u8]) -> nom::IResult<&[u8], SignedPayload, SignedPayloadStreamError> { + use nom::bytes::streaming::{tag, take}; + + macro_rules! try_parse { + ($expr:expr) => { + $expr.map_err(nom::Err::convert)? + }; + } + + let (input, header) = try_parse!(payload::Header::parse(input)); + + // 0-sized chunk is the last + if header.size == 0 { + return Ok(( + input, + SignedPayload { + header, + data: Bytes::new(), + }, + )); + } + + let (input, data) = try_parse!(take::<_, _, nom::error::Error<_>>(header.size)(input)); + let (input, _) = try_parse!(tag::<_, _, nom::error::Error<_>>("\r\n")(input)); + + let data = Bytes::from(data.to_vec()); + + Ok((input, SignedPayload { header, data })) + } +} + +impl Stream for SignedPayloadStream +where + S: Stream> + Unpin, +{ + type Item = Result; + + fn poll_next( + self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + ) -> task::Poll> { + use std::task::Poll; + + let mut this = self.project(); + + loop { + let (input, payload) = match Self::parse_next(this.buf) { + Ok(res) => res, + Err(nom::Err::Incomplete(_)) => { + match futures::ready!(this.stream.as_mut().poll_next(cx)) { + Some(Ok(bytes)) => { + this.buf.extend(bytes); + continue; + } + Some(Err(e)) => { + return Poll::Ready(Some(Err(SignedPayloadStreamError::Stream(e)))) + } + None => { + return Poll::Ready(Some(Err(SignedPayloadStreamError::message( + "Unexpected EOF", + )))); + } + } + } + Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { + return Poll::Ready(Some(Err(e))) + } + }; + + // 0-sized chunk is the last + if payload.data.is_empty() { + return Poll::Ready(None); + } + + let data_sha256sum = sha256sum(&payload.data); + + let expected_signature = compute_streaming_payload_signature( + this.signing_hmac, + *this.datetime, + this.scope, + *this.previous_signature, + data_sha256sum, + ) + .map_err(|e| { + SignedPayloadStreamError::Message(format!("Could not build signature: {}", e)) + })?; + + if payload.header.signature != expected_signature { + return Poll::Ready(Some(Err(SignedPayloadStreamError::InvalidSignature))); + } + + *this.buf = input.into(); + *this.previous_signature = payload.header.signature; + + return Poll::Ready(Some(Ok(payload.data))); + } + } + + fn size_hint(&self) -> (usize, Option) { + self.stream.size_hint() + } +} + +#[cfg(test)] +mod tests { + use futures::prelude::*; + + use super::{SignedPayloadStream, SignedPayloadStreamError}; + + #[tokio::test] + async fn test_interrupted_signed_payload_stream() { + use chrono::{DateTime, Utc}; + + use garage_util::data::Hash; + + let datetime = DateTime::parse_from_rfc3339("2021-12-13T13:12:42+01:00") // TODO UNIX 0 + .unwrap() + .with_timezone(&Utc); + let secret_key = "test"; + let region = "test"; + let scope = crate::signature::compute_scope(&datetime, region, "s3"); + let signing_hmac = + crate::signature::signing_hmac(&datetime, secret_key, region, "s3").unwrap(); + + let data: &[&[u8]] = &[b"1"]; + let body = futures::stream::iter(data.iter().map(|block| Ok(block.to_vec().into()))); + + let seed_signature = Hash::default(); + + let mut stream = + SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature); + + assert!(stream.try_next().await.is_err()); + match stream.try_next().await { + Err(SignedPayloadStreamError::Message(msg)) if msg == "Unexpected EOF" => {} + item => panic!( + "Unexpected result, expected early EOF error, got {:?}", + item + ), + } + } +} diff --git a/src/api/common_error.rs b/src/api/common_error.rs deleted file mode 100644 index 0c8006dc..00000000 --- a/src/api/common_error.rs +++ /dev/null @@ -1,217 +0,0 @@ -use std::convert::TryFrom; - -use err_derive::Error; -use hyper::StatusCode; - -use garage_util::error::Error as GarageError; - -use garage_model::helper::error::Error as HelperError; - -/// Errors of this crate -#[derive(Debug, Error)] -pub enum CommonError { - // ---- INTERNAL ERRORS ---- - /// Error related to deeper parts of Garage - #[error(display = "Internal error: {}", _0)] - InternalError(#[error(source)] GarageError), - - /// Error related to Hyper - #[error(display = "Internal error (Hyper error): {}", _0)] - Hyper(#[error(source)] hyper::Error), - - /// Error related to HTTP - #[error(display = "Internal error (HTTP error): {}", _0)] - Http(#[error(source)] http::Error), - - // ---- GENERIC CLIENT ERRORS ---- - /// Proper authentication was not provided - #[error(display = "Forbidden: {}", _0)] - Forbidden(String), - - /// Generic bad request response with custom message - #[error(display = "Bad request: {}", _0)] - BadRequest(String), - - /// The client sent a header with invalid value - #[error(display = "Invalid header value: {}", _0)] - InvalidHeader(#[error(source)] hyper::header::ToStrError), - - // ---- SPECIFIC ERROR CONDITIONS ---- - // These have to be error codes referenced in the S3 spec here: - // https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList - /// The bucket requested don't exists - #[error(display = "Bucket not found: {}", _0)] - NoSuchBucket(String), - - /// Tried to create a bucket that already exist - #[error(display = "Bucket already exists")] - BucketAlreadyExists, - - /// Tried to delete a non-empty bucket - #[error(display = "Tried to delete a non-empty bucket")] - BucketNotEmpty, - - // Category: bad request - /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name: {}", _0)] - InvalidBucketName(String), -} - -impl CommonError { - pub fn http_status_code(&self) -> StatusCode { - match self { - CommonError::InternalError( - GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..), - ) => StatusCode::SERVICE_UNAVAILABLE, - CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { - StatusCode::INTERNAL_SERVER_ERROR - } - CommonError::BadRequest(_) => StatusCode::BAD_REQUEST, - CommonError::Forbidden(_) => StatusCode::FORBIDDEN, - CommonError::NoSuchBucket(_) => StatusCode::NOT_FOUND, - CommonError::BucketNotEmpty | CommonError::BucketAlreadyExists => StatusCode::CONFLICT, - CommonError::InvalidBucketName(_) | CommonError::InvalidHeader(_) => { - StatusCode::BAD_REQUEST - } - } - } - - pub fn aws_code(&self) -> &'static str { - match self { - CommonError::Forbidden(_) => "AccessDenied", - CommonError::InternalError( - GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(..), - ) => "ServiceUnavailable", - CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { - "InternalError" - } - CommonError::BadRequest(_) => "InvalidRequest", - CommonError::NoSuchBucket(_) => "NoSuchBucket", - CommonError::BucketAlreadyExists => "BucketAlreadyExists", - CommonError::BucketNotEmpty => "BucketNotEmpty", - CommonError::InvalidBucketName(_) => "InvalidBucketName", - CommonError::InvalidHeader(_) => "InvalidHeaderValue", - } - } - - pub fn bad_request(msg: M) -> Self { - CommonError::BadRequest(msg.to_string()) - } -} - -impl TryFrom for CommonError { - type Error = HelperError; - - fn try_from(err: HelperError) -> Result { - match err { - HelperError::Internal(i) => Ok(Self::InternalError(i)), - HelperError::BadRequest(b) => Ok(Self::BadRequest(b)), - HelperError::InvalidBucketName(n) => Ok(Self::InvalidBucketName(n)), - HelperError::NoSuchBucket(n) => Ok(Self::NoSuchBucket(n)), - e => Err(e), - } - } -} - -/// This function converts HelperErrors into CommonErrors, -/// for variants that exist in CommonError. -/// This is used for helper functions that might return InvalidBucketName -/// or NoSuchBucket for instance, and we want to pass that error -/// up to our caller. -pub(crate) fn pass_helper_error(err: HelperError) -> CommonError { - match CommonError::try_from(err) { - Ok(e) => e, - Err(e) => panic!("Helper error `{}` should hot have happenned here", e), - } -} - -pub(crate) fn helper_error_as_internal(err: HelperError) -> CommonError { - match err { - HelperError::Internal(e) => CommonError::InternalError(e), - e => CommonError::InternalError(GarageError::Message(e.to_string())), - } -} - -pub trait CommonErrorDerivative: From { - fn internal_error(msg: M) -> Self { - Self::from(CommonError::InternalError(GarageError::Message( - msg.to_string(), - ))) - } - - fn bad_request(msg: M) -> Self { - Self::from(CommonError::BadRequest(msg.to_string())) - } - - fn forbidden(msg: M) -> Self { - Self::from(CommonError::Forbidden(msg.to_string())) - } -} - -/// Trait to map error to the Bad Request error code -pub trait OkOrBadRequest { - type S; - fn ok_or_bad_request>(self, reason: M) -> Result; -} - -impl OkOrBadRequest for Result -where - E: std::fmt::Display, -{ - type S = T; - fn ok_or_bad_request>(self, reason: M) -> Result { - match self { - Ok(x) => Ok(x), - Err(e) => Err(CommonError::BadRequest(format!( - "{}: {}", - reason.as_ref(), - e - ))), - } - } -} - -impl OkOrBadRequest for Option { - type S = T; - fn ok_or_bad_request>(self, reason: M) -> Result { - match self { - Some(x) => Ok(x), - None => Err(CommonError::BadRequest(reason.as_ref().to_string())), - } - } -} - -/// Trait to map an error to an Internal Error code -pub trait OkOrInternalError { - type S; - fn ok_or_internal_error>(self, reason: M) -> Result; -} - -impl OkOrInternalError for Result -where - E: std::fmt::Display, -{ - type S = T; - fn ok_or_internal_error>(self, reason: M) -> Result { - match self { - Ok(x) => Ok(x), - Err(e) => Err(CommonError::InternalError(GarageError::Message(format!( - "{}: {}", - reason.as_ref(), - e - )))), - } - } -} - -impl OkOrInternalError for Option { - type S = T; - fn ok_or_internal_error>(self, reason: M) -> Result { - match self { - Some(x) => Ok(x), - None => Err(CommonError::InternalError(GarageError::Message( - reason.as_ref().to_string(), - ))), - } - } -} diff --git a/src/api/encoding.rs b/src/api/encoding.rs deleted file mode 100644 index e286a784..00000000 --- a/src/api/encoding.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Module containing various helpers for encoding - -/// Encode &str for use in a URI -pub fn uri_encode(string: &str, encode_slash: bool) -> String { - let mut result = String::with_capacity(string.len() * 2); - for c in string.chars() { - match c { - 'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' | '~' | '.' => result.push(c), - '/' if encode_slash => result.push_str("%2F"), - '/' if !encode_slash => result.push('/'), - _ => { - result.push_str( - &format!("{}", c) - .bytes() - .map(|b| format!("%{:02X}", b)) - .collect::(), - ); - } - } - } - result -} diff --git a/src/api/generic_server.rs b/src/api/generic_server.rs deleted file mode 100644 index 283abdd4..00000000 --- a/src/api/generic_server.rs +++ /dev/null @@ -1,378 +0,0 @@ -use std::convert::Infallible; -use std::fs::{self, Permissions}; -use std::os::unix::fs::PermissionsExt; -use std::sync::Arc; -use std::time::Duration; - -use async_trait::async_trait; - -use futures::future::Future; -use futures::stream::{futures_unordered::FuturesUnordered, StreamExt}; - -use http_body_util::BodyExt; -use hyper::header::HeaderValue; -use hyper::server::conn::http1; -use hyper::service::service_fn; -use hyper::{body::Incoming as IncomingBody, Request, Response}; -use hyper::{HeaderMap, StatusCode}; -use hyper_util::rt::TokioIo; - -use tokio::io::{AsyncRead, AsyncWrite}; -use tokio::net::{TcpListener, TcpStream, UnixListener, UnixStream}; -use tokio::sync::watch; -use tokio::time::{sleep_until, Instant}; - -use opentelemetry::{ - global, - metrics::{Counter, ValueRecorder}, - trace::{FutureExt, SpanRef, TraceContextExt, Tracer}, - Context, KeyValue, -}; - -use garage_util::error::Error as GarageError; -use garage_util::forwarded_headers; -use garage_util::metrics::{gen_trace_id, RecordDuration}; -use garage_util::socket_address::UnixOrTCPSocketAddress; - -use crate::helpers::{BoxBody, ErrorBody}; - -pub(crate) trait ApiEndpoint: Send + Sync + 'static { - fn name(&self) -> &'static str; - fn add_span_attributes(&self, span: SpanRef<'_>); -} - -pub trait ApiError: std::error::Error + Send + Sync + 'static { - fn http_status_code(&self) -> StatusCode; - fn add_http_headers(&self, header_map: &mut HeaderMap); - fn http_body(&self, garage_region: &str, path: &str) -> ErrorBody; -} - -#[async_trait] -pub(crate) trait ApiHandler: Send + Sync + 'static { - const API_NAME: &'static str; - const API_NAME_DISPLAY: &'static str; - - type Endpoint: ApiEndpoint; - type Error: ApiError; - - fn parse_endpoint(&self, r: &Request) -> Result; - async fn handle( - &self, - req: Request, - endpoint: Self::Endpoint, - ) -> Result>, Self::Error>; -} - -pub(crate) struct ApiServer { - region: String, - api_handler: A, - - // Metrics - request_counter: Counter, - error_counter: Counter, - request_duration: ValueRecorder, -} - -impl ApiServer { - pub fn new(region: String, api_handler: A) -> Arc { - let meter = global::meter("garage/api"); - Arc::new(Self { - region, - api_handler, - request_counter: meter - .u64_counter(format!("api.{}.request_counter", A::API_NAME)) - .with_description(format!( - "Number of API calls to the various {} API endpoints", - A::API_NAME_DISPLAY - )) - .init(), - error_counter: meter - .u64_counter(format!("api.{}.error_counter", A::API_NAME)) - .with_description(format!( - "Number of API calls to the various {} API endpoints that resulted in errors", - A::API_NAME_DISPLAY - )) - .init(), - request_duration: meter - .f64_value_recorder(format!("api.{}.request_duration", A::API_NAME)) - .with_description(format!( - "Duration of API calls to the various {} API endpoints", - A::API_NAME_DISPLAY - )) - .init(), - }) - } - - pub async fn run_server( - self: Arc, - bind_addr: UnixOrTCPSocketAddress, - unix_bind_addr_mode: Option, - must_exit: watch::Receiver, - ) -> Result<(), GarageError> { - let server_name = format!("{} API", A::API_NAME_DISPLAY); - info!("{} server listening on {}", server_name, bind_addr); - - match bind_addr { - UnixOrTCPSocketAddress::TCPSocket(addr) => { - let listener = TcpListener::bind(addr).await?; - - let handler = move |request, socketaddr| self.clone().handler(request, socketaddr); - server_loop(server_name, listener, handler, must_exit).await - } - UnixOrTCPSocketAddress::UnixSocket(ref path) => { - if path.exists() { - fs::remove_file(path)? - } - - let listener = UnixListener::bind(path)?; - let listener = UnixListenerOn(listener, path.display().to_string()); - - fs::set_permissions( - path, - Permissions::from_mode(unix_bind_addr_mode.unwrap_or(0o222)), - )?; - - let handler = move |request, socketaddr| self.clone().handler(request, socketaddr); - server_loop(server_name, listener, handler, must_exit).await - } - } - } - - async fn handler( - self: Arc, - req: Request, - addr: String, - ) -> Result>, http::Error> { - let uri = req.uri().clone(); - - if let Ok(forwarded_for_ip_addr) = - forwarded_headers::handle_forwarded_for_headers(req.headers()) - { - info!( - "{} (via {}) {} {}", - forwarded_for_ip_addr, - addr, - req.method(), - uri - ); - } else { - info!("{} {} {}", addr, req.method(), uri); - } - debug!("{:?}", req); - - let tracer = opentelemetry::global::tracer("garage"); - let span = tracer - .span_builder(format!("{} API call (unknown)", A::API_NAME_DISPLAY)) - .with_trace_id(gen_trace_id()) - .with_attributes(vec![ - KeyValue::new("method", format!("{}", req.method())), - KeyValue::new("uri", req.uri().to_string()), - ]) - .start(&tracer); - - let res = self - .handler_stage2(req) - .with_context(Context::current_with_span(span)) - .await; - - match res { - Ok(x) => { - debug!("{} {:?}", x.status(), x.headers()); - Ok(x) - } - Err(e) => { - let body = e.http_body(&self.region, uri.path()); - let mut http_error_builder = Response::builder().status(e.http_status_code()); - - if let Some(header_map) = http_error_builder.headers_mut() { - e.add_http_headers(header_map) - } - - let http_error = http_error_builder.body(body)?; - - if e.http_status_code().is_server_error() { - warn!("Response: error {}, {}", e.http_status_code(), e); - } else { - info!("Response: error {}, {}", e.http_status_code(), e); - } - Ok(http_error - .map(|body| BoxBody::new(body.map_err(|_: Infallible| unreachable!())))) - } - } - } - - async fn handler_stage2( - &self, - req: Request, - ) -> Result>, A::Error> { - let endpoint = self.api_handler.parse_endpoint(&req)?; - debug!("Endpoint: {}", endpoint.name()); - - let current_context = Context::current(); - let current_span = current_context.span(); - current_span.update_name::(format!( - "{} API {}", - A::API_NAME_DISPLAY, - endpoint.name() - )); - current_span.set_attribute(KeyValue::new("endpoint", endpoint.name())); - endpoint.add_span_attributes(current_span); - - let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())]; - - let res = self - .api_handler - .handle(req, endpoint) - .record_duration(&self.request_duration, &metrics_tags[..]) - .await; - - self.request_counter.add(1, &metrics_tags[..]); - - let status_code = match &res { - Ok(r) => r.status(), - Err(e) => e.http_status_code(), - }; - if status_code.is_client_error() || status_code.is_server_error() { - self.error_counter.add( - 1, - &[ - metrics_tags[0].clone(), - KeyValue::new("status_code", status_code.as_str().to_string()), - ], - ); - } - - res - } -} - -// ==== helper functions ==== - -#[async_trait] -pub trait Accept: Send + Sync + 'static { - type Stream: AsyncRead + AsyncWrite + Send + Sync + 'static; - async fn accept(&self) -> std::io::Result<(Self::Stream, String)>; -} - -#[async_trait] -impl Accept for TcpListener { - type Stream = TcpStream; - async fn accept(&self) -> std::io::Result<(Self::Stream, String)> { - self.accept() - .await - .map(|(stream, addr)| (stream, addr.to_string())) - } -} - -pub struct UnixListenerOn(pub UnixListener, pub String); - -#[async_trait] -impl Accept for UnixListenerOn { - type Stream = UnixStream; - async fn accept(&self) -> std::io::Result<(Self::Stream, String)> { - self.0 - .accept() - .await - .map(|(stream, _addr)| (stream, self.1.clone())) - } -} - -pub async fn server_loop( - server_name: String, - listener: A, - handler: H, - mut must_exit: watch::Receiver, -) -> Result<(), GarageError> -where - A: Accept, - H: Fn(Request, String) -> F + Send + Sync + Clone + 'static, - F: Future>, http::Error>> + Send + 'static, - E: Send + Sync + std::error::Error + 'static, -{ - let (conn_in, mut conn_out) = tokio::sync::mpsc::unbounded_channel(); - let connection_collector = tokio::spawn({ - let server_name = server_name.clone(); - async move { - let mut connections = FuturesUnordered::>::new(); - loop { - let collect_next = async { - if connections.is_empty() { - futures::future::pending().await - } else { - connections.next().await - } - }; - tokio::select! { - result = collect_next => { - trace!("{} server: HTTP connection finished: {:?}", server_name, result); - } - new_fut = conn_out.recv() => { - match new_fut { - Some(f) => connections.push(f), - None => break, - } - } - } - } - let deadline = Instant::now() + Duration::from_secs(10); - while !connections.is_empty() { - info!( - "{} server: {} connections still open, deadline in {:.2}s", - server_name, - connections.len(), - (deadline - Instant::now()).as_secs_f32(), - ); - tokio::select! { - conn_res = connections.next() => { - trace!( - "{} server: HTTP connection finished: {:?}", - server_name, - conn_res.unwrap(), - ); - } - _ = sleep_until(deadline) => { - warn!("{} server: exit deadline reached with {} connections still open, killing them now", - server_name, - connections.len()); - for conn in connections.iter() { - conn.abort(); - } - for conn in connections { - assert!(conn.await.unwrap_err().is_cancelled()); - } - break; - } - } - } - } - }); - - while !*must_exit.borrow() { - let (stream, client_addr) = tokio::select! { - acc = listener.accept() => acc?, - _ = must_exit.changed() => continue, - }; - - let io = TokioIo::new(stream); - - let handler = handler.clone(); - let serve = move |req: Request| handler(req, client_addr.clone()); - - let fut = tokio::task::spawn(async move { - let io = Box::pin(io); - if let Err(e) = http1::Builder::new() - .serve_connection(io, service_fn(serve)) - .await - { - debug!("Error handling HTTP connection: {}", e); - } - }); - conn_in.send(fut)?; - } - - info!("{} server exiting", server_name); - drop(conn_in); - connection_collector.await?; - - Ok(()) -} diff --git a/src/api/helpers.rs b/src/api/helpers.rs deleted file mode 100644 index cf60005d..00000000 --- a/src/api/helpers.rs +++ /dev/null @@ -1,371 +0,0 @@ -use std::convert::Infallible; -use std::sync::Arc; - -use futures::{Stream, StreamExt, TryStreamExt}; - -use http_body_util::{BodyExt, Full as FullBody}; -use hyper::{ - body::{Body, Bytes}, - Request, Response, -}; -use idna::domain_to_unicode; -use serde::{Deserialize, Serialize}; - -use garage_model::bucket_table::BucketParams; -use garage_model::garage::Garage; -use garage_model::key_table::Key; -use garage_util::data::Uuid; -use garage_util::error::Error as GarageError; - -use crate::common_error::{CommonError as Error, *}; - -/// What kind of authorization is required to perform a given action -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Authorization { - /// No authorization is required - None, - /// Having Read permission on bucket - Read, - /// Having Write permission on bucket - Write, - /// Having Owner permission on bucket - Owner, -} - -/// The values which are known for each request related to a bucket -pub struct ReqCtx { - pub garage: Arc, - pub bucket_id: Uuid, - pub bucket_name: String, - pub bucket_params: BucketParams, - pub api_key: Key, -} - -/// Host to bucket -/// -/// Convert a host, like "bucket.garage-site.tld" to the corresponding bucket "bucket", -/// considering that ".garage-site.tld" is the "root domain". For domains not matching -/// the provided root domain, no bucket is returned -/// This behavior has been chosen to follow AWS S3 semantic. -pub fn host_to_bucket<'a>(host: &'a str, root: &str) -> Option<&'a str> { - let root = root.trim_start_matches('.'); - let label_root = root.chars().filter(|c| c == &'.').count() + 1; - let root = root.rsplit('.'); - let mut host = host.rsplitn(label_root + 1, '.'); - for root_part in root { - let host_part = host.next()?; - if root_part != host_part { - return None; - } - } - host.next() -} - -/// Extract host from the authority section given by the HTTP host header -/// -/// The HTTP host contains both a host and a port. -/// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6 -/// We do not use the collect pattern as there is no way in std rust to collect over a stack allocated value -/// check here: -pub fn authority_to_host(authority: &str) -> Result { - let mut iter = authority.chars().enumerate(); - let (_, first_char) = iter - .next() - .ok_or_else(|| Error::bad_request("Authority is empty".to_string()))?; - - let split = match first_char { - '[' => { - let mut iter = iter.skip_while(|(_, c)| c != &']'); - match iter.next() { - Some((_, ']')) => iter.next(), - _ => { - return Err(Error::bad_request(format!( - "Authority {} has an illegal format", - authority - ))) - } - } - } - _ => iter.find(|(_, c)| *c == ':'), - }; - - let authority = match split { - Some((i, ':')) => Ok(&authority[..i]), - None => Ok(authority), - Some((_, _)) => Err(Error::bad_request(format!( - "Authority {} has an illegal format", - authority - ))), - }; - authority.map(|h| domain_to_unicode(h).0) -} - -/// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in -/// the host header of the request -/// -/// S3 internally manages only buckets and keys. This function splits -/// an HTTP path to get the corresponding bucket name and key. -pub fn parse_bucket_key<'a>( - path: &'a str, - host_bucket: Option<&'a str>, -) -> Result<(&'a str, Option<&'a str>), Error> { - let path = path.trim_start_matches('/'); - - if let Some(bucket) = host_bucket { - if !path.is_empty() { - return Ok((bucket, Some(path))); - } else { - return Ok((bucket, None)); - } - } - - let (bucket, key) = match path.find('/') { - Some(i) => { - let key = &path[i + 1..]; - if !key.is_empty() { - (&path[..i], Some(key)) - } else { - (&path[..i], None) - } - } - None => (path, None), - }; - if bucket.is_empty() { - return Err(Error::bad_request("No bucket specified")); - } - Ok((bucket, key)) -} - -const UTF8_BEFORE_LAST_CHAR: char = '\u{10FFFE}'; - -/// Compute the key after the prefix -pub fn key_after_prefix(pfx: &str) -> Option { - let mut next = pfx.to_string(); - while !next.is_empty() { - let tail = next.pop().unwrap(); - if tail >= char::MAX { - continue; - } - - // Circumvent a limitation of RangeFrom that overflow earlier than needed - // See: https://doc.rust-lang.org/core/ops/struct.RangeFrom.html - let new_tail = if tail == UTF8_BEFORE_LAST_CHAR { - char::MAX - } else { - (tail..).nth(1).unwrap() - }; - - next.push(new_tail); - return Some(next); - } - - None -} - -// =============== body helpers ================= - -pub type EmptyBody = http_body_util::Empty; -pub type ErrorBody = FullBody; -pub type BoxBody = http_body_util::combinators::BoxBody; - -pub fn string_body(s: String) -> BoxBody { - bytes_body(bytes::Bytes::from(s.into_bytes())) -} -pub fn bytes_body(b: bytes::Bytes) -> BoxBody { - BoxBody::new(FullBody::new(b).map_err(|_: Infallible| unreachable!())) -} -pub fn empty_body() -> BoxBody { - BoxBody::new(http_body_util::Empty::new().map_err(|_: Infallible| unreachable!())) -} -pub fn error_body(s: String) -> ErrorBody { - ErrorBody::from(bytes::Bytes::from(s.into_bytes())) -} - -pub async fn parse_json_body(req: Request) -> Result -where - T: for<'de> Deserialize<'de>, - B: Body, - E: From<::Error> + From, -{ - let body = req.into_body().collect().await?.to_bytes(); - let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; - Ok(resp) -} - -pub fn json_ok_response(res: &T) -> Result>, E> -where - E: From, -{ - let resp_json = serde_json::to_string_pretty(res) - .map_err(GarageError::from) - .map_err(Error::from)?; - Ok(Response::builder() - .status(hyper::StatusCode::OK) - .header(http::header::CONTENT_TYPE, "application/json") - .body(string_body(resp_json)) - .unwrap()) -} - -pub fn body_stream(body: B) -> impl Stream> -where - B: Body, - ::Error: Into, - E: From, -{ - let stream = http_body_util::BodyStream::new(body); - let stream = TryStreamExt::map_err(stream, Into::into); - stream.map(|x| { - x.and_then(|f| { - f.into_data() - .map_err(|_| E::from(Error::bad_request("non-data frame"))) - }) - }) -} - -pub fn is_default(v: &T) -> bool { - *v == T::default() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_bucket_containing_a_key() -> Result<(), Error> { - let (bucket, key) = parse_bucket_key("/my_bucket/a/super/file.jpg", None)?; - assert_eq!(bucket, "my_bucket"); - assert_eq!(key.expect("key must be set"), "a/super/file.jpg"); - Ok(()) - } - - #[test] - fn parse_bucket_containing_no_key() -> Result<(), Error> { - let (bucket, key) = parse_bucket_key("/my_bucket/", None)?; - assert_eq!(bucket, "my_bucket"); - assert!(key.is_none()); - let (bucket, key) = parse_bucket_key("/my_bucket", None)?; - assert_eq!(bucket, "my_bucket"); - assert!(key.is_none()); - Ok(()) - } - - #[test] - fn parse_bucket_containing_no_bucket() { - let parsed = parse_bucket_key("", None); - assert!(parsed.is_err()); - let parsed = parse_bucket_key("/", None); - assert!(parsed.is_err()); - let parsed = parse_bucket_key("////", None); - assert!(parsed.is_err()); - } - - #[test] - fn parse_bucket_with_vhost_and_key() -> Result<(), Error> { - let (bucket, key) = parse_bucket_key("/a/super/file.jpg", Some("my-bucket"))?; - assert_eq!(bucket, "my-bucket"); - assert_eq!(key.expect("key must be set"), "a/super/file.jpg"); - Ok(()) - } - - #[test] - fn parse_bucket_with_vhost_no_key() -> Result<(), Error> { - let (bucket, key) = parse_bucket_key("", Some("my-bucket"))?; - assert_eq!(bucket, "my-bucket"); - assert!(key.is_none()); - let (bucket, key) = parse_bucket_key("/", Some("my-bucket"))?; - assert_eq!(bucket, "my-bucket"); - assert!(key.is_none()); - Ok(()) - } - - #[test] - fn authority_to_host_with_port() -> Result<(), Error> { - let domain = authority_to_host("[::1]:3902")?; - assert_eq!(domain, "[::1]"); - let domain2 = authority_to_host("garage.tld:65200")?; - assert_eq!(domain2, "garage.tld"); - let domain3 = authority_to_host("127.0.0.1:80")?; - assert_eq!(domain3, "127.0.0.1"); - Ok(()) - } - - #[test] - fn authority_to_host_without_port() -> Result<(), Error> { - let domain = authority_to_host("[::1]")?; - assert_eq!(domain, "[::1]"); - let domain2 = authority_to_host("garage.tld")?; - assert_eq!(domain2, "garage.tld"); - let domain3 = authority_to_host("127.0.0.1")?; - assert_eq!(domain3, "127.0.0.1"); - assert!(authority_to_host("[").is_err()); - assert!(authority_to_host("[hello").is_err()); - Ok(()) - } - - #[test] - fn host_to_bucket_test() { - assert_eq!( - host_to_bucket("john.doe.garage.tld", ".garage.tld").unwrap(), - "john.doe" - ); - - assert_eq!( - host_to_bucket("john.doe.garage.tld", "garage.tld").unwrap(), - "john.doe" - ); - - assert_eq!(host_to_bucket("john.doe.com", "garage.tld"), None); - - assert_eq!(host_to_bucket("john.doe.com", ".garage.tld"), None); - - assert_eq!(host_to_bucket("garage.tld", "garage.tld"), None); - - assert_eq!(host_to_bucket("garage.tld", ".garage.tld"), None); - - assert_eq!(host_to_bucket("not-garage.tld", "garage.tld"), None); - assert_eq!(host_to_bucket("not-garage.tld", ".garage.tld"), None); - } - - #[test] - fn test_key_after_prefix() { - use std::iter::FromIterator; - - assert_eq!(UTF8_BEFORE_LAST_CHAR as u32, (char::MAX as u32) - 1); - assert_eq!(key_after_prefix("a/b/").unwrap().as_str(), "a/b0"); - assert_eq!(key_after_prefix("€").unwrap().as_str(), "₭"); - assert_eq!( - key_after_prefix("􏿽").unwrap().as_str(), - String::from(char::from_u32(0x10FFFE).unwrap()) - ); - - // When the last character is the biggest UTF8 char - let a = String::from_iter(['a', char::MAX].iter()); - assert_eq!(key_after_prefix(a.as_str()).unwrap().as_str(), "b"); - - // When all characters are the biggest UTF8 char - let b = String::from_iter([char::MAX; 3].iter()); - assert!(key_after_prefix(b.as_str()).is_none()); - - // Check utf8 surrogates - let c = String::from('\u{D7FF}'); - assert_eq!( - key_after_prefix(c.as_str()).unwrap().as_str(), - String::from('\u{E000}') - ); - - // Check the character before the biggest one - let d = String::from('\u{10FFFE}'); - assert_eq!( - key_after_prefix(d.as_str()).unwrap().as_str(), - String::from(char::MAX) - ); - } -} - -#[derive(Serialize)] -pub(crate) struct CustomApiErrorBody { - pub(crate) code: String, - pub(crate) message: String, - pub(crate) region: String, - pub(crate) path: String, -} diff --git a/src/api/k2v/Cargo.toml b/src/api/k2v/Cargo.toml new file mode 100644 index 00000000..86d12c2d --- /dev/null +++ b/src/api/k2v/Cargo.toml @@ -0,0 +1,75 @@ +[package] +name = "garage_api_k2v" +version = "1.0.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "S3 API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_block.workspace = true +garage_net.workspace = true +garage_util.workspace = true +garage_rpc.workspace = true +garage_api_common.workspace = true +garage_api_s3.workspace = true + +aes-gcm.workspace = true +argon2.workspace = true +async-compression.workspace = true +async-trait.workspace = true +base64.workspace = true +bytes.workspace = true +chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true +crypto-common.workspace = true +err-derive.workspace = true +hex.workspace = true +hmac.workspace = true +idna.workspace = true +tracing.workspace = true +md-5.workspace = true +nom.workspace = true +pin-project.workspace = true +sha1.workspace = true +sha2.workspace = true + +futures.workspace = true +futures-util.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tokio-util.workspace = true + +form_urlencoded.workspace = true +http.workspace = true +httpdate.workspace = true +http-range.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +hyper-util.workspace = true +multer.workspace = true +percent-encoding.workspace = true +roxmltree.workspace = true +url.workspace = true + +serde.workspace = true +serde_bytes.workspace = true +serde_json.workspace = true +quick-xml.workspace = true + +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } + +[features] +default = [ "garage_util/k2v", "garage_model/k2v" ] diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index de6e5f06..1fc512f9 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -12,19 +12,19 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; -use crate::generic_server::*; -use crate::k2v::error::*; +use crate::error::*; +use garage_api_common::generic_server::*; -use crate::signature::verify_request; +use garage_api_common::signature::verify_request; -use crate::helpers::*; -use crate::k2v::batch::*; -use crate::k2v::index::*; -use crate::k2v::item::*; -use crate::k2v::router::Endpoint; -use crate::s3::cors::*; +use crate::batch::*; +use crate::index::*; +use crate::item::*; +use crate::router::Endpoint; +use garage_api_common::helpers::*; +use garage_api_s3::cors::*; -pub use crate::signature::streaming::ReqBody; +pub use garage_api_common::signature::streaming::ReqBody; pub type ResBody = BoxBody; pub struct K2VApiServer { diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index e4d0b0e5..1dd90456 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -6,11 +6,11 @@ use garage_table::{EnumerationOrder, TableSchema}; use garage_model::k2v::item_table::*; -use crate::helpers::*; use crate::k2v::api_server::{ReqBody, ResBody}; use crate::k2v::error::*; use crate::k2v::item::parse_causality_token; use crate::k2v::range::read_range; +use garage_api_common::helpers::*; pub async fn handle_insert_batch( ctx: ReqCtx, diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index dbe4be2c..a4d3be1c 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -2,12 +2,14 @@ use err_derive::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; -use crate::common_error::CommonError; -pub(crate) use crate::common_error::{helper_error_as_internal, pass_helper_error}; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::helpers::*; -use crate::signature::error::Error as SignatureError; +use garage_api_common::common_error::CommonError; +pub(crate) use garage_api_common::common_error::{helper_error_as_internal, pass_helper_error}; +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; +use garage_api_common::signature::error::Error as SignatureError; /// Errors of this crate #[derive(Debug, Error)] diff --git a/src/api/k2v/index.rs b/src/api/k2v/index.rs index e3397238..423c1f97 100644 --- a/src/api/k2v/index.rs +++ b/src/api/k2v/index.rs @@ -5,10 +5,10 @@ use garage_table::util::*; use garage_model::k2v::item_table::{BYTES, CONFLICTS, ENTRIES, VALUES}; -use crate::helpers::*; -use crate::k2v::api_server::ResBody; -use crate::k2v::error::*; -use crate::k2v::range::read_range; +use crate::api_server::ResBody; +use crate::error::*; +use crate::range::read_range; +use garage_api_common::helpers::*; pub async fn handle_read_index( ctx: ReqCtx, diff --git a/src/api/k2v/item.rs b/src/api/k2v/item.rs index 87371727..315f647c 100644 --- a/src/api/k2v/item.rs +++ b/src/api/k2v/item.rs @@ -6,9 +6,9 @@ use hyper::{Request, Response, StatusCode}; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::helpers::*; -use crate::k2v::api_server::{ReqBody, ResBody}; -use crate::k2v::error::*; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use garage_api_common::helpers::*; pub const X_GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token"; diff --git a/src/api/k2v/lib.rs b/src/api/k2v/lib.rs new file mode 100644 index 00000000..334ae46b --- /dev/null +++ b/src/api/k2v/lib.rs @@ -0,0 +1,12 @@ +#[macro_use] +extern crate tracing; + +pub mod api_server; +mod error; +mod router; + +mod batch; +mod index; +mod item; + +mod range; diff --git a/src/api/k2v/mod.rs b/src/api/k2v/mod.rs deleted file mode 100644 index b6a8c5cf..00000000 --- a/src/api/k2v/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -pub mod api_server; -mod error; -mod router; - -mod batch; -mod index; -mod item; - -mod range; diff --git a/src/api/k2v/range.rs b/src/api/k2v/range.rs index bb9d3be5..fdb17e22 100644 --- a/src/api/k2v/range.rs +++ b/src/api/k2v/range.rs @@ -7,8 +7,8 @@ use std::sync::Arc; use garage_table::replication::TableShardedReplication; use garage_table::*; -use crate::helpers::key_after_prefix; -use crate::k2v::error::*; +use crate::error::*; +use garage_api_common::helpers::key_after_prefix; /// Read range in a Garage table. /// Returns (entries, more?, nextStart) diff --git a/src/api/k2v/router.rs b/src/api/k2v/router.rs index 1cc58be5..a04b0f81 100644 --- a/src/api/k2v/router.rs +++ b/src/api/k2v/router.rs @@ -1,11 +1,11 @@ -use crate::k2v::error::*; +use crate::error::*; use std::borrow::Cow; use hyper::{Method, Request}; -use crate::helpers::Authorization; -use crate::router_macros::{generateQueryParameters, router_match}; +use garage_api_common::helpers::Authorization; +use garage_api_common::router_macros::{generateQueryParameters, router_match}; router_match! {@func diff --git a/src/api/lib.rs b/src/api/lib.rs deleted file mode 100644 index 370dfd7a..00000000 --- a/src/api/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Crate for serving a S3 compatible API -#[macro_use] -extern crate tracing; - -pub mod common_error; - -mod encoding; -pub mod generic_server; -pub mod helpers; -mod router_macros; -/// This mode is public only to help testing. Don't expect stability here -pub mod signature; - -pub mod admin; -#[cfg(feature = "k2v")] -pub mod k2v; -pub mod s3; diff --git a/src/api/router_macros.rs b/src/api/router_macros.rs deleted file mode 100644 index 8f10a4f5..00000000 --- a/src/api/router_macros.rs +++ /dev/null @@ -1,224 +0,0 @@ -/// This macro is used to generate very repetitive match {} blocks in this module -/// It is _not_ made to be used anywhere else -macro_rules! router_match { - (@match $enum:expr , [ $($endpoint:ident,)* ]) => {{ - // usage: router_match {@match my_enum, [ VariantWithField1, VariantWithField2 ..] } - // returns true if the variant was one of the listed variants, false otherwise. - match $enum { - $( - Endpoint::$endpoint { .. } => true, - )* - _ => false - } - }}; - (@extract $enum:expr , $param:ident, [ $($endpoint:ident,)* ]) => {{ - // usage: router_match {@extract my_enum, field_name, [ VariantWithField1, VariantWithField2 ..] } - // returns Some(field_value), or None if the variant was not one of the listed variants. - match $enum { - $( - Endpoint::$endpoint {$param, ..} => Some($param), - )* - _ => None - } - }}; - (@gen_path_parser ($method:expr, $reqpath:expr, $query:expr) - [ - $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)* - ]) => {{ - { - #[allow(unused_parens)] - match ($method, $reqpath) { - $( - (&Method::$meth, $path) if true $(&& $query.$required.is_some())? => Endpoint::$api { - $($( - $param: router_match!(@@parse_param $query, $conv, $param), - )*)? - }, - )* - (m, p) => { - return Err(Error::bad_request(format!( - "Unknown API endpoint: {} {}", - m, p - ))) - } - } - } - }}; - (@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr), - key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*], - no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{ - // usage: router_match {@gen_parser (keyword, key, query, header), - // key: [ - // SOME_KEYWORD => VariantWithKey, - // ... - // ], - // no_key: [ - // SOME_KEYWORD => VariantWithoutKey, - // ... - // ] - // } - // See in from_{method} for more detailed usage. - match ($keyword, !$key.is_empty()){ - $( - (Keyword::$kw_k, true) if true $(&& $query.$required_k.is_some())? $(&& $header.contains_key($header_k))? => Ok(Endpoint::$api_k { - $key, - $($( - $param_k: router_match!(@@parse_param $query, $conv_k, $param_k), - )*)? - }), - )* - $( - (Keyword::$kw_nk, false) $(if $query.$required_nk.is_some())? $(if $header.contains($header_nk))? => Ok(Endpoint::$api_nk { - $($( - $param_nk: router_match!(@@parse_param $query, $conv_nk, $param_nk), - )*)? - }), - )* - (kw, _) => Err(Error::bad_request(format!("Invalid endpoint: {}", kw))) - } - }}; - - (@@parse_param $query:expr, query_opt, $param:ident) => {{ - // extract optional query parameter - $query.$param.take().map(|param| param.into_owned()) - }}; - (@@parse_param $query:expr, query, $param:ident) => {{ - // extract mendatory query parameter - $query.$param.take().ok_or_bad_request("Missing argument for endpoint")?.into_owned() - }}; - (@@parse_param $query:expr, opt_parse, $param:ident) => {{ - // extract and parse optional query parameter - // missing parameter is file, however parse error is reported as an error - $query.$param - .take() - .map(|param| param.parse()) - .transpose() - .map_err(|_| Error::bad_request("Failed to parse query parameter"))? - }}; - (@@parse_param $query:expr, parse, $param:ident) => {{ - // extract and parse mandatory query parameter - // both missing and un-parseable parameters are reported as errors - $query.$param.take().ok_or_bad_request("Missing argument for endpoint")? - .parse() - .map_err(|_| Error::bad_request("Failed to parse query parameter"))? - }}; - (@func - $(#[$doc:meta])* - pub enum Endpoint { - $( - $(#[$outer:meta])* - $variant:ident $({ - $($name:ident: $ty:ty,)* - })?, - )* - }) => { - $(#[$doc])* - pub enum Endpoint { - $( - $(#[$outer])* - $variant $({ - $($name: $ty, )* - })?, - )* - } - impl Endpoint { - pub fn name(&self) -> &'static str { - match self { - $(Endpoint::$variant $({ $($name: _,)* .. })? => stringify!($variant),)* - } - } - } - }; -} - -/// This macro is used to generate part of the code in this module. It must be called only one, and -/// is useless outside of this module. -macro_rules! generateQueryParameters { - ( - keywords: [ $($kw_param:expr => $kw_name: ident),* ], - fields: [ $($f_param:expr => $f_name:ident),* ] - ) => { - #[derive(Debug)] - #[allow(non_camel_case_types)] - #[allow(clippy::upper_case_acronyms)] - enum Keyword { - EMPTY, - $( $kw_name, )* - } - - impl std::fmt::Display for Keyword { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self { - Keyword::EMPTY => write!(f, "``"), - $( Keyword::$kw_name => write!(f, "`{}`", $kw_param), )* - } - } - } - - impl Default for Keyword { - fn default() -> Self { - Keyword::EMPTY - } - } - - /// Struct containing all query parameters used in endpoints. Think of it as an HashMap, - /// but with keys statically known. - #[derive(Debug, Default)] - struct QueryParameters<'a> { - keyword: Option, - $( - $f_name: Option>, - )* - } - - impl<'a> QueryParameters<'a> { - /// Build this struct from the query part of an URI. - fn from_query(query: &'a str) -> Result { - let mut res: Self = Default::default(); - for (k, v) in url::form_urlencoded::parse(query.as_bytes()) { - match k.as_ref() { - $( - $kw_param => if let Some(prev_kw) = res.keyword.replace(Keyword::$kw_name) { - return Err(Error::bad_request(format!( - "Multiple keywords: '{}' and '{}'", prev_kw, $kw_param - ))); - }, - )* - $( - $f_param => if !v.is_empty() { - if res.$f_name.replace(v).is_some() { - return Err(Error::bad_request(format!( - "Query parameter repeated: '{}'", k - ))); - } - }, - )* - _ => { - if !(k.starts_with("response-") || k.starts_with("X-Amz-")) { - debug!("Received an unknown query parameter: '{}'", k); - } - } - }; - } - Ok(res) - } - - /// Get an error message in case not all parameters where used when extracting them to - /// build an Endpoint variant - fn nonempty_message(&self) -> Option<&str> { - if self.keyword.is_some() { - Some("Keyword not used") - } $( - else if self.$f_name.is_some() { - Some(concat!("'", $f_param, "'")) - } - )* else { - None - } - } - } - } -} - -pub(crate) use generateQueryParameters; -pub(crate) use router_match; diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml new file mode 100644 index 00000000..c610b43a --- /dev/null +++ b/src/api/s3/Cargo.toml @@ -0,0 +1,71 @@ +[package] +name = "garage_api_s3" +version = "1.0.1" +authors = ["Alex Auvolat "] +edition = "2018" +license = "AGPL-3.0" +description = "S3 API server crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" +readme = "../../README.md" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model.workspace = true +garage_table.workspace = true +garage_block.workspace = true +garage_net.workspace = true +garage_util.workspace = true +garage_rpc.workspace = true +garage_api_common.workspace = true + +aes-gcm.workspace = true +argon2.workspace = true +async-compression.workspace = true +async-trait.workspace = true +base64.workspace = true +bytes.workspace = true +chrono.workspace = true +crc32fast.workspace = true +crc32c.workspace = true +crypto-common.workspace = true +err-derive.workspace = true +hex.workspace = true +hmac.workspace = true +idna.workspace = true +tracing.workspace = true +md-5.workspace = true +nom.workspace = true +pin-project.workspace = true +sha1.workspace = true +sha2.workspace = true + +futures.workspace = true +futures-util.workspace = true +tokio.workspace = true +tokio-stream.workspace = true +tokio-util.workspace = true + +form_urlencoded.workspace = true +http.workspace = true +httpdate.workspace = true +http-range.workspace = true +http-body-util.workspace = true +hyper = { workspace = true, default-features = false, features = ["server", "http1"] } +hyper-util.workspace = true +multer.workspace = true +percent-encoding.workspace = true +roxmltree.workspace = true +url.workspace = true + +serde.workspace = true +serde_bytes.workspace = true +serde_json.workspace = true +quick-xml.workspace = true + +opentelemetry.workspace = true +opentelemetry-prometheus = { workspace = true, optional = true } +prometheus = { workspace = true, optional = true } diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index f9dafa10..d24f6a0c 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -14,26 +14,26 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; use garage_model::key_table::Key; -use crate::generic_server::*; -use crate::s3::error::*; - -use crate::signature::verify_request; - -use crate::helpers::*; -use crate::s3::bucket::*; -use crate::s3::copy::*; -use crate::s3::cors::*; -use crate::s3::delete::*; -use crate::s3::get::*; -use crate::s3::lifecycle::*; -use crate::s3::list::*; -use crate::s3::multipart::*; -use crate::s3::post_object::handle_post_object; -use crate::s3::put::*; -use crate::s3::router::Endpoint; -use crate::s3::website::*; - -pub use crate::signature::streaming::ReqBody; +use crate::error::*; +use garage_api_common::generic_server::*; + +use garage_api_common::signature::verify_request; + +use crate::bucket::*; +use crate::copy::*; +use crate::cors::*; +use crate::delete::*; +use crate::get::*; +use crate::lifecycle::*; +use crate::list::*; +use crate::multipart::*; +use crate::post_object::handle_post_object; +use crate::put::*; +use crate::router::Endpoint; +use crate::website::*; +use garage_api_common::helpers::*; + +pub use garage_api_common::signature::streaming::ReqBody; pub type ResBody = BoxBody; pub struct S3ApiServer { diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 6a12aa9c..09c5742b 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -13,12 +13,12 @@ use garage_util::crdt::*; use garage_util::data::*; use garage_util::time::*; -use crate::common_error::CommonError; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml as s3_xml; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result, Error> { let ReqCtx { garage, .. } = ctx; diff --git a/src/api/s3/checksum.rs b/src/api/s3/checksum.rs index c7527163..02fb55ec 100644 --- a/src/api/s3/checksum.rs +++ b/src/api/s3/checksum.rs @@ -15,7 +15,7 @@ use garage_util::error::OkOrMessage; use garage_model::s3::object_table::*; -use crate::s3::error::*; +use crate::error::*; pub const X_AMZ_CHECKSUM_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-checksum-algorithm"); diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index b67ace88..1a474fd0 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -20,15 +20,15 @@ use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::checksum::*; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; -use crate::s3::get::full_object_byte_stream; -use crate::s3::multipart; -use crate::s3::put::{get_headers, save_stream, ChecksumMode, SaveStreamResult}; -use crate::s3::xml::{self as s3_xml, xmlns_tag}; +use crate::api_server::{ReqBody, ResBody}; +use crate::checksum::*; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::get::full_object_byte_stream; +use crate::multipart; +use crate::put::{get_headers, save_stream, ChecksumMode, SaveStreamResult}; +use crate::xml::{self as s3_xml, xmlns_tag}; +use garage_api_common::helpers::*; // -------- CopyObject --------- diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index 32dcc0d5..ae8352c3 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -15,12 +15,12 @@ use http_body_util::BodyExt; use serde::{Deserialize, Serialize}; -use crate::common_error::{helper_error_as_internal, CommonError}; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_api_common::common_error::{helper_error_as_internal, CommonError}; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule}; use garage_model::garage::Garage; diff --git a/src/api/s3/delete.rs b/src/api/s3/delete.rs index 57f6f948..1711a9b4 100644 --- a/src/api/s3/delete.rs +++ b/src/api/s3/delete.rs @@ -5,12 +5,12 @@ use garage_util::data::*; use garage_model::s3::object_table::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::put::next_timestamp; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::put::next_timestamp; +use crate::xml as s3_xml; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> { let ReqCtx { diff --git a/src/api/s3/encryption.rs b/src/api/s3/encryption.rs index 2e6ed65c..c54d487b 100644 --- a/src/api/s3/encryption.rs +++ b/src/api/s3/encryption.rs @@ -28,9 +28,9 @@ use garage_util::migrate::Migrate; use garage_model::garage::Garage; use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner}; -use crate::common_error::*; -use crate::s3::checksum::Md5Checksum; -use crate::s3::error::Error; +use crate::checksum::Md5Checksum; +use crate::error::Error; +use garage_api_common::common_error::*; const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm"); diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 22d2fe14..77dc07c8 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -6,13 +6,18 @@ use hyper::{HeaderMap, StatusCode}; use garage_model::helper::error::Error as HelperError; -pub(crate) use crate::common_error::pass_helper_error; -use crate::common_error::{helper_error_as_internal, CommonError}; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -use crate::generic_server::ApiError; -use crate::helpers::*; -use crate::s3::xml as s3_xml; -use crate::signature::error::Error as SignatureError; +pub(crate) use garage_api_common::common_error::pass_helper_error; + +use garage_api_common::common_error::{helper_error_as_internal, CommonError}; + +pub use garage_api_common::common_error::{ + CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, +}; + +use crate::xml as s3_xml; +use garage_api_common::generic_server::ApiError; +use garage_api_common::helpers::*; +use garage_api_common::signature::error::Error as SignatureError; /// Errors of this crate #[derive(Debug, Error)] diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index f61aae11..c4cd9d48 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -25,11 +25,11 @@ use garage_model::garage::Garage; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::*; -use crate::s3::api_server::ResBody; -use crate::s3::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; +use crate::api_server::ResBody; +use crate::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use garage_api_common::helpers::*; const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count"; diff --git a/src/api/s3/lib.rs b/src/api/s3/lib.rs new file mode 100644 index 00000000..fd99b443 --- /dev/null +++ b/src/api/s3/lib.rs @@ -0,0 +1,22 @@ +#[macro_use] +extern crate tracing; + +pub mod api_server; +pub mod error; + +mod bucket; +mod copy; +pub mod cors; +mod delete; +pub mod get; +mod lifecycle; +mod list; +mod multipart; +mod post_object; +mod put; +mod website; + +mod checksum; +mod encryption; +mod router; +pub mod xml; diff --git a/src/api/s3/lifecycle.rs b/src/api/s3/lifecycle.rs index 7eb1c2cb..da211585 100644 --- a/src/api/s3/lifecycle.rs +++ b/src/api/s3/lifecycle.rs @@ -5,11 +5,11 @@ use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; use garage_model::bucket_table::{ parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration, diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index 68d6cbe6..de808c32 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -13,13 +13,13 @@ use garage_model::s3::object_table::*; use garage_table::EnumerationOrder; -use crate::encoding::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; -use crate::s3::multipart as s3_multipart; -use crate::s3::xml as s3_xml; +use crate::api_server::{ReqBody, ResBody}; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::multipart as s3_multipart; +use crate::xml as s3_xml; +use garage_api_common::encoding::*; +use garage_api_common::helpers::*; const DUMMY_NAME: &str = "Dummy Key"; const DUMMY_KEY: &str = "GKDummyKey"; diff --git a/src/api/s3/mod.rs b/src/api/s3/mod.rs deleted file mode 100644 index b9bb1a6f..00000000 --- a/src/api/s3/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -pub mod api_server; -pub mod error; - -mod bucket; -mod copy; -pub mod cors; -mod delete; -pub mod get; -mod lifecycle; -mod list; -mod multipart; -mod post_object; -mod put; -mod website; - -mod checksum; -mod encryption; -mod router; -pub mod xml; diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs index 3db3e8aa..047ed06a 100644 --- a/src/api/s3/multipart.rs +++ b/src/api/s3/multipart.rs @@ -15,14 +15,14 @@ use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::checksum::*; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; -use crate::s3::put::*; -use crate::s3::xml as s3_xml; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::checksum::*; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::put::*; +use crate::xml as s3_xml; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; // ---- diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 5279ec6a..6416c523 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -16,15 +16,15 @@ use serde::Deserialize; use garage_model::garage::Garage; use garage_model::s3::object_table::*; -use crate::helpers::*; -use crate::s3::api_server::ResBody; -use crate::s3::checksum::*; -use crate::s3::cors::*; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; -use crate::s3::put::{get_headers, save_stream, ChecksumMode}; -use crate::s3::xml as s3_xml; -use crate::signature::payload::{verify_v4, Authorization}; +use crate::api_server::ResBody; +use crate::checksum::*; +use crate::cors::*; +use crate::encryption::EncryptionParams; +use crate::error::*; +use crate::put::{get_headers, save_stream, ChecksumMode}; +use crate::xml as s3_xml; +use garage_api_common::helpers::*; +use garage_api_common::signature::payload::{verify_v4, Authorization}; pub async fn handle_post_object( garage: Arc, diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index bfb0dc9b..47dcb8f7 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -30,11 +30,11 @@ use garage_model::s3::block_ref_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::checksum::*; -use crate::s3::encryption::EncryptionParams; -use crate::s3::error::*; +use crate::api_server::{ReqBody, ResBody}; +use crate::checksum::*; +use crate::encryption::EncryptionParams; +use crate::error::*; +use garage_api_common::helpers::*; const PUT_BLOCKS_MAX_PARALLEL: usize = 3; diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index e7ac1d77..94951e80 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -3,9 +3,9 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; -use crate::helpers::Authorization; -use crate::router_macros::{generateQueryParameters, router_match}; -use crate::s3::error::*; +use crate::error::*; +use garage_api_common::helpers::Authorization; +use garage_api_common::router_macros::{generateQueryParameters, router_match}; router_match! {@func diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index fa36bc32..46decccf 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -4,11 +4,11 @@ use http_body_util::BodyExt; use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::helpers::*; -use crate::s3::api_server::{ReqBody, ResBody}; -use crate::s3::error::*; -use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use crate::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; use garage_model::bucket_table::*; use garage_util::data::*; diff --git a/src/api/s3/xml.rs b/src/api/s3/xml.rs index 1e569ade..e8af3ec0 100644 --- a/src/api/s3/xml.rs +++ b/src/api/s3/xml.rs @@ -1,7 +1,7 @@ use quick_xml::se::to_string; use serde::{Deserialize, Serialize, Serializer}; -use crate::s3::error::Error as ApiError; +use crate::error::Error as ApiError; pub fn to_xml_with_header(x: &T) -> Result { let mut xml = r#""#.to_string(); diff --git a/src/api/signature/error.rs b/src/api/signature/error.rs deleted file mode 100644 index 2d92a072..00000000 --- a/src/api/signature/error.rs +++ /dev/null @@ -1,32 +0,0 @@ -use err_derive::Error; - -use crate::common_error::CommonError; -pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; - -/// Errors of this crate -#[derive(Debug, Error)] -pub enum Error { - #[error(display = "{}", _0)] - /// Error from common error - Common(CommonError), - - /// Authorization Header Malformed - #[error(display = "Authorization header malformed, unexpected scope: {}", _0)] - AuthorizationHeaderMalformed(String), - - // Category: bad request - /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8Str(#[error(source)] std::str::Utf8Error), -} - -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) - } -} - -impl CommonErrorDerivative for Error {} diff --git a/src/api/signature/mod.rs b/src/api/signature/mod.rs deleted file mode 100644 index 6514da43..00000000 --- a/src/api/signature/mod.rs +++ /dev/null @@ -1,78 +0,0 @@ -use chrono::{DateTime, Utc}; -use hmac::{Hmac, Mac}; -use sha2::Sha256; - -use hyper::{body::Incoming as IncomingBody, Request}; - -use garage_model::garage::Garage; -use garage_model::key_table::Key; -use garage_util::data::{sha256sum, Hash}; - -use error::*; - -pub mod error; -pub mod payload; -pub mod streaming; - -pub const SHORT_DATE: &str = "%Y%m%d"; -pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ"; - -type HmacSha256 = Hmac; - -pub async fn verify_request( - garage: &Garage, - mut req: Request, - service: &'static str, -) -> Result<(Request, Key, Option), Error> { - let (api_key, mut content_sha256) = - payload::check_payload_signature(&garage, &mut req, service).await?; - let api_key = - api_key.ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; - - let req = streaming::parse_streaming_body( - &api_key, - req, - &mut content_sha256, - &garage.config.s3_api.s3_region, - service, - )?; - - Ok((req, api_key, content_sha256)) -} - -pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> { - if expected_sha256 != sha256sum(body) { - return Err(Error::bad_request( - "Request content hash does not match signed hash".to_string(), - )); - } - Ok(()) -} - -pub fn signing_hmac( - datetime: &DateTime, - secret_key: &str, - region: &str, - service: &str, -) -> Result { - let secret = String::from("AWS4") + secret_key; - let mut date_hmac = HmacSha256::new_from_slice(secret.as_bytes())?; - date_hmac.update(datetime.format(SHORT_DATE).to_string().as_bytes()); - let mut region_hmac = HmacSha256::new_from_slice(&date_hmac.finalize().into_bytes())?; - region_hmac.update(region.as_bytes()); - let mut service_hmac = HmacSha256::new_from_slice(®ion_hmac.finalize().into_bytes())?; - service_hmac.update(service.as_bytes()); - let mut signing_hmac = HmacSha256::new_from_slice(&service_hmac.finalize().into_bytes())?; - signing_hmac.update(b"aws4_request"); - let hmac = HmacSha256::new_from_slice(&signing_hmac.finalize().into_bytes())?; - Ok(hmac) -} - -pub fn compute_scope(datetime: &DateTime, region: &str, service: &str) -> String { - format!( - "{}/{}/{}/aws4_request", - datetime.format(SHORT_DATE), - region, - service - ) -} diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs deleted file mode 100644 index 9e5a6043..00000000 --- a/src/api/signature/payload.rs +++ /dev/null @@ -1,562 +0,0 @@ -use std::collections::HashMap; -use std::convert::TryFrom; - -use chrono::{DateTime, Duration, NaiveDateTime, TimeZone, Utc}; -use hmac::Mac; -use hyper::header::{HeaderMap, HeaderName, HeaderValue, AUTHORIZATION, HOST}; -use hyper::{body::Incoming as IncomingBody, Method, Request}; -use sha2::{Digest, Sha256}; - -use garage_table::*; -use garage_util::data::Hash; - -use garage_model::garage::Garage; -use garage_model::key_table::*; - -use super::LONG_DATETIME; -use super::{compute_scope, signing_hmac}; - -use crate::encoding::uri_encode; -use crate::signature::error::*; - -pub const X_AMZ_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-algorithm"); -pub const X_AMZ_CREDENTIAL: HeaderName = HeaderName::from_static("x-amz-credential"); -pub const X_AMZ_DATE: HeaderName = HeaderName::from_static("x-amz-date"); -pub const X_AMZ_EXPIRES: HeaderName = HeaderName::from_static("x-amz-expires"); -pub const X_AMZ_SIGNEDHEADERS: HeaderName = HeaderName::from_static("x-amz-signedheaders"); -pub const X_AMZ_SIGNATURE: HeaderName = HeaderName::from_static("x-amz-signature"); -pub const X_AMZ_CONTENT_SH256: HeaderName = HeaderName::from_static("x-amz-content-sha256"); - -pub const AWS4_HMAC_SHA256: &str = "AWS4-HMAC-SHA256"; -pub const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD"; -pub const STREAMING_AWS4_HMAC_SHA256_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; - -pub type QueryMap = HeaderMap; -pub struct QueryValue { - /// Original key with potential uppercase characters, - /// for use in signature calculation - key: String, - value: String, -} - -pub async fn check_payload_signature( - garage: &Garage, - request: &mut Request, - service: &'static str, -) -> Result<(Option, Option), Error> { - let query = parse_query_map(request.uri())?; - - if query.contains_key(&X_AMZ_ALGORITHM) { - // We check for presigned-URL-style authentication first, because - // the browser or something else could inject an Authorization header - // that is totally unrelated to AWS signatures. - check_presigned_signature(garage, service, request, query).await - } else if request.headers().contains_key(AUTHORIZATION) { - check_standard_signature(garage, service, request, query).await - } else { - // Unsigned (anonymous) request - let content_sha256 = request - .headers() - .get("x-amz-content-sha256") - .filter(|c| c.as_bytes() != UNSIGNED_PAYLOAD.as_bytes()); - if let Some(content_sha256) = content_sha256 { - let sha256 = hex::decode(content_sha256) - .ok() - .and_then(|bytes| Hash::try_from(&bytes)) - .ok_or_bad_request("Invalid content sha256 hash")?; - Ok((None, Some(sha256))) - } else { - Ok((None, None)) - } - } -} - -async fn check_standard_signature( - garage: &Garage, - service: &'static str, - request: &Request, - query: QueryMap, -) -> Result<(Option, Option), Error> { - let authorization = Authorization::parse_header(request.headers())?; - - // Verify that all necessary request headers are included in signed_headers - // The following must be included for all signatures: - // - the Host header (mandatory) - // - all x-amz-* headers used in the request - // AWS also indicates that the Content-Type header should be signed if - // it is used, but Minio client doesn't sign it so we don't check it for compatibility. - let signed_headers = split_signed_headers(&authorization)?; - verify_signed_headers(request.headers(), &signed_headers)?; - - let canonical_request = canonical_request( - service, - request.method(), - request.uri().path(), - &query, - request.headers(), - &signed_headers, - &authorization.content_sha256, - )?; - let string_to_sign = string_to_sign( - &authorization.date, - &authorization.scope, - &canonical_request, - ); - - trace!("canonical request:\n{}", canonical_request); - trace!("string to sign:\n{}", string_to_sign); - - let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; - - let content_sha256 = if authorization.content_sha256 == UNSIGNED_PAYLOAD { - None - } else if authorization.content_sha256 == STREAMING_AWS4_HMAC_SHA256_PAYLOAD { - let bytes = hex::decode(authorization.signature).ok_or_bad_request("Invalid signature")?; - Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid signature")?) - } else { - let bytes = hex::decode(authorization.content_sha256) - .ok_or_bad_request("Invalid content sha256 hash")?; - Some(Hash::try_from(&bytes).ok_or_bad_request("Invalid content sha256 hash")?) - }; - - Ok((Some(key), content_sha256)) -} - -async fn check_presigned_signature( - garage: &Garage, - service: &'static str, - request: &mut Request, - mut query: QueryMap, -) -> Result<(Option, Option), Error> { - let algorithm = query.get(&X_AMZ_ALGORITHM).unwrap(); - let authorization = Authorization::parse_presigned(&algorithm.value, &query)?; - - // Verify that all necessary request headers are included in signed_headers - // For AWSv4 pre-signed URLs, the following must be included: - // - the Host header (mandatory) - // - all x-amz-* headers used in the request - let signed_headers = split_signed_headers(&authorization)?; - verify_signed_headers(request.headers(), &signed_headers)?; - - // The X-Amz-Signature value is passed as a query parameter, - // but the signature cannot be computed from a string that contains itself. - // AWS specifies that all query params except X-Amz-Signature are included - // in the canonical request. - query.remove(&X_AMZ_SIGNATURE); - let canonical_request = canonical_request( - service, - request.method(), - request.uri().path(), - &query, - request.headers(), - &signed_headers, - &authorization.content_sha256, - )?; - let string_to_sign = string_to_sign( - &authorization.date, - &authorization.scope, - &canonical_request, - ); - - trace!("canonical request (presigned url):\n{}", canonical_request); - trace!("string to sign (presigned url):\n{}", string_to_sign); - - let key = verify_v4(garage, service, &authorization, string_to_sign.as_bytes()).await?; - - // In the page on presigned URLs, AWS specifies that if a signed query - // parameter and a signed header of the same name have different values, - // then an InvalidRequest error is raised. - let headers_mut = request.headers_mut(); - for (name, value) in query.iter() { - if let Some(existing) = headers_mut.get(name) { - if signed_headers.contains(&name) && existing.as_bytes() != value.value.as_bytes() { - return Err(Error::bad_request(format!( - "Conflicting values for `{}` in query parameters and request headers", - name - ))); - } - } - if name.as_str().starts_with("x-amz-") { - // Query parameters that start by x-amz- are actually intended to stand in for - // headers that can't be added at the time the request is made. - // What we do is just add them to the Request object as regular headers, - // that will be handled downstream as if they were included like in a normal request. - // (Here we allow such query parameters to override headers with the same name - // that are not signed, however there is not much reason that this would happen) - headers_mut.insert( - name, - HeaderValue::from_bytes(value.value.as_bytes()) - .ok_or_bad_request("invalid query parameter value")?, - ); - } - } - - // Presigned URLs always use UNSIGNED-PAYLOAD, - // so there is no sha256 hash to return. - Ok((Some(key), None)) -} - -pub fn parse_query_map(uri: &http::uri::Uri) -> Result { - let mut query = QueryMap::with_capacity(0); - if let Some(query_str) = uri.query() { - let query_pairs = url::form_urlencoded::parse(query_str.as_bytes()); - for (key, val) in query_pairs { - let name = - HeaderName::from_bytes(key.as_bytes()).ok_or_bad_request("Invalid header name")?; - - let value = QueryValue { - key: key.to_string(), - value: val.into_owned(), - }; - - if query.insert(name, value).is_some() { - return Err(Error::bad_request(format!( - "duplicate query parameter: `{}`", - key - ))); - } - } - } - Ok(query) -} - -fn parse_credential(cred: &str) -> Result<(String, String), Error> { - let first_slash = cred - .find('/') - .ok_or_bad_request("Credentials does not contain '/' in authorization field")?; - let (key_id, scope) = cred.split_at(first_slash); - Ok(( - key_id.to_string(), - scope.trim_start_matches('/').to_string(), - )) -} - -fn split_signed_headers(authorization: &Authorization) -> Result, Error> { - let mut signed_headers = authorization - .signed_headers - .split(';') - .map(HeaderName::try_from) - .collect::, _>>() - .ok_or_bad_request("invalid header name")?; - signed_headers.sort_by(|h1, h2| h1.as_str().cmp(h2.as_str())); - Ok(signed_headers) -} - -fn verify_signed_headers(headers: &HeaderMap, signed_headers: &[HeaderName]) -> Result<(), Error> { - if !signed_headers.contains(&HOST) { - return Err(Error::bad_request("Header `Host` should be signed")); - } - for (name, _) in headers.iter() { - if name.as_str().starts_with("x-amz-") { - if !signed_headers.contains(name) { - return Err(Error::bad_request(format!( - "Header `{}` should be signed", - name - ))); - } - } - } - Ok(()) -} - -pub fn string_to_sign(datetime: &DateTime, scope_string: &str, canonical_req: &str) -> String { - let mut hasher = Sha256::default(); - hasher.update(canonical_req.as_bytes()); - [ - AWS4_HMAC_SHA256, - &datetime.format(LONG_DATETIME).to_string(), - scope_string, - &hex::encode(hasher.finalize().as_slice()), - ] - .join("\n") -} - -pub fn canonical_request( - service: &'static str, - method: &Method, - canonical_uri: &str, - query: &QueryMap, - headers: &HeaderMap, - signed_headers: &[HeaderName], - content_sha256: &str, -) -> Result { - // There seems to be evidence that in AWSv4 signatures, the path component is url-encoded - // a second time when building the canonical request, as specified in this documentation page: - // -> https://docs.aws.amazon.com/rolesanywhere/latest/userguide/authentication-sign-process.html - // However this documentation page is for a specific service ("roles anywhere"), and - // in the S3 service we know for a fact that there is no double-urlencoding, because all of - // the tests we made with external software work without it. - // - // The theory is that double-urlencoding occurs for all services except S3, - // which is what is implemented in rusoto_signature: - // -> https://docs.rs/rusoto_signature/latest/src/rusoto_signature/signature.rs.html#464 - // - // Digging into the code of the official AWS Rust SDK, we learn that double-URI-encoding can - // be set or unset on a per-request basis (the signature crates, aws-sigv4 and aws-sig-auth, - // are agnostic to this). Grepping the codebase confirms that S3 is the only API for which - // double_uri_encode is set to false, meaning it is true (its default value) for all other - // AWS services. We will therefore implement this behavior in Garage as well. - // - // Note that this documentation page, which is touted as the "authoritative reference" on - // AWSv4 signatures, makes no mention of either single- or double-urlencoding: - // -> https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html - // This page of the S3 documentation does also not mention anything specific: - // -> https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-header-based-auth.html - // - // Note that there is also the issue of path normalization, which I hope is unrelated to the - // one of URI-encoding. At least in aws-sigv4 both parameters can be set independently, - // and rusoto_signature does not seem to do any effective path normalization, even though - // it mentions it in the comments (same link to the source code as above). - // We make the explicit choice of NOT normalizing paths in the K2V API because doing so - // would make non-normalized paths invalid K2V partition keys, and we don't want that. - let canonical_uri: std::borrow::Cow = if service != "s3" { - uri_encode(canonical_uri, false).into() - } else { - canonical_uri.into() - }; - - // Canonical query string from passed HeaderMap - let canonical_query_string = { - let mut items = Vec::with_capacity(query.len()); - for (_, QueryValue { key, value }) in query.iter() { - items.push(uri_encode(&key, true) + "=" + &uri_encode(&value, true)); - } - items.sort(); - items.join("&") - }; - - // Canonical header string calculated from signed headers - let canonical_header_string = signed_headers - .iter() - .map(|name| { - let value = headers - .get(name) - .ok_or_bad_request(format!("signed header `{}` is not present", name))?; - let value = std::str::from_utf8(value.as_bytes())?; - Ok(format!("{}:{}", name.as_str(), value.trim())) - }) - .collect::, Error>>()? - .join("\n"); - let signed_headers = signed_headers.join(";"); - - let list = [ - method.as_str(), - &canonical_uri, - &canonical_query_string, - &canonical_header_string, - "", - &signed_headers, - content_sha256, - ]; - Ok(list.join("\n")) -} - -pub fn parse_date(date: &str) -> Result, Error> { - let date: NaiveDateTime = - NaiveDateTime::parse_from_str(date, LONG_DATETIME).ok_or_bad_request("Invalid date")?; - Ok(Utc.from_utc_datetime(&date)) -} - -pub async fn verify_v4( - garage: &Garage, - service: &str, - auth: &Authorization, - payload: &[u8], -) -> Result { - let scope_expected = compute_scope(&auth.date, &garage.config.s3_api.s3_region, service); - if auth.scope != scope_expected { - return Err(Error::AuthorizationHeaderMalformed(auth.scope.to_string())); - } - - let key = garage - .key_table - .get(&EmptyKey, &auth.key_id) - .await? - .filter(|k| !k.state.is_deleted()) - .ok_or_else(|| Error::forbidden(format!("No such key: {}", &auth.key_id)))?; - let key_p = key.params().unwrap(); - - let mut hmac = signing_hmac( - &auth.date, - &key_p.secret_key, - &garage.config.s3_api.s3_region, - service, - ) - .ok_or_internal_error("Unable to build signing HMAC")?; - hmac.update(payload); - let signature = - hex::decode(&auth.signature).map_err(|_| Error::forbidden("Invalid signature"))?; - if hmac.verify_slice(&signature).is_err() { - return Err(Error::forbidden("Invalid signature")); - } - - Ok(key) -} - -// ============ Authorization header, or X-Amz-* query params ========= - -pub struct Authorization { - key_id: String, - scope: String, - signed_headers: String, - signature: String, - content_sha256: String, - date: DateTime, -} - -impl Authorization { - fn parse_header(headers: &HeaderMap) -> Result { - let authorization = headers - .get(AUTHORIZATION) - .ok_or_bad_request("Missing authorization header")? - .to_str()?; - - let (auth_kind, rest) = authorization - .split_once(' ') - .ok_or_bad_request("Authorization field to short")?; - - if auth_kind != AWS4_HMAC_SHA256 { - return Err(Error::bad_request("Unsupported authorization method")); - } - - let mut auth_params = HashMap::new(); - for auth_part in rest.split(',') { - let auth_part = auth_part.trim(); - let eq = auth_part - .find('=') - .ok_or_bad_request("Field without value in authorization header")?; - let (key, value) = auth_part.split_at(eq); - auth_params.insert(key.to_string(), value.trim_start_matches('=').to_string()); - } - - let cred = auth_params - .get("Credential") - .ok_or_bad_request("Could not find Credential in Authorization field")?; - let signed_headers = auth_params - .get("SignedHeaders") - .ok_or_bad_request("Could not find SignedHeaders in Authorization field")? - .to_string(); - let signature = auth_params - .get("Signature") - .ok_or_bad_request("Could not find Signature in Authorization field")? - .to_string(); - - let content_sha256 = headers - .get(X_AMZ_CONTENT_SH256) - .ok_or_bad_request("Missing X-Amz-Content-Sha256 field")?; - - let date = headers - .get(X_AMZ_DATE) - .ok_or_bad_request("Missing X-Amz-Date field") - .map_err(Error::from)? - .to_str()?; - let date = parse_date(date)?; - - if Utc::now() - date > Duration::hours(24) { - return Err(Error::bad_request("Date is too old".to_string())); - } - - let (key_id, scope) = parse_credential(cred)?; - let auth = Authorization { - key_id, - scope, - signed_headers, - signature, - content_sha256: content_sha256.to_str()?.to_string(), - date, - }; - Ok(auth) - } - - fn parse_presigned(algorithm: &str, query: &QueryMap) -> Result { - if algorithm != AWS4_HMAC_SHA256 { - return Err(Error::bad_request( - "Unsupported authorization method".to_string(), - )); - } - - let cred = query - .get(&X_AMZ_CREDENTIAL) - .ok_or_bad_request("X-Amz-Credential not found in query parameters")?; - let signed_headers = query - .get(&X_AMZ_SIGNEDHEADERS) - .ok_or_bad_request("X-Amz-SignedHeaders not found in query parameters")?; - let signature = query - .get(&X_AMZ_SIGNATURE) - .ok_or_bad_request("X-Amz-Signature not found in query parameters")?; - - let duration = query - .get(&X_AMZ_EXPIRES) - .ok_or_bad_request("X-Amz-Expires not found in query parameters")? - .value - .parse() - .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; - - if duration > 7 * 24 * 3600 { - return Err(Error::bad_request( - "X-Amz-Expires may not exceed a week".to_string(), - )); - } - - let date = query - .get(&X_AMZ_DATE) - .ok_or_bad_request("Missing X-Amz-Date field")?; - let date = parse_date(&date.value)?; - - if Utc::now() - date > Duration::seconds(duration) { - return Err(Error::bad_request("Date is too old".to_string())); - } - - let (key_id, scope) = parse_credential(&cred.value)?; - Ok(Authorization { - key_id, - scope, - signed_headers: signed_headers.value.clone(), - signature: signature.value.clone(), - content_sha256: UNSIGNED_PAYLOAD.to_string(), - date, - }) - } - - pub(crate) fn parse_form(params: &HeaderMap) -> Result { - let algorithm = params - .get(X_AMZ_ALGORITHM) - .ok_or_bad_request("Missing X-Amz-Algorithm header")? - .to_str()?; - if algorithm != AWS4_HMAC_SHA256 { - return Err(Error::bad_request( - "Unsupported authorization method".to_string(), - )); - } - - let credential = params - .get(X_AMZ_CREDENTIAL) - .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? - .to_str()?; - let signature = params - .get(X_AMZ_SIGNATURE) - .ok_or_bad_request("No signature was provided")? - .to_str()? - .to_string(); - let date = params - .get(X_AMZ_DATE) - .ok_or_bad_request("No date was provided")? - .to_str()?; - let date = parse_date(date)?; - - if Utc::now() - date > Duration::hours(24) { - return Err(Error::bad_request("Date is too old".to_string())); - } - - let (key_id, scope) = parse_credential(credential)?; - let auth = Authorization { - key_id, - scope, - signed_headers: "".to_string(), - signature, - content_sha256: UNSIGNED_PAYLOAD.to_string(), - date, - }; - Ok(auth) - } -} diff --git a/src/api/signature/streaming.rs b/src/api/signature/streaming.rs deleted file mode 100644 index e223d1b1..00000000 --- a/src/api/signature/streaming.rs +++ /dev/null @@ -1,373 +0,0 @@ -use std::pin::Pin; - -use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; -use futures::prelude::*; -use futures::task; -use garage_model::key_table::Key; -use hmac::Mac; -use http_body_util::StreamBody; -use hyper::body::{Bytes, Incoming as IncomingBody}; -use hyper::Request; - -use garage_util::data::Hash; - -use super::{compute_scope, sha256sum, HmacSha256, LONG_DATETIME}; - -use crate::helpers::*; -use crate::signature::error::*; -use crate::signature::payload::{ - STREAMING_AWS4_HMAC_SHA256_PAYLOAD, X_AMZ_CONTENT_SH256, X_AMZ_DATE, -}; - -pub const AWS4_HMAC_SHA256_PAYLOAD: &str = "AWS4-HMAC-SHA256-PAYLOAD"; - -pub type ReqBody = BoxBody; - -pub fn parse_streaming_body( - api_key: &Key, - req: Request, - content_sha256: &mut Option, - region: &str, - service: &str, -) -> Result, Error> { - match req.headers().get(X_AMZ_CONTENT_SH256) { - Some(header) if header == STREAMING_AWS4_HMAC_SHA256_PAYLOAD => { - let signature = content_sha256 - .take() - .ok_or_bad_request("No signature provided")?; - - let secret_key = &api_key - .state - .as_option() - .ok_or_internal_error("Deleted key state")? - .secret_key; - - let date = req - .headers() - .get(X_AMZ_DATE) - .ok_or_bad_request("Missing X-Amz-Date field")? - .to_str()?; - let date: NaiveDateTime = NaiveDateTime::parse_from_str(date, LONG_DATETIME) - .ok_or_bad_request("Invalid date")?; - let date: DateTime = Utc.from_utc_datetime(&date); - - let scope = compute_scope(&date, region, service); - let signing_hmac = crate::signature::signing_hmac(&date, secret_key, region, service) - .ok_or_internal_error("Unable to build signing HMAC")?; - - Ok(req.map(move |body| { - let stream = body_stream::<_, Error>(body); - let signed_payload_stream = - SignedPayloadStream::new(stream, signing_hmac, date, &scope, signature) - .map(|x| x.map(hyper::body::Frame::data)) - .map_err(Error::from); - ReqBody::new(StreamBody::new(signed_payload_stream)) - })) - } - _ => Ok(req.map(|body| ReqBody::new(http_body_util::BodyExt::map_err(body, Error::from)))), - } -} - -/// Result of `sha256("")` -const EMPTY_STRING_HEX_DIGEST: &str = - "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - -fn compute_streaming_payload_signature( - signing_hmac: &HmacSha256, - date: DateTime, - scope: &str, - previous_signature: Hash, - content_sha256: Hash, -) -> Result { - let string_to_sign = [ - AWS4_HMAC_SHA256_PAYLOAD, - &date.format(LONG_DATETIME).to_string(), - scope, - &hex::encode(previous_signature), - EMPTY_STRING_HEX_DIGEST, - &hex::encode(content_sha256), - ] - .join("\n"); - - let mut hmac = signing_hmac.clone(); - hmac.update(string_to_sign.as_bytes()); - - Ok(Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature")?) -} - -mod payload { - use garage_util::data::Hash; - - pub enum Error { - Parser(nom::error::Error), - BadSignature, - } - - impl Error { - pub fn description(&self) -> &str { - match *self { - Error::Parser(ref e) => e.code.description(), - Error::BadSignature => "Bad signature", - } - } - } - - #[derive(Debug, Clone)] - pub struct Header { - pub size: usize, - pub signature: Hash, - } - - impl Header { - pub fn parse(input: &[u8]) -> nom::IResult<&[u8], Self, Error<&[u8]>> { - use nom::bytes::streaming::tag; - use nom::character::streaming::hex_digit1; - use nom::combinator::map_res; - use nom::number::streaming::hex_u32; - - macro_rules! try_parse { - ($expr:expr) => { - $expr.map_err(|e| e.map(Error::Parser))? - }; - } - - let (input, size) = try_parse!(hex_u32(input)); - let (input, _) = try_parse!(tag(";")(input)); - - let (input, _) = try_parse!(tag("chunk-signature=")(input)); - let (input, data) = try_parse!(map_res(hex_digit1, hex::decode)(input)); - let signature = Hash::try_from(&data).ok_or(nom::Err::Failure(Error::BadSignature))?; - - let (input, _) = try_parse!(tag("\r\n")(input)); - - let header = Header { - size: size as usize, - signature, - }; - - Ok((input, header)) - } - } -} - -#[derive(Debug)] -pub enum SignedPayloadStreamError { - Stream(Error), - InvalidSignature, - Message(String), -} - -impl SignedPayloadStreamError { - fn message(msg: &str) -> Self { - SignedPayloadStreamError::Message(msg.into()) - } -} - -impl From for Error { - fn from(err: SignedPayloadStreamError) -> Self { - match err { - SignedPayloadStreamError::Stream(e) => e, - SignedPayloadStreamError::InvalidSignature => { - Error::bad_request("Invalid payload signature") - } - SignedPayloadStreamError::Message(e) => { - Error::bad_request(format!("Chunk format error: {}", e)) - } - } - } -} - -impl From> for SignedPayloadStreamError { - fn from(err: payload::Error) -> Self { - Self::message(err.description()) - } -} - -impl From> for SignedPayloadStreamError { - fn from(err: nom::error::Error) -> Self { - Self::message(err.code.description()) - } -} - -struct SignedPayload { - header: payload::Header, - data: Bytes, -} - -#[pin_project::pin_project] -pub struct SignedPayloadStream -where - S: Stream>, -{ - #[pin] - stream: S, - buf: bytes::BytesMut, - datetime: DateTime, - scope: String, - signing_hmac: HmacSha256, - previous_signature: Hash, -} - -impl SignedPayloadStream -where - S: Stream>, -{ - pub fn new( - stream: S, - signing_hmac: HmacSha256, - datetime: DateTime, - scope: &str, - seed_signature: Hash, - ) -> Self { - Self { - stream, - buf: bytes::BytesMut::new(), - datetime, - scope: scope.into(), - signing_hmac, - previous_signature: seed_signature, - } - } - - fn parse_next(input: &[u8]) -> nom::IResult<&[u8], SignedPayload, SignedPayloadStreamError> { - use nom::bytes::streaming::{tag, take}; - - macro_rules! try_parse { - ($expr:expr) => { - $expr.map_err(nom::Err::convert)? - }; - } - - let (input, header) = try_parse!(payload::Header::parse(input)); - - // 0-sized chunk is the last - if header.size == 0 { - return Ok(( - input, - SignedPayload { - header, - data: Bytes::new(), - }, - )); - } - - let (input, data) = try_parse!(take::<_, _, nom::error::Error<_>>(header.size)(input)); - let (input, _) = try_parse!(tag::<_, _, nom::error::Error<_>>("\r\n")(input)); - - let data = Bytes::from(data.to_vec()); - - Ok((input, SignedPayload { header, data })) - } -} - -impl Stream for SignedPayloadStream -where - S: Stream> + Unpin, -{ - type Item = Result; - - fn poll_next( - self: Pin<&mut Self>, - cx: &mut task::Context<'_>, - ) -> task::Poll> { - use std::task::Poll; - - let mut this = self.project(); - - loop { - let (input, payload) = match Self::parse_next(this.buf) { - Ok(res) => res, - Err(nom::Err::Incomplete(_)) => { - match futures::ready!(this.stream.as_mut().poll_next(cx)) { - Some(Ok(bytes)) => { - this.buf.extend(bytes); - continue; - } - Some(Err(e)) => { - return Poll::Ready(Some(Err(SignedPayloadStreamError::Stream(e)))) - } - None => { - return Poll::Ready(Some(Err(SignedPayloadStreamError::message( - "Unexpected EOF", - )))); - } - } - } - Err(nom::Err::Error(e)) | Err(nom::Err::Failure(e)) => { - return Poll::Ready(Some(Err(e))) - } - }; - - // 0-sized chunk is the last - if payload.data.is_empty() { - return Poll::Ready(None); - } - - let data_sha256sum = sha256sum(&payload.data); - - let expected_signature = compute_streaming_payload_signature( - this.signing_hmac, - *this.datetime, - this.scope, - *this.previous_signature, - data_sha256sum, - ) - .map_err(|e| { - SignedPayloadStreamError::Message(format!("Could not build signature: {}", e)) - })?; - - if payload.header.signature != expected_signature { - return Poll::Ready(Some(Err(SignedPayloadStreamError::InvalidSignature))); - } - - *this.buf = input.into(); - *this.previous_signature = payload.header.signature; - - return Poll::Ready(Some(Ok(payload.data))); - } - } - - fn size_hint(&self) -> (usize, Option) { - self.stream.size_hint() - } -} - -#[cfg(test)] -mod tests { - use futures::prelude::*; - - use super::{SignedPayloadStream, SignedPayloadStreamError}; - - #[tokio::test] - async fn test_interrupted_signed_payload_stream() { - use chrono::{DateTime, Utc}; - - use garage_util::data::Hash; - - let datetime = DateTime::parse_from_rfc3339("2021-12-13T13:12:42+01:00") // TODO UNIX 0 - .unwrap() - .with_timezone(&Utc); - let secret_key = "test"; - let region = "test"; - let scope = crate::signature::compute_scope(&datetime, region, "s3"); - let signing_hmac = - crate::signature::signing_hmac(&datetime, secret_key, region, "s3").unwrap(); - - let data: &[&[u8]] = &[b"1"]; - let body = futures::stream::iter(data.iter().map(|block| Ok(block.to_vec().into()))); - - let seed_signature = Hash::default(); - - let mut stream = - SignedPayloadStream::new(body, signing_hmac, datetime, &scope, seed_signature); - - assert!(stream.try_next().await.is_err()); - match stream.try_next().await { - Err(SignedPayloadStreamError::Message(msg)) if msg == "Unexpected EOF" => {} - item => panic!( - "Unexpected result, expected early EOF error, got {:?}", - item - ), - } - } -} -- cgit v1.2.3 From 84f1db91c4e53a8d0c037fd01adb695fd9400ed5 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 18:34:57 +0100 Subject: fix things up --- src/api/admin/Cargo.toml | 3 +++ src/api/admin/api_server.rs | 2 +- src/api/admin/bucket.rs | 5 +++-- src/api/admin/cluster.rs | 3 ++- src/api/admin/error.rs | 15 +++------------ src/api/admin/key.rs | 3 ++- src/api/admin/router_v0.rs | 3 ++- src/api/admin/router_v1.rs | 3 ++- src/api/common/common_error.rs | 29 +++++++++++++++++++++++++++++ src/api/k2v/api_server.rs | 9 ++++----- src/api/k2v/batch.rs | 9 +++++---- src/api/k2v/error.rs | 15 +++------------ src/api/k2v/index.rs | 3 ++- src/api/k2v/item.rs | 3 ++- src/api/k2v/range.rs | 3 ++- src/api/s3/api_server.rs | 7 +++---- src/api/s3/bucket.rs | 7 ++++--- src/api/s3/copy.rs | 3 ++- src/api/s3/cors.rs | 13 +++++++------ src/api/s3/delete.rs | 5 +++-- src/api/s3/encryption.rs | 3 ++- src/api/s3/error.rs | 20 +++++++------------- src/api/s3/get.rs | 3 ++- src/api/s3/lifecycle.rs | 5 +++-- src/api/s3/list.rs | 5 +++-- src/api/s3/multipart.rs | 5 +++-- src/api/s3/post_object.rs | 5 +++-- src/api/s3/put.rs | 3 ++- src/api/s3/router.rs | 3 ++- src/api/s3/website.rs | 11 ++++++----- 30 files changed, 117 insertions(+), 89 deletions(-) (limited to 'src/api') diff --git a/src/api/admin/Cargo.toml b/src/api/admin/Cargo.toml index 02cbfc3d..804166b3 100644 --- a/src/api/admin/Cargo.toml +++ b/src/api/admin/Cargo.toml @@ -69,3 +69,6 @@ quick-xml.workspace = true opentelemetry.workspace = true opentelemetry-prometheus = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } + +[features] +metrics = [ "opentelemetry-prometheus", "prometheus", "garage_api_common/metrics" ] diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 7f8a51a6..e39fa1ba 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -21,6 +21,7 @@ use garage_util::error::Error as GarageError; use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_api_common::generic_server::*; +use garage_api_common::helpers::*; use crate::bucket::*; use crate::cluster::*; @@ -28,7 +29,6 @@ use crate::error::*; use crate::key::*; use crate::router_v0; use crate::router_v1::{Authorization, Endpoint}; -use garage_api_common::helpers::*; pub type ResBody = BoxBody; diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 3afed694..2537bfc9 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -17,11 +17,12 @@ use garage_model::permission::*; use garage_model::s3::mpu_table; use garage_model::s3::object_table::*; +use garage_api_common::common_error::CommonError; +use garage_api_common::helpers::*; + use crate::api_server::ResBody; use crate::error::*; use crate::key::ApiBucketKeyPerm; -use garage_api_common::common_error::CommonError; -use garage_api_common::helpers::*; pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { let buckets = garage diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index d4a645a2..ffa0fa71 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -12,9 +12,10 @@ use garage_rpc::layout; use garage_model::garage::Garage; +use garage_api_common::helpers::{json_ok_response, parse_json_body}; + use crate::api_server::ResBody; use crate::error::*; -use garage_api_common::helpers::{json_ok_response, parse_json_body}; pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { let layout = garage.system.cluster_layout(); diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 1c962776..201f9b40 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -6,7 +6,7 @@ use hyper::{HeaderMap, StatusCode}; pub use garage_model::helper::error::Error as HelperError; -use garage_api_common::common_error::CommonError; +use garage_api_common::common_error::{commonErrorDerivative, CommonError}; pub use garage_api_common::common_error::{ CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, }; @@ -18,7 +18,7 @@ use garage_api_common::helpers::*; pub enum Error { #[error(display = "{}", _0)] /// Error from common error - Common(CommonError), + Common(#[error(source)] CommonError), // Category: cannot process /// The API access key does not exist @@ -33,14 +33,7 @@ pub enum Error { KeyAlreadyExists(String), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) - } -} +commonErrorDerivative!(Error); /// FIXME: helper errors are transformed into their corresponding variants /// in the Error struct, but in many case a helper error should be considered @@ -55,8 +48,6 @@ impl From for Error { } } -impl CommonErrorDerivative for Error {} - impl Error { fn code(&self) -> &'static str { match self { diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 0c017a26..bebf3063 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -9,9 +9,10 @@ use garage_table::*; use garage_model::garage::Garage; use garage_model::key_table::*; +use garage_api_common::helpers::*; + use crate::api_server::ResBody; use crate::error::*; -use garage_api_common::helpers::*; pub async fn handle_list_keys(garage: &Arc) -> Result, Error> { let res = garage diff --git a/src/api/admin/router_v0.rs b/src/api/admin/router_v0.rs index 0c832fe1..9dd742ba 100644 --- a/src/api/admin/router_v0.rs +++ b/src/api/admin/router_v0.rs @@ -2,9 +2,10 @@ use std::borrow::Cow; use hyper::{Method, Request}; -use crate::error::*; use garage_api_common::router_macros::*; +use crate::error::*; + router_match! {@func /// List of all Admin API endpoints. diff --git a/src/api/admin/router_v1.rs b/src/api/admin/router_v1.rs index d9febd34..0b4901ea 100644 --- a/src/api/admin/router_v1.rs +++ b/src/api/admin/router_v1.rs @@ -2,9 +2,10 @@ use std::borrow::Cow; use hyper::{Method, Request}; +use garage_api_common::router_macros::*; + use crate::error::*; use crate::router_v0; -use garage_api_common::router_macros::*; pub enum Authorization { None, diff --git a/src/api/common/common_error.rs b/src/api/common/common_error.rs index 1e3f9feb..597a3511 100644 --- a/src/api/common/common_error.rs +++ b/src/api/common/common_error.rs @@ -57,6 +57,35 @@ pub enum CommonError { InvalidBucketName(String), } +#[macro_export] +macro_rules! commonErrorDerivative { + ( $error_struct: ident ) => { + impl From for $error_struct { + fn from(err: garage_util::error::Error) -> Self { + Self::Common(CommonError::InternalError(err)) + } + } + impl From for $error_struct { + fn from(err: http::Error) -> Self { + Self::Common(CommonError::Http(err)) + } + } + impl From for $error_struct { + fn from(err: hyper::Error) -> Self { + Self::Common(CommonError::Hyper(err)) + } + } + impl From for $error_struct { + fn from(err: hyper::header::ToStrError) -> Self { + Self::Common(CommonError::InvalidHeader(err)) + } + } + impl CommonErrorDerivative for $error_struct {} + }; +} + +pub use commonErrorDerivative; + impl CommonError { pub fn http_status_code(&self) -> StatusCode { match self { diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index 1fc512f9..0791c07d 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -12,17 +12,16 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; -use crate::error::*; use garage_api_common::generic_server::*; - +use garage_api_common::helpers::*; use garage_api_common::signature::verify_request; +use garage_api_s3::cors::*; use crate::batch::*; +use crate::error::*; use crate::index::*; use crate::item::*; use crate::router::Endpoint; -use garage_api_common::helpers::*; -use garage_api_s3::cors::*; pub use garage_api_common::signature::streaming::ReqBody; pub type ResBody = BoxBody; @@ -31,7 +30,7 @@ pub struct K2VApiServer { garage: Arc, } -pub(crate) struct K2VApiEndpoint { +pub struct K2VApiEndpoint { bucket_name: String, endpoint: Endpoint, } diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index 1dd90456..c284dbd4 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -6,12 +6,13 @@ use garage_table::{EnumerationOrder, TableSchema}; use garage_model::k2v::item_table::*; -use crate::k2v::api_server::{ReqBody, ResBody}; -use crate::k2v::error::*; -use crate::k2v::item::parse_causality_token; -use crate::k2v::range::read_range; use garage_api_common::helpers::*; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::item::parse_causality_token; +use crate::range::read_range; + pub async fn handle_insert_batch( ctx: ReqCtx, req: Request, diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index a4d3be1c..3cd0e6f7 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -2,7 +2,7 @@ use err_derive::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; -use garage_api_common::common_error::CommonError; +use garage_api_common::common_error::{commonErrorDerivative, CommonError}; pub(crate) use garage_api_common::common_error::{helper_error_as_internal, pass_helper_error}; pub use garage_api_common::common_error::{ CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, @@ -16,7 +16,7 @@ use garage_api_common::signature::error::Error as SignatureError; pub enum Error { #[error(display = "{}", _0)] /// Error from common error - Common(CommonError), + Common(#[error(source)] CommonError), // Category: cannot process /// Authorization Header Malformed @@ -44,16 +44,7 @@ pub enum Error { InvalidUtf8Str(#[error(source)] std::str::Utf8Error), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) - } -} - -impl CommonErrorDerivative for Error {} +commonErrorDerivative!(Error); impl From for Error { fn from(err: SignatureError) -> Self { diff --git a/src/api/k2v/index.rs b/src/api/k2v/index.rs index 423c1f97..fbfaad98 100644 --- a/src/api/k2v/index.rs +++ b/src/api/k2v/index.rs @@ -5,10 +5,11 @@ use garage_table::util::*; use garage_model::k2v::item_table::{BYTES, CONFLICTS, ENTRIES, VALUES}; +use garage_api_common::helpers::*; + use crate::api_server::ResBody; use crate::error::*; use crate::range::read_range; -use garage_api_common::helpers::*; pub async fn handle_read_index( ctx: ReqCtx, diff --git a/src/api/k2v/item.rs b/src/api/k2v/item.rs index 315f647c..4e28b499 100644 --- a/src/api/k2v/item.rs +++ b/src/api/k2v/item.rs @@ -6,9 +6,10 @@ use hyper::{Request, Response, StatusCode}; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; +use garage_api_common::helpers::*; + use crate::api_server::{ReqBody, ResBody}; use crate::error::*; -use garage_api_common::helpers::*; pub const X_GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token"; diff --git a/src/api/k2v/range.rs b/src/api/k2v/range.rs index fdb17e22..eb4738db 100644 --- a/src/api/k2v/range.rs +++ b/src/api/k2v/range.rs @@ -7,9 +7,10 @@ use std::sync::Arc; use garage_table::replication::TableShardedReplication; use garage_table::*; -use crate::error::*; use garage_api_common::helpers::key_after_prefix; +use crate::error::*; + /// Read range in a Garage table. /// Returns (entries, more?, nextStart) #[allow(clippy::too_many_arguments)] diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index d24f6a0c..a0dbf52c 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -14,15 +14,15 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; use garage_model::key_table::Key; -use crate::error::*; use garage_api_common::generic_server::*; - +use garage_api_common::helpers::*; use garage_api_common::signature::verify_request; use crate::bucket::*; use crate::copy::*; use crate::cors::*; use crate::delete::*; +use crate::error::*; use crate::get::*; use crate::lifecycle::*; use crate::list::*; @@ -31,7 +31,6 @@ use crate::post_object::handle_post_object; use crate::put::*; use crate::router::Endpoint; use crate::website::*; -use garage_api_common::helpers::*; pub use garage_api_common::signature::streaming::ReqBody; pub type ResBody = BoxBody; @@ -40,7 +39,7 @@ pub struct S3ApiServer { garage: Arc, } -pub(crate) struct S3ApiEndpoint { +pub struct S3ApiEndpoint { bucket_name: Option, endpoint: Endpoint, } diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 09c5742b..0a192ba6 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -13,13 +13,14 @@ use garage_util::crdt::*; use garage_util::data::*; use garage_util::time::*; -use crate::api_server::{ReqBody, ResBody}; -use crate::error::*; -use crate::xml as s3_xml; use garage_api_common::common_error::CommonError; use garage_api_common::helpers::*; use garage_api_common::signature::verify_signed_content; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml as s3_xml; + pub fn handle_get_bucket_location(ctx: ReqCtx) -> Result, Error> { let ReqCtx { garage, .. } = ctx; let loc = s3_xml::LocationConstraint { diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 1a474fd0..e4992a18 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -20,6 +20,8 @@ use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use garage_api_common::helpers::*; + use crate::api_server::{ReqBody, ResBody}; use crate::checksum::*; use crate::encryption::EncryptionParams; @@ -28,7 +30,6 @@ use crate::get::full_object_byte_stream; use crate::multipart; use crate::put::{get_headers, save_stream, ChecksumMode, SaveStreamResult}; use crate::xml::{self as s3_xml, xmlns_tag}; -use garage_api_common::helpers::*; // -------- CopyObject --------- diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index ae8352c3..4bd81e32 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -15,16 +15,17 @@ use http_body_util::BodyExt; use serde::{Deserialize, Serialize}; -use crate::api_server::{ReqBody, ResBody}; -use crate::error::*; -use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule}; +use garage_model::garage::Garage; +use garage_util::data::*; + use garage_api_common::common_error::{helper_error_as_internal, CommonError}; use garage_api_common::helpers::*; use garage_api_common::signature::verify_signed_content; -use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule}; -use garage_model::garage::Garage; -use garage_util::data::*; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; pub async fn handle_get_cors(ctx: ReqCtx) -> Result, Error> { let ReqCtx { bucket_params, .. } = ctx; diff --git a/src/api/s3/delete.rs b/src/api/s3/delete.rs index 1711a9b4..b799e67a 100644 --- a/src/api/s3/delete.rs +++ b/src/api/s3/delete.rs @@ -5,12 +5,13 @@ use garage_util::data::*; use garage_model::s3::object_table::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; + use crate::api_server::{ReqBody, ResBody}; use crate::error::*; use crate::put::next_timestamp; use crate::xml as s3_xml; -use garage_api_common::helpers::*; -use garage_api_common::signature::verify_signed_content; async fn handle_delete_internal(ctx: &ReqCtx, key: &str) -> Result<(Uuid, Uuid), Error> { let ReqCtx { diff --git a/src/api/s3/encryption.rs b/src/api/s3/encryption.rs index c54d487b..b38d7792 100644 --- a/src/api/s3/encryption.rs +++ b/src/api/s3/encryption.rs @@ -28,9 +28,10 @@ use garage_util::migrate::Migrate; use garage_model::garage::Garage; use garage_model::s3::object_table::{ObjectVersionEncryption, ObjectVersionMetaInner}; +use garage_api_common::common_error::*; + use crate::checksum::Md5Checksum; use crate::error::Error; -use garage_api_common::common_error::*; const X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM: HeaderName = HeaderName::from_static("x-amz-server-side-encryption-customer-algorithm"); diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 77dc07c8..1bb8909c 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -8,23 +8,26 @@ use garage_model::helper::error::Error as HelperError; pub(crate) use garage_api_common::common_error::pass_helper_error; -use garage_api_common::common_error::{helper_error_as_internal, CommonError}; +use garage_api_common::common_error::{ + commonErrorDerivative, helper_error_as_internal, CommonError, +}; pub use garage_api_common::common_error::{ CommonErrorDerivative, OkOrBadRequest, OkOrInternalError, }; -use crate::xml as s3_xml; use garage_api_common::generic_server::ApiError; use garage_api_common::helpers::*; use garage_api_common::signature::error::Error as SignatureError; +use crate::xml as s3_xml; + /// Errors of this crate #[derive(Debug, Error)] pub enum Error { #[error(display = "{}", _0)] /// Error from common error - Common(CommonError), + Common(#[error(source)] CommonError), // Category: cannot process /// Authorization Header Malformed @@ -86,14 +89,7 @@ pub enum Error { NotImplemented(String), } -impl From for Error -where - CommonError: From, -{ - fn from(err: T) -> Self { - Error::Common(CommonError::from(err)) - } -} +commonErrorDerivative!(Error); // Helper errors are always passed as internal errors by default. // To pass the specific error code back to the client, use `pass_helper_error`. @@ -103,8 +99,6 @@ impl From for Error { } } -impl CommonErrorDerivative for Error {} - impl From for Error { fn from(err: roxmltree::Error) -> Self { Self::InvalidXml(format!("{}", err)) diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index c4cd9d48..c2393a51 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -25,11 +25,12 @@ use garage_model::garage::Garage; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use garage_api_common::helpers::*; + use crate::api_server::ResBody; use crate::checksum::{add_checksum_response_headers, X_AMZ_CHECKSUM_MODE}; use crate::encryption::EncryptionParams; use crate::error::*; -use garage_api_common::helpers::*; const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count"; diff --git a/src/api/s3/lifecycle.rs b/src/api/s3/lifecycle.rs index da211585..c35047ed 100644 --- a/src/api/s3/lifecycle.rs +++ b/src/api/s3/lifecycle.rs @@ -5,11 +5,12 @@ use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; + use crate::api_server::{ReqBody, ResBody}; use crate::error::*; use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; -use garage_api_common::helpers::*; -use garage_api_common::signature::verify_signed_content; use garage_model::bucket_table::{ parse_lifecycle_date, Bucket, LifecycleExpiration as GarageLifecycleExpiration, diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index de808c32..a5cc03b0 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -13,13 +13,14 @@ use garage_model::s3::object_table::*; use garage_table::EnumerationOrder; +use garage_api_common::encoding::*; +use garage_api_common::helpers::*; + use crate::api_server::{ReqBody, ResBody}; use crate::encryption::EncryptionParams; use crate::error::*; use crate::multipart as s3_multipart; use crate::xml as s3_xml; -use garage_api_common::encoding::*; -use garage_api_common::helpers::*; const DUMMY_NAME: &str = "Dummy Key"; const DUMMY_KEY: &str = "GKDummyKey"; diff --git a/src/api/s3/multipart.rs b/src/api/s3/multipart.rs index 047ed06a..fe39fc93 100644 --- a/src/api/s3/multipart.rs +++ b/src/api/s3/multipart.rs @@ -15,14 +15,15 @@ use garage_model::s3::mpu_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::verify_signed_content; + use crate::api_server::{ReqBody, ResBody}; use crate::checksum::*; use crate::encryption::EncryptionParams; use crate::error::*; use crate::put::*; use crate::xml as s3_xml; -use garage_api_common::helpers::*; -use garage_api_common::signature::verify_signed_content; // ---- diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 6416c523..2bcabf1d 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -16,6 +16,9 @@ use serde::Deserialize; use garage_model::garage::Garage; use garage_model::s3::object_table::*; +use garage_api_common::helpers::*; +use garage_api_common::signature::payload::{verify_v4, Authorization}; + use crate::api_server::ResBody; use crate::checksum::*; use crate::cors::*; @@ -23,8 +26,6 @@ use crate::encryption::EncryptionParams; use crate::error::*; use crate::put::{get_headers, save_stream, ChecksumMode}; use crate::xml as s3_xml; -use garage_api_common::helpers::*; -use garage_api_common::signature::payload::{verify_v4, Authorization}; pub async fn handle_post_object( garage: Arc, diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 47dcb8f7..530b4e7b 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -30,11 +30,12 @@ use garage_model::s3::block_ref_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use garage_api_common::helpers::*; + use crate::api_server::{ReqBody, ResBody}; use crate::checksum::*; use crate::encryption::EncryptionParams; use crate::error::*; -use garage_api_common::helpers::*; const PUT_BLOCKS_MAX_PARALLEL: usize = 3; diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index 94951e80..9de84b2b 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -3,10 +3,11 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; -use crate::error::*; use garage_api_common::helpers::Authorization; use garage_api_common::router_macros::{generateQueryParameters, router_match}; +use crate::error::*; + router_match! {@func /// List of all S3 API endpoints. diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index 46decccf..b55bb345 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -4,14 +4,15 @@ use http_body_util::BodyExt; use hyper::{Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::api_server::{ReqBody, ResBody}; -use crate::error::*; -use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; +use garage_model::bucket_table::*; +use garage_util::data::*; + use garage_api_common::helpers::*; use garage_api_common::signature::verify_signed_content; -use garage_model::bucket_table::*; -use garage_util::data::*; +use crate::api_server::{ReqBody, ResBody}; +use crate::error::*; +use crate::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; pub async fn handle_get_website(ctx: ReqCtx) -> Result, Error> { let ReqCtx { bucket_params, .. } = ctx; -- cgit v1.2.3 From afa28706e5566737376f8448bcc548f780f0f57f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 18:42:14 +0100 Subject: split s3/cors.rs into also common/cors.rs --- src/api/admin/Cargo.toml | 2 - src/api/common/Cargo.toml | 2 - src/api/common/cors.rs | 170 ++++++++++++++++++++++++++++++++++++++++++++++ src/api/common/lib.rs | 2 +- src/api/k2v/Cargo.toml | 3 - src/api/k2v/api_server.rs | 2 +- src/api/s3/api_server.rs | 1 + src/api/s3/cors.rs | 170 +--------------------------------------------- src/api/s3/post_object.rs | 2 +- 9 files changed, 176 insertions(+), 178 deletions(-) create mode 100644 src/api/common/cors.rs (limited to 'src/api') diff --git a/src/api/admin/Cargo.toml b/src/api/admin/Cargo.toml index 804166b3..c816a6a9 100644 --- a/src/api/admin/Cargo.toml +++ b/src/api/admin/Cargo.toml @@ -58,13 +58,11 @@ hyper = { workspace = true, default-features = false, features = ["server", "htt hyper-util.workspace = true multer.workspace = true percent-encoding.workspace = true -roxmltree.workspace = true url.workspace = true serde.workspace = true serde_bytes.workspace = true serde_json.workspace = true -quick-xml.workspace = true opentelemetry.workspace = true opentelemetry-prometheus = { workspace = true, optional = true } diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml index e5dc57d4..7be16a09 100644 --- a/src/api/common/Cargo.toml +++ b/src/api/common/Cargo.toml @@ -57,13 +57,11 @@ hyper = { workspace = true, default-features = false, features = ["server", "htt hyper-util.workspace = true multer.workspace = true percent-encoding.workspace = true -roxmltree.workspace = true url.workspace = true serde.workspace = true serde_bytes.workspace = true serde_json.workspace = true -quick-xml.workspace = true opentelemetry.workspace = true opentelemetry-prometheus = { workspace = true, optional = true } diff --git a/src/api/common/cors.rs b/src/api/common/cors.rs new file mode 100644 index 00000000..14369b56 --- /dev/null +++ b/src/api/common/cors.rs @@ -0,0 +1,170 @@ +use std::sync::Arc; + +use http::header::{ + ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, + ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, +}; +use hyper::{body::Body, body::Incoming as IncomingBody, Request, Response, StatusCode}; + +use garage_model::bucket_table::{BucketParams, CorsRule as GarageCorsRule}; +use garage_model::garage::Garage; + +use crate::common_error::{ + helper_error_as_internal, CommonError, OkOrBadRequest, OkOrInternalError, +}; +use crate::helpers::*; + +pub fn find_matching_cors_rule<'a>( + bucket_params: &'a BucketParams, + req: &Request, +) -> Result, CommonError> { + if let Some(cors_config) = bucket_params.cors_config.get() { + if let Some(origin) = req.headers().get("Origin") { + let origin = origin.to_str()?; + let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { + Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), + None => vec![], + }; + return Ok(cors_config.iter().find(|rule| { + cors_rule_matches(rule, origin, req.method().as_ref(), request_headers.iter()) + })); + } + } + Ok(None) +} + +pub fn cors_rule_matches<'a, HI, S>( + rule: &GarageCorsRule, + origin: &'a str, + method: &'a str, + mut request_headers: HI, +) -> bool +where + HI: Iterator, + S: AsRef, +{ + rule.allow_origins.iter().any(|x| x == "*" || x == origin) + && rule.allow_methods.iter().any(|x| x == "*" || x == method) + && request_headers.all(|h| { + rule.allow_headers + .iter() + .any(|x| x == "*" || x == h.as_ref()) + }) +} + +pub fn add_cors_headers( + resp: &mut Response, + rule: &GarageCorsRule, +) -> Result<(), http::header::InvalidHeaderValue> { + let h = resp.headers_mut(); + h.insert( + ACCESS_CONTROL_ALLOW_ORIGIN, + rule.allow_origins.join(", ").parse()?, + ); + h.insert( + ACCESS_CONTROL_ALLOW_METHODS, + rule.allow_methods.join(", ").parse()?, + ); + h.insert( + ACCESS_CONTROL_ALLOW_HEADERS, + rule.allow_headers.join(", ").parse()?, + ); + h.insert( + ACCESS_CONTROL_EXPOSE_HEADERS, + rule.expose_headers.join(", ").parse()?, + ); + Ok(()) +} + +pub async fn handle_options_api( + garage: Arc, + req: &Request, + bucket_name: Option, +) -> Result, CommonError> { + // FIXME: CORS rules of buckets with local aliases are + // not taken into account. + + // If the bucket name is a global bucket name, + // we try to apply the CORS rules of that bucket. + // If a user has a local bucket name that has + // the same name, its CORS rules won't be applied + // and will be shadowed by the rules of the globally + // existing bucket (but this is inevitable because + // OPTIONS calls are not auhtenticated). + if let Some(bn) = bucket_name { + let helper = garage.bucket_helper(); + let bucket_id = helper + .resolve_global_bucket_name(&bn) + .await + .map_err(helper_error_as_internal)?; + if let Some(id) = bucket_id { + let bucket = garage + .bucket_helper() + .get_existing_bucket(id) + .await + .map_err(helper_error_as_internal)?; + let bucket_params = bucket.state.into_option().unwrap(); + handle_options_for_bucket(req, &bucket_params) + } else { + // If there is a bucket name in the request, but that name + // does not correspond to a global alias for a bucket, + // then it's either a non-existing bucket or a local bucket. + // We have no way of knowing, because the request is not + // authenticated and thus we can't resolve local aliases. + // We take the permissive approach of allowing everything, + // because we don't want to prevent web apps that use + // local bucket names from making API calls. + Ok(Response::builder() + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(ACCESS_CONTROL_ALLOW_METHODS, "*") + .status(StatusCode::OK) + .body(EmptyBody::new())?) + } + } else { + // If there is no bucket name in the request, + // we are doing a ListBuckets call, which we want to allow + // for all origins. + Ok(Response::builder() + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .header(ACCESS_CONTROL_ALLOW_METHODS, "GET") + .status(StatusCode::OK) + .body(EmptyBody::new())?) + } +} + +pub fn handle_options_for_bucket( + req: &Request, + bucket_params: &BucketParams, +) -> Result, CommonError> { + let origin = req + .headers() + .get("Origin") + .ok_or_bad_request("Missing Origin header")? + .to_str()?; + let request_method = req + .headers() + .get(ACCESS_CONTROL_REQUEST_METHOD) + .ok_or_bad_request("Missing Access-Control-Request-Method header")? + .to_str()?; + let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { + Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), + None => vec![], + }; + + if let Some(cors_config) = bucket_params.cors_config.get() { + let matching_rule = cors_config + .iter() + .find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter())); + if let Some(rule) = matching_rule { + let mut resp = Response::builder() + .status(StatusCode::OK) + .body(EmptyBody::new())?; + add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?; + return Ok(resp); + } + } + + Err(CommonError::Forbidden( + "This CORS request is not allowed.".into(), + )) +} diff --git a/src/api/common/lib.rs b/src/api/common/lib.rs index 49d463d7..0e655a53 100644 --- a/src/api/common/lib.rs +++ b/src/api/common/lib.rs @@ -4,9 +4,9 @@ extern crate tracing; pub mod common_error; +pub mod cors; pub mod encoding; pub mod generic_server; pub mod helpers; pub mod router_macros; -/// This mode is public only to help testing. Don't expect stability here pub mod signature; diff --git a/src/api/k2v/Cargo.toml b/src/api/k2v/Cargo.toml index 86d12c2d..1e4c53ad 100644 --- a/src/api/k2v/Cargo.toml +++ b/src/api/k2v/Cargo.toml @@ -21,7 +21,6 @@ garage_net.workspace = true garage_util.workspace = true garage_rpc.workspace = true garage_api_common.workspace = true -garage_api_s3.workspace = true aes-gcm.workspace = true argon2.workspace = true @@ -59,13 +58,11 @@ hyper = { workspace = true, default-features = false, features = ["server", "htt hyper-util.workspace = true multer.workspace = true percent-encoding.workspace = true -roxmltree.workspace = true url.workspace = true serde.workspace = true serde_bytes.workspace = true serde_json.workspace = true -quick-xml.workspace = true opentelemetry.workspace = true opentelemetry-prometheus = { workspace = true, optional = true } diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index 0791c07d..31e07762 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -12,10 +12,10 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; +use garage_api_common::cors::*; use garage_api_common::generic_server::*; use garage_api_common::helpers::*; use garage_api_common::signature::verify_request; -use garage_api_s3::cors::*; use crate::batch::*; use crate::error::*; diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index a0dbf52c..ed71b108 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -14,6 +14,7 @@ use garage_util::socket_address::UnixOrTCPSocketAddress; use garage_model::garage::Garage; use garage_model::key_table::Key; +use garage_api_common::cors::*; use garage_api_common::generic_server::*; use garage_api_common::helpers::*; use garage_api_common::signature::verify_request; diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index 4bd81e32..625b84db 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -1,25 +1,14 @@ -use std::sync::Arc; - use quick_xml::de::from_reader; -use http::header::{ - ACCESS_CONTROL_ALLOW_HEADERS, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, - ACCESS_CONTROL_EXPOSE_HEADERS, ACCESS_CONTROL_REQUEST_HEADERS, ACCESS_CONTROL_REQUEST_METHOD, -}; -use hyper::{ - body::Body, body::Incoming as IncomingBody, header::HeaderName, Method, Request, Response, - StatusCode, -}; +use hyper::{header::HeaderName, Method, Request, Response, StatusCode}; use http_body_util::BodyExt; use serde::{Deserialize, Serialize}; -use garage_model::bucket_table::{Bucket, BucketParams, CorsRule as GarageCorsRule}; -use garage_model::garage::Garage; +use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule}; use garage_util::data::*; -use garage_api_common::common_error::{helper_error_as_internal, CommonError}; use garage_api_common::helpers::*; use garage_api_common::signature::verify_signed_content; @@ -101,161 +90,6 @@ pub async fn handle_put_cors( .body(empty_body())?) } -pub async fn handle_options_api( - garage: Arc, - req: &Request, - bucket_name: Option, -) -> Result, CommonError> { - // FIXME: CORS rules of buckets with local aliases are - // not taken into account. - - // If the bucket name is a global bucket name, - // we try to apply the CORS rules of that bucket. - // If a user has a local bucket name that has - // the same name, its CORS rules won't be applied - // and will be shadowed by the rules of the globally - // existing bucket (but this is inevitable because - // OPTIONS calls are not auhtenticated). - if let Some(bn) = bucket_name { - let helper = garage.bucket_helper(); - let bucket_id = helper - .resolve_global_bucket_name(&bn) - .await - .map_err(helper_error_as_internal)?; - if let Some(id) = bucket_id { - let bucket = garage - .bucket_helper() - .get_existing_bucket(id) - .await - .map_err(helper_error_as_internal)?; - let bucket_params = bucket.state.into_option().unwrap(); - handle_options_for_bucket(req, &bucket_params) - } else { - // If there is a bucket name in the request, but that name - // does not correspond to a global alias for a bucket, - // then it's either a non-existing bucket or a local bucket. - // We have no way of knowing, because the request is not - // authenticated and thus we can't resolve local aliases. - // We take the permissive approach of allowing everything, - // because we don't want to prevent web apps that use - // local bucket names from making API calls. - Ok(Response::builder() - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(ACCESS_CONTROL_ALLOW_METHODS, "*") - .status(StatusCode::OK) - .body(EmptyBody::new())?) - } - } else { - // If there is no bucket name in the request, - // we are doing a ListBuckets call, which we want to allow - // for all origins. - Ok(Response::builder() - .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") - .header(ACCESS_CONTROL_ALLOW_METHODS, "GET") - .status(StatusCode::OK) - .body(EmptyBody::new())?) - } -} - -pub fn handle_options_for_bucket( - req: &Request, - bucket_params: &BucketParams, -) -> Result, CommonError> { - let origin = req - .headers() - .get("Origin") - .ok_or_bad_request("Missing Origin header")? - .to_str()?; - let request_method = req - .headers() - .get(ACCESS_CONTROL_REQUEST_METHOD) - .ok_or_bad_request("Missing Access-Control-Request-Method header")? - .to_str()?; - let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { - Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), - None => vec![], - }; - - if let Some(cors_config) = bucket_params.cors_config.get() { - let matching_rule = cors_config - .iter() - .find(|rule| cors_rule_matches(rule, origin, request_method, request_headers.iter())); - if let Some(rule) = matching_rule { - let mut resp = Response::builder() - .status(StatusCode::OK) - .body(EmptyBody::new())?; - add_cors_headers(&mut resp, rule).ok_or_internal_error("Invalid CORS configuration")?; - return Ok(resp); - } - } - - Err(CommonError::Forbidden( - "This CORS request is not allowed.".into(), - )) -} - -pub fn find_matching_cors_rule<'a>( - bucket_params: &'a BucketParams, - req: &Request, -) -> Result, Error> { - if let Some(cors_config) = bucket_params.cors_config.get() { - if let Some(origin) = req.headers().get("Origin") { - let origin = origin.to_str()?; - let request_headers = match req.headers().get(ACCESS_CONTROL_REQUEST_HEADERS) { - Some(h) => h.to_str()?.split(',').map(|h| h.trim()).collect::>(), - None => vec![], - }; - return Ok(cors_config.iter().find(|rule| { - cors_rule_matches(rule, origin, req.method().as_ref(), request_headers.iter()) - })); - } - } - Ok(None) -} - -fn cors_rule_matches<'a, HI, S>( - rule: &GarageCorsRule, - origin: &'a str, - method: &'a str, - mut request_headers: HI, -) -> bool -where - HI: Iterator, - S: AsRef, -{ - rule.allow_origins.iter().any(|x| x == "*" || x == origin) - && rule.allow_methods.iter().any(|x| x == "*" || x == method) - && request_headers.all(|h| { - rule.allow_headers - .iter() - .any(|x| x == "*" || x == h.as_ref()) - }) -} - -pub fn add_cors_headers( - resp: &mut Response, - rule: &GarageCorsRule, -) -> Result<(), http::header::InvalidHeaderValue> { - let h = resp.headers_mut(); - h.insert( - ACCESS_CONTROL_ALLOW_ORIGIN, - rule.allow_origins.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_ALLOW_METHODS, - rule.allow_methods.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_ALLOW_HEADERS, - rule.allow_headers.join(", ").parse()?, - ); - h.insert( - ACCESS_CONTROL_EXPOSE_HEADERS, - rule.expose_headers.join(", ").parse()?, - ); - Ok(()) -} - // ---- SERIALIZATION AND DESERIALIZATION TO/FROM S3 XML ---- #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)] diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 2bcabf1d..6c0e73d4 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -16,12 +16,12 @@ use serde::Deserialize; use garage_model::garage::Garage; use garage_model::s3::object_table::*; +use garage_api_common::cors::*; use garage_api_common::helpers::*; use garage_api_common::signature::payload::{verify_v4, Authorization}; use crate::api_server::ResBody; use crate::checksum::*; -use crate::cors::*; use crate::encryption::EncryptionParams; use crate::error::*; use crate::put::{get_headers, save_stream, ChecksumMode}; -- cgit v1.2.3 From 4563313f87af4b7be26152164a5ce09a451da0d9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 18:47:30 +0100 Subject: use cargo-shear to remove many unused dependencies between crates --- src/api/admin/Cargo.toml | 32 +------------------------------- src/api/common/Cargo.toml | 27 --------------------------- src/api/k2v/Cargo.toml | 32 -------------------------------- src/api/s3/Cargo.toml | 10 ---------- 4 files changed, 1 insertion(+), 100 deletions(-) (limited to 'src/api') diff --git a/src/api/admin/Cargo.toml b/src/api/admin/Cargo.toml index c816a6a9..55767dcf 100644 --- a/src/api/admin/Cargo.toml +++ b/src/api/admin/Cargo.toml @@ -16,52 +16,22 @@ path = "lib.rs" [dependencies] garage_model.workspace = true garage_table.workspace = true -garage_block.workspace = true -garage_net.workspace = true garage_util.workspace = true garage_rpc.workspace = true garage_api_common.workspace = true - -aes-gcm.workspace = true argon2.workspace = true -async-compression.workspace = true async-trait.workspace = true -base64.workspace = true -bytes.workspace = true -chrono.workspace = true -crc32fast.workspace = true -crc32c.workspace = true -crypto-common.workspace = true err-derive.workspace = true hex.workspace = true -hmac.workspace = true -idna.workspace = true tracing.workspace = true -md-5.workspace = true -nom.workspace = true -pin-project.workspace = true -sha1.workspace = true -sha2.workspace = true futures.workspace = true -futures-util.workspace = true tokio.workspace = true -tokio-stream.workspace = true -tokio-util.workspace = true - -form_urlencoded.workspace = true http.workspace = true -httpdate.workspace = true -http-range.workspace = true -http-body-util.workspace = true hyper = { workspace = true, default-features = false, features = ["server", "http1"] } -hyper-util.workspace = true -multer.workspace = true -percent-encoding.workspace = true url.workspace = true serde.workspace = true -serde_bytes.workspace = true serde_json.workspace = true opentelemetry.workspace = true @@ -69,4 +39,4 @@ opentelemetry-prometheus = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } [features] -metrics = [ "opentelemetry-prometheus", "prometheus", "garage_api_common/metrics" ] +metrics = [ "opentelemetry-prometheus", "prometheus" ] diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml index 7be16a09..eea43efe 100644 --- a/src/api/common/Cargo.toml +++ b/src/api/common/Cargo.toml @@ -16,56 +16,29 @@ path = "lib.rs" [dependencies] garage_model.workspace = true garage_table.workspace = true -garage_block.workspace = true -garage_net.workspace = true garage_util.workspace = true -garage_rpc.workspace = true - -aes-gcm.workspace = true -argon2.workspace = true -async-compression.workspace = true async-trait.workspace = true -base64.workspace = true bytes.workspace = true chrono.workspace = true -crc32fast.workspace = true -crc32c.workspace = true crypto-common.workspace = true err-derive.workspace = true hex.workspace = true hmac.workspace = true idna.workspace = true tracing.workspace = true -md-5.workspace = true nom.workspace = true pin-project.workspace = true -sha1.workspace = true sha2.workspace = true futures.workspace = true -futures-util.workspace = true tokio.workspace = true -tokio-stream.workspace = true -tokio-util.workspace = true - -form_urlencoded.workspace = true http.workspace = true -httpdate.workspace = true -http-range.workspace = true http-body-util.workspace = true hyper = { workspace = true, default-features = false, features = ["server", "http1"] } hyper-util.workspace = true -multer.workspace = true -percent-encoding.workspace = true url.workspace = true serde.workspace = true -serde_bytes.workspace = true serde_json.workspace = true opentelemetry.workspace = true -opentelemetry-prometheus = { workspace = true, optional = true } -prometheus = { workspace = true, optional = true } - -[features] -metrics = [ "opentelemetry-prometheus", "prometheus" ] diff --git a/src/api/k2v/Cargo.toml b/src/api/k2v/Cargo.toml index 1e4c53ad..01376020 100644 --- a/src/api/k2v/Cargo.toml +++ b/src/api/k2v/Cargo.toml @@ -16,57 +16,25 @@ path = "lib.rs" [dependencies] garage_model.workspace = true garage_table.workspace = true -garage_block.workspace = true -garage_net.workspace = true garage_util.workspace = true -garage_rpc.workspace = true garage_api_common.workspace = true - -aes-gcm.workspace = true -argon2.workspace = true -async-compression.workspace = true async-trait.workspace = true base64.workspace = true -bytes.workspace = true -chrono.workspace = true -crc32fast.workspace = true -crc32c.workspace = true -crypto-common.workspace = true err-derive.workspace = true -hex.workspace = true -hmac.workspace = true -idna.workspace = true tracing.workspace = true -md-5.workspace = true -nom.workspace = true -pin-project.workspace = true -sha1.workspace = true -sha2.workspace = true futures.workspace = true -futures-util.workspace = true tokio.workspace = true -tokio-stream.workspace = true -tokio-util.workspace = true - -form_urlencoded.workspace = true http.workspace = true -httpdate.workspace = true -http-range.workspace = true http-body-util.workspace = true hyper = { workspace = true, default-features = false, features = ["server", "http1"] } -hyper-util.workspace = true -multer.workspace = true percent-encoding.workspace = true url.workspace = true serde.workspace = true -serde_bytes.workspace = true serde_json.workspace = true opentelemetry.workspace = true -opentelemetry-prometheus = { workspace = true, optional = true } -prometheus = { workspace = true, optional = true } [features] default = [ "garage_util/k2v", "garage_model/k2v" ] diff --git a/src/api/s3/Cargo.toml b/src/api/s3/Cargo.toml index c610b43a..a1751c9f 100644 --- a/src/api/s3/Cargo.toml +++ b/src/api/s3/Cargo.toml @@ -23,7 +23,6 @@ garage_rpc.workspace = true garage_api_common.workspace = true aes-gcm.workspace = true -argon2.workspace = true async-compression.workspace = true async-trait.workspace = true base64.workspace = true @@ -31,20 +30,15 @@ bytes.workspace = true chrono.workspace = true crc32fast.workspace = true crc32c.workspace = true -crypto-common.workspace = true err-derive.workspace = true hex.workspace = true -hmac.workspace = true -idna.workspace = true tracing.workspace = true md-5.workspace = true -nom.workspace = true pin-project.workspace = true sha1.workspace = true sha2.workspace = true futures.workspace = true -futures-util.workspace = true tokio.workspace = true tokio-stream.workspace = true tokio-util.workspace = true @@ -55,17 +49,13 @@ httpdate.workspace = true http-range.workspace = true http-body-util.workspace = true hyper = { workspace = true, default-features = false, features = ["server", "http1"] } -hyper-util.workspace = true multer.workspace = true percent-encoding.workspace = true roxmltree.workspace = true url.workspace = true serde.workspace = true -serde_bytes.workspace = true serde_json.workspace = true quick-xml.workspace = true opentelemetry.workspace = true -opentelemetry-prometheus = { workspace = true, optional = true } -prometheus = { workspace = true, optional = true } -- cgit v1.2.3 From 3d5e9a027e67a924edffe300e06122cc4a24e02d Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 18:52:42 +0100 Subject: cargo defs: simplify and fix descriptions --- src/api/admin/Cargo.toml | 3 ++- src/api/common/Cargo.toml | 3 ++- src/api/k2v/Cargo.toml | 10 ++++------ 3 files changed, 8 insertions(+), 8 deletions(-) (limited to 'src/api') diff --git a/src/api/admin/Cargo.toml b/src/api/admin/Cargo.toml index 55767dcf..adddf306 100644 --- a/src/api/admin/Cargo.toml +++ b/src/api/admin/Cargo.toml @@ -4,7 +4,7 @@ version = "1.0.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" -description = "S3 API server crate for the Garage object store" +description = "Admin API server crate for the Garage object store" repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" readme = "../../README.md" @@ -19,6 +19,7 @@ garage_table.workspace = true garage_util.workspace = true garage_rpc.workspace = true garage_api_common.workspace = true + argon2.workspace = true async-trait.workspace = true err-derive.workspace = true diff --git a/src/api/common/Cargo.toml b/src/api/common/Cargo.toml index eea43efe..842662c4 100644 --- a/src/api/common/Cargo.toml +++ b/src/api/common/Cargo.toml @@ -4,7 +4,7 @@ version = "1.0.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" -description = "S3 API server crate for the Garage object store" +description = "Common functions for the API server crates for the Garage object store" repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" readme = "../../README.md" @@ -17,6 +17,7 @@ path = "lib.rs" garage_model.workspace = true garage_table.workspace = true garage_util.workspace = true + async-trait.workspace = true bytes.workspace = true chrono.workspace = true diff --git a/src/api/k2v/Cargo.toml b/src/api/k2v/Cargo.toml index 01376020..d4e26efa 100644 --- a/src/api/k2v/Cargo.toml +++ b/src/api/k2v/Cargo.toml @@ -4,7 +4,7 @@ version = "1.0.1" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" -description = "S3 API server crate for the Garage object store" +description = "K2V API server crate for the Garage object store" repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" readme = "../../README.md" @@ -14,10 +14,11 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_model.workspace = true +garage_model = { workspace = true, features = [ "k2v" ] } garage_table.workspace = true -garage_util.workspace = true +garage_util = { workspace = true, features = [ "k2v" ] } garage_api_common.workspace = true + async-trait.workspace = true base64.workspace = true err-derive.workspace = true @@ -35,6 +36,3 @@ serde.workspace = true serde_json.workspace = true opentelemetry.workspace = true - -[features] -default = [ "garage_util/k2v", "garage_model/k2v" ] -- cgit v1.2.3 From d18c5ad0ffc9ffe3ec38d234445ef1a826a57e67 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 31 Jan 2025 19:12:05 +0100 Subject: fix tests --- src/api/s3/copy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/api') diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index e4992a18..07d50ea5 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -863,7 +863,7 @@ pub struct CopyPartResult { #[cfg(test)] mod tests { use super::*; - use crate::s3::xml::to_xml_with_header; + use crate::xml::to_xml_with_header; #[test] fn copy_object_result() -> Result<(), Error> { -- cgit v1.2.3