diff options
Diffstat (limited to 'src')
29 files changed, 276 insertions, 161 deletions
diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 747f70ab..a9279c37 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api" -version = "0.8.2" +version = "0.8.3" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" @@ -28,7 +28,7 @@ crypto-common = "0.1" err-derive = "0.3" hex = "0.4" hmac = "0.12" -idna = "0.3" +idna = "0.4" tracing = "0.1" md-5 = "0.10" nom = "7.1" @@ -47,7 +47,7 @@ http-range = "0.1" hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } multer = "2.0" percent-encoding = "2.1.0" -roxmltree = "0.14" +roxmltree = "0.18" serde = { version = "1.0", features = ["derive"] } serde_bytes = "0.11" serde_json = "1.0" diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 6819e28e..cc04d81f 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -26,6 +26,7 @@ use crate::admin::cluster::*; use crate::admin::error::*; use crate::admin::key::*; use crate::admin::router::{Authorization, Endpoint}; +use crate::helpers::host_to_bucket; pub struct AdminApiServer { garage: Arc<Garage>, @@ -78,10 +79,7 @@ impl AdminApiServer { .body(Body::empty())?) } - async fn handle_check_website_enabled( - &self, - req: Request<Body>, - ) -> Result<Response<Body>, Error> { + async fn handle_check_domain(&self, req: Request<Body>) -> Result<Response<Body>, Error> { let query_params: HashMap<String, String> = req .uri() .query() @@ -102,12 +100,56 @@ impl AdminApiServer { .get("domain") .ok_or_internal_error("Could not parse domain query string")?; - let bucket_id = self + if self.check_domain(domain).await? { + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(format!( + "Domain '{domain}' is managed by Garage" + )))?) + } else { + Err(Error::bad_request(format!( + "Domain '{domain}' is not managed by Garage" + ))) + } + } + + async fn check_domain(&self, domain: &str) -> Result<bool, Error> { + // Resolve bucket from domain name, inferring if the website must be activated for the + // domain to be valid. + let (bucket_name, must_check_website) = if let Some(bname) = self + .garage + .config + .s3_api + .root_domain + .as_ref() + .and_then(|rd| host_to_bucket(domain, rd)) + { + (bname.to_string(), false) + } else if let Some(bname) = self + .garage + .config + .s3_web + .as_ref() + .and_then(|sw| host_to_bucket(domain, sw.root_domain.as_str())) + { + (bname.to_string(), true) + } else { + (domain.to_string(), true) + }; + + let bucket_id = match self .garage .bucket_helper() - .resolve_global_bucket_name(domain) + .resolve_global_bucket_name(&bucket_name) .await? - .ok_or(HelperError::NoSuchBucket(domain.to_string()))?; + { + Some(bucket_id) => bucket_id, + None => return Ok(false), + }; + + if !must_check_website { + return Ok(true); + } let bucket = self .garage @@ -119,16 +161,8 @@ impl AdminApiServer { let bucket_website_config = bucket_state.website_config.get(); match bucket_website_config { - Some(_v) => { - Ok(Response::builder() - .status(StatusCode::OK) - .body(Body::from(format!( - "Bucket '{domain}' is authorized for website hosting" - )))?) - } - None => Err(Error::bad_request(format!( - "Bucket '{domain}' is not authorized for website hosting" - ))), + Some(_v) => Ok(true), + None => Ok(false), } } @@ -229,7 +263,7 @@ impl ApiHandler for AdminApiServer { match endpoint { Endpoint::Options => self.handle_options(&req), - Endpoint::CheckWebsiteEnabled => self.handle_check_website_enabled(req).await, + Endpoint::CheckDomain => self.handle_check_domain(req).await, Endpoint::Health => self.handle_health(), Endpoint::Metrics => self.handle_metrics(), Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await, diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index d54dabe8..254aff12 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -17,7 +17,7 @@ router_match! {@func #[derive(Debug, Clone, PartialEq, Eq)] pub enum Endpoint { Options, - CheckWebsiteEnabled, + CheckDomain, Health, Metrics, GetClusterStatus, @@ -93,7 +93,7 @@ impl Endpoint { let res = router_match!(@gen_path_parser (req.method(), path, query) [ OPTIONS _ => Options, - GET "/check" => CheckWebsiteEnabled, + GET "/check" => CheckDomain, GET "/health" => Health, GET "/metrics" => Metrics, GET "/v1/status" => GetClusterStatus, @@ -139,7 +139,7 @@ impl Endpoint { pub fn authorization_type(&self) -> Authorization { match self { Self::Health => Authorization::None, - Self::CheckWebsiteEnabled => Authorization::None, + Self::CheckDomain => Authorization::None, Self::Metrics => Authorization::MetricsToken, _ => Authorization::AdminToken, } diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index f2098ab0..542b7a81 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -30,7 +30,7 @@ pub async fn handle_post_object( .get(header::CONTENT_TYPE) .and_then(|ct| ct.to_str().ok()) .and_then(|ct| multer::parse_boundary(ct).ok()) - .ok_or_bad_request("Counld not get multipart boundary")?; + .ok_or_bad_request("Could not get multipart boundary")?; // 16k seems plenty for a header. 5G is the max size of a single part, so it seems reasonable // for a PostObject @@ -64,15 +64,13 @@ pub async fn handle_post_object( "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */), "acl" => { if params.insert("x-amz-acl", content).is_some() { - return Err(Error::bad_request( - "Field 'acl' provided more than one time", - )); + return Err(Error::bad_request("Field 'acl' provided more than once")); } } _ => { if params.insert(&name, content).is_some() { return Err(Error::bad_request(format!( - "Field '{}' provided more than one time", + "Field '{}' provided more than once", name ))); } @@ -149,7 +147,7 @@ pub async fn handle_post_object( .ok_or_bad_request("Invalid expiration date")? .into(); if Utc::now() - expiration > Duration::zero() { - return Err(Error::bad_request("Expiration date is in the paste")); + return Err(Error::bad_request("Expiration date is in the past")); } let mut conditions = decoded_policy.into_conditions()?; @@ -330,7 +328,7 @@ impl Policy { if map.len() != 1 { return Err(Error::bad_request("Invalid policy item")); } - let (mut k, v) = map.into_iter().next().expect("size was verified"); + let (mut k, v) = map.into_iter().next().expect("Size could not be verified"); k.make_ascii_lowercase(); params.entry(k).or_default().push(Operation::Equal(v)); } diff --git a/src/block/Cargo.toml b/src/block/Cargo.toml index 1f8d738b..1057b699 100644 --- a/src/block/Cargo.toml +++ b/src/block/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_block" -version = "0.8.2" +version = "0.8.3" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" @@ -28,7 +28,7 @@ hex = "0.4" tracing = "0.1" rand = "0.8" -async-compression = { version = "0.3", features = ["tokio", "zstd"] } +async-compression = { version = "0.4", features = ["tokio", "zstd"] } zstd = { version = "0.12", default-features = false } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml index 7401c9cd..0327f403 100644 --- a/src/db/Cargo.toml +++ b/src/db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_db" -version = "0.8.2" +version = "0.8.3" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" @@ -22,19 +22,19 @@ hexdump = "0.1" tracing = "0.1" heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true } -rusqlite = { version = "0.28", optional = true } +rusqlite = { version = "0.29", optional = true } sled = { version = "0.34", optional = true } # cli deps clap = { version = "4.1", optional = true, features = ["derive", "env"] } -pretty_env_logger = { version = "0.4", optional = true } +pretty_env_logger = { version = "0.5", optional = true } [dev-dependencies] mktemp = "0.5" [features] default = [ "sled", "lmdb", "sqlite" ] -bundled-libs = [ "rusqlite/bundled" ] +bundled-libs = [ "rusqlite?/bundled" ] cli = ["clap", "pretty_env_logger"] lmdb = [ "heed" ] sqlite = [ "rusqlite" ] diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index f9fc206b..1b23285e 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.8.2" +version = "0.8.3" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" @@ -33,7 +33,7 @@ garage_web.workspace = true backtrace = "0.3" bytes = "1.0" -bytesize = "1.1" +bytesize = "1.2" timeago = { version = "0.4", default-features = false } parse_duration = "2.1" hex = "0.4" @@ -61,7 +61,8 @@ opentelemetry-otlp = { version = "0.10", optional = true } prometheus = { version = "0.13", optional = true } [dev-dependencies] -aws-sdk-s3 = "0.19" +aws-config = "0.55.2" +aws-sdk-s3 = "0.28" chrono = "0.4" http = "0.2" hmac = "0.12" diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index 05d2ea31..9ca4a059 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -227,7 +227,7 @@ pub struct WebsiteOpt { #[structopt(short = "i", long = "index-document", default_value = "index.html")] pub index_document: String, - /// Error document: the optionnal document returned when an error occurs + /// Error document: the optional document returned when an error occurs #[structopt(short = "e", long = "error-document")] pub error_document: Option<String>, } diff --git a/src/garage/tests/bucket.rs b/src/garage/tests/bucket.rs index 0dec3cfa..2dda7e6f 100644 --- a/src/garage/tests/bucket.rs +++ b/src/garage/tests/bucket.rs @@ -1,7 +1,6 @@ use crate::common; use crate::common::ext::CommandExt; -use aws_sdk_s3::model::BucketLocationConstraint; -use aws_sdk_s3::output::DeleteBucketOutput; +use aws_sdk_s3::operation::delete_bucket::DeleteBucketOutput; #[tokio::test] async fn test_bucket_all() { @@ -63,10 +62,7 @@ async fn test_bucket_all() { .await .unwrap(); - match r.location_constraint.unwrap() { - BucketLocationConstraint::Unknown(v) if v.as_str() == "garage-integ-test" => (), - _ => unreachable!("wrong region"), - } + assert_eq!(r.location_constraint.unwrap().as_str(), "garage-integ-test"); } { // (Stub) check GetVersioning diff --git a/src/garage/tests/common/client.rs b/src/garage/tests/common/client.rs index e9d4849a..ef4daa5d 100644 --- a/src/garage/tests/common/client.rs +++ b/src/garage/tests/common/client.rs @@ -1,15 +1,16 @@ -use aws_sdk_s3::{Client, Config, Credentials, Endpoint}; +use aws_sdk_s3::config::Credentials; +use aws_sdk_s3::{Client, Config}; -use super::garage::{Instance, Key}; +use super::garage::Key; +use crate::common::garage::DEFAULT_PORT; -pub fn build_client(instance: &Instance, key: &Key) -> Client { +pub fn build_client(key: &Key) -> Client { let credentials = Credentials::new(&key.id, &key.secret, None, None, "garage-integ-test"); - let endpoint = Endpoint::immutable(instance.s3_uri()); let config = Config::builder() + .endpoint_url(format!("http://127.0.0.1:{}", DEFAULT_PORT)) .region(super::REGION) .credentials_provider(credentials) - .endpoint_resolver(endpoint) .build(); Client::from_conf(config) diff --git a/src/garage/tests/common/mod.rs b/src/garage/tests/common/mod.rs index 0b8c6755..1273bad1 100644 --- a/src/garage/tests/common/mod.rs +++ b/src/garage/tests/common/mod.rs @@ -1,5 +1,7 @@ -use aws_sdk_s3::{Client, Region}; +use aws_sdk_s3::config::Region; +use aws_sdk_s3::Client; use ext::*; +#[cfg(feature = "k2v")] use k2v_client::K2vClient; #[macro_use] @@ -20,6 +22,7 @@ pub struct Context { pub key: garage::Key, pub client: Client, pub custom_request: CustomRequester, + #[cfg(feature = "k2v")] pub k2v: K2VContext, } @@ -32,8 +35,9 @@ impl Context { fn new() -> Self { let garage = garage::instance(); let key = garage.key(None); - let client = client::build_client(garage, &key); + let client = client::build_client(&key); let custom_request = CustomRequester::new_s3(garage, &key); + #[cfg(feature = "k2v")] let k2v_request = CustomRequester::new_k2v(garage, &key); Context { @@ -41,6 +45,7 @@ impl Context { client, key, custom_request, + #[cfg(feature = "k2v")] k2v: K2VContext { request: k2v_request, }, @@ -71,6 +76,7 @@ impl Context { } /// Build a K2vClient for a given bucket + #[cfg(feature = "k2v")] pub fn k2v_client(&self, bucket: &str) -> K2vClient { let config = k2v_client::K2vClientConfig { region: REGION.to_string(), diff --git a/src/garage/tests/k2v/batch.rs b/src/garage/tests/k2v/batch.rs index 595d0ba8..71de91bf 100644 --- a/src/garage/tests/k2v/batch.rs +++ b/src/garage/tests/k2v/batch.rs @@ -6,7 +6,7 @@ use assert_json_diff::assert_json_eq; use base64::prelude::*; use serde_json::json; -use super::json_body; +use crate::json_body; use hyper::{Method, StatusCode}; #[tokio::test] diff --git a/src/garage/tests/k2v/item.rs b/src/garage/tests/k2v/item.rs index 588836c7..25d9cce4 100644 --- a/src/garage/tests/k2v/item.rs +++ b/src/garage/tests/k2v/item.rs @@ -6,7 +6,7 @@ use assert_json_diff::assert_json_eq; use base64::prelude::*; use serde_json::json; -use super::json_body; +use crate::json_body; use hyper::{Method, StatusCode}; #[tokio::test] diff --git a/src/garage/tests/k2v/mod.rs b/src/garage/tests/k2v/mod.rs index a009460e..241e3dc2 100644 --- a/src/garage/tests/k2v/mod.rs +++ b/src/garage/tests/k2v/mod.rs @@ -3,16 +3,3 @@ pub mod errorcodes; pub mod item; pub mod poll; pub mod simple; - -use hyper::{Body, Response}; - -pub async fn json_body(res: Response<Body>) -> serde_json::Value { - let res_body: serde_json::Value = serde_json::from_slice( - &hyper::body::to_bytes(res.into_body()) - .await - .unwrap() - .to_vec()[..], - ) - .unwrap(); - res_body -} diff --git a/src/garage/tests/k2v/poll.rs b/src/garage/tests/k2v/poll.rs index dd44aed9..452317c2 100644 --- a/src/garage/tests/k2v/poll.rs +++ b/src/garage/tests/k2v/poll.rs @@ -5,8 +5,8 @@ use std::time::Duration; use assert_json_diff::assert_json_eq; use serde_json::json; -use super::json_body; use crate::common; +use crate::json_body; #[tokio::test] async fn test_poll_item() { diff --git a/src/garage/tests/lib.rs b/src/garage/tests/lib.rs index e450baac..ab92bc0a 100644 --- a/src/garage/tests/lib.rs +++ b/src/garage/tests/lib.rs @@ -10,3 +10,16 @@ mod s3; mod k2v; #[cfg(feature = "k2v")] mod k2v_client; + +use hyper::{Body, Response}; + +pub async fn json_body(res: Response<Body>) -> serde_json::Value { + let res_body: serde_json::Value = serde_json::from_slice( + &hyper::body::to_bytes(res.into_body()) + .await + .unwrap() + .to_vec()[..], + ) + .unwrap(); + res_body +} diff --git a/src/garage/tests/s3/multipart.rs b/src/garage/tests/s3/multipart.rs index 8ae6b66e..09ae5e5b 100644 --- a/src/garage/tests/s3/multipart.rs +++ b/src/garage/tests/s3/multipart.rs @@ -1,6 +1,6 @@ use crate::common; -use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart}; -use aws_sdk_s3::types::ByteStream; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart}; const SZ_5MB: usize = 5 * 1024 * 1024; const SZ_10MB: usize = 10 * 1024 * 1024; diff --git a/src/garage/tests/s3/objects.rs b/src/garage/tests/s3/objects.rs index 65f9e867..27697d45 100644 --- a/src/garage/tests/s3/objects.rs +++ b/src/garage/tests/s3/objects.rs @@ -1,6 +1,6 @@ use crate::common; -use aws_sdk_s3::model::{Delete, ObjectIdentifier}; -use aws_sdk_s3::types::ByteStream; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{Delete, ObjectIdentifier}; const STD_KEY: &str = "hello world"; const CTRL_KEY: &str = "\x00\x01\x02\x00"; diff --git a/src/garage/tests/s3/simple.rs b/src/garage/tests/s3/simple.rs index f54ae9ac..41ec44c6 100644 --- a/src/garage/tests/s3/simple.rs +++ b/src/garage/tests/s3/simple.rs @@ -2,7 +2,7 @@ use crate::common; #[tokio::test] async fn test_simple() { - use aws_sdk_s3::types::ByteStream; + use aws_sdk_s3::primitives::ByteStream; let ctx = common::context(); let bucket = ctx.create_bucket("test-simple"); diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs index f61838e4..eeafb5fa 100644 --- a/src/garage/tests/s3/website.rs +++ b/src/garage/tests/s3/website.rs @@ -1,11 +1,11 @@ use crate::common; use crate::common::ext::*; -use crate::k2v::json_body; +use crate::json_body; use assert_json_diff::assert_json_eq; use aws_sdk_s3::{ - model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, - types::ByteStream, + primitives::ByteStream, + types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, }; use http::{Request, StatusCode}; use hyper::{ @@ -72,7 +72,7 @@ async fn test_website() { res_body, json!({ "code": "InvalidRequest", - "message": "Bad request: Bucket 'my-website' is not authorized for website hosting", + "message": "Bad request: Domain 'my-website' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) @@ -91,24 +91,29 @@ async fn test_website() { BODY.as_ref() ); - let admin_req = || { - Request::builder() - .method("GET") - .uri(format!( - "http://127.0.0.1:{0}/check?domain={1}", - ctx.garage.admin_port, - BCKT_NAME.to_string() - )) - .body(Body::empty()) - .unwrap() - }; - - let mut admin_resp = client.request(admin_req()).await.unwrap(); - assert_eq!(admin_resp.status(), StatusCode::OK); - assert_eq!( - to_bytes(admin_resp.body_mut()).await.unwrap().as_ref(), - format!("Bucket '{BCKT_NAME}' is authorized for website hosting").as_bytes() - ); + for bname in [ + BCKT_NAME.to_string(), + format!("{BCKT_NAME}.web.garage"), + format!("{BCKT_NAME}.s3.garage"), + ] { + let admin_req = || { + Request::builder() + .method("GET") + .uri(format!( + "http://127.0.0.1:{0}/check?domain={1}", + ctx.garage.admin_port, bname + )) + .body(Body::empty()) + .unwrap() + }; + + let mut admin_resp = client.request(admin_req()).await.unwrap(); + assert_eq!(admin_resp.status(), StatusCode::OK); + assert_eq!( + to_bytes(admin_resp.body_mut()).await.unwrap().as_ref(), + format!("Domain '{bname}' is managed by Garage").as_bytes() + ); + } ctx.garage .command() @@ -142,7 +147,7 @@ async fn test_website() { res_body, json!({ "code": "InvalidRequest", - "message": "Bad request: Bucket 'my-website' is not authorized for website hosting", + "message": "Bad request: Domain 'my-website' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) @@ -397,7 +402,7 @@ async fn test_website_s3_api() { } #[tokio::test] -async fn test_website_check_website_enabled() { +async fn test_website_check_domain() { let ctx = common::context(); let client = Client::new(); @@ -435,13 +440,13 @@ async fn test_website_check_website_enabled() { }; let admin_resp = client.request(admin_req()).await.unwrap(); - assert_eq!(admin_resp.status(), StatusCode::NOT_FOUND); + assert_eq!(admin_resp.status(), StatusCode::BAD_REQUEST); let res_body = json_body(admin_resp).await; assert_json_eq!( res_body, json!({ - "code": "NoSuchBucket", - "message": "Bucket not found: ", + "code": "InvalidRequest", + "message": "Bad request: Domain '' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) @@ -459,13 +464,13 @@ async fn test_website_check_website_enabled() { }; let admin_resp = client.request(admin_req()).await.unwrap(); - assert_eq!(admin_resp.status(), StatusCode::NOT_FOUND); + assert_eq!(admin_resp.status(), StatusCode::BAD_REQUEST); let res_body = json_body(admin_resp).await; assert_json_eq!( res_body, json!({ - "code": "NoSuchBucket", - "message": "Bucket not found: foobar", + "code": "InvalidRequest", + "message": "Bad request: Domain 'foobar' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) @@ -483,13 +488,13 @@ async fn test_website_check_website_enabled() { }; let admin_resp = client.request(admin_req()).await.unwrap(); - assert_eq!(admin_resp.status(), StatusCode::NOT_FOUND); + assert_eq!(admin_resp.status(), StatusCode::BAD_REQUEST); let res_body = json_body(admin_resp).await; assert_json_eq!( res_body, json!({ - "code": "NoSuchBucket", - "message": "Bucket not found: ☹", + "code": "InvalidRequest", + "message": "Bad request: Domain '☹' is not managed by Garage", "region": "garage-integ-test", "path": "/check", }) diff --git a/src/k2v-client/bin/k2v-cli.rs b/src/k2v-client/bin/k2v-cli.rs index 5a2422ab..b9461c89 100644 --- a/src/k2v-client/bin/k2v-cli.rs +++ b/src/k2v-client/bin/k2v-cli.rs @@ -311,23 +311,19 @@ impl BatchOutputKind { .collect::<Vec<_>>() } - fn display_poll_range_output( - &self, - seen_marker: String, - values: BTreeMap<String, CausalValue>, - ) -> ! { + fn display_poll_range_output(&self, poll_range: PollRangeResult) -> ! { if self.json { let json = serde_json::json!({ - "values": self.values_json(values), - "seen_marker": seen_marker, + "values": self.values_json(poll_range.items), + "seen_marker": poll_range.seen_marker, }); let stdout = std::io::stdout(); serde_json::to_writer_pretty(stdout, &json).unwrap(); exit(0) } else { - println!("seen marker: {}", seen_marker); - self.display_human_output(values) + println!("seen marker: {}", poll_range.seen_marker); + self.display_human_output(poll_range.items) } } @@ -501,8 +497,8 @@ async fn main() -> Result<(), Error> { ) .await?; match res { - Some((items, seen_marker)) => { - output_kind.display_poll_range_output(seen_marker, items); + Some(poll_range_output) => { + output_kind.display_poll_range_output(poll_range_output); } None => { if output_kind.json { diff --git a/src/k2v-client/lib.rs b/src/k2v-client/lib.rs index 425c351f..4aa7a20a 100644 --- a/src/k2v-client/lib.rs +++ b/src/k2v-client/lib.rs @@ -182,7 +182,7 @@ impl K2vClient { filter: Option<PollRangeFilter<'_>>, seen_marker: Option<&str>, timeout: Option<Duration>, - ) -> Result<Option<(BTreeMap<String, CausalValue>, String)>, Error> { + ) -> Result<Option<PollRangeResult>, Error> { let timeout = timeout.unwrap_or(DEFAULT_POLL_TIMEOUT); let request = PollRangeRequest { @@ -217,7 +217,10 @@ impl K2vClient { }) .collect::<BTreeMap<_, _>>(); - Ok(Some((items, resp.seen_marker))) + Ok(Some(PollRangeResult { + items, + seen_marker: resp.seen_marker, + })) } /// Perform an InsertItem request, inserting a value for a single pk+sk. @@ -570,6 +573,7 @@ pub struct Filter<'a> { pub reverse: bool, } +/// Filter for a poll range operations. #[derive(Debug, Default, Clone, Serialize)] pub struct PollRangeFilter<'a> { pub start: Option<&'a str>, @@ -577,6 +581,15 @@ pub struct PollRangeFilter<'a> { pub prefix: Option<&'a str>, } +/// Response to a poll_range query +#[derive(Debug, Default, Clone, Serialize)] +pub struct PollRangeResult { + /// List of items that have changed since last PollRange call. + pub items: BTreeMap<String, CausalValue>, + /// opaque string representing items already seen for future PollRange calls. + pub seen_marker: String, +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] struct PollRangeRequest<'a> { diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index c4291c32..69f7eea4 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.8.2" +version = "0.8.3" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 69e81f38..0aeacb54 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_rpc" -version = "0.8.2" +version = "0.8.3" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" @@ -14,12 +14,13 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +garage_db.workspace = true garage_util.workspace = true arc-swap = "1.0" bytes = "1.0" bytesize = "1.1" -gethostname = "0.2" +gethostname = "0.4" hex = "0.4" tracing = "0.1" rand = "0.8" diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index d0776945..69908b6e 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "0.8.2" +version = "0.8.3" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 27ef4550..bf3560f6 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "0.8.2" +version = "0.8.3" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "AGPL-3.0" @@ -31,7 +31,7 @@ rand = "0.8" sha2 = "0.10" chrono = "0.4" -rmp-serde = "1.1" +rmp-serde = "1.1.2" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde_json = "1.0" toml = "0.6" diff --git a/src/util/migrate.rs b/src/util/migrate.rs index 5b708cc8..45147c74 100644 --- a/src/util/migrate.rs +++ b/src/util/migrate.rs @@ -27,7 +27,7 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { Self::Previous::decode(bytes).map(Self::migrate) } - /// Encode this type with optionnal version marker + /// Encode this type with optional version marker fn encode(&self) -> Result<Vec<u8>, rmp_serde::encode::Error> { let mut wr = Vec::with_capacity(128); wr.extend_from_slice(Self::VERSION_MARKER); diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 423d3829..354b5fb0 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_web" -version = "0.8.2" +version = "0.8.3" authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"] edition = "2018" license = "AGPL-3.0" diff --git a/src/web/web_server.rs b/src/web/web_server.rs index de63b842..287aef1a 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,4 +1,4 @@ -use std::{borrow::Cow, convert::Infallible, net::SocketAddr, sync::Arc}; +use std::{convert::Infallible, net::SocketAddr, sync::Arc}; use futures::future::Future; @@ -6,7 +6,7 @@ use hyper::{ header::{HeaderValue, HOST}, server::conn::AddrStream, service::{make_service_fn, service_fn}, - Body, Method, Request, Response, Server, + Body, Method, Request, Response, Server, StatusCode, }; use opentelemetry::{ @@ -28,6 +28,7 @@ use garage_api::s3::get::{handle_get, handle_head}; use garage_model::garage::Garage; use garage_table::*; +use garage_util::data::Uuid; use garage_util::error::Error as GarageError; use garage_util::forwarded_headers; use garage_util::metrics::{gen_trace_id, RecordDuration}; @@ -168,6 +169,17 @@ impl WebServer { } } + async fn check_key_exists(self: &Arc<Self>, bucket_id: Uuid, key: &str) -> Result<bool, Error> { + let exists = self + .garage + .object_table + .get(&bucket_id, &key.to_string()) + .await? + .map(|object| object.versions().iter().any(|v| v.is_data())) + .unwrap_or(false); + Ok(exists) + } + async fn serve_file(self: &Arc<Self>, req: &Request<Body>) -> Result<Response<Body>, Error> { // Get http authority string (eg. [::1]:3902 or garage.tld:80) let authority = req @@ -207,11 +219,11 @@ impl WebServer { // Get path let path = req.uri().path().to_string(); let index = &website_config.index_document; - let key = path_to_key(&path, index)?; + let (key, may_redirect) = path_to_keys(&path, index)?; debug!( - "Selected bucket: \"{}\" {:?}, selected key: \"{}\"", - bucket_name, bucket_id, key + "Selected bucket: \"{}\" {:?}, target key: \"{}\", may redirect to: {:?}", + bucket_name, bucket_id, key, may_redirect ); let ret_doc = match *req.method() { @@ -219,10 +231,23 @@ impl WebServer { Method::HEAD => handle_head(self.garage.clone(), req, bucket_id, &key, None).await, Method::GET => handle_get(self.garage.clone(), req, bucket_id, &key, None).await, _ => Err(ApiError::bad_request("HTTP method not supported")), - } - .map_err(Error::from); + }; + + // Try implicit redirect on error + let ret_doc_with_redir = match (&ret_doc, may_redirect) { + (Err(ApiError::NoSuchKey), ImplicitRedirect::To { key, url }) + if self.check_key_exists(bucket_id, key.as_str()).await? => + { + Ok(Response::builder() + .status(StatusCode::FOUND) + .header("Location", url) + .body(Body::empty()) + .unwrap()) + } + _ => ret_doc, + }; - match ret_doc { + match ret_doc_with_redir.map_err(Error::from) { Err(error) => { // For a HEAD or OPTIONS method, and for non-4xx errors, // we don't return the error document as content, @@ -308,30 +333,45 @@ fn error_to_res(e: Error) -> Response<Body> { http_error } +#[derive(Debug, PartialEq)] +enum ImplicitRedirect { + No, + To { key: String, url: String }, +} + /// Path to key /// /// Convert the provided path to the internal key /// When a path ends with "/", we append the index name to match traditional web server behavior /// which is also AWS S3 behavior. -fn path_to_key<'a>(path: &'a str, index: &str) -> Result<Cow<'a, str>, Error> { +/// +/// Check: https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html +fn path_to_keys<'a>(path: &'a str, index: &str) -> Result<(String, ImplicitRedirect), Error> { let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?; - if !path_utf8.starts_with('/') { - return Err(Error::BadRequest("Path must start with a / (slash)".into())); - } - - match path_utf8.chars().last() { - None => unreachable!(), - Some('/') => { - let mut key = String::with_capacity(path_utf8.len() + index.len()); - key.push_str(&path_utf8[1..]); - key.push_str(index); - Ok(key.into()) - } - Some(_) => match path_utf8 { - Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()), - Cow::Owned(pu8) => Ok(pu8[1..].to_string().into()), - }, + let base_key = match path_utf8.strip_prefix("/") { + Some(bk) => bk, + None => return Err(Error::BadRequest("Path must start with a / (slash)".into())), + }; + let is_bucket_root = base_key.len() == 0; + let is_trailing_slash = path_utf8.ends_with("/"); + + match (is_bucket_root, is_trailing_slash) { + // It is not possible to store something at the root of the bucket (ie. empty key), + // the only option is to fetch the index + (true, _) => Ok((index.to_string(), ImplicitRedirect::No)), + + // "If you create a folder structure in your bucket, you must have an index document at each level. In each folder, the index document must have the same name, for example, index.html. When a user specifies a URL that resembles a folder lookup, the presence or absence of a trailing slash determines the behavior of the website. For example, the following URL, with a trailing slash, returns the photos/index.html index document." + (false, true) => Ok((format!("{base_key}{index}"), ImplicitRedirect::No)), + + // "However, if you exclude the trailing slash from the preceding URL, Amazon S3 first looks for an object photos in the bucket. If the photos object is not found, it searches for an index document, photos/index.html. If that document is found, Amazon S3 returns a 302 Found message and points to the photos/ key. For subsequent requests to photos/, Amazon S3 returns photos/index.html. If the index document is not found, Amazon S3 returns an error." + (false, false) => Ok(( + base_key.to_string(), + ImplicitRedirect::To { + key: format!("{base_key}/{index}"), + url: format!("{path}/"), + }, + )), } } @@ -340,13 +380,37 @@ mod tests { use super::*; #[test] - fn path_to_key_test() -> Result<(), Error> { - assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "file .jpg"); - assert_eq!(path_to_key("/%20t/", "index.html")?, " t/index.html"); - assert_eq!(path_to_key("/", "index.html")?, "index.html"); - assert_eq!(path_to_key("/hello", "index.html")?, "hello"); - assert!(path_to_key("", "index.html").is_err()); - assert!(path_to_key("i/am/relative", "index.html").is_err()); + fn path_to_keys_test() -> Result<(), Error> { + assert_eq!( + path_to_keys("/file%20.jpg", "index.html")?, + ( + "file .jpg".to_string(), + ImplicitRedirect::To { + key: "file .jpg/index.html".to_string(), + url: "/file%20.jpg/".to_string() + } + ) + ); + assert_eq!( + path_to_keys("/%20t/", "index.html")?, + (" t/index.html".to_string(), ImplicitRedirect::No) + ); + assert_eq!( + path_to_keys("/", "index.html")?, + ("index.html".to_string(), ImplicitRedirect::No) + ); + assert_eq!( + path_to_keys("/hello", "index.html")?, + ( + "hello".to_string(), + ImplicitRedirect::To { + key: "hello/index.html".to_string(), + url: "/hello/".to_string() + } + ) + ); + assert!(path_to_keys("", "index.html").is_err()); + assert!(path_to_keys("i/am/relative", "index.html").is_err()); Ok(()) } } |