From 104e2ce0a25917dfaaab7e62042cc611fc05125a Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 31 Oct 2020 17:28:56 +0100 Subject: Add "web" configuration entry --- src/util/config.rs | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src') diff --git a/src/util/config.rs b/src/util/config.rs index b985114d..a78ef830 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -35,6 +35,8 @@ pub struct Config { pub rpc_tls: Option, pub s3_api: ApiConfig, + + pub s3_web: WebConfig, } #[derive(Deserialize, Debug, Clone)] @@ -50,6 +52,11 @@ pub struct ApiConfig { pub s3_region: String, } +#[derive(Deserialize, Debug, Clone)] +pub struct WebConfig { + pub website_bind_addr: SocketAddr, +} + fn default_max_concurrent_rpc_requests() -> usize { 12 } -- cgit v1.2.3 From cea871d944e36222f5fdabe3e907cb8cf86d26e8 Mon Sep 17 00:00:00 2001 From: Quentin Date: Mon, 2 Nov 2020 15:48:39 +0100 Subject: Skeleton to the new web API --- src/garage/server.rs | 6 ++++++ src/util/config.rs | 2 +- src/web/Cargo.toml | 43 +++++++++++++++++++++++++++++++++++++++++++ src/web/lib.rs | 5 +++++ src/web/web_server.rs | 37 +++++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 src/web/Cargo.toml create mode 100644 src/web/lib.rs create mode 100644 src/web/web_server.rs (limited to 'src') diff --git a/src/garage/server.rs b/src/garage/server.rs index 6caea5eb..8962a8da 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -9,6 +9,7 @@ use garage_util::config::*; use garage_util::error::Error; use garage_api::api_server; +use garage_web::web_server; use garage_model::garage::Garage; use garage_rpc::rpc_server::RpcServer; @@ -56,6 +57,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Initializing RPC and API servers..."); let run_rpc_server = Arc::new(rpc_server).run(wait_from(watch_cancel.clone())); let api_server = api_server::run_api_server(garage.clone(), wait_from(watch_cancel.clone())); + let web_server = web_server::run_web_server(garage.clone(), wait_from(watch_cancel.clone())); futures::try_join!( garage @@ -78,6 +80,10 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("API server exited"); rv }), + web_server.map(|rv| { + info!("Web server exited"); + rv + }), background.run().map(|rv| { info!("Background runner exited"); Ok(rv) diff --git a/src/util/config.rs b/src/util/config.rs index a78ef830..a5fbe4b4 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -54,7 +54,7 @@ pub struct ApiConfig { #[derive(Deserialize, Debug, Clone)] pub struct WebConfig { - pub website_bind_addr: SocketAddr, + pub web_bind_addr: SocketAddr, } fn default_max_concurrent_rpc_requests() -> usize { diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml new file mode 100644 index 00000000..8b3743dc --- /dev/null +++ b/src/web/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "garage_web" +version = "0.1.0" +authors = ["Alex Auvolat ", "Quentin Dufour "] +edition = "2018" +license = "GPL-3.0" +description = "Utility crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_util = { version = "0.1", path = "../util" } +garage_table = { version = "0.1.1", path = "../table" } +garage_model = { version = "0.1.1", path = "../model" } + +rand = "0.7" +hex = "0.3" +sha2 = "0.8" +err-derive = "0.2.3" +log = "0.4" + +sled = "0.31" + +toml = "0.5" +rmp-serde = "0.14.3" +serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } +serde_json = "1.0" + +futures = "0.3" +futures-util = "0.3" +tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } + +http = "0.2" +hyper = "0.13" +rustls = "0.17" +webpki = "0.21" + +roxmltree = "0.11" + diff --git a/src/web/lib.rs b/src/web/lib.rs new file mode 100644 index 00000000..80957669 --- /dev/null +++ b/src/web/lib.rs @@ -0,0 +1,5 @@ +#[macro_use] +extern crate log; + +pub mod web_server; + diff --git a/src/web/web_server.rs b/src/web/web_server.rs new file mode 100644 index 00000000..cb81e433 --- /dev/null +++ b/src/web/web_server.rs @@ -0,0 +1,37 @@ +use std::sync::Arc; + +use futures::future::Future; + +use hyper::server::conn::AddrStream; +use hyper::{Body,Request,Response,Server}; +use hyper::service::{make_service_fn, service_fn}; + +use garage_util::error::Error; +use garage_model::garage::Garage; + +pub async fn run_web_server( + garage: Arc, + shutdown_signal: impl Future, +) -> Result<(), Error> { + let addr = &garage.config.s3_web.web_bind_addr; + + let service = make_service_fn(|conn: &AddrStream| { + let garage = garage.clone(); + let client_addr = conn.remote_addr(); + info!("{:?}", client_addr); + async move { + Ok::<_, Error>(service_fn(move |req: Request| { + let garage = garage.clone(); + //handler(garage, req, client_addr) + async move { Ok::, Error>(Response::new(Body::from("hello world\n"))) } + })) + } + }); + + let server = Server::bind(&addr).serve(service); + let graceful = server.with_graceful_shutdown(shutdown_signal); + info!("Web server listening on http://{}", addr); + + graceful.await?; + Ok(()) +} -- cgit v1.2.3 From b3caa3628dbe26c76494333472815c9b59a1104c Mon Sep 17 00:00:00 2001 From: Quentin Date: Mon, 2 Nov 2020 15:57:23 +0100 Subject: Fix description of the crate --- src/web/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 8b3743dc..796478ae 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" authors = ["Alex Auvolat ", "Quentin Dufour "] edition = "2018" license = "GPL-3.0" -description = "Utility crate for the Garage object store" +description = "S3-like website endpoint crate for the Garage object store" repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" [lib] -- cgit v1.2.3 From 0d3bc169ee66d937bf88a8b7ee284043ce4a9bcd Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 3 Nov 2020 12:37:16 +0100 Subject: It compiles! --- src/garage/Cargo.toml | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index cb16bcd4..39288f40 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -19,6 +19,7 @@ garage_rpc = { version = "0.1", path = "../rpc" } garage_table = { version = "0.1.1", path = "../table" } garage_model = { version = "0.1.1", path = "../model" } garage_api = { version = "0.1.1", path = "../api" } +garage_web = { version = "0.1", path = "../web" } bytes = "0.4" rand = "0.7" -- cgit v1.2.3 From 0791e7164e77147b785232adfe91ec01f5c6c6af Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 8 Nov 2020 15:47:25 +0100 Subject: Parse host header --- src/web/web_server.rs | 71 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index cb81e433..a615ec8f 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,9 +1,11 @@ use std::sync::Arc; +use std::net::SocketAddr; use futures::future::Future; use hyper::server::conn::AddrStream; -use hyper::{Body,Request,Response,Server}; +use hyper::{Body,Request,Response,Server,Uri}; +use hyper::header::HOST; use hyper::service::{make_service_fn, service_fn}; use garage_util::error::Error; @@ -22,8 +24,7 @@ pub async fn run_web_server( async move { Ok::<_, Error>(service_fn(move |req: Request| { let garage = garage.clone(); - //handler(garage, req, client_addr) - async move { Ok::, Error>(Response::new(Body::from("hello world\n"))) } + handler(garage, req, client_addr) })) } }); @@ -35,3 +36,67 @@ pub async fn run_web_server( graceful.await?; Ok(()) } + +async fn handler( + garage: Arc, + req: Request, + addr: SocketAddr, +) -> Result, Error> { + + // Get http authority string (eg. [::1]:3902) + let authority = req + .headers() + .get(HOST) + .ok_or(Error::BadRequest(format!("HOST header required")))? + .to_str()?; + info!("authority is {}", authority); + + // Get HTTP domain/ip from host + //let domain = host.to_socket_ + + + Ok(Response::new(Body::from("hello world\n"))) +} + +fn authority_to_host(authority: &str) -> Result { + let mut uri_str: String = "fake://".to_owned(); + uri_str.push_str(authority); + + match uri_str.parse::() { + Ok(uri) => { + let host = uri + .host() + .ok_or(Error::BadRequest(format!("Unable to extract host from authority as string")))?; + Ok(String::from(host)) + } + _ => Err(Error::BadRequest(format!("Unable to parse authority (host HTTP header)"))), + } +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn authority_to_host_with_port() -> Result<(), Error> { + let domain = authority_to_host("[::1]:3902")?; + assert_eq!(domain, "[::1]"); + let domain2 = authority_to_host("garage.tld:65200")?; + assert_eq!(domain2, "garage.tld"); + let domain3 = authority_to_host("127.0.0.1:80")?; + assert_eq!(domain3, "127.0.0.1"); + Ok(()) + } + + #[test] + fn authority_to_host_without_port() -> Result<(), Error> { + let domain = authority_to_host("[::1]")?; + assert_eq!(domain, "[::1]"); + let domain2 = authority_to_host("garage.tld")?; + assert_eq!(domain2, "garage.tld"); + let domain3 = authority_to_host("127.0.0.1")?; + assert_eq!(domain3, "127.0.0.1"); + Ok(()) + } +} -- cgit v1.2.3 From c78df603d7355563c9f726f97bf318273fc5bb83 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 8 Nov 2020 16:02:16 +0100 Subject: Add some documentation --- src/web/web_server.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index a615ec8f..ce1f7ee1 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -58,7 +58,15 @@ async fn handler( Ok(Response::new(Body::from("hello world\n"))) } +/// Extract host from the authority section given by the HTTP host header +/// +/// The HTTP host contains both a host and a port. +/// Extracting the port is more complex than just finding the colon (:) symbol. +/// An example of a case where it does not work: [::1]:3902 +/// Instead, we use the Uri module provided by Hyper that correctl parses this "authority" section fn authority_to_host(authority: &str) -> Result { + // Hyper can not directly parse authority section so we build a fake URL + // that contains our authority section let mut uri_str: String = "fake://".to_owned(); uri_str.push_str(authority); @@ -66,7 +74,7 @@ fn authority_to_host(authority: &str) -> Result { Ok(uri) => { let host = uri .host() - .ok_or(Error::BadRequest(format!("Unable to extract host from authority as string")))?; + .ok_or(Error::BadRequest(format!("Unable to extract host from authority")))?; Ok(String::from(host)) } _ => Err(Error::BadRequest(format!("Unable to parse authority (host HTTP header)"))), -- cgit v1.2.3 From 09137fd6b5ff30639addcac837bc1c6e6ff78fcf Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 8 Nov 2020 16:06:52 +0100 Subject: Log host --- src/web/web_server.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index ce1f7ee1..a712a5bd 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -52,8 +52,8 @@ async fn handler( info!("authority is {}", authority); // Get HTTP domain/ip from host - //let domain = host.to_socket_ - + let host = authority_to_host(authority)?; + info!("host is {}", host); Ok(Response::new(Body::from("hello world\n"))) } -- cgit v1.2.3 From 4093833ae854df16bc893a21617b0902a5beae47 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 09:57:07 +0100 Subject: Extract bucket --- src/util/config.rs | 3 ++- src/web/Cargo.toml | 1 + src/web/web_server.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 53 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/util/config.rs b/src/util/config.rs index a5fbe4b4..72f7c319 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -54,7 +54,8 @@ pub struct ApiConfig { #[derive(Deserialize, Debug, Clone)] pub struct WebConfig { - pub web_bind_addr: SocketAddr, + pub bind_addr: SocketAddr, + pub root_domain: String, } fn default_max_concurrent_rpc_requests() -> usize { diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 796478ae..8eddf047 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -36,6 +36,7 @@ tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt- http = "0.2" hyper = "0.13" +percent-encoding = "2.1.0" rustls = "0.17" webpki = "0.21" diff --git a/src/web/web_server.rs b/src/web/web_server.rs index a712a5bd..432d9752 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -15,7 +15,7 @@ pub async fn run_web_server( garage: Arc, shutdown_signal: impl Future, ) -> Result<(), Error> { - let addr = &garage.config.s3_web.web_bind_addr; + let addr = &garage.config.s3_web.bind_addr; let service = make_service_fn(|conn: &AddrStream| { let garage = garage.clone(); @@ -43,17 +43,23 @@ async fn handler( addr: SocketAddr, ) -> Result, Error> { - // Get http authority string (eg. [::1]:3902) + // Get http authority string (eg. [::1]:3902 or garage.tld:80) let authority = req .headers() .get(HOST) .ok_or(Error::BadRequest(format!("HOST header required")))? .to_str()?; - info!("authority is {}", authority); - // Get HTTP domain/ip from host + // Get bucket let host = authority_to_host(authority)?; - info!("host is {}", host); + let root = &garage.config.s3_web.root_domain; + let bucket = host_to_bucket(&host, root); + + // Get path + let path = req.uri().path().to_string(); + let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; + + info!("host: {}, bucket: {}, key: {}", host, bucket, key); Ok(Response::new(Body::from("hello world\n"))) } @@ -81,6 +87,18 @@ fn authority_to_host(authority: &str) -> Result { } } +fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { + if root.len() >= host.len() || !host.ends_with(root) { + return host; + } + + let len_diff = host.len() - root.len(); + let missing_starting_dot = root.chars().next() != Some('.'); + let cursor = if missing_starting_dot { len_diff - 1 } else { len_diff }; + &host[..cursor] +} + + #[cfg(test)] mod tests { @@ -107,4 +125,31 @@ mod tests { assert_eq!(domain3, "127.0.0.1"); Ok(()) } + + #[test] + fn host_to_bucket_test() { + assert_eq!( + host_to_bucket("john.doe.garage.tld", ".garage.tld"), + "john.doe"); + + assert_eq!( + host_to_bucket("john.doe.garage.tld", "garage.tld"), + "john.doe"); + + assert_eq!( + host_to_bucket("john.doe.com", "garage.tld"), + "john.doe.com"); + + assert_eq!( + host_to_bucket("john.doe.com", ".garage.tld"), + "john.doe.com"); + + assert_eq!( + host_to_bucket("garage.tld", "garage.tld"), + "garage.tld"); + + assert_eq!( + host_to_bucket("garage.tld", ".garage.tld"), + "garage.tld"); + } } -- cgit v1.2.3 From 27795a390ced369a5fda353c046cdd4b7ca98bd0 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 09:59:52 +0100 Subject: Fix formatting --- src/garage/server.rs | 2 +- src/web/lib.rs | 1 - src/web/web_server.rs | 70 +++++++++++++++++++++++++-------------------------- 3 files changed, 36 insertions(+), 37 deletions(-) (limited to 'src') diff --git a/src/garage/server.rs b/src/garage/server.rs index 8962a8da..ec78c067 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -9,9 +9,9 @@ use garage_util::config::*; use garage_util::error::Error; use garage_api::api_server; -use garage_web::web_server; use garage_model::garage::Garage; use garage_rpc::rpc_server::RpcServer; +use garage_web::web_server; use crate::admin_rpc::*; diff --git a/src/web/lib.rs b/src/web/lib.rs index 80957669..c0c668a1 100644 --- a/src/web/lib.rs +++ b/src/web/lib.rs @@ -2,4 +2,3 @@ extern crate log; pub mod web_server; - diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 432d9752..ca9b559a 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,15 +1,15 @@ -use std::sync::Arc; use std::net::SocketAddr; +use std::sync::Arc; use futures::future::Future; -use hyper::server::conn::AddrStream; -use hyper::{Body,Request,Response,Server,Uri}; use hyper::header::HOST; +use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; +use hyper::{Body, Request, Response, Server, Uri}; -use garage_util::error::Error; use garage_model::garage::Garage; +use garage_util::error::Error; pub async fn run_web_server( garage: Arc, @@ -21,7 +21,7 @@ pub async fn run_web_server( let garage = garage.clone(); let client_addr = conn.remote_addr(); info!("{:?}", client_addr); - async move { + async move { Ok::<_, Error>(service_fn(move |req: Request| { let garage = garage.clone(); handler(garage, req, client_addr) @@ -42,7 +42,6 @@ async fn handler( req: Request, addr: SocketAddr, ) -> Result, Error> { - // Get http authority string (eg. [::1]:3902 or garage.tld:80) let authority = req .headers() @@ -52,13 +51,13 @@ async fn handler( // Get bucket let host = authority_to_host(authority)?; - let root = &garage.config.s3_web.root_domain; - let bucket = host_to_bucket(&host, root); + let root = &garage.config.s3_web.root_domain; + let bucket = host_to_bucket(&host, root); // Get path let path = req.uri().path().to_string(); let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; - + info!("host: {}, bucket: {}, key: {}", host, bucket, key); Ok(Response::new(Body::from("hello world\n"))) @@ -78,12 +77,14 @@ fn authority_to_host(authority: &str) -> Result { match uri_str.parse::() { Ok(uri) => { - let host = uri - .host() - .ok_or(Error::BadRequest(format!("Unable to extract host from authority")))?; + let host = uri.host().ok_or(Error::BadRequest(format!( + "Unable to extract host from authority" + )))?; Ok(String::from(host)) } - _ => Err(Error::BadRequest(format!("Unable to parse authority (host HTTP header)"))), + _ => Err(Error::BadRequest(format!( + "Unable to parse authority (host HTTP header)" + ))), } } @@ -94,12 +95,14 @@ fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { let len_diff = host.len() - root.len(); let missing_starting_dot = root.chars().next() != Some('.'); - let cursor = if missing_starting_dot { len_diff - 1 } else { len_diff }; - &host[..cursor] + let cursor = if missing_starting_dot { + len_diff - 1 + } else { + len_diff + }; + &host[..cursor] } - - #[cfg(test)] mod tests { use super::*; @@ -128,28 +131,25 @@ mod tests { #[test] fn host_to_bucket_test() { - assert_eq!( + assert_eq!( host_to_bucket("john.doe.garage.tld", ".garage.tld"), - "john.doe"); + "john.doe" + ); - assert_eq!( - host_to_bucket("john.doe.garage.tld", "garage.tld"), - "john.doe"); - assert_eq!( - host_to_bucket("john.doe.com", "garage.tld"), - "john.doe.com"); - + host_to_bucket("john.doe.garage.tld", "garage.tld"), + "john.doe" + ); + + assert_eq!(host_to_bucket("john.doe.com", "garage.tld"), "john.doe.com"); + assert_eq!( host_to_bucket("john.doe.com", ".garage.tld"), - "john.doe.com"); - - assert_eq!( - host_to_bucket("garage.tld", "garage.tld"), - "garage.tld"); - - assert_eq!( - host_to_bucket("garage.tld", ".garage.tld"), - "garage.tld"); + "john.doe.com" + ); + + assert_eq!(host_to_bucket("garage.tld", "garage.tld"), "garage.tld"); + + assert_eq!(host_to_bucket("garage.tld", ".garage.tld"), "garage.tld"); } } -- cgit v1.2.3 From 1e52ee9f5b7532df79c16a9c6e71adbcceaed187 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:26:48 +0100 Subject: Rewrite authority to host while staying on stack --- src/web/web_server.rs | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index ca9b559a..f38d8fd2 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -6,7 +6,7 @@ use futures::future::Future; use hyper::header::HOST; use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Request, Response, Server, Uri}; +use hyper::{Body, Request, Response, Server}; use garage_model::garage::Garage; use garage_util::error::Error; @@ -58,7 +58,7 @@ async fn handler( let path = req.uri().path().to_string(); let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; - info!("host: {}, bucket: {}, key: {}", host, bucket, key); + info!("Selected bucket: {}, selected key: {}", bucket, key); Ok(Response::new(Body::from("hello world\n"))) } @@ -66,28 +66,29 @@ async fn handler( /// Extract host from the authority section given by the HTTP host header /// /// The HTTP host contains both a host and a port. -/// Extracting the port is more complex than just finding the colon (:) symbol. -/// An example of a case where it does not work: [::1]:3902 -/// Instead, we use the Uri module provided by Hyper that correctl parses this "authority" section -fn authority_to_host(authority: &str) -> Result { - // Hyper can not directly parse authority section so we build a fake URL - // that contains our authority section - let mut uri_str: String = "fake://".to_owned(); - uri_str.push_str(authority); - - match uri_str.parse::() { - Ok(uri) => { - let host = uri.host().ok_or(Error::BadRequest(format!( - "Unable to extract host from authority" - )))?; - Ok(String::from(host)) - } - _ => Err(Error::BadRequest(format!( - "Unable to parse authority (host HTTP header)" - ))), +/// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6 +/// we do not use the collect pattern as there is no way in std rust to collect over a stack allocated value +/// check here: https://docs.rs/collect_slice/1.2.0/collect_slice/ +fn authority_to_host(authority: &str) -> Result<&str, Error> { + let mut iter = authority.chars().enumerate(); + let split = match iter.next() { + Some((_, '[')) => { + let mut niter = iter.skip_while(|(_, c)| c != &']'); + niter.next().ok_or(Error::BadRequest(format!("Authority {} has an illegal format", authority)))?; + niter.next() + }, + Some((_, _)) => iter.skip_while(|(_, c)| c != &':').next(), + None => return Err(Error::BadRequest(format!("Authority is empty"))), + }; + + match split { + Some((i, ':')) => Ok(&authority[..i]), + None => Ok(authority), + Some((_, _)) => Err(Error::BadRequest(format!("Authority {} has an illegal format", authority))), } } + fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { if root.len() >= host.len() || !host.ends_with(root) { return host; -- cgit v1.2.3 From 8797eed0abdefac9a550b7ef55f60ba5899a17bf Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:32:04 +0100 Subject: Fixes due to integration tests --- src/web/web_server.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index f38d8fd2..2c6185a1 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -20,7 +20,6 @@ pub async fn run_web_server( let service = make_service_fn(|conn: &AddrStream| { let garage = garage.clone(); let client_addr = conn.remote_addr(); - info!("{:?}", client_addr); async move { Ok::<_, Error>(service_fn(move |req: Request| { let garage = garage.clone(); @@ -58,7 +57,7 @@ async fn handler( let path = req.uri().path().to_string(); let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; - info!("Selected bucket: {}, selected key: {}", bucket, key); + info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key); Ok(Response::new(Body::from("hello world\n"))) } -- cgit v1.2.3 From ab62c59acb49d1f3563d546eb6af13bf40739c2f Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:40:33 +0100 Subject: Fix indent again --- src/web/web_server.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 2c6185a1..cda7f52e 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -71,23 +71,28 @@ async fn handler( fn authority_to_host(authority: &str) -> Result<&str, Error> { let mut iter = authority.chars().enumerate(); let split = match iter.next() { - Some((_, '[')) => { + Some((_, '[')) => { let mut niter = iter.skip_while(|(_, c)| c != &']'); - niter.next().ok_or(Error::BadRequest(format!("Authority {} has an illegal format", authority)))?; + niter.next().ok_or(Error::BadRequest(format!( + "Authority {} has an illegal format", + authority + )))?; niter.next() - }, + } Some((_, _)) => iter.skip_while(|(_, c)| c != &':').next(), None => return Err(Error::BadRequest(format!("Authority is empty"))), }; - match split { + match split { Some((i, ':')) => Ok(&authority[..i]), None => Ok(authority), - Some((_, _)) => Err(Error::BadRequest(format!("Authority {} has an illegal format", authority))), + Some((_, _)) => Err(Error::BadRequest(format!( + "Authority {} has an illegal format", + authority + ))), } } - fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { if root.len() >= host.len() || !host.ends_with(root) { return host; -- cgit v1.2.3 From d1b2fcc1e7d54025625c62bff7ef8cb573fab456 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:48:40 +0100 Subject: Rewrite for clarity --- src/web/web_server.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index cda7f52e..e4d15872 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -66,21 +66,24 @@ async fn handler( /// /// The HTTP host contains both a host and a port. /// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6 -/// we do not use the collect pattern as there is no way in std rust to collect over a stack allocated value +/// We do not use the collect pattern as there is no way in std rust to collect over a stack allocated value /// check here: https://docs.rs/collect_slice/1.2.0/collect_slice/ fn authority_to_host(authority: &str) -> Result<&str, Error> { let mut iter = authority.chars().enumerate(); - let split = match iter.next() { - Some((_, '[')) => { - let mut niter = iter.skip_while(|(_, c)| c != &']'); - niter.next().ok_or(Error::BadRequest(format!( + let (_, first_char) = iter + .next() + .ok_or(Error::BadRequest(format!("Authority is empty")))?; + + let split = match first_char { + '[' => { + let mut iter = iter.skip_while(|(_, c)| c != &']'); + iter.next().ok_or(Error::BadRequest(format!( "Authority {} has an illegal format", authority )))?; - niter.next() + iter.next() } - Some((_, _)) => iter.skip_while(|(_, c)| c != &':').next(), - None => return Err(Error::BadRequest(format!("Authority is empty"))), + _ => iter.skip_while(|(_, c)| c != &':').next(), }; match split { -- cgit v1.2.3 From cacf8ddf2da9c80574647aeb0d61dd15f9f8c5d5 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:52:20 +0100 Subject: Panic when it is a logical error --- src/web/web_server.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index e4d15872..73aa6648 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -77,10 +77,7 @@ fn authority_to_host(authority: &str) -> Result<&str, Error> { let split = match first_char { '[' => { let mut iter = iter.skip_while(|(_, c)| c != &']'); - iter.next().ok_or(Error::BadRequest(format!( - "Authority {} has an illegal format", - authority - )))?; + iter.next().expect("Authority parsing logic error"); iter.next() } _ => iter.skip_while(|(_, c)| c != &':').next(), -- cgit v1.2.3 From 3cb3994cd2005231f8cc60ce02c55762a7b293f3 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 17:05:10 +0100 Subject: Add documentation to host_to_bucket --- src/web/web_server.rs | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 73aa6648..2440857d 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -93,6 +93,12 @@ fn authority_to_host(authority: &str) -> Result<&str, Error> { } } +/// Host to bucket +/// +/// Convert a host, like "bucket.garage-site.tld" or "john.doe.com" +/// to the corresponding bucket, resp. "bucket" and "john.doe.com" +/// considering that ".garage-site.tld" is the "root domain". +/// This behavior has been chosen to follow AWS S3 semantic. fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { if root.len() >= host.len() || !host.ends_with(root) { return host; -- cgit v1.2.3 From d445c4ef9cd6835ec7e2e543e9e462adcd0f58bf Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 11 Nov 2020 15:24:25 +0100 Subject: WIP fetch object --- src/web/web_server.rs | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 2440857d..cbb2aaac 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -57,6 +57,13 @@ async fn handler( let path = req.uri().path().to_string(); let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; + // Get bucket descriptor + let object = garage + .object_table + .get(&bucket.to_string(), &key.to_string()) + .await? + .ok_or(Error::NotFound)?; + info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key); Ok(Response::new(Body::from("hello world\n"))) -- cgit v1.2.3 From 2765291796de1b94401e462dc5136fdfce867596 Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 11 Nov 2020 19:48:01 +0100 Subject: Build path correctly --- src/util/config.rs | 1 + src/web/web_server.rs | 40 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/util/config.rs b/src/util/config.rs index 72f7c319..f4c841b7 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -56,6 +56,7 @@ pub struct ApiConfig { pub struct WebConfig { pub bind_addr: SocketAddr, pub root_domain: String, + pub index: String, } fn default_max_concurrent_rpc_requests() -> usize { diff --git a/src/web/web_server.rs b/src/web/web_server.rs index cbb2aaac..16b27cef 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::net::SocketAddr; use std::sync::Arc; @@ -55,17 +56,18 @@ async fn handler( // Get path let path = req.uri().path().to_string(); - let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; + let index = &garage.config.s3_web.index; + let key = path_to_key(&path, &index)?; - // Get bucket descriptor + info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key); + + // Get bucket descriptor let object = garage .object_table .get(&bucket.to_string(), &key.to_string()) .await? .ok_or(Error::NotFound)?; - info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key); - Ok(Response::new(Body::from("hello world\n"))) } @@ -121,6 +123,27 @@ fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { &host[..cursor] } +/// Path to key +/// +/// Convert the provided path to the internal key +/// When a path ends with "/", we append the index name to match traditional web server behavior +/// which is also AWS S3 behavior. +fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { + let path_utf8 = percent_encoding::percent_decode_str(&path).decode_utf8()?; + match path_utf8.chars().last() { + None => Err(Error::BadRequest(format!( + "Path must have at least a character" + ))), + Some('/') => { + let mut key = String::with_capacity(path_utf8.len() + index.len()); + key.push_str(&path_utf8); + key.push_str(index); + Ok(key.into()) + } + Some(_) => Ok(path_utf8.into()), + } +} + #[cfg(test)] mod tests { use super::*; @@ -170,4 +193,13 @@ mod tests { assert_eq!(host_to_bucket("garage.tld", ".garage.tld"), "garage.tld"); } + + #[test] + fn path_to_key_test() -> Result<(), Error> { + assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "/file .jpg"); + assert_eq!(path_to_key("/%20t/", "index.html")?, "/ t/index.html"); + assert_eq!(path_to_key("/", "index.html")?, "/index.html"); + assert!(path_to_key("", "index.html").is_err()); + Ok(()) + } } -- cgit v1.2.3 From 6076d869b14aa38059d54a2dece222ad7b9da3bc Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 11 Nov 2020 21:17:34 +0100 Subject: Build error --- src/web/web_server.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 16b27cef..3cc0fa43 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; @@ -24,7 +25,7 @@ pub async fn run_web_server( async move { Ok::<_, Error>(service_fn(move |req: Request| { let garage = garage.clone(); - handler(garage, req, client_addr) + handle_request(garage, req, client_addr) })) } }); @@ -37,11 +38,29 @@ pub async fn run_web_server( Ok(()) } -async fn handler( +async fn handle_request( garage: Arc, req: Request, addr: SocketAddr, -) -> Result, Error> { +) -> Result, Infallible> { + info!("{} {} {}", addr, req.method(), req.uri()); + let res = serve_file(garage, req).await; + match &res { + Ok(r) => debug!("{} {:?}", r.status(), r.headers()), + Err(e) => warn!("Response: error {}, {}", e.http_status_code(), e), + } + + Ok(res.unwrap_or_else(error_to_res)) +} + +fn error_to_res(e: Error) -> Response { + let body: Body = Body::from(format!("{}\n", e)); + let mut http_error = Response::new(body); + *http_error.status_mut() = e.http_status_code(); + http_error +} + +async fn serve_file(garage: Arc, req: Request) -> Result, Error> { // Get http authority string (eg. [::1]:3902 or garage.tld:80) let authority = req .headers() -- cgit v1.2.3 From 04f455ff7f673ae9449b374183b8aafb9347581f Mon Sep 17 00:00:00 2001 From: Quentin Date: Thu, 19 Nov 2020 14:56:00 +0100 Subject: Make it compile again --- src/web/error.rs | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/web/lib.rs | 2 ++ src/web/web_server.rs | 5 +++-- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 src/web/error.rs (limited to 'src') diff --git a/src/web/error.rs b/src/web/error.rs new file mode 100644 index 00000000..094b22d0 --- /dev/null +++ b/src/web/error.rs @@ -0,0 +1,51 @@ +use err_derive::Error; +use hyper::StatusCode; + +use garage_util::error::Error as GarageError; + +#[derive(Debug, Error)] +pub enum Error { + // Category: internal error + #[error(display = "Internal error: {}", _0)] + InternalError(#[error(source)] GarageError), + + #[error(display = "Internal error (Hyper error): {}", _0)] + Hyper(#[error(source)] hyper::Error), + + #[error(display = "Internal error (HTTP error): {}", _0)] + HTTP(#[error(source)] http::Error), + + // Category: cannot process + #[error(display = "Forbidden: {}", _0)] + Forbidden(String), + + #[error(display = "Not found")] + NotFound, + + // Category: bad request + #[error(display = "Invalid UTF-8: {}", _0)] + InvalidUTF8(#[error(source)] std::str::Utf8Error), + + #[error(display = "Invalid XML: {}", _0)] + InvalidXML(#[error(source)] roxmltree::Error), + + #[error(display = "Invalid header value: {}", _0)] + InvalidHeader(#[error(source)] hyper::header::ToStrError), + + #[error(display = "Bad request: {}", _0)] + BadRequest(String), +} + +impl Error { + pub fn http_status_code(&self) -> StatusCode { + match self { + Error::NotFound => StatusCode::NOT_FOUND, + Error::Forbidden(_) => StatusCode::FORBIDDEN, + Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE, + Error::InternalError(_) | Error::Hyper(_) | Error::HTTP(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } + _ => StatusCode::BAD_REQUEST, + } + } +} diff --git a/src/web/lib.rs b/src/web/lib.rs index c0c668a1..f28937b9 100644 --- a/src/web/lib.rs +++ b/src/web/lib.rs @@ -1,4 +1,6 @@ #[macro_use] extern crate log; +pub mod error; + pub mod web_server; diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 3cc0fa43..4771d209 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -11,12 +11,13 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response, Server}; use garage_model::garage::Garage; -use garage_util::error::Error; +use garage_util::error::Error as GarageError; +use crate::error::*; pub async fn run_web_server( garage: Arc, shutdown_signal: impl Future, -) -> Result<(), Error> { +) -> Result<(), GarageError> { let addr = &garage.config.s3_web.bind_addr; let service = make_service_fn(|conn: &AddrStream| { -- cgit v1.2.3 From 5b363626f4803b3e43cdb450fd6ee04ac9429c4d Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 20 Nov 2020 21:23:32 +0100 Subject: Support punnycode --- src/web/Cargo.toml | 2 +- src/web/web_server.rs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 8eddf047..252ee58d 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -41,4 +41,4 @@ rustls = "0.17" webpki = "0.21" roxmltree = "0.11" - +idna = "0.2" diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 4771d209..1c5619fa 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -10,6 +10,8 @@ use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response, Server}; +use idna::domain_to_unicode; + use garage_model::garage::Garage; use garage_util::error::Error as GarageError; use crate::error::*; @@ -70,7 +72,7 @@ async fn serve_file(garage: Arc, req: Request) -> Result Date: Sat, 21 Nov 2020 12:01:02 +0100 Subject: Fix host to key --- src/web/web_server.rs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 1c5619fa..7172f222 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -152,17 +152,29 @@ fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { /// which is also AWS S3 behavior. fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { let path_utf8 = percent_encoding::percent_decode_str(&path).decode_utf8()?; + + if path_utf8.chars().next() != Some('/') { + return Err(Error::BadRequest(format!( + "Path must start with a / (slash)" + ))) + } + match path_utf8.chars().last() { None => Err(Error::BadRequest(format!( "Path must have at least a character" ))), Some('/') => { let mut key = String::with_capacity(path_utf8.len() + index.len()); - key.push_str(&path_utf8); + key.push_str(&path_utf8[1..]); key.push_str(index); Ok(key.into()) } - Some(_) => Ok(path_utf8.into()), + Some(_) => { + match path_utf8 { + Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()), + Cow::Owned(pu8) => Ok((&pu8[1..]).to_string().into()), + } + } } } @@ -218,10 +230,12 @@ mod tests { #[test] fn path_to_key_test() -> Result<(), Error> { - assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "/file .jpg"); - assert_eq!(path_to_key("/%20t/", "index.html")?, "/ t/index.html"); - assert_eq!(path_to_key("/", "index.html")?, "/index.html"); + assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "file .jpg"); + assert_eq!(path_to_key("/%20t/", "index.html")?, " t/index.html"); + assert_eq!(path_to_key("/", "index.html")?, "index.html"); + assert_eq!(path_to_key("/hello", "index.html")?, "hello"); assert!(path_to_key("", "index.html").is_err()); + assert!(path_to_key("i/am/relative", "index.html").is_err()); Ok(()) } } -- cgit v1.2.3 From 0f33231ee6154a2b08f67f8107cc686ee9e9c678 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 21 Nov 2020 15:15:25 +0100 Subject: We are able to serve a file --- src/web/Cargo.toml | 2 + src/web/web_server.rs | 111 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 252ee58d..819b51c1 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -42,3 +42,5 @@ webpki = "0.21" roxmltree = "0.11" idna = "0.2" + +httpdate = "0.3" diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 7172f222..8a222738 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -2,17 +2,23 @@ use std::borrow::Cow; use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; +use std::time::{Duration, UNIX_EPOCH}; use futures::future::Future; +use futures::stream::*; -use hyper::header::HOST; -use hyper::server::conn::AddrStream; -use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Request, Response, Server}; +use hyper::{ + header::HOST, + body::Bytes, + server::conn::AddrStream, + service::{make_service_fn, service_fn}, + Body, Request, Response, Server, StatusCode}; use idna::domain_to_unicode; use garage_model::garage::Garage; +use garage_model::object_table::*; +use garage_table::EmptyKey; use garage_util::error::Error as GarageError; use crate::error::*; @@ -90,7 +96,102 @@ async fn serve_file(garage: Arc, req: Request) -> Result x, + _ => unreachable!(), + }; + + // Get metadata from version + let last_v_meta = match last_v_data { + ObjectVersionData::DeleteMarker => return Err(Error::NotFound), + ObjectVersionData::Inline(meta, _) => meta, + ObjectVersionData::FirstBlock(meta, _) => meta, + }; + + // @FIXME Support range + + + // Set headers + let resp_builder = object_headers(&last_v, last_v_meta).status(StatusCode::OK); + + + // Stream body + match &last_v_data { + ObjectVersionData::DeleteMarker => unreachable!(), + ObjectVersionData::Inline(_, bytes) => { + let body: Body = Body::from(bytes.to_vec()); + Ok(resp_builder.body(body)?) + } + ObjectVersionData::FirstBlock(_, first_block_hash) => { + let read_first_block = garage.block_manager.rpc_get_block(&first_block_hash); + let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey); + + let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?; + let version = version.ok_or(Error::NotFound)?; + + let mut blocks = version + .blocks() + .iter() + .map(|vb| (vb.hash, None)) + .collect::>(); + blocks[0].1 = Some(first_block); + + let body_stream = futures::stream::iter(blocks) + .map(move |(hash, data_opt)| { + let garage = garage.clone(); + async move { + if let Some(data) = data_opt { + Ok(Bytes::from(data)) + } else { + garage + .block_manager + .rpc_get_block(&hash) + .await + .map(Bytes::from) + } + } + }) + .buffered(2); + //let body: Body = Box::new(StreamBody::new(Box::pin(body_stream))); + let body = hyper::body::Body::wrap_stream(body_stream); + Ok(resp_builder.body(body)?) + } + } +} + +// Copied from api/s3_get.rs +fn object_headers( + version: &ObjectVersion, + version_meta: &ObjectVersionMeta, +) -> http::response::Builder { + let date = UNIX_EPOCH + Duration::from_millis(version.timestamp); + let date_str = httpdate::fmt_http_date(date); + + let mut resp = Response::builder() + .header( + "Content-Type", + version_meta.headers.content_type.to_string(), + ) + .header("Content-Length", format!("{}", version_meta.size)) + .header("ETag", version_meta.etag.to_string()) + .header("Last-Modified", date_str) + .header("Accept-Ranges", format!("bytes")); + + for (k, v) in version_meta.headers.other.iter() { + resp = resp.header(k, v.to_string()); + } + + resp } /// Extract host from the authority section given by the HTTP host header -- cgit v1.2.3 From a88fd49f71844f04013970a678201a65ab89fb19 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 21 Nov 2020 17:50:19 +0100 Subject: Use handle_get --- src/api/s3_put.rs | 2 +- src/model/block.rs | 2 +- src/model/bucket_table.rs | 1 - src/table/lib.rs | 2 +- src/table/schema.rs | 4 -- src/web/Cargo.toml | 1 + src/web/error.rs | 3 ++ src/web/web_server.rs | 134 +++++----------------------------------------- 8 files changed, 20 insertions(+), 129 deletions(-) (limited to 'src') diff --git a/src/api/s3_put.rs b/src/api/s3_put.rs index 72613323..a1681d77 100644 --- a/src/api/s3_put.rs +++ b/src/api/s3_put.rs @@ -322,7 +322,7 @@ pub async fn handle_put_part( let (object, first_block) = futures::try_join!(get_object_fut, get_first_block_fut)?; // Check object is valid and multipart block can be accepted - let first_block = first_block.ok_or(Error::BadRequest(format!("Empty body")))?; + let first_block = first_block.ok_or(Error::BadRequest(format!("Empty body")))?; let object = object.ok_or(Error::BadRequest(format!("Object not found")))?; if !object diff --git a/src/model/block.rs b/src/model/block.rs index 6a5d9c5b..8a513a3c 100644 --- a/src/model/block.rs +++ b/src/model/block.rs @@ -20,7 +20,7 @@ use garage_rpc::rpc_client::*; use garage_rpc::rpc_server::*; use garage_table::table_sharded::TableShardedReplication; -use garage_table::{TableReplication, DeletedFilter}; +use garage_table::{DeletedFilter, TableReplication}; use crate::block_ref_table::*; diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index 35c0cc27..11f853f9 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -104,7 +104,6 @@ impl Entry for Bucket { pub struct BucketTable; - #[async_trait] impl TableSchema for BucketTable { type P = EmptyKey; diff --git a/src/table/lib.rs b/src/table/lib.rs index 7684fe9d..a10f78c2 100644 --- a/src/table/lib.rs +++ b/src/table/lib.rs @@ -12,5 +12,5 @@ pub mod table_sharded; pub mod table_sync; pub use schema::*; -pub use util::*; pub use table::*; +pub use util::*; diff --git a/src/table/schema.rs b/src/table/schema.rs index 49cede0a..d2ec9450 100644 --- a/src/table/schema.rs +++ b/src/table/schema.rs @@ -20,7 +20,6 @@ impl PartitionKey for Hash { } } - pub trait SortKey { fn sort_key(&self) -> &[u8]; } @@ -37,7 +36,6 @@ impl SortKey for Hash { } } - pub trait Entry: PartialEq + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync { @@ -47,7 +45,6 @@ pub trait Entry: fn merge(&mut self, other: &Self); } - #[async_trait] pub trait TableSchema: Send + Sync { type P: PartitionKey + Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync; @@ -66,4 +63,3 @@ pub trait TableSchema: Send + Sync { true } } - diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 819b51c1..0d08fdbf 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -16,6 +16,7 @@ path = "lib.rs" garage_util = { version = "0.1", path = "../util" } garage_table = { version = "0.1.1", path = "../table" } garage_model = { version = "0.1.1", path = "../model" } +garage_api = { version = "0.1.1", path = "../api" } rand = "0.7" hex = "0.3" diff --git a/src/web/error.rs b/src/web/error.rs index 094b22d0..59810f0f 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -5,6 +5,9 @@ use garage_util::error::Error as GarageError; #[derive(Debug, Error)] pub enum Error { + #[error(display = "API error: {}", _0)] + ApiError(#[error(source)] garage_api::error::Error), + // Category: internal error #[error(display = "Internal error: {}", _0)] InternalError(#[error(source)] GarageError), diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 8a222738..4f79a9ec 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,26 +1,20 @@ -use std::borrow::Cow; -use std::convert::Infallible; -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::{Duration, UNIX_EPOCH}; +use std::{borrow::Cow, convert::Infallible, net::SocketAddr, sync::Arc}; use futures::future::Future; -use futures::stream::*; use hyper::{ header::HOST, - body::Bytes, server::conn::AddrStream, service::{make_service_fn, service_fn}, - Body, Request, Response, Server, StatusCode}; + Body, Request, Response, Server, +}; use idna::domain_to_unicode; +use crate::error::*; +use garage_api::s3_get::handle_get; use garage_model::garage::Garage; -use garage_model::object_table::*; -use garage_table::EmptyKey; use garage_util::error::Error as GarageError; -use crate::error::*; pub async fn run_web_server( garage: Arc, @@ -89,109 +83,9 @@ async fn serve_file(garage: Arc, req: Request) -> Result x, - _ => unreachable!(), - }; - - // Get metadata from version - let last_v_meta = match last_v_data { - ObjectVersionData::DeleteMarker => return Err(Error::NotFound), - ObjectVersionData::Inline(meta, _) => meta, - ObjectVersionData::FirstBlock(meta, _) => meta, - }; - - // @FIXME Support range - - - // Set headers - let resp_builder = object_headers(&last_v, last_v_meta).status(StatusCode::OK); - + let r = handle_get(garage, &req, bucket, &key).await?; - // Stream body - match &last_v_data { - ObjectVersionData::DeleteMarker => unreachable!(), - ObjectVersionData::Inline(_, bytes) => { - let body: Body = Body::from(bytes.to_vec()); - Ok(resp_builder.body(body)?) - } - ObjectVersionData::FirstBlock(_, first_block_hash) => { - let read_first_block = garage.block_manager.rpc_get_block(&first_block_hash); - let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey); - - let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?; - let version = version.ok_or(Error::NotFound)?; - - let mut blocks = version - .blocks() - .iter() - .map(|vb| (vb.hash, None)) - .collect::>(); - blocks[0].1 = Some(first_block); - - let body_stream = futures::stream::iter(blocks) - .map(move |(hash, data_opt)| { - let garage = garage.clone(); - async move { - if let Some(data) = data_opt { - Ok(Bytes::from(data)) - } else { - garage - .block_manager - .rpc_get_block(&hash) - .await - .map(Bytes::from) - } - } - }) - .buffered(2); - //let body: Body = Box::new(StreamBody::new(Box::pin(body_stream))); - let body = hyper::body::Body::wrap_stream(body_stream); - Ok(resp_builder.body(body)?) - } - } -} - -// Copied from api/s3_get.rs -fn object_headers( - version: &ObjectVersion, - version_meta: &ObjectVersionMeta, -) -> http::response::Builder { - let date = UNIX_EPOCH + Duration::from_millis(version.timestamp); - let date_str = httpdate::fmt_http_date(date); - - let mut resp = Response::builder() - .header( - "Content-Type", - version_meta.headers.content_type.to_string(), - ) - .header("Content-Length", format!("{}", version_meta.size)) - .header("ETag", version_meta.etag.to_string()) - .header("Last-Modified", date_str) - .header("Accept-Ranges", format!("bytes")); - - for (k, v) in version_meta.headers.other.iter() { - resp = resp.header(k, v.to_string()); - } - - resp + Ok(r) } /// Extract host from the authority section given by the HTTP host header @@ -253,11 +147,11 @@ fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { /// which is also AWS S3 behavior. fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { let path_utf8 = percent_encoding::percent_decode_str(&path).decode_utf8()?; - + if path_utf8.chars().next() != Some('/') { return Err(Error::BadRequest(format!( "Path must start with a / (slash)" - ))) + ))); } match path_utf8.chars().last() { @@ -270,12 +164,10 @@ fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { key.push_str(index); Ok(key.into()) } - Some(_) => { - match path_utf8 { - Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()), - Cow::Owned(pu8) => Ok((&pu8[1..]).to_string().into()), - } - } + Some(_) => match path_utf8 { + Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()), + Cow::Owned(pu8) => Ok((&pu8[1..]).to_string().into()), + }, } } -- cgit v1.2.3 From b7a377308bbcbb7285a5b11cdcb07361eff93a28 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 21 Nov 2020 17:58:14 +0100 Subject: Handle HEAD --- src/web/web_server.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 4f79a9ec..f8a5cd14 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -6,13 +6,13 @@ use hyper::{ header::HOST, server::conn::AddrStream, service::{make_service_fn, service_fn}, - Body, Request, Response, Server, + Body, Method, Request, Response, Server, }; use idna::domain_to_unicode; use crate::error::*; -use garage_api::s3_get::handle_get; +use garage_api::s3_get::{handle_get, handle_head}; use garage_model::garage::Garage; use garage_util::error::Error as GarageError; @@ -83,9 +83,13 @@ async fn serve_file(garage: Arc, req: Request) -> Result handle_head(garage, &bucket, &key).await?, + &Method::GET => handle_get(garage, &req, bucket, &key).await?, + _ => return Err(Error::BadRequest(format!("HTTP method not supported"))), + }; - Ok(r) + Ok(res) } /// Extract host from the authority section given by the HTTP host header -- cgit v1.2.3 From fb18f5e17a34830d094fc591ee1d8accde2a85ad Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 21 Nov 2020 18:14:02 +0100 Subject: Fix wrong http status code --- src/web/error.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/web/error.rs b/src/web/error.rs index 59810f0f..220bacfe 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -43,6 +43,7 @@ impl Error { pub fn http_status_code(&self) -> StatusCode { match self { Error::NotFound => StatusCode::NOT_FOUND, + Error::ApiError(e) => e.http_status_code(), Error::Forbidden(_) => StatusCode::FORBIDDEN, Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE, Error::InternalError(_) | Error::Hyper(_) | Error::HTTP(_) => { -- cgit v1.2.3 From 51d0c14e440f00f24dbed6c3bce915a183a2bb65 Mon Sep 17 00:00:00 2001 From: Quentin Date: Thu, 10 Dec 2020 18:13:32 +0100 Subject: CLI structure --- src/garage/admin_rpc.rs | 3 +++ src/garage/main.rs | 15 +++++++++++++++ 2 files changed, 18 insertions(+) (limited to 'src') diff --git a/src/garage/admin_rpc.rs b/src/garage/admin_rpc.rs index a23d3e95..65bd24c0 100644 --- a/src/garage/admin_rpc.rs +++ b/src/garage/admin_rpc.rs @@ -155,6 +155,9 @@ impl AdminRpcHandler { &query.key_id, &query.bucket, allow_read, allow_write ))) } + BucketOperation::Website(query) => { + Ok(AdminRPC::Ok(format!("test"))) + } } } diff --git a/src/garage/main.rs b/src/garage/main.rs index 1185871f..7996d1f9 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -141,6 +141,21 @@ pub enum BucketOperation { /// Allow key to read or write to bucket #[structopt(name = "deny")] Deny(PermBucketOpt), + + /// Expose as website or not + #[structopt(name = "website")] + Website(WebsiteOpt), +} + +#[derive(Serialize, Deserialize, StructOpt, Debug)] +pub struct WebsiteOpt { + /// Create + #[structopt(long = "create")] + pub create: bool, + + /// Delete + #[structopt(long = "delete")] + pub delete: bool, } #[derive(Serialize, Deserialize, StructOpt, Debug)] -- cgit v1.2.3 From e1ce2b228aaacd5984bf4e1b73a0a6c1276f78e5 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 12 Dec 2020 17:00:31 +0100 Subject: WIP table migration --- src/garage/admin_rpc.rs | 33 ++++++++++++++++++++++++++++++++- src/garage/main.rs | 8 ++++---- src/model/bucket_table.rs | 28 ++++++++++++++++++++++------ 3 files changed, 58 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/garage/admin_rpc.rs b/src/garage/admin_rpc.rs index 65bd24c0..f9d398c2 100644 --- a/src/garage/admin_rpc.rs +++ b/src/garage/admin_rpc.rs @@ -156,7 +156,36 @@ impl AdminRpcHandler { ))) } BucketOperation::Website(query) => { - Ok(AdminRPC::Ok(format!("test"))) + let bucket = self.get_existing_bucket(&query.bucket).await?; + if query.allow && query.deny { + return Err(Error::Message(format!("Website can not be both allowed and denied on a bucket"))); + } + + if query.allow || query.deny { + let exposed = query.allow; + if let BucketState::Present(ak) = bucket.state.get_mut() { + let old_ak = ak.take_and_clear(); + ak.merge(&old_ak.update_mutator( + key_id.to_string(), + PermissionSet { + allow_read, + allow_write, + }, + )); + } else { + return Err(Error::Message(format!( + "Bucket is deleted in update_bucket_key" + ))); + } + } + + let msg = if bucket.exposed { + "Bucket is exposed as a website." + } else { + "Bucket is not exposed." + }; + + Ok(AdminRPC::Ok(msg)) } } } @@ -240,6 +269,7 @@ impl AdminRpcHandler { .unwrap_or(Err(Error::BadRPC(format!("Key {} does not exist", id)))) } + /// Update **bucket table** to inform of the new linked key async fn update_bucket_key( &self, mut bucket: Bucket, @@ -265,6 +295,7 @@ impl AdminRpcHandler { Ok(()) } + /// Update **key table** to inform of the new linked bucket async fn update_key_bucket( &self, mut key: Key, diff --git a/src/garage/main.rs b/src/garage/main.rs index 7996d1f9..d97ca3b8 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -150,12 +150,12 @@ pub enum BucketOperation { #[derive(Serialize, Deserialize, StructOpt, Debug)] pub struct WebsiteOpt { /// Create - #[structopt(long = "create")] - pub create: bool, + #[structopt(long = "allow")] + pub allow: bool, /// Delete - #[structopt(long = "delete")] - pub delete: bool, + #[structopt(long = "deny")] + pub deny: bool, } #[derive(Serialize, Deserialize, StructOpt, Debug)] diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index b7f24d71..b6b0fceb 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -21,27 +21,43 @@ pub struct Bucket { #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub enum BucketState { Deleted, - Present(crdt::LWWMap), + Present(BucketParams), } impl CRDT for BucketState { fn merge(&mut self, o: &Self) { match o { BucketState::Deleted => *self = BucketState::Deleted, - BucketState::Present(other_ak) => { - if let BucketState::Present(ak) = self { - ak.merge(other_ak); + BucketState::Present(other_params) => { + if let BucketState::Present(params) = self { + params.merge(other_params); } } } } } +#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] +pub struct BucketParams { + pub authorized_keys: crdt::LWWMap, + pub website: crdt::LWW +} + +impl CRDT for BucketParams { + fn merge(&mut self, o: &Self) { + self.authorized_keys.merge(&o.authorized_keys); + self.website.merge(&o.website); + } +} + impl Bucket { pub fn new(name: String) -> Self { Bucket { name, - state: crdt::LWW::new(BucketState::Present(crdt::LWWMap::new())), + state: crdt::LWW::new(BucketState::Present(BucketParams { + authorized_keys: crdt::LWWMap::new(), + website: crdt::LWW::new(false) + })), } } pub fn is_deleted(&self) -> bool { @@ -50,7 +66,7 @@ impl Bucket { pub fn authorized_keys(&self) -> &[(String, u64, PermissionSet)] { match self.state.get() { BucketState::Deleted => &[], - BucketState::Present(ak) => ak.items(), + BucketState::Present(state) => state.authorized_keys.items(), } } } -- cgit v1.2.3 From 96388acf230d48877a037b21931f61e2c63d2574 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 12 Dec 2020 21:35:29 +0100 Subject: Implement migration --- src/garage/admin_rpc.rs | 9 +++++---- src/model/bucket_table.rs | 13 ++++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/garage/admin_rpc.rs b/src/garage/admin_rpc.rs index f9d398c2..6ebd04a9 100644 --- a/src/garage/admin_rpc.rs +++ b/src/garage/admin_rpc.rs @@ -156,14 +156,15 @@ impl AdminRpcHandler { ))) } BucketOperation::Website(query) => { - let bucket = self.get_existing_bucket(&query.bucket).await?; + /*let bucket = self.get_existing_bucket(&query.bucket).await?; if query.allow && query.deny { return Err(Error::Message(format!("Website can not be both allowed and denied on a bucket"))); } if query.allow || query.deny { let exposed = query.allow; - if let BucketState::Present(ak) = bucket.state.get_mut() { + if let BucketState::Present(state) = bucket.state.get_mut() { + let ak = state.authorized_keys; let old_ak = ak.take_and_clear(); ak.merge(&old_ak.update_mutator( key_id.to_string(), @@ -183,9 +184,9 @@ impl AdminRpcHandler { "Bucket is exposed as a website." } else { "Bucket is not exposed." - }; + };*/ - Ok(AdminRPC::Ok(msg)) + Ok(AdminRPC::Ok(/*msg*/"".to_string())) } } } diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index b6b0fceb..08d0d168 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -10,6 +10,11 @@ use crate::key_table::PermissionSet; use model010::bucket_table as prev; +/// A bucket is a collection of objects +/// +/// Its parameters are not directly accessible as: +/// - It must be possible to merge paramaters, hence the use of a LWW CRDT. +/// - A bucket has 2 states, Present or Deleted and parameters make sense only if present. #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Bucket { // Primary key @@ -123,9 +128,15 @@ impl TableSchema for BucketTable { }, )); } + + let params = BucketParams { + authorized_keys: keys, + website: crdt::LWW::new(false) + }; + Some(Bucket { name: old.name, - state: crdt::LWW::migrate_from_raw(old.timestamp, BucketState::Present(keys)), + state: crdt::LWW::migrate_from_raw(old.timestamp, BucketState::Present(params)), }) } } -- cgit v1.2.3 From d0eb6a457f6f83c8f49d262adf6150964a1249b2 Mon Sep 17 00:00:00 2001 From: Quentin Date: Mon, 14 Dec 2020 21:46:49 +0100 Subject: Migrate RPC to new schema --- src/garage/admin_rpc.rs | 6 ++++-- src/model/bucket_table.rs | 14 ++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/garage/admin_rpc.rs b/src/garage/admin_rpc.rs index 6ebd04a9..8bf1dec2 100644 --- a/src/garage/admin_rpc.rs +++ b/src/garage/admin_rpc.rs @@ -89,7 +89,7 @@ impl AdminRpcHandler { } bucket .state - .update(BucketState::Present(crdt::LWWMap::new())); + .update(BucketState::Present(BucketParams::new())); bucket } None => Bucket::new(query.name.clone()), @@ -157,6 +157,7 @@ impl AdminRpcHandler { } BucketOperation::Website(query) => { /*let bucket = self.get_existing_bucket(&query.bucket).await?; + if query.allow && query.deny { return Err(Error::Message(format!("Website can not be both allowed and denied on a bucket"))); } @@ -278,7 +279,8 @@ impl AdminRpcHandler { allow_read: bool, allow_write: bool, ) -> Result<(), Error> { - if let BucketState::Present(ak) = bucket.state.get_mut() { + if let BucketState::Present(params) = bucket.state.get_mut() { + let ak = &mut params.authorized_keys; let old_ak = ak.take_and_clear(); ak.merge(&old_ak.update_mutator( key_id.to_string(), diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index 08d0d168..609490cb 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -55,14 +55,20 @@ impl CRDT for BucketParams { } } +impl BucketParams { + pub fn new() -> Self { + BucketParams { + authorized_keys: crdt::LWWMap::new(), + website: crdt::LWW::new(false) + } + } +} + impl Bucket { pub fn new(name: String) -> Self { Bucket { name, - state: crdt::LWW::new(BucketState::Present(BucketParams { - authorized_keys: crdt::LWWMap::new(), - website: crdt::LWW::new(false) - })), + state: crdt::LWW::new(BucketState::Present(BucketParams::new())), } } pub fn is_deleted(&self) -> bool { -- cgit v1.2.3 From a3566e49da406db9499a58a754af725a54d332af Mon Sep 17 00:00:00 2001 From: Quentin Date: Mon, 14 Dec 2020 21:50:40 +0100 Subject: Start to implement Website CLI --- src/garage/admin_rpc.rs | 5 ++--- src/garage/main.rs | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/garage/admin_rpc.rs b/src/garage/admin_rpc.rs index 8bf1dec2..1e363677 100644 --- a/src/garage/admin_rpc.rs +++ b/src/garage/admin_rpc.rs @@ -156,12 +156,11 @@ impl AdminRpcHandler { ))) } BucketOperation::Website(query) => { - /*let bucket = self.get_existing_bucket(&query.bucket).await?; - + let bucket = self.get_existing_bucket(&query.bucket).await?; if query.allow && query.deny { return Err(Error::Message(format!("Website can not be both allowed and denied on a bucket"))); } - + /* if query.allow || query.deny { let exposed = query.allow; if let BucketState::Present(state) = bucket.state.get_mut() { diff --git a/src/garage/main.rs b/src/garage/main.rs index d97ca3b8..7c8899a0 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -156,6 +156,9 @@ pub struct WebsiteOpt { /// Delete #[structopt(long = "deny")] pub deny: bool, + + /// Bucket name + pub bucket: String, } #[derive(Serialize, Deserialize, StructOpt, Debug)] -- cgit v1.2.3 From 3bc4d57a0f4a600c788a3c7ff51d633d1b7e6f09 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 15 Dec 2020 12:48:24 +0100 Subject: First implementation of the CLI --- src/garage/admin_rpc.rs | 47 +++++++++++++++++++---------------------------- src/model/bucket_table.rs | 8 ++++---- 2 files changed, 23 insertions(+), 32 deletions(-) (limited to 'src') diff --git a/src/garage/admin_rpc.rs b/src/garage/admin_rpc.rs index 1e363677..602e0b09 100644 --- a/src/garage/admin_rpc.rs +++ b/src/garage/admin_rpc.rs @@ -156,37 +156,28 @@ impl AdminRpcHandler { ))) } BucketOperation::Website(query) => { - let bucket = self.get_existing_bucket(&query.bucket).await?; - if query.allow && query.deny { - return Err(Error::Message(format!("Website can not be both allowed and denied on a bucket"))); + let mut bucket = self.get_existing_bucket(&query.bucket).await?; + + if !(query.allow ^ query.deny) { + return Err(Error::Message(format!( + "You must specify exactly one flag, either --allow or --deny" + ))); } - /* - if query.allow || query.deny { - let exposed = query.allow; - if let BucketState::Present(state) = bucket.state.get_mut() { - let ak = state.authorized_keys; - let old_ak = ak.take_and_clear(); - ak.merge(&old_ak.update_mutator( - key_id.to_string(), - PermissionSet { - allow_read, - allow_write, - }, - )); + + if let BucketState::Present(state) = bucket.state.get_mut() { + state.website.update(query.allow); + let msg = if query.allow { + format!("Website access allowed for {}", &query.bucket) } else { - return Err(Error::Message(format!( - "Bucket is deleted in update_bucket_key" - ))); - } - } + format!("Website access denied for {}", &query.bucket) + }; - let msg = if bucket.exposed { - "Bucket is exposed as a website." + Ok(AdminRPC::Ok(msg.to_string())) } else { - "Bucket is not exposed." - };*/ - - Ok(AdminRPC::Ok(/*msg*/"".to_string())) + return Err(Error::Message(format!( + "Bucket is deleted in update_bucket_key" + ))); + } } } } @@ -270,7 +261,7 @@ impl AdminRpcHandler { .unwrap_or(Err(Error::BadRPC(format!("Key {} does not exist", id)))) } - /// Update **bucket table** to inform of the new linked key + /// Update **bucket table** to inform of the new linked key async fn update_bucket_key( &self, mut bucket: Bucket, diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index 609490cb..78b0416f 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -45,7 +45,7 @@ impl CRDT for BucketState { #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct BucketParams { pub authorized_keys: crdt::LWWMap, - pub website: crdt::LWW + pub website: crdt::LWW, } impl CRDT for BucketParams { @@ -59,7 +59,7 @@ impl BucketParams { pub fn new() -> Self { BucketParams { authorized_keys: crdt::LWWMap::new(), - website: crdt::LWW::new(false) + website: crdt::LWW::new(false), } } } @@ -134,10 +134,10 @@ impl TableSchema for BucketTable { }, )); } - + let params = BucketParams { authorized_keys: keys, - website: crdt::LWW::new(false) + website: crdt::LWW::new(false), }; Some(Bucket { -- cgit v1.2.3 From 011ff87b5fd7cd1eea8713c7e21fcd827b69c149 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 15 Dec 2020 13:23:22 +0100 Subject: Push update --- src/garage/admin_rpc.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/garage/admin_rpc.rs b/src/garage/admin_rpc.rs index 602e0b09..39dc0ed8 100644 --- a/src/garage/admin_rpc.rs +++ b/src/garage/admin_rpc.rs @@ -166,6 +166,7 @@ impl AdminRpcHandler { if let BucketState::Present(state) = bucket.state.get_mut() { state.website.update(query.allow); + self.garage.bucket_table.insert(&bucket).await?; let msg = if query.allow { format!("Website access allowed for {}", &query.bucket) } else { -- cgit v1.2.3 From 3132deca5808905ce3956b40a6175b7714e11819 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 17 Dec 2020 20:43:14 +0100 Subject: Web server access control --- src/web/web_server.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index f8a5cd14..9effa86c 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -13,6 +13,8 @@ use idna::domain_to_unicode; use crate::error::*; use garage_api::s3_get::{handle_get, handle_head}; +use garage_table::*; +use garage_model::bucket_table::*; use garage_model::garage::Garage; use garage_util::error::Error as GarageError; @@ -76,6 +78,20 @@ async fn serve_file(garage: Arc, req: Request) -> Result Err(Error::NotFound), + BucketState::Present(params) if !params.website.get() => Err(Error::NotFound), + _ => Ok(()), + }?; + // Get path let path = req.uri().path().to_string(); let index = &garage.config.s3_web.index; -- cgit v1.2.3 From 2f4378a9c42cc3a78b992905e980e8977e6c5e58 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 17 Dec 2020 22:51:44 +0100 Subject: Fix formatting --- src/web/web_server.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 9effa86c..25a7cd5f 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -13,9 +13,9 @@ use idna::domain_to_unicode; use crate::error::*; use garage_api::s3_get::{handle_get, handle_head}; -use garage_table::*; use garage_model::bucket_table::*; use garage_model::garage::Garage; +use garage_table::*; use garage_util::error::Error as GarageError; pub async fn run_web_server( @@ -78,19 +78,19 @@ async fn serve_file(garage: Arc, req: Request) -> Result Err(Error::NotFound), - BucketState::Present(params) if !params.website.get() => Err(Error::NotFound), - _ => Ok(()), - }?; + // Check bucket is exposed as a website + let bucket_desc = garage + .bucket_table + .get(&EmptyKey, &bucket.to_string()) + .await? + .filter(|b| !b.is_deleted()) + .ok_or(Error::NotFound)?; + + match bucket_desc.state.get() { + BucketState::Deleted => Err(Error::NotFound), + BucketState::Present(params) if !params.website.get() => Err(Error::NotFound), + _ => Ok(()), + }?; // Get path let path = req.uri().path().to_string(); -- cgit v1.2.3 From f496e41ef4298a579810046bcff794a24cdb7e07 Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 15:44:44 +0100 Subject: Replace an already done check by unreachable!() --- src/garage/admin_rpc.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src') diff --git a/src/garage/admin_rpc.rs b/src/garage/admin_rpc.rs index 39dc0ed8..e1981e3a 100644 --- a/src/garage/admin_rpc.rs +++ b/src/garage/admin_rpc.rs @@ -175,9 +175,7 @@ impl AdminRpcHandler { Ok(AdminRPC::Ok(msg.to_string())) } else { - return Err(Error::Message(format!( - "Bucket is deleted in update_bucket_key" - ))); + unreachable!(); } } } -- cgit v1.2.3 From c441a358cdc8447d7ff7c6ff8b066ae2d0d99409 Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 16:16:32 +0100 Subject: Remove unused dependencies --- src/web/Cargo.toml | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'src') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 0d08fdbf..751b9ace 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -18,30 +18,11 @@ garage_table = { version = "0.1.1", path = "../table" } garage_model = { version = "0.1.1", path = "../model" } garage_api = { version = "0.1.1", path = "../api" } -rand = "0.7" -hex = "0.3" -sha2 = "0.8" err-derive = "0.2.3" log = "0.4" - -sled = "0.31" - -toml = "0.5" -rmp-serde = "0.14.3" -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_json = "1.0" - futures = "0.3" -futures-util = "0.3" -tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } - http = "0.2" hyper = "0.13" percent-encoding = "2.1.0" -rustls = "0.17" -webpki = "0.21" - roxmltree = "0.11" idna = "0.2" - -httpdate = "0.3" -- cgit v1.2.3 From 11a79a95dd9b2f2e0dd2d9dc999abd24f4ee232b Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 16:24:27 +0100 Subject: Simplify Error file --- src/web/error.rs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) (limited to 'src') diff --git a/src/web/error.rs b/src/web/error.rs index 220bacfe..14bc3b75 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -12,16 +12,6 @@ pub enum Error { #[error(display = "Internal error: {}", _0)] InternalError(#[error(source)] GarageError), - #[error(display = "Internal error (Hyper error): {}", _0)] - Hyper(#[error(source)] hyper::Error), - - #[error(display = "Internal error (HTTP error): {}", _0)] - HTTP(#[error(source)] http::Error), - - // Category: cannot process - #[error(display = "Forbidden: {}", _0)] - Forbidden(String), - #[error(display = "Not found")] NotFound, @@ -29,9 +19,6 @@ pub enum Error { #[error(display = "Invalid UTF-8: {}", _0)] InvalidUTF8(#[error(source)] std::str::Utf8Error), - #[error(display = "Invalid XML: {}", _0)] - InvalidXML(#[error(source)] roxmltree::Error), - #[error(display = "Invalid header value: {}", _0)] InvalidHeader(#[error(source)] hyper::header::ToStrError), @@ -44,11 +31,8 @@ impl Error { match self { Error::NotFound => StatusCode::NOT_FOUND, Error::ApiError(e) => e.http_status_code(), - Error::Forbidden(_) => StatusCode::FORBIDDEN, Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE, - Error::InternalError(_) | Error::Hyper(_) | Error::HTTP(_) => { - StatusCode::INTERNAL_SERVER_ERROR - } + Error::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR, _ => StatusCode::BAD_REQUEST, } } -- cgit v1.2.3 From 1e10c6a61cf59c11d72e941c5a6d4ea6e159b8ce Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 17:03:38 +0100 Subject: Doc tests that do not compile/work must be tagged with ignore --- src/table/crdt.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/table/crdt.rs b/src/table/crdt.rs index 386e478b..4cba10ce 100644 --- a/src/table/crdt.rs +++ b/src/table/crdt.rs @@ -239,7 +239,7 @@ where /// /// Typically, to update the value associated to a key in the map, you would do the following: /// - /// ``` + /// ```ignore /// let my_update = my_crdt.update_mutator(key_to_modify, new_value); /// my_crdt.merge(&my_update); /// ``` @@ -261,7 +261,7 @@ where /// empty map. This is very usefull to produce in-place a new map that contains only a delta /// that modifies a certain value: /// - /// ``` + /// ```ignore /// let mut a = get_my_crdt_value(); /// let old_a = a.take_and_clear(); /// a.merge(&old_a.update_mutator(key_to_modify, new_value)); @@ -273,7 +273,7 @@ where /// but in the case where the map is a field in a struct for instance (as is always the case), /// this becomes very handy: /// - /// ``` + /// ```ignore /// let mut a = get_my_crdt_value(); /// let old_a_map = a.map_field.take_and_clear(); /// a.map_field.merge(&old_a_map.update_mutator(key_to_modify, new_value)); -- cgit v1.2.3 From fad7bc405bd8b3cf1dc9a9319a7e3ee0e1eb3461 Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 17:03:54 +0100 Subject: Behavior problem: do not panic anymore + add tests --- src/web/web_server.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 25a7cd5f..246c045f 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -87,9 +87,8 @@ async fn serve_file(garage: Arc, req: Request) -> Result Err(Error::NotFound), - BucketState::Present(params) if !params.website.get() => Err(Error::NotFound), - _ => Ok(()), + BucketState::Present(params) if *params.website.get() => Ok(()), + _ => Err(Error::NotFound), }?; // Get path @@ -123,8 +122,10 @@ fn authority_to_host(authority: &str) -> Result<&str, Error> { let split = match first_char { '[' => { let mut iter = iter.skip_while(|(_, c)| c != &']'); - iter.next().expect("Authority parsing logic error"); - iter.next() + match iter.next() { + Some((_, ']')) => iter.next(), + _ => None, + } } _ => iter.skip_while(|(_, c)| c != &':').next(), }; @@ -214,6 +215,10 @@ mod tests { assert_eq!(domain2, "garage.tld"); let domain3 = authority_to_host("127.0.0.1")?; assert_eq!(domain3, "127.0.0.1"); + let domain4 = authority_to_host("[")?; + assert_eq!(domain4, "["); + let domain5 = authority_to_host("[hello")?; + assert_eq!(domain5, "[hello"); Ok(()) } -- cgit v1.2.3 From f8a40e8c4f69c20045aaffc4caf51158d697e792 Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 17:11:15 +0100 Subject: Explicitly set code path unreachable --- src/web/web_server.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 246c045f..aab7e8de 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -176,9 +176,7 @@ fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { } match path_utf8.chars().last() { - None => Err(Error::BadRequest(format!( - "Path must have at least a character" - ))), + None => unreachable!(), Some('/') => { let mut key = String::with_capacity(path_utf8.len() + index.len()); key.push_str(&path_utf8[1..]); -- cgit v1.2.3 From 851893a3f299da9eeb0ef3c745be1f30164fd6cf Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 15 Jan 2021 17:49:10 +0100 Subject: Do not accept domains such as [hello --- src/web/web_server.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index aab7e8de..24d111a9 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -124,7 +124,12 @@ fn authority_to_host(authority: &str) -> Result<&str, Error> { let mut iter = iter.skip_while(|(_, c)| c != &']'); match iter.next() { Some((_, ']')) => iter.next(), - _ => None, + _ => { + return Err(Error::BadRequest(format!( + "Authority {} has an illegal format", + authority + ))) + } } } _ => iter.skip_while(|(_, c)| c != &':').next(), @@ -213,10 +218,8 @@ mod tests { assert_eq!(domain2, "garage.tld"); let domain3 = authority_to_host("127.0.0.1")?; assert_eq!(domain3, "127.0.0.1"); - let domain4 = authority_to_host("[")?; - assert_eq!(domain4, "["); - let domain5 = authority_to_host("[hello")?; - assert_eq!(domain5, "[hello"); + assert!(authority_to_host("[").is_err()); + assert!(authority_to_host("[hello").is_err()); Ok(()) } -- cgit v1.2.3