From cea871d944e36222f5fdabe3e907cb8cf86d26e8 Mon Sep 17 00:00:00 2001 From: Quentin Date: Mon, 2 Nov 2020 15:48:39 +0100 Subject: Skeleton to the new web API --- src/web/Cargo.toml | 43 +++++++++++++++++++++++++++++++++++++++++++ src/web/lib.rs | 5 +++++ src/web/web_server.rs | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 85 insertions(+) create mode 100644 src/web/Cargo.toml create mode 100644 src/web/lib.rs create mode 100644 src/web/web_server.rs (limited to 'src/web') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml new file mode 100644 index 00000000..8b3743dc --- /dev/null +++ b/src/web/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "garage_web" +version = "0.1.0" +authors = ["Alex Auvolat ", "Quentin Dufour "] +edition = "2018" +license = "GPL-3.0" +description = "Utility crate for the Garage object store" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_util = { version = "0.1", path = "../util" } +garage_table = { version = "0.1.1", path = "../table" } +garage_model = { version = "0.1.1", path = "../model" } + +rand = "0.7" +hex = "0.3" +sha2 = "0.8" +err-derive = "0.2.3" +log = "0.4" + +sled = "0.31" + +toml = "0.5" +rmp-serde = "0.14.3" +serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } +serde_json = "1.0" + +futures = "0.3" +futures-util = "0.3" +tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } + +http = "0.2" +hyper = "0.13" +rustls = "0.17" +webpki = "0.21" + +roxmltree = "0.11" + diff --git a/src/web/lib.rs b/src/web/lib.rs new file mode 100644 index 00000000..80957669 --- /dev/null +++ b/src/web/lib.rs @@ -0,0 +1,5 @@ +#[macro_use] +extern crate log; + +pub mod web_server; + diff --git a/src/web/web_server.rs b/src/web/web_server.rs new file mode 100644 index 00000000..cb81e433 --- /dev/null +++ b/src/web/web_server.rs @@ -0,0 +1,37 @@ +use std::sync::Arc; + +use futures::future::Future; + +use hyper::server::conn::AddrStream; +use hyper::{Body,Request,Response,Server}; +use hyper::service::{make_service_fn, service_fn}; + +use garage_util::error::Error; +use garage_model::garage::Garage; + +pub async fn run_web_server( + garage: Arc, + shutdown_signal: impl Future, +) -> Result<(), Error> { + let addr = &garage.config.s3_web.web_bind_addr; + + let service = make_service_fn(|conn: &AddrStream| { + let garage = garage.clone(); + let client_addr = conn.remote_addr(); + info!("{:?}", client_addr); + async move { + Ok::<_, Error>(service_fn(move |req: Request| { + let garage = garage.clone(); + //handler(garage, req, client_addr) + async move { Ok::, Error>(Response::new(Body::from("hello world\n"))) } + })) + } + }); + + let server = Server::bind(&addr).serve(service); + let graceful = server.with_graceful_shutdown(shutdown_signal); + info!("Web server listening on http://{}", addr); + + graceful.await?; + Ok(()) +} -- cgit v1.2.3 From b3caa3628dbe26c76494333472815c9b59a1104c Mon Sep 17 00:00:00 2001 From: Quentin Date: Mon, 2 Nov 2020 15:57:23 +0100 Subject: Fix description of the crate --- src/web/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/web') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 8b3743dc..796478ae 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" authors = ["Alex Auvolat ", "Quentin Dufour "] edition = "2018" license = "GPL-3.0" -description = "Utility crate for the Garage object store" +description = "S3-like website endpoint crate for the Garage object store" repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" [lib] -- cgit v1.2.3 From 0791e7164e77147b785232adfe91ec01f5c6c6af Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 8 Nov 2020 15:47:25 +0100 Subject: Parse host header --- src/web/web_server.rs | 71 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 3 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index cb81e433..a615ec8f 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,9 +1,11 @@ use std::sync::Arc; +use std::net::SocketAddr; use futures::future::Future; use hyper::server::conn::AddrStream; -use hyper::{Body,Request,Response,Server}; +use hyper::{Body,Request,Response,Server,Uri}; +use hyper::header::HOST; use hyper::service::{make_service_fn, service_fn}; use garage_util::error::Error; @@ -22,8 +24,7 @@ pub async fn run_web_server( async move { Ok::<_, Error>(service_fn(move |req: Request| { let garage = garage.clone(); - //handler(garage, req, client_addr) - async move { Ok::, Error>(Response::new(Body::from("hello world\n"))) } + handler(garage, req, client_addr) })) } }); @@ -35,3 +36,67 @@ pub async fn run_web_server( graceful.await?; Ok(()) } + +async fn handler( + garage: Arc, + req: Request, + addr: SocketAddr, +) -> Result, Error> { + + // Get http authority string (eg. [::1]:3902) + let authority = req + .headers() + .get(HOST) + .ok_or(Error::BadRequest(format!("HOST header required")))? + .to_str()?; + info!("authority is {}", authority); + + // Get HTTP domain/ip from host + //let domain = host.to_socket_ + + + Ok(Response::new(Body::from("hello world\n"))) +} + +fn authority_to_host(authority: &str) -> Result { + let mut uri_str: String = "fake://".to_owned(); + uri_str.push_str(authority); + + match uri_str.parse::() { + Ok(uri) => { + let host = uri + .host() + .ok_or(Error::BadRequest(format!("Unable to extract host from authority as string")))?; + Ok(String::from(host)) + } + _ => Err(Error::BadRequest(format!("Unable to parse authority (host HTTP header)"))), + } +} + + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn authority_to_host_with_port() -> Result<(), Error> { + let domain = authority_to_host("[::1]:3902")?; + assert_eq!(domain, "[::1]"); + let domain2 = authority_to_host("garage.tld:65200")?; + assert_eq!(domain2, "garage.tld"); + let domain3 = authority_to_host("127.0.0.1:80")?; + assert_eq!(domain3, "127.0.0.1"); + Ok(()) + } + + #[test] + fn authority_to_host_without_port() -> Result<(), Error> { + let domain = authority_to_host("[::1]")?; + assert_eq!(domain, "[::1]"); + let domain2 = authority_to_host("garage.tld")?; + assert_eq!(domain2, "garage.tld"); + let domain3 = authority_to_host("127.0.0.1")?; + assert_eq!(domain3, "127.0.0.1"); + Ok(()) + } +} -- cgit v1.2.3 From c78df603d7355563c9f726f97bf318273fc5bb83 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 8 Nov 2020 16:02:16 +0100 Subject: Add some documentation --- src/web/web_server.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index a615ec8f..ce1f7ee1 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -58,7 +58,15 @@ async fn handler( Ok(Response::new(Body::from("hello world\n"))) } +/// Extract host from the authority section given by the HTTP host header +/// +/// The HTTP host contains both a host and a port. +/// Extracting the port is more complex than just finding the colon (:) symbol. +/// An example of a case where it does not work: [::1]:3902 +/// Instead, we use the Uri module provided by Hyper that correctl parses this "authority" section fn authority_to_host(authority: &str) -> Result { + // Hyper can not directly parse authority section so we build a fake URL + // that contains our authority section let mut uri_str: String = "fake://".to_owned(); uri_str.push_str(authority); @@ -66,7 +74,7 @@ fn authority_to_host(authority: &str) -> Result { Ok(uri) => { let host = uri .host() - .ok_or(Error::BadRequest(format!("Unable to extract host from authority as string")))?; + .ok_or(Error::BadRequest(format!("Unable to extract host from authority")))?; Ok(String::from(host)) } _ => Err(Error::BadRequest(format!("Unable to parse authority (host HTTP header)"))), -- cgit v1.2.3 From 09137fd6b5ff30639addcac837bc1c6e6ff78fcf Mon Sep 17 00:00:00 2001 From: Quentin Date: Sun, 8 Nov 2020 16:06:52 +0100 Subject: Log host --- src/web/web_server.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index ce1f7ee1..a712a5bd 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -52,8 +52,8 @@ async fn handler( info!("authority is {}", authority); // Get HTTP domain/ip from host - //let domain = host.to_socket_ - + let host = authority_to_host(authority)?; + info!("host is {}", host); Ok(Response::new(Body::from("hello world\n"))) } -- cgit v1.2.3 From 4093833ae854df16bc893a21617b0902a5beae47 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 09:57:07 +0100 Subject: Extract bucket --- src/web/Cargo.toml | 1 + src/web/web_server.rs | 55 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 5 deletions(-) (limited to 'src/web') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 796478ae..8eddf047 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -36,6 +36,7 @@ tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt- http = "0.2" hyper = "0.13" +percent-encoding = "2.1.0" rustls = "0.17" webpki = "0.21" diff --git a/src/web/web_server.rs b/src/web/web_server.rs index a712a5bd..432d9752 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -15,7 +15,7 @@ pub async fn run_web_server( garage: Arc, shutdown_signal: impl Future, ) -> Result<(), Error> { - let addr = &garage.config.s3_web.web_bind_addr; + let addr = &garage.config.s3_web.bind_addr; let service = make_service_fn(|conn: &AddrStream| { let garage = garage.clone(); @@ -43,17 +43,23 @@ async fn handler( addr: SocketAddr, ) -> Result, Error> { - // Get http authority string (eg. [::1]:3902) + // Get http authority string (eg. [::1]:3902 or garage.tld:80) let authority = req .headers() .get(HOST) .ok_or(Error::BadRequest(format!("HOST header required")))? .to_str()?; - info!("authority is {}", authority); - // Get HTTP domain/ip from host + // Get bucket let host = authority_to_host(authority)?; - info!("host is {}", host); + let root = &garage.config.s3_web.root_domain; + let bucket = host_to_bucket(&host, root); + + // Get path + let path = req.uri().path().to_string(); + let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; + + info!("host: {}, bucket: {}, key: {}", host, bucket, key); Ok(Response::new(Body::from("hello world\n"))) } @@ -81,6 +87,18 @@ fn authority_to_host(authority: &str) -> Result { } } +fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { + if root.len() >= host.len() || !host.ends_with(root) { + return host; + } + + let len_diff = host.len() - root.len(); + let missing_starting_dot = root.chars().next() != Some('.'); + let cursor = if missing_starting_dot { len_diff - 1 } else { len_diff }; + &host[..cursor] +} + + #[cfg(test)] mod tests { @@ -107,4 +125,31 @@ mod tests { assert_eq!(domain3, "127.0.0.1"); Ok(()) } + + #[test] + fn host_to_bucket_test() { + assert_eq!( + host_to_bucket("john.doe.garage.tld", ".garage.tld"), + "john.doe"); + + assert_eq!( + host_to_bucket("john.doe.garage.tld", "garage.tld"), + "john.doe"); + + assert_eq!( + host_to_bucket("john.doe.com", "garage.tld"), + "john.doe.com"); + + assert_eq!( + host_to_bucket("john.doe.com", ".garage.tld"), + "john.doe.com"); + + assert_eq!( + host_to_bucket("garage.tld", "garage.tld"), + "garage.tld"); + + assert_eq!( + host_to_bucket("garage.tld", ".garage.tld"), + "garage.tld"); + } } -- cgit v1.2.3 From 27795a390ced369a5fda353c046cdd4b7ca98bd0 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 09:59:52 +0100 Subject: Fix formatting --- src/web/lib.rs | 1 - src/web/web_server.rs | 70 +++++++++++++++++++++++++-------------------------- 2 files changed, 35 insertions(+), 36 deletions(-) (limited to 'src/web') diff --git a/src/web/lib.rs b/src/web/lib.rs index 80957669..c0c668a1 100644 --- a/src/web/lib.rs +++ b/src/web/lib.rs @@ -2,4 +2,3 @@ extern crate log; pub mod web_server; - diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 432d9752..ca9b559a 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,15 +1,15 @@ -use std::sync::Arc; use std::net::SocketAddr; +use std::sync::Arc; use futures::future::Future; -use hyper::server::conn::AddrStream; -use hyper::{Body,Request,Response,Server,Uri}; use hyper::header::HOST; +use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; +use hyper::{Body, Request, Response, Server, Uri}; -use garage_util::error::Error; use garage_model::garage::Garage; +use garage_util::error::Error; pub async fn run_web_server( garage: Arc, @@ -21,7 +21,7 @@ pub async fn run_web_server( let garage = garage.clone(); let client_addr = conn.remote_addr(); info!("{:?}", client_addr); - async move { + async move { Ok::<_, Error>(service_fn(move |req: Request| { let garage = garage.clone(); handler(garage, req, client_addr) @@ -42,7 +42,6 @@ async fn handler( req: Request, addr: SocketAddr, ) -> Result, Error> { - // Get http authority string (eg. [::1]:3902 or garage.tld:80) let authority = req .headers() @@ -52,13 +51,13 @@ async fn handler( // Get bucket let host = authority_to_host(authority)?; - let root = &garage.config.s3_web.root_domain; - let bucket = host_to_bucket(&host, root); + let root = &garage.config.s3_web.root_domain; + let bucket = host_to_bucket(&host, root); // Get path let path = req.uri().path().to_string(); let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; - + info!("host: {}, bucket: {}, key: {}", host, bucket, key); Ok(Response::new(Body::from("hello world\n"))) @@ -78,12 +77,14 @@ fn authority_to_host(authority: &str) -> Result { match uri_str.parse::() { Ok(uri) => { - let host = uri - .host() - .ok_or(Error::BadRequest(format!("Unable to extract host from authority")))?; + let host = uri.host().ok_or(Error::BadRequest(format!( + "Unable to extract host from authority" + )))?; Ok(String::from(host)) } - _ => Err(Error::BadRequest(format!("Unable to parse authority (host HTTP header)"))), + _ => Err(Error::BadRequest(format!( + "Unable to parse authority (host HTTP header)" + ))), } } @@ -94,12 +95,14 @@ fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { let len_diff = host.len() - root.len(); let missing_starting_dot = root.chars().next() != Some('.'); - let cursor = if missing_starting_dot { len_diff - 1 } else { len_diff }; - &host[..cursor] + let cursor = if missing_starting_dot { + len_diff - 1 + } else { + len_diff + }; + &host[..cursor] } - - #[cfg(test)] mod tests { use super::*; @@ -128,28 +131,25 @@ mod tests { #[test] fn host_to_bucket_test() { - assert_eq!( + assert_eq!( host_to_bucket("john.doe.garage.tld", ".garage.tld"), - "john.doe"); + "john.doe" + ); - assert_eq!( - host_to_bucket("john.doe.garage.tld", "garage.tld"), - "john.doe"); - assert_eq!( - host_to_bucket("john.doe.com", "garage.tld"), - "john.doe.com"); - + host_to_bucket("john.doe.garage.tld", "garage.tld"), + "john.doe" + ); + + assert_eq!(host_to_bucket("john.doe.com", "garage.tld"), "john.doe.com"); + assert_eq!( host_to_bucket("john.doe.com", ".garage.tld"), - "john.doe.com"); - - assert_eq!( - host_to_bucket("garage.tld", "garage.tld"), - "garage.tld"); - - assert_eq!( - host_to_bucket("garage.tld", ".garage.tld"), - "garage.tld"); + "john.doe.com" + ); + + assert_eq!(host_to_bucket("garage.tld", "garage.tld"), "garage.tld"); + + assert_eq!(host_to_bucket("garage.tld", ".garage.tld"), "garage.tld"); } } -- cgit v1.2.3 From 1e52ee9f5b7532df79c16a9c6e71adbcceaed187 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:26:48 +0100 Subject: Rewrite authority to host while staying on stack --- src/web/web_server.rs | 43 ++++++++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 21 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index ca9b559a..f38d8fd2 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -6,7 +6,7 @@ use futures::future::Future; use hyper::header::HOST; use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Request, Response, Server, Uri}; +use hyper::{Body, Request, Response, Server}; use garage_model::garage::Garage; use garage_util::error::Error; @@ -58,7 +58,7 @@ async fn handler( let path = req.uri().path().to_string(); let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; - info!("host: {}, bucket: {}, key: {}", host, bucket, key); + info!("Selected bucket: {}, selected key: {}", bucket, key); Ok(Response::new(Body::from("hello world\n"))) } @@ -66,28 +66,29 @@ async fn handler( /// Extract host from the authority section given by the HTTP host header /// /// The HTTP host contains both a host and a port. -/// Extracting the port is more complex than just finding the colon (:) symbol. -/// An example of a case where it does not work: [::1]:3902 -/// Instead, we use the Uri module provided by Hyper that correctl parses this "authority" section -fn authority_to_host(authority: &str) -> Result { - // Hyper can not directly parse authority section so we build a fake URL - // that contains our authority section - let mut uri_str: String = "fake://".to_owned(); - uri_str.push_str(authority); - - match uri_str.parse::() { - Ok(uri) => { - let host = uri.host().ok_or(Error::BadRequest(format!( - "Unable to extract host from authority" - )))?; - Ok(String::from(host)) - } - _ => Err(Error::BadRequest(format!( - "Unable to parse authority (host HTTP header)" - ))), +/// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6 +/// we do not use the collect pattern as there is no way in std rust to collect over a stack allocated value +/// check here: https://docs.rs/collect_slice/1.2.0/collect_slice/ +fn authority_to_host(authority: &str) -> Result<&str, Error> { + let mut iter = authority.chars().enumerate(); + let split = match iter.next() { + Some((_, '[')) => { + let mut niter = iter.skip_while(|(_, c)| c != &']'); + niter.next().ok_or(Error::BadRequest(format!("Authority {} has an illegal format", authority)))?; + niter.next() + }, + Some((_, _)) => iter.skip_while(|(_, c)| c != &':').next(), + None => return Err(Error::BadRequest(format!("Authority is empty"))), + }; + + match split { + Some((i, ':')) => Ok(&authority[..i]), + None => Ok(authority), + Some((_, _)) => Err(Error::BadRequest(format!("Authority {} has an illegal format", authority))), } } + fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { if root.len() >= host.len() || !host.ends_with(root) { return host; -- cgit v1.2.3 From 8797eed0abdefac9a550b7ef55f60ba5899a17bf Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:32:04 +0100 Subject: Fixes due to integration tests --- src/web/web_server.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index f38d8fd2..2c6185a1 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -20,7 +20,6 @@ pub async fn run_web_server( let service = make_service_fn(|conn: &AddrStream| { let garage = garage.clone(); let client_addr = conn.remote_addr(); - info!("{:?}", client_addr); async move { Ok::<_, Error>(service_fn(move |req: Request| { let garage = garage.clone(); @@ -58,7 +57,7 @@ async fn handler( let path = req.uri().path().to_string(); let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; - info!("Selected bucket: {}, selected key: {}", bucket, key); + info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key); Ok(Response::new(Body::from("hello world\n"))) } -- cgit v1.2.3 From ab62c59acb49d1f3563d546eb6af13bf40739c2f Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:40:33 +0100 Subject: Fix indent again --- src/web/web_server.rs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 2c6185a1..cda7f52e 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -71,23 +71,28 @@ async fn handler( fn authority_to_host(authority: &str) -> Result<&str, Error> { let mut iter = authority.chars().enumerate(); let split = match iter.next() { - Some((_, '[')) => { + Some((_, '[')) => { let mut niter = iter.skip_while(|(_, c)| c != &']'); - niter.next().ok_or(Error::BadRequest(format!("Authority {} has an illegal format", authority)))?; + niter.next().ok_or(Error::BadRequest(format!( + "Authority {} has an illegal format", + authority + )))?; niter.next() - }, + } Some((_, _)) => iter.skip_while(|(_, c)| c != &':').next(), None => return Err(Error::BadRequest(format!("Authority is empty"))), }; - match split { + match split { Some((i, ':')) => Ok(&authority[..i]), None => Ok(authority), - Some((_, _)) => Err(Error::BadRequest(format!("Authority {} has an illegal format", authority))), + Some((_, _)) => Err(Error::BadRequest(format!( + "Authority {} has an illegal format", + authority + ))), } } - fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { if root.len() >= host.len() || !host.ends_with(root) { return host; -- cgit v1.2.3 From d1b2fcc1e7d54025625c62bff7ef8cb573fab456 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:48:40 +0100 Subject: Rewrite for clarity --- src/web/web_server.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index cda7f52e..e4d15872 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -66,21 +66,24 @@ async fn handler( /// /// The HTTP host contains both a host and a port. /// Extracting the port is more complex than just finding the colon (:) symbol due to IPv6 -/// we do not use the collect pattern as there is no way in std rust to collect over a stack allocated value +/// We do not use the collect pattern as there is no way in std rust to collect over a stack allocated value /// check here: https://docs.rs/collect_slice/1.2.0/collect_slice/ fn authority_to_host(authority: &str) -> Result<&str, Error> { let mut iter = authority.chars().enumerate(); - let split = match iter.next() { - Some((_, '[')) => { - let mut niter = iter.skip_while(|(_, c)| c != &']'); - niter.next().ok_or(Error::BadRequest(format!( + let (_, first_char) = iter + .next() + .ok_or(Error::BadRequest(format!("Authority is empty")))?; + + let split = match first_char { + '[' => { + let mut iter = iter.skip_while(|(_, c)| c != &']'); + iter.next().ok_or(Error::BadRequest(format!( "Authority {} has an illegal format", authority )))?; - niter.next() + iter.next() } - Some((_, _)) => iter.skip_while(|(_, c)| c != &':').next(), - None => return Err(Error::BadRequest(format!("Authority is empty"))), + _ => iter.skip_while(|(_, c)| c != &':').next(), }; match split { -- cgit v1.2.3 From cacf8ddf2da9c80574647aeb0d61dd15f9f8c5d5 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 15:52:20 +0100 Subject: Panic when it is a logical error --- src/web/web_server.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index e4d15872..73aa6648 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -77,10 +77,7 @@ fn authority_to_host(authority: &str) -> Result<&str, Error> { let split = match first_char { '[' => { let mut iter = iter.skip_while(|(_, c)| c != &']'); - iter.next().ok_or(Error::BadRequest(format!( - "Authority {} has an illegal format", - authority - )))?; + iter.next().expect("Authority parsing logic error"); iter.next() } _ => iter.skip_while(|(_, c)| c != &':').next(), -- cgit v1.2.3 From 3cb3994cd2005231f8cc60ce02c55762a7b293f3 Mon Sep 17 00:00:00 2001 From: Quentin Date: Tue, 10 Nov 2020 17:05:10 +0100 Subject: Add documentation to host_to_bucket --- src/web/web_server.rs | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 73aa6648..2440857d 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -93,6 +93,12 @@ fn authority_to_host(authority: &str) -> Result<&str, Error> { } } +/// Host to bucket +/// +/// Convert a host, like "bucket.garage-site.tld" or "john.doe.com" +/// to the corresponding bucket, resp. "bucket" and "john.doe.com" +/// considering that ".garage-site.tld" is the "root domain". +/// This behavior has been chosen to follow AWS S3 semantic. fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { if root.len() >= host.len() || !host.ends_with(root) { return host; -- cgit v1.2.3 From d445c4ef9cd6835ec7e2e543e9e462adcd0f58bf Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 11 Nov 2020 15:24:25 +0100 Subject: WIP fetch object --- src/web/web_server.rs | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 2440857d..cbb2aaac 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -57,6 +57,13 @@ async fn handler( let path = req.uri().path().to_string(); let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; + // Get bucket descriptor + let object = garage + .object_table + .get(&bucket.to_string(), &key.to_string()) + .await? + .ok_or(Error::NotFound)?; + info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key); Ok(Response::new(Body::from("hello world\n"))) -- cgit v1.2.3 From 2765291796de1b94401e462dc5136fdfce867596 Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 11 Nov 2020 19:48:01 +0100 Subject: Build path correctly --- src/web/web_server.rs | 40 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index cbb2aaac..16b27cef 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::net::SocketAddr; use std::sync::Arc; @@ -55,17 +56,18 @@ async fn handler( // Get path let path = req.uri().path().to_string(); - let key = percent_encoding::percent_decode_str(&path).decode_utf8()?; + let index = &garage.config.s3_web.index; + let key = path_to_key(&path, &index)?; - // Get bucket descriptor + info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key); + + // Get bucket descriptor let object = garage .object_table .get(&bucket.to_string(), &key.to_string()) .await? .ok_or(Error::NotFound)?; - info!("Selected bucket: \"{}\", selected key: \"{}\"", bucket, key); - Ok(Response::new(Body::from("hello world\n"))) } @@ -121,6 +123,27 @@ fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { &host[..cursor] } +/// Path to key +/// +/// Convert the provided path to the internal key +/// When a path ends with "/", we append the index name to match traditional web server behavior +/// which is also AWS S3 behavior. +fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { + let path_utf8 = percent_encoding::percent_decode_str(&path).decode_utf8()?; + match path_utf8.chars().last() { + None => Err(Error::BadRequest(format!( + "Path must have at least a character" + ))), + Some('/') => { + let mut key = String::with_capacity(path_utf8.len() + index.len()); + key.push_str(&path_utf8); + key.push_str(index); + Ok(key.into()) + } + Some(_) => Ok(path_utf8.into()), + } +} + #[cfg(test)] mod tests { use super::*; @@ -170,4 +193,13 @@ mod tests { assert_eq!(host_to_bucket("garage.tld", ".garage.tld"), "garage.tld"); } + + #[test] + fn path_to_key_test() -> Result<(), Error> { + assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "/file .jpg"); + assert_eq!(path_to_key("/%20t/", "index.html")?, "/ t/index.html"); + assert_eq!(path_to_key("/", "index.html")?, "/index.html"); + assert!(path_to_key("", "index.html").is_err()); + Ok(()) + } } -- cgit v1.2.3 From 6076d869b14aa38059d54a2dece222ad7b9da3bc Mon Sep 17 00:00:00 2001 From: Quentin Date: Wed, 11 Nov 2020 21:17:34 +0100 Subject: Build error --- src/web/web_server.rs | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 16b27cef..3cc0fa43 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,4 +1,5 @@ use std::borrow::Cow; +use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; @@ -24,7 +25,7 @@ pub async fn run_web_server( async move { Ok::<_, Error>(service_fn(move |req: Request| { let garage = garage.clone(); - handler(garage, req, client_addr) + handle_request(garage, req, client_addr) })) } }); @@ -37,11 +38,29 @@ pub async fn run_web_server( Ok(()) } -async fn handler( +async fn handle_request( garage: Arc, req: Request, addr: SocketAddr, -) -> Result, Error> { +) -> Result, Infallible> { + info!("{} {} {}", addr, req.method(), req.uri()); + let res = serve_file(garage, req).await; + match &res { + Ok(r) => debug!("{} {:?}", r.status(), r.headers()), + Err(e) => warn!("Response: error {}, {}", e.http_status_code(), e), + } + + Ok(res.unwrap_or_else(error_to_res)) +} + +fn error_to_res(e: Error) -> Response { + let body: Body = Body::from(format!("{}\n", e)); + let mut http_error = Response::new(body); + *http_error.status_mut() = e.http_status_code(); + http_error +} + +async fn serve_file(garage: Arc, req: Request) -> Result, Error> { // Get http authority string (eg. [::1]:3902 or garage.tld:80) let authority = req .headers() -- cgit v1.2.3 From 04f455ff7f673ae9449b374183b8aafb9347581f Mon Sep 17 00:00:00 2001 From: Quentin Date: Thu, 19 Nov 2020 14:56:00 +0100 Subject: Make it compile again --- src/web/error.rs | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/web/lib.rs | 2 ++ src/web/web_server.rs | 5 +++-- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 src/web/error.rs (limited to 'src/web') diff --git a/src/web/error.rs b/src/web/error.rs new file mode 100644 index 00000000..094b22d0 --- /dev/null +++ b/src/web/error.rs @@ -0,0 +1,51 @@ +use err_derive::Error; +use hyper::StatusCode; + +use garage_util::error::Error as GarageError; + +#[derive(Debug, Error)] +pub enum Error { + // Category: internal error + #[error(display = "Internal error: {}", _0)] + InternalError(#[error(source)] GarageError), + + #[error(display = "Internal error (Hyper error): {}", _0)] + Hyper(#[error(source)] hyper::Error), + + #[error(display = "Internal error (HTTP error): {}", _0)] + HTTP(#[error(source)] http::Error), + + // Category: cannot process + #[error(display = "Forbidden: {}", _0)] + Forbidden(String), + + #[error(display = "Not found")] + NotFound, + + // Category: bad request + #[error(display = "Invalid UTF-8: {}", _0)] + InvalidUTF8(#[error(source)] std::str::Utf8Error), + + #[error(display = "Invalid XML: {}", _0)] + InvalidXML(#[error(source)] roxmltree::Error), + + #[error(display = "Invalid header value: {}", _0)] + InvalidHeader(#[error(source)] hyper::header::ToStrError), + + #[error(display = "Bad request: {}", _0)] + BadRequest(String), +} + +impl Error { + pub fn http_status_code(&self) -> StatusCode { + match self { + Error::NotFound => StatusCode::NOT_FOUND, + Error::Forbidden(_) => StatusCode::FORBIDDEN, + Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE, + Error::InternalError(_) | Error::Hyper(_) | Error::HTTP(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } + _ => StatusCode::BAD_REQUEST, + } + } +} diff --git a/src/web/lib.rs b/src/web/lib.rs index c0c668a1..f28937b9 100644 --- a/src/web/lib.rs +++ b/src/web/lib.rs @@ -1,4 +1,6 @@ #[macro_use] extern crate log; +pub mod error; + pub mod web_server; diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 3cc0fa43..4771d209 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -11,12 +11,13 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response, Server}; use garage_model::garage::Garage; -use garage_util::error::Error; +use garage_util::error::Error as GarageError; +use crate::error::*; pub async fn run_web_server( garage: Arc, shutdown_signal: impl Future, -) -> Result<(), Error> { +) -> Result<(), GarageError> { let addr = &garage.config.s3_web.bind_addr; let service = make_service_fn(|conn: &AddrStream| { -- cgit v1.2.3 From 5b363626f4803b3e43cdb450fd6ee04ac9429c4d Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 20 Nov 2020 21:23:32 +0100 Subject: Support punnycode --- src/web/Cargo.toml | 2 +- src/web/web_server.rs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'src/web') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 8eddf047..252ee58d 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -41,4 +41,4 @@ rustls = "0.17" webpki = "0.21" roxmltree = "0.11" - +idna = "0.2" diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 4771d209..1c5619fa 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -10,6 +10,8 @@ use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response, Server}; +use idna::domain_to_unicode; + use garage_model::garage::Garage; use garage_util::error::Error as GarageError; use crate::error::*; @@ -70,7 +72,7 @@ async fn serve_file(garage: Arc, req: Request) -> Result Date: Sat, 21 Nov 2020 12:01:02 +0100 Subject: Fix host to key --- src/web/web_server.rs | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 1c5619fa..7172f222 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -152,17 +152,29 @@ fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { /// which is also AWS S3 behavior. fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { let path_utf8 = percent_encoding::percent_decode_str(&path).decode_utf8()?; + + if path_utf8.chars().next() != Some('/') { + return Err(Error::BadRequest(format!( + "Path must start with a / (slash)" + ))) + } + match path_utf8.chars().last() { None => Err(Error::BadRequest(format!( "Path must have at least a character" ))), Some('/') => { let mut key = String::with_capacity(path_utf8.len() + index.len()); - key.push_str(&path_utf8); + key.push_str(&path_utf8[1..]); key.push_str(index); Ok(key.into()) } - Some(_) => Ok(path_utf8.into()), + Some(_) => { + match path_utf8 { + Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()), + Cow::Owned(pu8) => Ok((&pu8[1..]).to_string().into()), + } + } } } @@ -218,10 +230,12 @@ mod tests { #[test] fn path_to_key_test() -> Result<(), Error> { - assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "/file .jpg"); - assert_eq!(path_to_key("/%20t/", "index.html")?, "/ t/index.html"); - assert_eq!(path_to_key("/", "index.html")?, "/index.html"); + assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "file .jpg"); + assert_eq!(path_to_key("/%20t/", "index.html")?, " t/index.html"); + assert_eq!(path_to_key("/", "index.html")?, "index.html"); + assert_eq!(path_to_key("/hello", "index.html")?, "hello"); assert!(path_to_key("", "index.html").is_err()); + assert!(path_to_key("i/am/relative", "index.html").is_err()); Ok(()) } } -- cgit v1.2.3 From 0f33231ee6154a2b08f67f8107cc686ee9e9c678 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 21 Nov 2020 15:15:25 +0100 Subject: We are able to serve a file --- src/web/Cargo.toml | 2 + src/web/web_server.rs | 111 +++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 108 insertions(+), 5 deletions(-) (limited to 'src/web') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 252ee58d..819b51c1 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -42,3 +42,5 @@ webpki = "0.21" roxmltree = "0.11" idna = "0.2" + +httpdate = "0.3" diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 7172f222..8a222738 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -2,17 +2,23 @@ use std::borrow::Cow; use std::convert::Infallible; use std::net::SocketAddr; use std::sync::Arc; +use std::time::{Duration, UNIX_EPOCH}; use futures::future::Future; +use futures::stream::*; -use hyper::header::HOST; -use hyper::server::conn::AddrStream; -use hyper::service::{make_service_fn, service_fn}; -use hyper::{Body, Request, Response, Server}; +use hyper::{ + header::HOST, + body::Bytes, + server::conn::AddrStream, + service::{make_service_fn, service_fn}, + Body, Request, Response, Server, StatusCode}; use idna::domain_to_unicode; use garage_model::garage::Garage; +use garage_model::object_table::*; +use garage_table::EmptyKey; use garage_util::error::Error as GarageError; use crate::error::*; @@ -90,7 +96,102 @@ async fn serve_file(garage: Arc, req: Request) -> Result x, + _ => unreachable!(), + }; + + // Get metadata from version + let last_v_meta = match last_v_data { + ObjectVersionData::DeleteMarker => return Err(Error::NotFound), + ObjectVersionData::Inline(meta, _) => meta, + ObjectVersionData::FirstBlock(meta, _) => meta, + }; + + // @FIXME Support range + + + // Set headers + let resp_builder = object_headers(&last_v, last_v_meta).status(StatusCode::OK); + + + // Stream body + match &last_v_data { + ObjectVersionData::DeleteMarker => unreachable!(), + ObjectVersionData::Inline(_, bytes) => { + let body: Body = Body::from(bytes.to_vec()); + Ok(resp_builder.body(body)?) + } + ObjectVersionData::FirstBlock(_, first_block_hash) => { + let read_first_block = garage.block_manager.rpc_get_block(&first_block_hash); + let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey); + + let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?; + let version = version.ok_or(Error::NotFound)?; + + let mut blocks = version + .blocks() + .iter() + .map(|vb| (vb.hash, None)) + .collect::>(); + blocks[0].1 = Some(first_block); + + let body_stream = futures::stream::iter(blocks) + .map(move |(hash, data_opt)| { + let garage = garage.clone(); + async move { + if let Some(data) = data_opt { + Ok(Bytes::from(data)) + } else { + garage + .block_manager + .rpc_get_block(&hash) + .await + .map(Bytes::from) + } + } + }) + .buffered(2); + //let body: Body = Box::new(StreamBody::new(Box::pin(body_stream))); + let body = hyper::body::Body::wrap_stream(body_stream); + Ok(resp_builder.body(body)?) + } + } +} + +// Copied from api/s3_get.rs +fn object_headers( + version: &ObjectVersion, + version_meta: &ObjectVersionMeta, +) -> http::response::Builder { + let date = UNIX_EPOCH + Duration::from_millis(version.timestamp); + let date_str = httpdate::fmt_http_date(date); + + let mut resp = Response::builder() + .header( + "Content-Type", + version_meta.headers.content_type.to_string(), + ) + .header("Content-Length", format!("{}", version_meta.size)) + .header("ETag", version_meta.etag.to_string()) + .header("Last-Modified", date_str) + .header("Accept-Ranges", format!("bytes")); + + for (k, v) in version_meta.headers.other.iter() { + resp = resp.header(k, v.to_string()); + } + + resp } /// Extract host from the authority section given by the HTTP host header -- cgit v1.2.3 From a88fd49f71844f04013970a678201a65ab89fb19 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 21 Nov 2020 17:50:19 +0100 Subject: Use handle_get --- src/web/Cargo.toml | 1 + src/web/error.rs | 3 ++ src/web/web_server.rs | 134 +++++--------------------------------------------- 3 files changed, 17 insertions(+), 121 deletions(-) (limited to 'src/web') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 819b51c1..0d08fdbf 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -16,6 +16,7 @@ path = "lib.rs" garage_util = { version = "0.1", path = "../util" } garage_table = { version = "0.1.1", path = "../table" } garage_model = { version = "0.1.1", path = "../model" } +garage_api = { version = "0.1.1", path = "../api" } rand = "0.7" hex = "0.3" diff --git a/src/web/error.rs b/src/web/error.rs index 094b22d0..59810f0f 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -5,6 +5,9 @@ use garage_util::error::Error as GarageError; #[derive(Debug, Error)] pub enum Error { + #[error(display = "API error: {}", _0)] + ApiError(#[error(source)] garage_api::error::Error), + // Category: internal error #[error(display = "Internal error: {}", _0)] InternalError(#[error(source)] GarageError), diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 8a222738..4f79a9ec 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -1,26 +1,20 @@ -use std::borrow::Cow; -use std::convert::Infallible; -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::{Duration, UNIX_EPOCH}; +use std::{borrow::Cow, convert::Infallible, net::SocketAddr, sync::Arc}; use futures::future::Future; -use futures::stream::*; use hyper::{ header::HOST, - body::Bytes, server::conn::AddrStream, service::{make_service_fn, service_fn}, - Body, Request, Response, Server, StatusCode}; + Body, Request, Response, Server, +}; use idna::domain_to_unicode; +use crate::error::*; +use garage_api::s3_get::handle_get; use garage_model::garage::Garage; -use garage_model::object_table::*; -use garage_table::EmptyKey; use garage_util::error::Error as GarageError; -use crate::error::*; pub async fn run_web_server( garage: Arc, @@ -89,109 +83,9 @@ async fn serve_file(garage: Arc, req: Request) -> Result x, - _ => unreachable!(), - }; - - // Get metadata from version - let last_v_meta = match last_v_data { - ObjectVersionData::DeleteMarker => return Err(Error::NotFound), - ObjectVersionData::Inline(meta, _) => meta, - ObjectVersionData::FirstBlock(meta, _) => meta, - }; - - // @FIXME Support range - - - // Set headers - let resp_builder = object_headers(&last_v, last_v_meta).status(StatusCode::OK); - + let r = handle_get(garage, &req, bucket, &key).await?; - // Stream body - match &last_v_data { - ObjectVersionData::DeleteMarker => unreachable!(), - ObjectVersionData::Inline(_, bytes) => { - let body: Body = Body::from(bytes.to_vec()); - Ok(resp_builder.body(body)?) - } - ObjectVersionData::FirstBlock(_, first_block_hash) => { - let read_first_block = garage.block_manager.rpc_get_block(&first_block_hash); - let get_next_blocks = garage.version_table.get(&last_v.uuid, &EmptyKey); - - let (first_block, version) = futures::try_join!(read_first_block, get_next_blocks)?; - let version = version.ok_or(Error::NotFound)?; - - let mut blocks = version - .blocks() - .iter() - .map(|vb| (vb.hash, None)) - .collect::>(); - blocks[0].1 = Some(first_block); - - let body_stream = futures::stream::iter(blocks) - .map(move |(hash, data_opt)| { - let garage = garage.clone(); - async move { - if let Some(data) = data_opt { - Ok(Bytes::from(data)) - } else { - garage - .block_manager - .rpc_get_block(&hash) - .await - .map(Bytes::from) - } - } - }) - .buffered(2); - //let body: Body = Box::new(StreamBody::new(Box::pin(body_stream))); - let body = hyper::body::Body::wrap_stream(body_stream); - Ok(resp_builder.body(body)?) - } - } -} - -// Copied from api/s3_get.rs -fn object_headers( - version: &ObjectVersion, - version_meta: &ObjectVersionMeta, -) -> http::response::Builder { - let date = UNIX_EPOCH + Duration::from_millis(version.timestamp); - let date_str = httpdate::fmt_http_date(date); - - let mut resp = Response::builder() - .header( - "Content-Type", - version_meta.headers.content_type.to_string(), - ) - .header("Content-Length", format!("{}", version_meta.size)) - .header("ETag", version_meta.etag.to_string()) - .header("Last-Modified", date_str) - .header("Accept-Ranges", format!("bytes")); - - for (k, v) in version_meta.headers.other.iter() { - resp = resp.header(k, v.to_string()); - } - - resp + Ok(r) } /// Extract host from the authority section given by the HTTP host header @@ -253,11 +147,11 @@ fn host_to_bucket<'a>(host: &'a str, root: &str) -> &'a str { /// which is also AWS S3 behavior. fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { let path_utf8 = percent_encoding::percent_decode_str(&path).decode_utf8()?; - + if path_utf8.chars().next() != Some('/') { return Err(Error::BadRequest(format!( "Path must start with a / (slash)" - ))) + ))); } match path_utf8.chars().last() { @@ -270,12 +164,10 @@ fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { key.push_str(index); Ok(key.into()) } - Some(_) => { - match path_utf8 { - Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()), - Cow::Owned(pu8) => Ok((&pu8[1..]).to_string().into()), - } - } + Some(_) => match path_utf8 { + Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()), + Cow::Owned(pu8) => Ok((&pu8[1..]).to_string().into()), + }, } } -- cgit v1.2.3 From b7a377308bbcbb7285a5b11cdcb07361eff93a28 Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 21 Nov 2020 17:58:14 +0100 Subject: Handle HEAD --- src/web/web_server.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 4f79a9ec..f8a5cd14 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -6,13 +6,13 @@ use hyper::{ header::HOST, server::conn::AddrStream, service::{make_service_fn, service_fn}, - Body, Request, Response, Server, + Body, Method, Request, Response, Server, }; use idna::domain_to_unicode; use crate::error::*; -use garage_api::s3_get::handle_get; +use garage_api::s3_get::{handle_get, handle_head}; use garage_model::garage::Garage; use garage_util::error::Error as GarageError; @@ -83,9 +83,13 @@ async fn serve_file(garage: Arc, req: Request) -> Result handle_head(garage, &bucket, &key).await?, + &Method::GET => handle_get(garage, &req, bucket, &key).await?, + _ => return Err(Error::BadRequest(format!("HTTP method not supported"))), + }; - Ok(r) + Ok(res) } /// Extract host from the authority section given by the HTTP host header -- cgit v1.2.3 From fb18f5e17a34830d094fc591ee1d8accde2a85ad Mon Sep 17 00:00:00 2001 From: Quentin Date: Sat, 21 Nov 2020 18:14:02 +0100 Subject: Fix wrong http status code --- src/web/error.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'src/web') diff --git a/src/web/error.rs b/src/web/error.rs index 59810f0f..220bacfe 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -43,6 +43,7 @@ impl Error { pub fn http_status_code(&self) -> StatusCode { match self { Error::NotFound => StatusCode::NOT_FOUND, + Error::ApiError(e) => e.http_status_code(), Error::Forbidden(_) => StatusCode::FORBIDDEN, Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE, Error::InternalError(_) | Error::Hyper(_) | Error::HTTP(_) => { -- cgit v1.2.3 From 3132deca5808905ce3956b40a6175b7714e11819 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 17 Dec 2020 20:43:14 +0100 Subject: Web server access control --- src/web/web_server.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index f8a5cd14..9effa86c 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -13,6 +13,8 @@ use idna::domain_to_unicode; use crate::error::*; use garage_api::s3_get::{handle_get, handle_head}; +use garage_table::*; +use garage_model::bucket_table::*; use garage_model::garage::Garage; use garage_util::error::Error as GarageError; @@ -76,6 +78,20 @@ async fn serve_file(garage: Arc, req: Request) -> Result Err(Error::NotFound), + BucketState::Present(params) if !params.website.get() => Err(Error::NotFound), + _ => Ok(()), + }?; + // Get path let path = req.uri().path().to_string(); let index = &garage.config.s3_web.index; -- cgit v1.2.3 From 2f4378a9c42cc3a78b992905e980e8977e6c5e58 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 17 Dec 2020 22:51:44 +0100 Subject: Fix formatting --- src/web/web_server.rs | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 9effa86c..25a7cd5f 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -13,9 +13,9 @@ use idna::domain_to_unicode; use crate::error::*; use garage_api::s3_get::{handle_get, handle_head}; -use garage_table::*; use garage_model::bucket_table::*; use garage_model::garage::Garage; +use garage_table::*; use garage_util::error::Error as GarageError; pub async fn run_web_server( @@ -78,19 +78,19 @@ async fn serve_file(garage: Arc, req: Request) -> Result Err(Error::NotFound), - BucketState::Present(params) if !params.website.get() => Err(Error::NotFound), - _ => Ok(()), - }?; + // Check bucket is exposed as a website + let bucket_desc = garage + .bucket_table + .get(&EmptyKey, &bucket.to_string()) + .await? + .filter(|b| !b.is_deleted()) + .ok_or(Error::NotFound)?; + + match bucket_desc.state.get() { + BucketState::Deleted => Err(Error::NotFound), + BucketState::Present(params) if !params.website.get() => Err(Error::NotFound), + _ => Ok(()), + }?; // Get path let path = req.uri().path().to_string(); -- cgit v1.2.3 From c441a358cdc8447d7ff7c6ff8b066ae2d0d99409 Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 16:16:32 +0100 Subject: Remove unused dependencies --- src/web/Cargo.toml | 19 ------------------- 1 file changed, 19 deletions(-) (limited to 'src/web') diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 0d08fdbf..751b9ace 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -18,30 +18,11 @@ garage_table = { version = "0.1.1", path = "../table" } garage_model = { version = "0.1.1", path = "../model" } garage_api = { version = "0.1.1", path = "../api" } -rand = "0.7" -hex = "0.3" -sha2 = "0.8" err-derive = "0.2.3" log = "0.4" - -sled = "0.31" - -toml = "0.5" -rmp-serde = "0.14.3" -serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } -serde_json = "1.0" - futures = "0.3" -futures-util = "0.3" -tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } - http = "0.2" hyper = "0.13" percent-encoding = "2.1.0" -rustls = "0.17" -webpki = "0.21" - roxmltree = "0.11" idna = "0.2" - -httpdate = "0.3" -- cgit v1.2.3 From 11a79a95dd9b2f2e0dd2d9dc999abd24f4ee232b Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 16:24:27 +0100 Subject: Simplify Error file --- src/web/error.rs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) (limited to 'src/web') diff --git a/src/web/error.rs b/src/web/error.rs index 220bacfe..14bc3b75 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -12,16 +12,6 @@ pub enum Error { #[error(display = "Internal error: {}", _0)] InternalError(#[error(source)] GarageError), - #[error(display = "Internal error (Hyper error): {}", _0)] - Hyper(#[error(source)] hyper::Error), - - #[error(display = "Internal error (HTTP error): {}", _0)] - HTTP(#[error(source)] http::Error), - - // Category: cannot process - #[error(display = "Forbidden: {}", _0)] - Forbidden(String), - #[error(display = "Not found")] NotFound, @@ -29,9 +19,6 @@ pub enum Error { #[error(display = "Invalid UTF-8: {}", _0)] InvalidUTF8(#[error(source)] std::str::Utf8Error), - #[error(display = "Invalid XML: {}", _0)] - InvalidXML(#[error(source)] roxmltree::Error), - #[error(display = "Invalid header value: {}", _0)] InvalidHeader(#[error(source)] hyper::header::ToStrError), @@ -44,11 +31,8 @@ impl Error { match self { Error::NotFound => StatusCode::NOT_FOUND, Error::ApiError(e) => e.http_status_code(), - Error::Forbidden(_) => StatusCode::FORBIDDEN, Error::InternalError(GarageError::RPC(_)) => StatusCode::SERVICE_UNAVAILABLE, - Error::InternalError(_) | Error::Hyper(_) | Error::HTTP(_) => { - StatusCode::INTERNAL_SERVER_ERROR - } + Error::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR, _ => StatusCode::BAD_REQUEST, } } -- cgit v1.2.3 From fad7bc405bd8b3cf1dc9a9319a7e3ee0e1eb3461 Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 17:03:54 +0100 Subject: Behavior problem: do not panic anymore + add tests --- src/web/web_server.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 25a7cd5f..246c045f 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -87,9 +87,8 @@ async fn serve_file(garage: Arc, req: Request) -> Result Err(Error::NotFound), - BucketState::Present(params) if !params.website.get() => Err(Error::NotFound), - _ => Ok(()), + BucketState::Present(params) if *params.website.get() => Ok(()), + _ => Err(Error::NotFound), }?; // Get path @@ -123,8 +122,10 @@ fn authority_to_host(authority: &str) -> Result<&str, Error> { let split = match first_char { '[' => { let mut iter = iter.skip_while(|(_, c)| c != &']'); - iter.next().expect("Authority parsing logic error"); - iter.next() + match iter.next() { + Some((_, ']')) => iter.next(), + _ => None, + } } _ => iter.skip_while(|(_, c)| c != &':').next(), }; @@ -214,6 +215,10 @@ mod tests { assert_eq!(domain2, "garage.tld"); let domain3 = authority_to_host("127.0.0.1")?; assert_eq!(domain3, "127.0.0.1"); + let domain4 = authority_to_host("[")?; + assert_eq!(domain4, "["); + let domain5 = authority_to_host("[hello")?; + assert_eq!(domain5, "[hello"); Ok(()) } -- cgit v1.2.3 From f8a40e8c4f69c20045aaffc4caf51158d697e792 Mon Sep 17 00:00:00 2001 From: Quentin Date: Fri, 15 Jan 2021 17:11:15 +0100 Subject: Explicitly set code path unreachable --- src/web/web_server.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 246c045f..aab7e8de 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -176,9 +176,7 @@ fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { } match path_utf8.chars().last() { - None => Err(Error::BadRequest(format!( - "Path must have at least a character" - ))), + None => unreachable!(), Some('/') => { let mut key = String::with_capacity(path_utf8.len() + index.len()); key.push_str(&path_utf8[1..]); -- cgit v1.2.3 From 851893a3f299da9eeb0ef3c745be1f30164fd6cf Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 15 Jan 2021 17:49:10 +0100 Subject: Do not accept domains such as [hello --- src/web/web_server.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'src/web') diff --git a/src/web/web_server.rs b/src/web/web_server.rs index aab7e8de..24d111a9 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -124,7 +124,12 @@ fn authority_to_host(authority: &str) -> Result<&str, Error> { let mut iter = iter.skip_while(|(_, c)| c != &']'); match iter.next() { Some((_, ']')) => iter.next(), - _ => None, + _ => { + return Err(Error::BadRequest(format!( + "Authority {} has an illegal format", + authority + ))) + } } } _ => iter.skip_while(|(_, c)| c != &':').next(), @@ -213,10 +218,8 @@ mod tests { assert_eq!(domain2, "garage.tld"); let domain3 = authority_to_host("127.0.0.1")?; assert_eq!(domain3, "127.0.0.1"); - let domain4 = authority_to_host("[")?; - assert_eq!(domain4, "["); - let domain5 = authority_to_host("[hello")?; - assert_eq!(domain5, "[hello"); + assert!(authority_to_host("[").is_err()); + assert!(authority_to_host("[hello").is_err()); Ok(()) } -- cgit v1.2.3