From 5768bf362262f78376af14517c4921941986192e Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 10 May 2022 13:16:57 +0200 Subject: First implementation of K2V (#293) **Specification:** View spec at [this URL](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md) - [x] Specify the structure of K2V triples - [x] Specify the DVVS format used for causality detection - [x] Specify the K2V index (just a counter of number of values per partition key) - [x] Specify single-item endpoints: ReadItem, InsertItem, DeleteItem - [x] Specify index endpoint: ReadIndex - [x] Specify multi-item endpoints: InsertBatch, ReadBatch, DeleteBatch - [x] Move to JSON objects instead of tuples - [x] Specify endpoints for polling for updates on single values (PollItem) **Implementation:** - [x] Table for K2V items, causal contexts - [x] Indexing mechanism and table for K2V index - [x] Make API handlers a bit more generic - [x] K2V API endpoint - [x] K2V API router - [x] ReadItem - [x] InsertItem - [x] DeleteItem - [x] PollItem - [x] ReadIndex - [x] InsertBatch - [x] ReadBatch - [x] DeleteBatch **Testing:** - [x] Just a simple Python script that does some requests to check visually that things are going right (does not contain parsing of results or assertions on returned values) - [x] Actual tests: - [x] Adapt testing framework - [x] Simple test with InsertItem + ReadItem - [x] Test with several Insert/Read/DeleteItem + ReadIndex - [x] Test all combinations of return formats for ReadItem - [x] Test with ReadBatch, InsertBatch, DeleteBatch - [x] Test with PollItem - [x] Test error codes - [ ] Fix most broken stuff - [x] test PollItem broken randomly - [x] when invalid causality tokens are given, errors should be 4xx not 5xx **Improvements:** - [x] Descending range queries - [x] Specify - [x] Implement - [x] Add test - [x] Batch updates to index counter - [x] Put K2V behind `k2v` feature flag Co-authored-by: Alex Auvolat Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/293 Co-authored-by: Alex Co-committed-by: Alex --- src/garage/tests/website.rs | 342 -------------------------------------------- 1 file changed, 342 deletions(-) delete mode 100644 src/garage/tests/website.rs (limited to 'src/garage/tests/website.rs') diff --git a/src/garage/tests/website.rs b/src/garage/tests/website.rs deleted file mode 100644 index 963d11ea..00000000 --- a/src/garage/tests/website.rs +++ /dev/null @@ -1,342 +0,0 @@ -use crate::common; -use crate::common::ext::*; -use aws_sdk_s3::{ - model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, - types::ByteStream, -}; -use http::Request; -use hyper::{ - body::{to_bytes, Body}, - Client, -}; - -const BODY: &[u8; 16] = b"

bonjour

"; -const BODY_ERR: &[u8; 6] = b"erreur"; - -#[tokio::test] -async fn test_website() { - const BCKT_NAME: &str = "my-website"; - let ctx = common::context(); - let bucket = ctx.create_bucket(BCKT_NAME); - - let data = ByteStream::from_static(BODY); - - ctx.client - .put_object() - .bucket(&bucket) - .key("index.html") - .body(data) - .send() - .await - .unwrap(); - - let client = Client::new(); - - let req = || { - Request::builder() - .method("GET") - .uri(format!( - "http://127.0.0.1:{}/", - common::garage::DEFAULT_PORT + 2 - )) - .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) - .unwrap() - }; - - let mut resp = client.request(req()).await.unwrap(); - - assert_eq!(resp.status(), 404); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); /* check that we do not leak body */ - - ctx.garage - .command() - .args(["bucket", "website", "--allow", BCKT_NAME]) - .quiet() - .expect_success_status("Could not allow website on bucket"); - - resp = client.request(req()).await.unwrap(); - assert_eq!(resp.status(), 200); - assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); - - ctx.garage - .command() - .args(["bucket", "website", "--deny", BCKT_NAME]) - .quiet() - .expect_success_status("Could not deny website on bucket"); - - resp = client.request(req()).await.unwrap(); - assert_eq!(resp.status(), 404); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); /* check that we do not leak body */ -} - -#[tokio::test] -async fn test_website_s3_api() { - const BCKT_NAME: &str = "my-cors"; - let ctx = common::context(); - let bucket = ctx.create_bucket(BCKT_NAME); - - let data = ByteStream::from_static(BODY); - - ctx.client - .put_object() - .bucket(&bucket) - .key("site/home.html") - .body(data) - .send() - .await - .unwrap(); - - ctx.client - .put_object() - .bucket(&bucket) - .key("err/error.html") - .body(ByteStream::from_static(BODY_ERR)) - .send() - .await - .unwrap(); - - let conf = WebsiteConfiguration::builder() - .index_document(IndexDocument::builder().suffix("home.html").build()) - .error_document(ErrorDocument::builder().key("err/error.html").build()) - .build(); - - ctx.client - .put_bucket_website() - .bucket(&bucket) - .website_configuration(conf) - .send() - .await - .unwrap(); - - let cors = CorsConfiguration::builder() - .cors_rules( - CorsRule::builder() - .id("main-rule") - .allowed_headers("*") - .allowed_methods("GET") - .allowed_methods("PUT") - .allowed_origins("*") - .build(), - ) - .build(); - - ctx.client - .put_bucket_cors() - .bucket(&bucket) - .cors_configuration(cors) - .send() - .await - .unwrap(); - - { - let cors_res = ctx - .client - .get_bucket_cors() - .bucket(&bucket) - .send() - .await - .unwrap(); - - let main_rule = cors_res.cors_rules().unwrap().iter().next().unwrap(); - - assert_eq!(main_rule.id.as_ref().unwrap(), "main-rule"); - assert_eq!( - main_rule.allowed_headers.as_ref().unwrap(), - &vec!["*".to_string()] - ); - assert_eq!( - main_rule.allowed_origins.as_ref().unwrap(), - &vec!["*".to_string()] - ); - assert_eq!( - main_rule.allowed_methods.as_ref().unwrap(), - &vec!["GET".to_string(), "PUT".to_string()] - ); - } - - let client = Client::new(); - - // Test direct requests with CORS - { - let req = Request::builder() - .method("GET") - .uri(format!( - "http://127.0.0.1:{}/site/", - common::garage::DEFAULT_PORT + 2 - )) - .header("Host", format!("{}.web.garage", BCKT_NAME)) - .header("Origin", "https://example.com") - .body(Body::empty()) - .unwrap(); - - let mut resp = client.request(req).await.unwrap(); - - assert_eq!(resp.status(), 200); - assert_eq!( - resp.headers().get("access-control-allow-origin").unwrap(), - "*" - ); - assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); - } - - // Test ErrorDocument on 404 - { - let req = Request::builder() - .method("GET") - .uri(format!( - "http://127.0.0.1:{}/wrong.html", - common::garage::DEFAULT_PORT + 2 - )) - .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) - .unwrap(); - - let mut resp = client.request(req).await.unwrap(); - - assert_eq!(resp.status(), 404); - assert_eq!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY_ERR.as_ref() - ); - } - - // Test CORS with an allowed preflight request - { - let req = Request::builder() - .method("OPTIONS") - .uri(format!( - "http://127.0.0.1:{}/site/", - common::garage::DEFAULT_PORT + 2 - )) - .header("Host", format!("{}.web.garage", BCKT_NAME)) - .header("Origin", "https://example.com") - .header("Access-Control-Request-Method", "PUT") - .body(Body::empty()) - .unwrap(); - - let mut resp = client.request(req).await.unwrap(); - - assert_eq!(resp.status(), 200); - assert_eq!( - resp.headers().get("access-control-allow-origin").unwrap(), - "*" - ); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); - } - - // Test CORS with a forbidden preflight request - { - let req = Request::builder() - .method("OPTIONS") - .uri(format!( - "http://127.0.0.1:{}/site/", - common::garage::DEFAULT_PORT + 2 - )) - .header("Host", format!("{}.web.garage", BCKT_NAME)) - .header("Origin", "https://example.com") - .header("Access-Control-Request-Method", "DELETE") - .body(Body::empty()) - .unwrap(); - - let mut resp = client.request(req).await.unwrap(); - - assert_eq!(resp.status(), 403); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); - } - - //@TODO test CORS on the S3 endpoint. We need to handle auth manually to check it. - - // Delete cors - ctx.client - .delete_bucket_cors() - .bucket(&bucket) - .send() - .await - .unwrap(); - - // Check CORS are deleted from the API - // @FIXME check what is the expected behavior when GetBucketCors is called on a bucket without - // any CORS. - assert!(ctx - .client - .get_bucket_cors() - .bucket(&bucket) - .send() - .await - .is_err()); - - // Test CORS are not sent anymore on a previously allowed request - { - let req = Request::builder() - .method("OPTIONS") - .uri(format!( - "http://127.0.0.1:{}/site/", - common::garage::DEFAULT_PORT + 2 - )) - .header("Host", format!("{}.web.garage", BCKT_NAME)) - .header("Origin", "https://example.com") - .header("Access-Control-Request-Method", "PUT") - .body(Body::empty()) - .unwrap(); - - let mut resp = client.request(req).await.unwrap(); - - assert_eq!(resp.status(), 403); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); - } - - // Disallow website from the API - ctx.client - .delete_bucket_website() - .bucket(&bucket) - .send() - .await - .unwrap(); - - // Check that the website is not served anymore - { - let req = Request::builder() - .method("GET") - .uri(format!( - "http://127.0.0.1:{}/site/", - common::garage::DEFAULT_PORT + 2 - )) - .header("Host", format!("{}.web.garage", BCKT_NAME)) - .body(Body::empty()) - .unwrap(); - - let mut resp = client.request(req).await.unwrap(); - - assert_eq!(resp.status(), 404); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY_ERR.as_ref() - ); - assert_ne!( - to_bytes(resp.body_mut()).await.unwrap().as_ref(), - BODY.as_ref() - ); - } -} -- cgit v1.2.3