From fcaee3bea0019123db05356fd82706f38498365c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 14 Oct 2022 18:10:36 +0200 Subject: definitively expunge openssl from dependencies everywhere --- src/garage/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/garage') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 5ce40ff2..ddc23170 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -58,7 +58,7 @@ opentelemetry-otlp = { version = "0.10", optional = true } prometheus = { version = "0.13", optional = true } [dev-dependencies] -aws-sdk-s3 = "0.8" +aws-sdk-s3 = "0.19" chrono = "0.4" http = "0.2" hmac = "0.12" -- cgit v1.2.3 From 786500332341c8e39f7e3e82e493e567ecaa70e4 Mon Sep 17 00:00:00 2001 From: Tobias Krischer Date: Sun, 16 Oct 2022 19:46:15 +0200 Subject: Use status code 204 No Content for empty responses --- src/garage/tests/k2v/batch.rs | 22 +++++++-------- src/garage/tests/k2v/errorcodes.rs | 20 ++++++------- src/garage/tests/k2v/item.rs | 58 +++++++++++++++++++------------------- src/garage/tests/k2v/poll.rs | 10 +++---- src/garage/tests/k2v/simple.rs | 6 ++-- src/garage/tests/s3/website.rs | 20 ++++++------- 6 files changed, 68 insertions(+), 68 deletions(-) (limited to 'src/garage') diff --git a/src/garage/tests/k2v/batch.rs b/src/garage/tests/k2v/batch.rs index acae1910..6abba1c5 100644 --- a/src/garage/tests/k2v/batch.rs +++ b/src/garage/tests/k2v/batch.rs @@ -6,7 +6,7 @@ use assert_json_diff::assert_json_eq; use serde_json::json; use super::json_body; -use hyper::Method; +use hyper::{Method, StatusCode}; #[tokio::test] async fn test_batch() { @@ -49,7 +49,7 @@ async fn test_batch() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); for sk in ["a", "b", "c", "d.1", "d.2", "e"] { let res = ctx @@ -62,7 +62,7 @@ async fn test_batch() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" @@ -104,7 +104,7 @@ async fn test_batch() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); let json_res = json_body(res).await; assert_json_eq!( json_res, @@ -266,7 +266,7 @@ async fn test_batch() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); for sk in ["b", "c", "d.1", "d.2"] { let res = ctx @@ -280,9 +280,9 @@ async fn test_batch() { .await .unwrap(); if sk == "b" { - assert_eq!(res.status(), 204); + assert_eq!(res.status(), StatusCode::NO_CONTENT); } else { - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); } ct.insert( sk, @@ -317,7 +317,7 @@ async fn test_batch() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); let json_res = json_body(res).await; assert_json_eq!( json_res, @@ -478,7 +478,7 @@ async fn test_batch() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); let json_res = json_body(res).await; assert_json_eq!( json_res, @@ -514,7 +514,7 @@ async fn test_batch() { .send() .await .unwrap(); - assert_eq!(res.status(), 204); + assert_eq!(res.status(), StatusCode::NO_CONTENT); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" @@ -547,7 +547,7 @@ async fn test_batch() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); let json_res = json_body(res).await; assert_json_eq!( json_res, diff --git a/src/garage/tests/k2v/errorcodes.rs b/src/garage/tests/k2v/errorcodes.rs index 2fcc45bc..ecb84729 100644 --- a/src/garage/tests/k2v/errorcodes.rs +++ b/src/garage/tests/k2v/errorcodes.rs @@ -1,13 +1,13 @@ use crate::common; -use hyper::Method; +use hyper::{Method, StatusCode}; #[tokio::test] async fn test_error_codes() { let ctx = common::context(); let bucket = ctx.create_bucket("test-k2v-error-codes"); - // Regular insert should work (code 200) + // Regular insert should work (code 204) let res = ctx .k2v .request @@ -19,7 +19,7 @@ async fn test_error_codes() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // Insert with trash causality token: invalid request let res = ctx @@ -34,7 +34,7 @@ async fn test_error_codes() { .send() .await .unwrap(); - assert_eq!(res.status(), 400); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); // Search without partition key: invalid request let res = ctx @@ -52,7 +52,7 @@ async fn test_error_codes() { .send() .await .unwrap(); - assert_eq!(res.status(), 400); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); // Search with start that is not in prefix: invalid request let res = ctx @@ -70,7 +70,7 @@ async fn test_error_codes() { .send() .await .unwrap(); - assert_eq!(res.status(), 400); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); // Search with invalid json: 400 let res = ctx @@ -88,7 +88,7 @@ async fn test_error_codes() { .send() .await .unwrap(); - assert_eq!(res.status(), 400); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); // Batch insert with invalid causality token: 400 let res = ctx @@ -105,7 +105,7 @@ async fn test_error_codes() { .send() .await .unwrap(); - assert_eq!(res.status(), 400); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); // Batch insert with invalid data: 400 let res = ctx @@ -122,7 +122,7 @@ async fn test_error_codes() { .send() .await .unwrap(); - assert_eq!(res.status(), 400); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); // Poll with invalid causality token: 400 let res = ctx @@ -137,5 +137,5 @@ async fn test_error_codes() { .send() .await .unwrap(); - assert_eq!(res.status(), 400); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); } diff --git a/src/garage/tests/k2v/item.rs b/src/garage/tests/k2v/item.rs index 32537336..2641386f 100644 --- a/src/garage/tests/k2v/item.rs +++ b/src/garage/tests/k2v/item.rs @@ -6,7 +6,7 @@ use assert_json_diff::assert_json_eq; use serde_json::json; use super::json_body; -use hyper::Method; +use hyper::{Method, StatusCode}; #[tokio::test] async fn test_items_and_indices() { @@ -56,7 +56,7 @@ async fn test_items_and_indices() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // Get value back let res = ctx @@ -69,7 +69,7 @@ async fn test_items_and_indices() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" @@ -132,7 +132,7 @@ async fn test_items_and_indices() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // Get value back let res = ctx @@ -145,7 +145,7 @@ async fn test_items_and_indices() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" @@ -201,7 +201,7 @@ async fn test_items_and_indices() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // Get value back let res = ctx @@ -214,7 +214,7 @@ async fn test_items_and_indices() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -271,7 +271,7 @@ async fn test_items_and_indices() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); let ct = res .headers() .get("x-garage-causality-token") @@ -292,7 +292,7 @@ async fn test_items_and_indices() { .send() .await .unwrap(); - assert_eq!(res.status(), 204); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // ReadIndex -- now there should be some stuff tokio::time::sleep(Duration::from_secs(1)).await; @@ -364,7 +364,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // f0: either let res = ctx @@ -377,7 +377,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" @@ -405,7 +405,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -424,7 +424,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/octet-stream" @@ -446,7 +446,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -466,7 +466,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // f0: either let res = ctx @@ -479,7 +479,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -503,7 +503,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -528,7 +528,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 409); // CONFLICT + assert_eq!(res.status(), StatusCode::CONFLICT); // CONFLICT // f3: json let res = ctx @@ -541,7 +541,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -568,7 +568,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 204); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // f0: either let res = ctx @@ -581,7 +581,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -599,7 +599,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -625,7 +625,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 409); // CONFLICT + assert_eq!(res.status(), StatusCode::CONFLICT); // CONFLICT // f3: json let res = ctx @@ -638,7 +638,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -658,7 +658,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 204); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // f0: either let res = ctx @@ -671,7 +671,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 204); // NO CONTENT + assert_eq!(res.status(), StatusCode::NO_CONTENT); // NO CONTENT // f1: not specified let res = ctx @@ -683,7 +683,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" @@ -702,7 +702,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 204); // NO CONTENT + assert_eq!(res.status(), StatusCode::NO_CONTENT); // NO CONTENT // f3: json let res = ctx @@ -715,7 +715,7 @@ async fn test_item_return_format() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::OK); assert_eq!( res.headers().get("content-type").unwrap().to_str().unwrap(), "application/json" diff --git a/src/garage/tests/k2v/poll.rs b/src/garage/tests/k2v/poll.rs index 70dc0410..e56705ae 100644 --- a/src/garage/tests/k2v/poll.rs +++ b/src/garage/tests/k2v/poll.rs @@ -1,4 +1,4 @@ -use hyper::Method; +use hyper::{Method, StatusCode}; use std::time::Duration; use crate::common; @@ -20,7 +20,7 @@ async fn test_poll() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // Retrieve initial value to get its causality token let res2 = ctx @@ -33,7 +33,7 @@ async fn test_poll() { .send() .await .unwrap(); - assert_eq!(res2.status(), 200); + assert_eq!(res2.status(), StatusCode::OK); let ct = res2 .headers() .get("x-garage-causality-token") @@ -80,7 +80,7 @@ async fn test_poll() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); // Check poll finishes with correct value let poll_res = tokio::select! { @@ -88,7 +88,7 @@ async fn test_poll() { res = poll => res.unwrap().unwrap(), }; - assert_eq!(poll_res.status(), 200); + assert_eq!(poll_res.status(), StatusCode::OK); let poll_res_body = hyper::body::to_bytes(poll_res.into_body()) .await diff --git a/src/garage/tests/k2v/simple.rs b/src/garage/tests/k2v/simple.rs index ae9a8674..465fc24d 100644 --- a/src/garage/tests/k2v/simple.rs +++ b/src/garage/tests/k2v/simple.rs @@ -1,6 +1,6 @@ use crate::common; -use hyper::Method; +use hyper::{Method, StatusCode}; #[tokio::test] async fn test_simple() { @@ -18,7 +18,7 @@ async fn test_simple() { .send() .await .unwrap(); - assert_eq!(res.status(), 200); + assert_eq!(res.status(), StatusCode::NO_CONTENT); let res2 = ctx .k2v @@ -30,7 +30,7 @@ async fn test_simple() { .send() .await .unwrap(); - assert_eq!(res2.status(), 200); + assert_eq!(res2.status(), StatusCode::OK); let res2_body = hyper::body::to_bytes(res2.into_body()) .await diff --git a/src/garage/tests/s3/website.rs b/src/garage/tests/s3/website.rs index 0570ac6a..244a2fa0 100644 --- a/src/garage/tests/s3/website.rs +++ b/src/garage/tests/s3/website.rs @@ -4,7 +4,7 @@ use aws_sdk_s3::{ model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, types::ByteStream, }; -use http::Request; +use http::{Request, StatusCode}; use hyper::{ body::{to_bytes, Body}, Client, @@ -43,7 +43,7 @@ async fn test_website() { let mut resp = client.request(req()).await.unwrap(); - assert_eq!(resp.status(), 404); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( to_bytes(resp.body_mut()).await.unwrap().as_ref(), BODY.as_ref() @@ -56,7 +56,7 @@ async fn test_website() { .expect_success_status("Could not allow website on bucket"); resp = client.request(req()).await.unwrap(); - assert_eq!(resp.status(), 200); + assert_eq!(resp.status(), StatusCode::OK); assert_eq!( to_bytes(resp.body_mut()).await.unwrap().as_ref(), BODY.as_ref() @@ -69,7 +69,7 @@ async fn test_website() { .expect_success_status("Could not deny website on bucket"); resp = client.request(req()).await.unwrap(); - assert_eq!(resp.status(), 404); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( to_bytes(resp.body_mut()).await.unwrap().as_ref(), BODY.as_ref() @@ -175,7 +175,7 @@ async fn test_website_s3_api() { let mut resp = client.request(req).await.unwrap(); - assert_eq!(resp.status(), 200); + assert_eq!(resp.status(), StatusCode::OK); assert_eq!( resp.headers().get("access-control-allow-origin").unwrap(), "*" @@ -200,7 +200,7 @@ async fn test_website_s3_api() { let mut resp = client.request(req).await.unwrap(); - assert_eq!(resp.status(), 404); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_eq!( to_bytes(resp.body_mut()).await.unwrap().as_ref(), BODY_ERR.as_ref() @@ -220,7 +220,7 @@ async fn test_website_s3_api() { let mut resp = client.request(req).await.unwrap(); - assert_eq!(resp.status(), 200); + assert_eq!(resp.status(), StatusCode::OK); assert_eq!( resp.headers().get("access-control-allow-origin").unwrap(), "*" @@ -244,7 +244,7 @@ async fn test_website_s3_api() { let mut resp = client.request(req).await.unwrap(); - assert_eq!(resp.status(), 403); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); assert_ne!( to_bytes(resp.body_mut()).await.unwrap().as_ref(), BODY.as_ref() @@ -285,7 +285,7 @@ async fn test_website_s3_api() { let mut resp = client.request(req).await.unwrap(); - assert_eq!(resp.status(), 403); + assert_eq!(resp.status(), StatusCode::FORBIDDEN); assert_ne!( to_bytes(resp.body_mut()).await.unwrap().as_ref(), BODY.as_ref() @@ -311,7 +311,7 @@ async fn test_website_s3_api() { let mut resp = client.request(req).await.unwrap(); - assert_eq!(resp.status(), 404); + assert_eq!(resp.status(), StatusCode::NOT_FOUND); assert_ne!( to_bytes(resp.body_mut()).await.unwrap().as_ref(), BODY_ERR.as_ref() -- cgit v1.2.3 From 002b9fc50c5b69e0e10c84e4db5ecea1b3941fad Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 18 Oct 2022 18:38:20 +0200 Subject: Add TLS support for Consul discovery + refactoring --- src/garage/Cargo.toml | 2 ++ src/garage/main.rs | 2 ++ 2 files changed, 4 insertions(+) (limited to 'src/garage') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index ddc23170..cbc0dc61 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -81,6 +81,8 @@ sled = [ "garage_model/sled" ] lmdb = [ "garage_model/lmdb" ] sqlite = [ "garage_model/sqlite" ] +# Automatic registration and discovery via Consul API +consul-discovery = [ "garage_rpc/consul-discovery" ] # Automatic registration and discovery via Kubernetes API kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ] # Prometheus exporter (/metrics endpoint). diff --git a/src/garage/main.rs b/src/garage/main.rs index e5cba553..5b2a85c0 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -90,6 +90,8 @@ async fn main() { "lmdb", #[cfg(feature = "sqlite")] "sqlite", + #[cfg(feature = "consul-discovery")] + "consul-discovery", #[cfg(feature = "kubernetes-discovery")] "kubernetes-discovery", #[cfg(feature = "metrics")] -- cgit v1.2.3 From 5b18fd8201af98798f0a47a95dfec2f3ab9777f8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 4 Nov 2022 11:55:59 +0100 Subject: Add garage bucket cleanup-incomplete-uploads command --- src/garage/Cargo.toml | 1 + src/garage/admin.rs | 39 +++++++++++++++++++++++++++++++++++++++ src/garage/cli/structs.rs | 15 +++++++++++++++ 3 files changed, 55 insertions(+) (limited to 'src/garage') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index cbc0dc61..35caf2c1 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -33,6 +33,7 @@ garage_web = { version = "0.8.0", path = "../web" } bytes = "1.0" bytesize = "1.1" timeago = "0.3" +parse_duration = "2.1" hex = "0.4" tracing = { version = "0.1.30", features = ["log-always"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/src/garage/admin.rs b/src/garage/admin.rs index 802a8261..c21286e5 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -85,6 +85,9 @@ impl AdminRpcHandler { BucketOperation::Deny(query) => self.handle_bucket_deny(query).await, BucketOperation::Website(query) => self.handle_bucket_website(query).await, BucketOperation::SetQuotas(query) => self.handle_bucket_set_quotas(query).await, + BucketOperation::CleanupIncompleteUploads(query) => { + self.handle_bucket_cleanup_incomplete_uploads(query).await + } } } @@ -512,6 +515,42 @@ impl AdminRpcHandler { ))) } + async fn handle_bucket_cleanup_incomplete_uploads( + &self, + query: &CleanupIncompleteUploadsOpt, + ) -> Result { + let mut bucket_ids = vec![]; + for b in query.buckets.iter() { + bucket_ids.push( + self.garage + .bucket_helper() + .resolve_global_bucket_name(b) + .await? + .ok_or_bad_request("Bucket not found")?, + ); + } + + let duration = parse_duration::parse::parse(&query.older_than) + .ok_or_bad_request("Invalid duration")?; + + let mut ret = String::new(); + for bucket in bucket_ids { + let count = self + .garage + .bucket_helper() + .cleanup_incomplete_uploads(&bucket, duration) + .await?; + writeln!( + &mut ret, + "Bucket {:?}: {} incomplete uploads aborted", + bucket, count + ) + .unwrap(); + } + + Ok(AdminRpc::Ok(ret)) + } + async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result { match cmd { KeyOperation::List => self.handle_list_keys().await, diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index 06548e89..cb085813 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -189,6 +189,10 @@ pub enum BucketOperation { /// Set the quotas for this bucket #[structopt(name = "set-quotas", version = garage_version())] SetQuotas(SetQuotasOpt), + + /// Clean up (abort) old incomplete multipart uploads + #[structopt(name = "cleanup-incomplete-uploads", version = garage_version())] + CleanupIncompleteUploads(CleanupIncompleteUploadsOpt), } #[derive(Serialize, Deserialize, StructOpt, Debug)] @@ -290,6 +294,17 @@ pub struct SetQuotasOpt { pub max_objects: Option, } +#[derive(Serialize, Deserialize, StructOpt, Debug)] +pub struct CleanupIncompleteUploadsOpt { + /// Abort multipart uploads older than this value + #[structopt(long = "older-than", default_value = "1d")] + pub older_than: String, + + /// Name of bucket(s) to clean up + #[structopt(required = true)] + pub buckets: Vec, +} + #[derive(Serialize, Deserialize, StructOpt, Debug)] pub enum KeyOperation { /// List keys -- cgit v1.2.3 From 8d3bbf5703f97ff1f468bc09fc515b6c12b027a3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 4 Nov 2022 16:07:33 +0100 Subject: Clearer error messsages --- src/garage/admin.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'src/garage') diff --git a/src/garage/admin.rs b/src/garage/admin.rs index c21286e5..e973cfe7 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -526,12 +526,12 @@ impl AdminRpcHandler { .bucket_helper() .resolve_global_bucket_name(b) .await? - .ok_or_bad_request("Bucket not found")?, + .ok_or_bad_request(format!("Bucket not found: {}", b))?, ); } let duration = parse_duration::parse::parse(&query.older_than) - .ok_or_bad_request("Invalid duration")?; + .ok_or_bad_request("Invalid duration passed for --older-than parameter")?; let mut ret = String::new(); for bucket in bucket_ids { -- cgit v1.2.3 From e03d9062f7f21dd0493dd82a7dcf82f2cd035943 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 4 Nov 2022 16:33:05 +0100 Subject: Show a nice message and a backtrace when Garage panics --- src/garage/Cargo.toml | 1 + src/garage/main.rs | 56 ++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 41 insertions(+), 16 deletions(-) (limited to 'src/garage') diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 35caf2c1..69852db7 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -30,6 +30,7 @@ garage_table = { version = "0.8.0", path = "../table" } garage_util = { version = "0.8.0", path = "../util" } garage_web = { version = "0.8.0", path = "../web" } +backtrace = "0.3" bytes = "1.0" bytesize = "1.1" timeago = "0.3" diff --git a/src/garage/main.rs b/src/garage/main.rs index 5b2a85c0..edda734b 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -65,21 +65,6 @@ struct Opt { #[tokio::main] async fn main() { - if std::env::var("RUST_LOG").is_err() { - std::env::set_var("RUST_LOG", "netapp=info,garage=info") - } - tracing_subscriber::fmt() - .with_writer(std::io::stderr) - .with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env()) - .init(); - sodiumoxide::init().expect("Unable to init sodiumoxide"); - - // Abort on panic (same behavior as in Go) - std::panic::set_hook(Box::new(|panic_info| { - error!("{}", panic_info.to_string()); - std::process::abort(); - })); - // Initialize version and features info let features = &[ #[cfg(feature = "k2v")] @@ -108,12 +93,51 @@ async fn main() { } garage_util::version::init_features(features); - // Parse arguments let version = format!( "{} [features: {}]", garage_util::version::garage_version(), features.join(", ") ); + + // Initialize panic handler that aborts on panic and shows a nice message. + // By default, Tokio continues runing normally when a task panics. We want + // to avoid this behavior in Garage as this would risk putting the process in an + // unknown/uncontrollable state. We prefer to exit the process and restart it + // from scratch, so that it boots back into a fresh, known state. + let panic_version_info = version.clone(); + std::panic::set_hook(Box::new(move |panic_info| { + eprintln!("======== PANIC (internal Garage error) ========"); + eprintln!("{}", panic_info); + eprintln!(); + eprintln!("Panics are internal errors that Garage is unable to handle on its own."); + eprintln!("They can be caused by bugs in Garage's code, or by corrupted data in"); + eprintln!("the node's storage. If you feel that this error is likely to be a bug"); + eprintln!("in Garage, please report it on our issue tracker a the following address:"); + eprintln!(); + eprintln!(" https://git.deuxfleurs.fr/Deuxfleurs/garage/issues"); + eprintln!(); + eprintln!("Please include the last log messages and the the full backtrace below in"); + eprintln!("your bug report, as well as any relevant information on the context in"); + eprintln!("which Garage was running when this error occurred."); + eprintln!(); + eprintln!("GARAGE VERSION: {}", panic_version_info); + eprintln!(); + eprintln!("BACKTRACE:"); + eprintln!("{:?}", backtrace::Backtrace::new()); + std::process::abort(); + })); + + // Initialize logging as well as other libraries used in Garage + if std::env::var("RUST_LOG").is_err() { + std::env::set_var("RUST_LOG", "netapp=info,garage=info") + } + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env()) + .init(); + sodiumoxide::init().expect("Unable to init sodiumoxide"); + + // Parse arguments and dispatch command line let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches()); let res = match opt.cmd { -- cgit v1.2.3