diff options
| -rw-r--r-- | Cargo.lock | 131 | ||||
| -rw-r--r-- | Cargo.nix | 192 | ||||
| -rw-r--r-- | src/garage/Cargo.toml | 2 | ||||
| -rw-r--r-- | src/garage/admin.rs | 39 | ||||
| -rw-r--r-- | src/garage/cli/structs.rs | 15 | ||||
| -rw-r--r-- | src/garage/main.rs | 56 | ||||
| -rw-r--r-- | src/model/helper/bucket.rs | 69 |
7 files changed, 486 insertions, 18 deletions
@@ -3,6 +3,21 @@ version = 3 [[package]] +name = "addr2line" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] name = "ahash" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -390,6 +405,21 @@ dependencies = [ ] [[package]] +name = "backtrace" +version = "0.3.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7" +dependencies = [ + "addr2line", + "cc", + "cfg-if 1.0.0", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] name = "base64" version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1023,6 +1053,7 @@ dependencies = [ "assert-json-diff", "async-trait", "aws-sdk-s3", + "backtrace", "base64", "bytes", "bytesize", @@ -1046,6 +1077,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-prometheus", + "parse_duration", "prometheus", "rand 0.8.5", "rmp-serde", @@ -1309,6 +1341,12 @@ dependencies = [ ] [[package]] +name = "gimli" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" + +[[package]] name = "git-version" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2011,6 +2049,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] +name = "miniz_oxide" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34" +dependencies = [ + "adler", +] + +[[package]] name = "mio" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2113,6 +2160,41 @@ dependencies = [ ] [[package]] +name = "num" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304" +dependencies = [ + "autocfg 1.1.0", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" +dependencies = [ + "autocfg 1.1.0", + "num-traits", +] + +[[package]] name = "num-integer" version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2123,6 +2205,29 @@ dependencies = [ ] [[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg 1.1.0", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" +dependencies = [ + "autocfg 1.1.0", + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] name = "num-traits" version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2151,6 +2256,15 @@ dependencies = [ ] [[package]] +name = "object" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53" +dependencies = [ + "memchr", +] + +[[package]] name = "once_cell" version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2304,6 +2418,17 @@ dependencies = [ ] [[package]] +name = "parse_duration" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d" +dependencies = [ + "lazy_static", + "num", + "regex", +] + +[[package]] name = "pem" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2955,6 +3080,12 @@ dependencies = [ ] [[package]] +name = "rustc-demangle" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" + +[[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -32,7 +32,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "45b04ec226341a714068633cf7a3a6b421aa70ac3f78c72931d43ab8301b3c7b"; + nixifiedLockHash = "90b29705f5037c7e1b33f4650841f1266f2e86fa03d5d0c87ad80be7619985c7"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -67,6 +67,23 @@ in garage = rustPackages.unknown.garage."0.8.0"; k2v-client = rustPackages.unknown.k2v-client."0.0.1"; }; + "registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" = overridableMkRustCrate (profileName: rec { + name = "addr2line"; + version = "0.17.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"; }; + dependencies = { + gimli = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".gimli."0.26.2" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".adler."1.0.2" = overridableMkRustCrate (profileName: rec { + name = "adler"; + version = "1.0.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".ahash."0.7.6" = overridableMkRustCrate (profileName: rec { name = "ahash"; version = "0.7.6"; @@ -555,6 +572,28 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".backtrace."0.3.66" = overridableMkRustCrate (profileName: rec { + name = "backtrace"; + version = "0.3.66"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "cab84319d616cfb654d03394f38ab7e6f0919e181b1b57e1fd15e7fb4077d9a7"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + dependencies = { + addr2line = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".addr2line."0.17.0" { inherit profileName; }).out; + cfg_if = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }).out; + libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }).out; + miniz_oxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".miniz_oxide."0.5.4" { inherit profileName; }).out; + object = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".object."0.29.0" { inherit profileName; }).out; + rustc_demangle = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rustc-demangle."0.1.21" { inherit profileName; }).out; + }; + buildDependencies = { + cc = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.73" { profileName = "__noProfile"; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" = overridableMkRustCrate (profileName: rec { name = "base64"; version = "0.13.0"; @@ -1478,6 +1517,7 @@ in ]; dependencies = { async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }).out; + backtrace = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".backtrace."0.3.66" { inherit profileName; }).out; bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.2.0" { inherit profileName; }).out; bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.1.0" { inherit profileName; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }).out; @@ -1496,6 +1536,7 @@ in opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/opentelemetry-otlp" || rootFeatures' ? "garage/telemetry-otlp" then "opentelemetry_otlp" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus" then "opentelemetry_prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }).out; + parse_duration = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus" then "prometheus" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }).out; rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; rmp_serde = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }).out; @@ -1844,6 +1885,17 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".gimli."0.26.2" = overridableMkRustCrate (profileName: rec { + name = "gimli"; + version = "0.26.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d"; }; + features = builtins.concatLists [ + [ "read" ] + [ "read-core" ] + ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" = overridableMkRustCrate (profileName: rec { name = "git-version"; version = "0.3.5"; @@ -2818,6 +2870,16 @@ in ]; }); + "registry+https://github.com/rust-lang/crates.io-index".miniz_oxide."0.5.4" = overridableMkRustCrate (profileName: rec { + name = "miniz_oxide"; + version = "0.5.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "96590ba8f175222643a85693f33d26e9c8a015f599c216509b1a6894af675d34"; }; + dependencies = { + adler = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".adler."1.0.2" { inherit profileName; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".mio."0.8.2" = overridableMkRustCrate (profileName: rec { name = "mio"; version = "0.8.2"; @@ -2954,6 +3016,59 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".num."0.2.1" = overridableMkRustCrate (profileName: rec { + name = "num"; + version = "0.2.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "b8536030f9fea7127f841b45bb6243b27255787fb4eb83958aa1ef9d2fdc0c36"; }; + features = builtins.concatLists [ + [ "default" ] + [ "num-bigint" ] + [ "std" ] + ]; + dependencies = { + num_bigint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" { inherit profileName; }).out; + num_complex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-complex."0.2.4" { inherit profileName; }).out; + num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }).out; + num_iter = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-iter."0.1.43" { inherit profileName; }).out; + num_rational = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-rational."0.2.4" { inherit profileName; }).out; + num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" = overridableMkRustCrate (profileName: rec { + name = "num-bigint"; + version = "0.2.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "090c7f9998ee0ff65aa5b723e4009f7b217707f1fb5ea551329cc4d6231fb304"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + dependencies = { + num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }).out; + num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out; + }; + buildDependencies = { + autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".num-complex."0.2.4" = overridableMkRustCrate (profileName: rec { + name = "num-complex"; + version = "0.2.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + dependencies = { + num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out; + }; + buildDependencies = { + autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" = overridableMkRustCrate (profileName: rec { name = "num-integer"; version = "0.1.44"; @@ -2971,6 +3086,43 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".num-iter."0.1.43" = overridableMkRustCrate (profileName: rec { + name = "num-iter"; + version = "0.1.43"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252"; }; + features = builtins.concatLists [ + [ "std" ] + ]; + dependencies = { + num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }).out; + num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out; + }; + buildDependencies = { + autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".num-rational."0.2.4" = overridableMkRustCrate (profileName: rec { + name = "num-rational"; + version = "0.2.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef"; }; + features = builtins.concatLists [ + [ "bigint" ] + [ "num-bigint" ] + [ "std" ] + ]; + dependencies = { + num_bigint = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-bigint."0.2.6" { inherit profileName; }).out; + num_integer = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-integer."0.1.44" { inherit profileName; }).out; + num_traits = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" { inherit profileName; }).out; + }; + buildDependencies = { + autocfg = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".num-traits."0.2.14" = overridableMkRustCrate (profileName: rec { name = "num-traits"; version = "0.2.14"; @@ -3006,6 +3158,25 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".object."0.29.0" = overridableMkRustCrate (profileName: rec { + name = "object"; + version = "0.29.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "21158b2c33aa6d4561f1c0a6ea283ca92bc54802a93b263e910746d679a7eb53"; }; + features = builtins.concatLists [ + [ "archive" ] + [ "coff" ] + [ "elf" ] + [ "macho" ] + [ "pe" ] + [ "read_core" ] + [ "unaligned" ] + ]; + dependencies = { + memchr = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" = overridableMkRustCrate (profileName: rec { name = "once_cell"; version = "1.10.0"; @@ -3220,6 +3391,18 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".parse_duration."2.1.1" = overridableMkRustCrate (profileName: rec { + name = "parse_duration"; + version = "2.1.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7037e5e93e0172a5a96874380bf73bc6ecef022e26fa25f2be26864d6b3ba95d"; }; + dependencies = { + lazy_static = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }).out; + num = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".num."0.2.1" { inherit profileName; }).out; + regex = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.5" { inherit profileName; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".pem."1.1.0" = overridableMkRustCrate (profileName: rec { name = "pem"; version = "1.1.0"; @@ -4078,6 +4261,13 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".rustc-demangle."0.1.21" = overridableMkRustCrate (profileName: rec { + name = "rustc-demangle"; + version = "0.1.21"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".rustc_version."0.4.0" = overridableMkRustCrate (profileName: rec { name = "rustc_version"; version = "0.4.0"; diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index cbc0dc61..69852db7 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -30,9 +30,11 @@ garage_table = { version = "0.8.0", path = "../table" } garage_util = { version = "0.8.0", path = "../util" } garage_web = { version = "0.8.0", path = "../web" } +backtrace = "0.3" bytes = "1.0" bytesize = "1.1" timeago = "0.3" +parse_duration = "2.1" hex = "0.4" tracing = { version = "0.1.30", features = ["log-always"] } tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/src/garage/admin.rs b/src/garage/admin.rs index 802a8261..e973cfe7 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -85,6 +85,9 @@ impl AdminRpcHandler { BucketOperation::Deny(query) => self.handle_bucket_deny(query).await, BucketOperation::Website(query) => self.handle_bucket_website(query).await, BucketOperation::SetQuotas(query) => self.handle_bucket_set_quotas(query).await, + BucketOperation::CleanupIncompleteUploads(query) => { + self.handle_bucket_cleanup_incomplete_uploads(query).await + } } } @@ -512,6 +515,42 @@ impl AdminRpcHandler { ))) } + async fn handle_bucket_cleanup_incomplete_uploads( + &self, + query: &CleanupIncompleteUploadsOpt, + ) -> Result<AdminRpc, Error> { + let mut bucket_ids = vec![]; + for b in query.buckets.iter() { + bucket_ids.push( + self.garage + .bucket_helper() + .resolve_global_bucket_name(b) + .await? + .ok_or_bad_request(format!("Bucket not found: {}", b))?, + ); + } + + let duration = parse_duration::parse::parse(&query.older_than) + .ok_or_bad_request("Invalid duration passed for --older-than parameter")?; + + let mut ret = String::new(); + for bucket in bucket_ids { + let count = self + .garage + .bucket_helper() + .cleanup_incomplete_uploads(&bucket, duration) + .await?; + writeln!( + &mut ret, + "Bucket {:?}: {} incomplete uploads aborted", + bucket, count + ) + .unwrap(); + } + + Ok(AdminRpc::Ok(ret)) + } + async fn handle_key_cmd(&self, cmd: &KeyOperation) -> Result<AdminRpc, Error> { match cmd { KeyOperation::List => self.handle_list_keys().await, diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index 06548e89..cb085813 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -189,6 +189,10 @@ pub enum BucketOperation { /// Set the quotas for this bucket #[structopt(name = "set-quotas", version = garage_version())] SetQuotas(SetQuotasOpt), + + /// Clean up (abort) old incomplete multipart uploads + #[structopt(name = "cleanup-incomplete-uploads", version = garage_version())] + CleanupIncompleteUploads(CleanupIncompleteUploadsOpt), } #[derive(Serialize, Deserialize, StructOpt, Debug)] @@ -291,6 +295,17 @@ pub struct SetQuotasOpt { } #[derive(Serialize, Deserialize, StructOpt, Debug)] +pub struct CleanupIncompleteUploadsOpt { + /// Abort multipart uploads older than this value + #[structopt(long = "older-than", default_value = "1d")] + pub older_than: String, + + /// Name of bucket(s) to clean up + #[structopt(required = true)] + pub buckets: Vec<String>, +} + +#[derive(Serialize, Deserialize, StructOpt, Debug)] pub enum KeyOperation { /// List keys #[structopt(name = "list", version = garage_version())] diff --git a/src/garage/main.rs b/src/garage/main.rs index 5b2a85c0..edda734b 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -65,21 +65,6 @@ struct Opt { #[tokio::main] async fn main() { - if std::env::var("RUST_LOG").is_err() { - std::env::set_var("RUST_LOG", "netapp=info,garage=info") - } - tracing_subscriber::fmt() - .with_writer(std::io::stderr) - .with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env()) - .init(); - sodiumoxide::init().expect("Unable to init sodiumoxide"); - - // Abort on panic (same behavior as in Go) - std::panic::set_hook(Box::new(|panic_info| { - error!("{}", panic_info.to_string()); - std::process::abort(); - })); - // Initialize version and features info let features = &[ #[cfg(feature = "k2v")] @@ -108,12 +93,51 @@ async fn main() { } garage_util::version::init_features(features); - // Parse arguments let version = format!( "{} [features: {}]", garage_util::version::garage_version(), features.join(", ") ); + + // Initialize panic handler that aborts on panic and shows a nice message. + // By default, Tokio continues runing normally when a task panics. We want + // to avoid this behavior in Garage as this would risk putting the process in an + // unknown/uncontrollable state. We prefer to exit the process and restart it + // from scratch, so that it boots back into a fresh, known state. + let panic_version_info = version.clone(); + std::panic::set_hook(Box::new(move |panic_info| { + eprintln!("======== PANIC (internal Garage error) ========"); + eprintln!("{}", panic_info); + eprintln!(); + eprintln!("Panics are internal errors that Garage is unable to handle on its own."); + eprintln!("They can be caused by bugs in Garage's code, or by corrupted data in"); + eprintln!("the node's storage. If you feel that this error is likely to be a bug"); + eprintln!("in Garage, please report it on our issue tracker a the following address:"); + eprintln!(); + eprintln!(" https://git.deuxfleurs.fr/Deuxfleurs/garage/issues"); + eprintln!(); + eprintln!("Please include the last log messages and the the full backtrace below in"); + eprintln!("your bug report, as well as any relevant information on the context in"); + eprintln!("which Garage was running when this error occurred."); + eprintln!(); + eprintln!("GARAGE VERSION: {}", panic_version_info); + eprintln!(); + eprintln!("BACKTRACE:"); + eprintln!("{:?}", backtrace::Backtrace::new()); + std::process::abort(); + })); + + // Initialize logging as well as other libraries used in Garage + if std::env::var("RUST_LOG").is_err() { + std::env::set_var("RUST_LOG", "netapp=info,garage=info") + } + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env()) + .init(); + sodiumoxide::init().expect("Unable to init sodiumoxide"); + + // Parse arguments and dispatch command line let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches()); let res = match opt.cmd { diff --git a/src/model/helper/bucket.rs b/src/model/helper/bucket.rs index 130ba5be..4a488d7f 100644 --- a/src/model/helper/bucket.rs +++ b/src/model/helper/bucket.rs @@ -1,3 +1,5 @@ +use std::time::Duration; + use garage_util::crdt::*; use garage_util::data::*; use garage_util::error::{Error as GarageError, OkOrMessage}; @@ -12,7 +14,7 @@ use crate::helper::error::*; use crate::helper::key::KeyHelper; use crate::key_table::*; use crate::permission::BucketKeyPerm; -use crate::s3::object_table::ObjectFilter; +use crate::s3::object_table::*; pub struct BucketHelper<'a>(pub(crate) &'a Garage); @@ -472,4 +474,69 @@ impl<'a> BucketHelper<'a> { Ok(true) } + + // ---- + + /// Deletes all incomplete multipart uploads that are older than a certain time. + /// Returns the number of uploads aborted + pub async fn cleanup_incomplete_uploads( + &self, + bucket_id: &Uuid, + older_than: Duration, + ) -> Result<usize, Error> { + let older_than = now_msec() - older_than.as_millis() as u64; + + let mut ret = 0usize; + let mut start = None; + + loop { + let objects = self + .0 + .object_table + .get_range( + bucket_id, + start, + Some(ObjectFilter::IsUploading), + 1000, + EnumerationOrder::Forward, + ) + .await?; + + let abortions = objects + .iter() + .filter_map(|object| { + let aborted_versions = object + .versions() + .iter() + .filter(|v| v.is_uploading() && v.timestamp < older_than) + .map(|v| ObjectVersion { + state: ObjectVersionState::Aborted, + uuid: v.uuid, + timestamp: v.timestamp, + }) + .collect::<Vec<_>>(); + if !aborted_versions.is_empty() { + Some(Object::new( + object.bucket_id, + object.key.clone(), + aborted_versions, + )) + } else { + None + } + }) + .collect::<Vec<_>>(); + + ret += abortions.len(); + self.0.object_table.insert_many(abortions).await?; + + if objects.len() < 1000 { + break; + } else { + start = Some(objects.last().unwrap().key.clone()); + } + } + + Ok(ret) + } } |