From 78f2d86fc8be8e923d91f069406acd48edcae180 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Fri, 22 Dec 2023 21:52:20 +0100 Subject: WIP k2v --- Cargo.lock | 392 ++++++++++++++++++-------------------------------- Cargo.toml | 2 +- src/storage/garage.rs | 118 ++++++++++++--- 3 files changed, 238 insertions(+), 274 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 99ba996..7611ef8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,7 +44,7 @@ dependencies = [ "eml-codec", "futures", "hex", - "hyper-rustls 0.24.1", + "hyper-rustls", "im", "imap-codec", "itertools", @@ -69,6 +69,15 @@ dependencies = [ "zstd", ] +[[package]] +name = "aho-corasick" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +dependencies = [ + "memchr", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -406,11 +415,11 @@ dependencies = [ "aws-sdk-ssooidc", "aws-sdk-sts", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.60.1", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "aws-types", "bytes", "fastrand 2.0.1", @@ -432,7 +441,7 @@ checksum = "70a1629320d319dc715c6189b172349186557e209d2a7b893ff3d14efd33a47c" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "zeroize", ] @@ -443,7 +452,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30e4199d5d62ab09be6a64650c06cc5c4aa45806fed4c74bc4a5c8eaf039a6fa" dependencies = [ "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "aws-types", "bytes", "http", @@ -460,12 +469,12 @@ checksum = "87116d357c905b53f1828d15366363fd27b330a0393cbef349e653f686d36bad" dependencies = [ "aws-credential-types", "aws-http", - "aws-sigv4", + "aws-sigv4 1.1.1", "aws-smithy-async", "aws-smithy-eventstream", - "aws-smithy-http", + "aws-smithy-http 0.60.1", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "aws-types", "fastrand 2.0.1", "http", @@ -483,15 +492,15 @@ dependencies = [ "aws-credential-types", "aws-http", "aws-runtime", - "aws-sigv4", + "aws-sigv4 1.1.1", "aws-smithy-async", "aws-smithy-checksums", "aws-smithy-eventstream", - "aws-smithy-http", + "aws-smithy-http 0.60.1", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "aws-smithy-xml", "aws-types", "bytes", @@ -514,11 +523,11 @@ dependencies = [ "aws-http", "aws-runtime", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.60.1", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "aws-types", "bytes", "http", @@ -537,11 +546,11 @@ dependencies = [ "aws-http", "aws-runtime", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.60.1", "aws-smithy-json", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "aws-types", "bytes", "http", @@ -560,12 +569,12 @@ dependencies = [ "aws-http", "aws-runtime", "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.60.1", "aws-smithy-json", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "aws-smithy-xml", "aws-types", "http", @@ -574,6 +583,25 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-sigv4" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" +dependencies = [ + "aws-smithy-http 0.55.3", + "form_urlencoded", + "hex", + "hmac", + "http", + "once_cell", + "percent-encoding", + "regex", + "sha2", + "time 0.3.23", + "tracing", +] + [[package]] name = "aws-sigv4" version = "1.1.1" @@ -582,20 +610,20 @@ checksum = "d222297ca90209dc62245f0a490355795f29de362eb5c19caea4f7f55fe69078" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", - "aws-smithy-http", + "aws-smithy-http 0.60.1", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "bytes", "crypto-bigint 0.5.5", "form_urlencoded", "hex", - "hmac 0.12.1", + "hmac", "http", "once_cell", "p256", "percent-encoding", "ring 0.17.7", - "sha2 0.10.8", + "sha2", "subtle", "time 0.3.23", "tracing", @@ -619,18 +647,18 @@ version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c2a63681f82fb85ca58d566534b7dc619c782fee0c61c1aa51e2b560c21cb4f" dependencies = [ - "aws-smithy-http", - "aws-smithy-types", + "aws-smithy-http 0.60.1", + "aws-smithy-types 1.1.1", "bytes", "crc32c", "crc32fast", "hex", "http", "http-body", - "md-5 0.10.6", + "md-5", "pin-project-lite 0.2.10", "sha1", - "sha2 0.10.8", + "sha2", "tracing", ] @@ -640,11 +668,31 @@ version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a85e16fa903c70c49ab3785e5f4ac2ad2171b36e0616f321011fa57962404bb6" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 1.1.1", "bytes", "crc32fast", ] +[[package]] +name = "aws-smithy-http" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" +dependencies = [ + "aws-smithy-types 0.55.3", + "bytes", + "bytes-utils", + "futures-core", + "http", + "http-body", + "hyper", + "once_cell", + "percent-encoding", + "pin-project-lite 0.2.10", + "pin-utils", + "tracing", +] + [[package]] name = "aws-smithy-http" version = "0.60.1" @@ -653,7 +701,7 @@ checksum = "e4e816425a6b9caea4929ac97d0cb33674849bd5f0086418abc0d02c63f7a1bf" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "bytes", "bytes-utils", "futures-core", @@ -672,7 +720,7 @@ version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ab3f6d49e08df2f8d05e1bb5b68998e1e67b76054d3c43e7b954becb9a5e9ac" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 1.1.1", ] [[package]] @@ -681,7 +729,7 @@ version = "0.60.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f94a7a3aa509ff9e8b8d80749851d04e5eee0954c43f2e7d6396c4740028737" dependencies = [ - "aws-smithy-types", + "aws-smithy-types 1.1.1", "urlencoding", ] @@ -692,16 +740,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8da5b0a3617390e769576321816112f711c13d7e1114685e022505cf51fe5e48" dependencies = [ "aws-smithy-async", - "aws-smithy-http", + "aws-smithy-http 0.60.1", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "bytes", "fastrand 2.0.1", "h2", "http", "http-body", "hyper", - "hyper-rustls 0.24.1", + "hyper-rustls", "once_cell", "pin-project-lite 0.2.10", "pin-utils", @@ -717,7 +765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2404c9eb08bfe9af255945254d9afc69a367b7ee008b8db75c05e3bca485fc65" dependencies = [ "aws-smithy-async", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "bytes", "http", "pin-project-lite 0.2.10", @@ -726,6 +774,19 @@ dependencies = [ "zeroize", ] +[[package]] +name = "aws-smithy-types" +version = "0.55.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" +dependencies = [ + "base64-simd", + "itoa", + "num-integer", + "ryu", + "time 0.3.23", +] + [[package]] name = "aws-smithy-types" version = "1.1.1" @@ -767,7 +828,7 @@ dependencies = [ "aws-credential-types", "aws-smithy-async", "aws-smithy-runtime-api", - "aws-smithy-types", + "aws-smithy-types 1.1.1", "http", "rustc_version", "tracing", @@ -861,16 +922,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest 0.10.7", -] - -[[package]] -name = "block-buffer" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" -dependencies = [ - "generic-array", + "digest", ] [[package]] @@ -971,7 +1023,6 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", - "serde", "time 0.1.45", "wasm-bindgen", "winapi", @@ -1173,16 +1224,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "crypto-mac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "data-encoding" version = "2.4.0" @@ -1224,47 +1265,17 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "digest" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" -dependencies = [ - "generic-array", -] - [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer 0.10.4", + "block-buffer", "crypto-common", "subtle", ] -[[package]] -name = "dirs-next" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" -dependencies = [ - "cfg-if", - "dirs-sys-next", -] - -[[package]] -name = "dirs-sys-next" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" -dependencies = [ - "libc", - "redox_users", - "winapi", -] - [[package]] name = "displaydoc" version = "0.2.4" @@ -1322,7 +1333,7 @@ dependencies = [ "base16ct", "crypto-bigint 0.4.9", "der", - "digest 0.10.7", + "digest", "ff", "generic-array", "group", @@ -1642,23 +1653,13 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" -[[package]] -name = "hmac" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" -dependencies = [ - "crypto-mac", - "digest 0.9.0", -] - [[package]] name = "hmac" version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest 0.10.7", + "digest", ] [[package]] @@ -1719,21 +1720,6 @@ dependencies = [ "want", ] -[[package]] -name = "hyper-rustls" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" -dependencies = [ - "http", - "hyper", - "log", - "rustls 0.20.8", - "rustls-native-certs", - "tokio", - "tokio-rustls 0.23.4", -] - [[package]] name = "hyper-rustls" version = "0.24.1" @@ -1885,18 +1871,20 @@ dependencies = [ [[package]] name = "k2v-client" -version = "0.1.1" -source = "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?tag=v0.8.2#000006d689d2d8075599fbe1653605566ae9d36e" +version = "0.0.4" +source = "git+https://git.deuxfleurs.fr/Deuxfleurs/garage.git?tag=v0.9.0#952c9570c494468643353ee1ae9052b510353665" dependencies = [ + "aws-sigv4 0.55.3", "base64 0.21.2", + "hex", "http", - "hyper-rustls 0.23.2", + "hyper", + "hyper-rustls", "log", - "rusoto_core", - "rusoto_credential", - "rusoto_signature", + "percent-encoding", "serde", "serde_json", + "sha2", "thiserror", "tokio", ] @@ -2016,17 +2004,6 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" -[[package]] -name = "md-5" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" -dependencies = [ - "block-buffer 0.9.0", - "digest 0.9.0", - "opaque-debug", -] - [[package]] name = "md-5" version = "0.10.6" @@ -2034,7 +2011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest 0.10.7", + "digest", ] [[package]] @@ -2215,12 +2192,6 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" -[[package]] -name = "opaque-debug" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" - [[package]] name = "openssl-probe" version = "0.1.5" @@ -2253,7 +2224,7 @@ checksum = "51f44edd08f51e2ade572f141051021c5af22677e42b7dd28a88155151c33594" dependencies = [ "ecdsa", "elliptic-curve", - "sha2 0.10.8", + "sha2", ] [[package]] @@ -2280,7 +2251,7 @@ checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.3.5", + "redox_syscall", "smallvec", "windows-targets", ] @@ -2473,40 +2444,43 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.16" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ "bitflags 1.3.2", ] [[package]] -name = "redox_syscall" -version = "0.3.5" +name = "regex" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ - "bitflags 1.3.2", + "aho-corasick", + "memchr", + "regex-automata 0.3.7", + "regex-syntax 0.7.5", ] [[package]] -name = "redox_users" -version = "0.4.3" +name = "regex-automata" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "getrandom", - "redox_syscall 0.2.16", - "thiserror", + "regex-syntax 0.6.29", ] [[package]] name = "regex-automata" -version = "0.1.10" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ - "regex-syntax", + "aho-corasick", + "memchr", + "regex-syntax 0.7.5", ] [[package]] @@ -2521,6 +2495,12 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" +[[package]] +name = "regex-syntax" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" + [[package]] name = "rfc6979" version = "0.3.1" @@ -2528,7 +2508,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7743f17af12fa0b03b803ba12cd6a8d9483a587e89c69445e3909655c0b9fabb" dependencies = [ "crypto-bigint 0.4.9", - "hmac 0.12.1", + "hmac", "zeroize", ] @@ -2604,75 +2584,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rusoto_core" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2" -dependencies = [ - "async-trait", - "base64 0.13.1", - "bytes", - "crc32fast", - "futures", - "http", - "hyper", - "hyper-rustls 0.23.2", - "lazy_static", - "log", - "rusoto_credential", - "rusoto_signature", - "rustc_version", - "serde", - "serde_json", - "tokio", - "xml-rs", -] - -[[package]] -name = "rusoto_credential" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05" -dependencies = [ - "async-trait", - "chrono", - "dirs-next", - "futures", - "hyper", - "serde", - "serde_json", - "shlex", - "tokio", - "zeroize", -] - -[[package]] -name = "rusoto_signature" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272" -dependencies = [ - "base64 0.13.1", - "bytes", - "chrono", - "digest 0.9.0", - "futures", - "hex", - "hmac 0.11.0", - "http", - "hyper", - "log", - "md-5 0.9.1", - "percent-encoding", - "pin-project-lite 0.2.10", - "rusoto_credential", - "rustc_version", - "serde", - "sha2 0.9.9", - "tokio", -] - [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2888,20 +2799,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", -] - -[[package]] -name = "sha2" -version = "0.9.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" -dependencies = [ - "block-buffer 0.9.0", - "cfg-if", - "cpufeatures", - "digest 0.9.0", - "opaque-debug", + "digest", ] [[package]] @@ -2912,7 +2810,7 @@ checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", - "digest 0.10.7", + "digest", ] [[package]] @@ -2924,12 +2822,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shlex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" - [[package]] name = "signal-hook" version = "0.3.17" @@ -2955,7 +2847,7 @@ version = "1.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74233d3b3b2f6d4b006dc19dee745e73e2a6bfb6f93607cd3b02bd5b00797d7c" dependencies = [ - "digest 0.10.7", + "digest", "rand_core", ] @@ -3012,7 +2904,7 @@ dependencies = [ "lazy_static", "nom 6.1.2", "pin-project", - "regex-automata", + "regex-automata 0.1.10", "serde", ] @@ -3802,12 +3694,6 @@ dependencies = [ "time 0.3.23", ] -[[package]] -name = "xml-rs" -version = "0.8.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47430998a7b5d499ccee752b41567bc3afc57e1327dc855b1a2aa44ce29b5fa1" - [[package]] name = "xmlparser" version = "0.13.6" diff --git a/Cargo.toml b/Cargo.toml index 0e606cf..ce8cc8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ tower = "0.4" imap-codec = { git = "https://github.com/superboum/imap-codec.git", branch = "v0.5.x" } chrono = { version = "0.4", default-features = false, features = ["alloc"] } -k2v-client = { git = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git", tag = "v0.8.2" } +k2v-client = { git = "https://git.deuxfleurs.fr/Deuxfleurs/garage.git", tag = "v0.9.0" } boitalettres = { git = "https://git.deuxfleurs.fr/quentin/boitalettres.git", branch = "expose-mydatetime" } smtp-message = { git = "http://github.com/Alexis211/kannader", branch = "feature/lmtp" } smtp-server = { git = "http://github.com/Alexis211/kannader", branch = "feature/lmtp" } diff --git a/src/storage/garage.rs b/src/storage/garage.rs index ec26e80..bc123b5 100644 --- a/src/storage/garage.rs +++ b/src/storage/garage.rs @@ -34,7 +34,7 @@ impl GarageBuilder { #[async_trait] impl IBuilder for GarageBuilder { async fn build(&self) -> Result { - let creds = s3::config::Credentials::new( + let s3_creds = s3::config::Credentials::new( self.conf.aws_access_key_id.clone(), self.conf.aws_secret_access_key.clone(), None, @@ -42,17 +42,35 @@ impl IBuilder for GarageBuilder { "aerogramme" ); - let config = aws_config::from_env() + let s3_config = aws_config::from_env() .region(aws_config::Region::new(self.conf.region.clone())) - .credentials_provider(creds) + .credentials_provider(s3_creds) .endpoint_url(self.conf.s3_endpoint.clone()) .load() .await; + let s3_client = aws_sdk_s3::Client::new(&s3_config); + + let k2v_config = k2v_client::K2vClientConfig { + endpoint: self.conf.k2v_endpoint.clone(), + region: self.conf.region.clone(), + aws_access_key_id: self.conf.aws_access_key_id.clone(), + aws_secret_access_key: self.conf.aws_secret_access_key.clone(), + bucket: self.conf.bucket.clone(), + user_agent: None, + }; + + let k2v_client = match k2v_client::K2vClient::new(k2v_config) { + Err(e) => { + tracing::error!("unable to build k2v client: {}", e); + return Err(StorageError::Internal); + } + Ok(v) => v, + }; - let s3_client = aws_sdk_s3::Client::new(&config); Ok(Box::new(GarageStore { - s3_bucket: self.conf.bucket.clone(), - s3: s3_client + bucket: self.conf.bucket.clone(), + s3: s3_client, + k2v: k2v_client, })) } fn unique(&self) -> UnicityBuffer { @@ -61,13 +79,73 @@ impl IBuilder for GarageBuilder { } pub struct GarageStore { - s3_bucket: String, + bucket: String, s3: s3::Client, + k2v: k2v_client::K2vClient, +} + +fn causal_to_row_val(row_ref: RowRef, causal_value: k2v_client::CausalValue) -> RowVal { + let new_row_ref = row_ref.with_causality(causal_value.causality.into()); + let row_values = causal_value.value.into_iter().map(|k2v_value| match k2v_value { + k2v_client::K2vValue::Tombstone => Alternative::Tombstone, + k2v_client::K2vValue::Value(v) => Alternative::Value(v), + }).collect::>(); + + RowVal { row_ref: new_row_ref, value: row_values } } #[async_trait] impl IStore for GarageStore { async fn row_fetch<'a>(&self, select: &Selector<'a>) -> Result, StorageError> { + let batch_op = match select { + Selector::Range { shard, sort_begin, sort_end } => vec![k2v_client::BatchReadOp { + partition_key: shard, + filter: k2v_client::Filter { + start: Some(sort_begin), + end: Some(sort_end), + ..k2v_client::Filter::default() + }, + ..k2v_client::BatchReadOp::default() + }], + Selector::List(row_ref_list) => row_ref_list.iter().map(|row_ref| k2v_client::BatchReadOp { + partition_key: &row_ref.uid.shard, + filter: k2v_client::Filter { + start: Some(&row_ref.uid.sort), + ..k2v_client::Filter::default() + }, + single_item: true, + ..k2v_client::BatchReadOp::default() + }).collect::>(), + Selector::Prefix { shard, sort_prefix } => vec![k2v_client::BatchReadOp { + partition_key: shard, + filter: k2v_client::Filter { + prefix: Some(sort_prefix), + ..k2v_client::Filter::default() + }, + ..k2v_client::BatchReadOp::default() + }], + Selector::Single(row_ref) => { + let causal_value = match self.k2v.read_item(&row_ref.uid.shard, &row_ref.uid.sort).await { + Err(e) => { + tracing::error!("K2V read item shard={}, sort={}, bucket={} failed: {}", row_ref.uid.shard, row_ref.uid.sort, self.bucket, e); + return Err(StorageError::Internal); + }, + Ok(v) => v, + }; + + let row_val = causal_to_row_val((*row_ref).clone(), causal_value); + return Ok(vec![row_val]) + }, + }; + + let all_res = match self.k2v.read_batch(&batch_op).await { + Err(e) => { + tracing::error!("k2v read batch failed for {:?}, bucket {} with err: {}", select, self.bucket, e); + return Err(StorageError::Internal); + }, + Ok(v) => v, + }; + unimplemented!(); } async fn row_rm<'a>(&self, select: &Selector<'a>) -> Result<(), StorageError> { @@ -89,7 +167,7 @@ impl IStore for GarageStore { async fn blob_fetch(&self, blob_ref: &BlobRef) -> Result { let maybe_out = self.s3 .get_object() - .bucket(self.s3_bucket.to_string()) + .bucket(self.bucket.to_string()) .key(blob_ref.0.to_string()) .send() .await; @@ -117,7 +195,7 @@ impl IStore for GarageStore { } }; - tracing::debug!("Fetched {}/{}", self.s3_bucket, blob_ref.0); + tracing::debug!("Fetched {}/{}", self.bucket, blob_ref.0); Ok(BlobVal::new(blob_ref.clone(), buffer)) } async fn blob_insert(&self, blob_val: BlobVal) -> Result<(), StorageError> { @@ -125,7 +203,7 @@ impl IStore for GarageStore { let maybe_send = self.s3 .put_object() - .bucket(self.s3_bucket.to_string()) + .bucket(self.bucket.to_string()) .key(blob_val.blob_ref.0.to_string()) .body(streamable_value) .send() @@ -137,7 +215,7 @@ impl IStore for GarageStore { Err(StorageError::Internal) } Ok(_) => { - tracing::debug!("Inserted {}/{}", self.s3_bucket, blob_val.blob_ref.0); + tracing::debug!("Inserted {}/{}", self.bucket, blob_val.blob_ref.0); Ok(()) } } @@ -145,19 +223,19 @@ impl IStore for GarageStore { async fn blob_copy(&self, src: &BlobRef, dst: &BlobRef) -> Result<(), StorageError> { let maybe_copy = self.s3 .copy_object() - .bucket(self.s3_bucket.to_string()) + .bucket(self.bucket.to_string()) .key(dst.0.clone()) - .copy_source(format!("/{}/{}", self.s3_bucket.to_string(), src.0.clone())) + .copy_source(format!("/{}/{}", self.bucket.to_string(), src.0.clone())) .send() .await; match maybe_copy { Err(e) => { - tracing::error!("unable to copy object {} to {} (bucket: {}), error: {}", src.0, dst.0, self.s3_bucket, e); + tracing::error!("unable to copy object {} to {} (bucket: {}), error: {}", src.0, dst.0, self.bucket, e); Err(StorageError::Internal) }, Ok(_) => { - tracing::debug!("copied {} to {} (bucket: {})", src.0, dst.0, self.s3_bucket); + tracing::debug!("copied {} to {} (bucket: {})", src.0, dst.0, self.bucket); Ok(()) } } @@ -166,7 +244,7 @@ impl IStore for GarageStore { async fn blob_list(&self, prefix: &str) -> Result, StorageError> { let maybe_list = self.s3 .list_objects_v2() - .bucket(self.s3_bucket.to_string()) + .bucket(self.bucket.to_string()) .prefix(prefix) .into_paginator() .send() @@ -175,7 +253,7 @@ impl IStore for GarageStore { match maybe_list { Err(e) => { - tracing::error!("listing prefix {} on bucket {} failed: {}", prefix, self.s3_bucket, e); + tracing::error!("listing prefix {} on bucket {} failed: {}", prefix, self.bucket, e); Err(StorageError::Internal) } Ok(pagin_list_out) => Ok(pagin_list_out @@ -189,18 +267,18 @@ impl IStore for GarageStore { async fn blob_rm(&self, blob_ref: &BlobRef) -> Result<(), StorageError> { let maybe_delete = self.s3 .delete_object() - .bucket(self.s3_bucket.to_string()) + .bucket(self.bucket.to_string()) .key(blob_ref.0.clone()) .send() .await; match maybe_delete { Err(e) => { - tracing::error!("unable to delete {} (bucket: {}), error {}", blob_ref.0, self.s3_bucket, e); + tracing::error!("unable to delete {} (bucket: {}), error {}", blob_ref.0, self.bucket, e); Err(StorageError::Internal) }, Ok(_) => { - tracing::debug!("deleted {} (bucket: {})", blob_ref.0, self.s3_bucket); + tracing::debug!("deleted {} (bucket: {})", blob_ref.0, self.bucket); Ok(()) } } -- cgit v1.2.3