From 64c193e3dbb536d5d3c2881bc9aebbb3e4e6272e Mon Sep 17 00:00:00 2001 From: trinity-1686a Date: Wed, 18 May 2022 22:24:09 +0200 Subject: Add a K2V client library and CLI (#303) lib.rs could use getting split in modules, but I'm not sure how exactly Co-authored-by: trinity-1686a Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/303 Co-authored-by: trinity-1686a Co-committed-by: trinity-1686a --- Cargo.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'Cargo.toml') diff --git a/Cargo.toml b/Cargo.toml index cfc48113..c64f7897 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,9 +8,12 @@ members = [ "src/admin", "src/api", "src/web", - "src/garage" + "src/garage", + "src/k2v-client", ] +default-members = ["src/garage"] + [profile.dev] lto = "off" -- cgit v1.2.3 From 382e74c798263d042b1c6ca3788c866a8c69c4f4 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 24 May 2022 12:16:39 +0200 Subject: First version of admin API (#298) **Spec:** - [x] Start writing - [x] Specify all layout endpoints - [x] Specify all endpoints for operations on keys - [x] Specify all endpoints for operations on key/bucket permissions - [x] Specify all endpoints for operations on buckets - [x] Specify all endpoints for operations on bucket aliases View rendered spec at **Code:** - [x] Refactor code for admin api to use common api code that was created for K2V **General endpoints:** - [x] Metrics - [x] GetClusterStatus - [x] ConnectClusterNodes - [x] GetClusterLayout - [x] UpdateClusterLayout - [x] ApplyClusterLayout - [x] RevertClusterLayout **Key-related endpoints:** - [x] ListKeys - [x] CreateKey - [x] ImportKey - [x] GetKeyInfo - [x] UpdateKey - [x] DeleteKey **Bucket-related endpoints:** - [x] ListBuckets - [x] CreateBucket - [x] GetBucketInfo - [x] DeleteBucket - [x] PutBucketWebsite - [x] DeleteBucketWebsite **Operations on key/bucket permissions:** - [x] BucketAllowKey - [x] BucketDenyKey **Operations on bucket aliases:** - [x] GlobalAliasBucket - [x] GlobalUnaliasBucket - [x] LocalAliasBucket - [x] LocalUnaliasBucket **And also:** - [x] Separate error type for the admin API (this PR includes a quite big refactoring of error handling) - [x] Add management of website access - [ ] Check that nothing is missing wrt what can be done using the CLI - [ ] Improve formatting of the spec - [x] Make sure everyone is cool with the API design Fix #231 Fix #295 Co-authored-by: Alex Auvolat Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/298 Co-authored-by: Alex Co-committed-by: Alex --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) (limited to 'Cargo.toml') diff --git a/Cargo.toml b/Cargo.toml index c64f7897..edd0e3f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ members = [ "src/table", "src/block", "src/model", - "src/admin", "src/api", "src/web", "src/garage", -- cgit v1.2.3 From b44d3fc796484a50cd6854f20c9b46e5fddedc9d Mon Sep 17 00:00:00 2001 From: Alex Date: Wed, 8 Jun 2022 10:01:44 +0200 Subject: Abstract database behind generic interface and implement alternative drivers (#322) - [x] Design interface - [x] Implement Sled backend - [x] Re-implement the SledCountedTree hack ~~on Sled backend~~ on all backends (i.e. over the abstraction) - [x] Convert Garage code to use generic interface - [x] Proof-read converted Garage code - [ ] Test everything well - [x] Implement sqlite backend - [x] Implement LMDB backend - [ ] (Implement Persy backend?) - [ ] (Implement other backends? (like RocksDB, ...)) - [x] Implement backend choice in config file and garage server module - [x] Add CLI for converting between DB formats - Exploit the new interface to put more things in transactions - [x] `.updated()` trigger on Garage tables Fix #284 **Bugs** - [x] When exporting sqlite, trees iterate empty?? - [x] LMDB doesn't work **Known issues for various back-ends** - Sled: - Eats all my RAM and also all my disk space - `.len()` has to traverse the whole table - Is actually quite slow on some operations - And is actually pretty bad code... - Sqlite: - Requires a lock to be taken on all operations. The lock is also taken when iterating on a table with `.iter()`, and the lock isn't released until the iterator is dropped. This means that we must be VERY carefull to not do anything else inside a `.iter()` loop or else we will have a deadlock! Most such cases have been eliminated from the Garage codebase, but there might still be some that remain. If your Garage-over-Sqlite seems to hang/freeze, this is the reason. - (adapter uses a bunch of unsafe code) - Heed (LMDB): - Not suited for 32-bit machines as it has to map the whole DB in memory. - (adpater uses a tiny bit of unsafe code) **My recommendation:** avoid 32-bit machines and use LMDB as much as possible. **Converting databases** is actually quite easy. For example from Sled to LMDB: ```bash cd src/db cargo run --features cli --bin convert -- -i path/to/garage/meta/db -a sled -o path/to/garage/meta/db.lmdb -b lmdb ``` Then, just add this to your `config.toml`: ```toml db_engine = "lmdb" ``` Co-authored-by: Alex Auvolat Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/322 Co-authored-by: Alex Co-committed-by: Alex --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) (limited to 'Cargo.toml') diff --git a/Cargo.toml b/Cargo.toml index edd0e3f9..122285db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "src/db", "src/util", "src/rpc", "src/table", -- cgit v1.2.3 From db72812f01027ab2abd2226a0edaf3161f32e274 Mon Sep 17 00:00:00 2001 From: Jakub Jirutka Date: Sun, 4 Sep 2022 15:26:19 +0200 Subject: Use the new cargo feature resolver "2" Garage currently uses the legacy resolver "1". The new one is used by default if the root package specifies 'edition = 2021', which Garage does not (yet). The problem with the legacy resolver is, among others, that features enabled by dev-dependencies are propagated to normal dependencies. This affects e.g. hyper - one of the dev-dependencies enables "http2" feature that adds many extra dependencies. If we build garage without opentelemetry-otlp (this is enabled in the following commit), there's no normal dependency enabling "http2" feature. See https://doc.rust-lang.org/cargo/reference/resolver.html#feature-resolver-version-2 --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) (limited to 'Cargo.toml') diff --git a/Cargo.toml b/Cargo.toml index 122285db..a9fd4423 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" members = [ "src/db", "src/util", -- cgit v1.2.3