From 5768bf362262f78376af14517c4921941986192e Mon Sep 17 00:00:00 2001
From: Alex <alex@adnab.me>
Date: Tue, 10 May 2022 13:16:57 +0200
Subject: First implementation of K2V (#293)

**Specification:**

View spec at [this URL](https://git.deuxfleurs.fr/Deuxfleurs/garage/src/branch/k2v/doc/drafts/k2v-spec.md)

- [x] Specify the structure of K2V triples
- [x] Specify the DVVS format used for causality detection
- [x] Specify the K2V index (just a counter of number of values per partition key)
- [x] Specify single-item endpoints: ReadItem, InsertItem, DeleteItem
- [x] Specify index endpoint: ReadIndex
- [x] Specify multi-item endpoints: InsertBatch, ReadBatch, DeleteBatch
- [x] Move to JSON objects instead of tuples
- [x] Specify endpoints for polling for updates on single values (PollItem)

**Implementation:**

- [x] Table for K2V items, causal contexts
- [x] Indexing mechanism and table for K2V index
- [x] Make API handlers a bit more generic
- [x] K2V API endpoint
- [x] K2V API router
- [x] ReadItem
- [x] InsertItem
- [x] DeleteItem
- [x] PollItem
- [x] ReadIndex
- [x] InsertBatch
- [x] ReadBatch
- [x] DeleteBatch

**Testing:**

- [x] Just a simple Python script that does some requests to check visually that things are going right (does not contain parsing of results or assertions on returned values)
- [x] Actual tests:
  - [x] Adapt testing framework
  - [x] Simple test with InsertItem + ReadItem
  - [x] Test with several Insert/Read/DeleteItem + ReadIndex
  - [x] Test all combinations of return formats for ReadItem
  - [x] Test with ReadBatch, InsertBatch, DeleteBatch
  - [x] Test with PollItem
  - [x] Test error codes
- [ ] Fix most broken stuff
  - [x] test PollItem broken randomly
  - [x] when invalid causality tokens are given, errors should be 4xx not 5xx

**Improvements:**

- [x] Descending range queries
  - [x] Specify
  - [x] Implement
  - [x] Add test
- [x] Batch updates to index counter
- [x] Put K2V behind `k2v` feature flag

Co-authored-by: Alex Auvolat <alex@adnab.me>
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/293
Co-authored-by: Alex <alex@adnab.me>
Co-committed-by: Alex <alex@adnab.me>
---
 src/model/s3/block_ref_table.rs | 74 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 src/model/s3/block_ref_table.rs

(limited to 'src/model/s3/block_ref_table.rs')

diff --git a/src/model/s3/block_ref_table.rs b/src/model/s3/block_ref_table.rs
new file mode 100644
index 00000000..9b3991bf
--- /dev/null
+++ b/src/model/s3/block_ref_table.rs
@@ -0,0 +1,74 @@
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+
+use garage_util::data::*;
+
+use garage_table::crdt::Crdt;
+use garage_table::*;
+
+use garage_block::manager::*;
+
+#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
+pub struct BlockRef {
+	/// Hash (blake2 sum) of the block, used as partition key
+	pub block: Hash,
+
+	/// Id of the Version for the object containing this block, used as sorting key
+	pub version: Uuid,
+
+	// Keep track of deleted status
+	/// Is the Version that contains this block deleted
+	pub deleted: crdt::Bool,
+}
+
+impl Entry<Hash, Uuid> for BlockRef {
+	fn partition_key(&self) -> &Hash {
+		&self.block
+	}
+	fn sort_key(&self) -> &Uuid {
+		&self.version
+	}
+	fn is_tombstone(&self) -> bool {
+		self.deleted.get()
+	}
+}
+
+impl Crdt for BlockRef {
+	fn merge(&mut self, other: &Self) {
+		self.deleted.merge(&other.deleted);
+	}
+}
+
+pub struct BlockRefTable {
+	pub block_manager: Arc<BlockManager>,
+}
+
+impl TableSchema for BlockRefTable {
+	const TABLE_NAME: &'static str = "block_ref";
+
+	type P = Hash;
+	type S = Uuid;
+	type E = BlockRef;
+	type Filter = DeletedFilter;
+
+	fn updated(&self, old: Option<&Self::E>, new: Option<&Self::E>) {
+		#[allow(clippy::or_fun_call)]
+		let block = &old.or(new).unwrap().block;
+		let was_before = old.map(|x| !x.deleted.get()).unwrap_or(false);
+		let is_after = new.map(|x| !x.deleted.get()).unwrap_or(false);
+		if is_after && !was_before {
+			if let Err(e) = self.block_manager.block_incref(block) {
+				warn!("block_incref failed for block {:?}: {}", block, e);
+			}
+		}
+		if was_before && !is_after {
+			if let Err(e) = self.block_manager.block_decref(block) {
+				warn!("block_decref failed for block {:?}: {}", block, e);
+			}
+		}
+	}
+
+	fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool {
+		filter.apply(entry.deleted.get())
+	}
+}
-- 
cgit v1.2.3