diff options
| -rw-r--r-- | Cargo.lock | 119 | ||||
| -rw-r--r-- | src/api/Cargo.toml | 6 | ||||
| -rw-r--r-- | src/garage/Cargo.toml | 8 | ||||
| -rw-r--r-- | src/model/Cargo.toml | 5 | ||||
| -rw-r--r-- | src/table/Cargo.toml | 2 | ||||
| -rw-r--r-- | src/table/lib.rs | 2 | ||||
| -rw-r--r-- | src/table/schema.rs | 77 | ||||
| -rw-r--r-- | src/table/table.rs | 92 |
8 files changed, 220 insertions, 91 deletions
@@ -340,15 +340,15 @@ dependencies = [ [[package]] name = "garage" -version = "0.1.0" +version = "0.1.1" dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "garage_api 0.1.0", - "garage_model 0.1.0", + "garage_api 0.1.1", + "garage_model 0.1.1", "garage_rpc 0.1.0", - "garage_table 0.1.0", + "garage_table 0.1.1", "garage_util 0.1.0", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -365,15 +365,15 @@ dependencies = [ [[package]] name = "garage_api" -version = "0.1.0" +version = "0.1.1" dependencies = [ "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "chrono 0.4.13 (registry+https://github.com/rust-lang/crates.io-index)", "crypto-mac 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "garage_model 0.1.0", - "garage_table 0.1.0", + "garage_model 0.1.1", + "garage_table 0.1.1", "garage_util 0.1.0", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hmac 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -392,14 +392,39 @@ dependencies = [ [[package]] name = "garage_model" version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arc-swap 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "async-trait 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_rpc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_table 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_bytes 0.11.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "garage_model" +version = "0.1.1" dependencies = [ "arc-swap 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "async-trait 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_model 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "garage_rpc 0.1.0", - "garage_table 0.1.0", + "garage_table 0.1.1", "garage_util 0.1.0", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -439,8 +464,57 @@ dependencies = [ ] [[package]] +name = "garage_rpc" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arc-swap 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gethostname 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper-rustls 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio-rustls 0.13.1 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "garage_table" version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "arc-swap 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", + "async-trait 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_rpc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "garage_util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_bytes 0.11.5 (registry+https://github.com/rust-lang/crates.io-index)", + "sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "garage_table" +version = "0.1.1" dependencies = [ "arc-swap 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", "async-trait 0.1.36 (registry+https://github.com/rust-lang/crates.io-index)", @@ -484,6 +558,31 @@ dependencies = [ ] [[package]] +name = "garage_util" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "err-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", + "futures 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", + "roxmltree 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.114 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.56 (registry+https://github.com/rust-lang/crates.io-index)", + "sha2 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", + "sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", + "webpki 0.21.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "generator" version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1629,6 +1728,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" "checksum futures-task 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "bdb66b5f09e22019b1ab0830f7785bcea8e7a42148683f99214f73f8ec21a626" "checksum futures-util 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "8764574ff08b701a084482c3c7031349104b07ac897393010494beaa18ce32c6" "checksum fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +"checksum garage_model 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b6403312b28077fd585e5f96184d2d56142f10eb42709d2893294d70a892d73f" +"checksum garage_rpc 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0ef794407808c89f300f73457bdc567c953b323d7fe30534aba686a28ebb4a0d" +"checksum garage_table 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "567d7e9f791a1d65d26e36840aa0570b3a2144302c32f62b5e63a39f147cbf57" +"checksum garage_util 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "77de76a4167c041094f3f3415c6d3d773373e0326668fbce70dfd3b024788800" "checksum generator 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "add72f17bb81521258fcc8a7a3245b1e184e916bfbe34f0ea89558f440df5c68" "checksum generic-array 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c68f0274ae0e023facc3c97b2e00f076be70e254bc851d972503b328db79b2ec" "checksum gethostname 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index f9cd32b2..c8c5bf22 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api" -version = "0.1.0" +version = "0.1.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "GPL-3.0" @@ -14,8 +14,8 @@ path = "lib.rs" [dependencies] garage_util = { version = "0.1", path = "../util" } -garage_table = { version = "0.1", path = "../table" } -garage_model = { version = "0.1", path = "../model" } +garage_table = { version = "0.1.1", path = "../table" } +garage_model = { version = "0.1.1", path = "../model" } bytes = "0.4" hex = "0.3" diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 8e64cf34..cb16bcd4 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.1.0" +version = "0.1.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "GPL-3.0" @@ -16,9 +16,9 @@ path = "main.rs" [dependencies] garage_util = { version = "0.1", path = "../util" } garage_rpc = { version = "0.1", path = "../rpc" } -garage_table = { version = "0.1", path = "../table" } -garage_model = { version = "0.1", path = "../model" } -garage_api = { version = "0.1", path = "../api" } +garage_table = { version = "0.1.1", path = "../table" } +garage_model = { version = "0.1.1", path = "../model" } +garage_api = { version = "0.1.1", path = "../api" } bytes = "0.4" rand = "0.7" diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 76d759fd..a138691d 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.1.0" +version = "0.1.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "GPL-3.0" @@ -15,7 +15,8 @@ path = "lib.rs" [dependencies] garage_util = { version = "0.1", path = "../util" } garage_rpc = { version = "0.1", path = "../rpc" } -garage_table = { version = "0.1", path = "../table" } +garage_table = { version = "0.1.1", path = "../table" } +model010 = { package = "garage_model", version = "0.1.0" } bytes = "0.4" rand = "0.7" diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 7efdbb08..1963f3da 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "0.1.0" +version = "0.1.1" authors = ["Alex Auvolat <alex@adnab.me>"] edition = "2018" license = "GPL-3.0" diff --git a/src/table/lib.rs b/src/table/lib.rs index f490b491..e30a6665 100644 --- a/src/table/lib.rs +++ b/src/table/lib.rs @@ -3,9 +3,11 @@ #[macro_use] extern crate log; +pub mod schema; pub mod table; pub mod table_fullcopy; pub mod table_sharded; pub mod table_sync; pub use table::*; +pub use schema::*; diff --git a/src/table/schema.rs b/src/table/schema.rs new file mode 100644 index 00000000..cedaacac --- /dev/null +++ b/src/table/schema.rs @@ -0,0 +1,77 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; + +use garage_util::data::*; +use garage_util::error::Error; + + +pub trait PartitionKey { + fn hash(&self) -> Hash; +} + +pub trait SortKey { + fn sort_key(&self) -> &[u8]; +} + +pub trait Entry<P: PartitionKey, S: SortKey>: + PartialEq + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync +{ + fn partition_key(&self) -> &P; + fn sort_key(&self) -> &S; + + fn merge(&mut self, other: &Self); +} + +#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EmptyKey; +impl SortKey for EmptyKey { + fn sort_key(&self) -> &[u8] { + &[] + } +} +impl PartitionKey for EmptyKey { + fn hash(&self) -> Hash { + [0u8; 32].into() + } +} + +impl PartitionKey for String { + fn hash(&self) -> Hash { + hash(self.as_bytes()) + } +} +impl SortKey for String { + fn sort_key(&self) -> &[u8] { + self.as_bytes() + } +} + +impl PartitionKey for Hash { + fn hash(&self) -> Hash { + self.clone() + } +} +impl SortKey for Hash { + fn sort_key(&self) -> &[u8] { + self.as_slice() + } +} + +#[async_trait] +pub trait TableSchema: Send + Sync { + type P: PartitionKey + Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync; + type S: SortKey + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync; + type E: Entry<Self::P, Self::S>; + type Filter: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync; + + // Action to take if not able to decode current version: + // try loading from an older version + fn try_migrate(_bytes: &[u8]) -> Option<Self::E> { + None + } + + async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error>; + fn matches_filter(_entry: &Self::E, _filter: &Self::Filter) -> bool { + true + } +} diff --git a/src/table/table.rs b/src/table/table.rs index 94bacc60..7a5caf4f 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -3,7 +3,6 @@ use std::sync::Arc; use std::time::Duration; use arc_swap::ArcSwapOption; -use async_trait::async_trait; use futures::stream::*; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; @@ -16,6 +15,7 @@ use garage_rpc::rpc_client::*; use garage_rpc::rpc_server::*; use crate::table_sync::*; +use crate::schema::*; const TABLE_RPC_TIMEOUT: Duration = Duration::from_secs(10); @@ -48,70 +48,6 @@ pub enum TableRPC<F: TableSchema> { impl<F: TableSchema> RpcMessage for TableRPC<F> {} -pub trait PartitionKey { - fn hash(&self) -> Hash; -} - -pub trait SortKey { - fn sort_key(&self) -> &[u8]; -} - -pub trait Entry<P: PartitionKey, S: SortKey>: - PartialEq + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync -{ - fn partition_key(&self) -> &P; - fn sort_key(&self) -> &S; - - fn merge(&mut self, other: &Self); -} - -#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct EmptyKey; -impl SortKey for EmptyKey { - fn sort_key(&self) -> &[u8] { - &[] - } -} -impl PartitionKey for EmptyKey { - fn hash(&self) -> Hash { - [0u8; 32].into() - } -} - -impl PartitionKey for String { - fn hash(&self) -> Hash { - hash(self.as_bytes()) - } -} -impl SortKey for String { - fn sort_key(&self) -> &[u8] { - self.as_bytes() - } -} - -impl PartitionKey for Hash { - fn hash(&self) -> Hash { - self.clone() - } -} -impl SortKey for Hash { - fn sort_key(&self) -> &[u8] { - self.as_slice() - } -} - -#[async_trait] -pub trait TableSchema: Send + Sync { - type P: PartitionKey + Clone + PartialEq + Serialize + for<'de> Deserialize<'de> + Send + Sync; - type S: SortKey + Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync; - type E: Entry<Self::P, Self::S>; - type Filter: Clone + Serialize + for<'de> Deserialize<'de> + Send + Sync; - - async fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) -> Result<(), Error>; - fn matches_filter(_entry: &Self::E, _filter: &Self::Filter) -> bool { - true - } -} pub trait TableReplication: Send + Sync { // See examples in table_sharded.rs and table_fullcopy.rs @@ -250,7 +186,7 @@ where for resp in resps { if let TableRPC::ReadEntryResponse(value) = resp { if let Some(v_bytes) = value { - let v = rmp_serde::decode::from_read_ref::<_, F::E>(v_bytes.as_slice())?; + let v = Self::decode_entry(v_bytes.as_slice())?; ret = match ret { None => Some(v), Some(mut x) => { @@ -306,8 +242,7 @@ where for resp in resps { if let TableRPC::Update(entries) = resp { for entry_bytes in entries.iter() { - let entry = - rmp_serde::decode::from_read_ref::<_, F::E>(entry_bytes.as_slice())?; + let entry = Self::decode_entry(entry_bytes.as_slice())?; let entry_key = self.tree_key(entry.partition_key(), entry.sort_key()); match ret.remove(&entry_key) { None => { @@ -429,7 +364,7 @@ where let keep = match filter { None => true, Some(f) => { - let entry = rmp_serde::decode::from_read_ref::<_, F::E>(value.as_ref())?; + let entry = Self::decode_entry(value.as_ref())?; F::matches_filter(&entry, f) } }; @@ -448,15 +383,14 @@ where let mut epidemic_propagate = vec![]; for update_bytes in entries.iter() { - let update = rmp_serde::decode::from_read_ref::<_, F::E>(update_bytes.as_slice())?; + let update = Self::decode_entry(update_bytes.as_slice())?; let tree_key = self.tree_key(update.partition_key(), update.sort_key()); let (old_entry, new_entry) = self.store.transaction(|db| { let (old_entry, new_entry) = match db.get(&tree_key)? { Some(prev_bytes) => { - let old_entry = rmp_serde::decode::from_read_ref::<_, F::E>(&prev_bytes) - .map_err(Error::RMPDecode) + let old_entry = Self::decode_entry(&prev_bytes) .map_err(sled::ConflictableTransactionError::Abort)?; let mut new_entry = old_entry.clone(); new_entry.merge(&update); @@ -504,7 +438,7 @@ where break; } if let Some(old_val) = self.store.remove(&key)? { - let old_entry = rmp_serde::decode::from_read_ref::<_, F::E>(&old_val)?; + let old_entry = Self::decode_entry(&old_val)?; self.instance.updated(Some(old_entry), None).await?; self.system .background @@ -521,4 +455,16 @@ where ret.extend(s.sort_key()); ret } + + fn decode_entry(bytes: &[u8]) -> Result<F::E, Error> { + match rmp_serde::decode::from_read_ref::<_, F::E>(bytes) { + Ok(x) => Ok(x), + Err(e) => { + match F::try_migrate(bytes) { + Some(x) => Ok(x), + None => Err(e.into()), + } + } + } + } } |