From 5b1117e582db16cc5aa50840a685875cbd5501f4 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 14 Dec 2021 13:55:11 +0100 Subject: New model for buckets --- src/model/Cargo.toml | 9 ++-- src/model/bucket_alias_table.rs | 68 +++++++++++++++++++++++++++ src/model/bucket_helper.rs | 41 ++++++++++++++++ src/model/bucket_table.rs | 94 ++++++++++++++++++------------------ src/model/garage.rs | 17 +++++++ src/model/key_table.rs | 102 +++++++++++++++++++++++----------------- src/model/lib.rs | 3 ++ src/model/object_table.rs | 16 +++---- src/model/permission.rs | 37 +++++++++++++++ src/model/version_table.rs | 12 ++--- 10 files changed, 291 insertions(+), 108 deletions(-) create mode 100644 src/model/bucket_alias_table.rs create mode 100644 src/model/bucket_helper.rs create mode 100644 src/model/permission.rs (limited to 'src/model') diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 1d695192..12c08719 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.5.0" +version = "0.6.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,9 +14,10 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_rpc = { version = "0.5.0", path = "../rpc" } -garage_table = { version = "0.5.0", path = "../table" } -garage_util = { version = "0.5.0", path = "../util" } +garage_rpc = { version = "0.6.0", path = "../rpc" } +garage_table = { version = "0.6.0", path = "../table" } +garage_util = { version = "0.6.0", path = "../util" } +garage_model_050 = { package = "garage_model", version = "0.5.0" } async-trait = "0.1.7" arc-swap = "1.0" diff --git a/src/model/bucket_alias_table.rs b/src/model/bucket_alias_table.rs new file mode 100644 index 00000000..4d300d05 --- /dev/null +++ b/src/model/bucket_alias_table.rs @@ -0,0 +1,68 @@ +use serde::{Deserialize, Serialize}; + +use garage_table::crdt::*; +use garage_table::*; +use garage_util::data::*; + +/// The bucket alias table holds the names given to buckets +/// in the global namespace. +#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] +pub struct BucketAlias { + pub name: String, + pub state: crdt::Lww>, +} + +#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Debug, Serialize, Deserialize)] +pub struct AliasParams { + pub bucket_id: Uuid, + pub website_access: bool, +} + +impl AutoCrdt for AliasParams { + const WARN_IF_DIFFERENT: bool = true; +} + +impl BucketAlias { + pub fn new(name: String, bucket_id: Uuid, website_access: bool) -> Self { + BucketAlias { + name, + state: crdt::Lww::new(crdt::Deletable::present(AliasParams { + bucket_id, + website_access, + })), + } + } + pub fn is_deleted(&self) -> bool { + self.state.get().is_deleted() + } +} + +impl Crdt for BucketAlias { + fn merge(&mut self, o: &Self) { + self.state.merge(&o.state); + } +} + +impl Entry for BucketAlias { + fn partition_key(&self) -> &EmptyKey { + &EmptyKey + } + fn sort_key(&self) -> &String { + &self.name + } +} + +pub struct BucketAliasTable; + +impl TableSchema for BucketAliasTable { + const TABLE_NAME: &'static str = "bucket_alias"; + + type P = EmptyKey; + type S = String; + type E = BucketAlias; + type Filter = DeletedFilter; + + fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { + filter.apply(entry.is_deleted()) + } +} diff --git a/src/model/bucket_helper.rs b/src/model/bucket_helper.rs new file mode 100644 index 00000000..e0720b4e --- /dev/null +++ b/src/model/bucket_helper.rs @@ -0,0 +1,41 @@ +use garage_util::data::*; +use garage_util::error::*; + +use garage_table::util::EmptyKey; + +use crate::bucket_table::Bucket; +use crate::garage::Garage; + +pub struct BucketHelper<'a>(pub(crate) &'a Garage); + +#[allow(clippy::ptr_arg)] +impl<'a> BucketHelper<'a> { + pub async fn resolve_global_bucket_name( + &self, + bucket_name: &String, + ) -> Result, Error> { + Ok(self + .0 + .bucket_alias_table + .get(&EmptyKey, bucket_name) + .await? + .map(|x| x.state.get().as_option().map(|x| x.bucket_id)) + .flatten()) + } + + #[allow(clippy::ptr_arg)] + pub async fn get_existing_bucket(&self, bucket_id: Uuid) -> Result { + self.0 + .bucket_table + .get(&bucket_id, &EmptyKey) + .await? + .filter(|b| !b.is_deleted()) + .map(Ok) + .unwrap_or_else(|| { + Err(Error::BadRpc(format!( + "Bucket {:?} does not exist", + bucket_id + ))) + }) + } +} diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index 2cb206ce..ac40407e 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -2,8 +2,10 @@ use serde::{Deserialize, Serialize}; use garage_table::crdt::Crdt; use garage_table::*; +use garage_util::data::*; +use garage_util::time::*; -use crate::key_table::PermissionSet; +use crate::permission::BucketKeyPerm; /// A bucket is a collection of objects /// @@ -12,49 +14,38 @@ use crate::key_table::PermissionSet; /// - A bucket has 2 states, Present or Deleted and parameters make sense only if present. #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Bucket { - /// Name of the bucket - pub name: String, + /// ID of the bucket + pub id: Uuid, /// State, and configuration if not deleted, of the bucket - pub state: crdt::Lww, -} - -/// State of a bucket -#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] -pub enum BucketState { - /// The bucket is deleted - Deleted, - /// The bucket exists - Present(BucketParams), -} - -impl Crdt for BucketState { - fn merge(&mut self, o: &Self) { - match o { - BucketState::Deleted => *self = BucketState::Deleted, - BucketState::Present(other_params) => { - if let BucketState::Present(params) = self { - params.merge(other_params); - } - } - } - } + pub state: crdt::Deletable, } /// Configuration for a bucket #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct BucketParams { + /// Bucket's creation date + pub creation_date: u64, /// Map of key with access to the bucket, and what kind of access they give - pub authorized_keys: crdt::LwwMap, - /// Is the bucket served as http - pub website: crdt::Lww, + pub authorized_keys: crdt::Map, + /// Map of aliases that are or have been given to this bucket + /// in the global namespace + /// (not authoritative: this is just used as an indication to + /// map back to aliases when doing ListBuckets) + pub aliases: crdt::LwwMap, + /// Map of aliases that are or have been given to this bucket + /// in namespaces local to keys + /// key = (access key id, alias name) + pub local_aliases: crdt::LwwMap<(String, String), bool>, } impl BucketParams { /// Create an empty BucketParams with no authorized keys and no website accesss pub fn new() -> Self { BucketParams { - authorized_keys: crdt::LwwMap::new(), - website: crdt::Lww::new(false), + creation_date: now_msec(), + authorized_keys: crdt::Map::new(), + aliases: crdt::LwwMap::new(), + local_aliases: crdt::LwwMap::new(), } } } @@ -62,7 +53,14 @@ impl BucketParams { impl Crdt for BucketParams { fn merge(&mut self, o: &Self) { self.authorized_keys.merge(&o.authorized_keys); - self.website.merge(&o.website); + self.aliases.merge(&o.aliases); + self.local_aliases.merge(&o.local_aliases); + } +} + +impl Default for Bucket { + fn default() -> Self { + Self::new() } } @@ -74,34 +72,34 @@ impl Default for BucketParams { impl Bucket { /// Initializes a new instance of the Bucket struct - pub fn new(name: String) -> Self { + pub fn new() -> Self { Bucket { - name, - state: crdt::Lww::new(BucketState::Present(BucketParams::new())), + id: gen_uuid(), + state: crdt::Deletable::present(BucketParams::new()), } } /// Returns true if this represents a deleted bucket pub fn is_deleted(&self) -> bool { - *self.state.get() == BucketState::Deleted + self.state.is_deleted() } /// Return the list of authorized keys, when each was updated, and the permission associated to /// the key - pub fn authorized_keys(&self) -> &[(String, u64, PermissionSet)] { - match self.state.get() { - BucketState::Deleted => &[], - BucketState::Present(state) => state.authorized_keys.items(), + pub fn authorized_keys(&self) -> &[(String, BucketKeyPerm)] { + match &self.state { + crdt::Deletable::Deleted => &[], + crdt::Deletable::Present(state) => state.authorized_keys.items(), } } } -impl Entry for Bucket { - fn partition_key(&self) -> &EmptyKey { - &EmptyKey +impl Entry for Bucket { + fn partition_key(&self) -> &Uuid { + &self.id } - fn sort_key(&self) -> &String { - &self.name + fn sort_key(&self) -> &EmptyKey { + &EmptyKey } } @@ -114,10 +112,10 @@ impl Crdt for Bucket { pub struct BucketTable; impl TableSchema for BucketTable { - const TABLE_NAME: &'static str = "bucket"; + const TABLE_NAME: &'static str = "bucket_v2"; - type P = EmptyKey; - type S = String; + type P = Uuid; + type S = EmptyKey; type E = Bucket; type Filter = DeletedFilter; diff --git a/src/model/garage.rs b/src/model/garage.rs index a874cca8..9db1843c 100644 --- a/src/model/garage.rs +++ b/src/model/garage.rs @@ -14,6 +14,8 @@ use garage_table::*; use crate::block::*; use crate::block_ref_table::*; +use crate::bucket_alias_table::*; +use crate::bucket_helper::*; use crate::bucket_table::*; use crate::key_table::*; use crate::object_table::*; @@ -35,6 +37,8 @@ pub struct Garage { /// Table containing informations about buckets pub bucket_table: Arc>, + /// Table containing informations about bucket aliases + pub bucket_alias_table: Arc>, /// Table containing informations about api keys pub key_table: Arc>, @@ -120,6 +124,14 @@ impl Garage { info!("Initialize bucket_table..."); let bucket_table = Table::new(BucketTable, control_rep_param.clone(), system.clone(), &db); + info!("Initialize bucket_alias_table..."); + let bucket_alias_table = Table::new( + BucketAliasTable, + control_rep_param.clone(), + system.clone(), + &db, + ); + info!("Initialize key_table_table..."); let key_table = Table::new(KeyTable, control_rep_param, system.clone(), &db); @@ -131,6 +143,7 @@ impl Garage { system, block_manager, bucket_table, + bucket_alias_table, key_table, object_table, version_table, @@ -148,4 +161,8 @@ impl Garage { pub fn break_reference_cycles(&self) { self.block_manager.garage.swap(None); } + + pub fn bucket_helper(&self) -> BucketHelper { + BucketHelper(self) + } } diff --git a/src/model/key_table.rs b/src/model/key_table.rs index 225f51c7..e87f5949 100644 --- a/src/model/key_table.rs +++ b/src/model/key_table.rs @@ -2,6 +2,9 @@ use serde::{Deserialize, Serialize}; use garage_table::crdt::*; use garage_table::*; +use garage_util::data::*; + +use crate::permission::BucketKeyPerm; /// An api key #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] @@ -15,12 +18,39 @@ pub struct Key { /// Name for the key pub name: crdt::Lww, - /// Is the key deleted - pub deleted: crdt::Bool, + /// If the key is present: it gives some permissions, + /// a map of bucket IDs (uuids) to permissions. + /// Otherwise no permissions are granted to key + pub state: crdt::Deletable, +} + +/// Configuration for a key +#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] +pub struct KeyParams { + pub authorized_buckets: crdt::Map, + pub local_aliases: crdt::LwwMap>, +} + +impl KeyParams { + pub fn new() -> Self { + KeyParams { + authorized_buckets: crdt::Map::new(), + local_aliases: crdt::LwwMap::new(), + } + } +} + +impl Default for KeyParams { + fn default() -> Self { + Self::new() + } +} - /// Buckets in which the key is authorized. Empty if `Key` is deleted - // CRDT interaction: deleted implies authorized_buckets is empty - pub authorized_buckets: crdt::LwwMap, +impl Crdt for KeyParams { + fn merge(&mut self, o: &Self) { + self.authorized_buckets.merge(&o.authorized_buckets); + self.local_aliases.merge(&o.local_aliases); + } } impl Key { @@ -32,8 +62,7 @@ impl Key { key_id, secret_key, name: crdt::Lww::new(name), - deleted: crdt::Bool::new(false), - authorized_buckets: crdt::LwwMap::new(), + state: crdt::Deletable::present(KeyParams::new()), } } @@ -43,8 +72,7 @@ impl Key { key_id: key_id.to_string(), secret_key: secret_key.to_string(), name: crdt::Lww::new(name.to_string()), - deleted: crdt::Bool::new(false), - authorized_buckets: crdt::LwwMap::new(), + state: crdt::Deletable::present(KeyParams::new()), } } @@ -54,41 +82,37 @@ impl Key { key_id, secret_key: "".into(), name: crdt::Lww::new("".to_string()), - deleted: crdt::Bool::new(true), - authorized_buckets: crdt::LwwMap::new(), + state: crdt::Deletable::Deleted, } } /// Check if `Key` is allowed to read in bucket - pub fn allow_read(&self, bucket: &str) -> bool { - self.authorized_buckets - .get(&bucket.to_string()) - .map(|x| x.allow_read) - .unwrap_or(false) + pub fn allow_read(&self, bucket: &Uuid) -> bool { + if let crdt::Deletable::Present(params) = &self.state { + params + .authorized_buckets + .get(bucket) + .map(|x| x.allow_read) + .unwrap_or(false) + } else { + false + } } /// Check if `Key` is allowed to write in bucket - pub fn allow_write(&self, bucket: &str) -> bool { - self.authorized_buckets - .get(&bucket.to_string()) - .map(|x| x.allow_write) - .unwrap_or(false) + pub fn allow_write(&self, bucket: &Uuid) -> bool { + if let crdt::Deletable::Present(params) = &self.state { + params + .authorized_buckets + .get(bucket) + .map(|x| x.allow_write) + .unwrap_or(false) + } else { + false + } } } -/// Permission given to a key in a bucket -#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] -pub struct PermissionSet { - /// The key can be used to read the bucket - pub allow_read: bool, - /// The key can be used to write in the bucket - pub allow_write: bool, -} - -impl AutoCrdt for PermissionSet { - const WARN_IF_DIFFERENT: bool = true; -} - impl Entry for Key { fn partition_key(&self) -> &EmptyKey { &EmptyKey @@ -101,13 +125,7 @@ impl Entry for Key { impl Crdt for Key { fn merge(&mut self, other: &Self) { self.name.merge(&other.name); - self.deleted.merge(&other.deleted); - - if self.deleted.get() { - self.authorized_buckets.clear(); - } else { - self.authorized_buckets.merge(&other.authorized_buckets); - } + self.state.merge(&other.state); } } @@ -129,7 +147,7 @@ impl TableSchema for KeyTable { fn matches_filter(entry: &Self::E, filter: &Self::Filter) -> bool { match filter { - KeyFilter::Deleted(df) => df.apply(entry.deleted.get()), + KeyFilter::Deleted(df) => df.apply(entry.state.is_deleted()), KeyFilter::Matches(pat) => { let pat = pat.to_lowercase(); entry.key_id.to_lowercase().starts_with(&pat) diff --git a/src/model/lib.rs b/src/model/lib.rs index b4a8ddb7..fe8cfdad 100644 --- a/src/model/lib.rs +++ b/src/model/lib.rs @@ -3,8 +3,11 @@ extern crate log; pub mod block; pub mod block_ref_table; +pub mod bucket_alias_table; +pub mod bucket_helper; pub mod bucket_table; pub mod garage; pub mod key_table; pub mod object_table; +pub mod permission; pub mod version_table; diff --git a/src/model/object_table.rs b/src/model/object_table.rs index 9eec47ff..285cb5a7 100644 --- a/src/model/object_table.rs +++ b/src/model/object_table.rs @@ -15,7 +15,7 @@ use crate::version_table::*; #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Object { /// The bucket in which the object is stored, used as partition key - pub bucket: String, + pub bucket_id: Uuid, /// The key at which the object is stored in its bucket, used as sorting key pub key: String, @@ -26,9 +26,9 @@ pub struct Object { impl Object { /// Initialize an Object struct from parts - pub fn new(bucket: String, key: String, versions: Vec) -> Self { + pub fn new(bucket_id: Uuid, key: String, versions: Vec) -> Self { let mut ret = Self { - bucket, + bucket_id, key, versions: vec![], }; @@ -164,9 +164,9 @@ impl ObjectVersion { } } -impl Entry for Object { - fn partition_key(&self) -> &String { - &self.bucket +impl Entry for Object { + fn partition_key(&self) -> &Uuid { + &self.bucket_id } fn sort_key(&self) -> &String { &self.key @@ -219,7 +219,7 @@ pub struct ObjectTable { impl TableSchema for ObjectTable { const TABLE_NAME: &'static str = "object"; - type P = String; + type P = Uuid; type S = String; type E = Object; type Filter = DeletedFilter; @@ -242,7 +242,7 @@ impl TableSchema for ObjectTable { }; if newly_deleted { let deleted_version = - Version::new(v.uuid, old_v.bucket.clone(), old_v.key.clone(), true); + Version::new(v.uuid, old_v.bucket_id, old_v.key.clone(), true); version_table.insert(&deleted_version).await?; } } diff --git a/src/model/permission.rs b/src/model/permission.rs new file mode 100644 index 00000000..b61c92ce --- /dev/null +++ b/src/model/permission.rs @@ -0,0 +1,37 @@ +use std::cmp::Ordering; + +use serde::{Deserialize, Serialize}; + +use garage_util::crdt::*; + +/// Permission given to a key in a bucket +#[derive(PartialOrd, Ord, PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize)] +pub struct BucketKeyPerm { + /// Timestamp at which the permission was given + pub timestamp: u64, + + /// The key can be used to read the bucket + pub allow_read: bool, + /// The key can be used to write in the bucket + pub allow_write: bool, +} + +impl Crdt for BucketKeyPerm { + fn merge(&mut self, other: &Self) { + match other.timestamp.cmp(&self.timestamp) { + Ordering::Greater => { + *self = *other; + } + Ordering::Equal if other != self => { + warn!("Different permission sets with same timestamp: {:?} and {:?}, merging to most restricted permission set.", self, other); + if !other.allow_read { + self.allow_read = false; + } + if !other.allow_write { + self.allow_write = false; + } + } + _ => (), + } + } +} diff --git a/src/model/version_table.rs b/src/model/version_table.rs index 18ec8e1d..4edea0b7 100644 --- a/src/model/version_table.rs +++ b/src/model/version_table.rs @@ -29,19 +29,19 @@ pub struct Version { // Back link to bucket+key so that we can figure if // this was deleted later on /// Bucket in which the related object is stored - pub bucket: String, + pub bucket_id: Uuid, /// Key in which the related object is stored pub key: String, } impl Version { - pub fn new(uuid: Uuid, bucket: String, key: String, deleted: bool) -> Self { + pub fn new(uuid: Uuid, bucket_id: Uuid, key: String, deleted: bool) -> Self { Self { uuid, deleted: deleted.into(), blocks: crdt::Map::new(), parts_etags: crdt::Map::new(), - bucket, + bucket_id, key, } } @@ -82,8 +82,8 @@ impl AutoCrdt for VersionBlock { const WARN_IF_DIFFERENT: bool = true; } -impl Entry for Version { - fn partition_key(&self) -> &Hash { +impl Entry for Version { + fn partition_key(&self) -> &Uuid { &self.uuid } fn sort_key(&self) -> &EmptyKey { @@ -116,7 +116,7 @@ pub struct VersionTable { impl TableSchema for VersionTable { const TABLE_NAME: &'static str = "version"; - type P = Hash; + type P = Uuid; type S = EmptyKey; type E = Version; type Filter = DeletedFilter; -- cgit v1.2.3