From d8ab5bdc3e20759e5ba8a6844393757da3539372 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 22 Dec 2021 18:50:08 +0100 Subject: New buckets for 0.6.0: fix model and migration --- src/model/bucket_alias_table.rs | 10 ++++++-- src/model/bucket_table.rs | 17 +++++++++--- src/model/key_table.rs | 9 +++---- src/model/lib.rs | 13 ++++++---- src/model/migrate.rs | 38 +++++++++++++++------------ src/model/object_table.rs | 7 ++--- src/model/version_table.rs | 57 ++++++++++++++++++++++------------------- 7 files changed, 88 insertions(+), 63 deletions(-) (limited to 'src/model') diff --git a/src/model/bucket_alias_table.rs b/src/model/bucket_alias_table.rs index 904a5255..caae76f1 100644 --- a/src/model/bucket_alias_table.rs +++ b/src/model/bucket_alias_table.rs @@ -1,8 +1,10 @@ use serde::{Deserialize, Serialize}; +use garage_util::data::*; +use garage_util::time::*; + use garage_table::crdt::*; use garage_table::*; -use garage_util::data::*; /// The bucket alias table holds the names given to buckets /// in the global namespace. @@ -23,15 +25,19 @@ impl AutoCrdt for AliasParams { impl BucketAlias { pub fn new(name: String, bucket_id: Uuid) -> Option { + Self::raw(name, now_msec(), bucket_id) + } + pub fn raw(name: String, ts: u64, bucket_id: Uuid) -> Option { if !is_valid_bucket_name(&name) { None } else { Some(BucketAlias { name, - state: crdt::Lww::new(crdt::Deletable::present(AliasParams { bucket_id })), + state: crdt::Lww::raw(ts, crdt::Deletable::present(AliasParams { bucket_id })), }) } } + pub fn is_deleted(&self) -> bool { self.state.get().is_deleted() } diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index 00e03899..8dcf6913 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -1,5 +1,4 @@ use serde::{Deserialize, Serialize}; -use serde_bytes::ByteBuf; use garage_table::crdt::Crdt; use garage_table::*; @@ -8,8 +7,6 @@ use garage_util::time::*; use crate::permission::BucketKeyPerm; -pub const DEFAULT_WEBSITE_CONFIGURATION: &[u8] = b""; // TODO (an XML WebsiteConfiguration document per the AWS spec) - /// A bucket is a collection of objects /// /// Its parameters are not directly accessible as: @@ -33,7 +30,7 @@ pub struct BucketParams { /// Whether this bucket is allowed for website access /// (under all of its global alias names), /// and if so, the website configuration XML document - pub website_config: crdt::Lww>, + pub website_config: crdt::Lww>, /// Map of aliases that are or have been given to this bucket /// in the global namespace /// (not authoritative: this is just used as an indication to @@ -45,6 +42,18 @@ pub struct BucketParams { pub local_aliases: crdt::LwwMap<(String, String), bool>, } +#[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize)] +pub enum WebsiteConfig { + RedirectAll { + hostname: String, + protocol: String, + }, + Website { + index_document: String, + error_document: Option, + }, +} + impl BucketParams { /// Create an empty BucketParams with no authorized keys and no website accesss pub fn new() -> Self { diff --git a/src/model/key_table.rs b/src/model/key_table.rs index 3285e355..daea5473 100644 --- a/src/model/key_table.rs +++ b/src/model/key_table.rs @@ -6,6 +6,8 @@ use garage_util::data::*; use crate::permission::BucketKeyPerm; +use garage_model_050::key_table as old; + /// An api key #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Key { @@ -173,11 +175,8 @@ impl TableSchema for KeyTable { } fn try_migrate(bytes: &[u8]) -> Option { - let old_k = - match rmp_serde::decode::from_read_ref::<_, garage_model_050::key_table::Key>(bytes) { - Ok(x) => x, - Err(_) => return None, - }; + let old_k = rmp_serde::decode::from_read_ref::<_, old::Key>(bytes).ok()?; + let state = if old_k.deleted.get() { crdt::Deletable::Deleted } else { diff --git a/src/model/lib.rs b/src/model/lib.rs index 3f6b5cd4..e7d7e98b 100644 --- a/src/model/lib.rs +++ b/src/model/lib.rs @@ -1,14 +1,17 @@ #[macro_use] extern crate log; -pub mod block; +pub mod permission; + pub mod block_ref_table; pub mod bucket_alias_table; -pub mod bucket_helper; pub mod bucket_table; -pub mod garage; pub mod key_table; -pub mod migrate; pub mod object_table; -pub mod permission; pub mod version_table; + +pub mod block; + +pub mod bucket_helper; +pub mod garage; +pub mod migrate; diff --git a/src/model/migrate.rs b/src/model/migrate.rs index e4469e64..6b20a01f 100644 --- a/src/model/migrate.rs +++ b/src/model/migrate.rs @@ -1,7 +1,5 @@ use std::sync::Arc; -use serde_bytes::ByteBuf; - use garage_table::util::EmptyKey; use garage_util::crdt::*; use garage_util::data::*; @@ -50,24 +48,32 @@ impl Migrate { hex::encode(&bucket_id.as_slice()[..16]) }; - let mut new_ak = Map::new(); - for (k, ts, perm) in old_bucket_p.authorized_keys.items().iter() { - new_ak.put( - k.to_string(), - BucketKeyPerm { - timestamp: *ts, - allow_read: perm.allow_read, - allow_write: perm.allow_write, - allow_owner: false, - }, - ); - } + let new_ak = old_bucket_p + .authorized_keys + .items() + .iter() + .map(|(k, ts, perm)| { + ( + k.to_string(), + BucketKeyPerm { + timestamp: *ts, + allow_read: perm.allow_read, + allow_write: perm.allow_write, + allow_owner: false, + }, + ) + }) + .collect::>(); let mut aliases = LwwMap::new(); aliases.update_in_place(new_name.clone(), true); + let alias_ts = aliases.get_timestamp(&new_name); let website = if *old_bucket_p.website.get() { - Some(ByteBuf::from(DEFAULT_WEBSITE_CONFIGURATION.to_vec())) + Some(WebsiteConfig::Website { + index_document: "index.html".into(), + error_document: None, + }) } else { None }; @@ -84,7 +90,7 @@ impl Migrate { }; self.garage.bucket_table.insert(&new_bucket).await?; - let new_alias = BucketAlias::new(new_name.clone(), new_bucket.id).unwrap(); + let new_alias = BucketAlias::raw(new_name.clone(), alias_ts, new_bucket.id).unwrap(); self.garage.bucket_alias_table.insert(&new_alias).await?; for (k, perm) in new_ak.items().iter() { diff --git a/src/model/object_table.rs b/src/model/object_table.rs index 45f0daf4..0c6c3a6d 100644 --- a/src/model/object_table.rs +++ b/src/model/object_table.rs @@ -259,11 +259,8 @@ impl TableSchema for ObjectTable { } fn try_migrate(bytes: &[u8]) -> Option { - let old_v = match rmp_serde::decode::from_read_ref::<_, old::Object>(bytes) { - Ok(x) => x, - Err(_) => return None, - }; - Some(migrate_object(old_v)) + let old_obj = rmp_serde::decode::from_read_ref::<_, old::Object>(bytes).ok()?; + Some(migrate_object(old_obj)) } } diff --git a/src/model/version_table.rs b/src/model/version_table.rs index 05cae831..e0b99770 100644 --- a/src/model/version_table.rs +++ b/src/model/version_table.rs @@ -10,6 +10,8 @@ use garage_table::*; use crate::block_ref_table::*; +use garage_model_050::version_table as old; + /// A version of an object #[derive(PartialEq, Clone, Debug, Serialize, Deserialize)] pub struct Version { @@ -149,35 +151,38 @@ impl TableSchema for VersionTable { } fn try_migrate(bytes: &[u8]) -> Option { - let old = - match rmp_serde::decode::from_read_ref::<_, garage_model_050::version_table::Version>( - bytes, - ) { - Ok(x) => x, - Err(_) => return None, - }; - let mut new_blocks = crdt::Map::new(); - for (k, v) in old.blocks.items().iter() { - new_blocks.put( - VersionBlockKey { - part_number: k.part_number, - offset: k.offset, - }, - VersionBlock { - hash: Hash::try_from(v.hash.as_slice()).unwrap(), - size: v.size, - }, - ); - } - let mut new_parts_etags = crdt::Map::new(); - for (k, v) in old.parts_etags.items().iter() { - new_parts_etags.put(*k, v.clone()); - } + let old = rmp_serde::decode::from_read_ref::<_, old::Version>(bytes).ok()?; + + let blocks = old + .blocks + .items() + .iter() + .map(|(k, v)| { + ( + VersionBlockKey { + part_number: k.part_number, + offset: k.offset, + }, + VersionBlock { + hash: Hash::try_from(v.hash.as_slice()).unwrap(), + size: v.size, + }, + ) + }) + .collect::>(); + + let parts_etags = old + .parts_etags + .items() + .iter() + .map(|(k, v)| (*k, v.clone())) + .collect::>(); + Some(Version { uuid: Hash::try_from(old.uuid.as_slice()).unwrap(), deleted: crdt::Bool::new(old.deleted.get()), - blocks: new_blocks, - parts_etags: new_parts_etags, + blocks, + parts_etags, bucket_id: blake2sum(old.bucket.as_bytes()), key: old.key, }) -- cgit v1.2.3