From cdb2a591e9d393d24ab5c49bb905b0589b193299 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 3 Jan 2023 14:44:47 +0100 Subject: Refactor how things are migrated --- src/util/migrate.rs | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 src/util/migrate.rs (limited to 'src/util/migrate.rs') diff --git a/src/util/migrate.rs b/src/util/migrate.rs new file mode 100644 index 00000000..199c68f6 --- /dev/null +++ b/src/util/migrate.rs @@ -0,0 +1,75 @@ +use serde::{Deserialize, Serialize}; + +pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { + /// A sequence of bytes to add at the beginning of the serialized + /// string, to identify that the data is of this version. + const MARKER: &'static [u8] = b""; + + /// The previous version of this data type, from which items of this version + /// can be migrated. Set `type Previous = NoPrevious` to indicate that this datatype + /// is the initial schema and cannot be migrated. + type Previous: Migrate; + + /// This function must be filled in by implementors to migrate from a previons iteration + /// of the data format. + fn migrate(previous: Self::Previous) -> Self; + + fn decode(bytes: &[u8]) -> Option { + if bytes.len() >= Self::MARKER.len() && &bytes[..Self::MARKER.len()] == Self::MARKER { + if let Ok(value) = + rmp_serde::decode::from_read_ref::<_, Self>(&bytes[Self::MARKER.len()..]) + { + return Some(value); + } + } + + Self::Previous::decode(bytes).map(Self::migrate) + } + + fn encode(&self) -> Result, rmp_serde::encode::Error> { + let mut wr = Vec::with_capacity(128); + wr.extend_from_slice(Self::MARKER); + let mut se = rmp_serde::Serializer::new(&mut wr) + .with_struct_map() + .with_string_variants(); + self.serialize(&mut se)?; + Ok(wr) + } +} + +pub trait InitialFormat: Serialize + for<'de> Deserialize<'de> + 'static { + /// A sequence of bytes to add at the beginning of the serialized + /// string, to identify that the data is of this version. + const MARKER: &'static [u8] = b""; +} + +// ---- + +impl Migrate for T { + const MARKER: &'static [u8] = ::MARKER; + + type Previous = NoPrevious; + + fn migrate(_previous: Self::Previous) -> Self { + unreachable!(); + } +} + +#[derive(Serialize, Deserialize)] +pub struct NoPrevious; + +impl Migrate for NoPrevious { + type Previous = NoPrevious; + + fn migrate(_previous: Self::Previous) -> Self { + unreachable!(); + } + + fn decode(_bytes: &[u8]) -> Option { + None + } + + fn encode(&self) -> Result, rmp_serde::encode::Error> { + unreachable!() + } +} -- cgit v1.2.3 From 8d5505514f950dc1ca1249a3385c9913b5b5e8e0 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 3 Jan 2023 15:27:36 +0100 Subject: Make it explicit when using nonversioned encoding --- src/util/migrate.rs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'src/util/migrate.rs') diff --git a/src/util/migrate.rs b/src/util/migrate.rs index 199c68f6..f6028bf4 100644 --- a/src/util/migrate.rs +++ b/src/util/migrate.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { /// A sequence of bytes to add at the beginning of the serialized /// string, to identify that the data is of this version. - const MARKER: &'static [u8] = b""; + const VERSION_MARKER: &'static [u8] = b""; /// The previous version of this data type, from which items of this version /// can be migrated. Set `type Previous = NoPrevious` to indicate that this datatype @@ -15,10 +15,9 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { fn migrate(previous: Self::Previous) -> Self; fn decode(bytes: &[u8]) -> Option { - if bytes.len() >= Self::MARKER.len() && &bytes[..Self::MARKER.len()] == Self::MARKER { - if let Ok(value) = - rmp_serde::decode::from_read_ref::<_, Self>(&bytes[Self::MARKER.len()..]) - { + let marker_len = Self::VERSION_MARKER.len(); + if bytes.len() >= marker_len && &bytes[..marker_len] == Self::VERSION_MARKER { + if let Ok(value) = rmp_serde::decode::from_read_ref::<_, Self>(&bytes[marker_len..]) { return Some(value); } } @@ -28,7 +27,7 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { fn encode(&self) -> Result, rmp_serde::encode::Error> { let mut wr = Vec::with_capacity(128); - wr.extend_from_slice(Self::MARKER); + wr.extend_from_slice(Self::VERSION_MARKER); let mut se = rmp_serde::Serializer::new(&mut wr) .with_struct_map() .with_string_variants(); @@ -40,13 +39,13 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { pub trait InitialFormat: Serialize + for<'de> Deserialize<'de> + 'static { /// A sequence of bytes to add at the beginning of the serialized /// string, to identify that the data is of this version. - const MARKER: &'static [u8] = b""; + const VERSION_MARKER: &'static [u8] = b""; } // ---- impl Migrate for T { - const MARKER: &'static [u8] = ::MARKER; + const VERSION_MARKER: &'static [u8] = ::VERSION_MARKER; type Previous = NoPrevious; -- cgit v1.2.3 From 33f25d26c7a81f7dc7ae3ab4fd2faa49fb053ceb Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 3 Jan 2023 15:53:13 +0100 Subject: fix doc and add tests for migrate.rs --- src/util/migrate.rs | 97 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 91 insertions(+), 6 deletions(-) (limited to 'src/util/migrate.rs') diff --git a/src/util/migrate.rs b/src/util/migrate.rs index f6028bf4..b7d6edc1 100644 --- a/src/util/migrate.rs +++ b/src/util/migrate.rs @@ -1,19 +1,21 @@ use serde::{Deserialize, Serialize}; +/// Indicates that this type has an encoding that can be migrated from +/// a previous version upon upgrades of Garage. pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { /// A sequence of bytes to add at the beginning of the serialized /// string, to identify that the data is of this version. const VERSION_MARKER: &'static [u8] = b""; /// The previous version of this data type, from which items of this version - /// can be migrated. Set `type Previous = NoPrevious` to indicate that this datatype - /// is the initial schema and cannot be migrated. + /// can be migrated. type Previous: Migrate; - /// This function must be filled in by implementors to migrate from a previons iteration - /// of the data format. + /// The migration function that transforms a value decoded in the old format + /// to an up-to-date value. fn migrate(previous: Self::Previous) -> Self; + /// Decode an encoded version of this type, going through a migration if necessary. fn decode(bytes: &[u8]) -> Option { let marker_len = Self::VERSION_MARKER.len(); if bytes.len() >= marker_len && &bytes[..marker_len] == Self::VERSION_MARKER { @@ -25,6 +27,7 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { Self::Previous::decode(bytes).map(Self::migrate) } + /// Encode this type with optionnal version marker fn encode(&self) -> Result, rmp_serde::encode::Error> { let mut wr = Vec::with_capacity(128); wr.extend_from_slice(Self::VERSION_MARKER); @@ -36,14 +39,13 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { } } +/// Indicates that this type has no previous encoding version to be migrated from. pub trait InitialFormat: Serialize + for<'de> Deserialize<'de> + 'static { /// A sequence of bytes to add at the beginning of the serialized /// string, to identify that the data is of this version. const VERSION_MARKER: &'static [u8] = b""; } -// ---- - impl Migrate for T { const VERSION_MARKER: &'static [u8] = ::VERSION_MARKER; @@ -54,6 +56,7 @@ impl Migrate for T { } } +/// Internal type used by InitialFormat, not meant for general use. #[derive(Serialize, Deserialize)] pub struct NoPrevious; @@ -72,3 +75,85 @@ impl Migrate for NoPrevious { unreachable!() } } + +#[cfg(test)] +mod test { + use super::*; + + #[derive(Serialize, Deserialize, PartialEq, Eq, Debug)] + struct V1 { + a: usize, + b: String, + } + impl InitialFormat for V1 {} + + #[derive(Serialize, Deserialize, PartialEq, Eq, Debug)] + struct V2 { + a: usize, + b: Vec, + c: String, + } + impl Migrate for V2 { + const VERSION_MARKER: &'static [u8] = b"GtestV2"; + type Previous = V1; + fn migrate(prev: V1) -> V2 { + V2 { + a: prev.a, + b: vec![prev.b], + c: String::new(), + } + } + } + + #[test] + fn test_v1() { + let x = V1 { + a: 12, + b: "hello".into(), + }; + let x_enc = x.encode().unwrap(); + let y = V1::decode(&x_enc).unwrap(); + assert_eq!(x, y); + } + + #[test] + fn test_v2() { + let x = V2 { + a: 12, + b: vec!["hello".into(), "world".into()], + c: "plop".into(), + }; + let x_enc = x.encode().unwrap(); + assert_eq!(&x_enc[..V2::VERSION_MARKER.len()], V2::VERSION_MARKER); + let y = V2::decode(&x_enc).unwrap(); + assert_eq!(x, y); + } + + #[test] + fn test_migrate() { + let x = V1 { + a: 12, + b: "hello".into(), + }; + let x_enc = x.encode().unwrap(); + + let xx = V1::decode(&x_enc).unwrap(); + assert_eq!(x, xx); + + let y = V2::decode(&x_enc).unwrap(); + assert_eq!( + y, + V2 { + a: 12, + b: vec!["hello".into()], + c: "".into(), + } + ); + + let y_enc = y.encode().unwrap(); + assert_eq!(&y_enc[..V2::VERSION_MARKER.len()], V2::VERSION_MARKER); + + let z = V2::decode(&y_enc).unwrap(); + assert_eq!(y, z); + } +} -- cgit v1.2.3 From c106304b9cd325238742be4366877ed7316e7e28 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 3 Jan 2023 16:00:19 +0100 Subject: more idiomatic and shorter --- src/util/migrate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/util/migrate.rs') diff --git a/src/util/migrate.rs b/src/util/migrate.rs index b7d6edc1..00d81181 100644 --- a/src/util/migrate.rs +++ b/src/util/migrate.rs @@ -18,7 +18,7 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static { /// Decode an encoded version of this type, going through a migration if necessary. fn decode(bytes: &[u8]) -> Option { let marker_len = Self::VERSION_MARKER.len(); - if bytes.len() >= marker_len && &bytes[..marker_len] == Self::VERSION_MARKER { + if bytes.get(..marker_len) == Some(Self::VERSION_MARKER) { if let Ok(value) = rmp_serde::decode::from_read_ref::<_, Self>(&bytes[marker_len..]) { return Some(value); } -- cgit v1.2.3 From 1d5bdc17a46648eb3494ff629d0d360d0217c1e2 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 3 Jan 2023 16:04:06 +0100 Subject: use impossible enum type --- src/util/migrate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/util/migrate.rs') diff --git a/src/util/migrate.rs b/src/util/migrate.rs index 00d81181..1229fd9c 100644 --- a/src/util/migrate.rs +++ b/src/util/migrate.rs @@ -58,7 +58,7 @@ impl Migrate for T { /// Internal type used by InitialFormat, not meant for general use. #[derive(Serialize, Deserialize)] -pub struct NoPrevious; +pub enum NoPrevious {} impl Migrate for NoPrevious { type Previous = NoPrevious; -- cgit v1.2.3