diff options
| -rw-r--r-- | doc/book/operations/durability-repairs.md | 4 | ||||
| -rw-r--r-- | doc/book/operations/layout.md | 12 | ||||
| -rw-r--r-- | doc/book/operations/upgrading.md | 2 | ||||
| -rw-r--r-- | src/db/lib.rs | 42 | ||||
| -rw-r--r-- | src/garage/cli/convert_db.rs | 58 |
5 files changed, 97 insertions, 21 deletions
diff --git a/doc/book/operations/durability-repairs.md b/doc/book/operations/durability-repairs.md index b0d2c78a..578899a8 100644 --- a/doc/book/operations/durability-repairs.md +++ b/doc/book/operations/durability-repairs.md @@ -49,7 +49,7 @@ verifications. Of course, scrubbing the entire data store will also take longer. ## Block check and resync In some cases, nodes hold a reference to a block but do not actually have the block -stored on disk. Conversely, they may also have on disk blocks that are not referenced +stored on disk. Conversely, they may also have on-disk blocks that are not referenced any more. To fix both cases, a block repair may be run with `garage repair blocks`. This will scan the entire block reference counter table to check that the blocks exist on disk, and will scan the entire disk store to check that stored blocks @@ -95,7 +95,7 @@ using the `garage block purge` command. In [multi-HDD setups](@/documentation/operations/multi-hdd.md), to ensure that data blocks are well balanced between storage locations, you may run a -rebalance operation using `garage repair rebalance`. This is usefull when +rebalance operation using `garage repair rebalance`. This is useful when adding storage locations or when capacities of the storage locations have been changed. Once this is finished, Garage will know for each block of a single possible location where it can be, which can increase access speed. This diff --git a/doc/book/operations/layout.md b/doc/book/operations/layout.md index ee05aba1..cf1372b0 100644 --- a/doc/book/operations/layout.md +++ b/doc/book/operations/layout.md @@ -13,7 +13,7 @@ In Garage, all of the data that can be stored in a given cluster is divided into slices which we call *partitions*. Each partition is stored by one or several nodes in the cluster (see [`replication_mode`](@/documentation/reference-manual/configuration.md#replication_mode)). -The layout determines the correspondence between these partition, +The layout determines the correspondence between these partitions, which exist on a logical level, and actual storage nodes. ## How cluster layouts work in Garage @@ -94,10 +94,10 @@ follow the following recommendations: ## Understanding unexpected layout calculations When adding, removing or modifying nodes in a cluster layout, sometimes -unexpected assigntations of partitions to node can occur. These assignations -are in fact normal and logical, given the objectives of the algorihtm. Indeed, -**the layout algorithm prioritizes moving less data between nodes over the fact -of achieving equal distribution of load. It also tries to use all links between +unexpected assignations of partitions to node can occur. These assignations +are in fact normal and logical, given the objectives of the algorithm. Indeed, +**the layout algorithm prioritizes moving less data between nodes over +achieving equal distribution of load. It also tries to use all links between pairs of nodes in equal proportions when moving data.** This section presents two examples and illustrates how one can control Garage's behavior to obtain the desired results. @@ -270,5 +270,5 @@ that is moved to node1). This illustrates the second principle of the layout computation: **if there is a choice in moving data out of some nodes, then all links between pairs of nodes are used in equal proportions** (this is approximately true, there is -randomness in the algorihtm to achieve this so there might be some small +randomness in the algorithm to achieve this so there might be some small fluctuations, as we see above). diff --git a/doc/book/operations/upgrading.md b/doc/book/operations/upgrading.md index 9a738282..6b6ea26d 100644 --- a/doc/book/operations/upgrading.md +++ b/doc/book/operations/upgrading.md @@ -9,7 +9,7 @@ On a new version release, there is 2 possibilities: - protocols and data structures remained the same ➡️ this is a **minor upgrade** - protocols or data structures changed ➡️ this is a **major upgrade** -You can quickly now what type of update you will have to operate by looking at the version identifier: +You can quickly know what type of update you will have to operate by looking at the version identifier: when we require our users to do a major upgrade, we will always bump the first nonzero component of the version identifier (e.g. from v0.7.2 to v0.8.0). Conversely, for versions that only require a minor upgrade, the first nonzero component will always stay the same (e.g. from v0.8.0 to v0.8.1). diff --git a/src/db/lib.rs b/src/db/lib.rs index fe44b01e..eef3e177 100644 --- a/src/db/lib.rs +++ b/src/db/lib.rs @@ -171,6 +171,48 @@ impl Db { } } +/// List of supported database engine types +/// +/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter +/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine +/// and valid engine, whose support is not enabled via feature flag. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Engine { + Lmdb, + Sqlite, + Sled, +} + +impl Engine { + /// Return variant name as static `&str` + pub fn as_str(&self) -> &'static str { + match self { + Self::Lmdb => "lmdb", + Self::Sqlite => "sqlite", + Self::Sled => "sled", + } + } +} + +impl std::fmt::Display for Engine { + fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { + self.as_str().fmt(fmt) + } +} + +impl std::str::FromStr for Engine { + type Err = Error; + + fn from_str(text: &str) -> Result<Engine> { + match text { + "lmdb" | "heed" => Ok(Self::Lmdb), + "sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite), + "sled" => Ok(Self::Sled), + kind => Err(Error(format!("Invalid DB engine: {}", kind).into())), + } + } +} + #[allow(clippy::len_without_is_empty)] impl Tree { #[inline] diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs index 3c6ce69c..044ccbb9 100644 --- a/src/garage/cli/convert_db.rs +++ b/src/garage/cli/convert_db.rs @@ -14,44 +14,73 @@ pub struct ConvertDbOpt { /// Input database engine (sled, lmdb or sqlite; limited by db engines /// enabled in this build) #[structopt(short = "a")] - input_engine: String, + input_engine: Engine, /// Output database path #[structopt(short = "o")] output_path: PathBuf, /// Output database engine #[structopt(short = "b")] - output_engine: String, + output_engine: Engine, + + #[structopt(flatten)] + db_open: OpenDbOpt, +} + +/// Overrides for database open operation +#[derive(StructOpt, Debug, Default)] +pub struct OpenDbOpt { + #[cfg(feature = "lmdb")] + #[structopt(flatten)] + lmdb: OpenLmdbOpt, +} + +/// Overrides for LMDB database open operation +#[cfg(feature = "lmdb")] +#[derive(StructOpt, Debug, Default)] +pub struct OpenLmdbOpt { + /// LMDB map size override + /// (supported suffixes: B, KiB, MiB, GiB, TiB, PiB) + #[cfg(feature = "lmdb")] + #[structopt(long = "lmdb-map-size", name = "bytes", display_order = 1_000)] + map_size: Option<bytesize::ByteSize>, } pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> { - let input = open_db(args.input_path, args.input_engine)?; - let output = open_db(args.output_path, args.output_engine)?; + if args.input_engine == args.output_engine { + return Err(Error("input and output database engine must differ".into())); + } + + let input = open_db(args.input_path, args.input_engine, &args.db_open)?; + let output = open_db(args.output_path, args.output_engine, &args.db_open)?; output.import(&input)?; Ok(()) } -fn open_db(path: PathBuf, engine: String) -> Result<Db> { - match engine.as_str() { +fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> { + match engine { #[cfg(feature = "sled")] - "sled" => { + Engine::Sled => { let db = sled_adapter::sled::Config::default().path(&path).open()?; Ok(sled_adapter::SledDb::init(db)) } #[cfg(feature = "sqlite")] - "sqlite" | "sqlite3" | "rusqlite" => { + Engine::Sqlite => { let db = sqlite_adapter::rusqlite::Connection::open(&path)?; db.pragma_update(None, "journal_mode", &"WAL")?; db.pragma_update(None, "synchronous", &"NORMAL")?; Ok(sqlite_adapter::SqliteDb::init(db)) } #[cfg(feature = "lmdb")] - "lmdb" | "heed" => { + Engine::Lmdb => { std::fs::create_dir_all(&path).map_err(|e| { Error(format!("Unable to create LMDB data directory: {}", e).into()) })?; - let map_size = lmdb_adapter::recommended_map_size(); + let map_size = match open.lmdb.map_size { + Some(c) => c.as_u64() as usize, + None => lmdb_adapter::recommended_map_size(), + }; let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new(); env_builder.max_dbs(100); @@ -62,8 +91,13 @@ fn open_db(path: PathBuf, engine: String) -> Result<Db> { let db = env_builder.open(&path)?; Ok(lmdb_adapter::LmdbDb::init(db)) } - e => Err(Error( - format!("Invalid or unsupported DB engine: {}", e).into(), + + // Pattern is unreachable when all supported DB engines are compiled into binary. The allow + // attribute is added so that we won't have to change this match in case stop building + // support for one or more engines by default. + #[allow(unreachable_patterns)] + engine => Err(Error( + format!("Engine support not available in this build: {}", engine).into(), )), } } |