aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/db/lib.rs47
-rw-r--r--src/db/open.rs153
-rw-r--r--src/garage/cli/convert_db.rs54
-rw-r--r--src/model/garage.rs112
4 files changed, 183 insertions, 183 deletions
diff --git a/src/db/lib.rs b/src/db/lib.rs
index eef3e177..0fb457ce 100644
--- a/src/db/lib.rs
+++ b/src/db/lib.rs
@@ -1,5 +1,4 @@
#[macro_use]
-#[cfg(feature = "sqlite")]
extern crate tracing;
#[cfg(feature = "lmdb")]
@@ -11,6 +10,8 @@ pub mod sqlite_adapter;
pub mod counted_tree_hack;
+pub mod open;
+
#[cfg(test)]
pub mod test;
@@ -22,6 +23,8 @@ use std::sync::Arc;
use err_derive::Error;
+pub use open::*;
+
pub(crate) type OnCommit = Vec<Box<dyn FnOnce()>>;
#[derive(Clone)]
@@ -171,48 +174,6 @@ impl Db {
}
}
-/// List of supported database engine types
-///
-/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
-/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
-/// and valid engine, whose support is not enabled via feature flag.
-#[derive(Clone, Copy, Debug, PartialEq, Eq)]
-pub enum Engine {
- Lmdb,
- Sqlite,
- Sled,
-}
-
-impl Engine {
- /// Return variant name as static `&str`
- pub fn as_str(&self) -> &'static str {
- match self {
- Self::Lmdb => "lmdb",
- Self::Sqlite => "sqlite",
- Self::Sled => "sled",
- }
- }
-}
-
-impl std::fmt::Display for Engine {
- fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
- self.as_str().fmt(fmt)
- }
-}
-
-impl std::str::FromStr for Engine {
- type Err = Error;
-
- fn from_str(text: &str) -> Result<Engine> {
- match text {
- "lmdb" | "heed" => Ok(Self::Lmdb),
- "sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
- "sled" => Ok(Self::Sled),
- kind => Err(Error(format!("Invalid DB engine: {}", kind).into())),
- }
- }
-}
-
#[allow(clippy::len_without_is_empty)]
impl Tree {
#[inline]
diff --git a/src/db/open.rs b/src/db/open.rs
new file mode 100644
index 00000000..ae135c4e
--- /dev/null
+++ b/src/db/open.rs
@@ -0,0 +1,153 @@
+use std::path::PathBuf;
+
+use crate::{Db, Error, Result};
+
+/// List of supported database engine types
+///
+/// The `enum` holds list of *all* database engines that are are be supported by crate, no matter
+/// if relevant feature is enabled or not. It allows us to distinguish between invalid engine
+/// and valid engine, whose support is not enabled via feature flag.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub enum Engine {
+ Lmdb,
+ Sqlite,
+ Sled,
+}
+
+impl Engine {
+ /// Return variant name as static `&str`
+ pub fn as_str(&self) -> &'static str {
+ match self {
+ Self::Lmdb => "lmdb",
+ Self::Sqlite => "sqlite",
+ Self::Sled => "sled",
+ }
+ }
+}
+
+impl std::fmt::Display for Engine {
+ fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+ self.as_str().fmt(fmt)
+ }
+}
+
+impl std::str::FromStr for Engine {
+ type Err = Error;
+
+ fn from_str(text: &str) -> Result<Engine> {
+ match text {
+ "lmdb" | "heed" => Ok(Self::Lmdb),
+ "sqlite" | "sqlite3" | "rusqlite" => Ok(Self::Sqlite),
+ "sled" => Ok(Self::Sled),
+ kind => Err(Error(
+ format!(
+ "Invalid DB engine: {} (options are: lmdb, sled, sqlite)",
+ kind
+ )
+ .into(),
+ )),
+ }
+ }
+}
+
+pub struct OpenOpt {
+ pub fsync: bool,
+ pub lmdb_map_size: Option<usize>,
+ pub sled_cache_capacity: usize,
+ pub sled_flush_every_ms: u64,
+}
+
+impl Default for OpenOpt {
+ fn default() -> Self {
+ Self {
+ fsync: false,
+ lmdb_map_size: None,
+ sled_cache_capacity: 1024 * 1024 * 1024,
+ sled_flush_every_ms: 2000,
+ }
+ }
+}
+
+pub fn open_db(path: &PathBuf, engine: Engine, opt: &OpenOpt) -> Result<Db> {
+ match engine {
+ // ---- Sled DB ----
+ #[cfg(feature = "sled")]
+ Engine::Sled => {
+ if opt.fsync {
+ return Err(Error(
+ "`metadata_fsync = true` is not supported with the Sled database engine".into(),
+ ));
+ }
+ info!("Opening Sled database at: {}", path.display());
+ let db = crate::sled_adapter::sled::Config::default()
+ .path(&path)
+ .cache_capacity(opt.sled_cache_capacity as u64)
+ .flush_every_ms(Some(opt.sled_flush_every_ms))
+ .open()?;
+ Ok(crate::sled_adapter::SledDb::init(db))
+ }
+
+ // ---- Sqlite DB ----
+ #[cfg(feature = "sqlite")]
+ Engine::Sqlite => {
+ info!("Opening Sqlite database at: {}", path.display());
+ let db = crate::sqlite_adapter::rusqlite::Connection::open(&path)?;
+ db.pragma_update(None, "journal_mode", "WAL")?;
+ if opt.fsync {
+ db.pragma_update(None, "synchronous", "NORMAL")?;
+ } else {
+ db.pragma_update(None, "synchronous", "OFF")?;
+ }
+ Ok(crate::sqlite_adapter::SqliteDb::init(db))
+ }
+
+ // ---- LMDB DB ----
+ #[cfg(feature = "lmdb")]
+ Engine::Lmdb => {
+ info!("Opening LMDB database at: {}", path.display());
+ if let Err(e) = std::fs::create_dir_all(&path) {
+ return Err(Error(
+ format!("Unable to create LMDB data directory: {}", e).into(),
+ ));
+ }
+
+ let map_size = match opt.lmdb_map_size {
+ None => crate::lmdb_adapter::recommended_map_size(),
+ Some(v) => v - (v % 4096),
+ };
+
+ let mut env_builder = heed::EnvOpenOptions::new();
+ env_builder.max_dbs(100);
+ env_builder.map_size(map_size);
+ env_builder.max_readers(2048);
+ unsafe {
+ env_builder.flag(crate::lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
+ if !opt.fsync {
+ env_builder.flag(heed::flags::Flags::MdbNoSync);
+ }
+ }
+ match env_builder.open(&path) {
+ Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
+ return Err(Error(
+ "OutOfMemory error while trying to open LMDB database. This can happen \
+ if your operating system is not allowing you to use sufficient virtual \
+ memory address space. Please check that no limit is set (ulimit -v). \
+ You may also try to set a smaller `lmdb_map_size` configuration parameter. \
+ On 32-bit machines, you should probably switch to another database engine."
+ .into(),
+ ))
+ }
+ Err(e) => Err(Error(format!("Cannot open LMDB database: {}", e).into())),
+ Ok(db) => Ok(crate::lmdb_adapter::LmdbDb::init(db)),
+ }
+ }
+
+ // Pattern is unreachable when all supported DB engines are compiled into binary. The allow
+ // attribute is added so that we won't have to change this match in case stop building
+ // support for one or more engines by default.
+ #[allow(unreachable_patterns)]
+ engine => Err(Error(
+ format!("DB engine support not available in this build: {}", engine).into(),
+ )),
+ }
+}
diff --git a/src/garage/cli/convert_db.rs b/src/garage/cli/convert_db.rs
index 6b854ccb..2aadb1d6 100644
--- a/src/garage/cli/convert_db.rs
+++ b/src/garage/cli/convert_db.rs
@@ -51,53 +51,13 @@ pub(crate) fn do_conversion(args: ConvertDbOpt) -> Result<()> {
return Err(Error("input and output database engine must differ".into()));
}
- let input = open_db(args.input_path, args.input_engine, &args.db_open)?;
- let output = open_db(args.output_path, args.output_engine, &args.db_open)?;
+ let opt = OpenOpt {
+ lmdb_map_size: args.db_open.lmdb.map_size.map(|x| x.as_u64() as usize),
+ ..Default::default()
+ };
+
+ let input = open_db(&args.input_path, args.input_engine, &opt)?;
+ let output = open_db(&args.output_path, args.output_engine, &opt)?;
output.import(&input)?;
Ok(())
}
-
-fn open_db(path: PathBuf, engine: Engine, open: &OpenDbOpt) -> Result<Db> {
- match engine {
- #[cfg(feature = "sled")]
- Engine::Sled => {
- let db = sled_adapter::sled::Config::default().path(&path).open()?;
- Ok(sled_adapter::SledDb::init(db))
- }
- #[cfg(feature = "sqlite")]
- Engine::Sqlite => {
- let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
- db.pragma_update(None, "journal_mode", "WAL")?;
- db.pragma_update(None, "synchronous", "NORMAL")?;
- Ok(sqlite_adapter::SqliteDb::init(db))
- }
- #[cfg(feature = "lmdb")]
- Engine::Lmdb => {
- std::fs::create_dir_all(&path).map_err(|e| {
- Error(format!("Unable to create LMDB data directory: {}", e).into())
- })?;
-
- let map_size = match open.lmdb.map_size {
- Some(c) => c.as_u64() as usize,
- None => lmdb_adapter::recommended_map_size(),
- };
-
- let mut env_builder = lmdb_adapter::heed::EnvOpenOptions::new();
- env_builder.max_dbs(100);
- env_builder.map_size(map_size);
- unsafe {
- env_builder.flag(lmdb_adapter::heed::flags::Flags::MdbNoMetaSync);
- }
- let db = env_builder.open(&path)?;
- Ok(lmdb_adapter::LmdbDb::init(db))
- }
-
- // Pattern is unreachable when all supported DB engines are compiled into binary. The allow
- // attribute is added so that we won't have to change this match in case stop building
- // support for one or more engines by default.
- #[allow(unreachable_patterns)]
- engine => Err(Error(
- format!("Engine support not available in this build: {}", engine).into(),
- )),
- }
-}
diff --git a/src/model/garage.rs b/src/model/garage.rs
index fe38a760..18421ca3 100644
--- a/src/model/garage.rs
+++ b/src/model/garage.rs
@@ -1,3 +1,4 @@
+use std::str::FromStr;
use std::sync::Arc;
use garage_net::NetworkKey;
@@ -113,108 +114,33 @@ impl Garage {
}
info!("Opening database...");
+ let db_engine = db::Engine::from_str(&config.db_engine)
+ .ok_or_message("Invalid `db_engine` value in configuration file")?;
let mut db_path = config.metadata_dir.clone();
- let db = match config.db_engine.as_str() {
- // ---- Sled DB ----
- #[cfg(feature = "sled")]
- "sled" => {
- if config.metadata_fsync {
- return Err(Error::Message(format!(
- "`metadata_fsync = true` is not supported with the Sled database engine"
- )));
- }
+ match db_engine {
+ db::Engine::Sled => {
db_path.push("db");
- info!("Opening Sled database at: {}", db_path.display());
- let db = db::sled_adapter::sled::Config::default()
- .path(&db_path)
- .cache_capacity(config.sled_cache_capacity as u64)
- .flush_every_ms(Some(config.sled_flush_every_ms))
- .open()
- .ok_or_message("Unable to open sled DB")?;
- db::sled_adapter::SledDb::init(db)
}
- #[cfg(not(feature = "sled"))]
- "sled" => return Err(Error::Message("sled db not available in this build".into())),
- // ---- Sqlite DB ----
- #[cfg(feature = "sqlite")]
- "sqlite" | "sqlite3" | "rusqlite" => {
+ db::Engine::Sqlite => {
db_path.push("db.sqlite");
- info!("Opening Sqlite database at: {}", db_path.display());
- let db = db::sqlite_adapter::rusqlite::Connection::open(db_path)
- .and_then(|db| {
- db.pragma_update(None, "journal_mode", &"WAL")?;
- if config.metadata_fsync {
- db.pragma_update(None, "synchronous", &"NORMAL")?;
- } else {
- db.pragma_update(None, "synchronous", &"OFF")?;
- }
- Ok(db)
- })
- .ok_or_message("Unable to open sqlite DB")?;
- db::sqlite_adapter::SqliteDb::init(db)
- }
- #[cfg(not(feature = "sqlite"))]
- "sqlite" | "sqlite3" | "rusqlite" => {
- return Err(Error::Message(
- "sqlite db not available in this build".into(),
- ))
}
- // ---- LMDB DB ----
- #[cfg(feature = "lmdb")]
- "lmdb" | "heed" => {
+ db::Engine::Lmdb => {
db_path.push("db.lmdb");
- info!("Opening LMDB database at: {}", db_path.display());
- std::fs::create_dir_all(&db_path)
- .ok_or_message("Unable to create LMDB data directory")?;
- let map_size = match config.lmdb_map_size {
- v if v == usize::default() => garage_db::lmdb_adapter::recommended_map_size(),
- v => v - (v % 4096),
- };
-
- use db::lmdb_adapter::heed;
- let mut env_builder = heed::EnvOpenOptions::new();
- env_builder.max_dbs(100);
- env_builder.max_readers(500);
- env_builder.map_size(map_size);
- unsafe {
- env_builder.flag(heed::flags::Flags::MdbNoMetaSync);
- if !config.metadata_fsync {
- env_builder.flag(heed::flags::Flags::MdbNoSync);
- }
- }
- let db = match env_builder.open(&db_path) {
- Err(heed::Error::Io(e)) if e.kind() == std::io::ErrorKind::OutOfMemory => {
- return Err(Error::Message(
- "OutOfMemory error while trying to open LMDB database. This can happen \
- if your operating system is not allowing you to use sufficient virtual \
- memory address space. Please check that no limit is set (ulimit -v). \
- You may also try to set a smaller `lmdb_map_size` configuration parameter. \
- On 32-bit machines, you should probably switch to another database engine.".into()))
- }
- x => x.ok_or_message("Unable to open LMDB DB")?,
- };
- db::lmdb_adapter::LmdbDb::init(db)
- }
- #[cfg(not(feature = "lmdb"))]
- "lmdb" | "heed" => return Err(Error::Message("lmdb db not available in this build".into())),
- // ---- Unavailable DB engine ----
- e => {
- return Err(Error::Message(format!(
- "Unsupported DB engine: {} (options: {})",
- e,
- vec![
- #[cfg(feature = "sled")]
- "sled",
- #[cfg(feature = "sqlite")]
- "sqlite",
- #[cfg(feature = "lmdb")]
- "lmdb",
- ]
- .join(", ")
- )));
}
+ }
+ let db_opt = db::OpenOpt {
+ fsync: config.metadata_fsync,
+ lmdb_map_size: match config.lmdb_map_size {
+ v if v == usize::default() => None,
+ v => Some(v),
+ },
+ sled_cache_capacity: config.sled_cache_capacity,
+ sled_flush_every_ms: config.sled_flush_every_ms,
};
+ let db = db::open_db(&db_path, db_engine, &db_opt)
+ .ok_or_message("Unable to open metadata db")?;
+ info!("Initializing RPC...");
let network_key = hex::decode(config.rpc_secret.as_ref().ok_or_message(
"rpc_secret value is missing, not present in config file or in environment",
)?)
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-08 21:44:18 +0000