aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Cargo.lock1
-rw-r--r--src/db/Cargo.toml11
-rw-r--r--src/db/bin/convert.rs55
-rw-r--r--src/db/lib.rs43
-rw-r--r--src/db/sled_adapter.rs64
-rw-r--r--src/db/sqlite_adapter.rs35
6 files changed, 131 insertions, 78 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 9ff2c02d..73879369 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1014,6 +1014,7 @@ dependencies = [
name = "garage_db"
version = "0.8.0"
dependencies = [
+ "clap 3.1.18",
"err-derive 0.3.1",
"hexdump",
"mktemp",
diff --git a/src/db/Cargo.toml b/src/db/Cargo.toml
index ca189a67..22abc0b9 100644
--- a/src/db/Cargo.toml
+++ b/src/db/Cargo.toml
@@ -11,7 +11,10 @@ readme = "../../README.md"
[lib]
path = "lib.rs"
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[[bin]]
+name = "convert"
+path = "bin/convert.rs"
+required-features = ["cli"]
[dependencies]
err-derive = "0.3"
@@ -20,5 +23,11 @@ hexdump = "0.1"
sled = "0.34"
rusqlite = "0.27"
+# cli deps
+clap = { version = "3.1.18", optional = true, features = ["derive", "env"] }
+
[dev-dependencies]
mktemp = "0.4"
+
+[features]
+cli = ["clap"]
diff --git a/src/db/bin/convert.rs b/src/db/bin/convert.rs
new file mode 100644
index 00000000..8c4f0ddc
--- /dev/null
+++ b/src/db/bin/convert.rs
@@ -0,0 +1,55 @@
+use std::path::PathBuf;
+
+use garage_db::*;
+
+use clap::{Parser};
+
+/// K2V command line interface
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+struct Args {
+ /// Input DB path
+ #[clap(short = 'i')]
+ input_path: PathBuf,
+ /// Input DB engine
+ #[clap(short = 'a')]
+ input_engine: String,
+
+ /// Output DB path
+ #[clap(short = 'o')]
+ output_path: PathBuf,
+ /// Output DB engine
+ #[clap(short = 'b')]
+ output_engine: String,
+}
+
+fn main() {
+ let args = Args::parse();
+ match do_conversion(args) {
+ Ok(()) => println!("Success!"),
+ Err(e) => eprintln!("Error: {}", e),
+ }
+}
+
+fn do_conversion(args: Args) -> Result<()> {
+ let input = open_db(args.input_path, args.input_engine)?;
+ let output = open_db(args.output_path, args.output_engine)?;
+ output.import(&input)?;
+ Ok(())
+}
+
+fn open_db(path: PathBuf, engine: String) -> Result<Db> {
+ match engine.as_str() {
+ "sled" => {
+ let db = sled_adapter::sled::Config::default()
+ .path(&path)
+ .open()?;
+ Ok(sled_adapter::SledDb::init(db))
+ }
+ "sqlite" | "rusqlite" => {
+ let db = sqlite_adapter::rusqlite::Connection::open(&path)?;
+ Ok(sqlite_adapter::SqliteDb::init(db))
+ }
+ e => Err(Error(format!("Invalid DB engine: {}", e).into())),
+ }
+}
diff --git a/src/db/lib.rs b/src/db/lib.rs
index 95d2c16b..49ec0765 100644
--- a/src/db/lib.rs
+++ b/src/db/lib.rs
@@ -23,8 +23,6 @@ pub struct Tree(pub(crate) Arc<dyn IDb>, pub(crate) usize);
pub type ValueIter<'a> = Box<dyn std::iter::Iterator<Item = Result<(Value<'a>, Value<'a>)>> + 'a>;
-pub type Exporter<'a> = Box<dyn std::iter::Iterator<Item = Result<(String, ValueIter<'a>)>> + 'a>;
-
// ----
pub struct Value<'a>(pub(crate) Box<dyn IValue<'a> + 'a>);
@@ -115,7 +113,7 @@ impl<'a> IValue<'a> for &'a [u8] {
#[derive(Debug, Error)]
#[error(display = "{}", _0)]
-pub struct Error(Cow<'static, str>);
+pub struct Error(pub Cow<'static, str>);
pub type Result<T> = std::result::Result<T, Error>;
@@ -140,6 +138,10 @@ impl Db {
Ok(Tree(self.0.clone(), tree_id))
}
+ pub fn list_trees(&self) -> Result<Vec<String>> {
+ self.0.list_trees()
+ }
+
pub fn transaction<R, E, F>(&self, fun: F) -> TxResult<R, E>
where
F: Fn(Transaction<'_>) -> TxResult<R, E>,
@@ -175,12 +177,33 @@ impl Db {
}
}
- pub fn export(&self) -> Result<Exporter<'_>> {
- self.0.export()
- }
+ pub fn import(&self, other: &Db) -> Result<()> {
+ let existing_trees = self.list_trees()?;
+ if !existing_trees.is_empty() {
+ return Err(Error(format!("destination database already contains data: {:?}", existing_trees).into()));
+ }
+
+ let tree_names = other.list_trees()?;
+ for name in tree_names {
+ let tree = self.open_tree(&name)?;
+ if tree.len()? > 0 {
+ return Err(Error(format!("tree {} already contains data", name).into()));
+ }
+
+ let ex_tree = other.open_tree(&name)?;
- pub fn import(&self, ex: Exporter<'_>) -> Result<()> {
- self.0.import(ex)
+ let mut i = 0;
+ for item in ex_tree.iter()? {
+ let (k, v) = item?;
+ tree.insert(k, v)?;
+ i += 1;
+ if i % 1000 == 0 {
+ println!("{}: imported {}", name, i);
+ }
+ }
+ println!("{}: finished importing, {} items", name, i);
+ }
+ Ok(())
}
}
@@ -293,6 +316,7 @@ impl<'a> Transaction<'a> {
pub(crate) trait IDb: Send + Sync {
fn open_tree(&self, name: &str) -> Result<usize>;
+ fn list_trees(&self) -> Result<Vec<String>>;
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>>;
fn len(&self, tree: usize) -> Result<usize>;
@@ -317,9 +341,6 @@ pub(crate) trait IDb: Send + Sync {
) -> Result<ValueIter<'_>>;
fn transaction(&self, f: &dyn ITxFn) -> TxResult<(), ()>;
-
- fn export(&self) -> Result<Exporter<'_>>;
- fn import(&self, ex: Exporter<'_>) -> Result<()>;
}
pub(crate) trait ITx<'a> {
diff --git a/src/db/sled_adapter.rs b/src/db/sled_adapter.rs
index 7382776f..3388b0ca 100644
--- a/src/db/sled_adapter.rs
+++ b/src/db/sled_adapter.rs
@@ -10,8 +10,7 @@ use sled::transaction::{
};
use crate::{
- Db, Error, Exporter, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value,
- ValueIter,
+ Db, Error, IDb, ITx, ITxFn, IValue, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
};
pub use sled;
@@ -85,6 +84,19 @@ impl IDb for SledDb {
}
}
+ fn list_trees(&self) -> Result<Vec<String>> {
+ let mut trees = vec![];
+ for name in self.db.tree_names() {
+ let name = std::str::from_utf8(&name)
+ .map_err(|e| Error(format!("{}", e).into()))?
+ .to_string();
+ if name != "__sled__default" {
+ trees.push(name);
+ }
+ }
+ Ok(trees)
+ }
+
// ----
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> {
@@ -175,54 +187,6 @@ impl IDb for SledDb {
Err(TransactionError::Storage(s)) => Err(TxError::Db(s.into())),
}
}
-
- // ----
-
- fn export(&self) -> Result<Exporter<'_>> {
- let mut trees = vec![];
- for name in self.db.tree_names() {
- let name = std::str::from_utf8(&name)
- .map_err(|e| Error(format!("{}", e).into()))?
- .to_string();
- let tree = self.open_tree(&name)?;
- let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
- trees.push((name, tree));
- }
- let trees_exporter: Exporter<'_> = Box::new(trees.into_iter().map(|(name, tree)| {
- let iter: ValueIter<'_> = Box::new(
- tree.iter()
- .map(|v| v.map(|(x, y)| (x.into(), y.into())).map_err(Into::into)),
- );
- Ok((name, iter))
- }));
- Ok(trees_exporter)
- }
-
- fn import(&self, ex: Exporter<'_>) -> Result<()> {
- for ex_tree in ex {
- let (name, data) = ex_tree?;
-
- let tree = self.open_tree(&name)?;
- let tree = self.trees.read().unwrap().0.get(tree).unwrap().clone();
- if !tree.is_empty() {
- return Err(Error(format!("tree {} already contains data", name).into()));
- }
-
- let mut i = 0;
- for item in data {
- let (k, v) = item?;
- tree.insert(k, v)?;
- i += 1;
- if i % 1000 == 0 {
- println!("{}: imported {}", name, i);
- }
- }
- println!("{}: finished importing, {} items", name, i);
- }
- Ok(())
- }
-
- // ----
}
// ----
diff --git a/src/db/sqlite_adapter.rs b/src/db/sqlite_adapter.rs
index 31d20553..386eb951 100644
--- a/src/db/sqlite_adapter.rs
+++ b/src/db/sqlite_adapter.rs
@@ -7,9 +7,7 @@ use std::sync::{Arc, Mutex, MutexGuard, RwLock};
use rusqlite::{params, Connection, Rows, Statement, Transaction};
-use crate::{
- Db, Error, Exporter, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter,
-};
+use crate::{Db, Error, IDb, ITx, ITxFn, Result, TxError, TxFnResult, TxResult, Value, ValueIter};
pub use rusqlite;
@@ -55,8 +53,10 @@ impl SqliteDb {
impl IDb for SqliteDb {
fn open_tree(&self, name: &str) -> Result<usize> {
+ let name = format!("tree_{}", name.replace(":", "_COLON_"));
+
let mut trees = self.trees.write().unwrap();
- if let Some(i) = trees.iter().position(|x| x == name) {
+ if let Some(i) = trees.iter().position(|x| x == &name) {
Ok(i)
} else {
self.db.lock().unwrap().execute(
@@ -75,6 +75,21 @@ impl IDb for SqliteDb {
}
}
+ fn list_trees(&self) -> Result<Vec<String>> {
+ let mut trees = vec![];
+ let db = self.db.lock().unwrap();
+ let mut stmt = db.prepare(
+ "SELECT name FROM sqlite_schema WHERE type = 'table' AND name LIKE 'tree_%'",
+ )?;
+ let mut rows = stmt.query([])?;
+ while let Some(row) = rows.next()? {
+ let name = row.get::<_, String>(0)?;
+ let name = name.replace("_COLON_", ":");
+ trees.push(name);
+ }
+ Ok(trees)
+ }
+
// ----
fn get(&self, tree: usize, key: &[u8]) -> Result<Option<Value<'_>>> {
@@ -197,18 +212,6 @@ impl IDb for SqliteDb {
}
}
}
-
- // ----
-
- fn export(&self) -> Result<Exporter<'_>> {
- unimplemented!()
- }
-
- fn import(&self, ex: Exporter<'_>) -> Result<()> {
- unimplemented!()
- }
-
- // ----
}
// ----
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 00:00:21 +0000