aboutsummaryrefslogtreecommitdiff
path: root/src/util
diff options
context:
space:
mode:
Diffstat (limited to 'src/util')
-rw-r--r--src/util/Cargo.toml29
-rw-r--r--src/util/background.rs185
-rw-r--r--src/util/config.rs28
-rw-r--r--src/util/data.rs29
-rw-r--r--src/util/error.rs8
-rw-r--r--src/util/lib.rs1
-rw-r--r--src/util/time.rs16
7 files changed, 183 insertions, 113 deletions
diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml
index 35130c96..79611752 100644
--- a/src/util/Cargo.toml
+++ b/src/util/Cargo.toml
@@ -1,9 +1,9 @@
[package]
name = "garage_util"
-version = "0.1.1"
+version = "0.2.1"
authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018"
-license = "GPL-3.0"
+license = "AGPL-3.0"
description = "Utility crate for the Garage object store"
repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
@@ -13,29 +13,26 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
-rand = "0.7"
-hex = "0.3"
-sha2 = "0.8"
blake2 = "0.9"
-err-derive = "0.2.3"
+err-derive = "0.3"
+xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] }
+hex = "0.4"
log = "0.4"
-fasthash = "0.4"
+rand = "0.8"
+sha2 = "0.9"
sled = "0.34"
-toml = "0.5"
-rmp-serde = "0.14.3"
+chrono = "0.4"
+rmp-serde = "0.15"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_json = "1.0"
+toml = "0.5"
futures = "0.3"
-futures-util = "0.3"
-tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
+tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
http = "0.2"
-hyper = "0.13"
-rustls = "0.17"
+hyper = "0.14"
+rustls = "0.19"
webpki = "0.21"
-
-roxmltree = "0.11"
-
diff --git a/src/util/background.rs b/src/util/background.rs
index 937062dd..b5eb8bc8 100644
--- a/src/util/background.rs
+++ b/src/util/background.rs
@@ -1,12 +1,11 @@
use core::future::Future;
use std::pin::Pin;
+use std::sync::Arc;
+use std::time::Duration;
-use futures::future::join_all;
+use futures::future::*;
use futures::select;
-use futures_util::future::*;
-use std::sync::Arc;
-use tokio::sync::Mutex;
-use tokio::sync::{mpsc, watch, Notify};
+use tokio::sync::{mpsc, watch, Mutex};
use crate::error::Error;
@@ -14,54 +13,106 @@ type JobOutput = Result<(), Error>;
type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>;
pub struct BackgroundRunner {
- n_runners: usize,
pub stop_signal: watch::Receiver<bool>,
queue_in: mpsc::UnboundedSender<(Job, bool)>,
- queue_out: Mutex<mpsc::UnboundedReceiver<(Job, bool)>>,
- job_notify: Notify,
-
- workers: Mutex<Vec<tokio::task::JoinHandle<()>>>,
+ worker_in: mpsc::UnboundedSender<tokio::task::JoinHandle<()>>,
}
impl BackgroundRunner {
- pub fn new(n_runners: usize, stop_signal: watch::Receiver<bool>) -> Arc<Self> {
+ pub fn new(
+ n_runners: usize,
+ stop_signal: watch::Receiver<bool>,
+ ) -> (Arc<Self>, tokio::task::JoinHandle<()>) {
+ let (worker_in, mut worker_out) = mpsc::unbounded_channel();
+
+ let stop_signal_2 = stop_signal.clone();
+ let await_all_done = tokio::spawn(async move {
+ loop {
+ let wkr = {
+ select! {
+ item = worker_out.recv().fuse() => {
+ match item {
+ Some(x) => x,
+ None => break,
+ }
+ }
+ _ = tokio::time::sleep(Duration::from_secs(5)).fuse() => {
+ if *stop_signal_2.borrow() {
+ break;
+ } else {
+ continue;
+ }
+ }
+ }
+ };
+ if let Err(e) = wkr.await {
+ error!("Error while awaiting for worker: {}", e);
+ }
+ }
+ });
+
let (queue_in, queue_out) = mpsc::unbounded_channel();
- Arc::new(Self {
- n_runners,
- stop_signal,
- queue_in,
- queue_out: Mutex::new(queue_out),
- job_notify: Notify::new(),
- workers: Mutex::new(Vec::new()),
- })
- }
+ let queue_out = Arc::new(Mutex::new(queue_out));
- pub async fn run(self: Arc<Self>) {
- let mut workers = self.workers.lock().await;
- for i in 0..self.n_runners {
- workers.push(tokio::spawn(self.clone().runner(i)));
- }
- drop(workers);
+ for i in 0..n_runners {
+ let queue_out = queue_out.clone();
+ let stop_signal = stop_signal.clone();
- let mut stop_signal = self.stop_signal.clone();
- while let Some(exit_now) = stop_signal.recv().await {
- if exit_now {
- let mut workers = self.workers.lock().await;
- let workers_vec = workers.drain(..).collect::<Vec<_>>();
- join_all(workers_vec).await;
- return;
- }
+ worker_in
+ .send(tokio::spawn(async move {
+ loop {
+ let (job, cancellable) = {
+ select! {
+ item = wait_job(&queue_out).fuse() => match item {
+ // We received a task, process it
+ Some(x) => x,
+ // We received a signal that no more tasks will ever be sent
+ // because the sending side was dropped. Exit now.
+ None => break,
+ },
+ _ = tokio::time::sleep(Duration::from_secs(5)).fuse() => {
+ if *stop_signal.borrow() {
+ // Nothing has been going on for 5 secs, and we are shutting
+ // down. Exit now.
+ break;
+ } else {
+ // Nothing is going on but we don't want to exit.
+ continue;
+ }
+ }
+ }
+ };
+ if cancellable && *stop_signal.borrow() {
+ continue;
+ }
+ if let Err(e) = job.await {
+ error!("Job failed: {}", e)
+ }
+ }
+ info!("Background worker {} exiting", i);
+ }))
+ .unwrap();
}
+
+ let bgrunner = Arc::new(Self {
+ stop_signal,
+ queue_in,
+ worker_in,
+ });
+ (bgrunner, await_all_done)
}
+ // Spawn a task to be run in background
pub fn spawn<T>(&self, job: T)
where
T: Future<Output = JobOutput> + Send + 'static,
{
let boxed: Job = Box::pin(job);
- let _: Result<_, _> = self.queue_in.clone().send((boxed, false));
- self.job_notify.notify();
+ self.queue_in
+ .send((boxed, false))
+ .map_err(|_| "could not put job in queue")
+ .unwrap();
}
pub fn spawn_cancellable<T>(&self, job: T)
@@ -69,56 +120,30 @@ impl BackgroundRunner {
T: Future<Output = JobOutput> + Send + 'static,
{
let boxed: Job = Box::pin(job);
- let _: Result<_, _> = self.queue_in.clone().send((boxed, true));
- self.job_notify.notify();
+ self.queue_in
+ .send((boxed, true))
+ .map_err(|_| "could not put job in queue")
+ .unwrap();
}
- pub async fn spawn_worker<F, T>(&self, name: String, worker: F)
+ pub fn spawn_worker<F, T>(&self, name: String, worker: F)
where
F: FnOnce(watch::Receiver<bool>) -> T + Send + 'static,
- T: Future<Output = JobOutput> + Send + 'static,
+ T: Future<Output = ()> + Send + 'static,
{
- let mut workers = self.workers.lock().await;
let stop_signal = self.stop_signal.clone();
- workers.push(tokio::spawn(async move {
- if let Err(e) = worker(stop_signal).await {
- error!("Worker stopped with error: {}, error: {}", name, e);
- } else {
- info!("Worker exited successfully: {}", name);
- }
- }));
- }
-
- async fn runner(self: Arc<Self>, i: usize) {
- let mut stop_signal = self.stop_signal.clone();
- loop {
- let must_exit: bool = *stop_signal.borrow();
- if let Some(job) = self.dequeue_job(must_exit).await {
- if let Err(e) = job.await {
- error!("Job failed: {}", e)
- }
- } else {
- if must_exit {
- info!("Background runner {} exiting", i);
- return;
- }
- select! {
- _ = self.job_notify.notified().fuse() => (),
- _ = stop_signal.recv().fuse() => (),
- }
- }
- }
+ let task = tokio::spawn(async move {
+ info!("Worker started: {}", name);
+ worker(stop_signal).await;
+ info!("Worker exited: {}", name);
+ });
+ self.worker_in
+ .send(task)
+ .map_err(|_| "could not put job in queue")
+ .unwrap();
}
+}
- async fn dequeue_job(&self, must_exit: bool) -> Option<Job> {
- let mut queue = self.queue_out.lock().await;
- while let Ok((job, cancellable)) = queue.try_recv() {
- if cancellable && must_exit {
- continue;
- } else {
- return Some(job);
- }
- }
- None
- }
+async fn wait_job(q: &Mutex<mpsc::UnboundedReceiver<(Job, bool)>>) -> Option<(Job, bool)> {
+ q.lock().await.recv().await
}
diff --git a/src/util/config.rs b/src/util/config.rs
index cd65e009..9ff67711 100644
--- a/src/util/config.rs
+++ b/src/util/config.rs
@@ -2,7 +2,7 @@ use std::io::Read;
use std::net::SocketAddr;
use std::path::PathBuf;
-use serde::Deserialize;
+use serde::{de, Deserialize};
use crate::error::Error;
@@ -13,6 +13,7 @@ pub struct Config {
pub rpc_bind_addr: SocketAddr,
+ #[serde(deserialize_with = "deserialize_vec_addr")]
pub bootstrap_peers: Vec<SocketAddr>,
pub consul_host: Option<String>,
pub consul_service_name: Option<String>,
@@ -82,3 +83,28 @@ pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
Ok(toml::from_str(&config)?)
}
+
+fn deserialize_vec_addr<'de, D>(deserializer: D) -> Result<Vec<SocketAddr>, D::Error>
+where
+ D: de::Deserializer<'de>,
+{
+ use std::net::ToSocketAddrs;
+
+ Ok(<Vec<&str>>::deserialize(deserializer)?
+ .iter()
+ .filter_map(|&name| {
+ name.to_socket_addrs()
+ .map(|iter| (name, iter))
+ .map_err(|_| warn!("Error resolving \"{}\"", name))
+ .ok()
+ })
+ .map(|(name, iter)| {
+ let v = iter.collect::<Vec<_>>();
+ if v.is_empty() {
+ warn!("Error resolving \"{}\"", name)
+ }
+ v
+ })
+ .flatten()
+ .collect())
+}
diff --git a/src/util/data.rs b/src/util/data.rs
index f46454be..8cd6dd96 100644
--- a/src/util/data.rs
+++ b/src/util/data.rs
@@ -2,7 +2,6 @@ use rand::Rng;
use serde::de::{self, Visitor};
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use std::fmt;
-use std::time::{SystemTime, UNIX_EPOCH};
#[derive(Default, PartialOrd, Ord, Clone, Hash, PartialEq, Copy)]
pub struct FixedBytes32([u8; 32]);
@@ -71,6 +70,14 @@ impl FixedBytes32 {
pub fn to_vec(&self) -> Vec<u8> {
self.0.to_vec()
}
+ pub fn try_from(by: &[u8]) -> Option<Self> {
+ if by.len() != 32 {
+ return None;
+ }
+ let mut ret = [0u8; 32];
+ ret.copy_from_slice(by);
+ Some(Self(ret))
+ }
}
pub type UUID = FixedBytes32;
@@ -80,9 +87,9 @@ pub fn sha256sum(data: &[u8]) -> Hash {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
- hasher.input(data);
+ hasher.update(data);
let mut hash = [0u8; 32];
- hash.copy_from_slice(&hasher.result()[..]);
+ hash.copy_from_slice(&hasher.finalize()[..]);
hash.into()
}
@@ -99,25 +106,17 @@ pub fn blake2sum(data: &[u8]) -> Hash {
pub type FastHash = u64;
pub fn fasthash(data: &[u8]) -> FastHash {
- use fasthash::{xx::Hasher64, FastHasher};
- use std::hash::Hasher;
+ use xxhash_rust::xxh3::Xxh3;
- let mut h = Hasher64::new();
- h.write(data);
- h.finish()
+ let mut h = Xxh3::new();
+ h.update(data);
+ h.digest()
}
pub fn gen_uuid() -> UUID {
rand::thread_rng().gen::<[u8; 32]>().into()
}
-pub fn now_msec() -> u64 {
- SystemTime::now()
- .duration_since(UNIX_EPOCH)
- .expect("Fix your clock :o")
- .as_millis() as u64
-}
-
// RMP serialization with names of fields and variants
pub fn rmp_to_vec_all_named<T>(val: &T) -> Result<Vec<u8>, rmp_serde::encode::Error>
diff --git a/src/util/error.rs b/src/util/error.rs
index dbf71ac1..a9bf0824 100644
--- a/src/util/error.rs
+++ b/src/util/error.rs
@@ -8,16 +8,22 @@ use crate::data::*;
pub enum RPCError {
#[error(display = "Node is down: {:?}.", _0)]
NodeDown(UUID),
+
#[error(display = "Timeout: {}", _0)]
- Timeout(#[error(source)] tokio::time::Elapsed),
+ Timeout(#[error(source)] tokio::time::error::Elapsed),
+
#[error(display = "HTTP error: {}", _0)]
HTTP(#[error(source)] http::Error),
+
#[error(display = "Hyper error: {}", _0)]
Hyper(#[error(source)] hyper::Error),
+
#[error(display = "Messagepack encode error: {}", _0)]
RMPEncode(#[error(source)] rmp_serde::encode::Error),
+
#[error(display = "Messagepack decode error: {}", _0)]
RMPDecode(#[error(source)] rmp_serde::decode::Error),
+
#[error(display = "Too many errors: {:?}", _0)]
TooManyErrors(Vec<String>),
}
diff --git a/src/util/lib.rs b/src/util/lib.rs
index 0bf09bf6..e544a872 100644
--- a/src/util/lib.rs
+++ b/src/util/lib.rs
@@ -5,3 +5,4 @@ pub mod background;
pub mod config;
pub mod data;
pub mod error;
+pub mod time;
diff --git a/src/util/time.rs b/src/util/time.rs
new file mode 100644
index 00000000..148860e0
--- /dev/null
+++ b/src/util/time.rs
@@ -0,0 +1,16 @@
+use chrono::{SecondsFormat, TimeZone, Utc};
+use std::time::{SystemTime, UNIX_EPOCH};
+
+pub fn now_msec() -> u64 {
+ SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .expect("Fix your clock :o")
+ .as_millis() as u64
+}
+
+pub fn msec_to_rfc3339(msecs: u64) -> String {
+ let secs = msecs as i64 / 1000;
+ let nanos = (msecs as i64 % 1000) as u32 * 1_000_000;
+ let timestamp = Utc.timestamp(secs, nanos);
+ timestamp.to_rfc3339_opts(SecondsFormat::Secs, true)
+}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-22 11:16:13 +0000