WIP migrate to tokio 1

3 files changed, 78 insertions, 68 deletions

diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml
index 7bb7cb31..2ae4796c 100644
--- a/src/util/Cargo.toml
+++ b/src/util/Cargo.toml
@@ -28,14 +28,15 @@ rmp-serde = "0.14.3"
 serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
 serde_json = "1.0"
 chrono = "0.4"
+arc-swap = "1.2"
 
 futures = "0.3"
 futures-util = "0.3"
-tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
+tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
 
 http = "0.2"
-hyper = "0.13"
-rustls = "0.17"
+hyper = "0.14"
+rustls = "0.19"
 webpki = "0.21"
 
 roxmltree = "0.11"
diff --git a/src/util/background.rs b/src/util/background.rs
index 3e600fdf..0ec9779a 100644
--- a/src/util/background.rs
+++ b/src/util/background.rs
@@ -2,11 +2,9 @@ use core::future::Future;
 use std::pin::Pin;
 use std::sync::Mutex;
 
-use futures::future::join_all;
-use futures::select;
-use futures_util::future::*;
+use arc_swap::ArcSwapOption;
 use std::sync::Arc;
-use tokio::sync::{mpsc, watch, Notify};
+use tokio::sync::{mpsc, watch};
 
 use crate::error::Error;
 
@@ -14,12 +12,9 @@ type JobOutput = Result<(), Error>;
 type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>;
 
 pub struct BackgroundRunner {
-	n_runners: usize,
 	pub stop_signal: watch::Receiver<bool>,
 
-	queue_in: mpsc::UnboundedSender<(Job, bool)>,
-	queue_out: Mutex<mpsc::UnboundedReceiver<(Job, bool)>>,
-	job_notify: Notify,
+	queue_in: ArcSwapOption<mpsc::UnboundedSender<(Job, bool)>>,
 
 	workers: Mutex<Vec<tokio::task::JoinHandle<()>>>,
 }
@@ -27,50 +22,91 @@ pub struct BackgroundRunner {
 impl BackgroundRunner {
 	pub fn new(n_runners: usize, stop_signal: watch::Receiver<bool>) -> Arc<Self> {
 		let (queue_in, queue_out) = mpsc::unbounded_channel();
+
+		let mut workers = vec![];
+		let queue_out = Arc::new(tokio::sync::Mutex::new(queue_out));
+
+		for i in 0..n_runners {
+			let queue_out = queue_out.clone();
+			let stop_signal = stop_signal.clone();
+
+			workers.push(tokio::spawn(async move {
+				while let Some((job, cancellable)) = queue_out.lock().await.recv().await {
+					if cancellable && *stop_signal.borrow() {
+						continue;
+					}
+					if let Err(e) = job.await {
+						error!("Job failed: {}", e)
+					}
+				}
+				info!("Worker {} exiting", i);
+			}));
+		}
+
 		Arc::new(Self {
-			n_runners,
 			stop_signal,
-			queue_in,
-			queue_out: Mutex::new(queue_out),
-			job_notify: Notify::new(),
-			workers: Mutex::new(Vec::new()),
+			queue_in: ArcSwapOption::new(Some(Arc::new(queue_in))),
+			workers: Mutex::new(workers),
 		})
 	}
 
 	pub async fn run(self: Arc<Self>) {
-		let mut workers = self.workers.lock().unwrap();
-		for i in 0..self.n_runners {
-			workers.push(tokio::spawn(self.clone().runner(i)));
-		}
-		drop(workers);
-
 		let mut stop_signal = self.stop_signal.clone();
-		while let Some(exit_now) = stop_signal.recv().await {
+
+		loop {
+			let exit_now = match stop_signal.changed().await {
+				Ok(()) => *stop_signal.borrow(),
+				Err(e) => {
+					error!("Watch .changed() error: {}", e);
+					true
+				}
+			};
 			if exit_now {
-				let mut workers = self.workers.lock().unwrap();
-				let workers_vec = workers.drain(..).collect::<Vec<_>>();
-				join_all(workers_vec).await;
-				return;
+				break;
+			}
+		}
+
+		info!("Closing background job queue_in...");
+		drop(self.queue_in.swap(None));
+
+		info!("Waiting for all workers to terminate...");
+		while let Some(task) = self.workers.lock().unwrap().pop() {
+			if let Err(e) = task.await {
+				warn!("Error awaiting task: {}", e);
 			}
 		}
 	}
 
-	pub fn spawn<T>(&self, job: T)
+	// Spawn a task to be run in background
+	pub async fn spawn<T>(&self, job: T)
 	where
 		T: Future<Output = JobOutput> + Send + 'static,
 	{
-		let boxed: Job = Box::pin(job);
-		let _: Result<_, _> = self.queue_in.clone().send((boxed, false));
-		self.job_notify.notify();
+		match self.queue_in.load().as_ref() {
+			Some(chan) => {
+				let boxed: Job = Box::pin(job);
+				chan.send((boxed, false)).map_err(|_| "send error").unwrap();
+			}
+			None => {
+				warn!("Doing background job now because we are exiting...");
+				if let Err(e) = job.await {
+					warn!("Task failed: {}", e);
+				}
+			}
+		}
 	}
 
 	pub fn spawn_cancellable<T>(&self, job: T)
 	where
 		T: Future<Output = JobOutput> + Send + 'static,
 	{
-		let boxed: Job = Box::pin(job);
-		let _: Result<_, _> = self.queue_in.clone().send((boxed, true));
-		self.job_notify.notify();
+		match self.queue_in.load().as_ref() {
+			Some(chan) => {
+				let boxed: Job = Box::pin(job);
+				chan.send((boxed, false)).map_err(|_| "send error").unwrap();
+			}
+			None => (), // drop job if we are exiting
+		}
 	}
 
 	pub fn spawn_worker<F, T>(&self, name: String, worker: F)
@@ -85,37 +121,4 @@ impl BackgroundRunner {
 			info!("Worker exited: {}", name);
 		}));
 	}
-
-	async fn runner(self: Arc<Self>, i: usize) {
-		let mut stop_signal = self.stop_signal.clone();
-		loop {
-			let must_exit: bool = *stop_signal.borrow();
-			if let Some(job) = self.dequeue_job(must_exit) {
-				if let Err(e) = job.await {
-					error!("Job failed: {}", e)
-				}
-			} else {
-				if must_exit {
-					info!("Background runner {} exiting", i);
-					return;
-				}
-				select! {
-					_ = self.job_notify.notified().fuse() => (),
-					_ = stop_signal.recv().fuse() => (),
-				}
-			}
-		}
-	}
-
-	fn dequeue_job(&self, must_exit: bool) -> Option<Job> {
-		let mut queue = self.queue_out.lock().unwrap();
-		while let Ok((job, cancellable)) = queue.try_recv() {
-			if cancellable && must_exit {
-				continue;
-			} else {
-				return Some(job);
-			}
-		}
-		None
-	}
 }
diff --git a/src/util/error.rs b/src/util/error.rs
index dbf71ac1..a9bf0824 100644
--- a/src/util/error.rs
+++ b/src/util/error.rs
@@ -8,16 +8,22 @@ use crate::data::*;
 pub enum RPCError {
 	#[error(display = "Node is down: {:?}.", _0)]
 	NodeDown(UUID),
+
 	#[error(display = "Timeout: {}", _0)]
-	Timeout(#[error(source)] tokio::time::Elapsed),
+	Timeout(#[error(source)] tokio::time::error::Elapsed),
+
 	#[error(display = "HTTP error: {}", _0)]
 	HTTP(#[error(source)] http::Error),
+
 	#[error(display = "Hyper error: {}", _0)]
 	Hyper(#[error(source)] hyper::Error),
+
 	#[error(display = "Messagepack encode error: {}", _0)]
 	RMPEncode(#[error(source)] rmp_serde::encode::Error),
+
 	#[error(display = "Messagepack decode error: {}", _0)]
 	RMPDecode(#[error(source)] rmp_serde::decode::Error),
+
 	#[error(display = "Too many errors: {:?}", _0)]
 	TooManyErrors(Vec<String>),
 }