cli: new worker info command

6 files changed, 86 insertions, 11 deletions

diff --git a/src/block/repair.rs b/src/block/repair.rs
index eed40599..1878027e 100644
--- a/src/block/repair.rs
+++ b/src/block/repair.rs
@@ -67,14 +67,17 @@ impl Worker for RepairWorker {
 					idx_bytes
 				};
 				WorkerStatus {
-					progress: Some("Phase 1".into()),
-					freeform: vec![format!("Now at: {}", hex::encode(idx_bytes))],
+					progress: Some("0.00%".into()),
+					freeform: vec![format!(
+						"Currently in phase 1, iterator position: {}",
+						hex::encode(idx_bytes)
+					)],
 					..Default::default()
 				}
 			}
 			Some(bi) => WorkerStatus {
 				progress: Some(format!("{:.2}%", bi.progress() * 100.)),
-				freeform: vec!["Phase 2".into()],
+				freeform: vec!["Currently in phase 2".into()],
 				..Default::default()
 			},
 		}
@@ -291,11 +294,11 @@ impl Worker for ScrubWorker {
 			}
 			ScrubWorkerState::Paused(bsi, rt) => {
 				s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
-				s.freeform = vec![format!("Paused, resumes at {}", msec_to_rfc3339(*rt))];
+				s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
 			}
 			ScrubWorkerState::Finished => {
 				s.freeform = vec![format!(
-					"Completed {}",
+					"Last scrub completed at {}",
 					msec_to_rfc3339(self.persisted.time_last_complete_scrub)
 				)];
 			}
diff --git a/src/block/resync.rs b/src/block/resync.rs
index 875ead9b..55d28c14 100644
--- a/src/block/resync.rs
+++ b/src/block/resync.rs
@@ -257,7 +257,7 @@ impl BlockResyncManager {
 
 				if let Err(e) = &res {
 					manager.metrics.resync_error_counter.add(1);
-					warn!("Error when resyncing {:?}: {}", hash, e);
+					error!("Error when resyncing {:?}: {}", hash, e);
 
 					let err_counter = match self.errors.get(hash.as_slice())? {
 						Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
@@ -482,7 +482,7 @@ impl Worker for ResyncWorker {
 
 		if self.index >= persisted.n_workers {
 			return WorkerStatus {
-				freeform: vec!["(unused)".into()],
+				freeform: vec!["This worker is currently disabled".into()],
 				..Default::default()
 			};
 		}
diff --git a/src/garage/admin.rs b/src/garage/admin.rs
index da324882..e5bf5601 100644
--- a/src/garage/admin.rs
+++ b/src/garage/admin.rs
@@ -54,6 +54,7 @@ pub enum AdminRpc {
 		HashMap<usize, garage_util::background::WorkerInfo>,
 		WorkerListOpt,
 	),
+	WorkerInfo(usize, garage_util::background::WorkerInfo),
 }
 
 impl Rpc for AdminRpc {
@@ -880,6 +881,16 @@ impl AdminRpcHandler {
 				let workers = self.garage.background.get_worker_info();
 				Ok(AdminRpc::WorkerList(workers, opt))
 			}
+			WorkerCmd::Info { tid } => {
+				let info = self
+					.garage
+					.background
+					.get_worker_info()
+					.get(&tid)
+					.ok_or_bad_request(format!("No worker with TID {}", tid))?
+					.clone();
+				Ok(AdminRpc::WorkerInfo(tid, info))
+			}
 			WorkerCmd::Set { opt } => match opt {
 				WorkerSetCmd::ScrubTranquility { tranquility } => {
 					let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);
diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs
index c8b96489..6df15a48 100644
--- a/src/garage/cli/cmd.rs
+++ b/src/garage/cli/cmd.rs
@@ -186,7 +186,10 @@ pub async fn cmd_admin(
 			print_key_info(&key, &rb);
 		}
 		AdminRpc::WorkerList(wi, wlo) => {
-			print_worker_info(wi, wlo);
+			print_worker_list(wi, wlo);
+		}
+		AdminRpc::WorkerInfo(tid, wi) => {
+			print_worker_info(tid, wi);
 		}
 		r => {
 			error!("Unexpected response: {:?}", r);
diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs
index 59e6e34f..9334564b 100644
--- a/src/garage/cli/structs.rs
+++ b/src/garage/cli/structs.rs
@@ -516,6 +516,9 @@ pub enum WorkerCmd {
 		#[structopt(flatten)]
 		opt: WorkerListOpt,
 	},
+	/// Get detailed information about a worker
+	#[structopt(name = "info", version = garage_version())]
+	Info { tid: usize },
 	/// Set worker parameter
 	#[structopt(name = "set", version = garage_version())]
 	Set {
diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs
index 1f098b47..c1d03b8d 100644
--- a/src/garage/cli/util.rs
+++ b/src/garage/cli/util.rs
@@ -241,7 +241,7 @@ pub fn find_matching_node(
 	}
 }
 
-pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
+pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 	let mut wi = wi.into_iter().collect::<Vec<_>>();
 	wi.sort_by_key(|(tid, info)| {
 		(
@@ -284,13 +284,13 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 				.tranquility
 				.as_ref()
 				.map(ToString::to_string)
-				.unwrap_or("-".into()),
+				.unwrap_or_else(|| "-".into()),
 			info.status.progress.as_deref().unwrap_or("-"),
 			info.status
 				.queue_length
 				.as_ref()
 				.map(ToString::to_string)
-				.unwrap_or("-".into()),
+				.unwrap_or_else(|| "-".into()),
 			total_err,
 			consec_err,
 			err_ago,
@@ -298,3 +298,58 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
 	}
 	format_table(table);
 }
+
+pub fn print_worker_info(tid: usize, info: WorkerInfo) {
+	let mut table = vec![];
+	table.push(format!("Task id:\t{}", tid));
+	table.push(format!("Worker name:\t{}", info.name));
+	match info.state {
+		WorkerState::Throttled(t) => {
+			table.push(format!(
+				"Worker state:\tBusy (throttled, paused for {:.3}s)",
+				t
+			));
+		}
+		s => {
+			table.push(format!("Worker state:\t{}", s));
+		}
+	};
+	if let Some(tql) = info.status.tranquility {
+		table.push(format!("Tranquility:\t{}", tql));
+	}
+
+	table.push("".into());
+	table.push(format!("Total errors:\t{}", info.errors));
+	table.push(format!("Consecutive errs:\t{}", info.consecutive_errors));
+	if let Some((s, t)) = info.last_error {
+		table.push(format!("Last error:\t{}", s));
+		let tf = timeago::Formatter::new();
+		table.push(format!(
+			"Last error time:\t{}",
+			tf.convert(Duration::from_millis(now_msec() - t))
+		));
+	}
+
+	table.push("".into());
+	if let Some(p) = info.status.progress {
+		table.push(format!("Progress:\t{}", p));
+	}
+	if let Some(ql) = info.status.queue_length {
+		table.push(format!("Queue length:\t{}", ql));
+	}
+	if let Some(pe) = info.status.persistent_errors {
+		table.push(format!("Persistent errors:\t{}", pe));
+	}
+
+	for (i, s) in info.status.freeform.iter().enumerate() {
+		if i == 0 {
+			if table.last() != Some(&"".into()) {
+				table.push("".into());
+			}
+			table.push(format!("Message:\t{}", s));
+		} else {
+			table.push(format!("\t{}", s));
+		}
+	}
+	format_table(table);
+}