From 9d82196945f751c825621573657cfead992b356b Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 13 Dec 2022 12:24:30 +0100 Subject: cli: new worker info command --- src/block/repair.rs | 13 ++++++---- src/block/resync.rs | 4 ++-- src/garage/admin.rs | 11 +++++++++ src/garage/cli/cmd.rs | 5 +++- src/garage/cli/structs.rs | 3 +++ src/garage/cli/util.rs | 61 ++++++++++++++++++++++++++++++++++++++++++++--- 6 files changed, 86 insertions(+), 11 deletions(-) diff --git a/src/block/repair.rs b/src/block/repair.rs index eed40599..1878027e 100644 --- a/src/block/repair.rs +++ b/src/block/repair.rs @@ -67,14 +67,17 @@ impl Worker for RepairWorker { idx_bytes }; WorkerStatus { - progress: Some("Phase 1".into()), - freeform: vec![format!("Now at: {}", hex::encode(idx_bytes))], + progress: Some("0.00%".into()), + freeform: vec![format!( + "Currently in phase 1, iterator position: {}", + hex::encode(idx_bytes) + )], ..Default::default() } } Some(bi) => WorkerStatus { progress: Some(format!("{:.2}%", bi.progress() * 100.)), - freeform: vec!["Phase 2".into()], + freeform: vec!["Currently in phase 2".into()], ..Default::default() }, } @@ -291,11 +294,11 @@ impl Worker for ScrubWorker { } ScrubWorkerState::Paused(bsi, rt) => { s.progress = Some(format!("{:.2}%", bsi.progress() * 100.)); - s.freeform = vec![format!("Paused, resumes at {}", msec_to_rfc3339(*rt))]; + s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))]; } ScrubWorkerState::Finished => { s.freeform = vec![format!( - "Completed {}", + "Last scrub completed at {}", msec_to_rfc3339(self.persisted.time_last_complete_scrub) )]; } diff --git a/src/block/resync.rs b/src/block/resync.rs index 875ead9b..55d28c14 100644 --- a/src/block/resync.rs +++ b/src/block/resync.rs @@ -257,7 +257,7 @@ impl BlockResyncManager { if let Err(e) = &res { manager.metrics.resync_error_counter.add(1); - warn!("Error when resyncing {:?}: {}", hash, e); + error!("Error when resyncing {:?}: {}", hash, e); let err_counter = match self.errors.get(hash.as_slice())? { Some(ec) => ErrorCounter::decode(&ec).add1(now + 1), @@ -482,7 +482,7 @@ impl Worker for ResyncWorker { if self.index >= persisted.n_workers { return WorkerStatus { - freeform: vec!["(unused)".into()], + freeform: vec!["This worker is currently disabled".into()], ..Default::default() }; } diff --git a/src/garage/admin.rs b/src/garage/admin.rs index da324882..e5bf5601 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -54,6 +54,7 @@ pub enum AdminRpc { HashMap, WorkerListOpt, ), + WorkerInfo(usize, garage_util::background::WorkerInfo), } impl Rpc for AdminRpc { @@ -880,6 +881,16 @@ impl AdminRpcHandler { let workers = self.garage.background.get_worker_info(); Ok(AdminRpc::WorkerList(workers, opt)) } + WorkerCmd::Info { tid } => { + let info = self + .garage + .background + .get_worker_info() + .get(&tid) + .ok_or_bad_request(format!("No worker with TID {}", tid))? + .clone(); + Ok(AdminRpc::WorkerInfo(tid, info)) + } WorkerCmd::Set { opt } => match opt { WorkerSetCmd::ScrubTranquility { tranquility } => { let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility); diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs index c8b96489..6df15a48 100644 --- a/src/garage/cli/cmd.rs +++ b/src/garage/cli/cmd.rs @@ -186,7 +186,10 @@ pub async fn cmd_admin( print_key_info(&key, &rb); } AdminRpc::WorkerList(wi, wlo) => { - print_worker_info(wi, wlo); + print_worker_list(wi, wlo); + } + AdminRpc::WorkerInfo(tid, wi) => { + print_worker_info(tid, wi); } r => { error!("Unexpected response: {:?}", r); diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs index 59e6e34f..9334564b 100644 --- a/src/garage/cli/structs.rs +++ b/src/garage/cli/structs.rs @@ -516,6 +516,9 @@ pub enum WorkerCmd { #[structopt(flatten)] opt: WorkerListOpt, }, + /// Get detailed information about a worker + #[structopt(name = "info", version = garage_version())] + Info { tid: usize }, /// Set worker parameter #[structopt(name = "set", version = garage_version())] Set { diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs index 1f098b47..c1d03b8d 100644 --- a/src/garage/cli/util.rs +++ b/src/garage/cli/util.rs @@ -241,7 +241,7 @@ pub fn find_matching_node( } } -pub fn print_worker_info(wi: HashMap, wlo: WorkerListOpt) { +pub fn print_worker_list(wi: HashMap, wlo: WorkerListOpt) { let mut wi = wi.into_iter().collect::>(); wi.sort_by_key(|(tid, info)| { ( @@ -284,13 +284,13 @@ pub fn print_worker_info(wi: HashMap, wlo: WorkerListOpt) { .tranquility .as_ref() .map(ToString::to_string) - .unwrap_or("-".into()), + .unwrap_or_else(|| "-".into()), info.status.progress.as_deref().unwrap_or("-"), info.status .queue_length .as_ref() .map(ToString::to_string) - .unwrap_or("-".into()), + .unwrap_or_else(|| "-".into()), total_err, consec_err, err_ago, @@ -298,3 +298,58 @@ pub fn print_worker_info(wi: HashMap, wlo: WorkerListOpt) { } format_table(table); } + +pub fn print_worker_info(tid: usize, info: WorkerInfo) { + let mut table = vec![]; + table.push(format!("Task id:\t{}", tid)); + table.push(format!("Worker name:\t{}", info.name)); + match info.state { + WorkerState::Throttled(t) => { + table.push(format!( + "Worker state:\tBusy (throttled, paused for {:.3}s)", + t + )); + } + s => { + table.push(format!("Worker state:\t{}", s)); + } + }; + if let Some(tql) = info.status.tranquility { + table.push(format!("Tranquility:\t{}", tql)); + } + + table.push("".into()); + table.push(format!("Total errors:\t{}", info.errors)); + table.push(format!("Consecutive errs:\t{}", info.consecutive_errors)); + if let Some((s, t)) = info.last_error { + table.push(format!("Last error:\t{}", s)); + let tf = timeago::Formatter::new(); + table.push(format!( + "Last error time:\t{}", + tf.convert(Duration::from_millis(now_msec() - t)) + )); + } + + table.push("".into()); + if let Some(p) = info.status.progress { + table.push(format!("Progress:\t{}", p)); + } + if let Some(ql) = info.status.queue_length { + table.push(format!("Queue length:\t{}", ql)); + } + if let Some(pe) = info.status.persistent_errors { + table.push(format!("Persistent errors:\t{}", pe)); + } + + for (i, s) in info.status.freeform.iter().enumerate() { + if i == 0 { + if table.last() != Some(&"".into()) { + table.push("".into()); + } + table.push(format!("Message:\t{}", s)); + } else { + table.push(format!("\t{}", s)); + } + } + format_table(table); +} -- cgit v1.2.3