aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/block/repair.rs13
-rw-r--r--src/block/resync.rs4
-rw-r--r--src/garage/admin.rs11
-rw-r--r--src/garage/cli/cmd.rs5
-rw-r--r--src/garage/cli/structs.rs3
-rw-r--r--src/garage/cli/util.rs61
6 files changed, 86 insertions, 11 deletions
diff --git a/src/block/repair.rs b/src/block/repair.rs
index eed40599..1878027e 100644
--- a/src/block/repair.rs
+++ b/src/block/repair.rs
@@ -67,14 +67,17 @@ impl Worker for RepairWorker {
idx_bytes
};
WorkerStatus {
- progress: Some("Phase 1".into()),
- freeform: vec![format!("Now at: {}", hex::encode(idx_bytes))],
+ progress: Some("0.00%".into()),
+ freeform: vec![format!(
+ "Currently in phase 1, iterator position: {}",
+ hex::encode(idx_bytes)
+ )],
..Default::default()
}
}
Some(bi) => WorkerStatus {
progress: Some(format!("{:.2}%", bi.progress() * 100.)),
- freeform: vec!["Phase 2".into()],
+ freeform: vec!["Currently in phase 2".into()],
..Default::default()
},
}
@@ -291,11 +294,11 @@ impl Worker for ScrubWorker {
}
ScrubWorkerState::Paused(bsi, rt) => {
s.progress = Some(format!("{:.2}%", bsi.progress() * 100.));
- s.freeform = vec![format!("Paused, resumes at {}", msec_to_rfc3339(*rt))];
+ s.freeform = vec![format!("Scrub paused, resumes at {}", msec_to_rfc3339(*rt))];
}
ScrubWorkerState::Finished => {
s.freeform = vec![format!(
- "Completed {}",
+ "Last scrub completed at {}",
msec_to_rfc3339(self.persisted.time_last_complete_scrub)
)];
}
diff --git a/src/block/resync.rs b/src/block/resync.rs
index 875ead9b..55d28c14 100644
--- a/src/block/resync.rs
+++ b/src/block/resync.rs
@@ -257,7 +257,7 @@ impl BlockResyncManager {
if let Err(e) = &res {
manager.metrics.resync_error_counter.add(1);
- warn!("Error when resyncing {:?}: {}", hash, e);
+ error!("Error when resyncing {:?}: {}", hash, e);
let err_counter = match self.errors.get(hash.as_slice())? {
Some(ec) => ErrorCounter::decode(&ec).add1(now + 1),
@@ -482,7 +482,7 @@ impl Worker for ResyncWorker {
if self.index >= persisted.n_workers {
return WorkerStatus {
- freeform: vec!["(unused)".into()],
+ freeform: vec!["This worker is currently disabled".into()],
..Default::default()
};
}
diff --git a/src/garage/admin.rs b/src/garage/admin.rs
index da324882..e5bf5601 100644
--- a/src/garage/admin.rs
+++ b/src/garage/admin.rs
@@ -54,6 +54,7 @@ pub enum AdminRpc {
HashMap<usize, garage_util::background::WorkerInfo>,
WorkerListOpt,
),
+ WorkerInfo(usize, garage_util::background::WorkerInfo),
}
impl Rpc for AdminRpc {
@@ -880,6 +881,16 @@ impl AdminRpcHandler {
let workers = self.garage.background.get_worker_info();
Ok(AdminRpc::WorkerList(workers, opt))
}
+ WorkerCmd::Info { tid } => {
+ let info = self
+ .garage
+ .background
+ .get_worker_info()
+ .get(&tid)
+ .ok_or_bad_request(format!("No worker with TID {}", tid))?
+ .clone();
+ Ok(AdminRpc::WorkerInfo(tid, info))
+ }
WorkerCmd::Set { opt } => match opt {
WorkerSetCmd::ScrubTranquility { tranquility } => {
let scrub_command = ScrubWorkerCommand::SetTranquility(tranquility);
diff --git a/src/garage/cli/cmd.rs b/src/garage/cli/cmd.rs
index c8b96489..6df15a48 100644
--- a/src/garage/cli/cmd.rs
+++ b/src/garage/cli/cmd.rs
@@ -186,7 +186,10 @@ pub async fn cmd_admin(
print_key_info(&key, &rb);
}
AdminRpc::WorkerList(wi, wlo) => {
- print_worker_info(wi, wlo);
+ print_worker_list(wi, wlo);
+ }
+ AdminRpc::WorkerInfo(tid, wi) => {
+ print_worker_info(tid, wi);
}
r => {
error!("Unexpected response: {:?}", r);
diff --git a/src/garage/cli/structs.rs b/src/garage/cli/structs.rs
index 59e6e34f..9334564b 100644
--- a/src/garage/cli/structs.rs
+++ b/src/garage/cli/structs.rs
@@ -516,6 +516,9 @@ pub enum WorkerCmd {
#[structopt(flatten)]
opt: WorkerListOpt,
},
+ /// Get detailed information about a worker
+ #[structopt(name = "info", version = garage_version())]
+ Info { tid: usize },
/// Set worker parameter
#[structopt(name = "set", version = garage_version())]
Set {
diff --git a/src/garage/cli/util.rs b/src/garage/cli/util.rs
index 1f098b47..c1d03b8d 100644
--- a/src/garage/cli/util.rs
+++ b/src/garage/cli/util.rs
@@ -241,7 +241,7 @@ pub fn find_matching_node(
}
}
-pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
+pub fn print_worker_list(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
let mut wi = wi.into_iter().collect::<Vec<_>>();
wi.sort_by_key(|(tid, info)| {
(
@@ -284,13 +284,13 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
.tranquility
.as_ref()
.map(ToString::to_string)
- .unwrap_or("-".into()),
+ .unwrap_or_else(|| "-".into()),
info.status.progress.as_deref().unwrap_or("-"),
info.status
.queue_length
.as_ref()
.map(ToString::to_string)
- .unwrap_or("-".into()),
+ .unwrap_or_else(|| "-".into()),
total_err,
consec_err,
err_ago,
@@ -298,3 +298,58 @@ pub fn print_worker_info(wi: HashMap<usize, WorkerInfo>, wlo: WorkerListOpt) {
}
format_table(table);
}
+
+pub fn print_worker_info(tid: usize, info: WorkerInfo) {
+ let mut table = vec![];
+ table.push(format!("Task id:\t{}", tid));
+ table.push(format!("Worker name:\t{}", info.name));
+ match info.state {
+ WorkerState::Throttled(t) => {
+ table.push(format!(
+ "Worker state:\tBusy (throttled, paused for {:.3}s)",
+ t
+ ));
+ }
+ s => {
+ table.push(format!("Worker state:\t{}", s));
+ }
+ };
+ if let Some(tql) = info.status.tranquility {
+ table.push(format!("Tranquility:\t{}", tql));
+ }
+
+ table.push("".into());
+ table.push(format!("Total errors:\t{}", info.errors));
+ table.push(format!("Consecutive errs:\t{}", info.consecutive_errors));
+ if let Some((s, t)) = info.last_error {
+ table.push(format!("Last error:\t{}", s));
+ let tf = timeago::Formatter::new();
+ table.push(format!(
+ "Last error time:\t{}",
+ tf.convert(Duration::from_millis(now_msec() - t))
+ ));
+ }
+
+ table.push("".into());
+ if let Some(p) = info.status.progress {
+ table.push(format!("Progress:\t{}", p));
+ }
+ if let Some(ql) = info.status.queue_length {
+ table.push(format!("Queue length:\t{}", ql));
+ }
+ if let Some(pe) = info.status.persistent_errors {
+ table.push(format!("Persistent errors:\t{}", pe));
+ }
+
+ for (i, s) in info.status.freeform.iter().enumerate() {
+ if i == 0 {
+ if table.last() != Some(&"".into()) {
+ table.push("".into());
+ }
+ table.push(format!("Message:\t{}", s));
+ } else {
+ table.push(format!("\t{}", s));
+ }
+ }
+ format_table(table);
+}