diff options
| author | Alex Auvolat <alex@adnab.me> | 2022-06-24 10:31:11 +0200 |
|---|---|---|
| committer | Alex Auvolat <alex@adnab.me> | 2022-06-24 10:31:11 +0200 |
| commit | 95ffba343f14d7274e08099b9aca5a85da2259ed (patch) | |
| tree | f6a965fcce0a11dbab951e13acef24ab8ab7e9d9 /src/util | |
| parent | 59b43914d4a9ae9a50ae79fee61b1a46bff941f9 (diff) | |
| download | garage-95ffba343f14d7274e08099b9aca5a85da2259ed.tar.gz garage-95ffba343f14d7274e08099b9aca5a85da2259ed.zip | |
Error reporting
Diffstat (limited to 'src/util')
| -rw-r--r-- | src/util/background/mod.rs | 2 | ||||
| -rw-r--r-- | src/util/background/worker.rs | 15 |
2 files changed, 11 insertions, 6 deletions
diff --git a/src/util/background/mod.rs b/src/util/background/mod.rs index f7e15b80..636b9c13 100644 --- a/src/util/background/mod.rs +++ b/src/util/background/mod.rs @@ -33,7 +33,7 @@ pub struct WorkerInfo { pub status: WorkerStatus, pub errors: usize, pub consecutive_errors: usize, - pub last_error: Option<String>, + pub last_error: Option<(String, u64)>, } impl BackgroundRunner { diff --git a/src/util/background/worker.rs b/src/util/background/worker.rs index e4a04250..c08a0aaa 100644 --- a/src/util/background/worker.rs +++ b/src/util/background/worker.rs @@ -13,6 +13,7 @@ use tracing::*; use crate::background::WorkerInfo; use crate::error::Error; +use crate::time::now_msec; #[derive(PartialEq, Copy, Clone, Debug, Serialize, Deserialize)] pub enum WorkerStatus { @@ -167,7 +168,7 @@ impl WorkerProcessor { select! { _ = drain_everything => { - info!("All workers exited in time \\o/"); + info!("All workers exited peacefully \\o/"); } _ = tokio::time::sleep(Duration::from_secs(9)) => { error!("Some workers could not exit in time, we are cancelling some things in the middle"); @@ -176,7 +177,6 @@ impl WorkerProcessor { } } -// TODO add tranquilizer struct WorkerHandler { task_id: usize, stop_signal: watch::Receiver<bool>, @@ -185,7 +185,7 @@ struct WorkerHandler { status: WorkerStatus, errors: usize, consecutive_errors: usize, - last_error: Option<String>, + last_error: Option<(String, u64)>, } impl WorkerHandler { @@ -205,7 +205,7 @@ impl WorkerHandler { ); self.errors += 1; self.consecutive_errors += 1; - self.last_error = Some(format!("{}", e)); + self.last_error = Some((format!("{}", e), now_msec())); // Sleep a bit so that error won't repeat immediately, exponential backoff // strategy (min 1sec, max ~60sec) self.status = WorkerStatus::Throttled( @@ -215,7 +215,12 @@ impl WorkerHandler { }, WorkerStatus::Throttled(delay) => { // Sleep for given delay and go back to busy state - tokio::time::sleep(Duration::from_secs_f32(delay)).await; + if !*self.stop_signal.borrow() { + select! { + _ = tokio::time::sleep(Duration::from_secs_f32(delay)) => (), + _ = self.stop_signal.changed() => (), + } + } self.status = WorkerStatus::Busy; } WorkerStatus::Idle => { |