diff options
| author | Alex <alex@adnab.me> | 2024-03-28 12:40:27 +0000 |
|---|---|---|
| committer | Alex <alex@adnab.me> | 2024-03-28 12:40:27 +0000 |
| commit | ecf641d88c264f7278d13a6d988288feb24a5dfe (patch) | |
| tree | 5cd60dfa4f0d6d32a66d2e32d7912c9e289067c8 /src/block/metrics.rs | |
| parent | 75cd14926d8dec8c36289197822df78391686c6a (diff) | |
| parent | 85f580cbde4913fe8382316ff3c27b8443c61dd7 (diff) | |
| download | garage-ecf641d88c264f7278d13a6d988288feb24a5dfe.tar.gz garage-ecf641d88c264f7278d13a6d988288feb24a5dfe.zip | |
Merge pull request 'Fix unbounded buffering when one node has slower network' (#792) from fix-buffering into main
Reviewed-on: https://git.deuxfleurs.fr/Deuxfleurs/garage/pulls/792
Diffstat (limited to 'src/block/metrics.rs')
| -rw-r--r-- | src/block/metrics.rs | 15 |
1 files changed, 15 insertions, 0 deletions
diff --git a/src/block/metrics.rs b/src/block/metrics.rs index 6659df32..c989f940 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -1,3 +1,7 @@ +use std::sync::Arc; + +use tokio::sync::Semaphore; + use opentelemetry::{global, metrics::*}; use garage_db as db; @@ -9,6 +13,7 @@ pub struct BlockManagerMetrics { pub(crate) _rc_size: ValueObserver<u64>, pub(crate) _resync_queue_len: ValueObserver<u64>, pub(crate) _resync_errored_blocks: ValueObserver<u64>, + pub(crate) _buffer_free_kb: ValueObserver<u64>, pub(crate) resync_counter: BoundCounter<u64>, pub(crate) resync_error_counter: BoundCounter<u64>, @@ -31,6 +36,7 @@ impl BlockManagerMetrics { rc_tree: db::Tree, resync_queue: CountedTree, resync_errors: CountedTree, + buffer_semaphore: Arc<Semaphore>, ) -> Self { let meter = global::meter("garage_model/block"); Self { @@ -66,6 +72,15 @@ impl BlockManagerMetrics { .with_description("Number of block hashes whose last resync resulted in an error") .init(), + _buffer_free_kb: meter + .u64_value_observer("block.ram_buffer_free_kb", move |observer| { + observer.observe(buffer_semaphore.available_permits() as u64, &[]) + }) + .with_description( + "Available RAM in KiB to use for buffering data blocks to be written to remote nodes", + ) + .init(), + resync_counter: meter .u64_counter("block.resync_counter") .with_description("Number of calls to resync_block") |