diff options
| author | Alex Auvolat <alex@adnab.me> | 2024-03-27 15:26:08 +0100 |
|---|---|---|
| committer | Alex Auvolat <alex@adnab.me> | 2024-03-27 16:22:40 +0100 |
| commit | 0d3e285d133459fd53e28f879a86c0de1a0c36df (patch) | |
| tree | 4301ed7bd7ed72e79701c68bebdb4c5ea1db37a5 /src/block/metrics.rs | |
| parent | 95eb8808e8ede5439cf6352ce4f9a148fac2f236 (diff) | |
| download | garage-0d3e285d133459fd53e28f879a86c0de1a0c36df.tar.gz garage-0d3e285d133459fd53e28f879a86c0de1a0c36df.zip | |
[fix-buffering] implement `block_ram_buffer_max` to avoid excessive RAM usage
Diffstat (limited to 'src/block/metrics.rs')
| -rw-r--r-- | src/block/metrics.rs | 15 |
1 files changed, 15 insertions, 0 deletions
diff --git a/src/block/metrics.rs b/src/block/metrics.rs index 6659df32..c989f940 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -1,3 +1,7 @@ +use std::sync::Arc; + +use tokio::sync::Semaphore; + use opentelemetry::{global, metrics::*}; use garage_db as db; @@ -9,6 +13,7 @@ pub struct BlockManagerMetrics { pub(crate) _rc_size: ValueObserver<u64>, pub(crate) _resync_queue_len: ValueObserver<u64>, pub(crate) _resync_errored_blocks: ValueObserver<u64>, + pub(crate) _buffer_free_kb: ValueObserver<u64>, pub(crate) resync_counter: BoundCounter<u64>, pub(crate) resync_error_counter: BoundCounter<u64>, @@ -31,6 +36,7 @@ impl BlockManagerMetrics { rc_tree: db::Tree, resync_queue: CountedTree, resync_errors: CountedTree, + buffer_semaphore: Arc<Semaphore>, ) -> Self { let meter = global::meter("garage_model/block"); Self { @@ -66,6 +72,15 @@ impl BlockManagerMetrics { .with_description("Number of block hashes whose last resync resulted in an error") .init(), + _buffer_free_kb: meter + .u64_value_observer("block.ram_buffer_free_kb", move |observer| { + observer.observe(buffer_semaphore.available_permits() as u64, &[]) + }) + .with_description( + "Available RAM in KiB to use for buffering data blocks to be written to remote nodes", + ) + .init(), + resync_counter: meter .u64_counter("block.resync_counter") .with_description("Number of calls to resync_block") |