2022-02-16 13:23:04 +00:00
|
|
|
use opentelemetry::{global, metrics::*};
|
|
|
|
|
2022-12-13 14:54:03 +00:00
|
|
|
use garage_db as db;
|
2022-06-08 08:01:44 +00:00
|
|
|
use garage_db::counted_tree_hack::CountedTree;
|
2022-02-24 13:59:49 +00:00
|
|
|
|
2022-02-16 13:23:04 +00:00
|
|
|
/// TableMetrics reference all counter used for metrics
|
|
|
|
pub struct BlockManagerMetrics {
|
2023-01-09 18:49:33 +00:00
|
|
|
pub(crate) _compression_level: ValueObserver<u64>,
|
2022-12-13 14:54:03 +00:00
|
|
|
pub(crate) _rc_size: ValueObserver<u64>,
|
2022-02-16 13:23:04 +00:00
|
|
|
pub(crate) _resync_queue_len: ValueObserver<u64>,
|
2022-02-25 19:42:56 +00:00
|
|
|
pub(crate) _resync_errored_blocks: ValueObserver<u64>,
|
2022-02-16 13:23:04 +00:00
|
|
|
|
|
|
|
pub(crate) resync_counter: BoundCounter<u64>,
|
|
|
|
pub(crate) resync_error_counter: BoundCounter<u64>,
|
|
|
|
pub(crate) resync_duration: BoundValueRecorder<f64>,
|
|
|
|
pub(crate) resync_send_counter: Counter<u64>,
|
|
|
|
pub(crate) resync_recv_counter: BoundCounter<u64>,
|
|
|
|
|
|
|
|
pub(crate) bytes_read: BoundCounter<u64>,
|
|
|
|
pub(crate) block_read_duration: BoundValueRecorder<f64>,
|
|
|
|
pub(crate) bytes_written: BoundCounter<u64>,
|
|
|
|
pub(crate) block_write_duration: BoundValueRecorder<f64>,
|
|
|
|
pub(crate) delete_counter: BoundCounter<u64>,
|
|
|
|
|
|
|
|
pub(crate) corruption_counter: BoundCounter<u64>,
|
|
|
|
}
|
|
|
|
|
|
|
|
impl BlockManagerMetrics {
|
2023-01-09 18:49:33 +00:00
|
|
|
pub fn new(
|
|
|
|
compression_level: Option<i32>,
|
|
|
|
rc_tree: db::Tree,
|
|
|
|
resync_queue: CountedTree,
|
|
|
|
resync_errors: CountedTree,
|
|
|
|
) -> Self {
|
2022-02-16 13:23:04 +00:00
|
|
|
let meter = global::meter("garage_model/block");
|
|
|
|
Self {
|
2023-01-09 18:49:33 +00:00
|
|
|
_compression_level: meter
|
|
|
|
.u64_value_observer("block.compression_level", move |observer| {
|
|
|
|
match compression_level {
|
|
|
|
Some(v) => observer.observe(v as u64, &[]),
|
2023-01-26 16:26:32 +00:00
|
|
|
None => observer.observe(0_u64, &[]),
|
2023-01-09 18:49:33 +00:00
|
|
|
}
|
|
|
|
})
|
|
|
|
.with_description("Garage compression level for node")
|
|
|
|
.init(),
|
2022-12-13 14:54:03 +00:00
|
|
|
_rc_size: meter
|
|
|
|
.u64_value_observer("block.rc_size", move |observer| {
|
|
|
|
if let Ok(Some(v)) = rc_tree.fast_len() {
|
|
|
|
observer.observe(v as u64, &[])
|
|
|
|
}
|
|
|
|
})
|
|
|
|
.with_description("Number of blocks known to the reference counter")
|
|
|
|
.init(),
|
2022-02-16 13:23:04 +00:00
|
|
|
_resync_queue_len: meter
|
|
|
|
.u64_value_observer("block.resync_queue_length", move |observer| {
|
|
|
|
observer.observe(resync_queue.len() as u64, &[])
|
|
|
|
})
|
|
|
|
.with_description(
|
|
|
|
"Number of block hashes queued for local check and possible resync",
|
|
|
|
)
|
|
|
|
.init(),
|
2022-02-25 19:42:56 +00:00
|
|
|
_resync_errored_blocks: meter
|
|
|
|
.u64_value_observer("block.resync_errored_blocks", move |observer| {
|
|
|
|
observer.observe(resync_errors.len() as u64, &[])
|
|
|
|
})
|
|
|
|
.with_description("Number of block hashes whose last resync resulted in an error")
|
|
|
|
.init(),
|
2022-02-16 13:23:04 +00:00
|
|
|
|
|
|
|
resync_counter: meter
|
|
|
|
.u64_counter("block.resync_counter")
|
|
|
|
.with_description("Number of calls to resync_block")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
resync_error_counter: meter
|
|
|
|
.u64_counter("block.resync_error_counter")
|
|
|
|
.with_description("Number of calls to resync_block that returned an error")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
resync_duration: meter
|
|
|
|
.f64_value_recorder("block.resync_duration")
|
|
|
|
.with_description("Duration of resync_block operations")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
resync_send_counter: meter
|
|
|
|
.u64_counter("block.resync_send_counter")
|
|
|
|
.with_description("Number of blocks sent to another node in resync operations")
|
|
|
|
.init(),
|
|
|
|
resync_recv_counter: meter
|
|
|
|
.u64_counter("block.resync_recv_counter")
|
|
|
|
.with_description("Number of blocks received from other nodes in resync operations")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
|
|
|
|
bytes_read: meter
|
|
|
|
.u64_counter("block.bytes_read")
|
|
|
|
.with_description("Number of bytes read from disk")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
block_read_duration: meter
|
|
|
|
.f64_value_recorder("block.read_duration")
|
|
|
|
.with_description("Duration of block read operations")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
bytes_written: meter
|
|
|
|
.u64_counter("block.bytes_written")
|
|
|
|
.with_description("Number of bytes written to disk")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
block_write_duration: meter
|
|
|
|
.f64_value_recorder("block.write_duration")
|
|
|
|
.with_description("Duration of block write operations")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
delete_counter: meter
|
|
|
|
.u64_counter("block.delete_counter")
|
|
|
|
.with_description("Number of blocks deleted")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
|
|
|
|
corruption_counter: meter
|
|
|
|
.u64_counter("block.corruption_counter")
|
|
|
|
.with_description("Data corruptions detected on block reads")
|
|
|
|
.init()
|
|
|
|
.bind(&[]),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|