From aac348fe937442bcf750316d6ebd47936d86ed90 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 9 Jan 2023 17:13:41 +0000 Subject: [PATCH 1/3] Added system_metrics.rs file. --- src/rpc/lib.rs | 2 ++ src/rpc/system_metrics.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 src/rpc/system_metrics.rs diff --git a/src/rpc/lib.rs b/src/rpc/lib.rs index 86f63568..a8cc0030 100644 --- a/src/rpc/lib.rs +++ b/src/rpc/lib.rs @@ -17,3 +17,5 @@ mod metrics; pub mod rpc_helper; pub use rpc_helper::*; + +pub mod system_metrics; diff --git a/src/rpc/system_metrics.rs b/src/rpc/system_metrics.rs new file mode 100644 index 00000000..d96b67e4 --- /dev/null +++ b/src/rpc/system_metrics.rs @@ -0,0 +1,33 @@ +use opentelemetry::{global, metrics::*, KeyValue}; + +/// TableMetrics reference all counter used for metrics +pub struct SystemMetrics { + pub(crate) _garage_build_info: ValueObserver, + pub(crate) _replication_factor: ValueObserver, +} + +impl SystemMetrics { + pub fn new(replication_factor: usize) -> Self { + let meter = global::meter("garage_system"); + Self { + _garage_build_info: meter + .u64_value_observer("garage_build_info", move |observer| { + observer.observe( + 1, + &[KeyValue::new( + "version", + garage_util::version::garage_version(), + )], + ) + }) + .with_description("Garage build info") + .init(), + _replication_factor: meter + .u64_value_observer("garage_replication_factor", move |observer| { + observer.observe(replication_factor as u64, &[]) + }) + .with_description("Garage replication factor setting") + .init(), + } + } +} From df1d9a98737e8a38656de8ac71bb23e8d3ad2ce9 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 9 Jan 2023 17:15:55 +0000 Subject: [PATCH 2/3] system.rs: Integrated SystemMetrics into System implementation. --- src/rpc/system.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/rpc/system.rs b/src/rpc/system.rs index ed10aa4b..90f6a4c2 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -38,6 +38,9 @@ use crate::replication_mode::*; use crate::ring::*; use crate::rpc_helper::*; +#[cfg(feature = "metrics")] +use crate::system_metrics::*; + const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60); const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10); @@ -103,6 +106,8 @@ pub struct System { consul_discovery: Option, #[cfg(feature = "kubernetes-discovery")] kubernetes_discovery: Option, + #[cfg(feature = "metrics")] + metrics: SystemMetrics, replication_mode: ReplicationMode, replication_factor: usize, @@ -275,6 +280,9 @@ impl System { cluster_layout_staging_hash: cluster_layout.staging_hash, }; + #[cfg(feature = "metrics")] + let metrics = SystemMetrics::new(replication_factor); + let ring = Ring::new(cluster_layout, replication_factor); let (update_ring, ring) = watch::channel(Arc::new(ring)); @@ -365,6 +373,8 @@ impl System { consul_discovery, #[cfg(feature = "kubernetes-discovery")] kubernetes_discovery: config.kubernetes_discovery.clone(), + #[cfg(feature = "metrics")] + metrics, ring, update_ring: Mutex::new(update_ring), From 4cfb469d2bcce5bd41fe0af0022b34716ac7279a Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Mon, 9 Jan 2023 18:49:33 +0000 Subject: [PATCH 3/3] block/metrics.rs: Added compression_level metric. --- src/block/manager.rs | 8 ++++++-- src/block/metrics.rs | 17 ++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/block/manager.rs b/src/block/manager.rs index 1655be06..051a9f93 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -129,8 +129,12 @@ impl BlockManager { .netapp .endpoint("garage_block/manager.rs/Rpc".to_string()); - let metrics = - BlockManagerMetrics::new(rc.rc.clone(), resync.queue.clone(), resync.errors.clone()); + let metrics = BlockManagerMetrics::new( + compression_level, + rc.rc.clone(), + resync.queue.clone(), + resync.errors.clone(), + ); let scrub_persister = PersisterShared::new(&system.metadata_dir, "scrub_info"); diff --git a/src/block/metrics.rs b/src/block/metrics.rs index fbef95af..500022fc 100644 --- a/src/block/metrics.rs +++ b/src/block/metrics.rs @@ -5,6 +5,7 @@ use garage_db::counted_tree_hack::CountedTree; /// TableMetrics reference all counter used for metrics pub struct BlockManagerMetrics { + pub(crate) _compression_level: ValueObserver, pub(crate) _rc_size: ValueObserver, pub(crate) _resync_queue_len: ValueObserver, pub(crate) _resync_errored_blocks: ValueObserver, @@ -25,9 +26,23 @@ pub struct BlockManagerMetrics { } impl BlockManagerMetrics { - pub fn new(rc_tree: db::Tree, resync_queue: CountedTree, resync_errors: CountedTree) -> Self { + pub fn new( + compression_level: Option, + rc_tree: db::Tree, + resync_queue: CountedTree, + resync_errors: CountedTree, + ) -> Self { let meter = global::meter("garage_model/block"); Self { + _compression_level: meter + .u64_value_observer("block.compression_level", move |observer| { + match compression_level { + Some(v) => observer.observe(v as u64, &[]), + None => observer.observe(0 as u64, &[]), + } + }) + .with_description("Garage compression level for node") + .init(), _rc_size: meter .u64_value_observer("block.rc_size", move |observer| { if let Ok(Some(v)) = rc_tree.fast_len() {