Deuxfleurs/garage
48
56
Fork 73
Code Issues 100 Pull requests 13 Releases 30 Wiki Activity

Make OTLP exporter optional and allow building without Prometheus exporter (/metrics) #372

Merged
lx merged 3 commits from jirutka/garage:telemetry-and-metrics into improve-deps 2022-09-06 13:11:33 +00:00
Conversation 3 Commits 3 Files changed 6 +52 -27
6 changed files with 52 additions and 27 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Cargo.toml
View file

@ -1,4 +1,5 @@
[workspace] [workspace]
resolver = "2"
members = [ members = [
"src/db", "src/db",
"src/util", "src/util",

8
src/api/Cargo.toml
View file

@ -54,9 +54,11 @@ quick-xml = { version = "0.21", features = [ "serialize" ] }
url = "2.1" url = "2.1"
opentelemetry = "0.17" opentelemetry = "0.17"
opentelemetry-prometheus = "0.10" opentelemetry-prometheus = { version = "0.10", optional = true }
opentelemetry-otlp = "0.10" opentelemetry-otlp = { version = "0.10", optional = true }
prometheus = "0.13" prometheus = { version = "0.13", optional = true }
[features] [features]
k2v = [ "garage_util/k2v", "garage_model/k2v" ] k2v = [ "garage_util/k2v", "garage_model/k2v" ]
metrics = [ "opentelemetry-prometheus", "prometheus" ]
telemetry-otlp = ["opentelemetry-otlp"]

49
src/api/admin/api_server.rs
View file

@ -3,13 +3,14 @@ use std::sync::Arc;
use async_trait::async_trait; use async_trait::async_trait;
use futures::future::Future; use futures::future::Future;
use http::header::{ use http::header::{ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW};
ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW, CONTENT_TYPE,
};
use hyper::{Body, Request, Response}; use hyper::{Body, Request, Response};
use opentelemetry::trace::{SpanRef, Tracer}; use opentelemetry::trace::SpanRef;
#[cfg(feature = "metrics")]
use opentelemetry_prometheus::PrometheusExporter; use opentelemetry_prometheus::PrometheusExporter;
#[cfg(feature = "metrics")]
use prometheus::{Encoder, TextEncoder}; use prometheus::{Encoder, TextEncoder};
use garage_model::garage::Garage; use garage_model::garage::Garage;
@ -25,6 +26,7 @@ use crate::admin::router::{Authorization, Endpoint};
pub struct AdminApiServer { pub struct AdminApiServer {
garage: Arc<Garage>, garage: Arc<Garage>,
#[cfg(feature = "metrics")]
exporter: PrometheusExporter, exporter: PrometheusExporter,
metrics_token: Option<String>, metrics_token: Option<String>,
admin_token: Option<String>, admin_token: Option<String>,
@ -32,7 +34,6 @@ pub struct AdminApiServer {
impl AdminApiServer { impl AdminApiServer {
pub fn new(garage: Arc<Garage>) -> Self { pub fn new(garage: Arc<Garage>) -> Self {
let exporter = opentelemetry_prometheus::exporter().init();
let cfg = &garage.config.admin; let cfg = &garage.config.admin;
let metrics_token = cfg let metrics_token = cfg
.metrics_token .metrics_token
@ -44,7 +45,8 @@ impl AdminApiServer {
.map(|tok| format!("Bearer {}", tok)); .map(|tok| format!("Bearer {}", tok));
Self { Self {
garage, garage,
exporter, #[cfg(feature = "metrics")]
exporter: opentelemetry_prometheus::exporter().init(),
metrics_token, metrics_token,
admin_token, admin_token,
} }
@ -71,22 +73,31 @@ impl AdminApiServer {
} }
fn handle_metrics(&self) -> Result<Response<Body>, Error> { fn handle_metrics(&self) -> Result<Response<Body>, Error> {
let mut buffer = vec![]; #[cfg(feature = "metrics")]
let encoder = TextEncoder::new(); {
use opentelemetry::trace::Tracer;
let tracer = opentelemetry::global::tracer("garage"); let mut buffer = vec![];
let metric_families = tracer.in_span("admin/gather_metrics", |_| { let encoder = TextEncoder::new();
self.exporter.registry().gather()
});
encoder let tracer = opentelemetry::global::tracer("garage");
.encode(&metric_families, &mut buffer) let metric_families = tracer.in_span("admin/gather_metrics", |_| {
.ok_or_internal_error("Could not serialize metrics")?; self.exporter.registry().gather()
});
Ok(Response::builder() encoder
.status(200) .encode(&metric_families, &mut buffer)
.header(CONTENT_TYPE, encoder.format_type()) .ok_or_internal_error("Could not serialize metrics")?;
.body(Body::from(buffer))?)
Ok(Response::builder()
.status(200)
.header(http::header::CONTENT_TYPE, encoder.format_type())
.body(Body::from(buffer))?)
}
#[cfg(not(feature = "metrics"))]
Err(Error::bad_request(
"Garage was built without the metrics feature".to_string(),
))
} }
} }

12
src/garage/Cargo.toml
View file

@ -55,9 +55,9 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi
netapp = "0.4" netapp = "0.4"
opentelemetry = { version = "0.17", features = [ "rt-tokio" ] } opentelemetry = { version = "0.17", features = [ "rt-tokio" ] }
opentelemetry-prometheus = "0.10" opentelemetry-prometheus = { version = "0.10", optional = true }
opentelemetry-otlp = "0.10" opentelemetry-otlp = { version = "0.10", optional = true }
prometheus = "0.13" prometheus = { version = "0.13", optional = true }
[dev-dependencies] [dev-dependencies]
aws-sdk-s3 = "0.8" aws-sdk-s3 = "0.8"
@ -74,9 +74,13 @@ base64 = "0.13"
[features] [features]
default = [ "bundled-libs" ] default = [ "bundled-libs", "metrics" ]
kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ] kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
k2v = [ "garage_util/k2v", "garage_api/k2v" ] k2v = [ "garage_util/k2v", "garage_api/k2v" ]
# Prometheus exporter (/metrics endpoint).
metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ]
# Exporter for the OpenTelemetry Collector.
telemetry-otlp = [ "opentelemetry-otlp", "garage_api/telemetry-otlp" ]
# NOTE: bundled-libs and system-libs should be treat as mutually exclusive; # NOTE: bundled-libs and system-libs should be treat as mutually exclusive;
# exactly one of them should be enabled. # exactly one of them should be enabled.

1
src/garage/main.rs
View file

@ -8,6 +8,7 @@ mod admin;
mod cli; mod cli;
mod repair; mod repair;
mod server; mod server;
#[cfg(feature = "telemetry-otlp")]
mod tracing_setup; mod tracing_setup;
use std::net::SocketAddr; use std::net::SocketAddr;

8
src/garage/server.rs
View file

@ -15,6 +15,7 @@ use garage_web::run_web_server;
use garage_api::k2v::api_server::K2VApiServer; use garage_api::k2v::api_server::K2VApiServer;
use crate::admin::*; use crate::admin::*;
#[cfg(feature = "telemetry-otlp")]
use crate::tracing_setup::*; use crate::tracing_setup::*;
async fn wait_from(mut chan: watch::Receiver<bool>) { async fn wait_from(mut chan: watch::Receiver<bool>) {
@ -36,9 +37,14 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
info!("Initializing Garage main data store..."); info!("Initializing Garage main data store...");
let garage = Garage::new(config.clone(), background)?; let garage = Garage::new(config.clone(), background)?;
info!("Initialize tracing...");
if let Some(export_to) = config.admin.trace_sink { if let Some(export_to) = config.admin.trace_sink {
info!("Initialize tracing...");
#[cfg(feature = "telemetry-otlp")]
init_tracing(&export_to, garage.system.id)?; init_tracing(&export_to, garage.system.id)?;
#[cfg(not(feature = "telemetry-otlp"))]
warn!("Garage was built without OTLP exporter, admin.trace_sink is ignored.");
} }
info!("Initialize Admin API server and metrics collector..."); info!("Initialize Admin API server and metrics collector...");