2022-02-17 22:28:23 +00:00
|
|
|
use std::convert::Infallible;
|
2022-02-22 13:52:41 +00:00
|
|
|
use std::net::SocketAddr;
|
2022-02-17 22:28:23 +00:00
|
|
|
use std::sync::Arc;
|
|
|
|
|
use std::time::SystemTime;
|
|
|
|
|
|
|
|
|
|
use futures::future::*;
|
2021-09-28 06:57:20 +00:00
|
|
|
use hyper::{
|
|
|
|
|
header::CONTENT_TYPE,
|
|
|
|
|
service::{make_service_fn, service_fn},
|
|
|
|
|
Body, Method, Request, Response, Server,
|
|
|
|
|
};
|
2022-02-17 22:28:23 +00:00
|
|
|
|
2021-09-28 06:57:20 +00:00
|
|
|
use opentelemetry::{
|
|
|
|
|
global,
|
|
|
|
|
metrics::{BoundCounter, BoundValueRecorder},
|
2022-02-17 22:28:23 +00:00
|
|
|
trace::{FutureExt, TraceContextExt, Tracer},
|
|
|
|
|
Context,
|
2021-09-28 06:57:20 +00:00
|
|
|
};
|
|
|
|
|
use opentelemetry_prometheus::PrometheusExporter;
|
2022-02-17 22:28:23 +00:00
|
|
|
|
2021-09-28 06:57:20 +00:00
|
|
|
use prometheus::{Encoder, TextEncoder};
|
|
|
|
|
|
|
|
|
|
use garage_util::error::Error as GarageError;
|
2022-02-24 12:18:51 +00:00
|
|
|
use garage_util::metrics::*;
|
2021-09-28 06:57:20 +00:00
|
|
|
|
|
|
|
|
// serve_req on metric endpoint
|
|
|
|
|
async fn serve_req(
|
|
|
|
|
req: Request<Body>,
|
|
|
|
|
admin_server: Arc<AdminServer>,
|
|
|
|
|
) -> Result<Response<Body>, hyper::Error> {
|
2022-03-28 13:46:52 +00:00
|
|
|
debug!("Receiving request at path {}", req.uri());
|
2021-09-28 06:57:20 +00:00
|
|
|
let request_start = SystemTime::now();
|
|
|
|
|
|
|
|
|
|
admin_server.metrics.http_counter.add(1);
|
|
|
|
|
|
|
|
|
|
let response = match (req.method(), req.uri().path()) {
|
|
|
|
|
(&Method::GET, "/metrics") => {
|
|
|
|
|
let mut buffer = vec![];
|
|
|
|
|
let encoder = TextEncoder::new();
|
2022-02-17 22:28:23 +00:00
|
|
|
|
|
|
|
|
let tracer = opentelemetry::global::tracer("garage");
|
|
|
|
|
let metric_families = tracer.in_span("admin/gather_metrics", |_| {
|
|
|
|
|
admin_server.exporter.registry().gather()
|
|
|
|
|
});
|
|
|
|
|
|
2021-09-28 06:57:20 +00:00
|
|
|
encoder.encode(&metric_families, &mut buffer).unwrap();
|
|
|
|
|
admin_server
|
|
|
|
|
.metrics
|
|
|
|
|
.http_body_gauge
|
|
|
|
|
.record(buffer.len() as u64);
|
|
|
|
|
|
|
|
|
|
Response::builder()
|
|
|
|
|
.status(200)
|
|
|
|
|
.header(CONTENT_TYPE, encoder.format_type())
|
|
|
|
|
.body(Body::from(buffer))
|
|
|
|
|
.unwrap()
|
|
|
|
|
}
|
|
|
|
|
_ => Response::builder()
|
|
|
|
|
.status(404)
|
|
|
|
|
.body(Body::from("Not implemented"))
|
|
|
|
|
.unwrap(),
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
admin_server
|
|
|
|
|
.metrics
|
|
|
|
|
.http_req_histogram
|
|
|
|
|
.record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()));
|
|
|
|
|
Ok(response)
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// AdminServer hold the admin server internal admin_server and the metric exporter
|
|
|
|
|
pub struct AdminServer {
|
|
|
|
|
exporter: PrometheusExporter,
|
|
|
|
|
metrics: AdminServerMetrics,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// GarageMetricadmin_server holds the metrics counter definition for Garage
|
|
|
|
|
// FIXME: we would rather have that split up among the different libraries?
|
|
|
|
|
struct AdminServerMetrics {
|
|
|
|
|
http_counter: BoundCounter<u64>,
|
|
|
|
|
http_body_gauge: BoundValueRecorder<u64>,
|
|
|
|
|
http_req_histogram: BoundValueRecorder<f64>,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
impl AdminServer {
|
|
|
|
|
/// init initilialize the AdminServer and background metric server
|
|
|
|
|
pub fn init() -> AdminServer {
|
|
|
|
|
let exporter = opentelemetry_prometheus::exporter().init();
|
|
|
|
|
let meter = global::meter("garage/admin_server");
|
|
|
|
|
AdminServer {
|
|
|
|
|
exporter,
|
|
|
|
|
metrics: AdminServerMetrics {
|
|
|
|
|
http_counter: meter
|
2022-02-16 13:23:04 +00:00
|
|
|
.u64_counter("admin.http_requests_total")
|
2021-09-28 06:57:20 +00:00
|
|
|
.with_description("Total number of HTTP requests made.")
|
|
|
|
|
.init()
|
2022-02-16 13:23:04 +00:00
|
|
|
.bind(&[]),
|
2021-09-28 06:57:20 +00:00
|
|
|
http_body_gauge: meter
|
2022-02-16 13:23:04 +00:00
|
|
|
.u64_value_recorder("admin.http_response_size_bytes")
|
2021-09-28 06:57:20 +00:00
|
|
|
.with_description("The metrics HTTP response sizes in bytes.")
|
|
|
|
|
.init()
|
2022-02-16 13:23:04 +00:00
|
|
|
.bind(&[]),
|
2021-09-28 06:57:20 +00:00
|
|
|
http_req_histogram: meter
|
2022-02-16 13:23:04 +00:00
|
|
|
.f64_value_recorder("admin.http_request_duration_seconds")
|
2021-09-28 06:57:20 +00:00
|
|
|
.with_description("The HTTP request latencies in seconds.")
|
|
|
|
|
.init()
|
2022-02-16 13:23:04 +00:00
|
|
|
.bind(&[]),
|
2021-09-28 06:57:20 +00:00
|
|
|
},
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
/// run execute the admin server on the designated HTTP port and listen for requests
|
|
|
|
|
pub async fn run(
|
|
|
|
|
self,
|
2022-02-22 12:53:59 +00:00
|
|
|
bind_addr: SocketAddr,
|
2021-09-28 06:57:20 +00:00
|
|
|
shutdown_signal: impl Future<Output = ()>,
|
|
|
|
|
) -> Result<(), GarageError> {
|
|
|
|
|
let admin_server = Arc::new(self);
|
|
|
|
|
// For every connection, we must make a `Service` to handle all
|
|
|
|
|
// incoming HTTP requests on said connection.
|
|
|
|
|
let make_svc = make_service_fn(move |_conn| {
|
|
|
|
|
let admin_server = admin_server.clone();
|
|
|
|
|
// This is the `Service` that will handle the connection.
|
|
|
|
|
// `service_fn` is a helper to convert a function that
|
|
|
|
|
// returns a Response into a `Service`.
|
|
|
|
|
async move {
|
2022-02-17 22:28:23 +00:00
|
|
|
Ok::<_, Infallible>(service_fn(move |req| {
|
|
|
|
|
let tracer = opentelemetry::global::tracer("garage");
|
|
|
|
|
let span = tracer
|
|
|
|
|
.span_builder("admin/request")
|
2022-02-24 12:18:51 +00:00
|
|
|
.with_trace_id(gen_trace_id())
|
2022-02-17 22:28:23 +00:00
|
|
|
.start(&tracer);
|
|
|
|
|
|
|
|
|
|
serve_req(req, admin_server.clone())
|
|
|
|
|
.with_context(Context::current_with_span(span))
|
|
|
|
|
}))
|
2021-09-28 06:57:20 +00:00
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
|
2022-02-22 12:53:59 +00:00
|
|
|
let server = Server::bind(&bind_addr).serve(make_svc);
|
2021-09-28 06:57:20 +00:00
|
|
|
let graceful = server.with_graceful_shutdown(shutdown_signal);
|
2022-02-22 12:53:59 +00:00
|
|
|
info!("Admin server listening on http://{}", bind_addr);
|
2021-09-28 06:57:20 +00:00
|
|
|
|
|
|
|
|
graceful.await?;
|
|
|
|
|
Ok(())
|
|
|
|
|
}
|
|
|
|
|
}
|
