Add duration metric and improve others as well

This commit is contained in:
Alex 2022-12-05 19:10:15 +01:00
parent 5d38f2cf7f
commit ba5bf133f6
Signed by untrusted user: lx
GPG key ID: 0E496D15096376BE
2 changed files with 41 additions and 17 deletions

View file

@ -1,7 +1,7 @@
use std::convert::Infallible; use std::convert::Infallible;
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::{atomic::Ordering, Arc}; use std::sync::{atomic::Ordering, Arc};
use std::time::Duration; use std::time::{Duration, Instant};
use anyhow::Result; use anyhow::Result;
use log::*; use log::*;
@ -38,6 +38,7 @@ pub struct HttpsConfig {
struct HttpsMetrics { struct HttpsMetrics {
requests_received: metrics::Counter<u64>, requests_received: metrics::Counter<u64>,
requests_served: metrics::Counter<u64>, requests_served: metrics::Counter<u64>,
request_proxy_duration: metrics::ValueRecorder<f64>,
} }
pub async fn serve_https( pub async fn serve_https(
@ -58,6 +59,10 @@ pub async fn serve_https(
.u64_counter("https_requests_served") .u64_counter("https_requests_served")
.with_description("Total number of requests served over HTTPS") .with_description("Total number of requests served over HTTPS")
.init(), .init(),
request_proxy_duration: meter
.f64_value_recorder("https_request_proxy_duration")
.with_description("Duration between time when request was received, and time when backend returned status code and headers")
.init(),
}); });
let mut tls_cfg = rustls::ServerConfig::builder() let mut tls_cfg = rustls::ServerConfig::builder()
@ -164,7 +169,16 @@ async fn handle_outer(
]; ];
metrics.requests_received.add(1, &tags); metrics.requests_received.add(1, &tags);
let resp = match handle(remote_addr, req, https_config, proxy_config, &mut tags).await { let resp = match handle(
remote_addr,
req,
https_config,
proxy_config,
&mut tags,
&metrics,
)
.await
{
Err(e) => { Err(e) => {
warn!("Handler error: {}", e); warn!("Handler error: {}", e);
Response::builder() Response::builder()
@ -176,7 +190,7 @@ async fn handle_outer(
}; };
tags.push(KeyValue::new( tags.push(KeyValue::new(
"response_code", "status_code",
resp.status().as_u16().to_string(), resp.status().as_u16().to_string(),
)); ));
metrics.requests_served.add(1, &tags); metrics.requests_served.add(1, &tags);
@ -192,7 +206,10 @@ async fn handle(
https_config: Arc<HttpsConfig>, https_config: Arc<HttpsConfig>,
proxy_config: Arc<ProxyConfig>, proxy_config: Arc<ProxyConfig>,
tags: &mut Vec<KeyValue>, tags: &mut Vec<KeyValue>,
metrics: &HttpsMetrics,
) -> Result<Response<Body>, anyhow::Error> { ) -> Result<Response<Body>, anyhow::Error> {
let received_time = Instant::now();
let method = req.method().clone(); let method = req.method().clone();
let uri = req.uri().to_string(); let uri = req.uri().to_string();
@ -232,15 +249,11 @@ async fn handle(
}); });
if let Some(proxy_to) = best_match { if let Some(proxy_to) = best_match {
tags.push(KeyValue::new("service_name", proxy_to.service_name.clone())); tags.push(KeyValue::new("service", proxy_to.service_name.clone()));
tags.push(KeyValue::new( tags.push(KeyValue::new(
"target_addr", "target_addr",
proxy_to.target_addr.to_string(), proxy_to.target_addr.to_string(),
)); ));
tags.push(KeyValue::new(
"https_target",
proxy_to.https_target.to_string(),
));
tags.push(KeyValue::new("same_node", proxy_to.same_node.to_string())); tags.push(KeyValue::new("same_node", proxy_to.same_node.to_string()));
tags.push(KeyValue::new("same_site", proxy_to.same_site.to_string())); tags.push(KeyValue::new("same_site", proxy_to.same_site.to_string()));
@ -257,6 +270,10 @@ async fn handle(
handle_error(reverse_proxy::call(remote_addr.ip(), &to_addr, req).await) handle_error(reverse_proxy::call(remote_addr.ip(), &to_addr, req).await)
}; };
metrics
.request_proxy_duration
.record(received_time.elapsed().as_secs_f64(), &tags);
if response.status().is_success() { if response.status().is_success() {
// (TODO: maybe we want to add these headers even if it's not a success?) // (TODO: maybe we want to add these headers even if it's not a success?)
for (header, value) in proxy_to.add_headers.iter() { for (header, value) in proxy_to.add_headers.iter() {

View file

@ -189,7 +189,7 @@ fn parse_consul_catalog(
let mut entries = vec![]; let mut entries = vec![];
for (service_name, svc) in catalog.services.iter() { for (_, svc) in catalog.services.iter() {
let ip_addr = match svc.address.parse() { let ip_addr = match svc.address.parse() {
Ok(ip) => ip, Ok(ip) => ip,
_ => match catalog.node.address.parse() { _ => match catalog.node.address.parse() {
@ -222,7 +222,7 @@ fn parse_consul_catalog(
for tag in svc.tags.iter() { for tag in svc.tags.iter() {
if let Some(ent) = parse_tricot_tag( if let Some(ent) = parse_tricot_tag(
service_name.clone(), svc.service.clone(),
tag, tag,
addr, addr,
&add_headers[..], &add_headers[..],
@ -388,15 +388,22 @@ impl ProxyConfigMetrics {
.u64_value_observer("proxy_config_entries", move |observer| { .u64_value_observer("proxy_config_entries", move |observer| {
let mut patterns = HashMap::new(); let mut patterns = HashMap::new();
for ent in rx.borrow().entries.iter() { for ent in rx.borrow().entries.iter() {
let pat = format!( let attrs = (
"{}{}", ent.host.to_string(),
ent.host, ent.path_prefix.clone().unwrap_or_default(),
ent.path_prefix.as_deref().unwrap_or_default() ent.service_name.clone(),
); );
*patterns.entry(pat).or_default() += 1; *patterns.entry(attrs).or_default() += 1;
} }
for (pat, num) in patterns { for ((host, prefix, svc), num) in patterns {
observer.observe(num, &[KeyValue::new("host", pat)]); observer.observe(
num,
&[
KeyValue::new("host", host),
KeyValue::new("path_prefix", prefix),
KeyValue::new("service", svc),
],
);
} }
}) })
.with_description("Number of proxy entries (back-ends) configured in Tricot") .with_description("Number of proxy entries (back-ends) configured in Tricot")