Add duration metric and improve others as well

This commit is contained in:
Alex 2022-12-05 19:10:15 +01:00
parent 5d38f2cf7f
commit ba5bf133f6
Signed by untrusted user: lx
GPG key ID: 0E496D15096376BE
2 changed files with 41 additions and 17 deletions

View file

@ -1,7 +1,7 @@
use std::convert::Infallible;
use std::net::SocketAddr;
use std::sync::{atomic::Ordering, Arc};
use std::time::Duration;
use std::time::{Duration, Instant};
use anyhow::Result;
use log::*;
@ -38,6 +38,7 @@ pub struct HttpsConfig {
struct HttpsMetrics {
requests_received: metrics::Counter<u64>,
requests_served: metrics::Counter<u64>,
request_proxy_duration: metrics::ValueRecorder<f64>,
}
pub async fn serve_https(
@ -58,6 +59,10 @@ pub async fn serve_https(
.u64_counter("https_requests_served")
.with_description("Total number of requests served over HTTPS")
.init(),
request_proxy_duration: meter
.f64_value_recorder("https_request_proxy_duration")
.with_description("Duration between time when request was received, and time when backend returned status code and headers")
.init(),
});
let mut tls_cfg = rustls::ServerConfig::builder()
@ -164,7 +169,16 @@ async fn handle_outer(
];
metrics.requests_received.add(1, &tags);
let resp = match handle(remote_addr, req, https_config, proxy_config, &mut tags).await {
let resp = match handle(
remote_addr,
req,
https_config,
proxy_config,
&mut tags,
&metrics,
)
.await
{
Err(e) => {
warn!("Handler error: {}", e);
Response::builder()
@ -176,7 +190,7 @@ async fn handle_outer(
};
tags.push(KeyValue::new(
"response_code",
"status_code",
resp.status().as_u16().to_string(),
));
metrics.requests_served.add(1, &tags);
@ -192,7 +206,10 @@ async fn handle(
https_config: Arc<HttpsConfig>,
proxy_config: Arc<ProxyConfig>,
tags: &mut Vec<KeyValue>,
metrics: &HttpsMetrics,
) -> Result<Response<Body>, anyhow::Error> {
let received_time = Instant::now();
let method = req.method().clone();
let uri = req.uri().to_string();
@ -232,15 +249,11 @@ async fn handle(
});
if let Some(proxy_to) = best_match {
tags.push(KeyValue::new("service_name", proxy_to.service_name.clone()));
tags.push(KeyValue::new("service", proxy_to.service_name.clone()));
tags.push(KeyValue::new(
"target_addr",
proxy_to.target_addr.to_string(),
));
tags.push(KeyValue::new(
"https_target",
proxy_to.https_target.to_string(),
));
tags.push(KeyValue::new("same_node", proxy_to.same_node.to_string()));
tags.push(KeyValue::new("same_site", proxy_to.same_site.to_string()));
@ -257,6 +270,10 @@ async fn handle(
handle_error(reverse_proxy::call(remote_addr.ip(), &to_addr, req).await)
};
metrics
.request_proxy_duration
.record(received_time.elapsed().as_secs_f64(), &tags);
if response.status().is_success() {
// (TODO: maybe we want to add these headers even if it's not a success?)
for (header, value) in proxy_to.add_headers.iter() {

View file

@ -189,7 +189,7 @@ fn parse_consul_catalog(
let mut entries = vec![];
for (service_name, svc) in catalog.services.iter() {
for (_, svc) in catalog.services.iter() {
let ip_addr = match svc.address.parse() {
Ok(ip) => ip,
_ => match catalog.node.address.parse() {
@ -222,7 +222,7 @@ fn parse_consul_catalog(
for tag in svc.tags.iter() {
if let Some(ent) = parse_tricot_tag(
service_name.clone(),
svc.service.clone(),
tag,
addr,
&add_headers[..],
@ -388,15 +388,22 @@ impl ProxyConfigMetrics {
.u64_value_observer("proxy_config_entries", move |observer| {
let mut patterns = HashMap::new();
for ent in rx.borrow().entries.iter() {
let pat = format!(
"{}{}",
ent.host,
ent.path_prefix.as_deref().unwrap_or_default()
let attrs = (
ent.host.to_string(),
ent.path_prefix.clone().unwrap_or_default(),
ent.service_name.clone(),
);
*patterns.entry(pat).or_default() += 1;
*patterns.entry(attrs).or_default() += 1;
}
for (pat, num) in patterns {
observer.observe(num, &[KeyValue::new("host", pat)]);
for ((host, prefix, svc), num) in patterns {
observer.observe(
num,
&[
KeyValue::new("host", host),
KeyValue::new("path_prefix", prefix),
KeyValue::new("service", svc),
],
);
}
})
.with_description("Number of proxy entries (back-ends) configured in Tricot")