forked from Deuxfleurs/garage
Update dependencies and add admin module with metrics
- Global dependencies updated in Cargo.lock - New module created in src/admin to host: - the (future) admin REST API - the metric collection - add configuration block No metrics implemented yet
This commit is contained in:
parent
9d44127245
commit
e349af13a7
9 changed files with 284 additions and 2 deletions
81
Cargo.lock
generated
81
Cargo.lock
generated
|
@ -403,6 +403,16 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-channel"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-epoch"
|
||||
version = "0.9.7"
|
||||
|
@ -766,6 +776,7 @@ dependencies = [
|
|||
"bytes 1.1.0",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_admin",
|
||||
"garage_api",
|
||||
"garage_model 0.6.0",
|
||||
"garage_rpc 0.6.0",
|
||||
|
@ -791,6 +802,23 @@ dependencies = [
|
|||
"toml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_admin"
|
||||
version = "0.6.0"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"futures-util",
|
||||
"garage_model 0.6.0",
|
||||
"garage_util 0.6.0",
|
||||
"http",
|
||||
"hyper",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"opentelemetry",
|
||||
"opentelemetry-prometheus",
|
||||
"prometheus",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "garage_api"
|
||||
version = "0.6.0"
|
||||
|
@ -1824,6 +1852,38 @@ dependencies = [
|
|||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry"
|
||||
version = "0.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"crossbeam-channel",
|
||||
"dashmap",
|
||||
"fnv",
|
||||
"futures-channel",
|
||||
"futures-executor",
|
||||
"futures-util",
|
||||
"js-sys",
|
||||
"lazy_static",
|
||||
"percent-encoding",
|
||||
"pin-project 1.0.10",
|
||||
"rand",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opentelemetry-prometheus"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9328977e479cebe12ce0d3fcecdaea4721d234895a9440c5b5dfd113f0594ac6"
|
||||
dependencies = [
|
||||
"opentelemetry",
|
||||
"prometheus",
|
||||
"protobuf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "2.10.0"
|
||||
|
@ -2076,6 +2136,27 @@ dependencies = [
|
|||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prometheus"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"fnv",
|
||||
"lazy_static",
|
||||
"memchr",
|
||||
"parking_lot",
|
||||
"protobuf",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf"
|
||||
version = "2.27.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96"
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "1.2.3"
|
||||
|
|
|
@ -4,9 +4,10 @@ members = [
|
|||
"src/rpc",
|
||||
"src/table",
|
||||
"src/model",
|
||||
"src/admin",
|
||||
"src/api",
|
||||
"src/web",
|
||||
"src/garage",
|
||||
"src/garage"
|
||||
]
|
||||
|
||||
[profile.dev]
|
||||
|
|
|
@ -44,6 +44,9 @@ root_domain = ".s3.garage.localhost"
|
|||
bind_addr = "0.0.0.0:$((3920+$count))"
|
||||
root_domain = ".web.garage.localhost"
|
||||
index = "index.html"
|
||||
|
||||
[admin_api]
|
||||
bind_addr = "0.0.0.0:$((9900+$count))"
|
||||
EOF
|
||||
|
||||
echo -en "$LABEL configuration written to $CONF_PATH\n"
|
||||
|
|
28
src/admin/Cargo.toml
Normal file
28
src/admin/Cargo.toml
Normal file
|
@ -0,0 +1,28 @@
|
|||
[package]
|
||||
name = "garage_admin"
|
||||
version = "0.6.0"
|
||||
authors = ["Maximilien Richer <code@mricher.fr>"]
|
||||
edition = "2018"
|
||||
license = "AGPL-3.0"
|
||||
description = "Administration and metrics REST HTTP server for Garage"
|
||||
repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage"
|
||||
|
||||
[lib]
|
||||
path = "lib.rs"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
garage_model = { version = "0.6.0", path = "../model" }
|
||||
garage_util = { version = "0.6.0", path = "../util" }
|
||||
|
||||
futures = "0.3"
|
||||
futures-util = "0.3"
|
||||
http = "0.2"
|
||||
hyper = "0.14"
|
||||
log = "0.4"
|
||||
|
||||
opentelemetry = "0.17"
|
||||
opentelemetry-prometheus = "0.10"
|
||||
prometheus = "0.13"
|
||||
lazy_static = "1.4"
|
6
src/admin/lib.rs
Normal file
6
src/admin/lib.rs
Normal file
|
@ -0,0 +1,6 @@
|
|||
//! Crate for handling the admin and metric HTTP APIs
|
||||
#[macro_use]
|
||||
extern crate log;
|
||||
extern crate lazy_static;
|
||||
|
||||
pub mod metrics;
|
141
src/admin/metrics.rs
Normal file
141
src/admin/metrics.rs
Normal file
|
@ -0,0 +1,141 @@
|
|||
use hyper::{
|
||||
header::CONTENT_TYPE,
|
||||
service::{make_service_fn, service_fn},
|
||||
Body, Method, Request, Response, Server,
|
||||
};
|
||||
use lazy_static::lazy_static;
|
||||
use opentelemetry::{
|
||||
global,
|
||||
metrics::{BoundCounter, BoundValueRecorder},
|
||||
KeyValue,
|
||||
};
|
||||
use opentelemetry_prometheus::PrometheusExporter;
|
||||
use prometheus::{Encoder, TextEncoder};
|
||||
use std::convert::Infallible;
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use futures::future::*;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_util::error::Error as GarageError;
|
||||
|
||||
lazy_static! {
|
||||
// This defines the differennt tags that will be referenced by the object
|
||||
static ref HANDLER_ALL: [KeyValue; 1] = [KeyValue::new("handler", "all")];
|
||||
}
|
||||
|
||||
// serve_req on metric endpoint
|
||||
async fn serve_req(
|
||||
req: Request<Body>,
|
||||
admin_server: Arc<AdminServer>,
|
||||
) -> Result<Response<Body>, hyper::Error> {
|
||||
println!("Receiving request at path {}", req.uri());
|
||||
let request_start = SystemTime::now();
|
||||
|
||||
admin_server.metrics.http_counter.add(1);
|
||||
|
||||
let response = match (req.method(), req.uri().path()) {
|
||||
(&Method::GET, "/metrics") => {
|
||||
let mut buffer = vec![];
|
||||
let encoder = TextEncoder::new();
|
||||
let metric_families = admin_server.exporter.registry().gather();
|
||||
encoder.encode(&metric_families, &mut buffer).unwrap();
|
||||
admin_server
|
||||
.metrics
|
||||
.http_body_gauge
|
||||
.record(buffer.len() as u64);
|
||||
|
||||
Response::builder()
|
||||
.status(200)
|
||||
.header(CONTENT_TYPE, encoder.format_type())
|
||||
.body(Body::from(buffer))
|
||||
.unwrap()
|
||||
}
|
||||
_ => Response::builder()
|
||||
.status(404)
|
||||
.body(Body::from("Not implemented"))
|
||||
.unwrap(),
|
||||
};
|
||||
|
||||
admin_server
|
||||
.metrics
|
||||
.http_req_histogram
|
||||
.record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()));
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
// AdminServer hold the admin server internal admin_server and the metric exporter
|
||||
pub struct AdminServer {
|
||||
exporter: PrometheusExporter,
|
||||
metrics: AdminServerMetrics,
|
||||
}
|
||||
|
||||
// GarageMetricadmin_server holds the metrics counter definition for Garage
|
||||
// FIXME: we would rather have that split up among the different libraries?
|
||||
struct AdminServerMetrics {
|
||||
http_counter: BoundCounter<u64>,
|
||||
http_body_gauge: BoundValueRecorder<u64>,
|
||||
http_req_histogram: BoundValueRecorder<f64>,
|
||||
bucket_v2_merkle_updater_todo_queue_length: BoundValueRecorder<f64>,
|
||||
}
|
||||
|
||||
impl AdminServer {
|
||||
/// init initilialize the AdminServer and background metric server
|
||||
pub fn init() -> AdminServer {
|
||||
let exporter = opentelemetry_prometheus::exporter().init();
|
||||
let meter = global::meter("garage/admin_server");
|
||||
AdminServer {
|
||||
exporter,
|
||||
metrics: AdminServerMetrics {
|
||||
http_counter: meter
|
||||
.u64_counter("router.http_requests_total")
|
||||
.with_description("Total number of HTTP requests made.")
|
||||
.init()
|
||||
.bind(HANDLER_ALL.as_ref()),
|
||||
http_body_gauge: meter
|
||||
.u64_value_recorder("example.http_response_size_bytes")
|
||||
.with_description("The metrics HTTP response sizes in bytes.")
|
||||
.init()
|
||||
.bind(HANDLER_ALL.as_ref()),
|
||||
http_req_histogram: meter
|
||||
.f64_value_recorder("example.http_request_duration_seconds")
|
||||
.with_description("The HTTP request latencies in seconds.")
|
||||
.init()
|
||||
.bind(HANDLER_ALL.as_ref()),
|
||||
bucket_v2_merkle_updater_todo_queue_length: meter
|
||||
.f64_value_recorder("bucket_v2.merkle_updater.todo_queue_length")
|
||||
.with_description("Bucket merkle updater TODO queue length.")
|
||||
.init()
|
||||
.bind(HANDLER_ALL.as_ref()),
|
||||
},
|
||||
}
|
||||
}
|
||||
/// run execute the admin server on the designated HTTP port and listen for requests
|
||||
pub async fn run(
|
||||
self,
|
||||
garage: Arc<Garage>,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
let admin_server = Arc::new(self);
|
||||
// For every connection, we must make a `Service` to handle all
|
||||
// incoming HTTP requests on said connection.
|
||||
let make_svc = make_service_fn(move |_conn| {
|
||||
let admin_server = admin_server.clone();
|
||||
// This is the `Service` that will handle the connection.
|
||||
// `service_fn` is a helper to convert a function that
|
||||
// returns a Response into a `Service`.
|
||||
async move {
|
||||
Ok::<_, Infallible>(service_fn(move |req| serve_req(req, admin_server.clone())))
|
||||
}
|
||||
});
|
||||
|
||||
let addr = &garage.config.admin_api.bind_addr;
|
||||
|
||||
let server = Server::bind(&addr).serve(make_svc);
|
||||
let graceful = server.with_graceful_shutdown(shutdown_signal);
|
||||
info!("Admin server listening on http://{}", addr);
|
||||
|
||||
graceful.await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
|
@ -27,6 +27,7 @@ garage_rpc = { version = "0.6.0", path = "../rpc" }
|
|||
garage_table = { version = "0.6.0", path = "../table" }
|
||||
garage_util = { version = "0.6.0", path = "../util" }
|
||||
garage_web = { version = "0.6.0", path = "../web" }
|
||||
garage_admin = { version = "0.6.0", path = "../admin" }
|
||||
|
||||
bytes = "1.0"
|
||||
git-version = "0.3.4"
|
||||
|
|
|
@ -6,6 +6,7 @@ use garage_util::background::*;
|
|||
use garage_util::config::*;
|
||||
use garage_util::error::Error;
|
||||
|
||||
use garage_admin::metrics::*;
|
||||
use garage_api::run_api_server;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_web::run_web_server;
|
||||
|
@ -34,6 +35,9 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||
.open()
|
||||
.expect("Unable to open sled DB");
|
||||
|
||||
info!("Configure and run admin web server...");
|
||||
let admin_server_init = AdminServer::init();
|
||||
|
||||
info!("Initializing background runner...");
|
||||
let watch_cancel = netapp::util::watch_ctrl_c();
|
||||
let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone());
|
||||
|
@ -43,7 +47,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||
|
||||
let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone()));
|
||||
|
||||
info!("Crate admin RPC handler...");
|
||||
info!("Create admin RPC handler...");
|
||||
AdminRpcHandler::new(garage.clone());
|
||||
|
||||
info!("Initializing API server...");
|
||||
|
@ -58,6 +62,10 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||
wait_from(watch_cancel.clone()),
|
||||
));
|
||||
|
||||
info!("Configure and run admin web server...");
|
||||
let admin_server =
|
||||
tokio::spawn(admin_server_init.run(garage.clone(), wait_from(watch_cancel.clone())));
|
||||
|
||||
// Stuff runs
|
||||
|
||||
// When a cancel signal is sent, stuff stops
|
||||
|
@ -67,6 +75,9 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||
if let Err(e) = web_server.await? {
|
||||
warn!("Web server exited with error: {}", e);
|
||||
}
|
||||
if let Err(e) = admin_server.await? {
|
||||
warn!("Admin web server exited with error: {}", e);
|
||||
}
|
||||
|
||||
// Remove RPC handlers for system to break reference cycles
|
||||
garage.system.netapp.drop_all_handlers();
|
||||
|
|
|
@ -73,6 +73,9 @@ pub struct Config {
|
|||
|
||||
/// Configuration for serving files as normal web server
|
||||
pub s3_web: WebConfig,
|
||||
|
||||
/// Configuration for the admin API endpoint
|
||||
pub admin_api: AdminConfig,
|
||||
}
|
||||
|
||||
/// Configuration for S3 api
|
||||
|
@ -96,6 +99,13 @@ pub struct WebConfig {
|
|||
pub root_domain: String,
|
||||
}
|
||||
|
||||
/// Configuration for the admin and monitoring HTTP API
|
||||
#[derive(Deserialize, Debug, Clone)]
|
||||
pub struct AdminConfig {
|
||||
/// Address and port to bind for admin API serving
|
||||
pub bind_addr: SocketAddr,
|
||||
}
|
||||
|
||||
fn default_sled_cache_capacity() -> u64 {
|
||||
128 * 1024 * 1024
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue