From e349af13a7268d567c1bacc819af5b89c2d4231f Mon Sep 17 00:00:00 2001 From: mricher Date: Tue, 28 Sep 2021 08:57:20 +0200 Subject: [PATCH 01/19] Update dependencies and add admin module with metrics - Global dependencies updated in Cargo.lock - New module created in src/admin to host: - the (future) admin REST API - the metric collection - add configuration block No metrics implemented yet --- Cargo.lock | 81 ++++++++++++++++++++++++ Cargo.toml | 3 +- script/dev-cluster.sh | 3 + src/admin/Cargo.toml | 28 +++++++++ src/admin/lib.rs | 6 ++ src/admin/metrics.rs | 141 ++++++++++++++++++++++++++++++++++++++++++ src/garage/Cargo.toml | 1 + src/garage/server.rs | 13 +++- src/util/config.rs | 10 +++ 9 files changed, 284 insertions(+), 2 deletions(-) create mode 100644 src/admin/Cargo.toml create mode 100644 src/admin/lib.rs create mode 100644 src/admin/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index 09097857..de1035bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -403,6 +403,16 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa" +dependencies = [ + "cfg-if", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.7" @@ -766,6 +776,7 @@ dependencies = [ "bytes 1.1.0", "futures", "futures-util", + "garage_admin", "garage_api", "garage_model 0.6.0", "garage_rpc 0.6.0", @@ -791,6 +802,23 @@ dependencies = [ "toml", ] +[[package]] +name = "garage_admin" +version = "0.6.0" +dependencies = [ + "futures", + "futures-util", + "garage_model 0.6.0", + "garage_util 0.6.0", + "http", + "hyper", + "lazy_static", + "log", + "opentelemetry", + "opentelemetry-prometheus", + "prometheus", +] + [[package]] name = "garage_api" version = "0.6.0" @@ -1824,6 +1852,38 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8" +dependencies = [ + "async-trait", + "crossbeam-channel", + "dashmap", + "fnv", + "futures-channel", + "futures-executor", + "futures-util", + "js-sys", + "lazy_static", + "percent-encoding", + "pin-project 1.0.10", + "rand", + "thiserror", +] + +[[package]] +name = "opentelemetry-prometheus" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9328977e479cebe12ce0d3fcecdaea4721d234895a9440c5b5dfd113f0594ac6" +dependencies = [ + "opentelemetry", + "prometheus", + "protobuf", +] + [[package]] name = "ordered-float" version = "2.10.0" @@ -2076,6 +2136,27 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "prometheus" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror", +] + +[[package]] +name = "protobuf" +version = "2.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96" + [[package]] name = "quick-error" version = "1.2.3" diff --git a/Cargo.toml b/Cargo.toml index 739e698e..88c8ad7d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,9 +4,10 @@ members = [ "src/rpc", "src/table", "src/model", + "src/admin", "src/api", "src/web", - "src/garage", + "src/garage" ] [profile.dev] diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh index c1ffb355..5cc003ef 100755 --- a/script/dev-cluster.sh +++ b/script/dev-cluster.sh @@ -44,6 +44,9 @@ root_domain = ".s3.garage.localhost" bind_addr = "0.0.0.0:$((3920+$count))" root_domain = ".web.garage.localhost" index = "index.html" + +[admin_api] +bind_addr = "0.0.0.0:$((9900+$count))" EOF echo -en "$LABEL configuration written to $CONF_PATH\n" diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml new file mode 100644 index 00000000..9775b667 --- /dev/null +++ b/src/admin/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "garage_admin" +version = "0.6.0" +authors = ["Maximilien Richer "] +edition = "2018" +license = "AGPL-3.0" +description = "Administration and metrics REST HTTP server for Garage" +repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" + +[lib] +path = "lib.rs" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +garage_model = { version = "0.6.0", path = "../model" } +garage_util = { version = "0.6.0", path = "../util" } + +futures = "0.3" +futures-util = "0.3" +http = "0.2" +hyper = "0.14" +log = "0.4" + +opentelemetry = "0.17" +opentelemetry-prometheus = "0.10" +prometheus = "0.13" +lazy_static = "1.4" diff --git a/src/admin/lib.rs b/src/admin/lib.rs new file mode 100644 index 00000000..443361be --- /dev/null +++ b/src/admin/lib.rs @@ -0,0 +1,6 @@ +//! Crate for handling the admin and metric HTTP APIs +#[macro_use] +extern crate log; +extern crate lazy_static; + +pub mod metrics; diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs new file mode 100644 index 00000000..547ee4c8 --- /dev/null +++ b/src/admin/metrics.rs @@ -0,0 +1,141 @@ +use hyper::{ + header::CONTENT_TYPE, + service::{make_service_fn, service_fn}, + Body, Method, Request, Response, Server, +}; +use lazy_static::lazy_static; +use opentelemetry::{ + global, + metrics::{BoundCounter, BoundValueRecorder}, + KeyValue, +}; +use opentelemetry_prometheus::PrometheusExporter; +use prometheus::{Encoder, TextEncoder}; +use std::convert::Infallible; +use std::sync::Arc; +use std::time::SystemTime; + +use futures::future::*; +use garage_model::garage::Garage; +use garage_util::error::Error as GarageError; + +lazy_static! { + // This defines the differennt tags that will be referenced by the object + static ref HANDLER_ALL: [KeyValue; 1] = [KeyValue::new("handler", "all")]; +} + +// serve_req on metric endpoint +async fn serve_req( + req: Request, + admin_server: Arc, +) -> Result, hyper::Error> { + println!("Receiving request at path {}", req.uri()); + let request_start = SystemTime::now(); + + admin_server.metrics.http_counter.add(1); + + let response = match (req.method(), req.uri().path()) { + (&Method::GET, "/metrics") => { + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + let metric_families = admin_server.exporter.registry().gather(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + admin_server + .metrics + .http_body_gauge + .record(buffer.len() as u64); + + Response::builder() + .status(200) + .header(CONTENT_TYPE, encoder.format_type()) + .body(Body::from(buffer)) + .unwrap() + } + _ => Response::builder() + .status(404) + .body(Body::from("Not implemented")) + .unwrap(), + }; + + admin_server + .metrics + .http_req_histogram + .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); + Ok(response) +} + +// AdminServer hold the admin server internal admin_server and the metric exporter +pub struct AdminServer { + exporter: PrometheusExporter, + metrics: AdminServerMetrics, +} + +// GarageMetricadmin_server holds the metrics counter definition for Garage +// FIXME: we would rather have that split up among the different libraries? +struct AdminServerMetrics { + http_counter: BoundCounter, + http_body_gauge: BoundValueRecorder, + http_req_histogram: BoundValueRecorder, + bucket_v2_merkle_updater_todo_queue_length: BoundValueRecorder, +} + +impl AdminServer { + /// init initilialize the AdminServer and background metric server + pub fn init() -> AdminServer { + let exporter = opentelemetry_prometheus::exporter().init(); + let meter = global::meter("garage/admin_server"); + AdminServer { + exporter, + metrics: AdminServerMetrics { + http_counter: meter + .u64_counter("router.http_requests_total") + .with_description("Total number of HTTP requests made.") + .init() + .bind(HANDLER_ALL.as_ref()), + http_body_gauge: meter + .u64_value_recorder("example.http_response_size_bytes") + .with_description("The metrics HTTP response sizes in bytes.") + .init() + .bind(HANDLER_ALL.as_ref()), + http_req_histogram: meter + .f64_value_recorder("example.http_request_duration_seconds") + .with_description("The HTTP request latencies in seconds.") + .init() + .bind(HANDLER_ALL.as_ref()), + bucket_v2_merkle_updater_todo_queue_length: meter + .f64_value_recorder("bucket_v2.merkle_updater.todo_queue_length") + .with_description("Bucket merkle updater TODO queue length.") + .init() + .bind(HANDLER_ALL.as_ref()), + }, + } + } + /// run execute the admin server on the designated HTTP port and listen for requests + pub async fn run( + self, + garage: Arc, + shutdown_signal: impl Future, + ) -> Result<(), GarageError> { + let admin_server = Arc::new(self); + // For every connection, we must make a `Service` to handle all + // incoming HTTP requests on said connection. + let make_svc = make_service_fn(move |_conn| { + let admin_server = admin_server.clone(); + // This is the `Service` that will handle the connection. + // `service_fn` is a helper to convert a function that + // returns a Response into a `Service`. + async move { + Ok::<_, Infallible>(service_fn(move |req| serve_req(req, admin_server.clone()))) + } + }); + + let addr = &garage.config.admin_api.bind_addr; + + let server = Server::bind(&addr).serve(make_svc); + let graceful = server.with_graceful_shutdown(shutdown_signal); + info!("Admin server listening on http://{}", addr); + + graceful.await?; + Ok(()) + } +} diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 463f83e7..22e0f0f0 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -27,6 +27,7 @@ garage_rpc = { version = "0.6.0", path = "../rpc" } garage_table = { version = "0.6.0", path = "../table" } garage_util = { version = "0.6.0", path = "../util" } garage_web = { version = "0.6.0", path = "../web" } +garage_admin = { version = "0.6.0", path = "../admin" } bytes = "1.0" git-version = "0.3.4" diff --git a/src/garage/server.rs b/src/garage/server.rs index f4d62e91..923df1cd 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -6,6 +6,7 @@ use garage_util::background::*; use garage_util::config::*; use garage_util::error::Error; +use garage_admin::metrics::*; use garage_api::run_api_server; use garage_model::garage::Garage; use garage_web::run_web_server; @@ -34,6 +35,9 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { .open() .expect("Unable to open sled DB"); + info!("Configure and run admin web server..."); + let admin_server_init = AdminServer::init(); + info!("Initializing background runner..."); let watch_cancel = netapp::util::watch_ctrl_c(); let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone()); @@ -43,7 +47,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone())); - info!("Crate admin RPC handler..."); + info!("Create admin RPC handler..."); AdminRpcHandler::new(garage.clone()); info!("Initializing API server..."); @@ -58,6 +62,10 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { wait_from(watch_cancel.clone()), )); + info!("Configure and run admin web server..."); + let admin_server = + tokio::spawn(admin_server_init.run(garage.clone(), wait_from(watch_cancel.clone()))); + // Stuff runs // When a cancel signal is sent, stuff stops @@ -67,6 +75,9 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { if let Err(e) = web_server.await? { warn!("Web server exited with error: {}", e); } + if let Err(e) = admin_server.await? { + warn!("Admin web server exited with error: {}", e); + } // Remove RPC handlers for system to break reference cycles garage.system.netapp.drop_all_handlers(); diff --git a/src/util/config.rs b/src/util/config.rs index 19c75478..2d15748f 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -73,6 +73,9 @@ pub struct Config { /// Configuration for serving files as normal web server pub s3_web: WebConfig, + + /// Configuration for the admin API endpoint + pub admin_api: AdminConfig, } /// Configuration for S3 api @@ -96,6 +99,13 @@ pub struct WebConfig { pub root_domain: String, } +/// Configuration for the admin and monitoring HTTP API +#[derive(Deserialize, Debug, Clone)] +pub struct AdminConfig { + /// Address and port to bind for admin API serving + pub bind_addr: SocketAddr, +} + fn default_sled_cache_capacity() -> u64 { 128 * 1024 * 1024 } -- 2.43.0 From 1e2cf26373ef1812a3152a0057774f6381e66914 Mon Sep 17 00:00:00 2001 From: Maximilien R Date: Tue, 15 Feb 2022 20:09:43 +0100 Subject: [PATCH 02/19] Implement basic metrics in table --- Cargo.lock | 1 + src/admin/metrics.rs | 6 ------ src/garage/server.rs | 2 +- src/table/Cargo.toml | 2 ++ src/table/data.rs | 6 ++++++ src/table/lib.rs | 1 + src/table/metrics.rs | 25 +++++++++++++++++++++++++ 7 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 src/table/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index de1035bd..c2608fd8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -998,6 +998,7 @@ dependencies = [ "garage_util 0.6.0", "hexdump", "log", + "opentelemetry", "rand", "rmp-serde 0.15.5", "serde", diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs index 547ee4c8..ccc26d26 100644 --- a/src/admin/metrics.rs +++ b/src/admin/metrics.rs @@ -76,7 +76,6 @@ struct AdminServerMetrics { http_counter: BoundCounter, http_body_gauge: BoundValueRecorder, http_req_histogram: BoundValueRecorder, - bucket_v2_merkle_updater_todo_queue_length: BoundValueRecorder, } impl AdminServer { @@ -102,11 +101,6 @@ impl AdminServer { .with_description("The HTTP request latencies in seconds.") .init() .bind(HANDLER_ALL.as_ref()), - bucket_v2_merkle_updater_todo_queue_length: meter - .f64_value_recorder("bucket_v2.merkle_updater.todo_queue_length") - .with_description("Bucket merkle updater TODO queue length.") - .init() - .bind(HANDLER_ALL.as_ref()), }, } } diff --git a/src/garage/server.rs b/src/garage/server.rs index 923df1cd..cd32d708 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -35,7 +35,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { .open() .expect("Unable to open sled DB"); - info!("Configure and run admin web server..."); + info!("Initialize admin web server and metric backend..."); let admin_server_init = AdminServer::init(); info!("Initializing background runner..."); diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 91d71ddd..f6f9974d 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -17,6 +17,8 @@ path = "lib.rs" garage_rpc = { version = "0.6.0", path = "../rpc" } garage_util = { version = "0.6.0", path = "../util" } +opentelemetry = "0.17" + async-trait = "0.1.7" bytes = "1.0" hexdump = "0.1" diff --git a/src/table/data.rs b/src/table/data.rs index d7787b6b..a5209c26 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -13,6 +13,7 @@ use garage_rpc::system::System; use crate::crdt::Crdt; use crate::gc::GcTodoEntry; +use crate::metrics::*; use crate::replication::*; use crate::schema::*; @@ -28,6 +29,8 @@ pub struct TableData { pub(crate) merkle_todo: sled::Tree, pub(crate) merkle_todo_notify: Notify, pub(crate) gc_todo: sled::Tree, + + pub(crate) metrics: TableMetrics, } impl TableData @@ -51,6 +54,8 @@ where .open_tree(&format!("{}:gc_todo_v2", F::TABLE_NAME)) .expect("Unable to open DB tree"); + let metrics = TableMetrics::new(F::TABLE_NAME, merkle_todo.clone()); + Arc::new(Self { system, instance, @@ -60,6 +65,7 @@ where merkle_todo, merkle_todo_notify: Notify::new(), gc_todo, + metrics, }) } diff --git a/src/table/lib.rs b/src/table/lib.rs index d6c19f1b..e0920357 100644 --- a/src/table/lib.rs +++ b/src/table/lib.rs @@ -4,6 +4,7 @@ #[macro_use] extern crate log; +mod metrics; pub mod schema; pub mod util; diff --git a/src/table/metrics.rs b/src/table/metrics.rs new file mode 100644 index 00000000..38e93904 --- /dev/null +++ b/src/table/metrics.rs @@ -0,0 +1,25 @@ +use opentelemetry::{global, metrics::*, KeyValue}; + +/// TableMetrics reference all counter used for metrics +pub struct TableMetrics { + merkle_updater_todo_queue_length: ValueObserver, +} +impl TableMetrics { + pub fn new(table_name: &'static str, merkle_todo: sled::Tree) -> Self { + let meter = global::meter(table_name); + TableMetrics { + merkle_updater_todo_queue_length: meter + .u64_value_observer( + format!("merkle_updater_todo_queue_length"), + move |observer| { + observer.observe( + merkle_todo.len() as u64, + &[KeyValue::new("table_name", table_name)], + ) + }, + ) + .with_description("Bucket merkle updater TODO queue length") + .init(), + } + } +} -- 2.43.0 From 2cab84b1fe423a41b356211e592a614c95ec4e0c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 16 Feb 2022 14:23:04 +0100 Subject: [PATCH 03/19] Add many metrics in table/ and rpc/ --- Cargo.lock | 3 +- Cargo.nix | 173 +++++++++++++++++++++++++----- src/admin/Cargo.toml | 1 - src/admin/lib.rs | 1 - src/admin/metrics.rs | 21 ++-- src/garage/tests/common/garage.rs | 4 + src/model/Cargo.toml | 1 + src/model/block.rs | 55 +++++++++- src/model/block_metrics.rs | 93 ++++++++++++++++ src/model/lib.rs | 1 + src/rpc/Cargo.toml | 3 +- src/rpc/lib.rs | 1 + src/rpc/metrics.rs | 55 ++++++++++ src/rpc/rpc_helper.rs | 41 ++++++- src/table/data.rs | 6 +- src/table/metrics.rs | 75 ++++++++++++- src/table/sync.rs | 32 +++++- src/table/table.rs | 34 +++++- 18 files changed, 537 insertions(+), 63 deletions(-) create mode 100644 src/model/block_metrics.rs create mode 100644 src/rpc/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index c2608fd8..2cebfd33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -812,7 +812,6 @@ dependencies = [ "garage_util 0.6.0", "http", "hyper", - "lazy_static", "log", "opentelemetry", "opentelemetry-prometheus", @@ -898,6 +897,7 @@ dependencies = [ "hex", "log", "netapp", + "opentelemetry", "rand", "rmp-serde 0.15.5", "serde", @@ -953,6 +953,7 @@ dependencies = [ "log", "netapp", "openssl", + "opentelemetry", "pnet", "rand", "rmp-serde 0.15.5", diff --git a/Cargo.nix b/Cargo.nix index 42416177..3426ee4e 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -8,6 +8,7 @@ args@{ "garage_rpc/default" "garage_table/default" "garage_model/default" + "garage_admin/default" "garage_api/default" "garage_web/default" "garage/default" @@ -47,6 +48,7 @@ in garage_rpc = rustPackages.unknown.garage_rpc."0.6.0"; garage_table = rustPackages.unknown.garage_table."0.6.0"; garage_model = rustPackages.unknown.garage_model."0.6.0"; + garage_admin = rustPackages.unknown.garage_admin."0.6.0"; garage_api = rustPackages.unknown.garage_api."0.6.0"; garage_web = rustPackages.unknown.garage_web."0.6.0"; garage = rustPackages.unknown.garage."0.6.0"; @@ -562,7 +564,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; }; dependencies = { - ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; }; }); @@ -580,6 +582,22 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.2" = overridableMkRustCrate (profileName: rec { + name = "crossbeam-channel"; + version = "0.5.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa"; }; + features = builtins.concatLists [ + [ "crossbeam-utils" ] + [ "default" ] + [ "std" ] + ]; + dependencies = { + cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; + crossbeam_utils = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-utils."0.8.7" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".crossbeam-epoch."0.9.7" = overridableMkRustCrate (profileName: rec { name = "crossbeam-epoch"; version = "0.9.7"; @@ -982,6 +1000,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6"; }; features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") [ "std" ] ]; dependencies = { @@ -1047,7 +1066,7 @@ in [ "async-await" ] [ "async-await-macro" ] [ "channel" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") [ "futures-channel" ] [ "futures-io" ] [ "futures-macro" ] @@ -1092,6 +1111,7 @@ in bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + garage_admin = rustPackages."unknown".garage_admin."0.6.0" { inherit profileName; }; garage_api = rustPackages."unknown".garage_api."0.6.0" { inherit profileName; }; garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; garage_rpc = rustPackages."unknown".garage_rpc."0.6.0" { inherit profileName; }; @@ -1121,6 +1141,25 @@ in }; }); + "unknown".garage_admin."0.6.0" = overridableMkRustCrate (profileName: rec { + name = "garage_admin"; + version = "0.6.0"; + registry = "unknown"; + src = fetchCrateLocal (workspaceSrc + "/src/admin"); + dependencies = { + futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; + futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; + hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; + opentelemetry_prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }; + prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }; + }; + }); + "unknown".garage_api."0.6.0" = overridableMkRustCrate (profileName: rec { name = "garage_api"; version = "0.6.0"; @@ -1206,6 +1245,7 @@ in hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; @@ -1265,6 +1305,7 @@ in log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; openssl = rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl."0.10.38" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; pnet = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet."0.28.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; @@ -1314,6 +1355,7 @@ in garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; @@ -1632,7 +1674,7 @@ in (lib.optional (rootFeatures' ? "garage") "h2") [ "http1" ] (lib.optional (rootFeatures' ? "garage") "http2") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "runtime") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "runtime") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "server") [ "socket2" ] [ "stream" ] @@ -2197,7 +2239,7 @@ in [ "os-poll" ] ]; dependencies = { - ${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; ${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; }; ${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; }; @@ -2396,7 +2438,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "vendored") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "vendored") ]; dependencies = { bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; @@ -2434,8 +2476,8 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "openssl-src") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "vendored") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "openssl-src") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "vendored") ]; dependencies = { libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; @@ -2443,12 +2485,58 @@ in buildDependencies = { autocfg = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }; cc = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.73" { profileName = "__noProfile"; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "openssl_src" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl-src."111.17.0+1.1.1m" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "openssl_src" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl-src."111.17.0+1.1.1m" { profileName = "__noProfile"; }; pkg_config = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.24" { profileName = "__noProfile"; }; ${ if hostPlatform.parsed.abi.name == "msvc" then "vcpkg" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }; }; }); + "registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" = overridableMkRustCrate (profileName: rec { + name = "opentelemetry"; + version = "0.17.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"; }; + features = builtins.concatLists [ + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "async-trait") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "crossbeam-channel") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "dashmap") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "fnv") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "metrics") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "percent-encoding") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "pin-project") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rand") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "trace") + ]; + dependencies = { + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "async_trait" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "crossbeam_channel" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.2" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "dashmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".dashmap."4.0.2" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "fnv" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_channel" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_executor" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + ${ if (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") && hostPlatform.parsed.cpu.name == "wasm32" then "js_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.56" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "lazy_static" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "percent_encoding" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "pin_project" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "thiserror" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" = overridableMkRustCrate (profileName: rec { + name = "opentelemetry-prometheus"; + version = "0.10.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9328977e479cebe12ce0d3fcecdaea4721d234895a9440c5b5dfd113f0594ac6"; }; + dependencies = { + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; + prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }; + protobuf = rustPackages."registry+https://github.com/rust-lang/crates.io-index".protobuf."2.27.1" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".ordered-float."2.10.0" = overridableMkRustCrate (profileName: rec { name = "ordered-float"; version = "2.10.0"; @@ -2756,6 +2844,33 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" = overridableMkRustCrate (profileName: rec { + name = "prometheus"; + version = "0.13.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "b7f64969ffd5dd8f39bd57a68ac53c163a095ed9d0fb707146da1b27025a3504"; }; + features = builtins.concatLists [ + [ "default" ] + [ "protobuf" ] + ]; + dependencies = { + cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; + fnv = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }; + lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; + memchr = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; + parking_lot = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }; + protobuf = rustPackages."registry+https://github.com/rust-lang/crates.io-index".protobuf."2.27.1" { inherit profileName; }; + thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".protobuf."2.27.1" = overridableMkRustCrate (profileName: rec { + name = "protobuf"; + version = "2.27.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "cf7e6d18738ecd0902d30d1ad232c9125985a3422929b16c65517b38adc14f96"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".quick-error."1.2.3" = overridableMkRustCrate (profileName: rec { name = "quick-error"; version = "1.2.3"; @@ -2870,27 +2985,27 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "aho-corasick") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "memchr") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf-cache") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf-dfa") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf-inline") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf-literal") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "aho-corasick") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "memchr") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf-cache") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf-dfa") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf-inline") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "perf-literal") [ "std" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-age") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-bool") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-case") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-gencat") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-perl") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-script") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-segment") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-age") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-bool") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-case") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-gencat") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-perl") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-script") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "unicode-segment") ]; dependencies = { - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "aho_corasick" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "memchr" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "aho_corasick" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "memchr" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; regex_syntax = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.25" { inherit profileName; }; }; }); @@ -2936,7 +3051,7 @@ in ]; dependencies = { ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; }; untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; }; @@ -3413,7 +3528,7 @@ in ]; dependencies = { bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" { inherit profileName; }; static_init_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; }; @@ -3725,7 +3840,7 @@ in src = fetchCratesIo { inherit name version; sha256 = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"; }; features = builtins.concatLists [ [ "default" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "net") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "net") [ "time" ] ]; dependencies = { diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 9775b667..6f646869 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -25,4 +25,3 @@ log = "0.4" opentelemetry = "0.17" opentelemetry-prometheus = "0.10" prometheus = "0.13" -lazy_static = "1.4" diff --git a/src/admin/lib.rs b/src/admin/lib.rs index 443361be..f1e8ddd7 100644 --- a/src/admin/lib.rs +++ b/src/admin/lib.rs @@ -1,6 +1,5 @@ //! Crate for handling the admin and metric HTTP APIs #[macro_use] extern crate log; -extern crate lazy_static; pub mod metrics; diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs index ccc26d26..44fd4cb2 100644 --- a/src/admin/metrics.rs +++ b/src/admin/metrics.rs @@ -3,11 +3,9 @@ use hyper::{ service::{make_service_fn, service_fn}, Body, Method, Request, Response, Server, }; -use lazy_static::lazy_static; use opentelemetry::{ global, metrics::{BoundCounter, BoundValueRecorder}, - KeyValue, }; use opentelemetry_prometheus::PrometheusExporter; use prometheus::{Encoder, TextEncoder}; @@ -19,11 +17,6 @@ use futures::future::*; use garage_model::garage::Garage; use garage_util::error::Error as GarageError; -lazy_static! { - // This defines the differennt tags that will be referenced by the object - static ref HANDLER_ALL: [KeyValue; 1] = [KeyValue::new("handler", "all")]; -} - // serve_req on metric endpoint async fn serve_req( req: Request, @@ -87,20 +80,20 @@ impl AdminServer { exporter, metrics: AdminServerMetrics { http_counter: meter - .u64_counter("router.http_requests_total") + .u64_counter("admin.http_requests_total") .with_description("Total number of HTTP requests made.") .init() - .bind(HANDLER_ALL.as_ref()), + .bind(&[]), http_body_gauge: meter - .u64_value_recorder("example.http_response_size_bytes") + .u64_value_recorder("admin.http_response_size_bytes") .with_description("The metrics HTTP response sizes in bytes.") .init() - .bind(HANDLER_ALL.as_ref()), + .bind(&[]), http_req_histogram: meter - .f64_value_recorder("example.http_request_duration_seconds") + .f64_value_recorder("admin.http_request_duration_seconds") .with_description("The HTTP request latencies in seconds.") .init() - .bind(HANDLER_ALL.as_ref()), + .bind(&[]), }, } } @@ -125,7 +118,7 @@ impl AdminServer { let addr = &garage.config.admin_api.bind_addr; - let server = Server::bind(&addr).serve(make_svc); + let server = Server::bind(addr).serve(make_svc); let graceful = server.with_graceful_shutdown(shutdown_signal); info!("Admin server listening on http://{}", addr); diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs index 92aa2edf..12cf946b 100644 --- a/src/garage/tests/common/garage.rs +++ b/src/garage/tests/common/garage.rs @@ -65,6 +65,9 @@ root_domain = ".s3.garage" bind_addr = "127.0.0.1:{web_port}" root_domain = ".web.garage" index = "index.html" + +[admin_api] +bind_addr = "127.0.0.1:{admin_port}" "#, path = path.display(), secret = GARAGE_TEST_SECRET, @@ -72,6 +75,7 @@ index = "index.html" api_port = port, rpc_port = port + 1, web_port = port + 2, + admin_port = port + 3, ); fs::write(path.join("config.toml"), config).expect("Could not write garage config file"); diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 14e49557..10a4c838 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -36,6 +36,7 @@ serde_bytes = "0.11" futures = "0.3" futures-util = "0.3" tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } +opentelemetry = "0.17" #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } netapp = "0.3.0" diff --git a/src/model/block.rs b/src/model/block.rs index 1173c7b3..9e939c24 100644 --- a/src/model/block.rs +++ b/src/model/block.rs @@ -1,12 +1,13 @@ use std::convert::TryInto; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use arc_swap::ArcSwapOption; use async_trait::async_trait; use futures::future::*; use futures::select; +use opentelemetry::KeyValue; use serde::{Deserialize, Serialize}; use tokio::fs; use tokio::io::{AsyncReadExt, AsyncWriteExt}; @@ -23,8 +24,8 @@ use garage_rpc::*; use garage_table::replication::{TableReplication, TableShardedReplication}; +use crate::block_metrics::*; use crate::block_ref_table::*; - use crate::garage::Garage; /// Size under which data will be stored inlined in database instead of as files @@ -154,6 +155,8 @@ pub struct BlockManager { system: Arc, endpoint: Arc>, pub(crate) garage: ArcSwapOption, + + metrics: BlockManagerMetrics, } // This custom struct contains functions that must only be ran @@ -182,6 +185,8 @@ impl BlockManager { let manager_locked = BlockManagerLocked(); + let metrics = BlockManagerMetrics::new(resync_queue.clone()); + let block_manager = Arc::new(Self { replication, data_dir, @@ -192,6 +197,7 @@ impl BlockManager { system, endpoint, garage: ArcSwapOption::from(None), + metrics, }); block_manager.endpoint.set_handler(block_manager.clone()); @@ -380,15 +386,28 @@ impl BlockManager { /// Write a block to disk async fn write_block(&self, hash: &Hash, data: &DataBlock) -> Result { - self.mutation_lock + let request_start = SystemTime::now(); + let write_size = data.inner_buffer().len() as u64; + + let res = self + .mutation_lock .lock() .await .write_block(hash, data, self) - .await + .await?; + + self.metrics.bytes_written.add(write_size); + self.metrics + .block_write_duration + .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); + + Ok(res) } /// Read block from disk, verifying it's integrity async fn read_block(&self, hash: &Hash) -> Result { + let request_start = SystemTime::now(); + let mut path = self.block_path(hash); let compressed = match self.is_block_compressed(hash).await { Ok(c) => c, @@ -414,6 +433,8 @@ impl BlockManager { }; if data.verify(*hash).is_err() { + self.metrics.corruption_counter.add(1); + self.mutation_lock .lock() .await @@ -423,6 +444,13 @@ impl BlockManager { return Err(Error::CorruptData(*hash)); } + self.metrics + .bytes_read + .add(data.inner_buffer().len() as u64); + self.metrics + .block_read_duration + .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); + Ok(BlockRpc::PutBlock { hash: *hash, data }) } @@ -521,9 +549,18 @@ impl BlockManager { let time_msec = u64::from_be_bytes(time_bytes[0..8].try_into().unwrap()); let now = now_msec(); if now >= time_msec { + let start_time = SystemTime::now(); + let hash = Hash::try_from(&hash_bytes[..]).unwrap(); let res = self.resync_block(&hash).await; + + self.metrics.resync_counter.add(1); + self.metrics + .resync_duration + .record(start_time.elapsed().map_or(0.0, |d| d.as_secs_f64())); + if let Err(e) = &res { + self.metrics.resync_error_counter.add(1); warn!("Error when resyncing {:?}: {}", hash, e); self.put_to_resync(&hash, RESYNC_RETRY_DELAY)?; } @@ -607,6 +644,12 @@ impl BlockManager { need_nodes.len() ); + for node in need_nodes.iter() { + self.metrics + .resync_send_counter + .add(1, &[KeyValue::new("to", format!("{:?}", node))]); + } + let put_block_message = self.read_block(hash).await?; self.system .rpc @@ -644,6 +687,9 @@ impl BlockManager { ); let block_data = self.rpc_get_raw_block(hash).await?; + + self.metrics.resync_recv_counter.add(1); + self.write_block(hash, &block_data).await?; } @@ -819,6 +865,7 @@ impl BlockManagerLocked { path.set_extension("zst"); } fs::remove_file(path).await?; + mgr.metrics.delete_counter.add(1); } Ok(()) } diff --git a/src/model/block_metrics.rs b/src/model/block_metrics.rs new file mode 100644 index 00000000..7ef9a117 --- /dev/null +++ b/src/model/block_metrics.rs @@ -0,0 +1,93 @@ +use opentelemetry::{global, metrics::*}; + +/// TableMetrics reference all counter used for metrics +pub struct BlockManagerMetrics { + pub(crate) _resync_queue_len: ValueObserver, + + pub(crate) resync_counter: BoundCounter, + pub(crate) resync_error_counter: BoundCounter, + pub(crate) resync_duration: BoundValueRecorder, + pub(crate) resync_send_counter: Counter, + pub(crate) resync_recv_counter: BoundCounter, + + pub(crate) bytes_read: BoundCounter, + pub(crate) block_read_duration: BoundValueRecorder, + pub(crate) bytes_written: BoundCounter, + pub(crate) block_write_duration: BoundValueRecorder, + pub(crate) delete_counter: BoundCounter, + + pub(crate) corruption_counter: BoundCounter, +} + +impl BlockManagerMetrics { + pub fn new(resync_queue: sled::Tree) -> Self { + let meter = global::meter("garage_model/block"); + Self { + _resync_queue_len: meter + .u64_value_observer("block.resync_queue_length", move |observer| { + observer.observe(resync_queue.len() as u64, &[]) + }) + .with_description( + "Number of block hashes queued for local check and possible resync", + ) + .init(), + + resync_counter: meter + .u64_counter("block.resync_counter") + .with_description("Number of calls to resync_block") + .init() + .bind(&[]), + resync_error_counter: meter + .u64_counter("block.resync_error_counter") + .with_description("Number of calls to resync_block that returned an error") + .init() + .bind(&[]), + resync_duration: meter + .f64_value_recorder("block.resync_duration") + .with_description("Duration of resync_block operations") + .init() + .bind(&[]), + resync_send_counter: meter + .u64_counter("block.resync_send_counter") + .with_description("Number of blocks sent to another node in resync operations") + .init(), + resync_recv_counter: meter + .u64_counter("block.resync_recv_counter") + .with_description("Number of blocks received from other nodes in resync operations") + .init() + .bind(&[]), + + bytes_read: meter + .u64_counter("block.bytes_read") + .with_description("Number of bytes read from disk") + .init() + .bind(&[]), + block_read_duration: meter + .f64_value_recorder("block.read_duration") + .with_description("Duration of block read operations") + .init() + .bind(&[]), + bytes_written: meter + .u64_counter("block.bytes_written") + .with_description("Number of bytes written to disk") + .init() + .bind(&[]), + block_write_duration: meter + .f64_value_recorder("block.write_duration") + .with_description("Duration of block write operations") + .init() + .bind(&[]), + delete_counter: meter + .u64_counter("block.delete_counter") + .with_description("Number of blocks deleted") + .init() + .bind(&[]), + + corruption_counter: meter + .u64_counter("block.corruption_counter") + .with_description("Data corruptions detected on block reads") + .init() + .bind(&[]), + } + } +} diff --git a/src/model/lib.rs b/src/model/lib.rs index 9deaae9d..c8677603 100644 --- a/src/model/lib.rs +++ b/src/model/lib.rs @@ -11,6 +11,7 @@ pub mod object_table; pub mod version_table; pub mod block; +mod block_metrics; pub mod garage; pub mod helper; diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index f06606e5..57b61a08 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -43,8 +43,9 @@ futures = "0.3" futures-util = "0.3" tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } tokio-stream = { version = "0.1", features = ["net"] } +opentelemetry = "0.17" #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } -netapp = "0.3.0" +netapp = "0.3.1" hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] } diff --git a/src/rpc/lib.rs b/src/rpc/lib.rs index 2c877a7f..736513f4 100644 --- a/src/rpc/lib.rs +++ b/src/rpc/lib.rs @@ -10,6 +10,7 @@ pub mod layout; pub mod ring; pub mod system; +mod metrics; pub mod rpc_helper; pub use rpc_helper::*; diff --git a/src/rpc/metrics.rs b/src/rpc/metrics.rs new file mode 100644 index 00000000..c900518c --- /dev/null +++ b/src/rpc/metrics.rs @@ -0,0 +1,55 @@ +use std::sync::Arc; + +use opentelemetry::{global, metrics::*}; +use tokio::sync::Semaphore; + +/// TableMetrics reference all counter used for metrics +pub struct RpcMetrics { + pub(crate) _rpc_available_permits: ValueObserver, + + pub(crate) rpc_counter: Counter, + pub(crate) rpc_timeout_counter: Counter, + pub(crate) rpc_netapp_error_counter: Counter, + pub(crate) rpc_garage_error_counter: Counter, + + pub(crate) rpc_duration: ValueRecorder, + pub(crate) rpc_queueing_time: ValueRecorder, +} +impl RpcMetrics { + pub fn new(sem: Arc) -> Self { + let meter = global::meter("garage_rpc"); + RpcMetrics { + _rpc_available_permits: meter + .u64_value_observer("rpc.available_permits", move |observer| { + observer.observe(sem.available_permits() as u64, &[]) + }) + .with_description("Number of available RPC permits") + .init(), + + rpc_counter: meter + .u64_counter("rpc.request_counter") + .with_description("Number of RPC requests emitted") + .init(), + rpc_timeout_counter: meter + .u64_counter("rpc.timeout_counter") + .with_description("Number of RPC timeouts") + .init(), + rpc_netapp_error_counter: meter + .u64_counter("rpc.netapp_error_counter") + .with_description("Number of communication errors (errors in the Netapp library)") + .init(), + rpc_garage_error_counter: meter + .u64_counter("rpc.garage_error_counter") + .with_description("Number of RPC errors (errors happening when handling the RPC)") + .init(), + rpc_duration: meter + .f64_value_recorder("rpc.duration") + .with_description("Duration of RPCs") + .init(), + rpc_queueing_time: meter + .f64_value_recorder("rpc.queueing_time") + .with_description("Time RPC requests were queued for before being sent") + .init(), + } + } +} diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index 68bdfc4f..0d722e43 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -1,11 +1,12 @@ //! Contain structs related to making RPCs use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use futures::future::join_all; use futures::stream::futures_unordered::FuturesUnordered; use futures::stream::StreamExt; use futures_util::future::FutureExt; +use opentelemetry::KeyValue; use tokio::select; use tokio::sync::{watch, Semaphore}; @@ -18,6 +19,7 @@ use garage_util::background::BackgroundRunner; use garage_util::data::*; use garage_util::error::Error; +use crate::metrics::RpcMetrics; use crate::ring::Ring; const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); @@ -76,7 +78,8 @@ struct RpcHelperInner { fullmesh: Arc, background: Arc, ring: watch::Receiver>, - request_buffer_semaphore: Semaphore, + request_buffer_semaphore: Arc, + metrics: RpcMetrics, } impl RpcHelper { @@ -86,12 +89,17 @@ impl RpcHelper { background: Arc, ring: watch::Receiver>, ) -> Self { + let sem = Arc::new(Semaphore::new(REQUEST_BUFFER_SIZE)); + + let metrics = RpcMetrics::new(sem.clone()); + Self(Arc::new(RpcHelperInner { our_node_id, fullmesh, background, ring, - request_buffer_semaphore: Semaphore::new(REQUEST_BUFFER_SIZE), + request_buffer_semaphore: sem, + metrics, })) } @@ -120,6 +128,9 @@ impl RpcHelper { M: Rpc>, H: EndpointHandler, { + let queueing_start_time = SystemTime::now(); + let metric_tags = [KeyValue::new("endpoint", endpoint.path().to_string())]; + let msg_size = rmp_to_vec_all_named(&msg)?.len() as u32; let permit = self .0 @@ -127,14 +138,36 @@ impl RpcHelper { .acquire_many(msg_size) .await?; + self.0.metrics.rpc_queueing_time.record( + queueing_start_time + .elapsed() + .map_or(0.0, |d| d.as_secs_f64()), + &metric_tags, + ); + self.0.metrics.rpc_counter.add(1, &metric_tags); + let rpc_start_time = SystemTime::now(); + let node_id = to.into(); select! { res = endpoint.call(&node_id, &msg, strat.rs_priority) => { drop(permit); - Ok(res??) + + if res.is_err() { + self.0.metrics.rpc_netapp_error_counter.add(1, &metric_tags); + } + let res = res?; + + self.0.metrics.rpc_duration + .record(rpc_start_time.elapsed().map_or(0.0, |d| d.as_secs_f64()), &metric_tags); + if res.is_err() { + self.0.metrics.rpc_garage_error_counter.add(1, &metric_tags); + } + + Ok(res?) } _ = tokio::time::sleep(strat.rs_timeout) => { drop(permit); + self.0.metrics.rpc_timeout_counter.add(1, &metric_tags); Err(Error::Timeout) } } diff --git a/src/table/data.rs b/src/table/data.rs index a5209c26..684afdcd 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -54,7 +54,7 @@ where .open_tree(&format!("{}:gc_todo_v2", F::TABLE_NAME)) .expect("Unable to open DB tree"); - let metrics = TableMetrics::new(F::TABLE_NAME, merkle_todo.clone()); + let metrics = TableMetrics::new(F::TABLE_NAME, merkle_todo.clone(), gc_todo.clone()); Arc::new(Self { system, @@ -171,6 +171,8 @@ where })?; if let Some((old_entry, new_entry, new_bytes_hash)) = changed { + self.metrics.internal_update_counter.add(1); + let is_tombstone = new_entry.is_tombstone(); self.instance.updated(old_entry, Some(new_entry)); self.merkle_todo_notify.notify_one(); @@ -205,6 +207,8 @@ where })?; if removed { + self.metrics.internal_delete_counter.add(1); + let old_entry = self.decode_entry(v)?; self.instance.updated(Some(old_entry), None); self.merkle_todo_notify.notify_one(); diff --git a/src/table/metrics.rs b/src/table/metrics.rs index 38e93904..548bf0d6 100644 --- a/src/table/metrics.rs +++ b/src/table/metrics.rs @@ -2,15 +2,27 @@ use opentelemetry::{global, metrics::*, KeyValue}; /// TableMetrics reference all counter used for metrics pub struct TableMetrics { - merkle_updater_todo_queue_length: ValueObserver, + pub(crate) _merkle_todo_len: ValueObserver, + pub(crate) _gc_todo_len: ValueObserver, + + pub(crate) get_request_counter: BoundCounter, + pub(crate) get_request_duration: BoundValueRecorder, + pub(crate) put_request_counter: BoundCounter, + pub(crate) put_request_duration: BoundValueRecorder, + + pub(crate) internal_update_counter: BoundCounter, + pub(crate) internal_delete_counter: BoundCounter, + + pub(crate) sync_items_sent: Counter, + pub(crate) sync_items_received: Counter, } impl TableMetrics { - pub fn new(table_name: &'static str, merkle_todo: sled::Tree) -> Self { + pub fn new(table_name: &'static str, merkle_todo: sled::Tree, gc_todo: sled::Tree) -> Self { let meter = global::meter(table_name); TableMetrics { - merkle_updater_todo_queue_length: meter + _merkle_todo_len: meter .u64_value_observer( - format!("merkle_updater_todo_queue_length"), + "table.merkle_updater_todo_queue_length", move |observer| { observer.observe( merkle_todo.len() as u64, @@ -18,7 +30,60 @@ impl TableMetrics { ) }, ) - .with_description("Bucket merkle updater TODO queue length") + .with_description("Merkle tree updater TODO queue length") + .init(), + _gc_todo_len: meter + .u64_value_observer( + "table.gc_todo_queue_length", + move |observer| { + observer.observe( + gc_todo.len() as u64, + &[KeyValue::new("table_name", table_name)], + ) + }, + ) + .with_description("Table garbage collector TODO queue length") + .init(), + + get_request_counter: meter + .u64_counter("table.get_request_counter") + .with_description("Number of get/get_range requests internally made on this table") + .init() + .bind(&[KeyValue::new("table_name", table_name)]), + get_request_duration: meter + .f64_value_recorder("table.get_request_duration") + .with_description("Duration of get/get_range requests internally made on this table, in seconds") + .init() + .bind(&[KeyValue::new("table_name", table_name)]), + put_request_counter: meter + .u64_counter("table.put_request_counter") + .with_description("Number of insert/insert_many requests internally made on this table") + .init() + .bind(&[KeyValue::new("table_name", table_name)]), + put_request_duration: meter + .f64_value_recorder("table.put_request_duration") + .with_description("Duration of insert/insert_many requests internally made on this table, in seconds") + .init() + .bind(&[KeyValue::new("table_name", table_name)]), + + internal_update_counter: meter + .u64_counter("table.internal_update_counter") + .with_description("Number of value updates where the value actually changes (includes creation of new key and update of existing key)") + .init() + .bind(&[KeyValue::new("table_name", table_name)]), + internal_delete_counter: meter + .u64_counter("table.internal_delete_counter") + .with_description("Number of value deletions in the tree (due to GC or repartitioning)") + .init() + .bind(&[KeyValue::new("table_name", table_name)]), + + sync_items_sent: meter + .u64_counter("table.sync_items_sent") + .with_description("Number of data items sent to other nodes during resync procedures") + .init(), + sync_items_received: meter + .u64_counter("table.sync_items_received") + .with_description("Number of data items received from other nodes during resync procedures") .init(), } } diff --git a/src/table/sync.rs b/src/table/sync.rs index 1df2b01d..08069ad0 100644 --- a/src/table/sync.rs +++ b/src/table/sync.rs @@ -6,6 +6,7 @@ use async_trait::async_trait; use futures::select; use futures_util::future::*; use futures_util::stream::*; +use opentelemetry::KeyValue; use rand::Rng; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; @@ -312,6 +313,16 @@ where ) -> Result<(), Error> { let values = items.iter().map(|(_k, v)| v.clone()).collect::>(); + for to in nodes.iter() { + self.data.metrics.sync_items_sent.add( + values.len() as u64, + &[ + KeyValue::new("table_name", F::TABLE_NAME), + KeyValue::new("to", format!("{:?}", to)), + ], + ); + } + self.system .rpc .try_call_many( @@ -500,6 +511,14 @@ where .map(|x| Arc::new(ByteBuf::from(x))) .collect::>(); + self.data.metrics.sync_items_sent.add( + values.len() as u64, + &[ + KeyValue::new("table_name", F::TABLE_NAME), + KeyValue::new("to", format!("{:?}", who)), + ], + ); + let rpc_resp = self .system .rpc @@ -527,7 +546,7 @@ where F: TableSchema + 'static, R: TableReplication + 'static, { - async fn handle(self: &Arc, message: &SyncRpc, _from: NodeID) -> Result { + async fn handle(self: &Arc, message: &SyncRpc, from: NodeID) -> Result { match message { SyncRpc::RootCkHash(range, h) => { let (_root_ck_key, root_ck) = self.get_root_ck(*range)?; @@ -539,6 +558,17 @@ where Ok(SyncRpc::Node(k.clone(), node)) } SyncRpc::Items(items) => { + self.data.metrics.sync_items_received.add( + items.len() as u64, + &[ + KeyValue::new("table_name", F::TABLE_NAME), + KeyValue::new( + "from", + format!("{:?}", Uuid::try_from(from.as_ref()).unwrap()), + ), + ], + ); + self.data.update_many(items)?; Ok(SyncRpc::Ok) } diff --git a/src/table/table.rs b/src/table/table.rs index 01789c11..3ac3bc5b 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, SystemTime}; use async_trait::async_trait; use futures::stream::*; @@ -81,6 +81,8 @@ where } pub async fn insert(&self, e: &F::E) -> Result<(), Error> { + let request_start = SystemTime::now(); + let hash = e.partition_key().hash(); let who = self.data.replication.write_nodes(&hash); //eprintln!("insert who: {:?}", who); @@ -99,10 +101,18 @@ where .with_timeout(TABLE_RPC_TIMEOUT), ) .await?; + + self.data.metrics.put_request_counter.add(1); + self.data + .metrics + .put_request_duration + .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); Ok(()) } pub async fn insert_many(&self, entries: &[F::E]) -> Result<(), Error> { + let request_start = SystemTime::now(); + let mut call_list: HashMap<_, Vec<_>> = HashMap::new(); for entry in entries.iter() { @@ -140,6 +150,12 @@ where if errors.len() > self.data.replication.max_write_errors() { Err(Error::Message("Too many errors".into())) } else { + self.data.metrics.put_request_counter.add(1); + self.data + .metrics + .put_request_duration + .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); + Ok(()) } } @@ -149,6 +165,8 @@ where partition_key: &F::P, sort_key: &F::S, ) -> Result, Error> { + let request_start = SystemTime::now(); + let hash = partition_key.hash(); let who = self.data.replication.read_nodes(&hash); //eprintln!("get who: {:?}", who); @@ -198,6 +216,12 @@ where .spawn_cancellable(async move { self2.repair_on_read(&who[..], ent2).await }); } } + + self.data.metrics.get_request_counter.add(1); + self.data + .metrics + .get_request_duration + .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); Ok(ret) } @@ -208,6 +232,8 @@ where filter: Option, limit: usize, ) -> Result, Error> { + let request_start = SystemTime::now(); + let hash = partition_key.hash(); let who = self.data.replication.read_nodes(&hash); @@ -265,6 +291,12 @@ where .take(limit) .map(|(_k, v)| v.take().unwrap()) .collect::>(); + + self.data.metrics.get_request_counter.add(1); + self.data + .metrics + .get_request_duration + .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); Ok(ret_vec) } -- 2.43.0 From b6561f6e1bcb6a8de13a186405a480e356df89d8 Mon Sep 17 00:00:00 2001 From: Maximilien Date: Thu, 17 Feb 2022 09:38:57 +0100 Subject: [PATCH 04/19] Add docker-compose for traces & metrics --- script/telemetry/elastic/.env | 3 + script/telemetry/elastic/apm-config.yaml | 10 +++ script/telemetry/elastic/docker-compose.yml | 69 +++++++++++++++++++ .../provisioning/datasources/elastic.yaml | 20 ++++++ script/telemetry/elastic/otel-config.yaml | 47 +++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 script/telemetry/elastic/.env create mode 100644 script/telemetry/elastic/apm-config.yaml create mode 100644 script/telemetry/elastic/docker-compose.yml create mode 100644 script/telemetry/elastic/grafana/provisioning/datasources/elastic.yaml create mode 100644 script/telemetry/elastic/otel-config.yaml diff --git a/script/telemetry/elastic/.env b/script/telemetry/elastic/.env new file mode 100644 index 00000000..c4d06423 --- /dev/null +++ b/script/telemetry/elastic/.env @@ -0,0 +1,3 @@ +COMPOSE_PROJECT_NAME=telemetry +OTEL_COLLECT_TAG=0.44.0 +ELASTIC_BUNDLE_TAG=7.17.0 diff --git a/script/telemetry/elastic/apm-config.yaml b/script/telemetry/elastic/apm-config.yaml new file mode 100644 index 00000000..1c1e6452 --- /dev/null +++ b/script/telemetry/elastic/apm-config.yaml @@ -0,0 +1,10 @@ +apm-server: + # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket. + host: "0.0.0.0:8200" +#-------------------------- Elasticsearch output -------------------------- +output.elasticsearch: + # Array of hosts to connect to. + # Scheme and port can be left out and will be set to the default (`http` and `9200`). + # In case you specify and additional path, the scheme is required: `http://localhost:9200/path`. + # IPv6 addresses should always be defined as: `https://[2001:db8::1]:9200`. + hosts: ["localhost:9200"] diff --git a/script/telemetry/elastic/docker-compose.yml b/script/telemetry/elastic/docker-compose.yml new file mode 100644 index 00000000..c93ab68e --- /dev/null +++ b/script/telemetry/elastic/docker-compose.yml @@ -0,0 +1,69 @@ +version: "2" +services: + + otel: + image: otel/opentelemetry-collector-contrib:${OTEL_COLLECT_TAG} + command: [ "--config=/etc/otel-config.yaml" ] + volumes: + - ./otel-config.yaml:/etc/otel-config.yaml + network_mode: "host" + + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_BUNDLE_TAG} + container_name: elastic + environment: + - "node.name=elastic" + - "http.port=9200" + - "cluster.name=es-docker-cluster" + - "discovery.type=single-node" + - "bootstrap.memory_lock=true" + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: 65536 + volumes: + - "es_data:/usr/share/elasticsearch/data" + network_mode: "host" + + # kibana instance and collectors + # see https://www.elastic.co/guide/en/elastic-stack-get-started/current/get-started-docker.html + kibana: + image: docker.elastic.co/kibana/kibana:${ELASTIC_BUNDLE_TAG} + container_name: kibana + environment: + SERVER_NAME: "kibana.local" + # ELASTICSEARCH_URL: "http://localhost:9700" + ELASTICSEARCH_HOSTS: "http://localhost:9200" + depends_on: [ 'elastic' ] + network_mode: "host" + + apm: + image: docker.elastic.co/apm/apm-server:${ELASTIC_BUNDLE_TAG} + container_name: apm + volumes: + - "./apm-config.yaml:/usr/share/apm-server/apm-server.yml:ro" + depends_on: [ 'elastic' ] + network_mode: "host" + + grafana: + # see https://grafana.com/docs/grafana/latest/installation/docker/ + image: "grafana/grafana:8.3.5" + container_name: grafana + # restart: unless-stopped + environment: + - "GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-simple-json-datasource,grafana-piechart-panel,grafana-worldmap-panel,grafana-polystat-panel" + network_mode: "host" + volumes: + # chown 472:472 if needed + - grafana:/var/lib/grafana + - ./grafana/provisioning/:/etc/grafana/provisioning/ + +volumes: + es_data: + driver: local + grafana: + driver: local + metricbeat: + driver: local diff --git a/script/telemetry/elastic/grafana/provisioning/datasources/elastic.yaml b/script/telemetry/elastic/grafana/provisioning/datasources/elastic.yaml new file mode 100644 index 00000000..e46e7ac9 --- /dev/null +++ b/script/telemetry/elastic/grafana/provisioning/datasources/elastic.yaml @@ -0,0 +1,20 @@ +apiVersion: 1 + +datasources: + - name: DS_ELASTICSEARCH + type: elasticsearch + access: proxy + url: http://elastic:9700 + password: '' + user: '' + database: metricbeat-* + basicAuth: false + isDefault: true + jsonData: + esVersion: 70 + logLevelField: '' + logMessageField: '' + maxConcurrentShardRequests: 5 + timeField: "@timestamp" + timeInterval: 10s + readOnly: false diff --git a/script/telemetry/elastic/otel-config.yaml b/script/telemetry/elastic/otel-config.yaml new file mode 100644 index 00000000..cd119f0a --- /dev/null +++ b/script/telemetry/elastic/otel-config.yaml @@ -0,0 +1,47 @@ +receivers: + # Data sources: metrics, traces + otlp: + protocols: + grpc: + endpoint: ":4317" + http: + endpoint: ":55681" + # Data sources: metrics + prometheus: + config: + scrape_configs: + - job_name: "garage" + scrape_interval: 5s + static_configs: + - targets: ["localhost:3909"] + +exporters: + logging: + logLevel: info + # see https://www.elastic.co/guide/en/apm/get-started/current/open-telemetry-elastic.html#open-telemetry-collector + otlp/elastic: + endpoint: "localhost:8200" + tls: + insecure: true + +processors: + batch: + +extensions: + health_check: + pprof: + endpoint: :1888 + zpages: + endpoint: :55679 + +service: + extensions: [pprof, zpages, health_check] + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [logging, otlp/elastic] + metrics: + receivers: [otlp, prometheus] + processors: [batch] + exporters: [logging, otlp/elastic] -- 2.43.0 From 8c2fb0c066af7f68fdcfcdec96fa030af059bf63 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 17 Feb 2022 23:28:23 +0100 Subject: [PATCH 05/19] Add tracing integration with opentelemetry --- Cargo.lock | 317 +++++++++++++++++++++++++-- Cargo.nix | 435 +++++++++++++++++++++++++++++++++++-- src/admin/Cargo.toml | 7 +- src/admin/lib.rs | 3 +- src/admin/metrics.rs | 54 +++-- src/admin/tracing_setup.rs | 37 ++++ src/api/Cargo.toml | 7 +- src/api/api_server.rs | 24 +- src/api/lib.rs | 2 +- src/api/s3_router.rs | 4 +- src/garage/Cargo.toml | 2 +- src/garage/cli/init.rs | 2 - src/garage/main.rs | 2 +- src/garage/server.rs | 6 + src/model/Cargo.toml | 2 +- src/model/block.rs | 8 +- src/model/lib.rs | 2 +- src/rpc/Cargo.toml | 2 +- src/rpc/consul.rs | 4 +- src/rpc/lib.rs | 2 +- src/rpc/rpc_helper.rs | 280 +++++++++++++----------- src/table/Cargo.toml | 2 +- src/table/data.rs | 1 - src/table/lib.rs | 2 +- src/table/merkle.rs | 1 - src/util/Cargo.toml | 2 +- src/util/config.rs | 2 + src/util/lib.rs | 2 +- src/web/Cargo.toml | 2 +- src/web/lib.rs | 2 +- 30 files changed, 1020 insertions(+), 198 deletions(-) create mode 100644 src/admin/tracing_setup.rs diff --git a/Cargo.lock b/Cargo.lock index 2cebfd33..1b7b3174 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "anyhow" +version = "1.0.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" + [[package]] name = "arc-swap" version = "1.5.0" @@ -23,6 +29,27 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "async-stream" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.52" @@ -628,6 +655,12 @@ dependencies = [ "instant", ] +[[package]] +name = "fixedbitset" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" + [[package]] name = "fnv" version = "1.0.7" @@ -788,7 +821,6 @@ dependencies = [ "http", "hyper", "kuska-sodiumoxide", - "log", "netapp", "pretty_env_logger", "rand", @@ -800,6 +832,7 @@ dependencies = [ "structopt", "tokio", "toml", + "tracing", ] [[package]] @@ -810,12 +843,14 @@ dependencies = [ "futures-util", "garage_model 0.6.0", "garage_util 0.6.0", + "hex", "http", "hyper", - "log", "opentelemetry", + "opentelemetry-otlp", "opentelemetry-prometheus", "prometheus", + "tracing", ] [[package]] @@ -840,10 +875,10 @@ dependencies = [ "httpdate 0.3.2", "hyper", "idna", - "log", "md-5", "multer", "nom", + "opentelemetry", "percent-encoding", "pin-project 1.0.10", "quick-xml", @@ -852,7 +887,10 @@ dependencies = [ "serde_bytes", "serde_json", "sha2", + "strum", + "strum_macros", "tokio", + "tracing", "url", ] @@ -895,7 +933,6 @@ dependencies = [ "garage_table 0.6.0", "garage_util 0.6.0", "hex", - "log", "netapp", "opentelemetry", "rand", @@ -904,6 +941,7 @@ dependencies = [ "serde_bytes", "sled", "tokio", + "tracing", "zstd", ] @@ -950,7 +988,6 @@ dependencies = [ "k8s-openapi", "kube", "kuska-sodiumoxide", - "log", "netapp", "openssl", "opentelemetry", @@ -963,6 +1000,7 @@ dependencies = [ "serde_json", "tokio", "tokio-stream", + "tracing", ] [[package]] @@ -998,7 +1036,6 @@ dependencies = [ "garage_rpc 0.6.0", "garage_util 0.6.0", "hexdump", - "log", "opentelemetry", "rand", "rmp-serde 0.15.5", @@ -1006,6 +1043,7 @@ dependencies = [ "serde_bytes", "sled", "tokio", + "tracing", ] [[package]] @@ -1047,7 +1085,6 @@ dependencies = [ "hyper", "k8s-openapi", "kube", - "log", "netapp", "rand", "rmp-serde 0.15.5", @@ -1057,6 +1094,7 @@ dependencies = [ "sled", "tokio", "toml", + "tracing", "xxhash-rust", ] @@ -1072,8 +1110,8 @@ dependencies = [ "garage_util 0.6.0", "http", "hyper", - "log", "percent-encoding", + "tracing", ] [[package]] @@ -1191,7 +1229,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e40283dadb02f3af778878be1d717b17b4e4ab92e1d935ab03a730b0542905f2" dependencies = [ "arrayvec", - "itertools", + "itertools 0.4.19", ] [[package]] @@ -1376,6 +1414,15 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f" +[[package]] +name = "itertools" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.1" @@ -1701,6 +1748,12 @@ dependencies = [ "version_check", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "native-tls" version = "0.2.8" @@ -1873,6 +1926,26 @@ dependencies = [ "pin-project 1.0.10", "rand", "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a" +dependencies = [ + "async-trait", + "futures", + "futures-util", + "http", + "opentelemetry", + "prost", + "thiserror", + "tokio", + "tonic", + "tonic-build", ] [[package]] @@ -1903,7 +1976,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.5", +] + +[[package]] +name = "parking_lot" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.1", ] [[package]] @@ -1920,6 +2003,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-sys", +] + [[package]] name = "pem" version = "0.8.3" @@ -1937,6 +2033,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "petgraph" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "pin-project" version = "0.4.29" @@ -2148,11 +2254,64 @@ dependencies = [ "fnv", "lazy_static", "memchr", - "parking_lot", + "parking_lot 0.11.2", "protobuf", "thiserror", ] +[[package]] +name = "prost" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001" +dependencies = [ + "bytes 1.1.0", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5" +dependencies = [ + "bytes 1.1.0", + "heck", + "itertools 0.10.3", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe" +dependencies = [ + "anyhow", + "itertools 0.10.3", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a" +dependencies = [ + "bytes 1.1.0", + "prost", +] + [[package]] name = "protobuf" version = "2.27.1" @@ -2564,7 +2723,7 @@ dependencies = [ "fxhash", "libc", "log", - "parking_lot", + "parking_lot 0.11.2", ] [[package]] @@ -2627,8 +2786,8 @@ dependencies = [ "bitflags", "cfg_aliases", "libc", - "parking_lot", - "parking_lot_core", + "parking_lot 0.11.2", + "parking_lot_core 0.8.5", "static_init_macro", "winapi", ] @@ -2676,6 +2835,25 @@ dependencies = [ "syn", ] +[[package]] +name = "strum" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" + +[[package]] +name = "strum_macros" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subtle" version = "2.4.1" @@ -2805,6 +2983,7 @@ dependencies = [ "mio", "num_cpus", "once_cell", + "parking_lot 0.12.0", "pin-project-lite", "signal-hook-registry", "socket2", @@ -2904,6 +3083,49 @@ dependencies = [ "serde", ] +[[package]] +name = "tonic" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a" +dependencies = [ + "async-stream", + "async-trait", + "base64", + "bytes 1.1.0", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project 1.0.10", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util 0.6.9", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757" +dependencies = [ + "proc-macro2", + "prost-build", + "quote", + "syn", +] + [[package]] name = "tower" version = "0.4.12" @@ -2912,8 +3134,11 @@ checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project 1.0.10", "pin-project-lite", + "rand", + "slab", "tokio", "tokio-util 0.7.0", "tower-layer", @@ -2984,6 +3209,16 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project 1.0.10", + "tracing", +] + [[package]] name = "treediff" version = "3.0.2" @@ -3175,6 +3410,17 @@ dependencies = [ "untrusted", ] +[[package]] +name = "which" +version = "4.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a5a7e487e921cf220206864a94a89b6c6905bfc19f1057fa26a4cb360e5c1d2" +dependencies = [ + "either", + "lazy_static", + "libc", +] + [[package]] name = "winapi" version = "0.3.9" @@ -3206,6 +3452,49 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6" +dependencies = [ + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5" + +[[package]] +name = "windows_i686_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615" + +[[package]] +name = "windows_i686_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316" + [[package]] name = "xmlparser" version = "0.13.3" diff --git a/Cargo.nix b/Cargo.nix index 3426ee4e..e417c86b 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -67,6 +67,17 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".anyhow."1.0.56" = overridableMkRustCrate (profileName: rec { + name = "anyhow"; + version = "1.0.56"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" = overridableMkRustCrate (profileName: rec { name = "arc-swap"; version = "1.5.0"; @@ -85,6 +96,29 @@ in ]; }); + "registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.3" = overridableMkRustCrate (profileName: rec { + name = "async-stream"; + version = "0.3.3"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"; }; + dependencies = { + async_stream_impl = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-stream-impl."0.3.3" { profileName = "__noProfile"; }; + futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".async-stream-impl."0.3.3" = overridableMkRustCrate (profileName: rec { + name = "async-stream-impl"; + version = "0.3.3"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"; }; + dependencies = { + proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; }; + quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.15" { inherit profileName; }; + syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.86" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" = overridableMkRustCrate (profileName: rec { name = "async-trait"; version = "0.1.52"; @@ -564,7 +598,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"; }; dependencies = { - ${ if hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; }; }); @@ -890,6 +924,13 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".fixedbitset."0.4.1" = overridableMkRustCrate (profileName: rec { + name = "fixedbitset"; + version = "0.4.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" = overridableMkRustCrate (profileName: rec { name = "fnv"; version = "1.0.7"; @@ -1121,7 +1162,6 @@ in git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; pretty_env_logger = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; @@ -1132,6 +1172,7 @@ in structopt = rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; toml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; devDependencies = { aws_sdk_s3 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aws-sdk-s3."0.6.0" { inherit profileName; }; @@ -1151,12 +1192,14 @@ in futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; + opentelemetry_otlp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }; opentelemetry_prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }; prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; }); @@ -1184,10 +1227,10 @@ in httpdate = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."0.3.2" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; idna = rustPackages."registry+https://github.com/rust-lang/crates.io-index".idna."0.2.3" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; md5 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".md-5."0.9.1" { inherit profileName; }; multer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".multer."2.0.2" { inherit profileName; }; nom = rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.0" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; quick_xml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.21.0" { inherit profileName; }; @@ -1196,7 +1239,10 @@ in serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.79" { inherit profileName; }; sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; }; + strum = rustPackages."registry+https://github.com/rust-lang/crates.io-index".strum."0.23.0" { inherit profileName; }; + strum_macros = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".strum_macros."0.23.1" { profileName = "__noProfile"; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; url = rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.2.2" { inherit profileName; }; }; }); @@ -1243,7 +1289,6 @@ in garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; @@ -1252,6 +1297,7 @@ in serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; zstd = rustPackages."registry+https://github.com/rust-lang/crates.io-index".zstd."0.9.2+zstd.1.5.1" { inherit profileName; }; }; }); @@ -1302,7 +1348,6 @@ in k8s_openapi = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; }; kube = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; openssl = rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl."0.10.38" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; @@ -1315,6 +1360,7 @@ in serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.79" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; }); @@ -1354,7 +1400,6 @@ in garage_rpc = rustPackages."unknown".garage_rpc."0.6.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; @@ -1362,6 +1407,7 @@ in serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; }); @@ -1407,7 +1453,6 @@ in hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; k8s_openapi = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; }; kube = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; @@ -1417,6 +1462,7 @@ in sled = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sled."0.34.7" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; toml = rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.5.8" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; xxhash_rust = rustPackages."registry+https://github.com/rust-lang/crates.io-index".xxhash-rust."0.8.3" { inherit profileName; }; }; }); @@ -1435,8 +1481,8 @@ in garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; - log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; }); @@ -1671,11 +1717,12 @@ in features = builtins.concatLists [ [ "client" ] [ "default" ] - (lib.optional (rootFeatures' ? "garage") "h2") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "full") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "h2") [ "http1" ] - (lib.optional (rootFeatures' ? "garage") "http2") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "http2") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "runtime") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "server") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "server") [ "socket2" ] [ "stream" ] [ "tcp" ] @@ -1685,7 +1732,7 @@ in futures_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - ${ if rootFeatures' ? "garage" then "h2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "h2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; }; httparse = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.6.0" { inherit profileName; }; @@ -1777,7 +1824,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage") "std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "std") ]; dependencies = { hashbrown = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.11.2" { inherit profileName; }; @@ -1818,6 +1865,21 @@ in src = fetchCratesIo { inherit name version; sha256 = "c4a9b56eb56058f43dc66e58f40a214b2ccbc9f3df51861b63d51dec7b65bc3f"; }; }); + "registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.3" = overridableMkRustCrate (profileName: rec { + name = "itertools"; + version = "0.10.3"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3"; }; + features = builtins.concatLists [ + [ "default" ] + [ "use_alloc" ] + [ "use_std" ] + ]; + dependencies = { + either = rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.6.1" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" = overridableMkRustCrate (profileName: rec { name = "itoa"; version = "1.0.1"; @@ -2239,7 +2301,7 @@ in [ "os-poll" ] ]; dependencies = { - ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; ${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; }; ${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; }; @@ -2282,6 +2344,13 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".multimap."0.8.3" = overridableMkRustCrate (profileName: rec { + name = "multimap"; + version = "0.8.3"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".native-tls."0.2.8" = overridableMkRustCrate (profileName: rec { name = "native-tls"; version = "0.2.8"; @@ -2506,6 +2575,9 @@ in (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "percent-encoding") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "pin-project") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rand") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "rt-tokio") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "tokio") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "tokio-stream") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "trace") ]; dependencies = { @@ -2522,6 +2594,36 @@ in ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "pin_project" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "thiserror" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "tokio_stream" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" = overridableMkRustCrate (profileName: rec { + name = "opentelemetry-otlp"; + version = "0.10.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9d1a6ca9de4c8b00aa7f1a153bd76cb263287155cec642680d79d98706f3d28a"; }; + features = builtins.concatLists [ + [ "default" ] + [ "prost" ] + [ "tokio" ] + [ "tonic" ] + [ "tonic-build" ] + ]; + dependencies = { + async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }; + futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; + futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; + prost = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }; + thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; + tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + tonic = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tonic."0.6.2" { inherit profileName; }; + }; + buildDependencies = { + tonic_build = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tonic-build."0.6.2" { profileName = "__noProfile"; }; }; }); @@ -2566,6 +2668,20 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.0" = overridableMkRustCrate (profileName: rec { + name = "parking_lot"; + version = "0.12.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "87f5ec2493a61ac0506c0f4199f99070cbe83857b0337006a30f3e6719b8ef58"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + lock_api = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lock_api."0.4.6" { inherit profileName; }; + parking_lot_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.1" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" = overridableMkRustCrate (profileName: rec { name = "parking_lot_core"; version = "0.8.5"; @@ -2581,6 +2697,20 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.9.1" = overridableMkRustCrate (profileName: rec { + name = "parking_lot_core"; + version = "0.9.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "28141e0cc4143da2443301914478dc976a61ffdb3f043058310c70df2fed8954"; }; + dependencies = { + cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; + ${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "redox" then "syscall" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".redox_syscall."0.2.11" { inherit profileName; }; + smallvec = rustPackages."registry+https://github.com/rust-lang/crates.io-index".smallvec."1.8.0" { inherit profileName; }; + ${ if hostPlatform.isWindows then "windows_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.32.0" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".pem."0.8.3" = overridableMkRustCrate (profileName: rec { name = "pem"; version = "0.8.3"; @@ -2600,6 +2730,17 @@ in src = fetchCratesIo { inherit name version; sha256 = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"; }; }); + "registry+https://github.com/rust-lang/crates.io-index".petgraph."0.6.0" = overridableMkRustCrate (profileName: rec { + name = "petgraph"; + version = "0.6.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "4a13a2fa9d0b63e5f22328828741e523766fff0ee9e779316902290dff3f824f"; }; + dependencies = { + fixedbitset = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fixedbitset."0.4.1" { inherit profileName; }; + indexmap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.8.0" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".pin-project."0.4.29" = overridableMkRustCrate (profileName: rec { name = "pin-project"; version = "0.4.29"; @@ -2864,6 +3005,70 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" = overridableMkRustCrate (profileName: rec { + name = "prost"; + version = "0.9.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "444879275cb4fd84958b1a1d5420d15e6fcf7c235fe47f053c9c2a80aceb6001"; }; + features = builtins.concatLists [ + [ "default" ] + [ "prost-derive" ] + [ "std" ] + ]; + dependencies = { + bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + prost_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" { profileName = "__noProfile"; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".prost-build."0.9.0" = overridableMkRustCrate (profileName: rec { + name = "prost-build"; + version = "0.9.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "62941722fb675d463659e49c4f3fe1fe792ff24fe5bbaa9c08cd3b98a1c354f5"; }; + dependencies = { + bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + heck = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; }; + itertools = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.3" { inherit profileName; }; + lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + multimap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".multimap."0.8.3" { inherit profileName; }; + petgraph = rustPackages."registry+https://github.com/rust-lang/crates.io-index".petgraph."0.6.0" { inherit profileName; }; + prost = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }; + prost_types = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-types."0.9.0" { inherit profileName; }; + regex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex."1.5.5" { inherit profileName; }; + tempfile = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tempfile."3.3.0" { inherit profileName; }; + }; + buildDependencies = { + which = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".which."4.2.4" { profileName = "__noProfile"; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" = overridableMkRustCrate (profileName: rec { + name = "prost-derive"; + version = "0.9.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "f9cc1a3263e07e0bf68e96268f37665207b49560d98739662cdfaae215c720fe"; }; + dependencies = { + anyhow = rustPackages."registry+https://github.com/rust-lang/crates.io-index".anyhow."1.0.56" { inherit profileName; }; + itertools = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itertools."0.10.3" { inherit profileName; }; + proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; }; + quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.15" { inherit profileName; }; + syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.86" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".prost-types."0.9.0" = overridableMkRustCrate (profileName: rec { + name = "prost-types"; + version = "0.9.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "534b7a0e836e3c482d2693070f982e39e7611da9695d4d1f5a4b186b51faef0a"; }; + dependencies = { + bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + prost = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".protobuf."2.27.1" = overridableMkRustCrate (profileName: rec { name = "protobuf"; version = "2.27.1"; @@ -2919,6 +3124,7 @@ in [ "getrandom" ] [ "libc" ] [ "rand_chacha" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "small_rng") [ "std" ] [ "std_rng" ] ]; @@ -3051,7 +3257,7 @@ in ]; dependencies = { ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; }; untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; }; @@ -3588,6 +3794,31 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".strum."0.23.0" = overridableMkRustCrate (profileName: rec { + name = "strum"; + version = "0.23.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"; }; + features = builtins.concatLists [ + [ "default" ] + [ "std" ] + ]; + }); + + "registry+https://github.com/rust-lang/crates.io-index".strum_macros."0.23.1" = overridableMkRustCrate (profileName: rec { + name = "strum_macros"; + version = "0.23.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38"; }; + dependencies = { + heck = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; }; + proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; }; + quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.15" { inherit profileName; }; + rustversion = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustversion."1.0.6" { profileName = "__noProfile"; }; + syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.86" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".subtle."2.4.1" = overridableMkRustCrate (profileName: rec { name = "subtle"; version = "2.4.1"; @@ -3754,6 +3985,8 @@ in [ "bytes" ] [ "default" ] [ "fs" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "full") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "io-std") [ "io-util" ] [ "libc" ] [ "macros" ] @@ -3762,6 +3995,8 @@ in [ "net" ] [ "num_cpus" ] [ "once_cell" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "parking_lot") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "process") [ "rt" ] [ "rt-multi-thread" ] [ "signal" ] @@ -3779,6 +4014,7 @@ in mio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mio."0.8.1" { inherit profileName; }; num_cpus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.13.1" { inherit profileName; }; once_cell = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.0" { inherit profileName; }; pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; ${ if hostPlatform.isUnix then "signal_hook_registry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.0" { inherit profileName; }; socket2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.4" { inherit profileName; }; @@ -3904,6 +4140,71 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".tonic."0.6.2" = overridableMkRustCrate (profileName: rec { + name = "tonic"; + version = "0.6.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "ff08f4649d10a70ffa3522ca559031285d8e421d727ac85c60825761818f5d0a"; }; + features = builtins.concatLists [ + [ "async-trait" ] + [ "codegen" ] + [ "default" ] + [ "h2" ] + [ "hyper" ] + [ "hyper-timeout" ] + [ "prost" ] + [ "prost-derive" ] + [ "prost1" ] + [ "tokio" ] + [ "tower" ] + [ "tracing-futures" ] + [ "transport" ] + ]; + dependencies = { + async_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".async-stream."0.3.3" { inherit profileName; }; + async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }; + base64 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.13.0" { inherit profileName; }; + bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; + futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; + futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + h2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; }; + http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; + http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; }; + hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; + hyper_timeout = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper-timeout."0.4.1" { inherit profileName; }; + percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; + pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; + prost1 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost."0.9.0" { inherit profileName; }; + prost_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-derive."0.9.0" { profileName = "__noProfile"; }; + tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }; + tokio_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.9" { inherit profileName; }; + tower = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower."0.4.12" { inherit profileName; }; + tower_layer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.1" { inherit profileName; }; + tower_service = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.1" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; + tracing_futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-futures."0.2.5" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".tonic-build."0.6.2" = overridableMkRustCrate (profileName: rec { + name = "tonic-build"; + version = "0.6.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9403f1bafde247186684b230dc6f38b5cd514584e8bec1dd32514be4745fa757"; }; + features = builtins.concatLists [ + [ "prost" ] + [ "prost-build" ] + [ "transport" ] + ]; + dependencies = { + proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; }; + prost_build = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prost-build."0.9.0" { inherit profileName; }; + quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.15" { inherit profileName; }; + syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.86" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".tower."0.4.12" = overridableMkRustCrate (profileName: rec { name = "tower"; version = "0.4.12"; @@ -3911,14 +4212,24 @@ in src = fetchCratesIo { inherit name version; sha256 = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e"; }; features = builtins.concatLists [ [ "__common" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "balance") [ "buffer" ] [ "default" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "discover") [ "futures-core" ] [ "futures-util" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "indexmap") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "limit") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "load") [ "log" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "make") [ "pin-project" ] [ "pin-project-lite" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "rand") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "ready-cache") (lib.optional (rootFeatures' ? "garage") "retry") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "slab") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "timeout") [ "tokio" ] [ "tokio-util" ] [ "tracing" ] @@ -3927,8 +4238,11 @@ in dependencies = { futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "indexmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.8.0" { inherit profileName; }; pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "slab" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; tokio_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.0" { inherit profileName; }; tower_layer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.1" { inherit profileName; }; @@ -3987,6 +4301,7 @@ in [ "attributes" ] [ "default" ] [ "log" ] + (lib.optional (rootFeatures' ? "garage") "log-always") [ "std" ] [ "tracing-attributes" ] ]; @@ -4025,6 +4340,23 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".tracing-futures."0.2.5" = overridableMkRustCrate (profileName: rec { + name = "tracing-futures"; + version = "0.2.5"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"; }; + features = builtins.concatLists [ + [ "default" ] + [ "pin-project" ] + [ "std" ] + [ "std-future" ] + ]; + dependencies = { + pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; + tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".treediff."3.0.2" = overridableMkRustCrate (profileName: rec { name = "treediff"; version = "3.0.2"; @@ -4289,6 +4621,18 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".which."4.2.4" = overridableMkRustCrate (profileName: rec { + name = "which"; + version = "4.2.4"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "2a5a7e487e921cf220206864a94a89b6c6905bfc19f1057fa26a4cb360e5c1d2"; }; + dependencies = { + either = rustPackages."registry+https://github.com/rust-lang/crates.io-index".either."1.6.1" { inherit profileName; }; + ${ if hostPlatform.isWindows then "lazy_static" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; + libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" = overridableMkRustCrate (profileName: rec { name = "winapi"; version = "0.3.9"; @@ -4325,6 +4669,7 @@ in [ "std" ] [ "synchapi" ] [ "sysinfoapi" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "threadpoollegacyapiset") [ "timezoneapi" ] [ "winbase" ] [ "wincon" ] @@ -4369,6 +4714,64 @@ in src = fetchCratesIo { inherit name version; sha256 = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"; }; }); + "registry+https://github.com/rust-lang/crates.io-index".windows-sys."0.32.0" = overridableMkRustCrate (profileName: rec { + name = "windows-sys"; + version = "0.32.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "3df6e476185f92a12c072be4a189a0210dcdcf512a1891d6dff9edb874deadc6"; }; + features = builtins.concatLists [ + [ "Win32" ] + [ "Win32_Foundation" ] + [ "Win32_System" ] + [ "Win32_System_LibraryLoader" ] + [ "Win32_System_SystemServices" ] + [ "Win32_System_WindowsProgramming" ] + [ "default" ] + ]; + dependencies = { + ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" = overridableMkRustCrate (profileName: rec { + name = "windows_aarch64_msvc"; + version = "0.32.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "d8e92753b1c443191654ec532f14c199742964a061be25d77d7a96f09db20bf5"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" = overridableMkRustCrate (profileName: rec { + name = "windows_i686_gnu"; + version = "0.32.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "6a711c68811799e017b6038e0922cb27a5e2f43a2ddb609fe0b6f3eeda9de615"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" = overridableMkRustCrate (profileName: rec { + name = "windows_i686_msvc"; + version = "0.32.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "146c11bb1a02615db74680b32a68e2d61f553cc24c4eb5b4ca10311740e44172"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" = overridableMkRustCrate (profileName: rec { + name = "windows_x86_64_gnu"; + version = "0.32.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "c912b12f7454c6620635bbff3450962753834be2a594819bd5e945af18ec64bc"; }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" = overridableMkRustCrate (profileName: rec { + name = "windows_x86_64_msvc"; + version = "0.32.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "504a2476202769977a040c6364301a3f65d0cc9e3fb08600b2bda150a0488316"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".xmlparser."0.13.3" = overridableMkRustCrate (profileName: rec { name = "xmlparser"; version = "0.13.3"; diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 6f646869..b6bf2b3b 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -16,12 +16,15 @@ path = "lib.rs" garage_model = { version = "0.6.0", path = "../model" } garage_util = { version = "0.6.0", path = "../util" } +hex = "0.4" + futures = "0.3" futures-util = "0.3" http = "0.2" hyper = "0.14" -log = "0.4" +tracing = "0.1.30" -opentelemetry = "0.17" +opentelemetry = { version = "0.17", features = [ "rt-tokio" ] } opentelemetry-prometheus = "0.10" +opentelemetry-otlp = "0.10" prometheus = "0.13" diff --git a/src/admin/lib.rs b/src/admin/lib.rs index f1e8ddd7..b5b0775b 100644 --- a/src/admin/lib.rs +++ b/src/admin/lib.rs @@ -1,5 +1,6 @@ //! Crate for handling the admin and metric HTTP APIs #[macro_use] -extern crate log; +extern crate tracing; pub mod metrics; +pub mod tracing_setup; diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs index 44fd4cb2..e12373ab 100644 --- a/src/admin/metrics.rs +++ b/src/admin/metrics.rs @@ -1,20 +1,26 @@ -use hyper::{ - header::CONTENT_TYPE, - service::{make_service_fn, service_fn}, - Body, Method, Request, Response, Server, -}; -use opentelemetry::{ - global, - metrics::{BoundCounter, BoundValueRecorder}, -}; -use opentelemetry_prometheus::PrometheusExporter; -use prometheus::{Encoder, TextEncoder}; use std::convert::Infallible; use std::sync::Arc; use std::time::SystemTime; use futures::future::*; +use hyper::{ + header::CONTENT_TYPE, + service::{make_service_fn, service_fn}, + Body, Method, Request, Response, Server, +}; + +use opentelemetry::{ + global, + metrics::{BoundCounter, BoundValueRecorder}, + trace::{FutureExt, TraceContextExt, Tracer}, + Context, +}; +use opentelemetry_prometheus::PrometheusExporter; + +use prometheus::{Encoder, TextEncoder}; + use garage_model::garage::Garage; +use garage_util::data::*; use garage_util::error::Error as GarageError; // serve_req on metric endpoint @@ -22,7 +28,7 @@ async fn serve_req( req: Request, admin_server: Arc, ) -> Result, hyper::Error> { - println!("Receiving request at path {}", req.uri()); + info!("Receiving request at path {}", req.uri()); let request_start = SystemTime::now(); admin_server.metrics.http_counter.add(1); @@ -31,7 +37,12 @@ async fn serve_req( (&Method::GET, "/metrics") => { let mut buffer = vec![]; let encoder = TextEncoder::new(); - let metric_families = admin_server.exporter.registry().gather(); + + let tracer = opentelemetry::global::tracer("garage"); + let metric_families = tracer.in_span("admin/gather_metrics", |_| { + admin_server.exporter.registry().gather() + }); + encoder.encode(&metric_families, &mut buffer).unwrap(); admin_server .metrics @@ -112,7 +123,22 @@ impl AdminServer { // `service_fn` is a helper to convert a function that // returns a Response into a `Service`. async move { - Ok::<_, Infallible>(service_fn(move |req| serve_req(req, admin_server.clone()))) + Ok::<_, Infallible>(service_fn(move |req| { + let tracer = opentelemetry::global::tracer("garage"); + let uuid = gen_uuid(); + let span = tracer + .span_builder("admin/request") + .with_trace_id( + opentelemetry::trace::TraceId::from_hex(&hex::encode( + &uuid.as_slice()[..16], + )) + .unwrap(), + ) + .start(&tracer); + + serve_req(req, admin_server.clone()) + .with_context(Context::current_with_span(span)) + })) } }); diff --git a/src/admin/tracing_setup.rs b/src/admin/tracing_setup.rs new file mode 100644 index 00000000..83fa5622 --- /dev/null +++ b/src/admin/tracing_setup.rs @@ -0,0 +1,37 @@ +use std::time::Duration; + +use opentelemetry::sdk::{ + trace::{self, IdGenerator, Sampler}, + Resource, +}; +use opentelemetry::KeyValue; +use opentelemetry_otlp::WithExportConfig; + +use garage_util::data::*; +use garage_util::error::*; + +pub fn init_tracing(export_to: &str, node_id: Uuid) -> Result<(), Error> { + let node_id = hex::encode(&node_id.as_slice()[..8]); + + opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter( + opentelemetry_otlp::new_exporter() + .tonic() + .with_endpoint(export_to) + .with_timeout(Duration::from_secs(3)), + ) + .with_trace_config( + trace::config() + .with_id_generator(IdGenerator::default()) + .with_sampler(Sampler::TraceIdRatioBased(0.01f64)) + .with_resource(Resource::new(vec![ + KeyValue::new("service.name", "garage"), + KeyValue::new("service.instance.id", node_id), + ])), + ) + .install_batch(opentelemetry::runtime::Tokio) + .ok_or_message("Unable to initialize tracing")?; + + Ok(()) +} diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index cc9635bb..968e8aab 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -26,11 +26,14 @@ err-derive = "0.3" hex = "0.4" hmac = "0.10" idna = "0.2" -log = "0.4" +tracing = "0.1.30" md-5 = "0.9" nom = "7.1" sha2 = "0.9" +strum = "0.23" +strum_macros = "0.23" + futures = "0.3" futures-util = "0.3" pin-project = "1.0" @@ -49,3 +52,5 @@ serde_bytes = "0.11" serde_json = "1.0" quick-xml = { version = "0.21", features = [ "serialize" ] } url = "2.1" + +opentelemetry = "0.17" diff --git a/src/api/api_server.rs b/src/api/api_server.rs index 322ea298..15b00dde 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -7,6 +7,11 @@ use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Request, Response, Server}; +use opentelemetry::{ + trace::{FutureExt, TraceContextExt, Tracer}, + Context, +}; + use garage_util::data::*; use garage_util::error::Error as GarageError; @@ -43,7 +48,20 @@ pub async fn run_api_server( async move { Ok::<_, GarageError>(service_fn(move |req: Request| { let garage = garage.clone(); - handler(garage, req, client_addr) + + let tracer = opentelemetry::global::tracer("garage"); + let uuid = gen_uuid(); + let span = tracer + .span_builder("S3 API call (unknown)") + .with_trace_id( + opentelemetry::trace::TraceId::from_hex(&hex::encode( + &uuid.as_slice()[..16], + )) + .unwrap(), + ) + .start(&tracer); + + handler(garage, req, client_addr).with_context(Context::current_with_span(span)) })) } }); @@ -111,6 +129,10 @@ async fn handler_inner(garage: Arc, req: Request) -> Result(format!("S3 API {}", endpoint)); + // Some endpoints are processed early, before we even check for an API key if let Endpoint::PostObject = endpoint { return handle_post_object(garage, req, bucket_name.unwrap()).await; diff --git a/src/api/lib.rs b/src/api/lib.rs index 071cd7a6..f865325e 100644 --- a/src/api/lib.rs +++ b/src/api/lib.rs @@ -1,6 +1,6 @@ //! Crate for serving a S3 compatible API #[macro_use] -extern crate log; +extern crate tracing; pub mod error; pub use error::Error; diff --git a/src/api/s3_router.rs b/src/api/s3_router.rs index 95a7eceb..c325805d 100644 --- a/src/api/s3_router.rs +++ b/src/api/s3_router.rs @@ -5,6 +5,8 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; +use strum_macros::Display; + /// This macro is used to generate very repetitive match {} blocks in this module /// It is _not_ made to be used anywhere else macro_rules! s3_match { @@ -133,7 +135,7 @@ s3_match! {@func /// query parameters). Parameters it may receive by header are left out, however headers are /// considered when required to determine between one endpoint or another (for CopyObject and /// UploadObject, for instance). -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Display)] pub enum Endpoint { AbortMultipartUpload { key: String, diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 22e0f0f0..351fa62f 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -32,7 +32,7 @@ garage_admin = { version = "0.6.0", path = "../admin" } bytes = "1.0" git-version = "0.3.4" hex = "0.4" -log = "0.4" +tracing = { version = "0.1.30", features = ["log-always"] } pretty_env_logger = "0.4" rand = "0.8" async-trait = "0.1.7" diff --git a/src/garage/cli/init.rs b/src/garage/cli/init.rs index 80283c8f..511b53a6 100644 --- a/src/garage/cli/init.rs +++ b/src/garage/cli/init.rs @@ -1,7 +1,5 @@ use std::path::PathBuf; -use log::warn; - use garage_util::error::*; pub const READ_KEY_ERROR: &str = "Unable to read node key. It will be generated by your garage node the first time is it launched. Ensure that your garage node is currently running. (The node key is supposed to be stored in your metadata directory.)"; diff --git a/src/garage/main.rs b/src/garage/main.rs index 870455e1..7de7740f 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -2,7 +2,7 @@ //! Garage CLI, used to interact with a running Garage instance, and to launch a Garage instance #[macro_use] -extern crate log; +extern crate tracing; mod admin; mod cli; diff --git a/src/garage/server.rs b/src/garage/server.rs index cd32d708..6ef36273 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -7,6 +7,7 @@ use garage_util::config::*; use garage_util::error::Error; use garage_admin::metrics::*; +use garage_admin::tracing_setup::*; use garage_api::run_api_server; use garage_model::garage::Garage; use garage_web::run_web_server; @@ -45,6 +46,11 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Initializing Garage main data store..."); let garage = Garage::new(config.clone(), db, background); + info!("Initialize tracing..."); + if let Some(export_to) = config.admin_api.otlp_export_traces_to { + init_tracing(&export_to, garage.system.id)?; + } + let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone())); info!("Create admin RPC handler..."); diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 10a4c838..4465af1c 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -23,7 +23,7 @@ async-trait = "0.1.7" arc-swap = "1.0" err-derive = "0.3" hex = "0.4" -log = "0.4" +tracing = "0.1.30" rand = "0.8" zstd = { version = "0.9", default-features = false } diff --git a/src/model/block.rs b/src/model/block.rs index 9e939c24..ddda5e57 100644 --- a/src/model/block.rs +++ b/src/model/block.rs @@ -5,14 +5,16 @@ use std::time::{Duration, SystemTime}; use arc_swap::ArcSwapOption; use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use zstd::stream::{decode_all as zstd_decode, Encoder}; + use futures::future::*; use futures::select; -use opentelemetry::KeyValue; -use serde::{Deserialize, Serialize}; use tokio::fs; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::{watch, Mutex, Notify}; -use zstd::stream::{decode_all as zstd_decode, Encoder}; + +use opentelemetry::KeyValue; use garage_util::data::*; use garage_util::error::*; diff --git a/src/model/lib.rs b/src/model/lib.rs index c8677603..6da86fc6 100644 --- a/src/model/lib.rs +++ b/src/model/lib.rs @@ -1,5 +1,5 @@ #[macro_use] -extern crate log; +extern crate tracing; pub mod permission; diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 57b61a08..00f609c9 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -20,7 +20,7 @@ arc-swap = "1.0" bytes = "1.0" gethostname = "0.2" hex = "0.4" -log = "0.4" +tracing = "0.1.30" rand = "0.8" sodiumoxide = { version = "0.2.5-0", package = "kuska-sodiumoxide" } diff --git a/src/rpc/consul.rs b/src/rpc/consul.rs index 82bf99ba..e70288dd 100644 --- a/src/rpc/consul.rs +++ b/src/rpc/consul.rs @@ -139,10 +139,10 @@ pub async fn publish_consul_service( let resp = client.request(req).await?; debug!("Response of advertising to Consul: {:?}", resp); let resp_code = resp.status(); + let resp_bytes = &hyper::body::to_bytes(resp.into_body()).await?; debug!( "{}", - std::str::from_utf8(&hyper::body::to_bytes(resp.into_body()).await?) - .unwrap_or("") + std::str::from_utf8(resp_bytes).unwrap_or("") ); if resp_code != StatusCode::OK { diff --git a/src/rpc/lib.rs b/src/rpc/lib.rs index 736513f4..b8fb9772 100644 --- a/src/rpc/lib.rs +++ b/src/rpc/lib.rs @@ -1,7 +1,7 @@ //! Crate containing rpc related functions and types used in Garage #[macro_use] -extern crate log; +extern crate tracing; mod consul; mod kubernetes; diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index 0d722e43..97716b18 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -6,10 +6,15 @@ use futures::future::join_all; use futures::stream::futures_unordered::FuturesUnordered; use futures::stream::StreamExt; use futures_util::future::FutureExt; -use opentelemetry::KeyValue; use tokio::select; use tokio::sync::{watch, Semaphore}; +use opentelemetry::KeyValue; +use opentelemetry::{ + trace::{FutureExt as OtelFutureExt, Span, TraceContextExt, Tracer}, + Context, +}; + pub use netapp::endpoint::{Endpoint, EndpointHandler, Message as Rpc}; use netapp::peering::fullmesh::FullMeshPeeringStrategy; pub use netapp::proto::*; @@ -147,9 +152,17 @@ impl RpcHelper { self.0.metrics.rpc_counter.add(1, &metric_tags); let rpc_start_time = SystemTime::now(); + let tracer = opentelemetry::global::tracer("garage"); + let mut span = tracer.start(format!("RPC {}", endpoint.path())); + span.set_attribute(KeyValue::new("to", format!("{:?}", to))); + let node_id = to.into(); + let rpc_call = endpoint + .call(&node_id, &msg, strat.rs_priority) + .with_context(Context::current_with_span(span)); + select! { - res = endpoint.call(&node_id, &msg, strat.rs_priority) => { + res = rpc_call => { drop(permit); if res.is_err() { @@ -228,149 +241,164 @@ impl RpcHelper { where M: Rpc> + 'static, H: EndpointHandler + 'static, - S: Send, + S: Send + 'static, { - let msg = Arc::new(msg); - - // Build future for each request - // They are not started now: they are added below in a FuturesUnordered - // object that will take care of polling them (see below) - let requests = to.iter().cloned().map(|to| { - let self2 = self.clone(); - let msg = msg.clone(); - let endpoint2 = endpoint.clone(); - (to, async move { - self2.call_arc(&endpoint2, to, msg, strategy).await - }) - }); let quorum = strategy.rs_quorum.unwrap_or(to.len()); - // Vectors in which success results and errors will be collected - let mut successes = vec![]; - let mut errors = vec![]; + let tracer = opentelemetry::global::tracer("garage"); + let mut span = tracer.start(format!("RPC {} to {:?}", endpoint.path(), to)); + span.set_attribute(KeyValue::new("to", format!("{:?}", to))); + span.set_attribute(KeyValue::new("quorum", quorum as i64)); - if strategy.rs_interrupt_after_quorum { - // Case 1: once quorum is reached, other requests don't matter. - // What we do here is only send the required number of requests - // to reach a quorum, priorizing nodes with the lowest latency. - // When there are errors, we start new requests to compensate. + async { + let msg = Arc::new(msg); - // Retrieve some status variables that we will use to sort requests - let peer_list = self.0.fullmesh.get_peer_list(); - let ring: Arc = self.0.ring.borrow().clone(); - let our_zone = match ring.layout.node_role(&self.0.our_node_id) { - Some(pc) => &pc.zone, - None => "", - }; - - // Augment requests with some information used to sort them. - // The tuples are as follows: - // (is another node?, is another zone?, latency, node ID, request future) - // We store all of these tuples in a vec that we can sort. - // By sorting this vec, we priorize ourself, then nodes in the same zone, - // and within a same zone we priorize nodes with the lowest latency. - let mut requests = requests - .map(|(to, fut)| { - let peer_zone = match ring.layout.node_role(&to) { - Some(pc) => &pc.zone, - None => "", - }; - let peer_avg_ping = peer_list - .iter() - .find(|x| x.id.as_ref() == to.as_slice()) - .map(|pi| pi.avg_ping) - .flatten() - .unwrap_or_else(|| Duration::from_secs(1)); - ( - to != self.0.our_node_id, - peer_zone != our_zone, - peer_avg_ping, - to, - fut, - ) + // Build future for each request + // They are not started now: they are added below in a FuturesUnordered + // object that will take care of polling them (see below) + let requests = to.iter().cloned().map(|to| { + let self2 = self.clone(); + let msg = msg.clone(); + let endpoint2 = endpoint.clone(); + (to, async move { + self2.call_arc(&endpoint2, to, msg, strategy).await }) - .collect::>(); + }); - // Sort requests by (priorize ourself, priorize same zone, priorize low latency) - requests - .sort_by_key(|(diffnode, diffzone, ping, _to, _fut)| (*diffnode, *diffzone, *ping)); + // Vectors in which success results and errors will be collected + let mut successes = vec![]; + let mut errors = vec![]; - // Make an iterator to take requests in their sorted order - let mut requests = requests.into_iter(); + if strategy.rs_interrupt_after_quorum { + // Case 1: once quorum is reached, other requests don't matter. + // What we do here is only send the required number of requests + // to reach a quorum, priorizing nodes with the lowest latency. + // When there are errors, we start new requests to compensate. - // resp_stream will contain all of the requests that are currently in flight. - // (for the moment none, they will be added in the loop below) - let mut resp_stream = FuturesUnordered::new(); + // Retrieve some status variables that we will use to sort requests + let peer_list = self.0.fullmesh.get_peer_list(); + let ring: Arc = self.0.ring.borrow().clone(); + let our_zone = match ring.layout.node_role(&self.0.our_node_id) { + Some(pc) => &pc.zone, + None => "", + }; - // Do some requests and collect results - 'request_loop: while successes.len() < quorum { - // If the current set of requests that are running is not enough to possibly - // reach quorum, start some new requests. - while successes.len() + resp_stream.len() < quorum { - if let Some((_, _, _, _to, fut)) = requests.next() { - resp_stream.push(fut); - } else { - // If we have no request to add, we know that we won't ever - // reach quorum: bail out now. - break 'request_loop; - } - } - assert!(!resp_stream.is_empty()); // because of loop invariants + // Augment requests with some information used to sort them. + // The tuples are as follows: + // (is another node?, is another zone?, latency, node ID, request future) + // We store all of these tuples in a vec that we can sort. + // By sorting this vec, we priorize ourself, then nodes in the same zone, + // and within a same zone we priorize nodes with the lowest latency. + let mut requests = requests + .map(|(to, fut)| { + let peer_zone = match ring.layout.node_role(&to) { + Some(pc) => &pc.zone, + None => "", + }; + let peer_avg_ping = peer_list + .iter() + .find(|x| x.id.as_ref() == to.as_slice()) + .map(|pi| pi.avg_ping) + .flatten() + .unwrap_or_else(|| Duration::from_secs(1)); + ( + to != self.0.our_node_id, + peer_zone != our_zone, + peer_avg_ping, + to, + fut, + ) + }) + .collect::>(); - // Wait for one request to terminate - match resp_stream.next().await.unwrap() { - Ok(msg) => { - successes.push(msg); - } - Err(e) => { - errors.push(e); - } - } - } - } else { - // Case 2: all of the requests need to be sent in all cases, - // and need to terminate. (this is the case for writes that - // must be spread to n nodes) - // Just start all the requests in parallel and return as soon - // as the quorum is reached. - let mut resp_stream = requests - .map(|(_, fut)| fut) - .collect::>(); + // Sort requests by (priorize ourself, priorize same zone, priorize low latency) + requests.sort_by_key(|(diffnode, diffzone, ping, _to, _fut)| { + (*diffnode, *diffzone, *ping) + }); - while let Some(resp) = resp_stream.next().await { - match resp { - Ok(msg) => { - successes.push(msg); - if successes.len() >= quorum { - break; + // Make an iterator to take requests in their sorted order + let mut requests = requests.into_iter(); + + // resp_stream will contain all of the requests that are currently in flight. + // (for the moment none, they will be added in the loop below) + let mut resp_stream = FuturesUnordered::new(); + + // Do some requests and collect results + 'request_loop: while successes.len() < quorum { + // If the current set of requests that are running is not enough to possibly + // reach quorum, start some new requests. + while successes.len() + resp_stream.len() < quorum { + if let Some((_, _, _, req_to, fut)) = requests.next() { + let tracer = opentelemetry::global::tracer("garage"); + let span = tracer.start(format!("RPC to {:?}", req_to)); + resp_stream.push(tokio::spawn( + fut.with_context(Context::current_with_span(span)), + )); + } else { + // If we have no request to add, we know that we won't ever + // reach quorum: bail out now. + break 'request_loop; } } - Err(e) => { - errors.push(e); + assert!(!resp_stream.is_empty()); // because of loop invariants + + // Wait for one request to terminate + match resp_stream.next().await.unwrap().unwrap() { + Ok(msg) => { + successes.push(msg); + } + Err(e) => { + errors.push(e); + } } } + } else { + // Case 2: all of the requests need to be sent in all cases, + // and need to terminate. (this is the case for writes that + // must be spread to n nodes) + // Just start all the requests in parallel and return as soon + // as the quorum is reached. + let mut resp_stream = requests + .map(|(_, fut)| fut) + .collect::>(); + + while let Some(resp) = resp_stream.next().await { + match resp { + Ok(msg) => { + successes.push(msg); + if successes.len() >= quorum { + break; + } + } + Err(e) => { + errors.push(e); + } + } + } + + if !resp_stream.is_empty() { + // Continue remaining requests in background. + // Continue the remaining requests immediately using tokio::spawn + // but enqueue a task in the background runner + // to ensure that the process won't exit until the requests are done + // (if we had just enqueued the resp_stream.collect directly in the background runner, + // the requests might have been put on hold in the background runner's queue, + // in which case they might timeout or otherwise fail) + let wait_finished_fut = tokio::spawn(async move { + resp_stream.collect::>>().await; + }); + self.0.background.spawn(wait_finished_fut.map(|_| Ok(()))); + } } - if !resp_stream.is_empty() { - // Continue remaining requests in background. - // Continue the remaining requests immediately using tokio::spawn - // but enqueue a task in the background runner - // to ensure that the process won't exit until the requests are done - // (if we had just enqueued the resp_stream.collect directly in the background runner, - // the requests might have been put on hold in the background runner's queue, - // in which case they might timeout or otherwise fail) - let wait_finished_fut = tokio::spawn(async move { - resp_stream.collect::>>().await; - }); - self.0.background.spawn(wait_finished_fut.map(|_| Ok(()))); + if successes.len() >= quorum { + Ok(successes) + } else { + let errors = errors.iter().map(|e| format!("{}", e)).collect::>(); + Err(Error::Quorum(quorum, successes.len(), to.len(), errors)) } } - - if successes.len() >= quorum { - Ok(successes) - } else { - let errors = errors.iter().map(|e| format!("{}", e)).collect::>(); - Err(Error::Quorum(quorum, successes.len(), to.len(), errors)) - } + .with_context(Context::current_with_span(span)) + .await } } diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index f6f9974d..bc09ca7c 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -22,7 +22,7 @@ opentelemetry = "0.17" async-trait = "0.1.7" bytes = "1.0" hexdump = "0.1" -log = "0.4" +tracing = "0.1.30" rand = "0.8" sled = "0.34" diff --git a/src/table/data.rs b/src/table/data.rs index 684afdcd..4293e395 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -1,7 +1,6 @@ use core::borrow::Borrow; use std::sync::Arc; -use log::warn; use serde_bytes::ByteBuf; use sled::Transactional; use tokio::sync::Notify; diff --git a/src/table/lib.rs b/src/table/lib.rs index e0920357..b0153e9a 100644 --- a/src/table/lib.rs +++ b/src/table/lib.rs @@ -2,7 +2,7 @@ #![allow(clippy::comparison_chain)] #[macro_use] -extern crate log; +extern crate tracing; mod metrics; pub mod schema; diff --git a/src/table/merkle.rs b/src/table/merkle.rs index 5ec6ab61..93bf7e47 100644 --- a/src/table/merkle.rs +++ b/src/table/merkle.rs @@ -3,7 +3,6 @@ use std::time::Duration; use futures::select; use futures_util::future::*; -use log::{debug, warn}; use serde::{Deserialize, Serialize}; use sled::transaction::{ ConflictableTransactionError, ConflictableTransactionResult, TransactionalTree, diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 4e3c8c25..3cc2031d 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -18,7 +18,7 @@ blake2 = "0.9" err-derive = "0.3" xxhash-rust = { version = "0.8", default-features = false, features = ["xxh3"] } hex = "0.4" -log = "0.4" +tracing = "0.1.30" rand = "0.8" sha2 = "0.9" diff --git a/src/util/config.rs b/src/util/config.rs index 2d15748f..e763fcd5 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -104,6 +104,8 @@ pub struct WebConfig { pub struct AdminConfig { /// Address and port to bind for admin API serving pub bind_addr: SocketAddr, + /// OTLP server to where to export traces + pub otlp_export_traces_to: Option, } fn default_sled_cache_capacity() -> u64 { diff --git a/src/util/lib.rs b/src/util/lib.rs index 64874095..6856f656 100644 --- a/src/util/lib.rs +++ b/src/util/lib.rs @@ -1,7 +1,7 @@ //! Crate containing common functions and types used in Garage #[macro_use] -extern crate log; +extern crate tracing; pub mod background; pub mod config; diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 54211f5d..269e29a5 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -20,7 +20,7 @@ garage_util = { version = "0.6.0", path = "../util" } garage_table = { version = "0.6.0", path = "../table" } err-derive = "0.3" -log = "0.4" +tracing = "0.1.30" percent-encoding = "2.1.0" futures = "0.3" diff --git a/src/web/lib.rs b/src/web/lib.rs index c06492a3..9b7c8573 100644 --- a/src/web/lib.rs +++ b/src/web/lib.rs @@ -1,6 +1,6 @@ //! Crate for handling web serving of s3 bucket #[macro_use] -extern crate log; +extern crate tracing; mod error; pub use error::Error; -- 2.43.0 From bb04d94fa92740eebeee513030faf136bfd26da7 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 18 Feb 2022 20:39:55 +0100 Subject: [PATCH 06/19] Update to Netapp 0.4 which supports distributed tracing --- Cargo.lock | 37 ++++++++++++++++++++++++++++------- Cargo.nix | 45 ++++++++++++++++++++++++++++++++++--------- src/api/api_server.rs | 10 +++++++--- src/garage/Cargo.toml | 5 +++-- src/garage/main.rs | 4 ++-- src/model/Cargo.toml | 3 ++- src/model/block.rs | 24 +++++++++++++++++++++-- src/rpc/Cargo.toml | 4 +++- src/rpc/rpc_helper.rs | 11 ++--------- src/rpc/system.rs | 16 +++++++-------- src/util/Cargo.toml | 3 ++- 11 files changed, 117 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b7b3174..b5c896c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -821,7 +821,7 @@ dependencies = [ "http", "hyper", "kuska-sodiumoxide", - "netapp", + "netapp 0.4.0", "pretty_env_logger", "rand", "rmp-serde 0.15.5", @@ -909,7 +909,7 @@ dependencies = [ "garage_util 0.5.1", "hex", "log", - "netapp", + "netapp 0.3.1", "rand", "rmp-serde 0.15.5", "serde", @@ -933,7 +933,7 @@ dependencies = [ "garage_table 0.6.0", "garage_util 0.6.0", "hex", - "netapp", + "netapp 0.4.0", "opentelemetry", "rand", "rmp-serde 0.15.5", @@ -962,7 +962,7 @@ dependencies = [ "hyper", "kuska-sodiumoxide", "log", - "netapp", + "netapp 0.3.1", "rand", "rmp-serde 0.15.5", "serde", @@ -988,7 +988,7 @@ dependencies = [ "k8s-openapi", "kube", "kuska-sodiumoxide", - "netapp", + "netapp 0.4.0", "openssl", "opentelemetry", "pnet", @@ -1060,7 +1060,7 @@ dependencies = [ "http", "hyper", "log", - "netapp", + "netapp 0.3.1", "rand", "rmp-serde 0.15.5", "serde", @@ -1085,7 +1085,7 @@ dependencies = [ "hyper", "k8s-openapi", "kube", - "netapp", + "netapp 0.4.0", "rand", "rmp-serde 0.15.5", "serde", @@ -1794,6 +1794,29 @@ dependencies = [ "tokio-util 0.6.9", ] +[[package]] +name = "netapp" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22c545a13b0c47b47e8052b35c4884dbe33c9ea62607371b0f4f1b0490cafd38" +dependencies = [ + "arc-swap", + "async-trait", + "bytes 0.6.0", + "cfg-if", + "err-derive 0.2.4", + "futures", + "hex", + "kuska-handshake", + "kuska-sodiumoxide", + "log", + "rmp-serde 0.14.4", + "serde", + "tokio", + "tokio-stream", + "tokio-util 0.6.9", +] + [[package]] name = "nom" version = "7.1.0" diff --git a/Cargo.nix b/Cargo.nix index e417c86b..c4d3d36f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1162,7 +1162,7 @@ in git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; - netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; pretty_env_logger = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; @@ -1289,7 +1289,7 @@ in garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; - netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; @@ -1348,7 +1348,7 @@ in k8s_openapi = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; }; kube = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; - netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; openssl = rustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl."0.10.38" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; pnet = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet."0.28.0" { inherit profileName; }; @@ -1453,7 +1453,7 @@ in hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; k8s_openapi = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; }; kube = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; }; - netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.3.1" { inherit profileName; }; + netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; @@ -2396,6 +2396,33 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" = overridableMkRustCrate (profileName: rec { + name = "netapp"; + version = "0.4.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "22c545a13b0c47b47e8052b35c4884dbe33c9ea62607371b0f4f1b0490cafd38"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + arc_swap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }; + async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }; + bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."0.6.0" { inherit profileName; }; + cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; + err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.2.4" { profileName = "__noProfile"; }; + futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; + hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; + kuska_handshake = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" { inherit profileName; }; + sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; + log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.14.4" { inherit profileName; }; + serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; + tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + tokio_stream = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }; + tokio_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.6.9" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".nom."7.1.0" = overridableMkRustCrate (profileName: rec { name = "nom"; version = "7.1.0"; @@ -3734,7 +3761,7 @@ in ]; dependencies = { bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" { inherit profileName; }; static_init_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; }; @@ -4729,11 +4756,11 @@ in [ "default" ] ]; dependencies = { - ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; }; }); diff --git a/src/api/api_server.rs b/src/api/api_server.rs index 15b00dde..00d582d1 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -9,7 +9,7 @@ use hyper::{Body, Method, Request, Response, Server}; use opentelemetry::{ trace::{FutureExt, TraceContextExt, Tracer}, - Context, + Context, KeyValue, }; use garage_util::data::*; @@ -50,15 +50,19 @@ pub async fn run_api_server( let garage = garage.clone(); let tracer = opentelemetry::global::tracer("garage"); - let uuid = gen_uuid(); + let trace_id = gen_uuid(); let span = tracer .span_builder("S3 API call (unknown)") .with_trace_id( opentelemetry::trace::TraceId::from_hex(&hex::encode( - &uuid.as_slice()[..16], + &trace_id.as_slice()[..16], )) .unwrap(), ) + .with_attributes(vec![ + KeyValue::new("method", format!("{}", req.method())), + KeyValue::new("uri", req.uri().path().to_string()), + ]) .start(&tracer); handler(garage, req, client_addr).with_context(Context::current_with_span(span)) diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 351fa62f..ff0666a6 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -50,8 +50,9 @@ futures = "0.3" futures-util = "0.3" tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } -#netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } -netapp = "0.3.0" +#netapp = { version = "0.3.2", git = "https://git.deuxfleurs.fr/lx/netapp" } +#netapp = { version = "0.4", path = "../../../netapp" } +netapp = "0.4" [dev-dependencies] aws-sdk-s3 = "0.6" diff --git a/src/garage/main.rs b/src/garage/main.rs index 7de7740f..08ec912b 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -55,7 +55,7 @@ struct Opt { #[tokio::main] async fn main() { if std::env::var("RUST_LOG").is_err() { - std::env::set_var("RUST_LOG", "garage=info") + std::env::set_var("RUST_LOG", "netapp=info,garage=info") } pretty_env_logger::init(); sodiumoxide::init().expect("Unable to init sodiumoxide"); @@ -106,7 +106,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { // Generate a temporary keypair for our RPC client let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair(); - let netapp = NetApp::new(network_key, sk); + let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, sk); // Find and parse the address of the target host let (id, addr) = if let Some(h) = opt.rpc_host { diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 4465af1c..8083445e 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -39,4 +39,5 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi opentelemetry = "0.17" #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } -netapp = "0.3.0" +#netapp = { version = "0.4", path = "../../../netapp" } +netapp = "0.4" diff --git a/src/model/block.rs b/src/model/block.rs index ddda5e57..3799c6aa 100644 --- a/src/model/block.rs +++ b/src/model/block.rs @@ -14,7 +14,10 @@ use tokio::fs; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::{watch, Mutex, Notify}; -use opentelemetry::KeyValue; +use opentelemetry::{ + trace::{FutureExt as OtelFutureExt, TraceContextExt, Tracer}, + Context, KeyValue, +}; use garage_util::data::*; use garage_util::error::*; @@ -554,7 +557,24 @@ impl BlockManager { let start_time = SystemTime::now(); let hash = Hash::try_from(&hash_bytes[..]).unwrap(); - let res = self.resync_block(&hash).await; + + let tracer = opentelemetry::global::tracer("garage"); + let trace_id = gen_uuid(); + let span = tracer + .span_builder("Resync block") + .with_trace_id( + opentelemetry::trace::TraceId::from_hex(&hex::encode( + &trace_id.as_slice()[..16], + )) + .unwrap(), + ) + .with_attributes(vec![KeyValue::new("block", format!("{:?}", hash))]) + .start(&tracer); + + let res = self + .resync_block(&hash) + .with_context(Context::current_with_span(span)) + .await; self.metrics.resync_counter.add(1); self.metrics diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 00f609c9..3ab9e7da 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -46,6 +46,8 @@ tokio-stream = { version = "0.1", features = ["net"] } opentelemetry = "0.17" #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } -netapp = "0.3.1" +#netapp = { version = "0.4", path = "../../../netapp", features = ["telemetry"] } +netapp = "0.4" + hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] } diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index 97716b18..4b4235f1 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -152,14 +152,8 @@ impl RpcHelper { self.0.metrics.rpc_counter.add(1, &metric_tags); let rpc_start_time = SystemTime::now(); - let tracer = opentelemetry::global::tracer("garage"); - let mut span = tracer.start(format!("RPC {}", endpoint.path())); - span.set_attribute(KeyValue::new("to", format!("{:?}", to))); - let node_id = to.into(); - let rpc_call = endpoint - .call(&node_id, &msg, strat.rs_priority) - .with_context(Context::current_with_span(span)); + let rpc_call = endpoint.call(&node_id, msg, strat.rs_priority); select! { res = rpc_call => { @@ -246,7 +240,7 @@ impl RpcHelper { let quorum = strategy.rs_quorum.unwrap_or(to.len()); let tracer = opentelemetry::global::tracer("garage"); - let mut span = tracer.start(format!("RPC {} to {:?}", endpoint.path(), to)); + let mut span = tracer.start(format!("RPC {} to {}", endpoint.path(), to.len())); span.set_attribute(KeyValue::new("to", format!("{:?}", to))); span.set_attribute(KeyValue::new("quorum", quorum as i64)); @@ -329,7 +323,6 @@ impl RpcHelper { // reach quorum, start some new requests. while successes.len() + resp_stream.len() < quorum { if let Some((_, _, _, req_to, fut)) = requests.next() { - let tracer = opentelemetry::global::tracer("garage"); let span = tracer.start(format!("RPC to {:?}", req_to)); resp_stream.push(tokio::spawn( fut.with_context(Context::current_with_span(span)), diff --git a/src/rpc/system.rs b/src/rpc/system.rs index c8fc0ad5..2123a37f 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -38,6 +38,9 @@ const DISCOVERY_INTERVAL: Duration = Duration::from_secs(60); const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10); const PING_TIMEOUT: Duration = Duration::from_secs(2); +/// Version tag used for version check upon Netapp connection +pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650006; // garage 0x0006 + /// RPC endpoint used for calls related to membership pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc"; @@ -194,7 +197,10 @@ impl System { ) -> Arc { let node_key = gen_node_key(&config.metadata_dir).expect("Unable to read or generate node ID"); - info!("Node public key: {}", hex::encode(&node_key.public_key())); + info!( + "Node ID of this node: {}", + hex::encode(&node_key.public_key()[..8]) + ); let persist_cluster_layout = Persister::new(&config.metadata_dir, "cluster_layout"); let persist_peer_list = Persister::new(&config.metadata_dir, "peer_list"); @@ -222,13 +228,7 @@ impl System { let ring = Ring::new(cluster_layout, replication_factor); let (update_ring, ring) = watch::channel(Arc::new(ring)); - if let Some(addr) = config.rpc_public_addr { - println!("{}@{}", hex::encode(&node_key.public_key()), addr); - } else { - println!("{}", hex::encode(&node_key.public_key())); - } - - let netapp = NetApp::new(network_key, node_key); + let netapp = NetApp::new(GARAGE_VERSION_TAG, network_key, node_key); let fullmesh = FullMeshPeeringStrategy::new( netapp.clone(), config.bootstrap_peers.clone(), diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 3cc2031d..76b73f4b 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -34,7 +34,8 @@ futures = "0.3" tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] } #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } -netapp = "0.3.0" +#netapp = { version = "0.4", path = "../../../netapp" } +netapp = "0.4" http = "0.2" hyper = "0.14" -- 2.43.0 From 55d44715998d01563fe830d1dbc35745bc9427bf Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 21 Feb 2022 16:59:37 +0100 Subject: [PATCH 07/19] Remove ... at end of hex IDs --- src/util/data.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/data.rs b/src/util/data.rs index f0744307..7715c2cc 100644 --- a/src/util/data.rs +++ b/src/util/data.rs @@ -22,7 +22,7 @@ impl std::convert::AsRef<[u8]> for FixedBytes32 { impl fmt::Debug for FixedBytes32 { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}…", hex::encode(&self.0[..8])) + write!(f, "{}", hex::encode(&self.0[..8])) } } -- 2.43.0 From f0d0cd9a20979f59db246e6a545ddc1b7bbb20b3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 21 Feb 2022 23:10:13 +0100 Subject: [PATCH 08/19] Remove strum crate dependency; add protobuf nix dependency --- Cargo.lock | 21 --------------------- Cargo.nix | 31 ++----------------------------- shell.nix | 1 + src/api/Cargo.toml | 3 --- src/api/api_server.rs | 2 +- src/api/s3_router.rs | 4 +--- 6 files changed, 5 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b5c896c9..e70c1ea7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -887,8 +887,6 @@ dependencies = [ "serde_bytes", "serde_json", "sha2", - "strum", - "strum_macros", "tokio", "tracing", "url", @@ -2858,25 +2856,6 @@ dependencies = [ "syn", ] -[[package]] -name = "strum" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb" - -[[package]] -name = "strum_macros" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn", -] - [[package]] name = "subtle" version = "2.4.1" diff --git a/Cargo.nix b/Cargo.nix index c4d3d36f..9a31fc8d 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -1239,8 +1239,6 @@ in serde_bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.5" { inherit profileName; }; serde_json = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.79" { inherit profileName; }; sha2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.9.9" { inherit profileName; }; - strum = rustPackages."registry+https://github.com/rust-lang/crates.io-index".strum."0.23.0" { inherit profileName; }; - strum_macros = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".strum_macros."0.23.1" { profileName = "__noProfile"; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; url = rustPackages."registry+https://github.com/rust-lang/crates.io-index".url."2.2.2" { inherit profileName; }; @@ -3761,7 +3759,7 @@ in ]; dependencies = { bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" { inherit profileName; }; static_init_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; }; @@ -3821,31 +3819,6 @@ in }; }); - "registry+https://github.com/rust-lang/crates.io-index".strum."0.23.0" = overridableMkRustCrate (profileName: rec { - name = "strum"; - version = "0.23.0"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "cae14b91c7d11c9a851d3fbc80a963198998c2a64eec840477fa92d8ce9b70bb"; }; - features = builtins.concatLists [ - [ "default" ] - [ "std" ] - ]; - }); - - "registry+https://github.com/rust-lang/crates.io-index".strum_macros."0.23.1" = overridableMkRustCrate (profileName: rec { - name = "strum_macros"; - version = "0.23.1"; - registry = "registry+https://github.com/rust-lang/crates.io-index"; - src = fetchCratesIo { inherit name version; sha256 = "5bb0dc7ee9c15cea6199cde9a127fa16a4c5819af85395457ad72d68edc85a38"; }; - dependencies = { - heck = rustPackages."registry+https://github.com/rust-lang/crates.io-index".heck."0.3.3" { inherit profileName; }; - proc_macro2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".proc-macro2."1.0.36" { inherit profileName; }; - quote = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quote."1.0.15" { inherit profileName; }; - rustversion = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".rustversion."1.0.6" { profileName = "__noProfile"; }; - syn = rustPackages."registry+https://github.com/rust-lang/crates.io-index".syn."1.0.86" { inherit profileName; }; - }; - }); - "registry+https://github.com/rust-lang/crates.io-index".subtle."2.4.1" = overridableMkRustCrate (profileName: rec { name = "subtle"; version = "2.4.1"; @@ -4760,7 +4733,7 @@ in ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; }; }); diff --git a/shell.nix b/shell.nix index 870a45db..867d7f48 100644 --- a/shell.nix +++ b/shell.nix @@ -78,6 +78,7 @@ function refresh_toolchain { pkgs.clippy pkgs.rustfmt pkgs.perl + pkgs.protobuf cargo2nix.packages.x86_64-linux.cargo2nix ] else []) ++ diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 968e8aab..f39c6d04 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -31,9 +31,6 @@ md-5 = "0.9" nom = "7.1" sha2 = "0.9" -strum = "0.23" -strum_macros = "0.23" - futures = "0.3" futures-util = "0.3" pin-project = "1.0" diff --git a/src/api/api_server.rs b/src/api/api_server.rs index 00d582d1..1bab3aaa 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -135,7 +135,7 @@ async fn handler_inner(garage: Arc, req: Request) -> Result(format!("S3 API {}", endpoint)); + .update_name::(format!("S3 API {}", endpoint.name())); // Some endpoints are processed early, before we even check for an API key if let Endpoint::PostObject = endpoint { diff --git a/src/api/s3_router.rs b/src/api/s3_router.rs index c325805d..95a7eceb 100644 --- a/src/api/s3_router.rs +++ b/src/api/s3_router.rs @@ -5,8 +5,6 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; -use strum_macros::Display; - /// This macro is used to generate very repetitive match {} blocks in this module /// It is _not_ made to be used anywhere else macro_rules! s3_match { @@ -135,7 +133,7 @@ s3_match! {@func /// query parameters). Parameters it may receive by header are left out, however headers are /// considered when required to determine between one endpoint or another (for CopyObject and /// UploadObject, for instance). -#[derive(Debug, Clone, PartialEq, Eq, Display)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum Endpoint { AbortMultipartUpload { key: String, -- 2.43.0 From 818daa5c786813fdf50fecb6022e29b18e509b62 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 22 Feb 2022 13:53:59 +0100 Subject: [PATCH 09/19] Refactor how durations are measured --- Cargo.lock | 3 ++- src/admin/Cargo.toml | 1 - src/admin/metrics.rs | 10 +++---- src/garage/server.rs | 2 +- src/model/block.rs | 33 ++++++++++------------- src/rpc/Cargo.toml | 1 + src/rpc/rpc_helper.rs | 17 ++++-------- src/table/table.rs | 62 ++++++++++++++++++++++++------------------- src/util/Cargo.toml | 2 ++ src/util/lib.rs | 1 + src/util/metrics.rs | 35 ++++++++++++++++++++++++ 11 files changed, 99 insertions(+), 68 deletions(-) create mode 100644 src/util/metrics.rs diff --git a/Cargo.lock b/Cargo.lock index e70c1ea7..2f05701f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -841,7 +841,6 @@ version = "0.6.0" dependencies = [ "futures", "futures-util", - "garage_model 0.6.0", "garage_util 0.6.0", "hex", "http", @@ -979,6 +978,7 @@ dependencies = [ "bytes 1.1.0", "futures", "futures-util", + "garage_admin", "garage_util 0.6.0", "gethostname", "hex", @@ -1084,6 +1084,7 @@ dependencies = [ "k8s-openapi", "kube", "netapp 0.4.0", + "opentelemetry", "rand", "rmp-serde 0.15.5", "serde", diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index b6bf2b3b..01c16c57 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -13,7 +13,6 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_model = { version = "0.6.0", path = "../model" } garage_util = { version = "0.6.0", path = "../util" } hex = "0.4" diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs index e12373ab..64721af2 100644 --- a/src/admin/metrics.rs +++ b/src/admin/metrics.rs @@ -1,6 +1,7 @@ use std::convert::Infallible; use std::sync::Arc; use std::time::SystemTime; +use std::net::SocketAddr; use futures::future::*; use hyper::{ @@ -19,7 +20,6 @@ use opentelemetry_prometheus::PrometheusExporter; use prometheus::{Encoder, TextEncoder}; -use garage_model::garage::Garage; use garage_util::data::*; use garage_util::error::Error as GarageError; @@ -111,7 +111,7 @@ impl AdminServer { /// run execute the admin server on the designated HTTP port and listen for requests pub async fn run( self, - garage: Arc, + bind_addr: SocketAddr, shutdown_signal: impl Future, ) -> Result<(), GarageError> { let admin_server = Arc::new(self); @@ -142,11 +142,9 @@ impl AdminServer { } }); - let addr = &garage.config.admin_api.bind_addr; - - let server = Server::bind(addr).serve(make_svc); + let server = Server::bind(&bind_addr).serve(make_svc); let graceful = server.with_graceful_shutdown(shutdown_signal); - info!("Admin server listening on http://{}", addr); + info!("Admin server listening on http://{}", bind_addr); graceful.await?; Ok(()) diff --git a/src/garage/server.rs b/src/garage/server.rs index 6ef36273..739dedbe 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -70,7 +70,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Configure and run admin web server..."); let admin_server = - tokio::spawn(admin_server_init.run(garage.clone(), wait_from(watch_cancel.clone()))); + tokio::spawn(admin_server_init.run(config.admin_api.bind_addr, wait_from(watch_cancel.clone()))); // Stuff runs diff --git a/src/model/block.rs b/src/model/block.rs index 3799c6aa..058c71fd 100644 --- a/src/model/block.rs +++ b/src/model/block.rs @@ -1,7 +1,7 @@ use std::convert::TryInto; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::{Duration, SystemTime}; +use std::time::{Duration}; use arc_swap::ArcSwapOption; use async_trait::async_trait; @@ -23,6 +23,7 @@ use garage_util::data::*; use garage_util::error::*; use garage_util::time::*; use garage_util::tranquilizer::Tranquilizer; +use garage_util::metrics::RecordDuration; use garage_rpc::system::System; use garage_rpc::*; @@ -391,7 +392,6 @@ impl BlockManager { /// Write a block to disk async fn write_block(&self, hash: &Hash, data: &DataBlock) -> Result { - let request_start = SystemTime::now(); let write_size = data.inner_buffer().len() as u64; let res = self @@ -399,20 +399,26 @@ impl BlockManager { .lock() .await .write_block(hash, data, self) + .bound_record_duration(&self.metrics.block_write_duration) .await?; self.metrics.bytes_written.add(write_size); - self.metrics - .block_write_duration - .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); Ok(res) } /// Read block from disk, verifying it's integrity async fn read_block(&self, hash: &Hash) -> Result { - let request_start = SystemTime::now(); + let data = self.read_block_internal(hash) + .bound_record_duration(&self.metrics.block_read_duration) + .await?; + self.metrics.bytes_read.add(data.inner_buffer().len() as u64); + + Ok(BlockRpc::PutBlock { hash: *hash, data }) + } + + async fn read_block_internal(&self, hash: &Hash) -> Result { let mut path = self.block_path(hash); let compressed = match self.is_block_compressed(hash).await { Ok(c) => c, @@ -449,14 +455,7 @@ impl BlockManager { return Err(Error::CorruptData(*hash)); } - self.metrics - .bytes_read - .add(data.inner_buffer().len() as u64); - self.metrics - .block_read_duration - .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); - - Ok(BlockRpc::PutBlock { hash: *hash, data }) + Ok(data) } /// Check if this node should have a block, but don't actually have it @@ -554,8 +553,6 @@ impl BlockManager { let time_msec = u64::from_be_bytes(time_bytes[0..8].try_into().unwrap()); let now = now_msec(); if now >= time_msec { - let start_time = SystemTime::now(); - let hash = Hash::try_from(&hash_bytes[..]).unwrap(); let tracer = opentelemetry::global::tracer("garage"); @@ -574,12 +571,10 @@ impl BlockManager { let res = self .resync_block(&hash) .with_context(Context::current_with_span(span)) + .bound_record_duration(&self.metrics.resync_duration) .await; self.metrics.resync_counter.add(1); - self.metrics - .resync_duration - .record(start_time.elapsed().map_or(0.0, |d| d.as_secs_f64())); if let Err(e) = &res { self.metrics.resync_error_counter.add(1); diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 3ab9e7da..94d6f682 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -15,6 +15,7 @@ path = "lib.rs" [dependencies] garage_util = { version = "0.6.0", path = "../util" } +garage_admin = { version = "0.6.0", path = "../admin" } arc-swap = "1.0" bytes = "1.0" diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index 4b4235f1..f8bef47f 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -1,6 +1,6 @@ //! Contain structs related to making RPCs use std::sync::Arc; -use std::time::{Duration, SystemTime}; +use std::time::{Duration}; use futures::future::join_all; use futures::stream::futures_unordered::FuturesUnordered; @@ -23,6 +23,7 @@ pub use netapp::{NetApp, NodeID}; use garage_util::background::BackgroundRunner; use garage_util::data::*; use garage_util::error::Error; +use garage_util::metrics::RecordDuration; use crate::metrics::RpcMetrics; use crate::ring::Ring; @@ -133,7 +134,6 @@ impl RpcHelper { M: Rpc>, H: EndpointHandler, { - let queueing_start_time = SystemTime::now(); let metric_tags = [KeyValue::new("endpoint", endpoint.path().to_string())]; let msg_size = rmp_to_vec_all_named(&msg)?.len() as u32; @@ -141,19 +141,14 @@ impl RpcHelper { .0 .request_buffer_semaphore .acquire_many(msg_size) + .record_duration(&self.0.metrics.rpc_queueing_time, &metric_tags) .await?; - self.0.metrics.rpc_queueing_time.record( - queueing_start_time - .elapsed() - .map_or(0.0, |d| d.as_secs_f64()), - &metric_tags, - ); self.0.metrics.rpc_counter.add(1, &metric_tags); - let rpc_start_time = SystemTime::now(); let node_id = to.into(); - let rpc_call = endpoint.call(&node_id, msg, strat.rs_priority); + let rpc_call = endpoint.call(&node_id, msg, strat.rs_priority) + .record_duration(&self.0.metrics.rpc_duration, &metric_tags); select! { res = rpc_call => { @@ -164,8 +159,6 @@ impl RpcHelper { } let res = res?; - self.0.metrics.rpc_duration - .record(rpc_start_time.elapsed().map_or(0.0, |d| d.as_secs_f64()), &metric_tags); if res.is_err() { self.0.metrics.rpc_garage_error_counter.add(1, &metric_tags); } diff --git a/src/table/table.rs b/src/table/table.rs index 3ac3bc5b..9ba243c0 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -1,6 +1,6 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use std::time::{Duration, SystemTime}; +use std::time::Duration; use async_trait::async_trait; use futures::stream::*; @@ -9,6 +9,7 @@ use serde_bytes::ByteBuf; use garage_util::data::*; use garage_util::error::Error; +use garage_util::metrics::RecordDuration; use garage_rpc::system::System; use garage_rpc::*; @@ -81,8 +82,6 @@ where } pub async fn insert(&self, e: &F::E) -> Result<(), Error> { - let request_start = SystemTime::now(); - let hash = e.partition_key().hash(); let who = self.data.replication.write_nodes(&hash); //eprintln!("insert who: {:?}", who); @@ -100,19 +99,22 @@ where .with_quorum(self.data.replication.write_quorum()) .with_timeout(TABLE_RPC_TIMEOUT), ) + .bound_record_duration(&self.data.metrics.put_request_duration) .await?; self.data.metrics.put_request_counter.add(1); - self.data - .metrics - .put_request_duration - .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); Ok(()) } pub async fn insert_many(&self, entries: &[F::E]) -> Result<(), Error> { - let request_start = SystemTime::now(); + self.insert_many_internal(entries) + .bound_record_duration(&self.data.metrics.put_request_duration) + .await?; + self.data.metrics.put_request_counter.add(1); + Ok(()) + } + async fn insert_many_internal(&self, entries: &[F::E]) -> Result<(), Error> { let mut call_list: HashMap<_, Vec<_>> = HashMap::new(); for entry in entries.iter() { @@ -150,12 +152,6 @@ where if errors.len() > self.data.replication.max_write_errors() { Err(Error::Message("Too many errors".into())) } else { - self.data.metrics.put_request_counter.add(1); - self.data - .metrics - .put_request_duration - .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); - Ok(()) } } @@ -165,11 +161,20 @@ where partition_key: &F::P, sort_key: &F::S, ) -> Result, Error> { - let request_start = SystemTime::now(); + let res = self.get_internal(partition_key, sort_key) + .bound_record_duration(&self.data.metrics.get_request_duration) + .await?; + self.data.metrics.get_request_counter.add(1); + Ok(res) + } + async fn get_internal( + self: &Arc, + partition_key: &F::P, + sort_key: &F::S, + ) -> Result, Error> { let hash = partition_key.hash(); let who = self.data.replication.read_nodes(&hash); - //eprintln!("get who: {:?}", who); let rpc = TableRpc::::ReadEntry(partition_key.clone(), sort_key.clone()); let resps = self @@ -217,11 +222,6 @@ where } } - self.data.metrics.get_request_counter.add(1); - self.data - .metrics - .get_request_duration - .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); Ok(ret) } @@ -232,8 +232,20 @@ where filter: Option, limit: usize, ) -> Result, Error> { - let request_start = SystemTime::now(); + let res = self.get_range_internal(partition_key, begin_sort_key, filter, limit) + .bound_record_duration(&self.data.metrics.get_request_duration) + .await?; + self.data.metrics.get_request_counter.add(1); + Ok(res) + } + async fn get_range_internal( + self: &Arc, + partition_key: &F::P, + begin_sort_key: Option, + filter: Option, + limit: usize, + ) -> Result, Error> { let hash = partition_key.hash(); let who = self.data.replication.read_nodes(&hash); @@ -291,12 +303,6 @@ where .take(limit) .map(|(_k, v)| v.take().unwrap()) .collect::>(); - - self.data.metrics.get_request_counter.add(1); - self.data - .metrics - .get_request_duration - .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); Ok(ret_vec) } diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 76b73f4b..e1d516de 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -42,3 +42,5 @@ hyper = "0.14" kube = { version = "0.62", features = ["runtime", "derive"] } k8s-openapi = { version = "0.13", features = ["v1_22"] } + +opentelemetry = "0.17" diff --git a/src/util/lib.rs b/src/util/lib.rs index 6856f656..7ed00034 100644 --- a/src/util/lib.rs +++ b/src/util/lib.rs @@ -8,6 +8,7 @@ pub mod config; pub mod crdt; pub mod data; pub mod error; +pub mod metrics; pub mod persister; pub mod time; pub mod token_bucket; diff --git a/src/util/metrics.rs b/src/util/metrics.rs new file mode 100644 index 00000000..b3b1fc3c --- /dev/null +++ b/src/util/metrics.rs @@ -0,0 +1,35 @@ +use std::time::SystemTime; + +use futures::{future::BoxFuture, Future, FutureExt}; + +use opentelemetry::{KeyValue, metrics::*}; + +pub trait RecordDuration<'a>: 'a { + type Output; + + fn record_duration(self, r: &'a ValueRecorder, attributes: &'a [KeyValue]) -> BoxFuture<'a, Self::Output>; + fn bound_record_duration(self, r: &'a BoundValueRecorder) -> BoxFuture<'a, Self::Output>; +} + +impl<'a, T, O> RecordDuration<'a> for T +where T: Future + Send + 'a { + type Output = O; + + fn record_duration(self, r: &'a ValueRecorder, attributes: &'a [KeyValue]) -> BoxFuture<'a, Self::Output> { + async move { + let request_start = SystemTime::now(); + let res = self.await; + r.record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()), attributes); + res + }.boxed() + } + + fn bound_record_duration(self, r: &'a BoundValueRecorder) -> BoxFuture<'a, Self::Output> { + async move { + let request_start = SystemTime::now(); + let res = self.await; + r.record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); + res + }.boxed() + } +} -- 2.43.0 From 2a5609b292de019085f93a79b7b73f7a8341bf51 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 22 Feb 2022 14:52:41 +0100 Subject: [PATCH 10/19] Add metrics to API endpoint --- src/admin/metrics.rs | 2 +- src/api/api_server.rs | 113 +++++++++++++++++++++++++++++++++++------- src/garage/server.rs | 5 +- src/model/block.rs | 11 ++-- src/rpc/rpc_helper.rs | 7 +-- src/table/table.rs | 6 ++- src/util/metrics.rs | 29 ++++++++--- 7 files changed, 135 insertions(+), 38 deletions(-) diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs index 64721af2..02549fe9 100644 --- a/src/admin/metrics.rs +++ b/src/admin/metrics.rs @@ -1,7 +1,7 @@ use std::convert::Infallible; +use std::net::SocketAddr; use std::sync::Arc; use std::time::SystemTime; -use std::net::SocketAddr; use futures::future::*; use hyper::{ diff --git a/src/api/api_server.rs b/src/api/api_server.rs index 1bab3aaa..8502d9d8 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -8,12 +8,15 @@ use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Method, Request, Response, Server}; use opentelemetry::{ + global, + metrics::{Counter, ValueRecorder}, trace::{FutureExt, TraceContextExt, Tracer}, Context, KeyValue, }; use garage_util::data::*; use garage_util::error::Error as GarageError; +use garage_util::metrics::RecordDuration; use garage_model::garage::Garage; use garage_model::key_table::Key; @@ -35,6 +38,34 @@ use crate::s3_put::*; use crate::s3_router::{Authorization, Endpoint}; use crate::s3_website::*; +struct ApiMetrics { + request_counter: Counter, + error_counter: Counter, + request_duration: ValueRecorder, +} + +impl ApiMetrics { + fn new() -> Self { + let meter = global::meter("garage/api"); + Self { + request_counter: meter + .u64_counter("api.request_counter") + .with_description("Number of API calls to the various S3 API endpoints") + .init(), + error_counter: meter + .u64_counter("api.error_counter") + .with_description( + "Number of API calls to the various S3 API endpoints that resulted in errors", + ) + .init(), + request_duration: meter + .f64_value_recorder("api.request_duration") + .with_description("Duration of API calls to the various S3 API endpoints") + .init(), + } + } +} + /// Run the S3 API server pub async fn run_api_server( garage: Arc, @@ -42,30 +73,19 @@ pub async fn run_api_server( ) -> Result<(), GarageError> { let addr = &garage.config.s3_api.api_bind_addr; + let metrics = Arc::new(ApiMetrics::new()); + let service = make_service_fn(|conn: &AddrStream| { let garage = garage.clone(); + let metrics = metrics.clone(); + let client_addr = conn.remote_addr(); async move { Ok::<_, GarageError>(service_fn(move |req: Request| { let garage = garage.clone(); + let metrics = metrics.clone(); - let tracer = opentelemetry::global::tracer("garage"); - let trace_id = gen_uuid(); - let span = tracer - .span_builder("S3 API call (unknown)") - .with_trace_id( - opentelemetry::trace::TraceId::from_hex(&hex::encode( - &trace_id.as_slice()[..16], - )) - .unwrap(), - ) - .with_attributes(vec![ - KeyValue::new("method", format!("{}", req.method())), - KeyValue::new("uri", req.uri().path().to_string()), - ]) - .start(&tracer); - - handler(garage, req, client_addr).with_context(Context::current_with_span(span)) + handler(garage, metrics, req, client_addr) })) } }); @@ -81,13 +101,33 @@ pub async fn run_api_server( async fn handler( garage: Arc, + metrics: Arc, req: Request, addr: SocketAddr, ) -> Result, GarageError> { let uri = req.uri().clone(); info!("{} {} {}", addr, req.method(), uri); debug!("{:?}", req); - match handler_inner(garage.clone(), req).await { + + let tracer = opentelemetry::global::tracer("garage"); + let trace_id = gen_uuid(); + let span = tracer + .span_builder("S3 API call (unknown)") + .with_trace_id( + opentelemetry::trace::TraceId::from_hex(&hex::encode(&trace_id.as_slice()[..16])) + .unwrap(), + ) + .with_attributes(vec![ + KeyValue::new("method", format!("{}", req.method())), + KeyValue::new("uri", req.uri().path().to_string()), + ]) + .start(&tracer); + + let res = handler_stage2(garage.clone(), metrics, req) + .with_context(Context::current_with_span(span)) + .await; + + match res { Ok(x) => { debug!("{} {:?}", x.status(), x.headers()); Ok(x) @@ -114,7 +154,11 @@ async fn handler( } } -async fn handler_inner(garage: Arc, req: Request) -> Result, Error> { +async fn handler_stage2( + garage: Arc, + metrics: Arc, + req: Request, +) -> Result, Error> { let authority = req .headers() .get(header::HOST) @@ -137,6 +181,37 @@ async fn handler_inner(garage: Arc, req: Request) -> Result(format!("S3 API {}", endpoint.name())); + let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())]; + + let res = handler_stage3(garage, req, endpoint, bucket_name) + .record_duration(&metrics.request_duration, &metrics_tags[..]) + .await; + + metrics.request_counter.add(1, &metrics_tags[..]); + + let status_code = match &res { + Ok(r) => r.status(), + Err(e) => e.http_status_code(), + }; + if status_code.is_client_error() || status_code.is_server_error() { + metrics.error_counter.add( + 1, + &[ + metrics_tags[0].clone(), + KeyValue::new("status_code", status_code.as_str().to_string()), + ], + ); + } + + res +} + +async fn handler_stage3( + garage: Arc, + req: Request, + endpoint: Endpoint, + bucket_name: Option, +) -> Result, Error> { // Some endpoints are processed early, before we even check for an API key if let Endpoint::PostObject = endpoint { return handle_post_object(garage, req, bucket_name.unwrap()).await; diff --git a/src/garage/server.rs b/src/garage/server.rs index 739dedbe..b11f8417 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -69,8 +69,9 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { )); info!("Configure and run admin web server..."); - let admin_server = - tokio::spawn(admin_server_init.run(config.admin_api.bind_addr, wait_from(watch_cancel.clone()))); + let admin_server = tokio::spawn( + admin_server_init.run(config.admin_api.bind_addr, wait_from(watch_cancel.clone())), + ); // Stuff runs diff --git a/src/model/block.rs b/src/model/block.rs index 058c71fd..d97e64a8 100644 --- a/src/model/block.rs +++ b/src/model/block.rs @@ -1,7 +1,7 @@ use std::convert::TryInto; use std::path::{Path, PathBuf}; use std::sync::Arc; -use std::time::{Duration}; +use std::time::Duration; use arc_swap::ArcSwapOption; use async_trait::async_trait; @@ -21,9 +21,9 @@ use opentelemetry::{ use garage_util::data::*; use garage_util::error::*; +use garage_util::metrics::RecordDuration; use garage_util::time::*; use garage_util::tranquilizer::Tranquilizer; -use garage_util::metrics::RecordDuration; use garage_rpc::system::System; use garage_rpc::*; @@ -409,11 +409,14 @@ impl BlockManager { /// Read block from disk, verifying it's integrity async fn read_block(&self, hash: &Hash) -> Result { - let data = self.read_block_internal(hash) + let data = self + .read_block_internal(hash) .bound_record_duration(&self.metrics.block_read_duration) .await?; - self.metrics.bytes_read.add(data.inner_buffer().len() as u64); + self.metrics + .bytes_read + .add(data.inner_buffer().len() as u64); Ok(BlockRpc::PutBlock { hash: *hash, data }) } diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index f8bef47f..099c6429 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -1,6 +1,6 @@ //! Contain structs related to making RPCs use std::sync::Arc; -use std::time::{Duration}; +use std::time::Duration; use futures::future::join_all; use futures::stream::futures_unordered::FuturesUnordered; @@ -134,7 +134,7 @@ impl RpcHelper { M: Rpc>, H: EndpointHandler, { - let metric_tags = [KeyValue::new("endpoint", endpoint.path().to_string())]; + let metric_tags = [KeyValue::new("rpc_endpoint", endpoint.path().to_string())]; let msg_size = rmp_to_vec_all_named(&msg)?.len() as u32; let permit = self @@ -147,7 +147,8 @@ impl RpcHelper { self.0.metrics.rpc_counter.add(1, &metric_tags); let node_id = to.into(); - let rpc_call = endpoint.call(&node_id, msg, strat.rs_priority) + let rpc_call = endpoint + .call(&node_id, msg, strat.rs_priority) .record_duration(&self.0.metrics.rpc_duration, &metric_tags); select! { diff --git a/src/table/table.rs b/src/table/table.rs index 9ba243c0..69cac41a 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -161,7 +161,8 @@ where partition_key: &F::P, sort_key: &F::S, ) -> Result, Error> { - let res = self.get_internal(partition_key, sort_key) + let res = self + .get_internal(partition_key, sort_key) .bound_record_duration(&self.data.metrics.get_request_duration) .await?; self.data.metrics.get_request_counter.add(1); @@ -232,7 +233,8 @@ where filter: Option, limit: usize, ) -> Result, Error> { - let res = self.get_range_internal(partition_key, begin_sort_key, filter, limit) + let res = self + .get_range_internal(partition_key, begin_sort_key, filter, limit) .bound_record_duration(&self.data.metrics.get_request_duration) .await?; self.data.metrics.get_request_counter.add(1); diff --git a/src/util/metrics.rs b/src/util/metrics.rs index b3b1fc3c..cd5aa182 100644 --- a/src/util/metrics.rs +++ b/src/util/metrics.rs @@ -2,26 +2,40 @@ use std::time::SystemTime; use futures::{future::BoxFuture, Future, FutureExt}; -use opentelemetry::{KeyValue, metrics::*}; +use opentelemetry::{metrics::*, KeyValue}; pub trait RecordDuration<'a>: 'a { type Output; - fn record_duration(self, r: &'a ValueRecorder, attributes: &'a [KeyValue]) -> BoxFuture<'a, Self::Output>; + fn record_duration( + self, + r: &'a ValueRecorder, + attributes: &'a [KeyValue], + ) -> BoxFuture<'a, Self::Output>; fn bound_record_duration(self, r: &'a BoundValueRecorder) -> BoxFuture<'a, Self::Output>; } impl<'a, T, O> RecordDuration<'a> for T -where T: Future + Send + 'a { +where + T: Future + Send + 'a, +{ type Output = O; - fn record_duration(self, r: &'a ValueRecorder, attributes: &'a [KeyValue]) -> BoxFuture<'a, Self::Output> { + fn record_duration( + self, + r: &'a ValueRecorder, + attributes: &'a [KeyValue], + ) -> BoxFuture<'a, Self::Output> { async move { let request_start = SystemTime::now(); let res = self.await; - r.record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()), attributes); + r.record( + request_start.elapsed().map_or(0.0, |d| d.as_secs_f64()), + attributes, + ); res - }.boxed() + } + .boxed() } fn bound_record_duration(self, r: &'a BoundValueRecorder) -> BoxFuture<'a, Self::Output> { @@ -30,6 +44,7 @@ where T: Future + Send + 'a { let res = self.await; r.record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); res - }.boxed() + } + .boxed() } } -- 2.43.0 From d9a35359bf8903f37f5f7aa77421ed35b626e0af Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 22 Feb 2022 15:21:06 +0100 Subject: [PATCH 11/19] Add metrics to web endpoint --- Cargo.lock | 1 + Cargo.nix | 136 +++++++++++++++++++------------------ src/admin/tracing_setup.rs | 3 +- src/api/api_server.rs | 21 +++--- src/util/metrics.rs | 9 ++- src/web/Cargo.toml | 2 + src/web/web_server.rs | 80 ++++++++++++++++++++-- 7 files changed, 168 insertions(+), 84 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f05701f..95a81b82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1109,6 +1109,7 @@ dependencies = [ "garage_util 0.6.0", "http", "hyper", + "opentelemetry", "percent-encoding", "tracing", ] diff --git a/Cargo.nix b/Cargo.nix index 9a31fc8d..38e4828c 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -6,9 +6,9 @@ args@{ rootFeatures ? [ "garage_util/default" "garage_rpc/default" + "garage_admin/default" "garage_table/default" "garage_model/default" - "garage_admin/default" "garage_api/default" "garage_web/default" "garage/default" @@ -46,9 +46,9 @@ in workspace = { garage_util = rustPackages.unknown.garage_util."0.6.0"; garage_rpc = rustPackages.unknown.garage_rpc."0.6.0"; + garage_admin = rustPackages.unknown.garage_admin."0.6.0"; garage_table = rustPackages.unknown.garage_table."0.6.0"; garage_model = rustPackages.unknown.garage_model."0.6.0"; - garage_admin = rustPackages.unknown.garage_admin."0.6.0"; garage_api = rustPackages.unknown.garage_api."0.6.0"; garage_web = rustPackages.unknown.garage_web."0.6.0"; garage = rustPackages.unknown.garage."0.6.0"; @@ -1041,7 +1041,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "9420b90cfa29e327d0429f19be13e7ddb68fa1cccb09d65e5706b8c7a749b8a6"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") + [ "default" ] [ "std" ] ]; dependencies = { @@ -1190,7 +1190,6 @@ in dependencies = { futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; @@ -1339,6 +1338,7 @@ in bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + garage_admin = rustPackages."unknown".garage_admin."0.6.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; gethostname = rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.2" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; @@ -1452,6 +1452,7 @@ in k8s_openapi = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; }; kube = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kube."0.62.0" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; @@ -1479,6 +1480,7 @@ in garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; @@ -1715,12 +1717,12 @@ in features = builtins.concatLists [ [ "client" ] [ "default" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "full") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "h2") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "full") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "h2") [ "http1" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "http2") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "http2") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "runtime") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "server") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "server") [ "socket2" ] [ "stream" ] [ "tcp" ] @@ -1730,7 +1732,7 @@ in futures_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "h2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "h2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; }; httparse = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.6.0" { inherit profileName; }; @@ -1822,7 +1824,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "std") ]; dependencies = { hashbrown = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.11.2" { inherit profileName; }; @@ -2299,7 +2301,7 @@ in [ "os-poll" ] ]; dependencies = { - ${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; ${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; }; ${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; }; @@ -2532,7 +2534,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "vendored") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "vendored") ]; dependencies = { bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; @@ -2570,8 +2572,8 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "openssl-src") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "vendored") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "openssl-src") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "vendored") ]; dependencies = { libc = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; @@ -2579,7 +2581,7 @@ in buildDependencies = { autocfg = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }; cc = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".cc."1.0.73" { profileName = "__noProfile"; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "openssl_src" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl-src."111.17.0+1.1.1m" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "openssl_src" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".openssl-src."111.17.0+1.1.1m" { profileName = "__noProfile"; }; pkg_config = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".pkg-config."0.3.24" { profileName = "__noProfile"; }; ${ if hostPlatform.parsed.abi.name == "msvc" then "vcpkg" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".vcpkg."0.2.15" { profileName = "__noProfile"; }; }; @@ -2591,36 +2593,36 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "6105e89802af13fdf48c49d7646d3b533a70e536d818aae7e78ba0433d01acb8"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "async-trait") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "crossbeam-channel") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "dashmap") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "fnv") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "metrics") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "percent-encoding") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "pin-project") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rand") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "rt-tokio") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "tokio") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "tokio-stream") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "trace") + [ "async-trait" ] + [ "crossbeam-channel" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "dashmap") + [ "default" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "fnv") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "metrics") + [ "percent-encoding" ] + [ "pin-project" ] + [ "rand" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rt-tokio") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "tokio") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "tokio-stream") + [ "trace" ] ]; dependencies = { - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "async_trait" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "crossbeam_channel" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.2" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "dashmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".dashmap."4.0.2" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "fnv" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_channel" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_executor" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.21" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - ${ if (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") && hostPlatform.parsed.cpu.name == "wasm32" then "js_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.56" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "lazy_static" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "percent_encoding" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "pin_project" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "thiserror" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "tokio_stream" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }; + async_trait = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.52" { profileName = "__noProfile"; }; + crossbeam_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".crossbeam-channel."0.5.2" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "dashmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".dashmap."4.0.2" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "fnv" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fnv."1.0.7" { inherit profileName; }; + futures_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; + futures_executor = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-executor."0.3.21" { inherit profileName; }; + futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + ${ if hostPlatform.parsed.cpu.name == "wasm32" then "js_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".js-sys."0.3.56" { inherit profileName; }; + lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; + percent_encoding = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; + pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; + rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; + thiserror = rustPackages."registry+https://github.com/rust-lang/crates.io-index".thiserror."1.0.30" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "tokio_stream" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-stream."0.1.8" { inherit profileName; }; }; }); @@ -3149,7 +3151,7 @@ in [ "getrandom" ] [ "libc" ] [ "rand_chacha" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "small_rng") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "small_rng") [ "std" ] [ "std_rng" ] ]; @@ -3985,8 +3987,8 @@ in [ "bytes" ] [ "default" ] [ "fs" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "full") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "io-std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "full") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "io-std") [ "io-util" ] [ "libc" ] [ "macros" ] @@ -3995,8 +3997,8 @@ in [ "net" ] [ "num_cpus" ] [ "once_cell" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "parking_lot") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "process") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "parking_lot") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "process") [ "rt" ] [ "rt-multi-thread" ] [ "signal" ] @@ -4014,7 +4016,7 @@ in mio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mio."0.8.1" { inherit profileName; }; num_cpus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.13.1" { inherit profileName; }; once_cell = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.0" { inherit profileName; }; pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; ${ if hostPlatform.isUnix then "signal_hook_registry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.0" { inherit profileName; }; socket2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.4" { inherit profileName; }; @@ -4076,7 +4078,7 @@ in src = fetchCratesIo { inherit name version; sha256 = "50145484efff8818b5ccd256697f36863f587da82cf8b409c53adf1e840798e3"; }; features = builtins.concatLists [ [ "default" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "net") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "net") [ "time" ] ]; dependencies = { @@ -4212,24 +4214,24 @@ in src = fetchCratesIo { inherit name version; sha256 = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e"; }; features = builtins.concatLists [ [ "__common" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "balance") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "balance") [ "buffer" ] [ "default" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "discover") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "discover") [ "futures-core" ] [ "futures-util" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "indexmap") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "limit") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "load") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "indexmap") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "limit") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "load") [ "log" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "make") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "make") [ "pin-project" ] [ "pin-project-lite" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "rand") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "ready-cache") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rand") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "ready-cache") (lib.optional (rootFeatures' ? "garage") "retry") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "slab") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "timeout") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "slab") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "timeout") [ "tokio" ] [ "tokio-util" ] [ "tracing" ] @@ -4238,11 +4240,11 @@ in dependencies = { futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "indexmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.8.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "indexmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.8.0" { inherit profileName; }; pin_project = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" then "slab" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "slab" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; tokio_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.0" { inherit profileName; }; tower_layer = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.1" { inherit profileName; }; @@ -4669,7 +4671,7 @@ in [ "std" ] [ "synchapi" ] [ "sysinfoapi" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin") "threadpoollegacyapiset") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "threadpoollegacyapiset") [ "timezoneapi" ] [ "winbase" ] [ "wincon" ] @@ -4729,11 +4731,11 @@ in [ "default" ] ]; dependencies = { - ${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; }; }); diff --git a/src/admin/tracing_setup.rs b/src/admin/tracing_setup.rs index 83fa5622..c561d568 100644 --- a/src/admin/tracing_setup.rs +++ b/src/admin/tracing_setup.rs @@ -1,7 +1,7 @@ use std::time::Duration; use opentelemetry::sdk::{ - trace::{self, IdGenerator, Sampler}, + trace::{self, IdGenerator}, Resource, }; use opentelemetry::KeyValue; @@ -24,7 +24,6 @@ pub fn init_tracing(export_to: &str, node_id: Uuid) -> Result<(), Error> { .with_trace_config( trace::config() .with_id_generator(IdGenerator::default()) - .with_sampler(Sampler::TraceIdRatioBased(0.01f64)) .with_resource(Resource::new(vec![ KeyValue::new("service.name", "garage"), KeyValue::new("service.instance.id", node_id), diff --git a/src/api/api_server.rs b/src/api/api_server.rs index 8502d9d8..a6bf5a44 100644 --- a/src/api/api_server.rs +++ b/src/api/api_server.rs @@ -16,7 +16,7 @@ use opentelemetry::{ use garage_util::data::*; use garage_util::error::Error as GarageError; -use garage_util::metrics::RecordDuration; +use garage_util::metrics::{gen_trace_id, RecordDuration}; use garage_model::garage::Garage; use garage_model::key_table::Key; @@ -110,16 +110,12 @@ async fn handler( debug!("{:?}", req); let tracer = opentelemetry::global::tracer("garage"); - let trace_id = gen_uuid(); let span = tracer .span_builder("S3 API call (unknown)") - .with_trace_id( - opentelemetry::trace::TraceId::from_hex(&hex::encode(&trace_id.as_slice()[..16])) - .unwrap(), - ) + .with_trace_id(gen_trace_id()) .with_attributes(vec![ KeyValue::new("method", format!("{}", req.method())), - KeyValue::new("uri", req.uri().path().to_string()), + KeyValue::new("uri", req.uri().to_string()), ]) .start(&tracer); @@ -177,9 +173,14 @@ async fn handler_stage2( let (endpoint, bucket_name) = Endpoint::from_request(&req, bucket_name.map(ToOwned::to_owned))?; debug!("Endpoint: {:?}", endpoint); - Context::current() - .span() - .update_name::(format!("S3 API {}", endpoint.name())); + let current_context = Context::current(); + let current_span = current_context.span(); + current_span.update_name::(format!("S3 API {}", endpoint.name())); + current_span.set_attribute(KeyValue::new("endpoint", endpoint.name())); + current_span.set_attribute(KeyValue::new( + "bucket", + bucket_name.clone().unwrap_or_default(), + )); let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())]; diff --git a/src/util/metrics.rs b/src/util/metrics.rs index cd5aa182..1b05eabe 100644 --- a/src/util/metrics.rs +++ b/src/util/metrics.rs @@ -1,8 +1,9 @@ use std::time::SystemTime; use futures::{future::BoxFuture, Future, FutureExt}; +use rand::Rng; -use opentelemetry::{metrics::*, KeyValue}; +use opentelemetry::{metrics::*, trace::TraceId, KeyValue}; pub trait RecordDuration<'a>: 'a { type Output; @@ -48,3 +49,9 @@ where .boxed() } } + +// ---- + +pub fn gen_trace_id() -> TraceId { + rand::thread_rng().gen::<[u8; 16]>().into() +} diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 269e29a5..81a8c995 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -27,3 +27,5 @@ futures = "0.3" http = "0.2" hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } + +opentelemetry = "0.17" diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 80d2feb9..c51347a3 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -9,6 +9,13 @@ use hyper::{ Body, Method, Request, Response, Server, }; +use opentelemetry::{ + global, + metrics::{Counter, ValueRecorder}, + trace::{FutureExt, TraceContextExt, Tracer}, + Context, KeyValue, +}; + use crate::error::*; use garage_api::error::{Error as ApiError, OkOrBadRequest, OkOrInternalError}; @@ -20,6 +27,33 @@ use garage_model::garage::Garage; use garage_table::*; use garage_util::error::Error as GarageError; +use garage_util::metrics::{gen_trace_id, RecordDuration}; + +struct WebMetrics { + request_counter: Counter, + error_counter: Counter, + request_duration: ValueRecorder, +} + +impl WebMetrics { + fn new() -> Self { + let meter = global::meter("garage/web"); + Self { + request_counter: meter + .u64_counter("web.request_counter") + .with_description("Number of requests to the web endpoint") + .init(), + error_counter: meter + .u64_counter("web.error_counter") + .with_description("Number of requests to the web endpoint resulting in errors") + .init(), + request_duration: meter + .f64_value_recorder("web.request_duration") + .with_description("Duration of requests to the web endpoint") + .init(), + } + } +} /// Run a web server pub async fn run_web_server( @@ -28,13 +62,19 @@ pub async fn run_web_server( ) -> Result<(), GarageError> { let addr = &garage.config.s3_web.bind_addr; + let metrics = Arc::new(WebMetrics::new()); + let service = make_service_fn(|conn: &AddrStream| { let garage = garage.clone(); + let metrics = metrics.clone(); + let client_addr = conn.remote_addr(); async move { Ok::<_, Error>(service_fn(move |req: Request| { let garage = garage.clone(); - handle_request(garage, req, client_addr) + let metrics = metrics.clone(); + + handle_request(garage, metrics, req, client_addr) })) } }); @@ -49,23 +89,55 @@ pub async fn run_web_server( async fn handle_request( garage: Arc, + metrics: Arc, req: Request, addr: SocketAddr, ) -> Result, Infallible> { info!("{} {} {}", addr, req.method(), req.uri()); - match serve_file(garage, &req).await { + + // Lots of instrumentation + let tracer = opentelemetry::global::tracer("garage"); + let span = tracer + .span_builder(format!("Web {} request", req.method())) + .with_trace_id(gen_trace_id()) + .with_attributes(vec![ + KeyValue::new("method", format!("{}", req.method())), + KeyValue::new("uri", req.uri().to_string()), + ]) + .start(&tracer); + + let metrics_tags = &[KeyValue::new("method", req.method().to_string())]; + + // The actual handler + let res = serve_file(garage, &req) + .with_context(Context::current_with_span(span)) + .record_duration(&metrics.request_duration, &metrics_tags[..]) + .await; + + // More instrumentation + metrics.request_counter.add(1, &metrics_tags[..]); + + // Returning the result + match res { Ok(res) => { - debug!("{} {} {}", req.method(), req.uri(), res.status()); + debug!("{} {} {}", req.method(), res.status(), req.uri()); Ok(res) } Err(error) => { info!( "{} {} {} {}", req.method(), - req.uri(), error.http_status_code(), + req.uri(), error ); + metrics.error_counter.add( + 1, + &[ + metrics_tags[0].clone(), + KeyValue::new("status_code", error.http_status_code().to_string()), + ], + ); Ok(error_to_res(error)) } } -- 2.43.0 From dc8d0496ccd1b577fe6c0a1c172a8794531fa0a6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 22 Feb 2022 15:25:13 +0100 Subject: [PATCH 12/19] Refactoring: rename config files, make modifications less invasive --- script/dev-cluster.sh | 4 +- src/garage/server.rs | 4 +- src/garage/tests/common/garage.rs | 4 +- src/rpc/rpc_helper.rs | 278 ++++++++++++++++-------------- src/util/config.rs | 6 +- 5 files changed, 155 insertions(+), 141 deletions(-) diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh index 5cc003ef..fa0a950e 100755 --- a/script/dev-cluster.sh +++ b/script/dev-cluster.sh @@ -45,8 +45,8 @@ bind_addr = "0.0.0.0:$((3920+$count))" root_domain = ".web.garage.localhost" index = "index.html" -[admin_api] -bind_addr = "0.0.0.0:$((9900+$count))" +[admin] +api_bind_addr = "0.0.0.0:$((9900+$count))" EOF echo -en "$LABEL configuration written to $CONF_PATH\n" diff --git a/src/garage/server.rs b/src/garage/server.rs index b11f8417..1e276e9e 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -47,7 +47,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { let garage = Garage::new(config.clone(), db, background); info!("Initialize tracing..."); - if let Some(export_to) = config.admin_api.otlp_export_traces_to { + if let Some(export_to) = config.admin.trace_sink { init_tracing(&export_to, garage.system.id)?; } @@ -70,7 +70,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Configure and run admin web server..."); let admin_server = tokio::spawn( - admin_server_init.run(config.admin_api.bind_addr, wait_from(watch_cancel.clone())), + admin_server_init.run(config.admin.api_bind_addr, wait_from(watch_cancel.clone())), ); // Stuff runs diff --git a/src/garage/tests/common/garage.rs b/src/garage/tests/common/garage.rs index 12cf946b..36adb55e 100644 --- a/src/garage/tests/common/garage.rs +++ b/src/garage/tests/common/garage.rs @@ -66,8 +66,8 @@ bind_addr = "127.0.0.1:{web_port}" root_domain = ".web.garage" index = "index.html" -[admin_api] -bind_addr = "127.0.0.1:{admin_port}" +[admin] +api_bind_addr = "127.0.0.1:{admin_port}" "#, path = path.display(), secret = GARAGE_TEST_SECRET, diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index 099c6429..4114724f 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -238,154 +238,168 @@ impl RpcHelper { span.set_attribute(KeyValue::new("to", format!("{:?}", to))); span.set_attribute(KeyValue::new("quorum", quorum as i64)); - async { - let msg = Arc::new(msg); + self.try_call_many_internal(endpoint, to, msg, strategy, quorum) + .with_context(Context::current_with_span(span)) + .await + } - // Build future for each request - // They are not started now: they are added below in a FuturesUnordered - // object that will take care of polling them (see below) - let requests = to.iter().cloned().map(|to| { - let self2 = self.clone(); - let msg = msg.clone(); - let endpoint2 = endpoint.clone(); - (to, async move { - self2.call_arc(&endpoint2, to, msg, strategy).await + async fn try_call_many_internal( + &self, + endpoint: &Arc>, + to: &[Uuid], + msg: M, + strategy: RequestStrategy, + quorum: usize, + ) -> Result, Error> + where + M: Rpc> + 'static, + H: EndpointHandler + 'static, + S: Send + 'static, + { + let msg = Arc::new(msg); + + // Build future for each request + // They are not started now: they are added below in a FuturesUnordered + // object that will take care of polling them (see below) + let requests = to.iter().cloned().map(|to| { + let self2 = self.clone(); + let msg = msg.clone(); + let endpoint2 = endpoint.clone(); + (to, async move { + self2.call_arc(&endpoint2, to, msg, strategy).await + }) + }); + + // Vectors in which success results and errors will be collected + let mut successes = vec![]; + let mut errors = vec![]; + + if strategy.rs_interrupt_after_quorum { + // Case 1: once quorum is reached, other requests don't matter. + // What we do here is only send the required number of requests + // to reach a quorum, priorizing nodes with the lowest latency. + // When there are errors, we start new requests to compensate. + + // Retrieve some status variables that we will use to sort requests + let peer_list = self.0.fullmesh.get_peer_list(); + let ring: Arc = self.0.ring.borrow().clone(); + let our_zone = match ring.layout.node_role(&self.0.our_node_id) { + Some(pc) => &pc.zone, + None => "", + }; + + // Augment requests with some information used to sort them. + // The tuples are as follows: + // (is another node?, is another zone?, latency, node ID, request future) + // We store all of these tuples in a vec that we can sort. + // By sorting this vec, we priorize ourself, then nodes in the same zone, + // and within a same zone we priorize nodes with the lowest latency. + let mut requests = requests + .map(|(to, fut)| { + let peer_zone = match ring.layout.node_role(&to) { + Some(pc) => &pc.zone, + None => "", + }; + let peer_avg_ping = peer_list + .iter() + .find(|x| x.id.as_ref() == to.as_slice()) + .map(|pi| pi.avg_ping) + .flatten() + .unwrap_or_else(|| Duration::from_secs(1)); + ( + to != self.0.our_node_id, + peer_zone != our_zone, + peer_avg_ping, + to, + fut, + ) }) - }); + .collect::>(); - // Vectors in which success results and errors will be collected - let mut successes = vec![]; - let mut errors = vec![]; + // Sort requests by (priorize ourself, priorize same zone, priorize low latency) + requests + .sort_by_key(|(diffnode, diffzone, ping, _to, _fut)| (*diffnode, *diffzone, *ping)); - if strategy.rs_interrupt_after_quorum { - // Case 1: once quorum is reached, other requests don't matter. - // What we do here is only send the required number of requests - // to reach a quorum, priorizing nodes with the lowest latency. - // When there are errors, we start new requests to compensate. + // Make an iterator to take requests in their sorted order + let mut requests = requests.into_iter(); - // Retrieve some status variables that we will use to sort requests - let peer_list = self.0.fullmesh.get_peer_list(); - let ring: Arc = self.0.ring.borrow().clone(); - let our_zone = match ring.layout.node_role(&self.0.our_node_id) { - Some(pc) => &pc.zone, - None => "", - }; + // resp_stream will contain all of the requests that are currently in flight. + // (for the moment none, they will be added in the loop below) + let mut resp_stream = FuturesUnordered::new(); - // Augment requests with some information used to sort them. - // The tuples are as follows: - // (is another node?, is another zone?, latency, node ID, request future) - // We store all of these tuples in a vec that we can sort. - // By sorting this vec, we priorize ourself, then nodes in the same zone, - // and within a same zone we priorize nodes with the lowest latency. - let mut requests = requests - .map(|(to, fut)| { - let peer_zone = match ring.layout.node_role(&to) { - Some(pc) => &pc.zone, - None => "", - }; - let peer_avg_ping = peer_list - .iter() - .find(|x| x.id.as_ref() == to.as_slice()) - .map(|pi| pi.avg_ping) - .flatten() - .unwrap_or_else(|| Duration::from_secs(1)); - ( - to != self.0.our_node_id, - peer_zone != our_zone, - peer_avg_ping, - to, - fut, - ) - }) - .collect::>(); - - // Sort requests by (priorize ourself, priorize same zone, priorize low latency) - requests.sort_by_key(|(diffnode, diffzone, ping, _to, _fut)| { - (*diffnode, *diffzone, *ping) - }); - - // Make an iterator to take requests in their sorted order - let mut requests = requests.into_iter(); - - // resp_stream will contain all of the requests that are currently in flight. - // (for the moment none, they will be added in the loop below) - let mut resp_stream = FuturesUnordered::new(); - - // Do some requests and collect results - 'request_loop: while successes.len() < quorum { - // If the current set of requests that are running is not enough to possibly - // reach quorum, start some new requests. - while successes.len() + resp_stream.len() < quorum { - if let Some((_, _, _, req_to, fut)) = requests.next() { - let span = tracer.start(format!("RPC to {:?}", req_to)); - resp_stream.push(tokio::spawn( - fut.with_context(Context::current_with_span(span)), - )); - } else { - // If we have no request to add, we know that we won't ever - // reach quorum: bail out now. - break 'request_loop; - } - } - assert!(!resp_stream.is_empty()); // because of loop invariants - - // Wait for one request to terminate - match resp_stream.next().await.unwrap().unwrap() { - Ok(msg) => { - successes.push(msg); - } - Err(e) => { - errors.push(e); - } + // Do some requests and collect results + 'request_loop: while successes.len() < quorum { + // If the current set of requests that are running is not enough to possibly + // reach quorum, start some new requests. + while successes.len() + resp_stream.len() < quorum { + if let Some((_, _, _, req_to, fut)) = requests.next() { + let tracer = opentelemetry::global::tracer("garage"); + let span = tracer.start(format!("RPC to {:?}", req_to)); + resp_stream.push(tokio::spawn( + fut.with_context(Context::current_with_span(span)), + )); + } else { + // If we have no request to add, we know that we won't ever + // reach quorum: bail out now. + break 'request_loop; } } - } else { - // Case 2: all of the requests need to be sent in all cases, - // and need to terminate. (this is the case for writes that - // must be spread to n nodes) - // Just start all the requests in parallel and return as soon - // as the quorum is reached. - let mut resp_stream = requests - .map(|(_, fut)| fut) - .collect::>(); + assert!(!resp_stream.is_empty()); // because of loop invariants - while let Some(resp) = resp_stream.next().await { - match resp { - Ok(msg) => { - successes.push(msg); - if successes.len() >= quorum { - break; - } - } - Err(e) => { - errors.push(e); - } + // Wait for one request to terminate + match resp_stream.next().await.unwrap().unwrap() { + Ok(msg) => { + successes.push(msg); + } + Err(e) => { + errors.push(e); } } + } + } else { + // Case 2: all of the requests need to be sent in all cases, + // and need to terminate. (this is the case for writes that + // must be spread to n nodes) + // Just start all the requests in parallel and return as soon + // as the quorum is reached. + let mut resp_stream = requests + .map(|(_, fut)| fut) + .collect::>(); - if !resp_stream.is_empty() { - // Continue remaining requests in background. - // Continue the remaining requests immediately using tokio::spawn - // but enqueue a task in the background runner - // to ensure that the process won't exit until the requests are done - // (if we had just enqueued the resp_stream.collect directly in the background runner, - // the requests might have been put on hold in the background runner's queue, - // in which case they might timeout or otherwise fail) - let wait_finished_fut = tokio::spawn(async move { - resp_stream.collect::>>().await; - }); - self.0.background.spawn(wait_finished_fut.map(|_| Ok(()))); + while let Some(resp) = resp_stream.next().await { + match resp { + Ok(msg) => { + successes.push(msg); + if successes.len() >= quorum { + break; + } + } + Err(e) => { + errors.push(e); + } } } - if successes.len() >= quorum { - Ok(successes) - } else { - let errors = errors.iter().map(|e| format!("{}", e)).collect::>(); - Err(Error::Quorum(quorum, successes.len(), to.len(), errors)) + if !resp_stream.is_empty() { + // Continue remaining requests in background. + // Continue the remaining requests immediately using tokio::spawn + // but enqueue a task in the background runner + // to ensure that the process won't exit until the requests are done + // (if we had just enqueued the resp_stream.collect directly in the background runner, + // the requests might have been put on hold in the background runner's queue, + // in which case they might timeout or otherwise fail) + let wait_finished_fut = tokio::spawn(async move { + resp_stream.collect::>>().await; + }); + self.0.background.spawn(wait_finished_fut.map(|_| Ok(()))); } } - .with_context(Context::current_with_span(span)) - .await + + if successes.len() >= quorum { + Ok(successes) + } else { + let errors = errors.iter().map(|e| format!("{}", e)).collect::>(); + Err(Error::Quorum(quorum, successes.len(), to.len(), errors)) + } } } diff --git a/src/util/config.rs b/src/util/config.rs index e763fcd5..7e8d4ba6 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -75,7 +75,7 @@ pub struct Config { pub s3_web: WebConfig, /// Configuration for the admin API endpoint - pub admin_api: AdminConfig, + pub admin: AdminConfig, } /// Configuration for S3 api @@ -103,9 +103,9 @@ pub struct WebConfig { #[derive(Deserialize, Debug, Clone)] pub struct AdminConfig { /// Address and port to bind for admin API serving - pub bind_addr: SocketAddr, + pub api_bind_addr: SocketAddr, /// OTLP server to where to export traces - pub otlp_export_traces_to: Option, + pub trace_sink: Option, } fn default_sled_cache_capacity() -> u64 { -- 2.43.0 From 0cc31ee16994d11f4446186c817d8492b0b54f6b Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 22 Feb 2022 17:34:46 +0100 Subject: [PATCH 13/19] add missing netapp telemetry feature --- Cargo.lock | 82 ++++++++++++++++++++++++----- Cargo.nix | 102 ++++++++++++++++++++++++++++++++++--- src/admin/tracing_setup.rs | 3 +- src/rpc/Cargo.toml | 2 +- 4 files changed, 168 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95a81b82..098d92c5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -396,6 +396,15 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + [[package]] name = "core-foundation" version = "0.9.3" @@ -702,6 +711,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fuchsia-cprng" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" + [[package]] name = "futures" version = "0.3.21" @@ -823,7 +838,7 @@ dependencies = [ "kuska-sodiumoxide", "netapp 0.4.0", "pretty_env_logger", - "rand", + "rand 0.8.5", "rmp-serde 0.15.5", "serde", "serde_bytes", @@ -907,7 +922,7 @@ dependencies = [ "hex", "log", "netapp 0.3.1", - "rand", + "rand 0.8.5", "rmp-serde 0.15.5", "serde", "serde_bytes", @@ -932,7 +947,7 @@ dependencies = [ "hex", "netapp 0.4.0", "opentelemetry", - "rand", + "rand 0.8.5", "rmp-serde 0.15.5", "serde", "serde_bytes", @@ -960,7 +975,7 @@ dependencies = [ "kuska-sodiumoxide", "log", "netapp 0.3.1", - "rand", + "rand 0.8.5", "rmp-serde 0.15.5", "serde", "serde_bytes", @@ -990,7 +1005,7 @@ dependencies = [ "openssl", "opentelemetry", "pnet", - "rand", + "rand 0.8.5", "rmp-serde 0.15.5", "schemars", "serde", @@ -1015,7 +1030,7 @@ dependencies = [ "garage_util 0.5.1", "hexdump", "log", - "rand", + "rand 0.8.5", "rmp-serde 0.15.5", "serde", "serde_bytes", @@ -1035,7 +1050,7 @@ dependencies = [ "garage_util 0.6.0", "hexdump", "opentelemetry", - "rand", + "rand 0.8.5", "rmp-serde 0.15.5", "serde", "serde_bytes", @@ -1059,7 +1074,7 @@ dependencies = [ "hyper", "log", "netapp 0.3.1", - "rand", + "rand 0.8.5", "rmp-serde 0.15.5", "serde", "serde_json", @@ -1085,7 +1100,7 @@ dependencies = [ "kube", "netapp 0.4.0", "opentelemetry", - "rand", + "rand 0.8.5", "rmp-serde 0.15.5", "serde", "serde_json", @@ -1810,6 +1825,9 @@ dependencies = [ "kuska-handshake", "kuska-sodiumoxide", "log", + "opentelemetry", + "opentelemetry-contrib", + "rand 0.5.6", "rmp-serde 0.14.4", "serde", "tokio", @@ -1947,12 +1965,22 @@ dependencies = [ "lazy_static", "percent-encoding", "pin-project 1.0.10", - "rand", + "rand 0.8.5", "thiserror", "tokio", "tokio-stream", ] +[[package]] +name = "opentelemetry-contrib" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85637add8f60bb4cac673469c14f47a329c6cec7365c72d72cd32f2d104a721a" +dependencies = [ + "lazy_static", + "opentelemetry", +] + [[package]] name = "opentelemetry-otlp" version = "0.10.0" @@ -2366,6 +2394,19 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rand" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9" +dependencies = [ + "cloudabi", + "fuchsia-cprng", + "libc", + "rand_core 0.3.1", + "winapi", +] + [[package]] name = "rand" version = "0.8.5" @@ -2374,7 +2415,7 @@ checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", "rand_chacha", - "rand_core", + "rand_core 0.6.3", ] [[package]] @@ -2384,9 +2425,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.3", ] +[[package]] +name = "rand_core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +dependencies = [ + "rand_core 0.4.2", +] + +[[package]] +name = "rand_core" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" + [[package]] name = "rand_core" version = "0.6.3" @@ -3141,7 +3197,7 @@ dependencies = [ "indexmap", "pin-project 1.0.10", "pin-project-lite", - "rand", + "rand 0.8.5", "slab", "tokio", "tokio-util 0.7.0", diff --git a/Cargo.nix b/Cargo.nix index 38e4828c..68f803c0 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -574,6 +574,20 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".cloudabi."0.0.3" = overridableMkRustCrate (profileName: rec { + name = "cloudabi"; + version = "0.0.3"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f"; }; + features = builtins.concatLists [ + [ "bitflags" ] + [ "default" ] + ]; + dependencies = { + bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".core-foundation."0.9.3" = overridableMkRustCrate (profileName: rec { name = "core-foundation"; version = "0.9.3"; @@ -981,6 +995,13 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".fuchsia-cprng."0.1.1" = overridableMkRustCrate (profileName: rec { + name = "fuchsia-cprng"; + version = "0.1.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"; }; + }); + "registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" = overridableMkRustCrate (profileName: rec { name = "futures"; version = "0.3.21"; @@ -2301,7 +2322,7 @@ in [ "os-poll" ] ]; dependencies = { - ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; ${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; }; ${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; }; @@ -2403,6 +2424,10 @@ in src = fetchCratesIo { inherit name version; sha256 = "22c545a13b0c47b47e8052b35c4884dbe33c9ea62607371b0f4f1b0490cafd38"; }; features = builtins.concatLists [ [ "default" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "opentelemetry") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "opentelemetry-contrib") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rand") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "telemetry") ]; dependencies = { arc_swap = rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.5.0" { inherit profileName; }; @@ -2415,6 +2440,9 @@ in kuska_handshake = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-handshake."0.2.0" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "opentelemetry_contrib" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-contrib."0.9.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.5.6" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.14.4" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; @@ -2626,6 +2654,20 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-contrib."0.9.0" = overridableMkRustCrate (profileName: rec { + name = "opentelemetry-contrib"; + version = "0.9.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "85637add8f60bb4cac673469c14f47a329c6cec7365c72d72cd32f2d104a721a"; }; + features = builtins.concatLists [ + [ "default" ] + ]; + dependencies = { + lazy_static = rustPackages."registry+https://github.com/rust-lang/crates.io-index".lazy_static."1.4.0" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" = overridableMkRustCrate (profileName: rec { name = "opentelemetry-otlp"; version = "0.10.0"; @@ -3140,6 +3182,29 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".rand."0.5.6" = overridableMkRustCrate (profileName: rec { + name = "rand"; + version = "0.5.6"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "c618c47cd3ebd209790115ab837de41425723956ad3ce2e6a7f09890947cacb9"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "cloudabi" ] + [ "default" ] + [ "fuchsia-cprng" ] + [ "libc" ] + [ "std" ] + [ "winapi" ] + ]; + dependencies = { + ${ if hostPlatform.parsed.kernel.name == "cloudabi" then "cloudabi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cloudabi."0.0.3" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "fuchsia" then "fuchsia_cprng" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".fuchsia-cprng."0.1.1" { inherit profileName; }; + ${ if hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.3.1" { inherit profileName; }; + ${ if hostPlatform.isWindows then "winapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" = overridableMkRustCrate (profileName: rec { name = "rand"; version = "0.8.5"; @@ -3176,6 +3241,31 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".rand_core."0.3.1" = overridableMkRustCrate (profileName: rec { + name = "rand_core"; + version = "0.3.1"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "std" ] + ]; + dependencies = { + rand_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand_core."0.4.2" { inherit profileName; }; + }; + }); + + "registry+https://github.com/rust-lang/crates.io-index".rand_core."0.4.2" = overridableMkRustCrate (profileName: rec { + name = "rand_core"; + version = "0.4.2"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc"; }; + features = builtins.concatLists [ + [ "alloc" ] + [ "std" ] + ]; + }); + "registry+https://github.com/rust-lang/crates.io-index".rand_core."0.6.3" = overridableMkRustCrate (profileName: rec { name = "rand_core"; version = "0.6.3"; @@ -3284,7 +3374,7 @@ in ]; dependencies = { ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "dragonfly" || hostPlatform.parsed.kernel.name == "freebsd" || hostPlatform.parsed.kernel.name == "illumos" || hostPlatform.parsed.kernel.name == "netbsd" || hostPlatform.parsed.kernel.name == "openbsd" || hostPlatform.parsed.kernel.name == "solaris" || hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "once_cell" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "i686" || hostPlatform.parsed.cpu.name == "x86_64" || (hostPlatform.parsed.cpu.name == "aarch64" || hostPlatform.parsed.cpu.name == "armv6l" || hostPlatform.parsed.cpu.name == "armv7l") && (hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "fuchsia" || hostPlatform.parsed.kernel.name == "linux") then "spin" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".spin."0.5.2" { inherit profileName; }; untrusted = rustPackages."registry+https://github.com/rust-lang/crates.io-index".untrusted."0.7.1" { inherit profileName; }; ${ if hostPlatform.parsed.cpu.name == "wasm32" && hostPlatform.parsed.vendor.name == "unknown" && hostPlatform.parsed.kernel.name == "unknown" && hostPlatform.parsed.abi.name == "" then "web_sys" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".web-sys."0.3.56" { inherit profileName; }; @@ -3761,7 +3851,7 @@ in ]; dependencies = { bitflags = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bitflags."1.3.2" { inherit profileName; }; - ${ if hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "android" || hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.11.2" { inherit profileName; }; ${ if !(hostPlatform.parsed.kernel.name == "linux" || hostPlatform.parsed.kernel.name == "android") then "parking_lot_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot_core."0.8.5" { inherit profileName; }; static_init_macro = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init_macro."1.0.2" { profileName = "__noProfile"; }; @@ -4732,10 +4822,10 @@ in ]; dependencies = { ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; }; }); diff --git a/src/admin/tracing_setup.rs b/src/admin/tracing_setup.rs index c561d568..55fc4094 100644 --- a/src/admin/tracing_setup.rs +++ b/src/admin/tracing_setup.rs @@ -1,7 +1,7 @@ use std::time::Duration; use opentelemetry::sdk::{ - trace::{self, IdGenerator}, + trace::{self, IdGenerator, Sampler}, Resource, }; use opentelemetry::KeyValue; @@ -24,6 +24,7 @@ pub fn init_tracing(export_to: &str, node_id: Uuid) -> Result<(), Error> { .with_trace_config( trace::config() .with_id_generator(IdGenerator::default()) + .with_sampler(Sampler::AlwaysOn) .with_resource(Resource::new(vec![ KeyValue::new("service.name", "garage"), KeyValue::new("service.instance.id", node_id), diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 94d6f682..72393921 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -48,7 +48,7 @@ opentelemetry = "0.17" #netapp = { version = "0.3.0", git = "https://git.deuxfleurs.fr/lx/netapp" } #netapp = { version = "0.4", path = "../../../netapp", features = ["telemetry"] } -netapp = "0.4" +netapp = { version = "0.4", features = ["telemetry"] } hyper = { version = "0.14", features = ["client", "http1", "runtime", "tcp"] } -- 2.43.0 From f869ca625d025d8c52af405392d2081bd81437a8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 24 Feb 2022 13:18:51 +0100 Subject: [PATCH 14/19] Add spans to table calls, change span names in RPC --- src/admin/metrics.rs | 10 ++-------- src/rpc/rpc_helper.rs | 16 +++++++++++++++- src/table/table.rs | 39 +++++++++++++++++++++++++++++++++++++-- 3 files changed, 54 insertions(+), 11 deletions(-) diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs index 02549fe9..cbc737d3 100644 --- a/src/admin/metrics.rs +++ b/src/admin/metrics.rs @@ -20,8 +20,8 @@ use opentelemetry_prometheus::PrometheusExporter; use prometheus::{Encoder, TextEncoder}; -use garage_util::data::*; use garage_util::error::Error as GarageError; +use garage_util::metrics::*; // serve_req on metric endpoint async fn serve_req( @@ -125,15 +125,9 @@ impl AdminServer { async move { Ok::<_, Infallible>(service_fn(move |req| { let tracer = opentelemetry::global::tracer("garage"); - let uuid = gen_uuid(); let span = tracer .span_builder("admin/request") - .with_trace_id( - opentelemetry::trace::TraceId::from_hex(&hex::encode( - &uuid.as_slice()[..16], - )) - .unwrap(), - ) + .with_trace_id(gen_trace_id()) .start(&tracer); serve_req(req, admin_server.clone()) diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index 4114724f..89225511 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -234,9 +234,23 @@ impl RpcHelper { let quorum = strategy.rs_quorum.unwrap_or(to.len()); let tracer = opentelemetry::global::tracer("garage"); - let mut span = tracer.start(format!("RPC {} to {}", endpoint.path(), to.len())); + let span_name = if strategy.rs_interrupt_after_quorum { + format!("RPC {} to {} of {}", endpoint.path(), quorum, to.len()) + } else { + format!( + "RPC {} to {} (quorum {})", + endpoint.path(), + to.len(), + quorum + ) + }; + let mut span = tracer.start(span_name); span.set_attribute(KeyValue::new("to", format!("{:?}", to))); span.set_attribute(KeyValue::new("quorum", quorum as i64)); + span.set_attribute(KeyValue::new( + "interrupt_after_quorum", + strategy.rs_interrupt_after_quorum.to_string(), + )); self.try_call_many_internal(endpoint, to, msg, strategy, quorum) .with_context(Context::current_with_span(span)) diff --git a/src/table/table.rs b/src/table/table.rs index 69cac41a..7f87a449 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -7,6 +7,11 @@ use futures::stream::*; use serde::{Deserialize, Serialize}; use serde_bytes::ByteBuf; +use opentelemetry::{ + trace::{FutureExt, TraceContextExt, Tracer}, + Context, +}; + use garage_util::data::*; use garage_util::error::Error; use garage_util::metrics::RecordDuration; @@ -82,6 +87,20 @@ where } pub async fn insert(&self, e: &F::E) -> Result<(), Error> { + let tracer = opentelemetry::global::tracer("garage_table"); + let span = tracer.start(format!("{} insert", F::TABLE_NAME)); + + self.insert_internal(e) + .bound_record_duration(&self.data.metrics.put_request_duration) + .with_context(Context::current_with_span(span)) + .await?; + + self.data.metrics.put_request_counter.add(1); + + Ok(()) + } + + async fn insert_internal(&self, e: &F::E) -> Result<(), Error> { let hash = e.partition_key().hash(); let who = self.data.replication.write_nodes(&hash); //eprintln!("insert who: {:?}", who); @@ -99,18 +118,22 @@ where .with_quorum(self.data.replication.write_quorum()) .with_timeout(TABLE_RPC_TIMEOUT), ) - .bound_record_duration(&self.data.metrics.put_request_duration) .await?; - self.data.metrics.put_request_counter.add(1); Ok(()) } pub async fn insert_many(&self, entries: &[F::E]) -> Result<(), Error> { + let tracer = opentelemetry::global::tracer("garage_table"); + let span = tracer.start(format!("{} insert_many {}", F::TABLE_NAME, entries.len())); + self.insert_many_internal(entries) .bound_record_duration(&self.data.metrics.put_request_duration) + .with_context(Context::current_with_span(span)) .await?; + self.data.metrics.put_request_counter.add(1); + Ok(()) } @@ -161,11 +184,17 @@ where partition_key: &F::P, sort_key: &F::S, ) -> Result, Error> { + let tracer = opentelemetry::global::tracer("garage_table"); + let span = tracer.start(format!("{} get", F::TABLE_NAME)); + let res = self .get_internal(partition_key, sort_key) .bound_record_duration(&self.data.metrics.get_request_duration) + .with_context(Context::current_with_span(span)) .await?; + self.data.metrics.get_request_counter.add(1); + Ok(res) } @@ -233,11 +262,17 @@ where filter: Option, limit: usize, ) -> Result, Error> { + let tracer = opentelemetry::global::tracer("garage_table"); + let span = tracer.start(format!("{} get_range", F::TABLE_NAME)); + let res = self .get_range_internal(partition_key, begin_sort_key, filter, limit) .bound_record_duration(&self.data.metrics.get_request_duration) + .with_context(Context::current_with_span(span)) .await?; + self.data.metrics.get_request_counter.add(1); + Ok(res) } -- 2.43.0 From 203e8d2c345dabee56e31aebf88b706df5aa72e5 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 24 Feb 2022 13:25:28 +0100 Subject: [PATCH 15/19] Bump version to 0.7 because of incompatible Netapp --- Cargo.lock | 50 ++++++++++----------- Cargo.nix | 102 +++++++++++++++++++++--------------------- src/admin/Cargo.toml | 4 +- src/api/Cargo.toml | 8 ++-- src/garage/Cargo.toml | 16 +++---- src/model/Cargo.toml | 8 ++-- src/rpc/Cargo.toml | 6 +-- src/rpc/system.rs | 2 +- src/table/Cargo.toml | 6 +-- src/util/Cargo.toml | 2 +- src/web/Cargo.toml | 10 ++--- 11 files changed, 107 insertions(+), 107 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 098d92c5..bbd79449 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -817,7 +817,7 @@ dependencies = [ [[package]] name = "garage" -version = "0.6.0" +version = "0.7.0" dependencies = [ "async-trait", "aws-sdk-s3", @@ -826,10 +826,10 @@ dependencies = [ "futures-util", "garage_admin", "garage_api", - "garage_model 0.6.0", - "garage_rpc 0.6.0", - "garage_table 0.6.0", - "garage_util 0.6.0", + "garage_model 0.7.0", + "garage_rpc 0.7.0", + "garage_table 0.7.0", + "garage_util 0.7.0", "garage_web", "git-version", "hex", @@ -852,11 +852,11 @@ dependencies = [ [[package]] name = "garage_admin" -version = "0.6.0" +version = "0.7.0" dependencies = [ "futures", "futures-util", - "garage_util 0.6.0", + "garage_util 0.7.0", "hex", "http", "hyper", @@ -869,7 +869,7 @@ dependencies = [ [[package]] name = "garage_api" -version = "0.6.0" +version = "0.7.0" dependencies = [ "base64", "bytes 1.1.0", @@ -879,9 +879,9 @@ dependencies = [ "form_urlencoded", "futures", "futures-util", - "garage_model 0.6.0", - "garage_table 0.6.0", - "garage_util 0.6.0", + "garage_model 0.7.0", + "garage_table 0.7.0", + "garage_util 0.7.0", "hex", "hmac", "http", @@ -933,7 +933,7 @@ dependencies = [ [[package]] name = "garage_model" -version = "0.6.0" +version = "0.7.0" dependencies = [ "arc-swap", "async-trait", @@ -941,9 +941,9 @@ dependencies = [ "futures", "futures-util", "garage_model 0.5.1", - "garage_rpc 0.6.0", - "garage_table 0.6.0", - "garage_util 0.6.0", + "garage_rpc 0.7.0", + "garage_table 0.7.0", + "garage_util 0.7.0", "hex", "netapp 0.4.0", "opentelemetry", @@ -986,7 +986,7 @@ dependencies = [ [[package]] name = "garage_rpc" -version = "0.6.0" +version = "0.7.0" dependencies = [ "arc-swap", "async-trait", @@ -994,7 +994,7 @@ dependencies = [ "futures", "futures-util", "garage_admin", - "garage_util 0.6.0", + "garage_util 0.7.0", "gethostname", "hex", "hyper", @@ -1040,14 +1040,14 @@ dependencies = [ [[package]] name = "garage_table" -version = "0.6.0" +version = "0.7.0" dependencies = [ "async-trait", "bytes 1.1.0", "futures", "futures-util", - "garage_rpc 0.6.0", - "garage_util 0.6.0", + "garage_rpc 0.7.0", + "garage_util 0.7.0", "hexdump", "opentelemetry", "rand 0.8.5", @@ -1087,7 +1087,7 @@ dependencies = [ [[package]] name = "garage_util" -version = "0.6.0" +version = "0.7.0" dependencies = [ "blake2", "chrono", @@ -1114,14 +1114,14 @@ dependencies = [ [[package]] name = "garage_web" -version = "0.6.0" +version = "0.7.0" dependencies = [ "err-derive 0.3.1", "futures", "garage_api", - "garage_model 0.6.0", - "garage_table 0.6.0", - "garage_util 0.6.0", + "garage_model 0.7.0", + "garage_table 0.7.0", + "garage_util 0.7.0", "http", "hyper", "opentelemetry", diff --git a/Cargo.nix b/Cargo.nix index 68f803c0..82ad7aba 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -44,14 +44,14 @@ in { cargo2nixVersion = "0.10.0"; workspace = { - garage_util = rustPackages.unknown.garage_util."0.6.0"; - garage_rpc = rustPackages.unknown.garage_rpc."0.6.0"; - garage_admin = rustPackages.unknown.garage_admin."0.6.0"; - garage_table = rustPackages.unknown.garage_table."0.6.0"; - garage_model = rustPackages.unknown.garage_model."0.6.0"; - garage_api = rustPackages.unknown.garage_api."0.6.0"; - garage_web = rustPackages.unknown.garage_web."0.6.0"; - garage = rustPackages.unknown.garage."0.6.0"; + garage_util = rustPackages.unknown.garage_util."0.7.0"; + garage_rpc = rustPackages.unknown.garage_rpc."0.7.0"; + garage_admin = rustPackages.unknown.garage_admin."0.7.0"; + garage_table = rustPackages.unknown.garage_table."0.7.0"; + garage_model = rustPackages.unknown.garage_model."0.7.0"; + garage_api = rustPackages.unknown.garage_api."0.7.0"; + garage_web = rustPackages.unknown.garage_web."0.7.0"; + garage = rustPackages.unknown.garage."0.7.0"; }; "registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" = overridableMkRustCrate (profileName: rec { name = "aho-corasick"; @@ -1163,9 +1163,9 @@ in }; }); - "unknown".garage."0.6.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage."0.7.0" = overridableMkRustCrate (profileName: rec { name = "garage"; - version = "0.6.0"; + version = "0.7.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/garage"); dependencies = { @@ -1173,13 +1173,13 @@ in bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - garage_admin = rustPackages."unknown".garage_admin."0.6.0" { inherit profileName; }; - garage_api = rustPackages."unknown".garage_api."0.6.0" { inherit profileName; }; - garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; - garage_rpc = rustPackages."unknown".garage_rpc."0.6.0" { inherit profileName; }; - garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; - garage_web = rustPackages."unknown".garage_web."0.6.0" { inherit profileName; }; + garage_admin = rustPackages."unknown".garage_admin."0.7.0" { inherit profileName; }; + garage_api = rustPackages."unknown".garage_api."0.7.0" { inherit profileName; }; + garage_model = rustPackages."unknown".garage_model."0.7.0" { inherit profileName; }; + garage_rpc = rustPackages."unknown".garage_rpc."0.7.0" { inherit profileName; }; + garage_table = rustPackages."unknown".garage_table."0.7.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; + garage_web = rustPackages."unknown".garage_web."0.7.0" { inherit profileName; }; git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; @@ -1203,15 +1203,15 @@ in }; }); - "unknown".garage_admin."0.6.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_admin."0.7.0" = overridableMkRustCrate (profileName: rec { name = "garage_admin"; - version = "0.6.0"; + version = "0.7.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/admin"); dependencies = { futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; @@ -1223,9 +1223,9 @@ in }; }); - "unknown".garage_api."0.6.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_api."0.7.0" = overridableMkRustCrate (profileName: rec { name = "garage_api"; - version = "0.6.0"; + version = "0.7.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/api"); dependencies = { @@ -1237,9 +1237,9 @@ in form_urlencoded = rustPackages."registry+https://github.com/rust-lang/crates.io-index".form_urlencoded."1.0.1" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; - garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + garage_model = rustPackages."unknown".garage_model."0.7.0" { inherit profileName; }; + garage_table = rustPackages."unknown".garage_table."0.7.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; hmac = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hmac."0.10.1" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; @@ -1291,9 +1291,9 @@ in }; }); - "unknown".garage_model."0.6.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_model."0.7.0" = overridableMkRustCrate (profileName: rec { name = "garage_model"; - version = "0.6.0"; + version = "0.7.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/model"); dependencies = { @@ -1303,9 +1303,9 @@ in futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; garage_model_050 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".garage_model."0.5.1" { inherit profileName; }; - garage_rpc = rustPackages."unknown".garage_rpc."0.6.0" { inherit profileName; }; - garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + garage_rpc = rustPackages."unknown".garage_rpc."0.7.0" { inherit profileName; }; + garage_table = rustPackages."unknown".garage_table."0.7.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.0" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; @@ -1348,9 +1348,9 @@ in }; }); - "unknown".garage_rpc."0.6.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_rpc."0.7.0" = overridableMkRustCrate (profileName: rec { name = "garage_rpc"; - version = "0.6.0"; + version = "0.7.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/rpc"); dependencies = { @@ -1359,8 +1359,8 @@ in bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - garage_admin = rustPackages."unknown".garage_admin."0.6.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + garage_admin = rustPackages."unknown".garage_admin."0.7.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; gethostname = rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.2" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; @@ -1406,9 +1406,9 @@ in }; }); - "unknown".garage_table."0.6.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_table."0.7.0" = overridableMkRustCrate (profileName: rec { name = "garage_table"; - version = "0.6.0"; + version = "0.7.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/table"); dependencies = { @@ -1416,8 +1416,8 @@ in bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - garage_rpc = rustPackages."unknown".garage_rpc."0.6.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + garage_rpc = rustPackages."unknown".garage_rpc."0.7.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; hexdump = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hexdump."0.1.1" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; @@ -1457,9 +1457,9 @@ in }; }); - "unknown".garage_util."0.6.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_util."0.7.0" = overridableMkRustCrate (profileName: rec { name = "garage_util"; - version = "0.6.0"; + version = "0.7.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/util"); dependencies = { @@ -1487,18 +1487,18 @@ in }; }); - "unknown".garage_web."0.6.0" = overridableMkRustCrate (profileName: rec { + "unknown".garage_web."0.7.0" = overridableMkRustCrate (profileName: rec { name = "garage_web"; - version = "0.6.0"; + version = "0.7.0"; registry = "unknown"; src = fetchCrateLocal (workspaceSrc + "/src/web"); dependencies = { err_derive = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; - garage_api = rustPackages."unknown".garage_api."0.6.0" { inherit profileName; }; - garage_model = rustPackages."unknown".garage_model."0.6.0" { inherit profileName; }; - garage_table = rustPackages."unknown".garage_table."0.6.0" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.6.0" { inherit profileName; }; + garage_api = rustPackages."unknown".garage_api."0.7.0" { inherit profileName; }; + garage_model = rustPackages."unknown".garage_model."0.7.0" { inherit profileName; }; + garage_table = rustPackages."unknown".garage_table."0.7.0" { inherit profileName; }; + garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.17" { inherit profileName; }; opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; @@ -2322,7 +2322,7 @@ in [ "os-poll" ] ]; dependencies = { - ${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; + ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.119" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.14" { inherit profileName; }; ${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; }; ${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; }; @@ -4821,11 +4821,11 @@ in [ "default" ] ]; dependencies = { - ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "i686-uwp-windows-msvc" || hostPlatform.config == "i686-pc-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-uwp-windows-msvc" || hostPlatform.config == "x86_64-pc-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; }; }); diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml index 01c16c57..2db4bb08 100644 --- a/src/admin/Cargo.toml +++ b/src/admin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_admin" -version = "0.6.0" +version = "0.7.0" authors = ["Maximilien Richer "] edition = "2018" license = "AGPL-3.0" @@ -13,7 +13,7 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_util = { version = "0.6.0", path = "../util" } +garage_util = { version = "0.7.0", path = "../util" } hex = "0.4" diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index f39c6d04..d61a986f 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_api" -version = "0.6.0" +version = "0.7.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,9 +14,9 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_model = { version = "0.6.0", path = "../model" } -garage_table = { version = "0.6.0", path = "../table" } -garage_util = { version = "0.6.0", path = "../util" } +garage_model = { version = "0.7.0", path = "../model" } +garage_table = { version = "0.7.0", path = "../table" } +garage_util = { version = "0.7.0", path = "../util" } base64 = "0.13" bytes = "1.0" diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index ff0666a6..2f67d418 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage" -version = "0.6.0" +version = "0.7.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -21,13 +21,13 @@ path = "tests/lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_api = { version = "0.6.0", path = "../api" } -garage_model = { version = "0.6.0", path = "../model" } -garage_rpc = { version = "0.6.0", path = "../rpc" } -garage_table = { version = "0.6.0", path = "../table" } -garage_util = { version = "0.6.0", path = "../util" } -garage_web = { version = "0.6.0", path = "../web" } -garage_admin = { version = "0.6.0", path = "../admin" } +garage_api = { version = "0.7.0", path = "../api" } +garage_model = { version = "0.7.0", path = "../model" } +garage_rpc = { version = "0.7.0", path = "../rpc" } +garage_table = { version = "0.7.0", path = "../table" } +garage_util = { version = "0.7.0", path = "../util" } +garage_web = { version = "0.7.0", path = "../web" } +garage_admin = { version = "0.7.0", path = "../admin" } bytes = "1.0" git-version = "0.3.4" diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 8083445e..64eae0f4 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_model" -version = "0.6.0" +version = "0.7.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,9 +14,9 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_rpc = { version = "0.6.0", path = "../rpc" } -garage_table = { version = "0.6.0", path = "../table" } -garage_util = { version = "0.6.0", path = "../util" } +garage_rpc = { version = "0.7.0", path = "../rpc" } +garage_table = { version = "0.7.0", path = "../table" } +garage_util = { version = "0.7.0", path = "../util" } garage_model_050 = { package = "garage_model", version = "0.5.1" } async-trait = "0.1.7" diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 72393921..a027f219 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_rpc" -version = "0.6.0" +version = "0.7.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,8 +14,8 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_util = { version = "0.6.0", path = "../util" } -garage_admin = { version = "0.6.0", path = "../admin" } +garage_util = { version = "0.7.0", path = "../util" } +garage_admin = { version = "0.7.0", path = "../admin" } arc-swap = "1.0" bytes = "1.0" diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 2123a37f..741f68e2 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -39,7 +39,7 @@ const STATUS_EXCHANGE_INTERVAL: Duration = Duration::from_secs(10); const PING_TIMEOUT: Duration = Duration::from_secs(2); /// Version tag used for version check upon Netapp connection -pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650006; // garage 0x0006 +pub const GARAGE_VERSION_TAG: u64 = 0x6761726167650007; // garage 0x0007 /// RPC endpoint used for calls related to membership pub const SYSTEM_RPC_PATH: &str = "garage_rpc/membership.rs/SystemRpc"; diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index bc09ca7c..ed1a213f 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_table" -version = "0.6.0" +version = "0.7.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" @@ -14,8 +14,8 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_rpc = { version = "0.6.0", path = "../rpc" } -garage_util = { version = "0.6.0", path = "../util" } +garage_rpc = { version = "0.7.0", path = "../rpc" } +garage_util = { version = "0.7.0", path = "../util" } opentelemetry = "0.17" diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index e1d516de..dfa4e822 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_util" -version = "0.6.0" +version = "0.7.0" authors = ["Alex Auvolat "] edition = "2018" license = "AGPL-3.0" diff --git a/src/web/Cargo.toml b/src/web/Cargo.toml index 81a8c995..59a1231d 100644 --- a/src/web/Cargo.toml +++ b/src/web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "garage_web" -version = "0.6.0" +version = "0.7.0" authors = ["Alex Auvolat ", "Quentin Dufour "] edition = "2018" license = "AGPL-3.0" @@ -14,10 +14,10 @@ path = "lib.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -garage_api = { version = "0.6.0", path = "../api" } -garage_model = { version = "0.6.0", path = "../model" } -garage_util = { version = "0.6.0", path = "../util" } -garage_table = { version = "0.6.0", path = "../table" } +garage_api = { version = "0.7.0", path = "../api" } +garage_model = { version = "0.7.0", path = "../model" } +garage_util = { version = "0.7.0", path = "../util" } +garage_table = { version = "0.7.0", path = "../table" } err-derive = "0.3" tracing = "0.1.30" -- 2.43.0 From 2377a92f6b64165be84f53722f30c21e9c8a0b3e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 24 Feb 2022 14:59:49 +0100 Subject: [PATCH 16/19] Add wrapper over sled tree to count items (used for big queues) --- src/model/block.rs | 4 +- src/model/block_metrics.rs | 4 +- src/rpc/rpc_helper.rs | 7 ++- src/table/data.rs | 4 +- src/table/gc.rs | 5 ++- src/table/metrics.rs | 8 +++- src/util/lib.rs | 1 + src/util/sled_counter.rs | 92 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 src/util/sled_counter.rs diff --git a/src/model/block.rs b/src/model/block.rs index d97e64a8..97e06f0e 100644 --- a/src/model/block.rs +++ b/src/model/block.rs @@ -22,6 +22,7 @@ use opentelemetry::{ use garage_util::data::*; use garage_util::error::*; use garage_util::metrics::RecordDuration; +use garage_util::sled_counter::SledCountedTree; use garage_util::time::*; use garage_util::tranquilizer::Tranquilizer; @@ -155,7 +156,7 @@ pub struct BlockManager { rc: sled::Tree, - resync_queue: sled::Tree, + resync_queue: SledCountedTree, resync_notify: Notify, system: Arc, @@ -184,6 +185,7 @@ impl BlockManager { let resync_queue = db .open_tree("block_local_resync_queue") .expect("Unable to open block_local_resync_queue tree"); + let resync_queue = SledCountedTree::new(resync_queue); let endpoint = system .netapp diff --git a/src/model/block_metrics.rs b/src/model/block_metrics.rs index 7ef9a117..819af241 100644 --- a/src/model/block_metrics.rs +++ b/src/model/block_metrics.rs @@ -1,5 +1,7 @@ use opentelemetry::{global, metrics::*}; +use garage_util::sled_counter::SledCountedTree; + /// TableMetrics reference all counter used for metrics pub struct BlockManagerMetrics { pub(crate) _resync_queue_len: ValueObserver, @@ -20,7 +22,7 @@ pub struct BlockManagerMetrics { } impl BlockManagerMetrics { - pub fn new(resync_queue: sled::Tree) -> Self { + pub fn new(resync_queue: SledCountedTree) -> Self { let meter = global::meter("garage_model/block"); Self { _resync_queue_len: meter diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index 89225511..1b351024 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -134,7 +134,11 @@ impl RpcHelper { M: Rpc>, H: EndpointHandler, { - let metric_tags = [KeyValue::new("rpc_endpoint", endpoint.path().to_string())]; + let metric_tags = [ + KeyValue::new("rpc_endpoint", endpoint.path().to_string()), + KeyValue::new("from", format!("{:?}", self.0.our_node_id)), + KeyValue::new("to", format!("{:?}", to)), + ]; let msg_size = rmp_to_vec_all_named(&msg)?.len() as u32; let permit = self @@ -245,6 +249,7 @@ impl RpcHelper { ) }; let mut span = tracer.start(span_name); + span.set_attribute(KeyValue::new("from", format!("{:?}", self.0.our_node_id))); span.set_attribute(KeyValue::new("to", format!("{:?}", to))); span.set_attribute(KeyValue::new("quorum", quorum as i64)); span.set_attribute(KeyValue::new( diff --git a/src/table/data.rs b/src/table/data.rs index 4293e395..ff7965f5 100644 --- a/src/table/data.rs +++ b/src/table/data.rs @@ -7,6 +7,7 @@ use tokio::sync::Notify; use garage_util::data::*; use garage_util::error::*; +use garage_util::sled_counter::SledCountedTree; use garage_rpc::system::System; @@ -27,7 +28,7 @@ pub struct TableData { pub(crate) merkle_tree: sled::Tree, pub(crate) merkle_todo: sled::Tree, pub(crate) merkle_todo_notify: Notify, - pub(crate) gc_todo: sled::Tree, + pub(crate) gc_todo: SledCountedTree, pub(crate) metrics: TableMetrics, } @@ -52,6 +53,7 @@ where let gc_todo = db .open_tree(&format!("{}:gc_todo_v2", F::TABLE_NAME)) .expect("Unable to open DB tree"); + let gc_todo = SledCountedTree::new(gc_todo); let metrics = TableMetrics::new(F::TABLE_NAME, merkle_todo.clone(), gc_todo.clone()); diff --git a/src/table/gc.rs b/src/table/gc.rs index 8d0a5bef..2a05b6ae 100644 --- a/src/table/gc.rs +++ b/src/table/gc.rs @@ -14,6 +14,7 @@ use tokio::sync::watch; use garage_util::data::*; use garage_util::error::*; +use garage_util::sled_counter::SledCountedTree; use garage_util::time::*; use garage_rpc::system::System; @@ -362,7 +363,7 @@ impl GcTodoEntry { } /// Saves the GcTodoEntry in the gc_todo tree - pub(crate) fn save(&self, gc_todo_tree: &sled::Tree) -> Result<(), Error> { + pub(crate) fn save(&self, gc_todo_tree: &SledCountedTree) -> Result<(), Error> { gc_todo_tree.insert(self.todo_table_key(), self.value_hash.as_slice())?; Ok(()) } @@ -372,7 +373,7 @@ impl GcTodoEntry { /// This is usefull to remove a todo entry only under the condition /// that it has not changed since the time it was read, i.e. /// what we have to do is still the same - pub(crate) fn remove_if_equal(&self, gc_todo_tree: &sled::Tree) -> Result<(), Error> { + pub(crate) fn remove_if_equal(&self, gc_todo_tree: &SledCountedTree) -> Result<(), Error> { let _ = gc_todo_tree.compare_and_swap::<_, _, Vec>( &self.todo_table_key()[..], Some(self.value_hash), diff --git a/src/table/metrics.rs b/src/table/metrics.rs index 548bf0d6..752a2a6d 100644 --- a/src/table/metrics.rs +++ b/src/table/metrics.rs @@ -1,5 +1,7 @@ use opentelemetry::{global, metrics::*, KeyValue}; +use garage_util::sled_counter::SledCountedTree; + /// TableMetrics reference all counter used for metrics pub struct TableMetrics { pub(crate) _merkle_todo_len: ValueObserver, @@ -17,7 +19,11 @@ pub struct TableMetrics { pub(crate) sync_items_received: Counter, } impl TableMetrics { - pub fn new(table_name: &'static str, merkle_todo: sled::Tree, gc_todo: sled::Tree) -> Self { + pub fn new( + table_name: &'static str, + merkle_todo: sled::Tree, + gc_todo: SledCountedTree, + ) -> Self { let meter = global::meter(table_name); TableMetrics { _merkle_todo_len: meter diff --git a/src/util/lib.rs b/src/util/lib.rs index 7ed00034..e83fc2e6 100644 --- a/src/util/lib.rs +++ b/src/util/lib.rs @@ -10,6 +10,7 @@ pub mod data; pub mod error; pub mod metrics; pub mod persister; +pub mod sled_counter; pub mod time; pub mod token_bucket; pub mod tranquilizer; diff --git a/src/util/sled_counter.rs b/src/util/sled_counter.rs new file mode 100644 index 00000000..8af04f50 --- /dev/null +++ b/src/util/sled_counter.rs @@ -0,0 +1,92 @@ +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +use sled::{CompareAndSwapError, IVec, Iter, Result, Tree}; + +#[derive(Clone)] +pub struct SledCountedTree(Arc); + +struct SledCountedTreeInternal { + tree: Tree, + len: AtomicUsize, +} + +impl SledCountedTree { + pub fn new(tree: Tree) -> Self { + let len = tree.len(); + Self(Arc::new(SledCountedTreeInternal { + tree, + len: AtomicUsize::new(len), + })) + } + + pub fn len(&self) -> usize { + self.0.len.load(Ordering::Relaxed) + } + + pub fn is_empty(&self) -> bool { + self.0.tree.is_empty() + } + + pub fn get>(&self, key: K) -> Result> { + self.0.tree.get(key) + } + + pub fn iter(&self) -> Iter { + self.0.tree.iter() + } + + // ---- writing functions ---- + + pub fn insert(&self, key: K, value: V) -> Result> + where + K: AsRef<[u8]>, + V: Into, + { + let res = self.0.tree.insert(key, value); + if res == Ok(None) { + self.0.len.fetch_add(1, Ordering::Relaxed); + } + res + } + + pub fn pop_min(&self) -> Result> { + let res = self.0.tree.pop_min(); + if let Ok(Some(_)) = &res { + self.0.len.fetch_sub(1, Ordering::Relaxed); + }; + res + } + + pub fn compare_and_swap( + &self, + key: K, + old: Option, + new: Option, + ) -> Result> + where + K: AsRef<[u8]>, + OV: AsRef<[u8]>, + NV: Into, + { + let old_some = old.is_some(); + let new_some = new.is_some(); + + let res = self.0.tree.compare_and_swap(key, old, new); + + if res == Ok(Ok(())) { + match (old_some, new_some) { + (false, true) => { + self.0.len.fetch_add(1, Ordering::Relaxed); + } + (true, false) => { + self.0.len.fetch_sub(1, Ordering::Relaxed); + } + _ => (), + } + } + res + } +} -- 2.43.0 From a19341b1885b87ab2ac0ee6fa8c19ba2fca5e3fc Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 24 Feb 2022 15:35:40 +0100 Subject: [PATCH 17/19] Add Grafana dashboard for Garage --- ...rafana-garage-dashboard-elasticsearch.json | 2639 +++++++++++++++++ 1 file changed, 2639 insertions(+) create mode 100644 script/telemetry/grafana-garage-dashboard-elasticsearch.json diff --git a/script/telemetry/grafana-garage-dashboard-elasticsearch.json b/script/telemetry/grafana-garage-dashboard-elasticsearch.json new file mode 100644 index 00000000..28648f99 --- /dev/null +++ b/script/telemetry/grafana-garage-dashboard-elasticsearch.json @@ -0,0 +1,2639 @@ +{ + "__inputs": [ + { + "name": "DS_DS_ELASTICSEARCH", + "label": "DS_ELASTICSEARCH", + "description": "", + "type": "datasource", + "pluginId": "elasticsearch", + "pluginName": "Elasticsearch" + } + ], + "__elements": [], + "__requires": [ + { + "type": "datasource", + "id": "elasticsearch", + "name": "Elasticsearch", + "version": "1.0.0" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "8.3.5" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.api_endpoint", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "api_request_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "5", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "API request rate", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.api_endpoint", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "api_request_duration", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric", + "refId": "E", + "timeField": "@timestamp" + } + ], + "title": "API request duration", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.api_endpoint", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "labels.status_code", + "id": "7", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "api_error_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "5", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "API error rate", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 8 + }, + "id": 18, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.method", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "web_request_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "5", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Web request rate", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 8 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.method", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "web_request_duration", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric", + "refId": "E", + "timeField": "@timestamp" + } + ], + "title": "Web request duration", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.method", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "labels.status_code", + "id": "7", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "web_error_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "5", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Web error rate", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 16 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.table_name", + "id": "5", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "table_get_request_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "B", + "timeField": "@timestamp" + } + ], + "title": "Table get and get_range calls", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 16 + }, + "id": 14, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.table_name", + "id": "5", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "table_put_request_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "4", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "C", + "timeField": "@timestamp" + } + ], + "title": "Table put and put_many calls", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 16 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "block_bytes_read", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "4", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "C", + "timeField": "@timestamp" + } + ], + "title": "Disk reads", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 16 + }, + "id": 15, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "block_bytes_written", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "4", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Disk writes", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": -1, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "block_resync_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "5", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "Resyncs", + "timeField": "@timestamp" + } + ], + "title": "Resyncs", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "trimEdges": "5" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "block_resync_queue_length", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric", + "refId": "Resync queue", + "timeField": "@timestamp" + } + ], + "title": "Resync queue", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 23 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.table_name", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "table_merkle_updater_todo_queue_length", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:\"metric\"", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Table Merkle todo queue", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 23 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.table_name", + "id": "3", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "table_gc_todo_queue_length", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:\"metric\"", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Table GC todo queue", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 29 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.rpc_endpoint", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "rpc_request_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "5", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "RPC counters", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 29 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.rpc_endpoint", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "4" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "rpc_duration", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "RPC duration", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 29 + }, + "id": 21, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.to", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "rpc_netapp_error_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "5", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "RPC errors (network layer)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 29 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.rpc_endpoint", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "5" + }, + "type": "terms" + }, + { + "field": "labels.to", + "id": "7", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "5" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "rpc_garage_error_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "5", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "RPC errors (Garage errors)", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 38 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "admin_http_requests_total", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "3", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Admin request rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 38 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "rpc_duration", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + }, + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "admin_http_request_duration_seconds", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric", + "refId": "B", + "timeField": "@timestamp" + }, + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "block_read_duration", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric", + "refId": "C", + "timeField": "@timestamp" + }, + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "block_resync_duration", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric", + "refId": "D", + "timeField": "@timestamp" + }, + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "hide": false, + "metrics": [ + { + "field": "block_write_duration", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric", + "refId": "E", + "timeField": "@timestamp" + } + ], + "title": "Event duration", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "block_corruption_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "3", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Data corruptions detected", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 34, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Garage", + "uid": "ODT8K4B7e", + "version": 12, + "weekStart": "" +} \ No newline at end of file -- 2.43.0 From 9b2b531f4d8d62897b0dc3f8691608a9298a2f85 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 10 Mar 2022 10:51:40 +0100 Subject: [PATCH 18/19] Make admin server optional --- src/garage/server.rs | 18 ++++++++++++------ src/util/config.rs | 5 +++-- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/src/garage/server.rs b/src/garage/server.rs index 1e276e9e..37c3535e 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -68,10 +68,14 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { wait_from(watch_cancel.clone()), )); - info!("Configure and run admin web server..."); - let admin_server = tokio::spawn( - admin_server_init.run(config.admin.api_bind_addr, wait_from(watch_cancel.clone())), - ); + let admin_server = if let Some(admin_bind_addr) = config.admin.api_bind_addr { + info!("Configure and run admin web server..."); + Some(tokio::spawn( + admin_server_init.run(admin_bind_addr, wait_from(watch_cancel.clone())), + )) + } else { + None + }; // Stuff runs @@ -82,8 +86,10 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { if let Err(e) = web_server.await? { warn!("Web server exited with error: {}", e); } - if let Err(e) = admin_server.await? { - warn!("Admin web server exited with error: {}", e); + if let Some(a) = admin_server { + if let Err(e) = a.await? { + warn!("Admin web server exited with error: {}", e); + } } // Remove RPC handlers for system to break reference cycles diff --git a/src/util/config.rs b/src/util/config.rs index 7e8d4ba6..f74a62d0 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -75,6 +75,7 @@ pub struct Config { pub s3_web: WebConfig, /// Configuration for the admin API endpoint + #[serde(default = "Default::default")] pub admin: AdminConfig, } @@ -100,10 +101,10 @@ pub struct WebConfig { } /// Configuration for the admin and monitoring HTTP API -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, Default)] pub struct AdminConfig { /// Address and port to bind for admin API serving - pub api_bind_addr: SocketAddr, + pub api_bind_addr: Option, /// OTLP server to where to export traces pub trace_sink: Option, } -- 2.43.0 From bfb4353df50f31fc3b00a9833c09b8de16b471fc Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 14 Mar 2022 10:55:30 +0100 Subject: [PATCH 19/19] Update Grafana dashboard --- ...rafana-garage-dashboard-elasticsearch.json | 1041 +++++++++++------ 1 file changed, 700 insertions(+), 341 deletions(-) diff --git a/script/telemetry/grafana-garage-dashboard-elasticsearch.json b/script/telemetry/grafana-garage-dashboard-elasticsearch.json index 28648f99..456ff84d 100644 --- a/script/telemetry/grafana-garage-dashboard-elasticsearch.json +++ b/script/telemetry/grafana-garage-dashboard-elasticsearch.json @@ -21,7 +21,7 @@ "type": "grafana", "id": "grafana", "name": "Grafana", - "version": "8.3.5" + "version": "8.4.3" }, { "type": "panel", @@ -53,6 +53,7 @@ "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, + "iteration": 1647250457308, "links": [], "liveNow": false, "panels": [ @@ -116,6 +117,7 @@ "y": 0 }, "id": 8, + "interval": ">$interval", "options": { "legend": { "calcs": [], @@ -123,7 +125,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -146,7 +149,8 @@ "id": "2", "settings": { "interval": "auto", - "min_doc_count": "0" + "min_doc_count": "0", + "trimEdges": "1" }, "type": "date_histogram" } @@ -170,7 +174,7 @@ "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } @@ -225,7 +229,8 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, @@ -236,6 +241,7 @@ "y": 0 }, "id": 11, + "interval": ">$interval", "options": { "legend": { "calcs": [], @@ -243,7 +249,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -282,7 +289,7 @@ "type": "avg" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "E", "timeField": "@timestamp" } @@ -297,7 +304,7 @@ "mode": "palette-classic" }, "custom": { - "axisLabel": "", + "axisLabel": "Error rate", "axisPlacement": "auto", "axisSoftMin": 0, "barAlignment": 0, @@ -350,6 +357,7 @@ "y": 0 }, "id": 17, + "interval": ">$interval", "options": { "legend": { "calcs": [], @@ -357,7 +365,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -391,7 +400,8 @@ "id": "2", "settings": { "interval": "auto", - "min_doc_count": "0" + "min_doc_count": "0", + "trimEdges": "1" }, "type": "date_histogram" } @@ -412,10 +422,13 @@ "hide": false, "id": "5", "pipelineAgg": "1", + "settings": { + "unit": "second" + }, "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } @@ -472,7 +485,8 @@ "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, @@ -483,6 +497,7 @@ "y": 8 }, "id": 18, + "interval": ">$interval", "options": { "legend": { "calcs": [], @@ -490,7 +505,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -504,7 +520,7 @@ "min_doc_count": "1", "order": "desc", "orderBy": "_term", - "size": "10" + "size": "15" }, "type": "terms" }, @@ -534,10 +550,13 @@ "hide": false, "id": "5", "pipelineAgg": "1", + "settings": { + "unit": "second" + }, "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } @@ -592,7 +611,8 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, @@ -603,6 +623,7 @@ "y": 8 }, "id": 19, + "interval": ">$interval", "options": { "legend": { "calcs": [], @@ -610,7 +631,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -649,7 +671,7 @@ "type": "avg" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "E", "timeField": "@timestamp" } @@ -706,7 +728,8 @@ "value": 80 } ] - } + }, + "unit": "reqps" }, "overrides": [] }, @@ -724,7 +747,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -758,7 +782,8 @@ "id": "2", "settings": { "interval": "auto", - "min_doc_count": "0" + "min_doc_count": "0", + "trimEdges": "1" }, "type": "date_histogram" } @@ -782,7 +807,7 @@ "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } @@ -857,7 +882,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -879,7 +905,8 @@ "field": "@timestamp", "id": "2", "settings": { - "interval": "auto" + "interval": "auto", + "trimEdges": "1" }, "type": "date_histogram" } @@ -903,7 +930,7 @@ "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "B", "timeField": "@timestamp" } @@ -978,7 +1005,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -1001,7 +1029,8 @@ "id": "2", "settings": { "interval": "auto", - "min_doc_count": "0" + "min_doc_count": "0", + "trimEdges": "1" }, "type": "date_histogram" } @@ -1025,7 +1054,7 @@ "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "C", "timeField": "@timestamp" } @@ -1081,7 +1110,8 @@ "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, @@ -1099,7 +1129,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -1111,7 +1142,7 @@ "id": "2", "settings": { "interval": "auto", - "trimEdges": "0" + "trimEdges": "1" }, "type": "date_histogram" } @@ -1133,10 +1164,13 @@ "hide": false, "id": "4", "pipelineAgg": "1", + "settings": { + "unit": "second" + }, "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "C", "timeField": "@timestamp" } @@ -1192,7 +1226,8 @@ "value": 80 } ] - } + }, + "unit": "binBps" }, "overrides": [] }, @@ -1203,6 +1238,7 @@ "y": 16 }, "id": 15, + "interval": ">$interval", "options": { "legend": { "calcs": [], @@ -1210,7 +1246,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -1243,10 +1280,13 @@ "hide": false, "id": "4", "pipelineAgg": "1", + "settings": { + "unit": "second" + }, "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } @@ -1314,6 +1354,7 @@ "y": 23 }, "id": 2, + "interval": ">$interval", "options": { "legend": { "calcs": [], @@ -1321,7 +1362,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -1332,7 +1374,8 @@ "field": "@timestamp", "id": "2", "settings": { - "interval": "auto" + "interval": "auto", + "trimEdges": "0" }, "type": "date_histogram" } @@ -1352,10 +1395,13 @@ "field": "1", "id": "5", "pipelineAgg": "1", + "settings": { + "unit": "second" + }, "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "Resyncs", "timeField": "@timestamp" } @@ -1398,6 +1444,7 @@ } }, "mappings": [], + "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -1428,7 +1475,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -1440,7 +1488,7 @@ "id": "2", "settings": { "interval": "auto", - "trimEdges": "5" + "trimEdges": "2" }, "type": "date_histogram" } @@ -1457,7 +1505,7 @@ "type": "avg" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "Resync queue", "timeField": "@timestamp" } @@ -1466,10 +1514,6 @@ "type": "timeseries" }, { - "datasource": { - "type": "elasticsearch", - "uid": "${DS_DS_ELASTICSEARCH}" - }, "fieldConfig": { "defaults": { "color": { @@ -1510,10 +1554,6 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } @@ -1527,6 +1567,7 @@ "y": 23 }, "id": 4, + "interval": ">$interval", "options": { "legend": { "calcs": [], @@ -1534,7 +1575,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -1573,7 +1615,7 @@ "type": "avg" } ], - "query": "processor.event:\"metric\"", + "query": "processor.event:\"metric\" AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } @@ -1646,7 +1688,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -1670,7 +1713,7 @@ "settings": { "interval": "auto", "min_doc_count": "0", - "trimEdges": "0" + "trimEdges": "2" }, "type": "date_histogram" } @@ -1686,7 +1729,7 @@ "type": "avg" } ], - "query": "processor.event:\"metric\"", + "query": "processor.event:\"metric\" AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } @@ -1703,7 +1746,6 @@ "custom": { "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -1737,13 +1779,10 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] - } + }, + "unit": "short" }, "overrides": [] }, @@ -1753,7 +1792,7 @@ "x": 0, "y": 29 }, - "id": 13, + "id": 24, "options": { "legend": { "calcs": [], @@ -1761,30 +1800,20 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ { "alias": "", "bucketAggs": [ - { - "field": "labels.rpc_endpoint", - "id": "6", - "settings": { - "min_doc_count": "1", - "order": "desc", - "orderBy": "_term", - "size": "10" - }, - "type": "terms" - }, { "field": "@timestamp", "id": "2", "settings": { "interval": "auto", - "min_doc_count": "0" + "trimEdges": "0" }, "type": "date_histogram" } @@ -1795,25 +1824,28 @@ }, "metrics": [ { - "field": "rpc_request_counter", + "field": "block_resync_error_counter", "hide": true, "id": "1", + "settings": {}, "type": "sum" }, { "field": "1", - "hide": false, - "id": "5", + "id": "3", "pipelineAgg": "1", + "settings": { + "unit": "second" + }, "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } ], - "title": "RPC counters", + "title": "Resync errors", "type": "timeseries" }, { @@ -1825,7 +1857,6 @@ "custom": { "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -1875,7 +1906,7 @@ "x": 6, "y": 29 }, - "id": 16, + "id": 25, "options": { "legend": { "calcs": [], @@ -1883,31 +1914,20 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ { "alias": "", "bucketAggs": [ - { - "field": "labels.rpc_endpoint", - "id": "6", - "settings": { - "min_doc_count": "1", - "order": "desc", - "orderBy": "_term", - "size": "10" - }, - "type": "terms" - }, { "field": "@timestamp", "id": "2", "settings": { "interval": "auto", - "min_doc_count": "0", - "trimEdges": "4" + "trimEdges": "2" }, "type": "date_histogram" } @@ -1918,17 +1938,18 @@ }, "metrics": [ { - "field": "rpc_duration", + "field": "block_resync_errored_blocks", + "hide": false, "id": "1", - "type": "avg" + "type": "sum" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } ], - "title": "RPC duration", + "title": "Blocks in error status", "type": "timeseries" }, { @@ -1940,7 +1961,6 @@ "custom": { "axisLabel": "", "axisPlacement": "auto", - "axisSoftMin": 0, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -1990,6 +2010,117 @@ "x": 12, "y": 29 }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "block_corruption_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "3", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric AND labels.job: garage", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Data corruptions detected", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "RPC error rate (per second)", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 18, + "y": 29 + }, "id": 21, "options": { "legend": { @@ -1998,7 +2129,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -2021,7 +2153,8 @@ "id": "2", "settings": { "interval": "auto", - "min_doc_count": "0" + "min_doc_count": "0", + "trimEdges": "1" }, "type": "date_histogram" } @@ -2042,10 +2175,13 @@ "hide": false, "id": "5", "pipelineAgg": "1", + "settings": { + "unit": "second" + }, "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } @@ -2053,6 +2189,358 @@ "title": "RPC errors (network layer)", "type": "timeseries" }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 0, + "y": 38 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.rpc_endpoint", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "1" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "rpc_request_counter", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "hide": false, + "id": "5", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric AND labels.job: garage", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "RPC counters", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 6, + "y": 38 + }, + "id": 16, + "interval": ">$interval", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "labels.rpc_endpoint", + "id": "6", + "settings": { + "min_doc_count": "1", + "order": "desc", + "orderBy": "_term", + "size": "10" + }, + "type": "terms" + }, + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "min_doc_count": "0", + "trimEdges": "0" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "rpc_duration", + "id": "1", + "type": "avg" + } + ], + "query": "processor.event:metric AND labels.job: garage", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Average RPC duration", + "type": "timeseries" + }, + { + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 6, + "x": 12, + "y": 38 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "alias": "", + "bucketAggs": [ + { + "field": "@timestamp", + "id": "2", + "settings": { + "interval": "auto", + "trimEdges": "2" + }, + "type": "date_histogram" + } + ], + "datasource": { + "type": "elasticsearch", + "uid": "${DS_DS_ELASTICSEARCH}" + }, + "metrics": [ + { + "field": "admin_http_requests_total", + "hide": true, + "id": "1", + "type": "sum" + }, + { + "field": "1", + "id": "3", + "pipelineAgg": "1", + "type": "derivative" + } + ], + "query": "processor.event:metric AND labels.job: garage", + "refId": "A", + "timeField": "@timestamp" + } + ], + "title": "Admin request rate", + "type": "timeseries" + }, { "fieldConfig": { "defaults": { @@ -2110,7 +2598,7 @@ "h": 9, "w": 6, "x": 18, - "y": 29 + "y": 38 }, "id": 22, "options": { @@ -2120,7 +2608,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -2178,7 +2667,7 @@ "type": "derivative" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" } @@ -2221,7 +2710,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -2234,7 +2722,8 @@ "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, @@ -2242,120 +2731,10 @@ "h": 9, "w": 6, "x": 0, - "y": 38 - }, - "id": 9, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "alias": "", - "bucketAggs": [ - { - "field": "@timestamp", - "id": "2", - "settings": { - "interval": "auto" - }, - "type": "date_histogram" - } - ], - "datasource": { - "type": "elasticsearch", - "uid": "${DS_DS_ELASTICSEARCH}" - }, - "metrics": [ - { - "field": "admin_http_requests_total", - "hide": true, - "id": "1", - "type": "sum" - }, - { - "field": "1", - "id": "3", - "pipelineAgg": "1", - "type": "derivative" - } - ], - "query": "processor.event:metric", - "refId": "A", - "timeField": "@timestamp" - } - ], - "title": "Admin request rate", - "type": "timeseries" - }, - { - "datasource": { - "type": "elasticsearch", - "uid": "${DS_DS_ELASTICSEARCH}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 6, - "x": 6, - "y": 38 + "y": 47 }, "id": 10, + "interval": ">$interval", "options": { "legend": { "calcs": [], @@ -2363,7 +2742,8 @@ "placement": "bottom" }, "tooltip": { - "mode": "single" + "mode": "single", + "sort": "none" } }, "targets": [ @@ -2390,7 +2770,7 @@ "type": "avg" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "A", "timeField": "@timestamp" }, @@ -2401,7 +2781,8 @@ "field": "@timestamp", "id": "2", "settings": { - "interval": "auto" + "interval": "auto", + "trimEdges": "2" }, "type": "date_histogram" } @@ -2418,7 +2799,7 @@ "type": "avg" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "B", "timeField": "@timestamp" }, @@ -2446,7 +2827,7 @@ "type": "avg" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "C", "timeField": "@timestamp" }, @@ -2474,7 +2855,7 @@ "type": "avg" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "D", "timeField": "@timestamp" }, @@ -2502,138 +2883,116 @@ "type": "avg" } ], - "query": "processor.event:metric", + "query": "processor.event:metric AND labels.job: garage", "refId": "E", "timeField": "@timestamp" } ], - "title": "Event duration", - "type": "timeseries" - }, - { - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 38 - }, - "id": 23, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ - { - "alias": "", - "bucketAggs": [ - { - "field": "@timestamp", - "id": "2", - "settings": { - "interval": "auto" - }, - "type": "date_histogram" - } - ], - "datasource": { - "type": "elasticsearch", - "uid": "${DS_DS_ELASTICSEARCH}" - }, - "metrics": [ - { - "field": "block_corruption_counter", - "hide": true, - "id": "1", - "type": "sum" - }, - { - "field": "1", - "id": "3", - "pipelineAgg": "1", - "type": "derivative" - } - ], - "query": "processor.event:metric", - "refId": "A", - "timeField": "@timestamp" - } - ], - "title": "Data corruptions detected", + "title": "Average event duration", "type": "timeseries" } ], "refresh": "10s", - "schemaVersion": 34, + "schemaVersion": 35, "style": "dark", "tags": [], "templating": { - "list": [] + "list": [ + { + "auto": true, + "auto_count": 50, + "auto_min": "10s", + "current": { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "30s,1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] }, "time": { - "from": "now-15m", + "from": "now-24h", "to": "now" }, "timepicker": {}, "timezone": "", - "title": "Garage", + "title": "Garage - Max", "uid": "ODT8K4B7e", - "version": 12, + "version": 7, "weekStart": "" } \ No newline at end of file -- 2.43.0