From 633958c7b1ce9c83df5159051fd299b484d0d797 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 5 May 2022 10:29:45 +0200 Subject: [PATCH 01/46] Refactor admin API to be in api/admin and use common code --- Cargo.lock | 26 ++--- Cargo.toml | 1 - src/admin/Cargo.toml | 29 ----- src/admin/lib.rs | 6 - src/admin/metrics.rs | 146 ------------------------- src/api/Cargo.toml | 3 + src/api/admin/api_server.rs | 128 ++++++++++++++++++++++ src/api/admin/mod.rs | 2 + src/api/admin/router.rs | 59 ++++++++++ src/api/lib.rs | 1 + src/garage/Cargo.toml | 6 +- src/garage/main.rs | 1 + src/garage/server.rs | 35 +++--- src/{admin => garage}/tracing_setup.rs | 0 src/rpc/Cargo.toml | 1 - src/util/config.rs | 4 + 16 files changed, 228 insertions(+), 220 deletions(-) delete mode 100644 src/admin/Cargo.toml delete mode 100644 src/admin/lib.rs delete mode 100644 src/admin/metrics.rs create mode 100644 src/api/admin/api_server.rs create mode 100644 src/api/admin/mod.rs create mode 100644 src/api/admin/router.rs rename src/{admin => garage}/tracing_setup.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index de1ae5cd..3f253b5b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -839,7 +839,6 @@ dependencies = [ "chrono", "futures", "futures-util", - "garage_admin", "garage_api", "garage_model 0.7.0", "garage_rpc 0.7.0", @@ -853,7 +852,11 @@ dependencies = [ "hyper", "kuska-sodiumoxide", "netapp 0.4.4", + "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-prometheus", "pretty_env_logger", + "prometheus", "rand 0.8.5", "rmp-serde 0.15.5", "serde", @@ -868,23 +871,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "garage_admin" -version = "0.7.0" -dependencies = [ - "futures", - "futures-util", - "garage_util 0.7.0", - "hex", - "http", - "hyper", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry-prometheus", - "prometheus", - "tracing", -] - [[package]] name = "garage_api" version = "0.7.0" @@ -914,8 +900,11 @@ dependencies = [ "multer", "nom", "opentelemetry", + "opentelemetry-otlp", + "opentelemetry-prometheus", "percent-encoding", "pin-project 1.0.10", + "prometheus", "quick-xml", "roxmltree", "serde", @@ -1040,7 +1029,6 @@ dependencies = [ "bytes 1.1.0", "futures", "futures-util", - "garage_admin", "garage_util 0.7.0", "gethostname", "hex", diff --git a/Cargo.toml b/Cargo.toml index cfc48113..3d42b11f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,6 @@ members = [ "src/table", "src/block", "src/model", - "src/admin", "src/api", "src/web", "src/garage" diff --git a/src/admin/Cargo.toml b/src/admin/Cargo.toml deleted file mode 100644 index 2db4bb08..00000000 --- a/src/admin/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "garage_admin" -version = "0.7.0" -authors = ["Maximilien Richer "] -edition = "2018" -license = "AGPL-3.0" -description = "Administration and metrics REST HTTP server for Garage" -repository = "https://git.deuxfleurs.fr/Deuxfleurs/garage" - -[lib] -path = "lib.rs" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -garage_util = { version = "0.7.0", path = "../util" } - -hex = "0.4" - -futures = "0.3" -futures-util = "0.3" -http = "0.2" -hyper = "0.14" -tracing = "0.1.30" - -opentelemetry = { version = "0.17", features = [ "rt-tokio" ] } -opentelemetry-prometheus = "0.10" -opentelemetry-otlp = "0.10" -prometheus = "0.13" diff --git a/src/admin/lib.rs b/src/admin/lib.rs deleted file mode 100644 index b5b0775b..00000000 --- a/src/admin/lib.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Crate for handling the admin and metric HTTP APIs -#[macro_use] -extern crate tracing; - -pub mod metrics; -pub mod tracing_setup; diff --git a/src/admin/metrics.rs b/src/admin/metrics.rs deleted file mode 100644 index 7edc36c6..00000000 --- a/src/admin/metrics.rs +++ /dev/null @@ -1,146 +0,0 @@ -use std::convert::Infallible; -use std::net::SocketAddr; -use std::sync::Arc; -use std::time::SystemTime; - -use futures::future::*; -use hyper::{ - header::CONTENT_TYPE, - service::{make_service_fn, service_fn}, - Body, Method, Request, Response, Server, -}; - -use opentelemetry::{ - global, - metrics::{BoundCounter, BoundValueRecorder}, - trace::{FutureExt, TraceContextExt, Tracer}, - Context, -}; -use opentelemetry_prometheus::PrometheusExporter; - -use prometheus::{Encoder, TextEncoder}; - -use garage_util::error::Error as GarageError; -use garage_util::metrics::*; - -// serve_req on metric endpoint -async fn serve_req( - req: Request, - admin_server: Arc, -) -> Result, hyper::Error> { - debug!("Receiving request at path {}", req.uri()); - let request_start = SystemTime::now(); - - admin_server.metrics.http_counter.add(1); - - let response = match (req.method(), req.uri().path()) { - (&Method::GET, "/metrics") => { - let mut buffer = vec![]; - let encoder = TextEncoder::new(); - - let tracer = opentelemetry::global::tracer("garage"); - let metric_families = tracer.in_span("admin/gather_metrics", |_| { - admin_server.exporter.registry().gather() - }); - - encoder.encode(&metric_families, &mut buffer).unwrap(); - admin_server - .metrics - .http_body_gauge - .record(buffer.len() as u64); - - Response::builder() - .status(200) - .header(CONTENT_TYPE, encoder.format_type()) - .body(Body::from(buffer)) - .unwrap() - } - _ => Response::builder() - .status(404) - .body(Body::from("Not implemented")) - .unwrap(), - }; - - admin_server - .metrics - .http_req_histogram - .record(request_start.elapsed().map_or(0.0, |d| d.as_secs_f64())); - Ok(response) -} - -// AdminServer hold the admin server internal admin_server and the metric exporter -pub struct AdminServer { - exporter: PrometheusExporter, - metrics: AdminServerMetrics, -} - -// GarageMetricadmin_server holds the metrics counter definition for Garage -// FIXME: we would rather have that split up among the different libraries? -struct AdminServerMetrics { - http_counter: BoundCounter, - http_body_gauge: BoundValueRecorder, - http_req_histogram: BoundValueRecorder, -} - -impl AdminServer { - /// init initilialize the AdminServer and background metric server - pub fn init() -> AdminServer { - let exporter = opentelemetry_prometheus::exporter().init(); - let meter = global::meter("garage/admin_server"); - AdminServer { - exporter, - metrics: AdminServerMetrics { - http_counter: meter - .u64_counter("admin.http_requests_total") - .with_description("Total number of HTTP requests made.") - .init() - .bind(&[]), - http_body_gauge: meter - .u64_value_recorder("admin.http_response_size_bytes") - .with_description("The metrics HTTP response sizes in bytes.") - .init() - .bind(&[]), - http_req_histogram: meter - .f64_value_recorder("admin.http_request_duration_seconds") - .with_description("The HTTP request latencies in seconds.") - .init() - .bind(&[]), - }, - } - } - /// run execute the admin server on the designated HTTP port and listen for requests - pub async fn run( - self, - bind_addr: SocketAddr, - shutdown_signal: impl Future, - ) -> Result<(), GarageError> { - let admin_server = Arc::new(self); - // For every connection, we must make a `Service` to handle all - // incoming HTTP requests on said connection. - let make_svc = make_service_fn(move |_conn| { - let admin_server = admin_server.clone(); - // This is the `Service` that will handle the connection. - // `service_fn` is a helper to convert a function that - // returns a Response into a `Service`. - async move { - Ok::<_, Infallible>(service_fn(move |req| { - let tracer = opentelemetry::global::tracer("garage"); - let span = tracer - .span_builder("admin/request") - .with_trace_id(gen_trace_id()) - .start(&tracer); - - serve_req(req, admin_server.clone()) - .with_context(Context::current_with_span(span)) - })) - } - }); - - let server = Server::bind(&bind_addr).serve(make_svc); - let graceful = server.with_graceful_shutdown(shutdown_signal); - info!("Admin server listening on http://{}", bind_addr); - - graceful.await?; - Ok(()) - } -} diff --git a/src/api/Cargo.toml b/src/api/Cargo.toml index 29b26e5e..db77cf38 100644 --- a/src/api/Cargo.toml +++ b/src/api/Cargo.toml @@ -54,6 +54,9 @@ quick-xml = { version = "0.21", features = [ "serialize" ] } url = "2.1" opentelemetry = "0.17" +opentelemetry-prometheus = "0.10" +opentelemetry-otlp = "0.10" +prometheus = "0.13" [features] k2v = [ "garage_util/k2v", "garage_model/k2v" ] diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs new file mode 100644 index 00000000..836b5158 --- /dev/null +++ b/src/api/admin/api_server.rs @@ -0,0 +1,128 @@ +use std::sync::Arc; + +use async_trait::async_trait; + +use futures::future::Future; +use http::header::CONTENT_TYPE; +use hyper::{Body, Request, Response}; + +use opentelemetry::trace::{SpanRef, Tracer}; +use opentelemetry_prometheus::PrometheusExporter; +use prometheus::{Encoder, TextEncoder}; + +use garage_model::garage::Garage; +use garage_util::error::Error as GarageError; + +use crate::error::*; +use crate::generic_server::*; + +use crate::admin::router::{Authorization, Endpoint}; + +pub struct AdminApiServer { + garage: Arc, + exporter: PrometheusExporter, + metrics_token: Option, + admin_token: Option, +} + +impl AdminApiServer { + pub fn new(garage: Arc) -> Self { + let exporter = opentelemetry_prometheus::exporter().init(); + let cfg = &garage.config.admin; + let metrics_token = cfg + .metrics_token + .as_ref() + .map(|tok| format!("Bearer {}", tok)); + let admin_token = cfg + .admin_token + .as_ref() + .map(|tok| format!("Bearer {}", tok)); + Self { + garage, + exporter, + metrics_token, + admin_token, + } + } + + pub async fn run(self, shutdown_signal: impl Future) -> Result<(), GarageError> { + if let Some(bind_addr) = self.garage.config.admin.api_bind_addr { + let region = self.garage.config.s3_api.s3_region.clone(); + ApiServer::new(region, self) + .run_server(bind_addr, shutdown_signal) + .await + } else { + Ok(()) + } + } + + fn handle_metrics(&self) -> Result, Error> { + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + + let tracer = opentelemetry::global::tracer("garage"); + let metric_families = tracer.in_span("admin/gather_metrics", |_| { + self.exporter.registry().gather() + }); + + encoder + .encode(&metric_families, &mut buffer) + .ok_or_internal_error("Could not serialize metrics")?; + + Ok(Response::builder() + .status(200) + .header(CONTENT_TYPE, encoder.format_type()) + .body(Body::from(buffer))?) + } +} + +#[async_trait] +impl ApiHandler for AdminApiServer { + const API_NAME: &'static str = "admin"; + const API_NAME_DISPLAY: &'static str = "Admin"; + + type Endpoint = Endpoint; + + fn parse_endpoint(&self, req: &Request) -> Result { + Endpoint::from_request(req) + } + + async fn handle( + &self, + req: Request, + endpoint: Endpoint, + ) -> Result, Error> { + let expected_auth_header = match endpoint.authorization_type() { + Authorization::MetricsToken => self.metrics_token.as_ref(), + Authorization::AdminToken => self.admin_token.as_ref(), + }; + + if let Some(h) = expected_auth_header { + match req.headers().get("Authorization") { + None => Err(Error::Forbidden( + "Authorization token must be provided".into(), + )), + Some(v) if v.to_str().map(|hv| hv == h).unwrap_or(false) => Ok(()), + _ => Err(Error::Forbidden( + "Invalid authorization token provided".into(), + )), + }?; + } + + match endpoint { + Endpoint::Metrics => self.handle_metrics(), + _ => Err(Error::NotImplemented(format!( + "Admin endpoint {} not implemented yet", + endpoint.name() + ))), + } + } +} + +impl ApiEndpoint for Endpoint { + fn name(&self) -> &'static str { + Endpoint::name(self) + } + + fn add_span_attributes(&self, _span: SpanRef<'_>) {} +} diff --git a/src/api/admin/mod.rs b/src/api/admin/mod.rs new file mode 100644 index 00000000..ff2cf4b1 --- /dev/null +++ b/src/api/admin/mod.rs @@ -0,0 +1,2 @@ +pub mod api_server; +mod router; diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs new file mode 100644 index 00000000..d0b30fc1 --- /dev/null +++ b/src/api/admin/router.rs @@ -0,0 +1,59 @@ +use crate::error::*; + +use hyper::{Method, Request}; + +use crate::router_macros::router_match; + +pub enum Authorization { + MetricsToken, + AdminToken, +} + +router_match! {@func + +/// List of all Admin API endpoints. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Endpoint { + Metrics, + Options, + GetClusterStatus, + GetClusterLayout, + UpdateClusterLayout, + ApplyClusterLayout, + RevertClusterLayout, +}} + +impl Endpoint { + /// Determine which S3 endpoint a request is for using the request, and a bucket which was + /// possibly extracted from the Host header. + /// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets + pub fn from_request(req: &Request) -> Result { + let path = req.uri().path(); + + use Endpoint::*; + let res = match (req.method(), path) { + (&Method::OPTIONS, _) => Options, + (&Method::GET, "/metrics") => Metrics, + (&Method::GET, "/status") => GetClusterStatus, + (&Method::GET, "/layout") => GetClusterLayout, + (&Method::POST, "/layout") => UpdateClusterLayout, + (&Method::POST, "/layout/apply") => ApplyClusterLayout, + (&Method::POST, "/layout/revert") => RevertClusterLayout, + (m, p) => { + return Err(Error::BadRequest(format!( + "Unknown API endpoint: {} {}", + m, p + ))) + } + }; + + Ok(res) + } + /// Get the kind of authorization which is required to perform the operation. + pub fn authorization_type(&self) -> Authorization { + match self { + Self::Metrics => Authorization::MetricsToken, + _ => Authorization::AdminToken, + } + } +} diff --git a/src/api/lib.rs b/src/api/lib.rs index 0078f7b5..5c522799 100644 --- a/src/api/lib.rs +++ b/src/api/lib.rs @@ -12,6 +12,7 @@ mod router_macros; /// This mode is public only to help testing. Don't expect stability here pub mod signature; +pub mod admin; #[cfg(feature = "k2v")] pub mod k2v; pub mod s3; diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 3b69d7bc..59566358 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -27,7 +27,6 @@ garage_rpc = { version = "0.7.0", path = "../rpc" } garage_table = { version = "0.7.0", path = "../table" } garage_util = { version = "0.7.0", path = "../util" } garage_web = { version = "0.7.0", path = "../web" } -garage_admin = { version = "0.7.0", path = "../admin" } bytes = "1.0" git-version = "0.3.4" @@ -54,6 +53,11 @@ tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi #netapp = { version = "0.4", path = "../../../netapp" } netapp = "0.4" +opentelemetry = { version = "0.17", features = [ "rt-tokio" ] } +opentelemetry-prometheus = "0.10" +opentelemetry-otlp = "0.10" +prometheus = "0.13" + [dev-dependencies] aws-sdk-s3 = "0.8" chrono = "0.4" diff --git a/src/garage/main.rs b/src/garage/main.rs index e898e680..69ab1147 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -8,6 +8,7 @@ mod admin; mod cli; mod repair; mod server; +mod tracing_setup; use std::net::SocketAddr; use std::path::PathBuf; diff --git a/src/garage/server.rs b/src/garage/server.rs index 24bb25b3..4c0f8653 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -6,8 +6,7 @@ use garage_util::background::*; use garage_util::config::*; use garage_util::error::Error; -use garage_admin::metrics::*; -use garage_admin::tracing_setup::*; +use garage_api::admin::api_server::AdminApiServer; use garage_api::s3::api_server::S3ApiServer; use garage_model::garage::Garage; use garage_web::run_web_server; @@ -16,6 +15,7 @@ use garage_web::run_web_server; use garage_api::k2v::api_server::K2VApiServer; use crate::admin::*; +use crate::tracing_setup::*; async fn wait_from(mut chan: watch::Receiver) { while !*chan.borrow() { @@ -39,9 +39,6 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { .open() .expect("Unable to open sled DB"); - info!("Initialize admin web server and metric backend..."); - let admin_server_init = AdminServer::init(); - info!("Initializing background runner..."); let watch_cancel = netapp::util::watch_ctrl_c(); let (background, await_background_done) = BackgroundRunner::new(16, watch_cancel.clone()); @@ -54,6 +51,9 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { init_tracing(&export_to, garage.system.id)?; } + info!("Initialize Admin API server and metrics collector..."); + let admin_server = AdminApiServer::new(garage.clone()); + let run_system = tokio::spawn(garage.system.clone().run(watch_cancel.clone())); info!("Create admin RPC handler..."); @@ -80,32 +80,32 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { wait_from(watch_cancel.clone()), )); - let admin_server = if let Some(admin_bind_addr) = config.admin.api_bind_addr { - info!("Configure and run admin web server..."); - Some(tokio::spawn( - admin_server_init.run(admin_bind_addr, wait_from(watch_cancel.clone())), - )) - } else { - None - }; + info!("Initializing Admin server..."); + let admin_server = tokio::spawn(admin_server.run(wait_from(watch_cancel.clone()))); // Stuff runs // When a cancel signal is sent, stuff stops if let Err(e) = s3_api_server.await? { warn!("S3 API server exited with error: {}", e); + } else { + info!("S3 API server exited without error."); } #[cfg(feature = "k2v")] if let Err(e) = k2v_api_server.await? { warn!("K2V API server exited with error: {}", e); + } else { + info!("K2V API server exited without error."); } if let Err(e) = web_server.await? { warn!("Web server exited with error: {}", e); + } else { + info!("Web server exited without error."); } - if let Some(a) = admin_server { - if let Err(e) = a.await? { - warn!("Admin web server exited with error: {}", e); - } + if let Err(e) = admin_server.await? { + warn!("Admin web server exited with error: {}", e); + } else { + info!("Admin API server exited without error."); } // Remove RPC handlers for system to break reference cycles @@ -113,6 +113,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { // Await for netapp RPC system to end run_system.await?; + info!("Netapp exited"); // Drop all references so that stuff can terminate properly drop(garage); diff --git a/src/admin/tracing_setup.rs b/src/garage/tracing_setup.rs similarity index 100% rename from src/admin/tracing_setup.rs rename to src/garage/tracing_setup.rs diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index bed7f44a..80a1975c 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -15,7 +15,6 @@ path = "lib.rs" [dependencies] garage_util = { version = "0.7.0", path = "../util" } -garage_admin = { version = "0.7.0", path = "../admin" } arc-swap = "1.0" bytes = "1.0" diff --git a/src/util/config.rs b/src/util/config.rs index 4d66bfe4..99ebce31 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -121,6 +121,10 @@ pub struct WebConfig { pub struct AdminConfig { /// Address and port to bind for admin API serving pub api_bind_addr: Option, + /// Bearer token to use to scrape metrics + pub metrics_token: Option, + /// Bearer token to use to access Admin API endpoints + pub admin_token: Option, /// OTLP server to where to export traces pub trace_sink: Option, } -- 2.43.4 From 99fcfa3844a346463e5739fec19ac2a6b560adfc Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 5 May 2022 10:56:44 +0200 Subject: [PATCH 02/46] Make background runner terminate correctly --- src/garage/server.rs | 1 + src/util/background.rs | 37 ++++++++++++++++++++++--------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/garage/server.rs b/src/garage/server.rs index 4c0f8653..ffbe97ec 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -110,6 +110,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { // Remove RPC handlers for system to break reference cycles garage.system.netapp.drop_all_handlers(); + opentelemetry::global::shutdown_tracer_provider(); // Await for netapp RPC system to end run_system.await?; diff --git a/src/util/background.rs b/src/util/background.rs index bfdaaf1e..d35425f5 100644 --- a/src/util/background.rs +++ b/src/util/background.rs @@ -6,7 +6,9 @@ use std::time::Duration; use futures::future::*; use futures::select; -use tokio::sync::{mpsc, watch, Mutex}; +use futures::stream::FuturesUnordered; +use futures::StreamExt; +use tokio::sync::{mpsc, mpsc::error::TryRecvError, watch, Mutex}; use crate::error::Error; @@ -30,26 +32,31 @@ impl BackgroundRunner { let stop_signal_2 = stop_signal.clone(); let await_all_done = tokio::spawn(async move { + let mut workers = FuturesUnordered::new(); + let mut shutdown_timer = 0; loop { - let wkr = { - select! { - item = worker_out.recv().fuse() => { - match item { - Some(x) => x, - None => break, - } + let closed = match worker_out.try_recv() { + Ok(wkr) => { + workers.push(wkr); + false + } + Err(TryRecvError::Empty) => false, + Err(TryRecvError::Disconnected) => true, + }; + select! { + res = workers.next() => { + if let Some(Err(e)) = res { + error!("Worker exited with error: {}", e); } - _ = tokio::time::sleep(Duration::from_secs(5)).fuse() => { - if *stop_signal_2.borrow() { + } + _ = tokio::time::sleep(Duration::from_secs(1)).fuse() => { + if closed || *stop_signal_2.borrow() { + shutdown_timer += 1; + if shutdown_timer >= 10 { break; - } else { - continue; } } } - }; - if let Err(e) = wkr.await { - error!("Error while awaiting for worker: {}", e); } } }); -- 2.43.4 From 7a19daafbd08b1372aa3c41eab9b3871c26bca80 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 5 May 2022 13:40:31 +0200 Subject: [PATCH 03/46] Implement /status Admin endpoint --- src/api/admin/api_server.rs | 14 ++++- src/api/admin/cluster.rs | 69 ++++++++++++++++++++++ src/api/admin/mod.rs | 2 + src/api/admin/router.rs | 2 +- src/rpc/system.rs | 114 +++++++++++++++++++++--------------- 5 files changed, 152 insertions(+), 49 deletions(-) create mode 100644 src/api/admin/cluster.rs diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 836b5158..57842548 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use async_trait::async_trait; use futures::future::Future; -use http::header::CONTENT_TYPE; +use http::header::{CONTENT_TYPE, ALLOW, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN}; use hyper::{Body, Request, Response}; use opentelemetry::trace::{SpanRef, Tracer}; @@ -17,6 +17,7 @@ use crate::error::*; use crate::generic_server::*; use crate::admin::router::{Authorization, Endpoint}; +use crate::admin::cluster::*; pub struct AdminApiServer { garage: Arc, @@ -56,6 +57,15 @@ impl AdminApiServer { } } + fn handle_options(&self, _req: &Request) -> Result, Error> { + Ok(Response::builder() + .status(204) + .header(ALLOW, "OPTIONS, GET, POST") + .header(ACCESS_CONTROL_ALLOW_METHODS, "OPTIONS, GET, POST") + .header(ACCESS_CONTROL_ALLOW_ORIGIN, "*") + .body(Body::empty())?) + } + fn handle_metrics(&self) -> Result, Error> { let mut buffer = vec![]; let encoder = TextEncoder::new(); @@ -110,7 +120,9 @@ impl ApiHandler for AdminApiServer { } match endpoint { + Endpoint::Options => self.handle_options(&req), Endpoint::Metrics => self.handle_metrics(), + Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await, _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs new file mode 100644 index 00000000..9ed41944 --- /dev/null +++ b/src/api/admin/cluster.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; +use std::collections::HashMap; +use std::net::{IpAddr, SocketAddr}; + +use serde::{Serialize}; + +use hyper::{Body, Request, Response, StatusCode}; + +use garage_util::data::*; +use garage_util::error::Error as GarageError; +use garage_rpc::layout::*; +use garage_rpc::system::*; + +use garage_model::garage::Garage; + + +use crate::error::*; + + +pub async fn handle_get_cluster_status( + garage: &Arc, +) -> Result, Error> { + let layout = garage.system.get_cluster_layout(); + + let res = GetClusterStatusResponse { + known_nodes: garage.system.get_known_nodes() + .into_iter() + .map(|i| (hex::encode(i.id), KnownNodeResp { + addr: i.addr, + is_up: i.is_up, + last_seen_secs_ago: i.last_seen_secs_ago, + hostname: i.status.hostname, + })) + .collect(), + roles: layout.roles.items() + .iter() + .filter(|(_, _, v)| v.0.is_some()) + .map(|(k, _, v)| (hex::encode(k), v.0.clone())) + .collect(), + staged_role_changes: layout.staging.items() + .iter() + .filter(|(k, _, v)| layout.roles.get(k) != Some(v)) + .map(|(k, _, v)| (hex::encode(k), v.0.clone())) + .collect(), + }; + + let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(resp_json))?) +} + + +#[derive(Serialize)] +struct GetClusterStatusResponse { + #[serde(rename = "knownNodes")] + known_nodes: HashMap, + roles: HashMap>, + #[serde(rename = "stagedRoleChanges")] + staged_role_changes: HashMap>, +} + +#[derive(Serialize)] +struct KnownNodeResp { + addr: SocketAddr, + is_up: bool, + last_seen_secs_ago: Option, + hostname: String, +} diff --git a/src/api/admin/mod.rs b/src/api/admin/mod.rs index ff2cf4b1..7e8d0635 100644 --- a/src/api/admin/mod.rs +++ b/src/api/admin/mod.rs @@ -1,2 +1,4 @@ pub mod api_server; mod router; + +mod cluster; diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index d0b30fc1..714af1e8 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -14,8 +14,8 @@ router_match! {@func /// List of all Admin API endpoints. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Endpoint { - Metrics, Options, + Metrics, GetClusterStatus, GetClusterLayout, UpdateClusterLayout, diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 68d94ea5..a5b8d4f4 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -312,6 +312,67 @@ impl System { ); } + // ---- Administrative operations (directly available and + // also available through RPC) ---- + + pub fn get_known_nodes(&self) -> Vec { + let node_status = self.node_status.read().unwrap(); + let known_nodes = self + .fullmesh + .get_peer_list() + .iter() + .map(|n| KnownNodeInfo { + id: n.id.into(), + addr: n.addr, + is_up: n.is_up(), + last_seen_secs_ago: n.last_seen.map(|t| (Instant::now() - t).as_secs()), + status: node_status + .get(&n.id.into()) + .cloned() + .map(|(_, st)| st) + .unwrap_or(NodeStatus { + hostname: "?".to_string(), + replication_factor: 0, + cluster_layout_version: 0, + cluster_layout_staging_hash: Hash::from([0u8; 32]), + }), + }) + .collect::>(); + known_nodes + } + + pub fn get_cluster_layout(&self) -> ClusterLayout { + self.ring.borrow().layout.clone() + } + + pub async fn connect(&self, node: &str) -> Result<(), Error> { + let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| { + Error::Message(format!( + "Unable to parse or resolve node specification: {}", + node + )) + })?; + let mut errors = vec![]; + for ip in addrs.iter() { + match self + .netapp + .clone() + .try_connect(*ip, pubkey) + .await + .err_context(CONNECT_ERROR_MESSAGE) + { + Ok(()) => return Ok(()), + Err(e) => { + errors.push((*ip, e)); + } + } + } + return Err(Error::Message(format!( + "Could not connect to specified peers. Errors: {:?}", + errors + ))); + } + // ---- INTERNALS ---- async fn advertise_to_consul(self: Arc) -> Result<(), Error> { @@ -384,32 +445,11 @@ impl System { self.local_status.swap(Arc::new(new_si)); } + // --- RPC HANDLERS --- + async fn handle_connect(&self, node: &str) -> Result { - let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| { - Error::Message(format!( - "Unable to parse or resolve node specification: {}", - node - )) - })?; - let mut errors = vec![]; - for ip in addrs.iter() { - match self - .netapp - .clone() - .try_connect(*ip, pubkey) - .await - .err_context(CONNECT_ERROR_MESSAGE) - { - Ok(()) => return Ok(SystemRpc::Ok), - Err(e) => { - errors.push((*ip, e)); - } - } - } - return Err(Error::Message(format!( - "Could not connect to specified peers. Errors: {:?}", - errors - ))); + self.connect(node).await?; + Ok(SystemRpc::Ok) } fn handle_pull_cluster_layout(&self) -> SystemRpc { @@ -418,31 +458,11 @@ impl System { } fn handle_get_known_nodes(&self) -> SystemRpc { - let node_status = self.node_status.read().unwrap(); - let known_nodes = self - .fullmesh - .get_peer_list() - .iter() - .map(|n| KnownNodeInfo { - id: n.id.into(), - addr: n.addr, - is_up: n.is_up(), - last_seen_secs_ago: n.last_seen.map(|t| (Instant::now() - t).as_secs()), - status: node_status - .get(&n.id.into()) - .cloned() - .map(|(_, st)| st) - .unwrap_or(NodeStatus { - hostname: "?".to_string(), - replication_factor: 0, - cluster_layout_version: 0, - cluster_layout_staging_hash: Hash::from([0u8; 32]), - }), - }) - .collect::>(); + let known_nodes = self.get_known_nodes(); SystemRpc::ReturnKnownNodes(known_nodes) } + async fn handle_advertise_status( self: &Arc, from: Uuid, -- 2.43.4 From ec03e3d16c86d5a2f1100d436d195033fc2dbed9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 5 May 2022 13:51:23 +0200 Subject: [PATCH 04/46] Fmt & cleanup --- src/api/admin/api_server.rs | 7 ++- src/api/admin/cluster.rs | 85 +++++++++++++++++++++++-------------- src/rpc/system.rs | 1 - 3 files changed, 59 insertions(+), 34 deletions(-) diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 57842548..dfaac015 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -3,7 +3,9 @@ use std::sync::Arc; use async_trait::async_trait; use futures::future::Future; -use http::header::{CONTENT_TYPE, ALLOW, ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN}; +use http::header::{ + ACCESS_CONTROL_ALLOW_METHODS, ACCESS_CONTROL_ALLOW_ORIGIN, ALLOW, CONTENT_TYPE, +}; use hyper::{Body, Request, Response}; use opentelemetry::trace::{SpanRef, Tracer}; @@ -16,8 +18,8 @@ use garage_util::error::Error as GarageError; use crate::error::*; use crate::generic_server::*; -use crate::admin::router::{Authorization, Endpoint}; use crate::admin::cluster::*; +use crate::admin::router::{Authorization, Endpoint}; pub struct AdminApiServer { garage: Arc, @@ -123,6 +125,7 @@ impl ApiHandler for AdminApiServer { Endpoint::Options => self.handle_options(&req), Endpoint::Metrics => self.handle_metrics(), Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await, + Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await, _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 9ed41944..f4835648 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -1,47 +1,37 @@ -use std::sync::Arc; use std::collections::HashMap; -use std::net::{IpAddr, SocketAddr}; +use std::net::SocketAddr; +use std::sync::Arc; -use serde::{Serialize}; +use serde::Serialize; -use hyper::{Body, Request, Response, StatusCode}; +use hyper::{Body, Response, StatusCode}; -use garage_util::data::*; -use garage_util::error::Error as GarageError; use garage_rpc::layout::*; -use garage_rpc::system::*; +use garage_util::error::Error as GarageError; use garage_model::garage::Garage; - use crate::error::*; - -pub async fn handle_get_cluster_status( - garage: &Arc, -) -> Result, Error> { - let layout = garage.system.get_cluster_layout(); - +pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { let res = GetClusterStatusResponse { - known_nodes: garage.system.get_known_nodes() + known_nodes: garage + .system + .get_known_nodes() .into_iter() - .map(|i| (hex::encode(i.id), KnownNodeResp { - addr: i.addr, - is_up: i.is_up, - last_seen_secs_ago: i.last_seen_secs_ago, - hostname: i.status.hostname, - })) - .collect(), - roles: layout.roles.items() - .iter() - .filter(|(_, _, v)| v.0.is_some()) - .map(|(k, _, v)| (hex::encode(k), v.0.clone())) - .collect(), - staged_role_changes: layout.staging.items() - .iter() - .filter(|(k, _, v)| layout.roles.get(k) != Some(v)) - .map(|(k, _, v)| (hex::encode(k), v.0.clone())) + .map(|i| { + ( + hex::encode(i.id), + KnownNodeResp { + addr: i.addr, + is_up: i.is_up, + last_seen_secs_ago: i.last_seen_secs_ago, + hostname: i.status.hostname, + }, + ) + }) .collect(), + layout: get_cluster_layout(garage), }; let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; @@ -50,11 +40,44 @@ pub async fn handle_get_cluster_status( .body(Body::from(resp_json))?) } +pub async fn handle_get_cluster_layout(garage: &Arc) -> Result, Error> { + let res = get_cluster_layout(garage); + let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(resp_json))?) +} + +fn get_cluster_layout(garage: &Arc) -> GetClusterLayoutResponse { + let layout = garage.system.get_cluster_layout(); + + GetClusterLayoutResponse { + roles: layout + .roles + .items() + .iter() + .filter(|(_, _, v)| v.0.is_some()) + .map(|(k, _, v)| (hex::encode(k), v.0.clone())) + .collect(), + staged_role_changes: layout + .staging + .items() + .iter() + .filter(|(k, _, v)| layout.roles.get(k) != Some(v)) + .map(|(k, _, v)| (hex::encode(k), v.0.clone())) + .collect(), + } +} #[derive(Serialize)] struct GetClusterStatusResponse { #[serde(rename = "knownNodes")] known_nodes: HashMap, + layout: GetClusterLayoutResponse, +} + +#[derive(Serialize)] +struct GetClusterLayoutResponse { roles: HashMap>, #[serde(rename = "stagedRoleChanges")] staged_role_changes: HashMap>, diff --git a/src/rpc/system.rs b/src/rpc/system.rs index a5b8d4f4..73c7b898 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -462,7 +462,6 @@ impl System { SystemRpc::ReturnKnownNodes(known_nodes) } - async fn handle_advertise_status( self: &Arc, from: Uuid, -- 2.43.4 From e4c61124d875574ec7c2969a3a3056b69eade3af Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 May 2022 16:01:25 +0200 Subject: [PATCH 05/46] Add first draft of admin api --- doc/drafts/admin-api.md | 135 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 doc/drafts/admin-api.md diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md new file mode 100644 index 00000000..1ba868ef --- /dev/null +++ b/doc/drafts/admin-api.md @@ -0,0 +1,135 @@ +# Access control + +The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section: + +- `metrics_token`: the token for accessing the Metrics endpoint (if this token is not set in the config file, the Metrics endpoint can be accessed without access control); +- `admin_token`: the token for accessing all of the other administration endpoints (if this token is not set in the config file, these endpoints can be accessed without access control). + +# Administration API endpoints + +## Metrics `GET /metrics` + +Returns internal Garage metrics in Prometheus format. + +## GetClusterStatus `GET /status` + +Returns the cluster's current status in JSON, including: + +- Live nodes +- Currently configured cluster layout +- Staged changes to the cluster layout + +Example response body: + +```json +{ + "knownNodes": { + "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": { + "addr": "10.0.0.11:3901", + "is_up": true, + "last_seen_secs_ago": 9, + "hostname": "node1" + }, + "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": { + "addr": "10.0.0.12:3901", + "is_up": true, + "last_seen_secs_ago": 1, + "hostname": "node2" + }, + "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": { + "addr": "10.0.0.21:3901", + "is_up": true, + "last_seen_secs_ago": 7, + "hostname": "node3" + }, + "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": { + "addr": "10.0.0.22:3901", + "is_up": true, + "last_seen_secs_ago": 1, + "hostname": "node4" + } + }, + "layout": { + "roles": { + "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": { + "zone": "dc1", + "capacity": 4, + "tags": [ + "node1" + ] + }, + "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": { + "zone": "dc1", + "capacity": 6, + "tags": [ + "node2" + ] + }, + "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": { + "zone": "dc2", + "capacity": 10, + "tags": [ + "node3" + ] + } + }, + "stagedRoleChanges": { + "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": { + "zone": "dc2", + "capacity": 5, + "tags": [ + "node4" + ] + } + } + } +} +``` + +## GetClusterLayout `GET /layout` + +Returns the cluster's current layout in JSON, including: + +- Currently configured cluster layout +- Staged changes to the cluster layout + +(the info returned by this endpoint is a subset of the info returned by GetClusterStatus) + +Example response body: + +```json +{ + "roles": { + "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": { + "zone": "dc1", + "capacity": 4, + "tags": [ + "node1" + ] + }, + "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff": { + "zone": "dc1", + "capacity": 6, + "tags": [ + "node2" + ] + }, + "23ffd0cdd375ebff573b20cc5cef38996b51c1a7d6dbcf2c6e619876e507cf27": { + "zone": "dc2", + "capacity": 10, + "tags": [ + "node3" + ] + } + }, + "stagedRoleChanges": { + "e2ee7984ee65b260682086ec70026165903c86e601a4a5a501c1900afe28d84b": { + "zone": "dc2", + "capacity": 5, + "tags": [ + "node4" + ] + } + } +} +``` -- 2.43.4 From 01c4876fb447b70106e934ad09cf9b921f33682a Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 May 2022 16:21:01 +0200 Subject: [PATCH 06/46] Specify remaining cluster-related endpoints --- doc/drafts/admin-api.md | 79 ++++++++++++++++++++++++++++++++++++++-- src/api/admin/cluster.rs | 2 + 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 1ba868ef..a518c93f 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -7,11 +7,15 @@ The admin API uses two different tokens for acces control, that are specified in # Administration API endpoints -## Metrics `GET /metrics` +## Metrics-related endpoints + +### Metrics `GET /metrics` Returns internal Garage metrics in Prometheus format. -## GetClusterStatus `GET /status` +## Cluster operations + +### GetClusterStatus `GET /status` Returns the cluster's current status in JSON, including: @@ -50,6 +54,7 @@ Example response body: } }, "layout": { + "version": 12, "roles": { "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": { "zone": "dc1", @@ -86,7 +91,7 @@ Example response body: } ``` -## GetClusterLayout `GET /layout` +### GetClusterLayout `GET /layout` Returns the cluster's current layout in JSON, including: @@ -99,6 +104,7 @@ Example response body: ```json { + "version": 12, "roles": { "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": { "zone": "dc1", @@ -133,3 +139,70 @@ Example response body: } } ``` + +### UpdateClusterLayout `POST /layout` + +Send modifications to the cluster layout. These modifications will +be included in the staged role changes, visible in subsequent calls +of `GetClusterLayout`. Once the set of staged changes is satisfactory, +the user may call `ApplyClusterLayout` to apply the changed changes, +or `Revert ClusterLayout` to clear all of the staged changes in +the layout. + +Request body format: + +```json +{ + : { + "capacity": , + "zone": , + "tags": [ + , + ... + ] + }, + : null, + ... +} +``` + +Contrary to the CLI that may update only a subset of the fields +`capacity`, `zone` and `tags`, when calling this API all of these +values must be specified. + + +### ApplyClusterLayout `POST /layout/apply` + +Applies to the cluster the layout changes currently registered as +staged layout changes. + +Request body format: + +```json +{ + "version": 13 +} +``` + +Similarly to the CLI, the body must include the version of the new layout +that will be created, which MUST be 1 + the value of the currently +existing layout in the cluster. + +### RevertClusterLayout `POST /layout/revert` + +Clears all of the staged layout changes. + +Request body format: + +```json +{ + "version": 13 +} +``` + +Reverting the staged changes is done by incrementing the version number +and clearing the contents of the staged change list. +Similarly to the CLI, the body must include the incremented +version number, which MUST be 1 + the value of the currently +existing layout in the cluster. + diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index f4835648..0eb754ac 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -52,6 +52,7 @@ fn get_cluster_layout(garage: &Arc) -> GetClusterLayoutResponse { let layout = garage.system.get_cluster_layout(); GetClusterLayoutResponse { + version: layout.version, roles: layout .roles .items() @@ -78,6 +79,7 @@ struct GetClusterStatusResponse { #[derive(Serialize)] struct GetClusterLayoutResponse { + version: u64, roles: HashMap>, #[serde(rename = "stagedRoleChanges")] staged_role_changes: HashMap>, -- 2.43.4 From dd54d0b2b13ecf1f95e60b107de9af20632335f6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 6 May 2022 17:14:09 +0200 Subject: [PATCH 07/46] Refactor code for apply/revert, implement Update/Apply/RevertLayout --- Cargo.lock | 2 +- doc/drafts/admin-api.md | 3 ++ src/api/admin/api_server.rs | 5 +++ src/api/admin/cluster.rs | 77 +++++++++++++++++++++++++++++++++++-- src/api/helpers.rs | 8 ++++ src/api/k2v/batch.rs | 13 ++----- src/garage/Cargo.toml | 1 - src/garage/admin.rs | 6 +-- src/garage/cli/layout.rs | 47 ++-------------------- src/rpc/Cargo.toml | 1 + src/rpc/layout.rs | 56 +++++++++++++++++++++++++++ src/rpc/system.rs | 18 ++++++++- src/util/crdt/lww_map.rs | 5 +++ 13 files changed, 179 insertions(+), 63 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3f253b5b..3eb24e4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -845,7 +845,6 @@ dependencies = [ "garage_table 0.7.0", "garage_util 0.7.0", "garage_web", - "git-version", "hex", "hmac", "http", @@ -1031,6 +1030,7 @@ dependencies = [ "futures-util", "garage_util 0.7.0", "gethostname", + "git-version", "hex", "hyper", "k8s-openapi", diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index a518c93f..ab24e18f 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -19,6 +19,7 @@ Returns internal Garage metrics in Prometheus format. Returns the cluster's current status in JSON, including: +- ID of the node being queried and its version of the Garage daemon - Live nodes - Currently configured cluster layout - Staged changes to the cluster layout @@ -27,6 +28,8 @@ Example response body: ```json { + "node": "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f", + "garage_version": "git:v0.8.0", "knownNodes": { "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f": { "addr": "10.0.0.11:3901", diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index dfaac015..d008f10a 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -126,10 +126,15 @@ impl ApiHandler for AdminApiServer { Endpoint::Metrics => self.handle_metrics(), Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await, Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await, + Endpoint::UpdateClusterLayout => handle_update_cluster_layout(&self.garage, req).await, + Endpoint::ApplyClusterLayout => handle_apply_cluster_layout(&self.garage, req).await, + Endpoint::RevertClusterLayout => handle_revert_cluster_layout(&self.garage, req).await, + /* _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() ))), + */ } } } diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 0eb754ac..b8e9d96c 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -2,19 +2,24 @@ use std::collections::HashMap; use std::net::SocketAddr; use std::sync::Arc; -use serde::Serialize; +use hyper::{Body, Request, Response, StatusCode}; +use serde::{Deserialize, Serialize}; -use hyper::{Body, Response, StatusCode}; +use garage_util::crdt::*; +use garage_util::data::*; +use garage_util::error::Error as GarageError; use garage_rpc::layout::*; -use garage_util::error::Error as GarageError; use garage_model::garage::Garage; use crate::error::*; +use crate::helpers::*; pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { let res = GetClusterStatusResponse { + node: hex::encode(garage.system.id), + garage_version: garage.system.garage_version(), known_nodes: garage .system .get_known_nodes() @@ -72,6 +77,8 @@ fn get_cluster_layout(garage: &Arc) -> GetClusterLayoutResponse { #[derive(Serialize)] struct GetClusterStatusResponse { + node: String, + garage_version: &'static str, #[serde(rename = "knownNodes")] known_nodes: HashMap, layout: GetClusterLayoutResponse, @@ -92,3 +99,67 @@ struct KnownNodeResp { last_seen_secs_ago: Option, hostname: String, } + +pub async fn handle_update_cluster_layout( + garage: &Arc, + req: Request, +) -> Result, Error> { + let updates = parse_json_body::(req).await?; + + let mut layout = garage.system.get_cluster_layout(); + + let mut roles = layout.roles.clone(); + roles.merge(&layout.staging); + + for (node, role) in updates { + let node = hex::decode(node).ok_or_bad_request("Invalid node identifier")?; + let node = Uuid::try_from(&node).ok_or_bad_request("Invalid node identifier")?; + + layout + .staging + .merge(&roles.update_mutator(node, NodeRoleV(role))); + } + + garage.system.update_cluster_layout(&layout).await?; + + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::empty())?) +} + +pub async fn handle_apply_cluster_layout( + garage: &Arc, + req: Request, +) -> Result, Error> { + let param = parse_json_body::(req).await?; + + let layout = garage.system.get_cluster_layout(); + let layout = layout.apply_staged_changes(Some(param.version))?; + garage.system.update_cluster_layout(&layout).await?; + + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::empty())?) +} + +pub async fn handle_revert_cluster_layout( + garage: &Arc, + req: Request, +) -> Result, Error> { + let param = parse_json_body::(req).await?; + + let layout = garage.system.get_cluster_layout(); + let layout = layout.revert_staged_changes(Some(param.version))?; + garage.system.update_cluster_layout(&layout).await?; + + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::empty())?) +} + +type UpdateClusterLayoutRequest = HashMap>; + +#[derive(Deserialize)] +struct ApplyRevertLayoutRequest { + version: u64, +} diff --git a/src/api/helpers.rs b/src/api/helpers.rs index a994b82f..5e249dae 100644 --- a/src/api/helpers.rs +++ b/src/api/helpers.rs @@ -1,4 +1,6 @@ +use hyper::{Body, Request}; use idna::domain_to_unicode; +use serde::Deserialize; use garage_util::data::*; @@ -163,6 +165,12 @@ pub fn key_after_prefix(pfx: &str) -> Option { None } +pub async fn parse_json_body Deserialize<'de>>(req: Request) -> Result { + let body = hyper::body::to_bytes(req.into_body()).await?; + let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; + Ok(resp) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index 4ecddeb9..a97bd7f2 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -13,6 +13,7 @@ use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; use crate::error::*; +use crate::helpers::*; use crate::k2v::range::read_range; pub async fn handle_insert_batch( @@ -20,9 +21,7 @@ pub async fn handle_insert_batch( bucket_id: Uuid, req: Request, ) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; - let items: Vec = - serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; + let items = parse_json_body::>(req).await?; let mut items2 = vec![]; for it in items { @@ -52,9 +51,7 @@ pub async fn handle_read_batch( bucket_id: Uuid, req: Request, ) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; - let queries: Vec = - serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; + let queries = parse_json_body::>(req).await?; let resp_results = futures::future::join_all( queries @@ -149,9 +146,7 @@ pub async fn handle_delete_batch( bucket_id: Uuid, req: Request, ) -> Result, Error> { - let body = hyper::body::to_bytes(req.into_body()).await?; - let queries: Vec = - serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; + let queries = parse_json_body::>(req).await?; let resp_results = futures::future::join_all( queries diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 59566358..902f67f8 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -29,7 +29,6 @@ garage_util = { version = "0.7.0", path = "../util" } garage_web = { version = "0.7.0", path = "../web" } bytes = "1.0" -git-version = "0.3.4" hex = "0.4" tracing = { version = "0.1.30", features = ["log-always"] } pretty_env_logger = "0.4" diff --git a/src/garage/admin.rs b/src/garage/admin.rs index af0c3f22..1a58a613 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -696,11 +696,7 @@ impl AdminRpcHandler { writeln!( &mut ret, "\nGarage version: {}", - option_env!("GIT_VERSION").unwrap_or(git_version::git_version!( - prefix = "git:", - cargo_prefix = "cargo:", - fallback = "unknown" - )) + self.garage.system.garage_version(), ) .unwrap(); diff --git a/src/garage/cli/layout.rs b/src/garage/cli/layout.rs index 88941d78..cdd3869b 100644 --- a/src/garage/cli/layout.rs +++ b/src/garage/cli/layout.rs @@ -1,5 +1,4 @@ use garage_util::crdt::Crdt; -use garage_util::data::*; use garage_util::error::*; use garage_rpc::layout::*; @@ -211,31 +210,9 @@ pub async fn cmd_apply_layout( rpc_host: NodeID, apply_opt: ApplyLayoutOpt, ) -> Result<(), Error> { - let mut layout = fetch_layout(rpc_cli, rpc_host).await?; + let layout = fetch_layout(rpc_cli, rpc_host).await?; - match apply_opt.version { - None => { - println!("Please pass the --version flag to ensure that you are writing the correct version of the cluster layout."); - println!("To know the correct value of the --version flag, invoke `garage layout show` and review the proposed changes."); - return Err(Error::Message("--version flag is missing".into())); - } - Some(v) => { - if v != layout.version + 1 { - return Err(Error::Message("Invalid value of --version flag".into())); - } - } - } - - layout.roles.merge(&layout.staging); - - if !layout.calculate_partition_assignation() { - return Err(Error::Message("Could not calculate new assignation of partitions to nodes. This can happen if there are less nodes than the desired number of copies of your data (see the replication_mode configuration parameter).".into())); - } - - layout.staging.clear(); - layout.staging_hash = blake2sum(&rmp_to_vec_all_named(&layout.staging).unwrap()[..]); - - layout.version += 1; + let layout = layout.apply_staged_changes(apply_opt.version)?; send_layout(rpc_cli, rpc_host, layout).await?; @@ -250,25 +227,9 @@ pub async fn cmd_revert_layout( rpc_host: NodeID, revert_opt: RevertLayoutOpt, ) -> Result<(), Error> { - let mut layout = fetch_layout(rpc_cli, rpc_host).await?; + let layout = fetch_layout(rpc_cli, rpc_host).await?; - match revert_opt.version { - None => { - println!("Please pass the --version flag to ensure that you are writing the correct version of the cluster layout."); - println!("To know the correct value of the --version flag, invoke `garage layout show` and review the proposed changes."); - return Err(Error::Message("--version flag is missing".into())); - } - Some(v) => { - if v != layout.version + 1 { - return Err(Error::Message("Invalid value of --version flag".into())); - } - } - } - - layout.staging.clear(); - layout.staging_hash = blake2sum(&rmp_to_vec_all_named(&layout.staging).unwrap()[..]); - - layout.version += 1; + let layout = layout.revert_staged_changes(revert_opt.version)?; send_layout(rpc_cli, rpc_host, layout).await?; diff --git a/src/rpc/Cargo.toml b/src/rpc/Cargo.toml index 80a1975c..73328993 100644 --- a/src/rpc/Cargo.toml +++ b/src/rpc/Cargo.toml @@ -19,6 +19,7 @@ garage_util = { version = "0.7.0", path = "../util" } arc-swap = "1.0" bytes = "1.0" gethostname = "0.2" +git-version = "0.3.4" hex = "0.4" tracing = "0.1.30" rand = "0.8" diff --git a/src/rpc/layout.rs b/src/rpc/layout.rs index b9c02c21..f517f36f 100644 --- a/src/rpc/layout.rs +++ b/src/rpc/layout.rs @@ -5,6 +5,7 @@ use serde::{Deserialize, Serialize}; use garage_util::crdt::{AutoCrdt, Crdt, LwwMap}; use garage_util::data::*; +use garage_util::error::*; use crate::ring::*; @@ -100,6 +101,61 @@ impl ClusterLayout { } } + pub fn apply_staged_changes(mut self, version: Option) -> Result { + match version { + None => { + let error = r#" +Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout. +To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes. + "#; + return Err(Error::Message(error.into())); + } + Some(v) => { + if v != self.version + 1 { + return Err(Error::Message("Invalid new layout version".into())); + } + } + } + + self.roles.merge(&self.staging); + self.roles.retain(|(_, _, v)| v.0.is_some()); + + if !self.calculate_partition_assignation() { + return Err(Error::Message("Could not calculate new assignation of partitions to nodes. This can happen if there are less nodes than the desired number of copies of your data (see the replication_mode configuration parameter).".into())); + } + + self.staging.clear(); + self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]); + + self.version += 1; + + Ok(self) + } + + pub fn revert_staged_changes(mut self, version: Option) -> Result { + match version { + None => { + let error = r#" +Please pass the new layout version number to ensure that you are writing the correct version of the cluster layout. +To know the correct value of the new layout version, invoke `garage layout show` and review the proposed changes. + "#; + return Err(Error::Message(error.into())); + } + Some(v) => { + if v != self.version + 1 { + return Err(Error::Message("Invalid new layout version".into())); + } + } + } + + self.staging.clear(); + self.staging_hash = blake2sum(&rmp_to_vec_all_named(&self.staging).unwrap()[..]); + + self.version += 1; + + Ok(self) + } + /// Returns a list of IDs of nodes that currently have /// a role in the cluster pub fn node_ids(&self) -> &[Uuid] { diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 73c7b898..eb2f2e42 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -315,6 +315,14 @@ impl System { // ---- Administrative operations (directly available and // also available through RPC) ---- + pub fn garage_version(&self) -> &'static str { + option_env!("GIT_VERSION").unwrap_or(git_version::git_version!( + prefix = "git:", + cargo_prefix = "cargo:", + fallback = "unknown" + )) + } + pub fn get_known_nodes(&self) -> Vec { let node_status = self.node_status.read().unwrap(); let known_nodes = self @@ -345,6 +353,14 @@ impl System { self.ring.borrow().layout.clone() } + pub async fn update_cluster_layout( + self: &Arc, + layout: &ClusterLayout, + ) -> Result<(), Error> { + self.handle_advertise_cluster_layout(layout).await?; + Ok(()) + } + pub async fn connect(&self, node: &str) -> Result<(), Error> { let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| { Error::Message(format!( @@ -495,7 +511,7 @@ impl System { } async fn handle_advertise_cluster_layout( - self: Arc, + self: &Arc, adv: &ClusterLayout, ) -> Result { let update_ring = self.update_ring.lock().await; diff --git a/src/util/crdt/lww_map.rs b/src/util/crdt/lww_map.rs index c155c3a8..91d24c7f 100644 --- a/src/util/crdt/lww_map.rs +++ b/src/util/crdt/lww_map.rs @@ -140,6 +140,11 @@ where self.vals.clear(); } + /// Retain only values that match a certain predicate + pub fn retain(&mut self, pred: impl FnMut(&(K, u64, V)) -> bool) { + self.vals.retain(pred); + } + /// Get a reference to the value assigned to a key pub fn get(&self, k: &K) -> Option<&V> { match self.vals.binary_search_by(|(k2, _, _)| k2.cmp(k)) { -- 2.43.4 From bb6ec9ebd979c168091c7b00cc8b97da4a1a8dc9 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 10 May 2022 13:36:35 +0200 Subject: [PATCH 08/46] Update Cargo.nix and improve log message --- Cargo.nix | 205 ++++++++++++++++++++----------------------- src/garage/server.rs | 2 +- 2 files changed, 95 insertions(+), 112 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index 39f409b6..acc62f2f 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -6,7 +6,6 @@ args@{ rootFeatures ? [ "garage_util/default" "garage_rpc/default" - "garage_admin/default" "garage_table/default" "garage_block/default" "garage_model/default" @@ -47,7 +46,6 @@ in workspace = { garage_util = rustPackages.unknown.garage_util."0.7.0"; garage_rpc = rustPackages.unknown.garage_rpc."0.7.0"; - garage_admin = rustPackages.unknown.garage_admin."0.7.0"; garage_table = rustPackages.unknown.garage_table."0.7.0"; garage_block = rustPackages.unknown.garage_block."0.7.0"; garage_model = rustPackages.unknown.garage_model."0.7.0"; @@ -630,7 +628,7 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"; }; dependencies = { - ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.config == "aarch64-apple-darwin" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-linux-android" || hostPlatform.parsed.cpu.name == "aarch64" && hostPlatform.parsed.kernel.name == "linux" || hostPlatform.config == "aarch64-apple-darwin" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; }; }); @@ -1149,7 +1147,7 @@ in [ "async-await" ] [ "async-await-macro" ] [ "channel" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") [ "futures-channel" ] [ "futures-io" ] [ "futures-macro" ] @@ -1198,18 +1196,20 @@ in bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - garage_admin = rustPackages."unknown".garage_admin."0.7.0" { inherit profileName; }; garage_api = rustPackages."unknown".garage_api."0.7.0" { inherit profileName; }; garage_model = rustPackages."unknown".garage_model."0.7.0" { inherit profileName; }; garage_rpc = rustPackages."unknown".garage_rpc."0.7.0" { inherit profileName; }; garage_table = rustPackages."unknown".garage_table."0.7.0" { inherit profileName; }; garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; garage_web = rustPackages."unknown".garage_web."0.7.0" { inherit profileName; }; - git_version = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }; hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; sodiumoxide = rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }; netapp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.4.4" { inherit profileName; }; + opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; + opentelemetry_otlp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }; + opentelemetry_prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }; pretty_env_logger = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pretty_env_logger."0.4.0" { inherit profileName; }; + prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }; rand = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; rmp_serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rmp-serde."0.15.5" { inherit profileName; }; serde = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; @@ -1234,26 +1234,6 @@ in }; }); - "unknown".garage_admin."0.7.0" = overridableMkRustCrate (profileName: rec { - name = "garage_admin"; - version = "0.7.0"; - registry = "unknown"; - src = fetchCrateLocal (workspaceSrc + "/src/admin"); - dependencies = { - futures = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; - futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - garage_util = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; - hex = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; - http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; - hyper = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }; - opentelemetry = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; - opentelemetry_otlp = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }; - opentelemetry_prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }; - prometheus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }; - tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; - }; - }); - "unknown".garage_api."0.7.0" = overridableMkRustCrate (profileName: rec { name = "garage_api"; version = "0.7.0"; @@ -1288,8 +1268,11 @@ in ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "multer" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".multer."2.0.2" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "nom" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".nom."7.1.1" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "opentelemetry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "opentelemetry_otlp" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-otlp."0.10.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "opentelemetry_prometheus" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry-prometheus."0.10.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "percent_encoding" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".percent-encoding."2.1.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "pin_project" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "prometheus" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".prometheus."0.13.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "quick_xml" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".quick-xml."0.21.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "roxmltree" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".roxmltree."0.14.1" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "serde" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde."1.0.136" { inherit profileName; }; @@ -1435,9 +1418,9 @@ in ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "bytes" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.21" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "garage_admin" else null } = rustPackages."unknown".garage_admin."0.7.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "garage_util" else null } = rustPackages."unknown".garage_util."0.7.0" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "gethostname" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".gethostname."0.2.3" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "git_version" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".git-version."0.3.5" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "hex" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hex."0.4.3" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "hyper" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.18" { inherit profileName; }; ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc" then "k8s_openapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".k8s-openapi."0.13.1" { inherit profileName; }; @@ -1806,31 +1789,31 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "client") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "client") [ "default" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "full") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "h2") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "http1") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "http2") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "runtime") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "server") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "socket2") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "stream") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "tcp") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "full") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "h2") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "http1") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "http2") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "runtime") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "server") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "socket2") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "stream") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "tcp") ]; dependencies = { bytes = rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.1.0" { inherit profileName; }; futures_channel = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-channel."0.3.21" { inherit profileName; }; futures_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; futures_util = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "h2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "h2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".h2."0.3.12" { inherit profileName; }; http = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.6" { inherit profileName; }; http_body = rustPackages."registry+https://github.com/rust-lang/crates.io-index".http-body."0.4.4" { inherit profileName; }; httparse = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httparse."1.6.0" { inherit profileName; }; httpdate = rustPackages."registry+https://github.com/rust-lang/crates.io-index".httpdate."1.0.2" { inherit profileName; }; itoa = rustPackages."registry+https://github.com/rust-lang/crates.io-index".itoa."1.0.1" { inherit profileName; }; pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "socket2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.4" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "socket2" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.4" { inherit profileName; }; tokio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; tower_service = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.1" { inherit profileName; }; tracing = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; @@ -1915,13 +1898,13 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"; }; features = builtins.concatLists [ - [ "std" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "std") ]; dependencies = { - hashbrown = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.11.2" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "hashbrown" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".hashbrown."0.11.2" { inherit profileName; }; }; buildDependencies = { - autocfg = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "autocfg" else null } = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".autocfg."1.1.0" { profileName = "__noProfile"; }; }; }); @@ -2392,7 +2375,7 @@ in [ "os-poll" ] ]; dependencies = { - ${ if hostPlatform.isUnix || hostPlatform.parsed.kernel.name == "wasi" then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; + ${ if hostPlatform.parsed.kernel.name == "wasi" || hostPlatform.isUnix then "libc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.121" { inherit profileName; }; log = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; ${ if hostPlatform.isWindows then "miow" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".miow."0.3.7" { inherit profileName; }; ${ if hostPlatform.isWindows then "ntapi" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".ntapi."0.3.7" { inherit profileName; }; @@ -3214,7 +3197,7 @@ in [ "getrandom" ] [ "libc" ] [ "rand_chacha" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "small_rng") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "small_rng") [ "std" ] [ "std_rng" ] ]; @@ -3307,28 +3290,28 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"; }; features = builtins.concatLists [ - [ "aho-corasick" ] - [ "default" ] - [ "memchr" ] - [ "perf" ] - [ "perf-cache" ] - [ "perf-dfa" ] - [ "perf-inline" ] - [ "perf-literal" ] - [ "std" ] - [ "unicode" ] - [ "unicode-age" ] - [ "unicode-bool" ] - [ "unicode-case" ] - [ "unicode-gencat" ] - [ "unicode-perl" ] - [ "unicode-script" ] - [ "unicode-segment" ] + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "aho-corasick") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "memchr") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf-cache") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf-dfa") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf-inline") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "perf-literal") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-age") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-bool") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-case") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-gencat") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-perl") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-script") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "unicode-segment") ]; dependencies = { - aho_corasick = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" { inherit profileName; }; - memchr = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; - regex_syntax = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.25" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "aho_corasick" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".aho-corasick."0.7.18" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "memchr" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".memchr."2.4.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "regex_syntax" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".regex-syntax."0.6.25" { inherit profileName; }; }; }); @@ -4076,8 +4059,8 @@ in [ "bytes" ] [ "default" ] [ "fs" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "full") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "io-std") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "full") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "io-std") [ "io-util" ] [ "libc" ] [ "macros" ] @@ -4086,8 +4069,8 @@ in [ "net" ] [ "num_cpus" ] [ "once_cell" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "parking_lot") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "process") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "parking_lot") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "process") [ "rt" ] [ "rt-multi-thread" ] [ "signal" ] @@ -4105,7 +4088,7 @@ in mio = rustPackages."registry+https://github.com/rust-lang/crates.io-index".mio."0.8.2" { inherit profileName; }; num_cpus = rustPackages."registry+https://github.com/rust-lang/crates.io-index".num_cpus."1.13.1" { inherit profileName; }; once_cell = rustPackages."registry+https://github.com/rust-lang/crates.io-index".once_cell."1.10.0" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "parking_lot" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".parking_lot."0.12.0" { inherit profileName; }; pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; ${ if hostPlatform.isUnix then "signal_hook_registry" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".signal-hook-registry."1.4.0" { inherit profileName; }; socket2 = rustPackages."registry+https://github.com/rust-lang/crates.io-index".socket2."0.4.4" { inherit profileName; }; @@ -4183,9 +4166,9 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "codec") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "codec") [ "compat" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "default") [ "futures-io" ] (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc") "io") (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_rpc") "slab") @@ -4302,43 +4285,43 @@ in registry = "registry+https://github.com/rust-lang/crates.io-index"; src = fetchCratesIo { inherit name version; sha256 = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e"; }; features = builtins.concatLists [ - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "__common") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "balance") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "buffer") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "default") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "discover") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "futures-core") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "futures-util") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "indexmap") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "limit") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "load") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "log") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "make") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "pin-project") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "pin-project-lite") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "rand") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "ready-cache") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "__common") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "balance") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "buffer") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "default") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "discover") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "futures-core") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "futures-util") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "indexmap") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "limit") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "load") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "log") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "make") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "pin-project") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "pin-project-lite") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "rand") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "ready-cache") (lib.optional (rootFeatures' ? "garage") "retry") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "slab") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "timeout") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "tokio") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "tokio-util") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "tracing") - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "util") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "slab") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "timeout") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "tokio") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "tokio-util") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "tracing") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "util") ]; dependencies = { - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "indexmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.8.0" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "pin_project" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "slab" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "tokio_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.0" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "tower_layer" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.1" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "tower_service" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.1" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "tracing" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "futures_core" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-core."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "futures_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.21" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "indexmap" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".indexmap."1.8.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "pin_project" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project."1.0.10" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "pin_project_lite" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "rand" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web" then "slab" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".slab."0.4.5" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "tokio" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.17.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "tokio_util" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio-util."0.7.0" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "tower_layer" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-layer."0.3.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "tower_service" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tower-service."0.3.1" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "tracing" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing."0.1.32" { inherit profileName; }; }; }); @@ -4391,14 +4374,14 @@ in features = builtins.concatLists [ [ "attributes" ] [ "default" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "log") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web") "log") (lib.optional (rootFeatures' ? "garage") "log-always") [ "std" ] [ "tracing-attributes" ] ]; dependencies = { cfg_if = rustPackages."registry+https://github.com/rust-lang/crates.io-index".cfg-if."1.0.0" { inherit profileName; }; - ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web" then "log" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; + ${ if rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_web" then "log" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".log."0.4.16" { inherit profileName; }; pin_project_lite = rustPackages."registry+https://github.com/rust-lang/crates.io-index".pin-project-lite."0.2.8" { inherit profileName; }; tracing_attributes = buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-attributes."0.1.20" { profileName = "__noProfile"; }; tracing_core = rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.23" { inherit profileName; }; @@ -4760,7 +4743,7 @@ in [ "std" ] [ "synchapi" ] [ "sysinfoapi" ] - (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_admin" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_block" || rootFeatures' ? "garage_model" || rootFeatures' ? "garage_rpc" || rootFeatures' ? "garage_table" || rootFeatures' ? "garage_web") "threadpoollegacyapiset") + (lib.optional (rootFeatures' ? "garage" || rootFeatures' ? "garage_api" || rootFeatures' ? "garage_web") "threadpoollegacyapiset") [ "timezoneapi" ] [ "winbase" ] [ "wincon" ] @@ -4820,10 +4803,10 @@ in [ "default" ] ]; dependencies = { - ${ if hostPlatform.config == "aarch64-pc-windows-msvc" || hostPlatform.config == "aarch64-uwp-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "i686-uwp-windows-gnu" || hostPlatform.config == "i686-pc-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "aarch64-uwp-windows-msvc" || hostPlatform.config == "aarch64-pc-windows-msvc" then "windows_aarch64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_aarch64_msvc."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "i686-pc-windows-gnu" || hostPlatform.config == "i686-uwp-windows-gnu" then "windows_i686_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "i686-pc-windows-msvc" || hostPlatform.config == "i686-uwp-windows-msvc" then "windows_i686_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_i686_msvc."0.32.0" { inherit profileName; }; - ${ if hostPlatform.config == "x86_64-pc-windows-gnu" || hostPlatform.config == "x86_64-uwp-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; + ${ if hostPlatform.config == "x86_64-uwp-windows-gnu" || hostPlatform.config == "x86_64-pc-windows-gnu" then "windows_x86_64_gnu" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_gnu."0.32.0" { inherit profileName; }; ${ if hostPlatform.config == "x86_64-pc-windows-msvc" || hostPlatform.config == "x86_64-uwp-windows-msvc" then "windows_x86_64_msvc" else null } = rustPackages."registry+https://github.com/rust-lang/crates.io-index".windows_x86_64_msvc."0.32.0" { inherit profileName; }; }; }); diff --git a/src/garage/server.rs b/src/garage/server.rs index ffbe97ec..b58ad286 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -80,7 +80,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { wait_from(watch_cancel.clone()), )); - info!("Initializing Admin server..."); + info!("Launching Admin API server..."); let admin_server = tokio::spawn(admin_server.run(wait_from(watch_cancel.clone()))); // Stuff runs -- 2.43.4 From f97a7845e9e9ab68c3b8afc0d1091765ed11439c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 11 May 2022 10:27:40 +0200 Subject: [PATCH 09/46] Add API access key admin endpoints --- doc/drafts/admin-api.md | 49 +++++++++++++++++++++++++++++ src/api/admin/api_server.rs | 2 -- src/api/admin/router.rs | 62 +++++++++++++++++++++++++------------ src/api/router_macros.rs | 23 ++++++++++++++ src/api/s3/router.rs | 3 +- 5 files changed, 116 insertions(+), 23 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index ab24e18f..baf87e61 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -209,3 +209,52 @@ Similarly to the CLI, the body must include the incremented version number, which MUST be 1 + the value of the currently existing layout in the cluster. + +## Access key operations + +### ListKeys `GET /key` + +Returns all API access keys in the cluster. + +Example response: + +```json +#TODO +``` + +### CreateKey `POST /key` + +Creates a new API access key. + +Request body format: + +```json +{ + "name": "NameOfMyKey" +} +``` + +### GetKeyInfo `GET /key?id=` + +Returns information about the requested API access key. + +Example response: + +```json +#TODO +``` + +### DeleteKey `DELETE /key?id=` + +Deletes an API access key. + +### UpdateKey `POST /key?id=` + +Updates information about the specified API access key. + +Request body format: + +```json +#TODO +``` + diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index d008f10a..3ae9f591 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -129,12 +129,10 @@ impl ApiHandler for AdminApiServer { Endpoint::UpdateClusterLayout => handle_update_cluster_layout(&self.garage, req).await, Endpoint::ApplyClusterLayout => handle_apply_cluster_layout(&self.garage, req).await, Endpoint::RevertClusterLayout => handle_revert_cluster_layout(&self.garage, req).await, - /* _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() ))), - */ } } } diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index 714af1e8..7ff34aaa 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -1,8 +1,9 @@ -use crate::error::*; +use std::borrow::Cow; use hyper::{Method, Request}; -use crate::router_macros::router_match; +use crate::error::*; +use crate::router_macros::*; pub enum Authorization { MetricsToken, @@ -21,6 +22,17 @@ pub enum Endpoint { UpdateClusterLayout, ApplyClusterLayout, RevertClusterLayout, + ListKeys, + CreateKey, + GetKeyInfo { + id: String, + }, + DeleteKey { + id: String, + }, + UpdateKey { + id: String, + }, }} impl Endpoint { @@ -28,24 +40,32 @@ impl Endpoint { /// possibly extracted from the Host header. /// Returns Self plus bucket name, if endpoint is not Endpoint::ListBuckets pub fn from_request(req: &Request) -> Result { - let path = req.uri().path(); + let uri = req.uri(); + let path = uri.path(); + let query = uri.query(); - use Endpoint::*; - let res = match (req.method(), path) { - (&Method::OPTIONS, _) => Options, - (&Method::GET, "/metrics") => Metrics, - (&Method::GET, "/status") => GetClusterStatus, - (&Method::GET, "/layout") => GetClusterLayout, - (&Method::POST, "/layout") => UpdateClusterLayout, - (&Method::POST, "/layout/apply") => ApplyClusterLayout, - (&Method::POST, "/layout/revert") => RevertClusterLayout, - (m, p) => { - return Err(Error::BadRequest(format!( - "Unknown API endpoint: {} {}", - m, p - ))) - } - }; + let mut query = QueryParameters::from_query(query.unwrap_or_default())?; + + let res = router_match!(@gen_path_parser (req.method(), path, query) [ + OPTIONS _ => Options, + GET "/metrics" => Metrics, + GET "/status" => GetClusterStatus, + // Layout endpoints + GET "/layout" => GetClusterLayout, + POST "/layout" => UpdateClusterLayout, + POST "/layout/apply" => ApplyClusterLayout, + POST "/layout/revert" => RevertClusterLayout, + // API key endpoints + GET "/key" if id => GetKeyInfo (query::id), + POST "/key" if id => UpdateKey (query::id), + POST "/key" => CreateKey, + DELETE "/key" if id => DeleteKey (query::id), + GET "/key" => ListKeys, + ]); + + if let Some(message) = query.nonempty_message() { + debug!("Unused query parameter: {}", message) + } Ok(res) } @@ -57,3 +77,7 @@ impl Endpoint { } } } + +generateQueryParameters! { + "id" => id +} diff --git a/src/api/router_macros.rs b/src/api/router_macros.rs index 8471407c..a3e885e6 100644 --- a/src/api/router_macros.rs +++ b/src/api/router_macros.rs @@ -23,6 +23,29 @@ macro_rules! router_match { _ => None } }}; + (@gen_path_parser ($method:expr, $reqpath:expr, $query:expr) + [ + $($meth:ident $path:pat $(if $required:ident)? => $api:ident $(($($conv:ident :: $param:ident),*))?,)* + ]) => {{ + { + use Endpoint::*; + match ($method, $reqpath) { + $( + (&Method::$meth, $path) if true $(&& $query.$required.is_some())? => $api { + $($( + $param: router_match!(@@parse_param $query, $conv, $param), + )*)? + }, + )* + (m, p) => { + return Err(Error::BadRequest(format!( + "Unknown API endpoint: {} {}", + m, p + ))) + } + } + } + }}; (@gen_parser ($keyword:expr, $key:ident, $query:expr, $header:expr), key: [$($kw_k:ident $(if $required_k:ident)? $(header $header_k:expr)? => $api_k:ident $(($($conv_k:ident :: $param_k:ident),*))?,)*], no_key: [$($kw_nk:ident $(if $required_nk:ident)? $(if_header $header_nk:expr)? => $api_nk:ident $(($($conv_nk:ident :: $param_nk:ident),*))?,)*]) => {{ diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index 0525c649..446ceb54 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -1,10 +1,9 @@ -use crate::error::{Error, OkOrBadRequest}; - use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; +use crate::error::{Error, OkOrBadRequest}; use crate::helpers::Authorization; use crate::router_macros::{generateQueryParameters, router_match}; -- 2.43.4 From 5c00c9fb46305b021b5fc45d7ae7b1e13b72030c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 11 May 2022 11:10:28 +0200 Subject: [PATCH 10/46] First key endpoints: ListKeys and GetKeyInfo --- doc/drafts/admin-api.md | 75 ++++++++++++++- src/api/admin/api_server.rs | 7 ++ src/api/admin/key.rs | 181 ++++++++++++++++++++++++++++++++++++ src/api/admin/mod.rs | 1 + src/api/admin/router.rs | 9 +- 5 files changed, 268 insertions(+), 5 deletions(-) create mode 100644 src/api/admin/key.rs diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index baf87e61..dc89014a 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -219,7 +219,16 @@ Returns all API access keys in the cluster. Example response: ```json -#TODO +[ + { + "id": "GK31c2f218a2e44f485b94239e", + "name": "test" + }, + { + "id": "GKe10061ac9c2921f09e4c5540", + "name": "test2" + } +] ``` ### CreateKey `POST /key` @@ -235,13 +244,75 @@ Request body format: ``` ### GetKeyInfo `GET /key?id=` +### GetKeyInfo `GET /key?search=` Returns information about the requested API access key. +If `id` is set, the key is looked up using its exact identifier (faster). +If `search` is set, the key is looked up using its name or prefix +of identifier (slower, all keys are enumerated to do this). + Example response: ```json -#TODO +{ + "name": "test", + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835", + "permissions": { + "createBucket": false + }, + "buckets": [ + { + "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033", + "globalAliases": [ + "test2" + ], + "localAliases": [], + "permissions": { + "read": true, + "write": true, + "owner": false + } + }, + { + "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995", + "globalAliases": [ + "test3" + ], + "localAliases": [], + "permissions": { + "read": true, + "write": true, + "owner": false + } + }, + { + "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b", + "globalAliases": [], + "localAliases": [ + "test" + ], + "permissions": { + "read": true, + "write": true, + "owner": true + } + }, + { + "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95", + "globalAliases": [ + "alex" + ], + "localAliases": [], + "permissions": { + "read": true, + "write": true, + "owner": true + } + } + ] +} ``` ### DeleteKey `DELETE /key?id=` diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 3ae9f591..e44443ff 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -19,6 +19,7 @@ use crate::error::*; use crate::generic_server::*; use crate::admin::cluster::*; +use crate::admin::key::*; use crate::admin::router::{Authorization, Endpoint}; pub struct AdminApiServer { @@ -125,10 +126,16 @@ impl ApiHandler for AdminApiServer { Endpoint::Options => self.handle_options(&req), Endpoint::Metrics => self.handle_metrics(), Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await, + // Layout Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await, Endpoint::UpdateClusterLayout => handle_update_cluster_layout(&self.garage, req).await, Endpoint::ApplyClusterLayout => handle_apply_cluster_layout(&self.garage, req).await, Endpoint::RevertClusterLayout => handle_revert_cluster_layout(&self.garage, req).await, + // Keys + Endpoint::ListKeys => handle_list_keys(&self.garage).await, + Endpoint::GetKeyInfo { id, search } => { + handle_get_key_info(&self.garage, id, search).await + } _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs new file mode 100644 index 00000000..224be6c1 --- /dev/null +++ b/src/api/admin/key.rs @@ -0,0 +1,181 @@ +use std::collections::HashMap; +use std::net::SocketAddr; +use std::sync::Arc; + +use hyper::{Body, Request, Response, StatusCode}; +use serde::{Deserialize, Serialize}; + +use garage_util::crdt::*; +use garage_util::data::*; +use garage_util::error::Error as GarageError; + +use garage_rpc::layout::*; + +use garage_table::*; + +use garage_model::garage::Garage; +use garage_model::key_table::*; + +use crate::error::*; +use crate::helpers::*; + +pub async fn handle_list_keys(garage: &Arc) -> Result, Error> { + let res = garage + .key_table + .get_range( + &EmptyKey, + None, + Some(KeyFilter::Deleted(DeletedFilter::NotDeleted)), + 10000, + EnumerationOrder::Forward, + ) + .await? + .iter() + .map(|k| ListKeyResultItem { + id: k.key_id.to_string(), + name: k.params().unwrap().name.get().clone(), + }) + .collect::>(); + + let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(resp_json))?) +} + +#[derive(Serialize)] +struct ListKeyResultItem { + id: String, + name: String, +} + +pub async fn handle_get_key_info( + garage: &Arc, + id: Option, + search: Option, +) -> Result, Error> { + let key = if let Some(id) = id { + garage + .key_table + .get(&EmptyKey, &id) + .await? + .ok_or(Error::NoSuchKey)? + } else if let Some(search) = search { + garage + .bucket_helper() + .get_existing_matching_key(&search) + .await + .map_err(|_| Error::NoSuchKey)? + } else { + unreachable!(); + }; + + let mut relevant_buckets = HashMap::new(); + + let key_state = key.state.as_option().unwrap(); + + for id in key_state + .authorized_buckets + .items() + .iter() + .map(|(id, _)| id) + .chain( + key_state + .local_aliases + .items() + .iter() + .filter_map(|(_, _, v)| v.as_ref()), + ) { + if !relevant_buckets.contains_key(id) { + if let Some(b) = garage.bucket_table.get(&EmptyKey, id).await? { + if b.state.as_option().is_some() { + relevant_buckets.insert(*id, b); + } + } + } + } + + let res = GetKeyInfoResult { + name: key_state.name.get().clone(), + access_key_id: key.key_id.clone(), + secret_access_key: key_state.secret_key.clone(), + permissions: KeyPermResult { + create_bucket: *key_state.allow_create_bucket.get(), + }, + buckets: relevant_buckets + .into_iter() + .map(|(_, bucket)| { + let state = bucket.state.as_option().unwrap(); + KeyInfoBucketResult { + id: hex::encode(bucket.id), + global_aliases: state + .aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + local_aliases: state + .local_aliases + .items() + .iter() + .filter(|((k, _), _, a)| *a && *k == key.key_id) + .map(|((_, n), _, _)| n.to_string()) + .collect::>(), + permissions: key_state + .authorized_buckets + .get(&bucket.id) + .map(|p| KeyBucketPermResult { + read: p.allow_read, + write: p.allow_write, + owner: p.allow_owner, + }) + .unwrap_or(KeyBucketPermResult { + read: false, + write: false, + owner: false, + }), + } + }) + .collect::>(), + }; + + let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(resp_json))?) +} + +#[derive(Serialize)] +struct GetKeyInfoResult { + name: String, + #[serde(rename = "accessKeyId")] + access_key_id: String, + #[serde(rename = "secretAccessKey")] + secret_access_key: String, + permissions: KeyPermResult, + buckets: Vec, +} + +#[derive(Serialize)] +struct KeyPermResult { + #[serde(rename = "createBucket")] + create_bucket: bool, +} + +#[derive(Serialize)] +struct KeyInfoBucketResult { + id: String, + #[serde(rename = "globalAliases")] + global_aliases: Vec, + #[serde(rename = "localAliases")] + local_aliases: Vec, + permissions: KeyBucketPermResult, +} + +#[derive(Serialize)] +struct KeyBucketPermResult { + read: bool, + write: bool, + owner: bool, +} diff --git a/src/api/admin/mod.rs b/src/api/admin/mod.rs index 7e8d0635..f6c3b2ee 100644 --- a/src/api/admin/mod.rs +++ b/src/api/admin/mod.rs @@ -2,3 +2,4 @@ pub mod api_server; mod router; mod cluster; +mod key; diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index 7ff34aaa..626cced1 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -25,7 +25,8 @@ pub enum Endpoint { ListKeys, CreateKey, GetKeyInfo { - id: String, + id: Option, + search: Option, }, DeleteKey { id: String, @@ -56,7 +57,8 @@ impl Endpoint { POST "/layout/apply" => ApplyClusterLayout, POST "/layout/revert" => RevertClusterLayout, // API key endpoints - GET "/key" if id => GetKeyInfo (query::id), + GET "/key" if id => GetKeyInfo (query_opt::id, query_opt::search), + GET "/key" if search => GetKeyInfo (query_opt::id, query_opt::search), POST "/key" if id => UpdateKey (query::id), POST "/key" => CreateKey, DELETE "/key" if id => DeleteKey (query::id), @@ -79,5 +81,6 @@ impl Endpoint { } generateQueryParameters! { - "id" => id + "id" => id, + "search" => search } -- 2.43.4 From 393b76ecba66ff11b80bf404691704568f2d1794 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 11 May 2022 11:40:26 +0200 Subject: [PATCH 11/46] Implement CreateKey, DeleteKey and rudimentary UpdateKey --- src/api/admin/api_server.rs | 5 ++ src/api/admin/key.rs | 95 +++++++++++++++++++++++++++++---- src/garage/admin.rs | 57 ++++++++------------ src/model/garage.rs | 4 ++ src/model/helper/bucket.rs | 68 ++++-------------------- src/model/helper/key.rs | 102 ++++++++++++++++++++++++++++++++++++ src/model/helper/mod.rs | 1 + 7 files changed, 227 insertions(+), 105 deletions(-) create mode 100644 src/model/helper/key.rs diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index e44443ff..952f6a73 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -136,10 +136,15 @@ impl ApiHandler for AdminApiServer { Endpoint::GetKeyInfo { id, search } => { handle_get_key_info(&self.garage, id, search).await } + Endpoint::CreateKey => handle_create_key(&self.garage, req).await, + Endpoint::UpdateKey { id } => handle_update_key(&self.garage, id, req).await, + Endpoint::DeleteKey { id } => handle_delete_key(&self.garage, id).await, + /* _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() ))), + */ } } } diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 224be6c1..7cfe3fce 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -1,16 +1,11 @@ use std::collections::HashMap; -use std::net::SocketAddr; use std::sync::Arc; use hyper::{Body, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use garage_util::crdt::*; -use garage_util::data::*; use garage_util::error::Error as GarageError; -use garage_rpc::layout::*; - use garage_table::*; use garage_model::garage::Garage; @@ -62,7 +57,7 @@ pub async fn handle_get_key_info( .ok_or(Error::NoSuchKey)? } else if let Some(search) = search { garage - .bucket_helper() + .key_helper() .get_existing_matching_key(&search) .await .map_err(|_| Error::NoSuchKey)? @@ -70,6 +65,84 @@ pub async fn handle_get_key_info( unreachable!(); }; + key_info_results(garage, key).await +} + +pub async fn handle_create_key( + garage: &Arc, + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; + + let key = Key::new(&req.name); + garage.key_table.insert(&key).await?; + + key_info_results(garage, key).await +} + +#[derive(Deserialize)] +struct CreateKeyRequest { + name: String, +} + +pub async fn handle_update_key( + garage: &Arc, + id: String, + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; + + let mut key = garage + .key_table + .get(&EmptyKey, &id) + .await? + .ok_or(Error::NoSuchKey)?; + + let key_state = key.state.as_option_mut().ok_or(Error::NoSuchKey)?; + + if let Some(new_name) = req.name { + key_state.name.update(new_name); + } + if let Some(allow) = req.allow { + if allow.create_bucket { + key_state.allow_create_bucket.update(true); + } + } + if let Some(deny) = req.deny { + if deny.create_bucket { + key_state.allow_create_bucket.update(false); + } + } + + garage.key_table.insert(&key).await?; + + key_info_results(garage, key).await +} + +#[derive(Deserialize)] +struct UpdateKeyRequest { + name: Option, + allow: Option, + deny: Option, +} + +pub async fn handle_delete_key(garage: &Arc, id: String) -> Result, Error> { + let mut key = garage + .key_table + .get(&EmptyKey, &id) + .await? + .ok_or(Error::NoSuchKey)?; + + key.state.as_option().ok_or(Error::NoSuchKey)?; + + garage.key_helper().delete_key(&mut key).await?; + + Ok(Response::builder() + .status(StatusCode::NO_CONTENT) + .body(Body::empty())?) +} + +async fn key_info_results(garage: &Arc, key: Key) -> Result, Error> { let mut relevant_buckets = HashMap::new(); let key_state = key.state.as_option().unwrap(); @@ -99,7 +172,7 @@ pub async fn handle_get_key_info( name: key_state.name.get().clone(), access_key_id: key.key_id.clone(), secret_access_key: key_state.secret_key.clone(), - permissions: KeyPermResult { + permissions: KeyPerm { create_bucket: *key_state.allow_create_bucket.get(), }, buckets: relevant_buckets @@ -153,13 +226,13 @@ struct GetKeyInfoResult { access_key_id: String, #[serde(rename = "secretAccessKey")] secret_access_key: String, - permissions: KeyPermResult, + permissions: KeyPerm, buckets: Vec, } -#[derive(Serialize)] -struct KeyPermResult { - #[serde(rename = "createBucket")] +#[derive(Serialize, Deserialize)] +struct KeyPerm { + #[serde(rename = "createBucket", default)] create_bucket: bool, } diff --git a/src/garage/admin.rs b/src/garage/admin.rs index 1a58a613..c1ba297b 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -261,6 +261,7 @@ impl AdminRpcHandler { async fn handle_alias_bucket(&self, query: &AliasBucketOpt) -> Result { let helper = self.garage.bucket_helper(); + let key_helper = self.garage.key_helper(); let bucket_id = helper .resolve_global_bucket_name(&query.existing_bucket) @@ -268,7 +269,7 @@ impl AdminRpcHandler { .ok_or_bad_request("Bucket not found")?; if let Some(key_pattern) = &query.local { - let key = helper.get_existing_matching_key(key_pattern).await?; + let key = key_helper.get_existing_matching_key(key_pattern).await?; helper .set_local_bucket_alias(bucket_id, &key.key_id, &query.new_name) @@ -290,9 +291,10 @@ impl AdminRpcHandler { async fn handle_unalias_bucket(&self, query: &UnaliasBucketOpt) -> Result { let helper = self.garage.bucket_helper(); + let key_helper = self.garage.key_helper(); if let Some(key_pattern) = &query.local { - let key = helper.get_existing_matching_key(key_pattern).await?; + let key = key_helper.get_existing_matching_key(key_pattern).await?; let bucket_id = key .state @@ -331,12 +333,15 @@ impl AdminRpcHandler { async fn handle_bucket_allow(&self, query: &PermBucketOpt) -> Result { let helper = self.garage.bucket_helper(); + let key_helper = self.garage.key_helper(); let bucket_id = helper .resolve_global_bucket_name(&query.bucket) .await? .ok_or_bad_request("Bucket not found")?; - let key = helper.get_existing_matching_key(&query.key_pattern).await?; + let key = key_helper + .get_existing_matching_key(&query.key_pattern) + .await?; let allow_read = query.read || key.allow_read(&bucket_id); let allow_write = query.write || key.allow_write(&bucket_id); @@ -363,12 +368,15 @@ impl AdminRpcHandler { async fn handle_bucket_deny(&self, query: &PermBucketOpt) -> Result { let helper = self.garage.bucket_helper(); + let key_helper = self.garage.key_helper(); let bucket_id = helper .resolve_global_bucket_name(&query.bucket) .await? .ok_or_bad_request("Bucket not found")?; - let key = helper.get_existing_matching_key(&query.key_pattern).await?; + let key = key_helper + .get_existing_matching_key(&query.key_pattern) + .await?; let allow_read = !query.read && key.allow_read(&bucket_id); let allow_write = !query.write && key.allow_write(&bucket_id); @@ -469,7 +477,7 @@ impl AdminRpcHandler { async fn handle_key_info(&self, query: &KeyOpt) -> Result { let key = self .garage - .bucket_helper() + .key_helper() .get_existing_matching_key(&query.key_pattern) .await?; self.key_info_result(key).await @@ -484,7 +492,7 @@ impl AdminRpcHandler { async fn handle_rename_key(&self, query: &KeyRenameOpt) -> Result { let mut key = self .garage - .bucket_helper() + .key_helper() .get_existing_matching_key(&query.key_pattern) .await?; key.params_mut() @@ -496,9 +504,11 @@ impl AdminRpcHandler { } async fn handle_delete_key(&self, query: &KeyDeleteOpt) -> Result { - let helper = self.garage.bucket_helper(); + let key_helper = self.garage.key_helper(); - let mut key = helper.get_existing_matching_key(&query.key_pattern).await?; + let mut key = key_helper + .get_existing_matching_key(&query.key_pattern) + .await?; if !query.yes { return Err(Error::BadRequest( @@ -506,32 +516,7 @@ impl AdminRpcHandler { )); } - let state = key.state.as_option_mut().unwrap(); - - // --- done checking, now commit --- - // (the step at unset_local_bucket_alias will fail if a bucket - // does not have another alias, the deletion will be - // interrupted in the middle if that happens) - - // 1. Delete local aliases - for (alias, _, to) in state.local_aliases.items().iter() { - if let Some(bucket_id) = to { - helper - .unset_local_bucket_alias(*bucket_id, &key.key_id, alias) - .await?; - } - } - - // 2. Remove permissions on all authorized buckets - for (ab_id, _auth) in state.authorized_buckets.items().iter() { - helper - .set_bucket_key_permissions(*ab_id, &key.key_id, BucketKeyPerm::NO_PERMISSIONS) - .await?; - } - - // 3. Actually delete key - key.state = Deletable::delete(); - self.garage.key_table.insert(&key).await?; + key_helper.delete_key(&mut key).await?; Ok(AdminRpc::Ok(format!( "Key {} was deleted successfully.", @@ -542,7 +527,7 @@ impl AdminRpcHandler { async fn handle_allow_key(&self, query: &KeyPermOpt) -> Result { let mut key = self .garage - .bucket_helper() + .key_helper() .get_existing_matching_key(&query.key_pattern) .await?; if query.create_bucket { @@ -555,7 +540,7 @@ impl AdminRpcHandler { async fn handle_deny_key(&self, query: &KeyPermOpt) -> Result { let mut key = self .garage - .bucket_helper() + .key_helper() .get_existing_matching_key(&query.key_pattern) .await?; if query.create_bucket { diff --git a/src/model/garage.rs b/src/model/garage.rs index 03e21f8a..2f99bd68 100644 --- a/src/model/garage.rs +++ b/src/model/garage.rs @@ -191,6 +191,10 @@ impl Garage { pub fn bucket_helper(&self) -> helper::bucket::BucketHelper { helper::bucket::BucketHelper(self) } + + pub fn key_helper(&self) -> helper::key::KeyHelper { + helper::key::KeyHelper(self) + } } #[cfg(feature = "k2v")] diff --git a/src/model/helper/bucket.rs b/src/model/helper/bucket.rs index 54d2f97b..7e81b946 100644 --- a/src/model/helper/bucket.rs +++ b/src/model/helper/bucket.rs @@ -8,7 +8,7 @@ use crate::bucket_alias_table::*; use crate::bucket_table::*; use crate::garage::Garage; use crate::helper::error::*; -use crate::key_table::{Key, KeyFilter}; +use crate::helper::key::KeyHelper; use crate::permission::BucketKeyPerm; pub struct BucketHelper<'a>(pub(crate) &'a Garage); @@ -77,60 +77,6 @@ impl<'a> BucketHelper<'a> { )) } - /// Returns a Key if it is present in key table, - /// even if it is in deleted state. Querying a non-existing - /// key ID returns an internal error. - pub async fn get_internal_key(&self, key_id: &String) -> Result { - Ok(self - .0 - .key_table - .get(&EmptyKey, key_id) - .await? - .ok_or_message(format!("Key {} does not exist", key_id))?) - } - - /// Returns a Key if it is present in key table, - /// only if it is in non-deleted state. - /// Querying a non-existing key ID or a deleted key - /// returns a bad request error. - pub async fn get_existing_key(&self, key_id: &String) -> Result { - self.0 - .key_table - .get(&EmptyKey, key_id) - .await? - .filter(|b| !b.state.is_deleted()) - .ok_or_bad_request(format!("Key {} does not exist or has been deleted", key_id)) - } - - /// Returns a Key if it is present in key table, - /// looking it up by key ID or by a match on its name, - /// only if it is in non-deleted state. - /// Querying a non-existing key ID or a deleted key - /// returns a bad request error. - pub async fn get_existing_matching_key(&self, pattern: &str) -> Result { - let candidates = self - .0 - .key_table - .get_range( - &EmptyKey, - None, - Some(KeyFilter::MatchesAndNotDeleted(pattern.to_string())), - 10, - EnumerationOrder::Forward, - ) - .await? - .into_iter() - .collect::>(); - if candidates.len() != 1 { - Err(Error::BadRequest(format!( - "{} matching keys", - candidates.len() - ))) - } else { - Ok(candidates.into_iter().next().unwrap()) - } - } - /// Sets a new alias for a bucket in global namespace. /// This function fails if: /// - alias name is not valid according to S3 spec @@ -303,6 +249,8 @@ impl<'a> BucketHelper<'a> { key_id: &String, alias_name: &String, ) -> Result<(), Error> { + let key_helper = KeyHelper(self.0); + if !is_valid_bucket_name(alias_name) { return Err(Error::BadRequest(format!( "{}: {}", @@ -311,7 +259,7 @@ impl<'a> BucketHelper<'a> { } let mut bucket = self.get_existing_bucket(bucket_id).await?; - let mut key = self.get_existing_key(key_id).await?; + let mut key = key_helper.get_existing_key(key_id).await?; let mut key_param = key.state.as_option_mut().unwrap(); @@ -360,8 +308,10 @@ impl<'a> BucketHelper<'a> { key_id: &String, alias_name: &String, ) -> Result<(), Error> { + let key_helper = KeyHelper(self.0); + let mut bucket = self.get_existing_bucket(bucket_id).await?; - let mut key = self.get_existing_key(key_id).await?; + let mut key = key_helper.get_existing_key(key_id).await?; let mut bucket_p = bucket.state.as_option_mut().unwrap(); @@ -429,8 +379,10 @@ impl<'a> BucketHelper<'a> { key_id: &String, mut perm: BucketKeyPerm, ) -> Result<(), Error> { + let key_helper = KeyHelper(self.0); + let mut bucket = self.get_internal_bucket(bucket_id).await?; - let mut key = self.get_internal_key(key_id).await?; + let mut key = key_helper.get_internal_key(key_id).await?; if let Some(bstate) = bucket.state.as_option() { if let Some(kp) = bstate.authorized_keys.get(key_id) { diff --git a/src/model/helper/key.rs b/src/model/helper/key.rs new file mode 100644 index 00000000..eea37f79 --- /dev/null +++ b/src/model/helper/key.rs @@ -0,0 +1,102 @@ +use garage_table::util::*; +use garage_util::crdt::*; +use garage_util::error::OkOrMessage; + +use crate::garage::Garage; +use crate::helper::bucket::BucketHelper; +use crate::helper::error::*; +use crate::key_table::{Key, KeyFilter}; +use crate::permission::BucketKeyPerm; + +pub struct KeyHelper<'a>(pub(crate) &'a Garage); + +#[allow(clippy::ptr_arg)] +impl<'a> KeyHelper<'a> { + /// Returns a Key if it is present in key table, + /// even if it is in deleted state. Querying a non-existing + /// key ID returns an internal error. + pub async fn get_internal_key(&self, key_id: &String) -> Result { + Ok(self + .0 + .key_table + .get(&EmptyKey, key_id) + .await? + .ok_or_message(format!("Key {} does not exist", key_id))?) + } + + /// Returns a Key if it is present in key table, + /// only if it is in non-deleted state. + /// Querying a non-existing key ID or a deleted key + /// returns a bad request error. + pub async fn get_existing_key(&self, key_id: &String) -> Result { + self.0 + .key_table + .get(&EmptyKey, key_id) + .await? + .filter(|b| !b.state.is_deleted()) + .ok_or_bad_request(format!("Key {} does not exist or has been deleted", key_id)) + } + + /// Returns a Key if it is present in key table, + /// looking it up by key ID or by a match on its name, + /// only if it is in non-deleted state. + /// Querying a non-existing key ID or a deleted key + /// returns a bad request error. + pub async fn get_existing_matching_key(&self, pattern: &str) -> Result { + let candidates = self + .0 + .key_table + .get_range( + &EmptyKey, + None, + Some(KeyFilter::MatchesAndNotDeleted(pattern.to_string())), + 10, + EnumerationOrder::Forward, + ) + .await? + .into_iter() + .collect::>(); + if candidates.len() != 1 { + Err(Error::BadRequest(format!( + "{} matching keys", + candidates.len() + ))) + } else { + Ok(candidates.into_iter().next().unwrap()) + } + } + + /// Deletes an API access key + pub async fn delete_key(&self, key: &mut Key) -> Result<(), Error> { + let bucket_helper = BucketHelper(self.0); + + let state = key.state.as_option_mut().unwrap(); + + // --- done checking, now commit --- + // (the step at unset_local_bucket_alias will fail if a bucket + // does not have another alias, the deletion will be + // interrupted in the middle if that happens) + + // 1. Delete local aliases + for (alias, _, to) in state.local_aliases.items().iter() { + if let Some(bucket_id) = to { + bucket_helper + .unset_local_bucket_alias(*bucket_id, &key.key_id, alias) + .await?; + } + } + + // 2. Remove permissions on all authorized buckets + for (ab_id, _auth) in state.authorized_buckets.items().iter() { + bucket_helper + .set_bucket_key_permissions(*ab_id, &key.key_id, BucketKeyPerm::NO_PERMISSIONS) + .await?; + } + + // 3. Actually delete key + key.state = Deletable::delete(); + self.0.key_table.insert(key).await?; + + Ok(()) + } +} diff --git a/src/model/helper/mod.rs b/src/model/helper/mod.rs index 2f4e8898..dd947c86 100644 --- a/src/model/helper/mod.rs +++ b/src/model/helper/mod.rs @@ -1,2 +1,3 @@ pub mod bucket; pub mod error; +pub mod key; -- 2.43.4 From aeb978552a10eb89d183cdec04d242369127d764 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 11 May 2022 11:51:11 +0200 Subject: [PATCH 12/46] Short doc on UpdateKey --- doc/drafts/admin-api.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index dc89014a..840dd4f7 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -326,6 +326,16 @@ Updates information about the specified API access key. Request body format: ```json -#TODO +{ + "name": "NameOfMyKey", + "allow": { + "createBucket": true, + }, + "deny": {} +} ``` +All fields (`name`, `allow` and `deny`) are optionnal. +If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed. +The possible flags in `allow` and `deny` are: `createBucket`. + -- 2.43.4 From 2b93a01d2bead391e15f529dc5b4db80dcbeeba8 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 12 May 2022 10:20:34 +0200 Subject: [PATCH 13/46] ListBucket and GetBucketInfo --- doc/drafts/admin-api.md | 114 ++++++++++++++++++++ src/api/admin/api_server.rs | 8 +- src/api/admin/bucket.rs | 208 ++++++++++++++++++++++++++++++++++++ src/api/admin/key.rs | 16 ++- src/api/admin/mod.rs | 1 + src/api/admin/router.rs | 21 +++- 6 files changed, 355 insertions(+), 13 deletions(-) create mode 100644 src/api/admin/bucket.rs diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 840dd4f7..edfdeae7 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -339,3 +339,117 @@ All fields (`name`, `allow` and `deny`) are optionnal. If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed. The possible flags in `allow` and `deny` are: `createBucket`. + +## Bucket operations + +### ListBuckets `GET /bucket` + +Returns all storage buckets in the cluster. + +Example response: + +```json +[ + { + "id": "70dc3bed7fe83a75e46b66e7ddef7d56e65f3c02f9f80b6749fb97eccb5e1033", + "globalAliases": [ + "test2" + ], + "localAliases": [] + }, + { + "id": "96470e0df00ec28807138daf01915cfda2bee8eccc91dea9558c0b4855b5bf95", + "globalAliases": [ + "alex" + ], + "localAliases": [] + }, + { + "id": "d7452a935e663fc1914f3a5515163a6d3724010ce8dfd9e4743ca8be5974f995", + "globalAliases": [ + "test3" + ], + "localAliases": [] + }, + { + "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b", + "globalAliases": [], + "localAliases": [ + { + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "alias": "test" + } + ] + } +] +``` + +### GetBucketInfo `GET /bucket?id=` +### GetBucketInfo `GET /bucket?globalAlias=` + +Returns information about the requested storage bucket. + +If `id` is set, the bucket is looked up using its exact identifier. +If `globalAlias` is set, the bucket is looked up using its global alias. +(both are fast) + +Example response: + +```json +{ + "id": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b", + "globalAliases": [ + "alex" + ], + "keys": [ + { + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "name": "alex", + "permissions": { + "read": true, + "write": true, + "owner": true + }, + "bucketLocalAliases": [ + "test" + ] + } + ] +} +``` + +### CreateBucket `POST /bucket` + +Creates a new storage bucket. + +Request body format: + +```json +{ + "globalAlias": "NameOfMyBucket" +} +``` + +OR + +```json +{ + "localAlias": { + "key": "GK31c2f218a2e44f485b94239e", + "alias": "NameOfMyBucket" + } +} +``` + +OR + +```json +{} +``` + +Creates a new bucket, either with a global alias, a local one, +or no alias at all. + +### DeleteBucket `DELETE /bucket?id=` + +Deletes a storage bucket. A bucket cannot be deleted if it is not empty. diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 952f6a73..8e5a8c6c 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -18,6 +18,7 @@ use garage_util::error::Error as GarageError; use crate::error::*; use crate::generic_server::*; +use crate::admin::bucket::*; use crate::admin::cluster::*; use crate::admin::key::*; use crate::admin::router::{Authorization, Endpoint}; @@ -139,12 +140,15 @@ impl ApiHandler for AdminApiServer { Endpoint::CreateKey => handle_create_key(&self.garage, req).await, Endpoint::UpdateKey { id } => handle_update_key(&self.garage, id, req).await, Endpoint::DeleteKey { id } => handle_delete_key(&self.garage, id).await, - /* + // Buckets + Endpoint::ListBuckets => handle_list_buckets(&self.garage).await, + Endpoint::GetBucketInfo { id, global_alias } => { + handle_get_bucket_info(&self.garage, id, global_alias).await + } _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() ))), - */ } } } diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs new file mode 100644 index 00000000..003203c1 --- /dev/null +++ b/src/api/admin/bucket.rs @@ -0,0 +1,208 @@ +use std::collections::HashMap; +use std::sync::Arc; + +use hyper::{Body, Request, Response, StatusCode}; +use serde::{Deserialize, Serialize}; + +use garage_util::data::*; +use garage_util::error::Error as GarageError; + +use garage_table::*; + +use garage_model::bucket_table::*; +use garage_model::garage::Garage; +use garage_model::key_table::*; + +use crate::admin::key::KeyBucketPermResult; +use crate::error::*; +use crate::helpers::*; + +pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { + let buckets = garage + .bucket_table + .get_range( + &EmptyKey, + None, + Some(DeletedFilter::NotDeleted), + 10000, + EnumerationOrder::Forward, + ) + .await?; + + let res = buckets + .into_iter() + .map(|b| { + let state = b.state.as_option().unwrap(); + ListBucketResultItem { + id: hex::encode(b.id), + global_aliases: state + .aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + local_aliases: state + .local_aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|((k, n), _, _)| ListBucketLocalAlias { + access_key_id: k.to_string(), + alias: n.to_string(), + }) + .collect::>(), + } + }) + .collect::>(); + + let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(resp_json))?) +} + +#[derive(Serialize)] +struct ListBucketResultItem { + id: String, + #[serde(rename = "globalAliases")] + global_aliases: Vec, + #[serde(rename = "localAliases")] + local_aliases: Vec, +} + +#[derive(Serialize)] +struct ListBucketLocalAlias { + #[serde(rename = "accessKeyId")] + access_key_id: String, + alias: String, +} + +pub async fn handle_get_bucket_info( + garage: &Arc, + id: Option, + global_alias: Option, +) -> Result, Error> { + let bucket_id = match (id, global_alias) { + (Some(id), None) => { + let id_hex = hex::decode(&id).ok_or_bad_request("Invalid bucket id")?; + Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")? + } + (None, Some(ga)) => garage + .bucket_helper() + .resolve_global_bucket_name(&ga) + .await? + .ok_or_bad_request("Bucket not found")?, + _ => { + return Err(Error::BadRequest( + "Either id or globalAlias must be provided (but not both)".into(), + )) + } + }; + + let bucket = garage + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; + + let mut relevant_keys = HashMap::new(); + for (k, _) in bucket + .state + .as_option() + .unwrap() + .authorized_keys + .items() + .iter() + { + if let Some(key) = garage + .key_table + .get(&EmptyKey, k) + .await? + .filter(|k| !k.is_deleted()) + { + if !key.state.is_deleted() { + relevant_keys.insert(k.clone(), key); + } + } + } + for ((k, _), _, _) in bucket + .state + .as_option() + .unwrap() + .local_aliases + .items() + .iter() + { + if relevant_keys.contains_key(k) { + continue; + } + if let Some(key) = garage.key_table.get(&EmptyKey, k).await? { + if !key.state.is_deleted() { + relevant_keys.insert(k.clone(), key); + } + } + } + + let state = bucket.state.as_option().unwrap(); + + let res = GetBucketInfoResult { + id: hex::encode(&bucket.id), + global_aliases: state + .aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + keys: relevant_keys + .into_iter() + .map(|(_, key)| { + let p = key.state.as_option().unwrap(); + GetBucketInfoKey { + access_key_id: key.key_id, + name: p.name.get().to_string(), + permissions: p + .authorized_buckets + .get(&bucket.id) + .map(|p| KeyBucketPermResult { + read: p.allow_read, + write: p.allow_write, + owner: p.allow_owner, + }) + .unwrap_or_default(), + bucket_local_aliases: p + .local_aliases + .items() + .iter() + .filter(|(_, _, b)| *b == Some(bucket.id)) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + } + }) + .collect::>(), + }; + + let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(resp_json))?) +} + +#[derive(Serialize)] +struct GetBucketInfoResult { + id: String, + #[serde(rename = "globalAliases")] + global_aliases: Vec, + keys: Vec, +} + +#[derive(Serialize)] +struct GetBucketInfoKey { + #[serde(rename = "accessKeyId")] + access_key_id: String, + #[serde(rename = "name")] + name: String, + permissions: KeyBucketPermResult, + #[serde(rename = "bucketLocalAliases")] + bucket_local_aliases: Vec, +} diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 7cfe3fce..1252d2c8 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -203,11 +203,7 @@ async fn key_info_results(garage: &Arc, key: Key) -> Result>(), @@ -246,9 +242,9 @@ struct KeyInfoBucketResult { permissions: KeyBucketPermResult, } -#[derive(Serialize)] -struct KeyBucketPermResult { - read: bool, - write: bool, - owner: bool, +#[derive(Serialize, Default)] +pub(crate) struct KeyBucketPermResult { + pub(crate) read: bool, + pub(crate) write: bool, + pub(crate) owner: bool, } diff --git a/src/api/admin/mod.rs b/src/api/admin/mod.rs index f6c3b2ee..05097c8b 100644 --- a/src/api/admin/mod.rs +++ b/src/api/admin/mod.rs @@ -1,5 +1,6 @@ pub mod api_server; mod router; +mod bucket; mod cluster; mod key; diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index 626cced1..a6e1c848 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -18,10 +18,12 @@ pub enum Endpoint { Options, Metrics, GetClusterStatus, + // Layout GetClusterLayout, UpdateClusterLayout, ApplyClusterLayout, RevertClusterLayout, + // Keys ListKeys, CreateKey, GetKeyInfo { @@ -34,6 +36,16 @@ pub enum Endpoint { UpdateKey { id: String, }, + // Buckets + ListBuckets, + CreateBucket, + GetBucketInfo { + id: Option, + global_alias: Option, + }, + DeleteBucket { + id: String, + }, }} impl Endpoint { @@ -63,6 +75,12 @@ impl Endpoint { POST "/key" => CreateKey, DELETE "/key" if id => DeleteKey (query::id), GET "/key" => ListKeys, + // Bucket endpoints + GET "/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/bucket" => ListBuckets, + POST "/bucket" => CreateBucket, + DELETE "/bucket" if id => DeleteBucket (query::id), ]); if let Some(message) = query.nonempty_message() { @@ -82,5 +100,6 @@ impl Endpoint { generateQueryParameters! { "id" => id, - "search" => search + "search" => search, + "globalAlias" => global_alias } -- 2.43.4 From de1a5b87b6980c66966fbe2b2f35b925fd83308b Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 12 May 2022 10:45:09 +0200 Subject: [PATCH 14/46] CreateBucket --- doc/drafts/admin-api.md | 5 +- src/api/admin/api_server.rs | 1 + src/api/admin/bucket.rs | 108 ++++++++++++++++++++++++++++++++++-- 3 files changed, 108 insertions(+), 6 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index edfdeae7..b24e4cf4 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -435,8 +435,9 @@ OR ```json { "localAlias": { - "key": "GK31c2f218a2e44f485b94239e", - "alias": "NameOfMyBucket" + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "alias": "NameOfMyBucket", + "allPermissions": true } } ``` diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 8e5a8c6c..783ecfcb 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -145,6 +145,7 @@ impl ApiHandler for AdminApiServer { Endpoint::GetBucketInfo { id, global_alias } => { handle_get_bucket_info(&self.garage, id, global_alias).await } + Endpoint::CreateBucket => handle_create_bucket(&self.garage, req).await, _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 003203c1..723391c5 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -9,9 +9,10 @@ use garage_util::error::Error as GarageError; use garage_table::*; +use garage_model::bucket_alias_table::*; use garage_model::bucket_table::*; use garage_model::garage::Garage; -use garage_model::key_table::*; +use garage_model::permission::*; use crate::admin::key::KeyBucketPermResult; use crate::error::*; @@ -47,7 +48,7 @@ pub async fn handle_list_buckets(garage: &Arc) -> Result, .items() .iter() .filter(|(_, _, a)| *a) - .map(|((k, n), _, _)| ListBucketLocalAlias { + .map(|((k, n), _, _)| BucketLocalAlias { access_key_id: k.to_string(), alias: n.to_string(), }) @@ -68,11 +69,11 @@ struct ListBucketResultItem { #[serde(rename = "globalAliases")] global_aliases: Vec, #[serde(rename = "localAliases")] - local_aliases: Vec, + local_aliases: Vec, } #[derive(Serialize)] -struct ListBucketLocalAlias { +struct BucketLocalAlias { #[serde(rename = "accessKeyId")] access_key_id: String, alias: String, @@ -105,6 +106,13 @@ pub async fn handle_get_bucket_info( .get_existing_bucket(bucket_id) .await?; + bucket_info_results(garage, bucket).await +} + +async fn bucket_info_results( + garage: &Arc, + bucket: Bucket, +) -> Result, Error> { let mut relevant_keys = HashMap::new(); for (k, _) in bucket .state @@ -206,3 +214,95 @@ struct GetBucketInfoKey { #[serde(rename = "bucketLocalAliases")] bucket_local_aliases: Vec, } + +pub async fn handle_create_bucket( + garage: &Arc, + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; + + if let Some(ga) = &req.global_alias { + if !is_valid_bucket_name(ga) { + return Err(Error::BadRequest(format!( + "{}: {}", + ga, INVALID_BUCKET_NAME_MESSAGE + ))); + } + + if let Some(alias) = garage.bucket_alias_table.get(&EmptyKey, ga).await? { + if alias.state.get().is_some() { + return Err(Error::BucketAlreadyExists); + } + } + } + + if let Some(la) = &req.local_alias { + if !is_valid_bucket_name(&la.alias) { + return Err(Error::BadRequest(format!( + "{}: {}", + la.alias, INVALID_BUCKET_NAME_MESSAGE + ))); + } + + let key = garage + .key_table + .get(&EmptyKey, &la.access_key_id) + .await? + .ok_or(Error::NoSuchKey)?; + let state = key.state.as_option().ok_or(Error::NoSuchKey)?; + if matches!(state.local_aliases.get(&la.alias), Some(_)) { + return Err(Error::BadRequest("Local alias already exists".into())); + } + } + + let bucket = Bucket::new(); + garage.bucket_table.insert(&bucket).await?; + + if let Some(ga) = &req.global_alias { + garage + .bucket_helper() + .set_global_bucket_alias(bucket.id, ga) + .await?; + } + + if let Some(la) = &req.local_alias { + garage + .bucket_helper() + .set_local_bucket_alias(bucket.id, &la.access_key_id, &la.alias) + .await?; + if la.all_permissions { + garage + .bucket_helper() + .set_bucket_key_permissions( + bucket.id, + &la.access_key_id, + BucketKeyPerm::ALL_PERMISSIONS, + ) + .await?; + } + } + + let bucket = garage + .bucket_table + .get(&EmptyKey, &bucket.id) + .await? + .ok_or_internal_error("Bucket should now exist but doesn't")?; + bucket_info_results(garage, bucket).await +} + +#[derive(Deserialize)] +struct CreateBucketRequest { + #[serde(rename = "globalAlias")] + global_alias: Option, + #[serde(rename = "localAlias")] + local_alias: Option, +} + +#[derive(Deserialize)] +struct CreateBucketLocalAlias { + #[serde(rename = "accessKeyId")] + access_key_id: String, + alias: String, + #[serde(rename = "allPermissions", default)] + all_permissions: bool, +} -- 2.43.4 From fe399a326506a9d8870cb7783a57495849793d2c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 12 May 2022 11:02:36 +0200 Subject: [PATCH 15/46] DeleteBucket --- Makefile | 2 +- doc/drafts/admin-api.md | 2 ++ src/api/admin/api_server.rs | 1 + src/api/admin/bucket.rs | 62 +++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c70be9da..eeeffedb 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ .PHONY: doc all release shell all: - clear; cargo build --features k2v + clear; cargo build --all-features doc: cd doc/book; mdbook build diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index b24e4cf4..048b77fb 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -454,3 +454,5 @@ or no alias at all. ### DeleteBucket `DELETE /bucket?id=` Deletes a storage bucket. A bucket cannot be deleted if it is not empty. + +Warning: this will delete all aliases associated with the bucket! diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 783ecfcb..4366cbd6 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -146,6 +146,7 @@ impl ApiHandler for AdminApiServer { handle_get_bucket_info(&self.garage, id, global_alias).await } Endpoint::CreateBucket => handle_create_bucket(&self.garage, req).await, + Endpoint::DeleteBucket { id } => handle_delete_bucket(&self.garage, id).await, _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 723391c5..8e6cc067 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use hyper::{Body, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; +use garage_util::crdt::*; use garage_util::data::*; use garage_util::error::Error as GarageError; @@ -13,6 +14,7 @@ use garage_model::bucket_alias_table::*; use garage_model::bucket_table::*; use garage_model::garage::Garage; use garage_model::permission::*; +use garage_model::s3::object_table::ObjectFilter; use crate::admin::key::KeyBucketPermResult; use crate::error::*; @@ -306,3 +308,63 @@ struct CreateBucketLocalAlias { #[serde(rename = "allPermissions", default)] all_permissions: bool, } + +pub async fn handle_delete_bucket( + garage: &Arc, + id: String, +) -> Result, Error> { + let helper = garage.bucket_helper(); + + let id_hex = hex::decode(&id).ok_or_bad_request("Invalid bucket id")?; + let bucket_id = Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")?; + + let mut bucket = helper.get_existing_bucket(bucket_id).await?; + let state = bucket.state.as_option().unwrap(); + + // Check bucket is empty + let objects = garage + .object_table + .get_range( + &bucket_id, + None, + Some(ObjectFilter::IsData), + 10, + EnumerationOrder::Forward, + ) + .await?; + if !objects.is_empty() { + return Err(Error::BadRequest("Bucket is not empty".into())); + } + + // --- done checking, now commit --- + // 1. delete authorization from keys that had access + for (key_id, perm) in bucket.authorized_keys() { + if perm.is_any() { + helper + .set_bucket_key_permissions(bucket.id, key_id, BucketKeyPerm::NO_PERMISSIONS) + .await?; + } + } + // 2. delete all local aliases + for ((key_id, alias), _, active) in state.local_aliases.items().iter() { + if *active { + helper + .unset_local_bucket_alias(bucket.id, &key_id, &alias) + .await?; + } + } + // 3. delete all global aliases + for (alias, _, active) in state.aliases.items().iter() { + if *active { + helper.purge_global_bucket_alias(bucket.id, &alias).await?; + } + } + + // 4. delete bucket + bucket.state = Deletable::delete(); + garage.bucket_table.insert(&bucket).await?; + + Ok(Response::builder() + .status(StatusCode::NO_CONTENT) + .body(Body::empty())?) +} -- 2.43.4 From fc2f73ddb5ecaca250daa7b034fe59fb8c47f570 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 12 May 2022 11:19:41 +0200 Subject: [PATCH 16/46] BucketAllowKey and BucketDenyKey --- doc/drafts/admin-api.md | 45 +++++++++++++++++++++ src/api/admin/api_server.rs | 3 ++ src/api/admin/bucket.rs | 79 +++++++++++++++++++++++++++++++++++-- src/api/admin/key.rs | 11 ++++-- src/api/admin/router.rs | 6 +++ 5 files changed, 137 insertions(+), 7 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 048b77fb..5dc3f127 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -456,3 +456,48 @@ or no alias at all. Deletes a storage bucket. A bucket cannot be deleted if it is not empty. Warning: this will delete all aliases associated with the bucket! + + +## Operations on permissions for keys on buckets + +### BucketAllowKey `POST /bucket/allow` + +Allows a key to do read/write/owner operations on a bucket. + +Request body format: + +```json +{ + "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b", + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "permissions": { + "read": true, + "write": true, + "owner": true + }, +} +``` + +Flags in `permissions` which have the value `true` will be activated. +Other flags will remain unchanged. + +### BucketDenyKey `POST /bucket/deny` + +Denies a key from doing read/write/owner operations on a bucket. + +Request body format: + +```json +{ + "bucketId": "e6a14cd6a27f48684579ec6b381c078ab11697e6bc8513b72b2f5307e25fff9b", + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "permissions": { + "read": false, + "write": false, + "owner": true + }, +} +``` + +Flags in `permissions` which have the value `true` will be deactivated. +Other flags will remain unchanged. diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 4366cbd6..6bdef56c 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -147,6 +147,9 @@ impl ApiHandler for AdminApiServer { } Endpoint::CreateBucket => handle_create_bucket(&self.garage, req).await, Endpoint::DeleteBucket { id } => handle_delete_bucket(&self.garage, id).await, + // Bucket-key permissions + Endpoint::BucketAllowKey => handle_bucket_allow_key(&self.garage, req).await, + Endpoint::BucketDenyKey => handle_bucket_deny_key(&self.garage, req).await, _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 8e6cc067..16e9c174 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -16,7 +16,7 @@ use garage_model::garage::Garage; use garage_model::permission::*; use garage_model::s3::object_table::ObjectFilter; -use crate::admin::key::KeyBucketPermResult; +use crate::admin::key::ApiBucketKeyPerm; use crate::error::*; use crate::helpers::*; @@ -174,7 +174,7 @@ async fn bucket_info_results( permissions: p .authorized_buckets .get(&bucket.id) - .map(|p| KeyBucketPermResult { + .map(|p| ApiBucketKeyPerm { read: p.allow_read, write: p.allow_write, owner: p.allow_owner, @@ -212,7 +212,7 @@ struct GetBucketInfoKey { access_key_id: String, #[serde(rename = "name")] name: String, - permissions: KeyBucketPermResult, + permissions: ApiBucketKeyPerm, #[serde(rename = "bucketLocalAliases")] bucket_local_aliases: Vec, } @@ -368,3 +368,76 @@ pub async fn handle_delete_bucket( .status(StatusCode::NO_CONTENT) .body(Body::empty())?) } + +pub async fn handle_bucket_allow_key( + garage: &Arc, + req: Request, +) -> Result, Error> { + handle_bucket_change_key_perm(garage, req, true).await +} + +pub async fn handle_bucket_deny_key( + garage: &Arc, + req: Request, +) -> Result, Error> { + handle_bucket_change_key_perm(garage, req, false).await +} + +pub async fn handle_bucket_change_key_perm( + garage: &Arc, + req: Request, + new_perm_flag: bool, +) -> Result, Error> { + let req = parse_json_body::(req).await?; + + let id_hex = hex::decode(&req.bucket_id).ok_or_bad_request("Invalid bucket id")?; + let bucket_id = Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")?; + + let bucket = garage + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; + let state = bucket.state.as_option().unwrap(); + + let key = garage + .key_helper() + .get_existing_key(&req.access_key_id) + .await?; + + let mut perm = state + .authorized_keys + .get(&key.key_id) + .cloned() + .unwrap_or(BucketKeyPerm::NO_PERMISSIONS); + + if req.permissions.read { + perm.allow_read = new_perm_flag; + } + if req.permissions.write { + perm.allow_write = new_perm_flag; + } + if req.permissions.owner { + perm.allow_owner = new_perm_flag; + } + + garage + .bucket_helper() + .set_bucket_key_permissions(bucket.id, &key.key_id, perm) + .await?; + + let bucket = garage + .bucket_table + .get(&EmptyKey, &bucket.id) + .await? + .ok_or_internal_error("Bucket should now exist but doesn't")?; + bucket_info_results(garage, bucket).await +} + +#[derive(Deserialize)] +struct BucketKeyPermChangeRequest { + #[serde(rename = "bucketId")] + bucket_id: String, + #[serde(rename = "accessKeyId")] + access_key_id: String, + permissions: ApiBucketKeyPerm, +} diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 1252d2c8..19ad5160 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -198,7 +198,7 @@ async fn key_info_results(garage: &Arc, key: Key) -> Result, #[serde(rename = "localAliases")] local_aliases: Vec, - permissions: KeyBucketPermResult, + permissions: ApiBucketKeyPerm, } -#[derive(Serialize, Default)] -pub(crate) struct KeyBucketPermResult { +#[derive(Serialize, Deserialize, Default)] +pub(crate) struct ApiBucketKeyPerm { + #[serde(default)] pub(crate) read: bool, + #[serde(default)] pub(crate) write: bool, + #[serde(default)] pub(crate) owner: bool, } diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index a6e1c848..6f787fe9 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -46,6 +46,9 @@ pub enum Endpoint { DeleteBucket { id: String, }, + // Bucket-Key Permissions + BucketAllowKey, + BucketDenyKey, }} impl Endpoint { @@ -81,6 +84,9 @@ impl Endpoint { GET "/bucket" => ListBuckets, POST "/bucket" => CreateBucket, DELETE "/bucket" if id => DeleteBucket (query::id), + // Bucket-key permissions + POST "/bucket/allow" => BucketAllowKey, + POST "/bucket/deny" => BucketDenyKey, ]); if let Some(message) = query.nonempty_message() { -- 2.43.4 From ed768935815851a7e4b8880f0cb8fc91e35e3027 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 12 May 2022 11:21:23 +0200 Subject: [PATCH 17/46] Simplify --- src/api/admin/api_server.rs | 8 ++++++-- src/api/admin/bucket.rs | 14 -------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 6bdef56c..b3bc9221 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -148,8 +148,12 @@ impl ApiHandler for AdminApiServer { Endpoint::CreateBucket => handle_create_bucket(&self.garage, req).await, Endpoint::DeleteBucket { id } => handle_delete_bucket(&self.garage, id).await, // Bucket-key permissions - Endpoint::BucketAllowKey => handle_bucket_allow_key(&self.garage, req).await, - Endpoint::BucketDenyKey => handle_bucket_deny_key(&self.garage, req).await, + Endpoint::BucketAllowKey => { + handle_bucket_change_key_perm(&self.garage, req, true).await + } + Endpoint::BucketDenyKey => { + handle_bucket_change_key_perm(&self.garage, req, false).await + } _ => Err(Error::NotImplemented(format!( "Admin endpoint {} not implemented yet", endpoint.name() diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 16e9c174..6901f139 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -369,20 +369,6 @@ pub async fn handle_delete_bucket( .body(Body::empty())?) } -pub async fn handle_bucket_allow_key( - garage: &Arc, - req: Request, -) -> Result, Error> { - handle_bucket_change_key_perm(garage, req, true).await -} - -pub async fn handle_bucket_deny_key( - garage: &Arc, - req: Request, -) -> Result, Error> { - handle_bucket_change_key_perm(garage, req, false).await -} - pub async fn handle_bucket_change_key_perm( garage: &Arc, req: Request, -- 2.43.4 From e7ddba53e38195744fd3bac29eda35fca87ab095 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 12 May 2022 17:10:25 +0200 Subject: [PATCH 18/46] Slightly more detailed error reporting from helper --- src/api/error.rs | 5 +++++ src/garage/main.rs | 1 + src/model/helper/bucket.rs | 19 +++++-------------- src/model/helper/error.rs | 10 ++++++++++ src/model/helper/key.rs | 2 +- 5 files changed, 22 insertions(+), 15 deletions(-) diff --git a/src/api/error.rs b/src/api/error.rs index 4b7254d2..f111c801 100644 --- a/src/api/error.rs +++ b/src/api/error.rs @@ -84,6 +84,10 @@ pub enum Error { #[error(display = "Invalid base64: {}", _0)] InvalidBase64(#[error(source)] base64::DecodeError), + /// Bucket name is not valid according to AWS S3 specs + #[error(display = "Invalid bucket name")] + InvalidBucketName, + /// The client sent invalid XML data #[error(display = "Invalid XML: {}", _0)] InvalidXml(String), @@ -126,6 +130,7 @@ impl From for Error { match err { HelperError::Internal(i) => Self::InternalError(i), HelperError::BadRequest(b) => Self::BadRequest(b), + e => Self::BadRequest(format!("{}", e)), } } } diff --git a/src/garage/main.rs b/src/garage/main.rs index 69ab1147..bd09b6ea 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -142,6 +142,7 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { match cli_command_dispatch(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await { Err(HelperError::Internal(i)) => Err(Error::Message(format!("Internal error: {}", i))), Err(HelperError::BadRequest(b)) => Err(Error::Message(b)), + Err(e) => Err(Error::Message(format!("{}", e))), Ok(x) => Ok(x), } } diff --git a/src/model/helper/bucket.rs b/src/model/helper/bucket.rs index 7e81b946..788bf3a6 100644 --- a/src/model/helper/bucket.rs +++ b/src/model/helper/bucket.rs @@ -71,10 +71,7 @@ impl<'a> BucketHelper<'a> { .get(&EmptyKey, &bucket_id) .await? .filter(|b| !b.is_deleted()) - .ok_or_bad_request(format!( - "Bucket {:?} does not exist or has been deleted", - bucket_id - )) + .ok_or_else(|| Error::NoSuchBucket(hex::encode(bucket_id))) } /// Sets a new alias for a bucket in global namespace. @@ -88,10 +85,7 @@ impl<'a> BucketHelper<'a> { alias_name: &String, ) -> Result<(), Error> { if !is_valid_bucket_name(alias_name) { - return Err(Error::BadRequest(format!( - "{}: {}", - alias_name, INVALID_BUCKET_NAME_MESSAGE - ))); + return Err(Error::InvalidBucketName(alias_name.to_string())); } let mut bucket = self.get_existing_bucket(bucket_id).await?; @@ -122,7 +116,7 @@ impl<'a> BucketHelper<'a> { let alias = match alias { None => BucketAlias::new(alias_name.clone(), alias_ts, Some(bucket_id)) - .ok_or_bad_request(format!("{}: {}", alias_name, INVALID_BUCKET_NAME_MESSAGE))?, + .ok_or_else(|| Error::InvalidBucketName(alias_name.clone()))?, Some(mut a) => { a.state = Lww::raw(alias_ts, Some(bucket_id)); a @@ -210,7 +204,7 @@ impl<'a> BucketHelper<'a> { .bucket_alias_table .get(&EmptyKey, alias_name) .await? - .ok_or_message(format!("Alias {} not found", alias_name))?; + .ok_or_else(|| Error::NoSuchBucket(alias_name.to_string()))?; // Checks ok, remove alias let alias_ts = match bucket.state.as_option() { @@ -252,10 +246,7 @@ impl<'a> BucketHelper<'a> { let key_helper = KeyHelper(self.0); if !is_valid_bucket_name(alias_name) { - return Err(Error::BadRequest(format!( - "{}: {}", - alias_name, INVALID_BUCKET_NAME_MESSAGE - ))); + return Err(Error::InvalidBucketName(alias_name.to_string())); } let mut bucket = self.get_existing_bucket(bucket_id).await?; diff --git a/src/model/helper/error.rs b/src/model/helper/error.rs index 30b2ba32..3ca8f55c 100644 --- a/src/model/helper/error.rs +++ b/src/model/helper/error.rs @@ -10,6 +10,16 @@ pub enum Error { #[error(display = "Bad request: {}", _0)] BadRequest(String), + + /// Bucket name is not valid according to AWS S3 specs + #[error(display = "Invalid bucket name: {}", _0)] + InvalidBucketName(String), + + #[error(display = "Access key not found: {}", _0)] + NoSuchAccessKey(String), + + #[error(display = "Bucket not found: {}", _0)] + NoSuchBucket(String), } impl From for Error { diff --git a/src/model/helper/key.rs b/src/model/helper/key.rs index eea37f79..c1a8e974 100644 --- a/src/model/helper/key.rs +++ b/src/model/helper/key.rs @@ -34,7 +34,7 @@ impl<'a> KeyHelper<'a> { .get(&EmptyKey, key_id) .await? .filter(|b| !b.state.is_deleted()) - .ok_or_bad_request(format!("Key {} does not exist or has been deleted", key_id)) + .ok_or_else(|| Error::NoSuchAccessKey(key_id.to_string())) } /// Returns a Key if it is present in key table, -- 2.43.4 From e4e1f8f0d60545d81c1082ca5f0194962a4a4f79 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Thu, 12 May 2022 17:11:45 +0200 Subject: [PATCH 19/46] Fix clippy --- src/api/admin/api_server.rs | 4 ---- src/api/admin/bucket.rs | 4 ++-- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index b3bc9221..15aa690f 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -154,10 +154,6 @@ impl ApiHandler for AdminApiServer { Endpoint::BucketDenyKey => { handle_bucket_change_key_perm(&self.garage, req, false).await } - _ => Err(Error::NotImplemented(format!( - "Admin endpoint {} not implemented yet", - endpoint.name() - ))), } } } diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 6901f139..2a25bb18 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -349,14 +349,14 @@ pub async fn handle_delete_bucket( for ((key_id, alias), _, active) in state.local_aliases.items().iter() { if *active { helper - .unset_local_bucket_alias(bucket.id, &key_id, &alias) + .unset_local_bucket_alias(bucket.id, key_id, alias) .await?; } } // 3. delete all global aliases for (alias, _, active) in state.aliases.items().iter() { if *active { - helper.purge_global_bucket_alias(bucket.id, &alias).await?; + helper.purge_global_bucket_alias(bucket.id, alias).await?; } } -- 2.43.4 From 983037d965fdcdf089b09fa90fac31501defae9e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 13:51:34 +0200 Subject: [PATCH 20/46] Possibility of different error types for different APIs --- src/api/admin/api_server.rs | 1 + src/api/error.rs | 85 +++++++++++++++++++------------------ src/api/generic_server.rs | 21 ++++++--- src/api/k2v/api_server.rs | 1 + src/api/lib.rs | 2 +- src/api/s3/api_server.rs | 1 + src/web/error.rs | 3 +- 7 files changed, 64 insertions(+), 50 deletions(-) diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 15aa690f..bffffd72 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -96,6 +96,7 @@ impl ApiHandler for AdminApiServer { const API_NAME_DISPLAY: &'static str = "Admin"; type Endpoint = Endpoint; + type Error = Error; fn parse_endpoint(&self, req: &Request) -> Result { Endpoint::from_request(req) diff --git a/src/api/error.rs b/src/api/error.rs index f111c801..5c33d763 100644 --- a/src/api/error.rs +++ b/src/api/error.rs @@ -2,11 +2,12 @@ use std::convert::TryInto; use err_derive::Error; use hyper::header::HeaderValue; -use hyper::{HeaderMap, StatusCode}; +use hyper::{Body, HeaderMap, StatusCode}; use garage_model::helper::error::Error as HelperError; use garage_util::error::Error as GarageError; +use crate::generic_server::ApiError; use crate::s3::xml as s3_xml; /// Errors of this crate @@ -142,28 +143,6 @@ impl From for Error { } impl Error { - /// Get the HTTP status code that best represents the meaning of the error for the client - pub fn http_status_code(&self) -> StatusCode { - match self { - Error::NoSuchKey | Error::NoSuchBucket | Error::NoSuchUpload => StatusCode::NOT_FOUND, - Error::BucketNotEmpty | Error::BucketAlreadyExists => StatusCode::CONFLICT, - Error::PreconditionFailed => StatusCode::PRECONDITION_FAILED, - Error::Forbidden(_) => StatusCode::FORBIDDEN, - Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE, - Error::InternalError( - GarageError::Timeout - | GarageError::RemoteError(_) - | GarageError::Quorum(_, _, _, _), - ) => StatusCode::SERVICE_UNAVAILABLE, - Error::InternalError(_) | Error::Hyper(_) | Error::Http(_) => { - StatusCode::INTERNAL_SERVER_ERROR - } - Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE, - Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED, - _ => StatusCode::BAD_REQUEST, - } - } - pub fn aws_code(&self) -> &'static str { match self { Error::NoSuchKey => "NoSuchKey", @@ -187,27 +166,32 @@ impl Error { _ => "InvalidRequest", } } +} - pub fn aws_xml(&self, garage_region: &str, path: &str) -> String { - let error = s3_xml::Error { - code: s3_xml::Value(self.aws_code().to_string()), - message: s3_xml::Value(format!("{}", self)), - resource: Some(s3_xml::Value(path.to_string())), - region: Some(s3_xml::Value(garage_region.to_string())), - }; - s3_xml::to_xml_with_header(&error).unwrap_or_else(|_| { - r#" - - - InternalError - XML encoding of error failed - - "# - .into() - }) +impl ApiError for Error { + /// Get the HTTP status code that best represents the meaning of the error for the client + fn http_status_code(&self) -> StatusCode { + match self { + Error::NoSuchKey | Error::NoSuchBucket | Error::NoSuchUpload => StatusCode::NOT_FOUND, + Error::BucketNotEmpty | Error::BucketAlreadyExists => StatusCode::CONFLICT, + Error::PreconditionFailed => StatusCode::PRECONDITION_FAILED, + Error::Forbidden(_) => StatusCode::FORBIDDEN, + Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE, + Error::InternalError( + GarageError::Timeout + | GarageError::RemoteError(_) + | GarageError::Quorum(_, _, _, _), + ) => StatusCode::SERVICE_UNAVAILABLE, + Error::InternalError(_) | Error::Hyper(_) | Error::Http(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } + Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE, + Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED, + _ => StatusCode::BAD_REQUEST, + } } - pub fn add_headers(&self, header_map: &mut HeaderMap) { + fn add_http_headers(&self, header_map: &mut HeaderMap) { use hyper::header; #[allow(clippy::single_match)] match self { @@ -222,6 +206,25 @@ impl Error { _ => (), } } + + fn http_body(&self, garage_region: &str, path: &str) -> Body { + let error = s3_xml::Error { + code: s3_xml::Value(self.aws_code().to_string()), + message: s3_xml::Value(format!("{}", self)), + resource: Some(s3_xml::Value(path.to_string())), + region: Some(s3_xml::Value(garage_region.to_string())), + }; + Body::from(s3_xml::to_xml_with_header(&error).unwrap_or_else(|_| { + r#" + + + InternalError + XML encoding of error failed + + "# + .into() + })) + } } /// Trait to map error to the Bad Request error code diff --git a/src/api/generic_server.rs b/src/api/generic_server.rs index 9281e596..77278908 100644 --- a/src/api/generic_server.rs +++ b/src/api/generic_server.rs @@ -5,9 +5,11 @@ use async_trait::async_trait; use futures::future::Future; +use hyper::header::HeaderValue; use hyper::server::conn::AddrStream; use hyper::service::{make_service_fn, service_fn}; use hyper::{Body, Request, Response, Server}; +use hyper::{HeaderMap, StatusCode}; use opentelemetry::{ global, @@ -19,26 +21,31 @@ use opentelemetry::{ use garage_util::error::Error as GarageError; use garage_util::metrics::{gen_trace_id, RecordDuration}; -use crate::error::*; - pub(crate) trait ApiEndpoint: Send + Sync + 'static { fn name(&self) -> &'static str; fn add_span_attributes(&self, span: SpanRef<'_>); } +pub trait ApiError: std::error::Error + Send + Sync + 'static { + fn http_status_code(&self) -> StatusCode; + fn add_http_headers(&self, header_map: &mut HeaderMap); + fn http_body(&self, garage_region: &str, path: &str) -> Body; +} + #[async_trait] pub(crate) trait ApiHandler: Send + Sync + 'static { const API_NAME: &'static str; const API_NAME_DISPLAY: &'static str; type Endpoint: ApiEndpoint; + type Error: ApiError; - fn parse_endpoint(&self, r: &Request) -> Result; + fn parse_endpoint(&self, r: &Request) -> Result; async fn handle( &self, req: Request, endpoint: Self::Endpoint, - ) -> Result, Error>; + ) -> Result, Self::Error>; } pub(crate) struct ApiServer { @@ -142,13 +149,13 @@ impl ApiServer { Ok(x) } Err(e) => { - let body: Body = Body::from(e.aws_xml(&self.region, uri.path())); + let body: Body = e.http_body(&self.region, uri.path()); let mut http_error_builder = Response::builder() .status(e.http_status_code()) .header("Content-Type", "application/xml"); if let Some(header_map) = http_error_builder.headers_mut() { - e.add_headers(header_map) + e.add_http_headers(header_map) } let http_error = http_error_builder.body(body)?; @@ -163,7 +170,7 @@ impl ApiServer { } } - async fn handler_stage2(&self, req: Request) -> Result, Error> { + async fn handler_stage2(&self, req: Request) -> Result, A::Error> { let endpoint = self.api_handler.parse_endpoint(&req)?; debug!("Endpoint: {}", endpoint.name()); diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index 5f5e9030..b14bcda6 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -60,6 +60,7 @@ impl ApiHandler for K2VApiServer { const API_NAME_DISPLAY: &'static str = "K2V"; type Endpoint = K2VApiEndpoint; + type Error = Error; fn parse_endpoint(&self, req: &Request) -> Result { let (endpoint, bucket_name) = Endpoint::from_request(req)?; diff --git a/src/api/lib.rs b/src/api/lib.rs index 5c522799..f8165d2a 100644 --- a/src/api/lib.rs +++ b/src/api/lib.rs @@ -6,7 +6,7 @@ pub mod error; pub use error::Error; mod encoding; -mod generic_server; +pub mod generic_server; pub mod helpers; mod router_macros; /// This mode is public only to help testing. Don't expect stability here diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 78a69d53..2bf4a1bc 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -75,6 +75,7 @@ impl ApiHandler for S3ApiServer { const API_NAME_DISPLAY: &'static str = "S3"; type Endpoint = S3ApiEndpoint; + type Error = Error; fn parse_endpoint(&self, req: &Request) -> Result { let authority = req diff --git a/src/web/error.rs b/src/web/error.rs index 55990e9d..e9ed5314 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -2,6 +2,7 @@ use err_derive::Error; use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; +use garage_api::generic_server::ApiError; use garage_util::error::Error as GarageError; /// Errors of this crate @@ -52,7 +53,7 @@ impl Error { pub fn add_headers(&self, header_map: &mut HeaderMap) { #[allow(clippy::single_match)] match self { - Error::ApiError(e) => e.add_headers(header_map), + Error::ApiError(e) => e.add_http_headers(header_map), _ => (), } } -- 2.43.4 From c0fb9fd0fe553e5eda39dcb1a09f059bcd631b6c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 14:30:30 +0200 Subject: [PATCH 21/46] Common error type and admin error type that uses it --- src/api/admin/api_server.rs | 2 +- src/api/admin/bucket.rs | 22 ++++---- src/api/admin/cluster.rs | 4 +- src/api/admin/error.rs | 94 +++++++++++++++++++++++++++++++ src/api/admin/key.rs | 16 +++--- src/api/admin/mod.rs | 13 +++++ src/api/admin/router.rs | 2 +- src/api/common_error.rs | 108 ++++++++++++++++++++++++++++++++++++ src/api/error.rs | 4 ++ src/api/lib.rs | 1 + src/api/router_macros.rs | 12 ++-- 11 files changed, 249 insertions(+), 29 deletions(-) create mode 100644 src/api/admin/error.rs create mode 100644 src/api/common_error.rs diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index bffffd72..b344a51b 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -15,9 +15,9 @@ use prometheus::{Encoder, TextEncoder}; use garage_model::garage::Garage; use garage_util::error::Error as GarageError; -use crate::error::*; use crate::generic_server::*; +use crate::admin::error::*; use crate::admin::bucket::*; use crate::admin::cluster::*; use crate::admin::key::*; diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 2a25bb18..1ecb66ab 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -17,8 +17,8 @@ use garage_model::permission::*; use garage_model::s3::object_table::ObjectFilter; use crate::admin::key::ApiBucketKeyPerm; -use crate::error::*; -use crate::helpers::*; +use crate::admin::error::*; +use crate::admin::parse_json_body; pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { let buckets = garage @@ -97,9 +97,9 @@ pub async fn handle_get_bucket_info( .await? .ok_or_bad_request("Bucket not found")?, _ => { - return Err(Error::BadRequest( - "Either id or globalAlias must be provided (but not both)".into(), - )) + return Err(Error::bad_request( + "Either id or globalAlias must be provided (but not both)" + )); } }; @@ -225,7 +225,7 @@ pub async fn handle_create_bucket( if let Some(ga) = &req.global_alias { if !is_valid_bucket_name(ga) { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "{}: {}", ga, INVALID_BUCKET_NAME_MESSAGE ))); @@ -240,7 +240,7 @@ pub async fn handle_create_bucket( if let Some(la) = &req.local_alias { if !is_valid_bucket_name(&la.alias) { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "{}: {}", la.alias, INVALID_BUCKET_NAME_MESSAGE ))); @@ -250,10 +250,10 @@ pub async fn handle_create_bucket( .key_table .get(&EmptyKey, &la.access_key_id) .await? - .ok_or(Error::NoSuchKey)?; - let state = key.state.as_option().ok_or(Error::NoSuchKey)?; + .ok_or(Error::NoSuchAccessKey)?; + let state = key.state.as_option().ok_or(Error::NoSuchAccessKey)?; if matches!(state.local_aliases.get(&la.alias), Some(_)) { - return Err(Error::BadRequest("Local alias already exists".into())); + return Err(Error::bad_request("Local alias already exists")); } } @@ -333,7 +333,7 @@ pub async fn handle_delete_bucket( ) .await?; if !objects.is_empty() { - return Err(Error::BadRequest("Bucket is not empty".into())); + return Err(Error::bad_request("Bucket is not empty")); } // --- done checking, now commit --- diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index b8e9d96c..db4d968d 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -13,8 +13,8 @@ use garage_rpc::layout::*; use garage_model::garage::Garage; -use crate::error::*; -use crate::helpers::*; +use crate::admin::error::*; +use crate::admin::parse_json_body; pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { let res = GetClusterStatusResponse { diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs new file mode 100644 index 00000000..3e488d8d --- /dev/null +++ b/src/api/admin/error.rs @@ -0,0 +1,94 @@ +use err_derive::Error; +use hyper::header::HeaderValue; +use hyper::{Body, HeaderMap, StatusCode}; + +use garage_model::helper::error::Error as HelperError; +use garage_util::error::Error as GarageError; + +use crate::generic_server::ApiError; +pub use crate::common_error::*; + +/// Errors of this crate +#[derive(Debug, Error)] +pub enum Error { + #[error(display = "{}", _0)] + /// Error from common error + CommonError(CommonError), + + // Category: cannot process + /// No proper api key was used, or the signature was invalid + #[error(display = "Forbidden: {}", _0)] + Forbidden(String), + + /// The API access key does not exist + #[error(display = "Access key not found")] + NoSuchAccessKey, + + /// The bucket requested don't exists + #[error(display = "Bucket not found")] + NoSuchBucket, + + /// Tried to create a bucket that already exist + #[error(display = "Bucket already exists")] + BucketAlreadyExists, + + /// Tried to delete a non-empty bucket + #[error(display = "Tried to delete a non-empty bucket")] + BucketNotEmpty, + + // Category: bad request + /// Bucket name is not valid according to AWS S3 specs + #[error(display = "Invalid bucket name")] + InvalidBucketName, + + /// The client sent a request for an action not supported by garage + #[error(display = "Unimplemented action: {}", _0)] + NotImplemented(String), +} + +impl From for Error +where CommonError: From { + fn from(err: T) -> Self { + Error::CommonError(CommonError::from(err)) + } +} + +impl From for Error { + fn from(err: HelperError) -> Self { + match err { + HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), + HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), + HelperError::InvalidBucketName(_) => Self::InvalidBucketName, + HelperError::NoSuchAccessKey(_) => Self::NoSuchAccessKey, + HelperError::NoSuchBucket(_) => Self::NoSuchBucket, + } + } +} + +impl ApiError for Error { + /// Get the HTTP status code that best represents the meaning of the error for the client + fn http_status_code(&self) -> StatusCode { + match self { + Error::CommonError(c) => c.http_status_code(), + Error::NoSuchAccessKey | Error::NoSuchBucket => StatusCode::NOT_FOUND, + Error::BucketNotEmpty | Error::BucketAlreadyExists => StatusCode::CONFLICT, + Error::Forbidden(_) => StatusCode::FORBIDDEN, + Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED, + Error::InvalidBucketName => StatusCode::BAD_REQUEST, + } + } + + fn add_http_headers(&self, _header_map: &mut HeaderMap) { + // nothing + } + + fn http_body(&self, garage_region: &str, path: &str) -> Body { + Body::from(format!("ERROR: {}\n\ngarage region: {}\npath: {}", self, garage_region, path)) + } +} + +impl Error { + pub fn bad_request(msg: M) -> Self { + Self::CommonError(CommonError::BadRequest(msg.to_string())) + } +} diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 19ad5160..e5f25601 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -11,8 +11,8 @@ use garage_table::*; use garage_model::garage::Garage; use garage_model::key_table::*; -use crate::error::*; -use crate::helpers::*; +use crate::admin::error::*; +use crate::admin::parse_json_body; pub async fn handle_list_keys(garage: &Arc) -> Result, Error> { let res = garage @@ -54,13 +54,13 @@ pub async fn handle_get_key_info( .key_table .get(&EmptyKey, &id) .await? - .ok_or(Error::NoSuchKey)? + .ok_or(Error::NoSuchAccessKey)? } else if let Some(search) = search { garage .key_helper() .get_existing_matching_key(&search) .await - .map_err(|_| Error::NoSuchKey)? + .map_err(|_| Error::NoSuchAccessKey)? } else { unreachable!(); }; @@ -96,9 +96,9 @@ pub async fn handle_update_key( .key_table .get(&EmptyKey, &id) .await? - .ok_or(Error::NoSuchKey)?; + .ok_or(Error::NoSuchAccessKey)?; - let key_state = key.state.as_option_mut().ok_or(Error::NoSuchKey)?; + let key_state = key.state.as_option_mut().ok_or(Error::NoSuchAccessKey)?; if let Some(new_name) = req.name { key_state.name.update(new_name); @@ -131,9 +131,9 @@ pub async fn handle_delete_key(garage: &Arc, id: String) -> Result Deserialize<'de>>(req: Request) -> Result { + let body = hyper::body::to_bytes(req.into_body()).await?; + let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; + Ok(resp) +} diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index 6f787fe9..2a5098bf 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use hyper::{Method, Request}; -use crate::error::*; +use crate::admin::error::*; use crate::router_macros::*; pub enum Authorization { diff --git a/src/api/common_error.rs b/src/api/common_error.rs new file mode 100644 index 00000000..8be85f97 --- /dev/null +++ b/src/api/common_error.rs @@ -0,0 +1,108 @@ +use err_derive::Error; +use hyper::header::HeaderValue; +use hyper::{Body, HeaderMap, StatusCode}; + +use garage_model::helper::error::Error as HelperError; +use garage_util::error::Error as GarageError; + +use crate::generic_server::ApiError; + +/// Errors of this crate +#[derive(Debug, Error)] +pub enum CommonError { + // Category: internal error + /// Error related to deeper parts of Garage + #[error(display = "Internal error: {}", _0)] + InternalError(#[error(source)] GarageError), + + /// Error related to Hyper + #[error(display = "Internal error (Hyper error): {}", _0)] + Hyper(#[error(source)] hyper::Error), + + /// Error related to HTTP + #[error(display = "Internal error (HTTP error): {}", _0)] + Http(#[error(source)] http::Error), + + /// The client sent an invalid request + #[error(display = "Bad request: {}", _0)] + BadRequest(String), +} + +impl CommonError { + pub fn http_status_code(&self) -> StatusCode { + match self { + CommonError::InternalError( + GarageError::Timeout + | GarageError::RemoteError(_) + | GarageError::Quorum(_, _, _, _), + ) => StatusCode::SERVICE_UNAVAILABLE, + CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => + StatusCode::INTERNAL_SERVER_ERROR, + CommonError::BadRequest(_) => StatusCode::BAD_REQUEST, + } + } +} + +/// Trait to map error to the Bad Request error code +pub trait OkOrBadRequest { + type S; + fn ok_or_bad_request>(self, reason: M) -> Result; +} + +impl OkOrBadRequest for Result +where + E: std::fmt::Display, +{ + type S = T; + fn ok_or_bad_request>(self, reason: M) -> Result { + match self { + Ok(x) => Ok(x), + Err(e) => Err(CommonError::BadRequest(format!("{}: {}", reason.as_ref(), e))), + } + } +} + +impl OkOrBadRequest for Option { + type S = T; + fn ok_or_bad_request>(self, reason: M) -> Result { + match self { + Some(x) => Ok(x), + None => Err(CommonError::BadRequest(reason.as_ref().to_string())), + } + } +} + +/// Trait to map an error to an Internal Error code +pub trait OkOrInternalError { + type S; + fn ok_or_internal_error>(self, reason: M) -> Result; +} + +impl OkOrInternalError for Result +where + E: std::fmt::Display, +{ + type S = T; + fn ok_or_internal_error>(self, reason: M) -> Result { + match self { + Ok(x) => Ok(x), + Err(e) => Err(CommonError::InternalError(GarageError::Message(format!( + "{}: {}", + reason.as_ref(), + e + )))), + } + } +} + +impl OkOrInternalError for Option { + type S = T; + fn ok_or_internal_error>(self, reason: M) -> Result { + match self { + Some(x) => Ok(x), + None => Err(CommonError::InternalError(GarageError::Message( + reason.as_ref().to_string(), + ))), + } + } +} diff --git a/src/api/error.rs b/src/api/error.rs index 5c33d763..90bfccef 100644 --- a/src/api/error.rs +++ b/src/api/error.rs @@ -166,6 +166,10 @@ impl Error { _ => "InvalidRequest", } } + + pub fn bad_request(msg: M) -> Self { + Self::BadRequest(msg.to_string()) + } } impl ApiError for Error { diff --git a/src/api/lib.rs b/src/api/lib.rs index f8165d2a..5bc2a18e 100644 --- a/src/api/lib.rs +++ b/src/api/lib.rs @@ -2,6 +2,7 @@ #[macro_use] extern crate tracing; +pub mod common_error; pub mod error; pub use error::Error; diff --git a/src/api/router_macros.rs b/src/api/router_macros.rs index a3e885e6..4c593300 100644 --- a/src/api/router_macros.rs +++ b/src/api/router_macros.rs @@ -38,7 +38,7 @@ macro_rules! router_match { }, )* (m, p) => { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Unknown API endpoint: {} {}", m, p ))) @@ -78,7 +78,7 @@ macro_rules! router_match { )*)? }), )* - (kw, _) => Err(Error::BadRequest(format!("Invalid endpoint: {}", kw))) + (kw, _) => Err(Error::bad_request(format!("Invalid endpoint: {}", kw))) } }}; @@ -97,14 +97,14 @@ macro_rules! router_match { .take() .map(|param| param.parse()) .transpose() - .map_err(|_| Error::BadRequest("Failed to parse query parameter".to_owned()))? + .map_err(|_| Error::bad_request("Failed to parse query parameter"))? }}; (@@parse_param $query:expr, parse, $param:ident) => {{ // extract and parse mandatory query parameter // both missing and un-parseable parameters are reported as errors $query.$param.take().ok_or_bad_request("Missing argument for endpoint")? .parse() - .map_err(|_| Error::BadRequest("Failed to parse query parameter".to_owned()))? + .map_err(|_| Error::bad_request("Failed to parse query parameter"))? }}; (@func $(#[$doc:meta])* @@ -173,7 +173,7 @@ macro_rules! generateQueryParameters { false } else if v.as_ref().is_empty() { if res.keyword.replace(k).is_some() { - return Err(Error::BadRequest("Multiple keywords".to_owned())); + return Err(Error::bad_request("Multiple keywords")); } continue; } else { @@ -183,7 +183,7 @@ macro_rules! generateQueryParameters { } }; if repeated { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Query parameter repeated: '{}'", k ))); -- 2.43.4 From 96b11524d53b3616a28f33e2b057655be1639f6f Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 15:04:53 +0200 Subject: [PATCH 22/46] Error refactoring --- src/api/admin/api_server.rs | 2 +- src/api/admin/bucket.rs | 4 +- src/api/admin/error.rs | 13 ++-- src/api/admin/mod.rs | 7 +- src/api/common_error.rs | 17 ++--- src/api/error.rs | 127 ++++++++------------------------- src/api/helpers.rs | 8 +-- src/api/k2v/batch.rs | 4 +- src/api/k2v/range.rs | 2 +- src/api/k2v/router.rs | 4 +- src/api/s3/api_server.rs | 2 +- src/api/s3/bucket.rs | 4 +- src/api/s3/copy.rs | 14 ++-- src/api/s3/get.rs | 8 +-- src/api/s3/list.rs | 2 +- src/api/s3/post_object.rs | 34 ++++----- src/api/s3/put.rs | 12 ++-- src/api/s3/router.rs | 2 +- src/api/s3/website.rs | 14 ++-- src/api/signature/mod.rs | 2 +- src/api/signature/payload.rs | 14 ++-- src/api/signature/streaming.rs | 6 +- src/web/error.rs | 33 +++------ src/web/web_server.rs | 6 +- 24 files changed, 135 insertions(+), 206 deletions(-) diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index b344a51b..b2effa62 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -17,9 +17,9 @@ use garage_util::error::Error as GarageError; use crate::generic_server::*; -use crate::admin::error::*; use crate::admin::bucket::*; use crate::admin::cluster::*; +use crate::admin::error::*; use crate::admin::key::*; use crate::admin::router::{Authorization, Endpoint}; diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 1ecb66ab..db1fda0f 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -16,8 +16,8 @@ use garage_model::garage::Garage; use garage_model::permission::*; use garage_model::s3::object_table::ObjectFilter; -use crate::admin::key::ApiBucketKeyPerm; use crate::admin::error::*; +use crate::admin::key::ApiBucketKeyPerm; use crate::admin::parse_json_body; pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { @@ -98,7 +98,7 @@ pub async fn handle_get_bucket_info( .ok_or_bad_request("Bucket not found")?, _ => { return Err(Error::bad_request( - "Either id or globalAlias must be provided (but not both)" + "Either id or globalAlias must be provided (but not both)", )); } }; diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 3e488d8d..1f49fed5 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -3,10 +3,10 @@ use hyper::header::HeaderValue; use hyper::{Body, HeaderMap, StatusCode}; use garage_model::helper::error::Error as HelperError; -use garage_util::error::Error as GarageError; use crate::generic_server::ApiError; -pub use crate::common_error::*; +use crate::common_error::CommonError; +pub use crate::common_error::{OkOrBadRequest, OkOrInternalError}; /// Errors of this crate #[derive(Debug, Error)] @@ -47,7 +47,9 @@ pub enum Error { } impl From for Error -where CommonError: From { +where + CommonError: From, +{ fn from(err: T) -> Self { Error::CommonError(CommonError::from(err)) } @@ -83,7 +85,10 @@ impl ApiError for Error { } fn http_body(&self, garage_region: &str, path: &str) -> Body { - Body::from(format!("ERROR: {}\n\ngarage region: {}\npath: {}", self, garage_region, path)) + Body::from(format!( + "ERROR: {}\n\ngarage region: {}\npath: {}", + self, garage_region, path + )) } } diff --git a/src/api/admin/mod.rs b/src/api/admin/mod.rs index 68839039..73700e6e 100644 --- a/src/api/admin/mod.rs +++ b/src/api/admin/mod.rs @@ -1,14 +1,13 @@ pub mod api_server; -mod router; mod error; +mod router; mod bucket; mod cluster; mod key; - -use serde::{Deserialize}; -use hyper::{Request, Body}; +use hyper::{Body, Request}; +use serde::Deserialize; use error::*; diff --git a/src/api/common_error.rs b/src/api/common_error.rs index 8be85f97..eca27e6b 100644 --- a/src/api/common_error.rs +++ b/src/api/common_error.rs @@ -1,12 +1,8 @@ use err_derive::Error; -use hyper::header::HeaderValue; -use hyper::{Body, HeaderMap, StatusCode}; +use hyper::StatusCode; -use garage_model::helper::error::Error as HelperError; use garage_util::error::Error as GarageError; -use crate::generic_server::ApiError; - /// Errors of this crate #[derive(Debug, Error)] pub enum CommonError { @@ -36,8 +32,9 @@ impl CommonError { | GarageError::RemoteError(_) | GarageError::Quorum(_, _, _, _), ) => StatusCode::SERVICE_UNAVAILABLE, - CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => - StatusCode::INTERNAL_SERVER_ERROR, + CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { + StatusCode::INTERNAL_SERVER_ERROR + } CommonError::BadRequest(_) => StatusCode::BAD_REQUEST, } } @@ -57,7 +54,11 @@ where fn ok_or_bad_request>(self, reason: M) -> Result { match self { Ok(x) => Ok(x), - Err(e) => Err(CommonError::BadRequest(format!("{}: {}", reason.as_ref(), e))), + Err(e) => Err(CommonError::BadRequest(format!( + "{}: {}", + reason.as_ref(), + e + ))), } } } diff --git a/src/api/error.rs b/src/api/error.rs index 90bfccef..3cb97019 100644 --- a/src/api/error.rs +++ b/src/api/error.rs @@ -7,24 +7,17 @@ use hyper::{Body, HeaderMap, StatusCode}; use garage_model::helper::error::Error as HelperError; use garage_util::error::Error as GarageError; +use crate::common_error::CommonError; +pub use crate::common_error::{OkOrBadRequest, OkOrInternalError}; use crate::generic_server::ApiError; use crate::s3::xml as s3_xml; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { - // Category: internal error - /// Error related to deeper parts of Garage - #[error(display = "Internal error: {}", _0)] - InternalError(#[error(source)] GarageError), - - /// Error related to Hyper - #[error(display = "Internal error (Hyper error): {}", _0)] - Hyper(#[error(source)] hyper::Error), - - /// Error related to HTTP - #[error(display = "Internal error (HTTP error): {}", _0)] - Http(#[error(source)] http::Error), + #[error(display = "{}", _0)] + /// Error from common error + CommonError(CommonError), // Category: cannot process /// No proper api key was used, or the signature was invalid @@ -101,10 +94,6 @@ pub enum Error { #[error(display = "Invalid HTTP range: {:?}", _0)] InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)), - /// The client sent an invalid request - #[error(display = "Bad request: {}", _0)] - BadRequest(String), - /// The client asked for an invalid return format (invalid Accept header) #[error(display = "Not acceptable: {}", _0)] NotAcceptable(String), @@ -114,6 +103,15 @@ pub enum Error { NotImplemented(String), } +impl From for Error +where + CommonError: From, +{ + fn from(err: T) -> Self { + Error::CommonError(CommonError::from(err)) + } +} + impl From for Error { fn from(err: roxmltree::Error) -> Self { Self::InvalidXml(format!("{}", err)) @@ -129,16 +127,16 @@ impl From for Error { impl From for Error { fn from(err: HelperError) -> Self { match err { - HelperError::Internal(i) => Self::InternalError(i), - HelperError::BadRequest(b) => Self::BadRequest(b), - e => Self::BadRequest(format!("{}", e)), + HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), + HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), + e => Self::CommonError(CommonError::BadRequest(format!("{}", e))), } } } impl From for Error { fn from(err: multer::Error) -> Self { - Self::BadRequest(err.to_string()) + Self::bad_request(err) } } @@ -157,18 +155,26 @@ impl Error { Error::Forbidden(_) => "AccessDenied", Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed", Error::NotImplemented(_) => "NotImplemented", - Error::InternalError( + Error::CommonError(CommonError::InternalError( GarageError::Timeout | GarageError::RemoteError(_) | GarageError::Quorum(_, _, _, _), - ) => "ServiceUnavailable", - Error::InternalError(_) | Error::Hyper(_) | Error::Http(_) => "InternalError", + )) => "ServiceUnavailable", + Error::CommonError( + CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_), + ) => "InternalError", _ => "InvalidRequest", } } + pub fn internal_error(msg: M) -> Self { + Self::CommonError(CommonError::InternalError(GarageError::Message( + msg.to_string(), + ))) + } + pub fn bad_request(msg: M) -> Self { - Self::BadRequest(msg.to_string()) + Self::CommonError(CommonError::BadRequest(msg.to_string())) } } @@ -176,19 +182,12 @@ impl ApiError for Error { /// Get the HTTP status code that best represents the meaning of the error for the client fn http_status_code(&self) -> StatusCode { match self { + Error::CommonError(c) => c.http_status_code(), Error::NoSuchKey | Error::NoSuchBucket | Error::NoSuchUpload => StatusCode::NOT_FOUND, Error::BucketNotEmpty | Error::BucketAlreadyExists => StatusCode::CONFLICT, Error::PreconditionFailed => StatusCode::PRECONDITION_FAILED, Error::Forbidden(_) => StatusCode::FORBIDDEN, Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE, - Error::InternalError( - GarageError::Timeout - | GarageError::RemoteError(_) - | GarageError::Quorum(_, _, _, _), - ) => StatusCode::SERVICE_UNAVAILABLE, - Error::InternalError(_) | Error::Hyper(_) | Error::Http(_) => { - StatusCode::INTERNAL_SERVER_ERROR - } Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE, Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED, _ => StatusCode::BAD_REQUEST, @@ -230,67 +229,3 @@ impl ApiError for Error { })) } } - -/// Trait to map error to the Bad Request error code -pub trait OkOrBadRequest { - type S; - fn ok_or_bad_request>(self, reason: M) -> Result; -} - -impl OkOrBadRequest for Result -where - E: std::fmt::Display, -{ - type S = T; - fn ok_or_bad_request>(self, reason: M) -> Result { - match self { - Ok(x) => Ok(x), - Err(e) => Err(Error::BadRequest(format!("{}: {}", reason.as_ref(), e))), - } - } -} - -impl OkOrBadRequest for Option { - type S = T; - fn ok_or_bad_request>(self, reason: M) -> Result { - match self { - Some(x) => Ok(x), - None => Err(Error::BadRequest(reason.as_ref().to_string())), - } - } -} - -/// Trait to map an error to an Internal Error code -pub trait OkOrInternalError { - type S; - fn ok_or_internal_error>(self, reason: M) -> Result; -} - -impl OkOrInternalError for Result -where - E: std::fmt::Display, -{ - type S = T; - fn ok_or_internal_error>(self, reason: M) -> Result { - match self { - Ok(x) => Ok(x), - Err(e) => Err(Error::InternalError(GarageError::Message(format!( - "{}: {}", - reason.as_ref(), - e - )))), - } - } -} - -impl OkOrInternalError for Option { - type S = T; - fn ok_or_internal_error>(self, reason: M) -> Result { - match self { - Some(x) => Ok(x), - None => Err(Error::InternalError(GarageError::Message( - reason.as_ref().to_string(), - ))), - } - } -} diff --git a/src/api/helpers.rs b/src/api/helpers.rs index 5e249dae..e94e8b00 100644 --- a/src/api/helpers.rs +++ b/src/api/helpers.rs @@ -52,7 +52,7 @@ pub fn authority_to_host(authority: &str) -> Result { let mut iter = authority.chars().enumerate(); let (_, first_char) = iter .next() - .ok_or_else(|| Error::BadRequest("Authority is empty".to_string()))?; + .ok_or_else(|| Error::bad_request("Authority is empty".to_string()))?; let split = match first_char { '[' => { @@ -60,7 +60,7 @@ pub fn authority_to_host(authority: &str) -> Result { match iter.next() { Some((_, ']')) => iter.next(), _ => { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Authority {} has an illegal format", authority ))) @@ -73,7 +73,7 @@ pub fn authority_to_host(authority: &str) -> Result { let authority = match split { Some((i, ':')) => Ok(&authority[..i]), None => Ok(authority), - Some((_, _)) => Err(Error::BadRequest(format!( + Some((_, _)) => Err(Error::bad_request(format!( "Authority {} has an illegal format", authority ))), @@ -134,7 +134,7 @@ pub fn parse_bucket_key<'a>( None => (path, None), }; if bucket.is_empty() { - return Err(Error::BadRequest("No bucket specified".to_string())); + return Err(Error::bad_request("No bucket specified")); } Ok((bucket, key)) } diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index a97bd7f2..26d3cef0 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -88,7 +88,7 @@ async fn handle_read_batch_query( let (items, more, next_start) = if query.single_item { if query.prefix.is_some() || query.end.is_some() || query.limit.is_some() || query.reverse { - return Err(Error::BadRequest("Batch query parameters 'prefix', 'end', 'limit' and 'reverse' must not be set when singleItem is true.".into())); + return Err(Error::bad_request("Batch query parameters 'prefix', 'end', 'limit' and 'reverse' must not be set when singleItem is true.")); } let sk = query .start @@ -183,7 +183,7 @@ async fn handle_delete_batch_query( let deleted_items = if query.single_item { if query.prefix.is_some() || query.end.is_some() { - return Err(Error::BadRequest("Batch query parameters 'prefix' and 'end' must not be set when singleItem is true.".into())); + return Err(Error::bad_request("Batch query parameters 'prefix' and 'end' must not be set when singleItem is true.")); } let sk = query .start diff --git a/src/api/k2v/range.rs b/src/api/k2v/range.rs index cd019723..8d4cefbc 100644 --- a/src/api/k2v/range.rs +++ b/src/api/k2v/range.rs @@ -31,7 +31,7 @@ where (None, Some(s)) => (Some(s.clone()), false), (Some(p), Some(s)) => { if !s.starts_with(p) { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Start key '{}' does not start with prefix '{}'", s, p ))); diff --git a/src/api/k2v/router.rs b/src/api/k2v/router.rs index f948ffce..611b6629 100644 --- a/src/api/k2v/router.rs +++ b/src/api/k2v/router.rs @@ -62,7 +62,7 @@ impl Endpoint { .unwrap_or((path.to_owned(), "")); if bucket.is_empty() { - return Err(Error::BadRequest("Missing bucket name".to_owned())); + return Err(Error::bad_request("Missing bucket name".to_owned())); } if *req.method() == Method::OPTIONS { @@ -83,7 +83,7 @@ impl Endpoint { Method::PUT => Self::from_put(partition_key, &mut query)?, Method::DELETE => Self::from_delete(partition_key, &mut query)?, _ if req.method() == method_search => Self::from_search(partition_key, &mut query)?, - _ => return Err(Error::BadRequest("Unknown method".to_owned())), + _ => return Err(Error::bad_request("Unknown method".to_owned())), }; if let Some(message) = query.nonempty_message() { diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 2bf4a1bc..af9f03e7 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -310,7 +310,7 @@ impl ApiHandler for S3ApiServer { ) .await } else { - Err(Error::BadRequest(format!( + Err(Error::bad_request(format!( "Invalid endpoint: list-type={}", list_type ))) diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 93048a8c..6ecda2cd 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -130,7 +130,7 @@ pub async fn handle_create_bucket( if let Some(location_constraint) = cmd { if location_constraint != garage.config.s3_api.s3_region { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Cannot satisfy location constraint `{}`: buckets can only be created in region `{}`", location_constraint, garage.config.s3_api.s3_region @@ -163,7 +163,7 @@ pub async fn handle_create_bucket( } else { // Create the bucket! if !is_valid_bucket_name(&bucket_name) { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "{}: {}", bucket_name, INVALID_BUCKET_NAME_MESSAGE ))); diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 4e94d887..825b8fc0 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -201,8 +201,8 @@ pub async fn handle_upload_part_copy( let mut ranges = http_range::HttpRange::parse(range_str, source_version_meta.size) .map_err(|e| (e, source_version_meta.size))?; if ranges.len() != 1 { - return Err(Error::BadRequest( - "Invalid x-amz-copy-source-range header: exactly 1 range must be given".into(), + return Err(Error::bad_request( + "Invalid x-amz-copy-source-range header: exactly 1 range must be given", )); } else { ranges.pop().unwrap() @@ -230,8 +230,8 @@ pub async fn handle_upload_part_copy( // This is only for small files, we don't bother handling this. // (in AWS UploadPartCopy works for parts at least 5MB which // is never the case of an inline object) - return Err(Error::BadRequest( - "Source object is too small (minimum part size is 5Mb)".into(), + return Err(Error::bad_request( + "Source object is too small (minimum part size is 5Mb)", )); } ObjectVersionData::FirstBlock(_meta, _first_block_hash) => (), @@ -250,7 +250,7 @@ pub async fn handle_upload_part_copy( // Check this part number hasn't yet been uploaded if let Some(dv) = dest_version { if dv.has_part_number(part_number) { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Part number {} has already been uploaded", part_number ))); @@ -536,8 +536,8 @@ impl CopyPreconditionHeaders { (None, None, None, Some(ims)) => v_date > *ims, (None, None, None, None) => true, _ => { - return Err(Error::BadRequest( - "Invalid combination of x-amz-copy-source-if-xxxxx headers".into(), + return Err(Error::bad_request( + "Invalid combination of x-amz-copy-source-if-xxxxx headers", )) } }; diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index 3edf22a6..794bd4e9 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -210,8 +210,8 @@ pub async fn handle_get( match (part_number, parse_range_header(req, last_v_meta.size)?) { (Some(_), Some(_)) => { - return Err(Error::BadRequest( - "Cannot specify both partNumber and Range header".into(), + return Err(Error::bad_request( + "Cannot specify both partNumber and Range header", )); } (Some(pn), None) => { @@ -302,9 +302,9 @@ async fn handle_get_range( let body: Body = Body::from(bytes[begin as usize..end as usize].to_vec()); Ok(resp_builder.body(body)?) } else { - None.ok_or_internal_error( + Err(Error::internal_error( "Requested range not present in inline bytes when it should have been", - ) + )) } } ObjectVersionData::FirstBlock(_meta, _first_block_hash) => { diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index e2848c57..d97aafe5 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -588,7 +588,7 @@ impl ListObjectsQuery { "]" => Ok(RangeBegin::AfterKey { key: String::from_utf8(base64::decode(token[1..].as_bytes())?)?, }), - _ => Err(Error::BadRequest("Invalid continuation token".to_string())), + _ => Err(Error::bad_request("Invalid continuation token".to_string())), }, // StartAfter has defined semantics in the spec: diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 86fa7880..91648a19 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -48,7 +48,7 @@ pub async fn handle_post_object( let field = if let Some(field) = multipart.next_field().await? { field } else { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Request did not contain a file".to_owned(), )); }; @@ -66,14 +66,14 @@ pub async fn handle_post_object( "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */), "acl" => { if params.insert("x-amz-acl", content).is_some() { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Field 'acl' provided more than one time".to_string(), )); } } _ => { if params.insert(&name, content).is_some() { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Field '{}' provided more than one time", name ))); @@ -145,7 +145,7 @@ pub async fn handle_post_object( .ok_or_bad_request("Invalid expiration date")? .into(); if Utc::now() - expiration > Duration::zero() { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Expiration date is in the paste".to_string(), )); } @@ -159,7 +159,7 @@ pub async fn handle_post_object( "policy" | "x-amz-signature" => (), // this is always accepted, as it's required to validate other fields "content-type" => { let conds = conditions.params.remove("content-type").ok_or_else(|| { - Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) + Error::bad_request(format!("Key '{}' is not allowed in policy", param_key)) })?; for cond in conds { let ok = match cond { @@ -169,7 +169,7 @@ pub async fn handle_post_object( } }; if !ok { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Key '{}' has value not allowed in policy", param_key ))); @@ -178,7 +178,7 @@ pub async fn handle_post_object( } "key" => { let conds = conditions.params.remove("key").ok_or_else(|| { - Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) + Error::bad_request(format!("Key '{}' is not allowed in policy", param_key)) })?; for cond in conds { let ok = match cond { @@ -186,7 +186,7 @@ pub async fn handle_post_object( Operation::StartsWith(s) => key.starts_with(&s), }; if !ok { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Key '{}' has value not allowed in policy", param_key ))); @@ -201,7 +201,7 @@ pub async fn handle_post_object( continue; } let conds = conditions.params.remove(¶m_key).ok_or_else(|| { - Error::BadRequest(format!("Key '{}' is not allowed in policy", param_key)) + Error::bad_request(format!("Key '{}' is not allowed in policy", param_key)) })?; for cond in conds { let ok = match cond { @@ -209,7 +209,7 @@ pub async fn handle_post_object( Operation::StartsWith(s) => value.to_str()?.starts_with(s.as_str()), }; if !ok { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Key '{}' has value not allowed in policy", param_key ))); @@ -220,7 +220,7 @@ pub async fn handle_post_object( } if let Some((param_key, _)) = conditions.params.iter().next() { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Key '{}' is required in policy, but no value was provided", param_key ))); @@ -326,7 +326,7 @@ impl Policy { match condition { PolicyCondition::Equal(map) => { if map.len() != 1 { - return Err(Error::BadRequest("Invalid policy item".to_owned())); + return Err(Error::bad_request("Invalid policy item".to_owned())); } let (mut k, v) = map.into_iter().next().expect("size was verified"); k.make_ascii_lowercase(); @@ -334,7 +334,7 @@ impl Policy { } PolicyCondition::OtherOp([cond, mut key, value]) => { if key.remove(0) != '$' { - return Err(Error::BadRequest("Invalid policy item".to_owned())); + return Err(Error::bad_request("Invalid policy item".to_owned())); } key.make_ascii_lowercase(); match cond.as_str() { @@ -347,7 +347,7 @@ impl Policy { .or_default() .push(Operation::StartsWith(value)); } - _ => return Err(Error::BadRequest("Invalid policy item".to_owned())), + _ => return Err(Error::bad_request("Invalid policy item".to_owned())), } } PolicyCondition::SizeRange(key, min, max) => { @@ -355,7 +355,7 @@ impl Policy { length.0 = length.0.max(min); length.1 = length.1.min(max); } else { - return Err(Error::BadRequest("Invalid policy item".to_owned())); + return Err(Error::bad_request("Invalid policy item".to_owned())); } } } @@ -420,14 +420,14 @@ where self.read += bytes.len() as u64; // optimization to fail early when we know before the end it's too long if self.length.end() < &self.read { - return Poll::Ready(Some(Err(Error::BadRequest( + return Poll::Ready(Some(Err(Error::bad_request( "File size does not match policy".to_owned(), )))); } } Poll::Ready(None) => { if !self.length.contains(&self.read) { - return Poll::Ready(Some(Err(Error::BadRequest( + return Poll::Ready(Some(Err(Error::bad_request( "File size does not match policy".to_owned(), )))); } diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 89aa8d84..d50e32b0 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -183,7 +183,7 @@ fn ensure_checksum_matches( ) -> Result<(), Error> { if let Some(expected_sha256) = content_sha256 { if expected_sha256 != data_sha256sum { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Unable to validate x-amz-content-sha256".to_string(), )); } else { @@ -192,7 +192,7 @@ fn ensure_checksum_matches( } if let Some(expected_md5) = content_md5 { if expected_md5.trim_matches('"') != base64::encode(data_md5sum) { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Unable to validate content-md5".to_string(), )); } else { @@ -428,7 +428,7 @@ pub async fn handle_put_part( // Check part hasn't already been uploaded if let Some(v) = version { if v.has_part_number(part_number) { - return Err(Error::BadRequest(format!( + return Err(Error::bad_request(format!( "Part number {} has already been uploaded", part_number ))); @@ -513,7 +513,7 @@ pub async fn handle_complete_multipart_upload( let version = version.ok_or(Error::NoSuchKey)?; if version.blocks.is_empty() { - return Err(Error::BadRequest("No data was uploaded".to_string())); + return Err(Error::bad_request("No data was uploaded".to_string())); } let headers = match object_version.state { @@ -574,8 +574,8 @@ pub async fn handle_complete_multipart_upload( .map(|x| x.part_number) .eq(block_parts.into_iter()); if !same_parts { - return Err(Error::BadRequest( - "Part numbers in block list and part list do not match. This can happen if a part was partially uploaded. Please abort the multipart upload and try again.".into(), + return Err(Error::bad_request( + "Part numbers in block list and part list do not match. This can happen if a part was partially uploaded. Please abort the multipart upload and try again." )); } diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index 446ceb54..e920e162 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -342,7 +342,7 @@ impl Endpoint { Method::POST => Self::from_post(key, &mut query)?, Method::PUT => Self::from_put(key, &mut query, req.headers())?, Method::DELETE => Self::from_delete(key, &mut query)?, - _ => return Err(Error::BadRequest("Unknown method".to_owned())), + _ => return Err(Error::bad_request("Unknown method".to_owned())), }; if let Some(message) = query.nonempty_message() { diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index 561130dc..4fc7b7bb 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -176,7 +176,7 @@ impl WebsiteConfiguration { || self.index_document.is_some() || self.routing_rules.is_some()) { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Bad XML: can't have RedirectAllRequestsTo and other fields".to_owned(), )); } @@ -222,7 +222,7 @@ impl WebsiteConfiguration { impl Key { pub fn validate(&self) -> Result<(), Error> { if self.key.0.is_empty() { - Err(Error::BadRequest( + Err(Error::bad_request( "Bad XML: error document specified but empty".to_owned(), )) } else { @@ -234,7 +234,7 @@ impl Key { impl Suffix { pub fn validate(&self) -> Result<(), Error> { if self.suffix.0.is_empty() | self.suffix.0.contains('/') { - Err(Error::BadRequest( + Err(Error::bad_request( "Bad XML: index document is empty or contains /".to_owned(), )) } else { @@ -247,7 +247,7 @@ impl Target { pub fn validate(&self) -> Result<(), Error> { if let Some(ref protocol) = self.protocol { if protocol.0 != "http" && protocol.0 != "https" { - return Err(Error::BadRequest("Bad XML: invalid protocol".to_owned())); + return Err(Error::bad_request("Bad XML: invalid protocol".to_owned())); } } Ok(()) @@ -269,19 +269,19 @@ impl Redirect { pub fn validate(&self, has_prefix: bool) -> Result<(), Error> { if self.replace_prefix.is_some() { if self.replace_full.is_some() { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set".to_owned(), )); } if !has_prefix { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Bad XML: ReplaceKeyPrefixWith is set, but KeyPrefixEquals isn't".to_owned(), )); } } if let Some(ref protocol) = self.protocol { if protocol.0 != "http" && protocol.0 != "https" { - return Err(Error::BadRequest("Bad XML: invalid protocol".to_owned())); + return Err(Error::bad_request("Bad XML: invalid protocol".to_owned())); } } // TODO there are probably more invalide cases, but which ones? diff --git a/src/api/signature/mod.rs b/src/api/signature/mod.rs index 5646f4fa..e3554080 100644 --- a/src/api/signature/mod.rs +++ b/src/api/signature/mod.rs @@ -16,7 +16,7 @@ type HmacSha256 = Hmac; pub fn verify_signed_content(expected_sha256: Hash, body: &[u8]) -> Result<(), Error> { if expected_sha256 != sha256sum(body) { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Request content hash does not match signed hash".to_string(), )); } diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 9137dd2d..52c4d401 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -105,7 +105,7 @@ fn parse_authorization( let (auth_kind, rest) = authorization.split_at(first_space); if auth_kind != "AWS4-HMAC-SHA256" { - return Err(Error::BadRequest("Unsupported authorization method".into())); + return Err(Error::bad_request("Unsupported authorization method")); } let mut auth_params = HashMap::new(); @@ -129,10 +129,11 @@ fn parse_authorization( let date = headers .get("x-amz-date") .ok_or_bad_request("Missing X-Amz-Date field") + .map_err(Error::from) .and_then(|d| parse_date(d))?; if Utc::now() - date > Duration::hours(24) { - return Err(Error::BadRequest("Date is too old".to_string())); + return Err(Error::bad_request("Date is too old".to_string())); } let auth = Authorization { @@ -156,7 +157,7 @@ fn parse_query_authorization( headers: &HashMap, ) -> Result { if algorithm != "AWS4-HMAC-SHA256" { - return Err(Error::BadRequest( + return Err(Error::bad_request( "Unsupported authorization method".to_string(), )); } @@ -179,10 +180,10 @@ fn parse_query_authorization( .get("x-amz-expires") .ok_or_bad_request("X-Amz-Expires not found in query parameters")? .parse() - .map_err(|_| Error::BadRequest("X-Amz-Expires is not a number".to_string()))?; + .map_err(|_| Error::bad_request("X-Amz-Expires is not a number".to_string()))?; if duration > 7 * 24 * 3600 { - return Err(Error::BadRequest( + return Err(Error::bad_request( "X-Amz-Exprires may not exceed a week".to_string(), )); } @@ -190,10 +191,11 @@ fn parse_query_authorization( let date = headers .get("x-amz-date") .ok_or_bad_request("Missing X-Amz-Date field") + .map_err(Error::from) .and_then(|d| parse_date(d))?; if Utc::now() - date > Duration::seconds(duration) { - return Err(Error::BadRequest("Date is too old".to_string())); + return Err(Error::bad_request("Date is too old".to_string())); } Ok(Authorization { diff --git a/src/api/signature/streaming.rs b/src/api/signature/streaming.rs index ded9d993..6c326c54 100644 --- a/src/api/signature/streaming.rs +++ b/src/api/signature/streaming.rs @@ -87,7 +87,7 @@ fn compute_streaming_payload_signature( let mut hmac = signing_hmac.clone(); hmac.update(string_to_sign.as_bytes()); - Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature") + Ok(Hash::try_from(&hmac.finalize().into_bytes()).ok_or_internal_error("Invalid signature")?) } mod payload { @@ -163,10 +163,10 @@ impl From for Error { match err { SignedPayloadStreamError::Stream(e) => e, SignedPayloadStreamError::InvalidSignature => { - Error::BadRequest("Invalid payload signature".into()) + Error::bad_request("Invalid payload signature") } SignedPayloadStreamError::Message(e) => { - Error::BadRequest(format!("Chunk format error: {}", e)) + Error::bad_request(format!("Chunk format error: {}", e)) } } } diff --git a/src/web/error.rs b/src/web/error.rs index e9ed5314..478731b5 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -3,50 +3,39 @@ use hyper::header::HeaderValue; use hyper::{HeaderMap, StatusCode}; use garage_api::generic_server::ApiError; -use garage_util::error::Error as GarageError; /// Errors of this crate #[derive(Debug, Error)] pub enum Error { /// An error received from the API crate #[error(display = "API error: {}", _0)] - ApiError(#[error(source)] garage_api::Error), - - // Category: internal error - /// Error internal to garage - #[error(display = "Internal error: {}", _0)] - InternalError(#[error(source)] GarageError), + ApiError(garage_api::Error), /// The file does not exist #[error(display = "Not found")] NotFound, - /// The request contained an invalid UTF-8 sequence in its path or in other parameters - #[error(display = "Invalid UTF-8: {}", _0)] - InvalidUtf8(#[error(source)] std::str::Utf8Error), - - /// The client send a header with invalid value - #[error(display = "Invalid header value: {}", _0)] - InvalidHeader(#[error(source)] hyper::header::ToStrError), - /// The client sent a request without host, or with unsupported method #[error(display = "Bad request: {}", _0)] BadRequest(String), } +impl From for Error +where + garage_api::Error: From, +{ + fn from(err: T) -> Self { + Error::ApiError(garage_api::Error::from(err)) + } +} + impl Error { /// Transform errors into http status code pub fn http_status_code(&self) -> StatusCode { match self { Error::NotFound => StatusCode::NOT_FOUND, Error::ApiError(e) => e.http_status_code(), - Error::InternalError( - GarageError::Timeout - | GarageError::RemoteError(_) - | GarageError::Quorum(_, _, _, _), - ) => StatusCode::SERVICE_UNAVAILABLE, - Error::InternalError(_) => StatusCode::INTERNAL_SERVER_ERROR, - _ => StatusCode::BAD_REQUEST, + Error::BadRequest(_) => StatusCode::BAD_REQUEST, } } diff --git a/src/web/web_server.rs b/src/web/web_server.rs index 867adc51..e83bc4cb 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -207,7 +207,7 @@ async fn serve_file(garage: Arc, req: &Request) -> Result handle_options_for_bucket(req, &bucket), Method::HEAD => handle_head(garage.clone(), req, bucket_id, &key, None).await, Method::GET => handle_get(garage.clone(), req, bucket_id, &key, None).await, - _ => Err(ApiError::BadRequest("HTTP method not supported".into())), + _ => Err(ApiError::bad_request("HTTP method not supported")), } .map_err(Error::from); @@ -290,9 +290,7 @@ fn path_to_key<'a>(path: &'a str, index: &str) -> Result, Error> { let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?; if !path_utf8.starts_with('/') { - return Err(Error::BadRequest( - "Path must start with a / (slash)".to_string(), - )); + return Err(Error::BadRequest("Path must start with a / (slash)".into())); } match path_utf8.chars().last() { -- 2.43.4 From f82b938033f1a01a136315b5f37ecb89b78bca0c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 15:10:52 +0200 Subject: [PATCH 23/46] Rename error::Error to s3::error::Error --- src/api/helpers.rs | 2 +- src/api/k2v/api_server.rs | 2 +- src/api/k2v/batch.rs | 2 +- src/api/k2v/index.rs | 2 +- src/api/k2v/item.rs | 2 +- src/api/k2v/range.rs | 2 +- src/api/k2v/router.rs | 2 +- src/api/lib.rs | 2 -- src/api/s3/api_server.rs | 2 +- src/api/s3/bucket.rs | 2 +- src/api/s3/copy.rs | 2 +- src/api/s3/cors.rs | 2 +- src/api/s3/delete.rs | 2 +- src/api/{ => s3}/error.rs | 0 src/api/s3/get.rs | 2 +- src/api/s3/list.rs | 2 +- src/api/s3/mod.rs | 1 + src/api/s3/post_object.rs | 2 +- src/api/s3/put.rs | 2 +- src/api/s3/router.rs | 2 +- src/api/s3/website.rs | 2 +- src/api/s3/xml.rs | 2 +- src/api/signature/mod.rs | 2 +- src/api/signature/payload.rs | 2 +- src/api/signature/streaming.rs | 2 +- src/web/error.rs | 6 +++--- src/web/web_server.rs | 2 +- 27 files changed, 27 insertions(+), 28 deletions(-) rename src/api/{ => s3}/error.rs (100%) diff --git a/src/api/helpers.rs b/src/api/helpers.rs index e94e8b00..121fbd5a 100644 --- a/src/api/helpers.rs +++ b/src/api/helpers.rs @@ -7,7 +7,7 @@ use garage_util::data::*; use garage_model::garage::Garage; use garage_model::key_table::Key; -use crate::error::*; +use crate::s3::error::*; /// What kind of authorization is required to perform a given action #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index b14bcda6..9b4ad882 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -12,7 +12,7 @@ use garage_util::error::Error as GarageError; use garage_model::garage::Garage; -use crate::error::*; +use crate::s3::error::*; use crate::generic_server::*; use crate::signature::payload::check_payload_signature; diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index 26d3cef0..dab3bfb2 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -12,7 +12,7 @@ use garage_model::garage::Garage; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::error::*; +use crate::s3::error::*; use crate::helpers::*; use crate::k2v::range::read_range; diff --git a/src/api/k2v/index.rs b/src/api/k2v/index.rs index 896dbcf0..e587841c 100644 --- a/src/api/k2v/index.rs +++ b/src/api/k2v/index.rs @@ -12,7 +12,7 @@ use garage_table::util::*; use garage_model::garage::Garage; use garage_model::k2v::counter_table::{BYTES, CONFLICTS, ENTRIES, VALUES}; -use crate::error::*; +use crate::s3::error::*; use crate::k2v::range::read_range; pub async fn handle_read_index( diff --git a/src/api/k2v/item.rs b/src/api/k2v/item.rs index 1860863e..95624d57 100644 --- a/src/api/k2v/item.rs +++ b/src/api/k2v/item.rs @@ -10,7 +10,7 @@ use garage_model::garage::Garage; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::error::*; +use crate::s3::error::*; pub const X_GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token"; diff --git a/src/api/k2v/range.rs b/src/api/k2v/range.rs index 8d4cefbc..cf6034b9 100644 --- a/src/api/k2v/range.rs +++ b/src/api/k2v/range.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use garage_table::replication::TableShardedReplication; use garage_table::*; -use crate::error::*; +use crate::s3::error::*; use crate::helpers::key_after_prefix; /// Read range in a Garage table. diff --git a/src/api/k2v/router.rs b/src/api/k2v/router.rs index 611b6629..c509a4da 100644 --- a/src/api/k2v/router.rs +++ b/src/api/k2v/router.rs @@ -1,4 +1,4 @@ -use crate::error::*; +use crate::s3::error::*; use std::borrow::Cow; diff --git a/src/api/lib.rs b/src/api/lib.rs index 5bc2a18e..370dfd7a 100644 --- a/src/api/lib.rs +++ b/src/api/lib.rs @@ -3,8 +3,6 @@ extern crate tracing; pub mod common_error; -pub mod error; -pub use error::Error; mod encoding; pub mod generic_server; diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index af9f03e7..77ac3879 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -14,7 +14,7 @@ use garage_util::error::Error as GarageError; use garage_model::garage::Garage; use garage_model::key_table::Key; -use crate::error::*; +use crate::s3::error::*; use crate::generic_server::*; use crate::signature::payload::check_payload_signature; diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 6ecda2cd..d4a6b0cb 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -14,7 +14,7 @@ use garage_util::crdt::*; use garage_util::data::*; use garage_util::time::*; -use crate::error::*; +use crate::s3::error::*; use crate::s3::xml as s3_xml; use crate::signature::verify_signed_content; diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 825b8fc0..abd90f4a 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -18,7 +18,7 @@ use garage_model::s3::block_ref_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::error::*; +use crate::s3::error::*; use crate::helpers::{parse_bucket_key, resolve_bucket}; use crate::s3::put::{decode_upload_id, get_headers}; use crate::s3::xml::{self as s3_xml, xmlns_tag}; diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index 37ea2e43..1ad4f2f8 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -9,7 +9,7 @@ use hyper::{header::HeaderName, Body, Method, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::error::*; +use crate::s3::error::*; use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; use crate::signature::verify_signed_content; diff --git a/src/api/s3/delete.rs b/src/api/s3/delete.rs index 1e3f1249..5065b285 100644 --- a/src/api/s3/delete.rs +++ b/src/api/s3/delete.rs @@ -8,7 +8,7 @@ use garage_util::time::*; use garage_model::garage::Garage; use garage_model::s3::object_table::*; -use crate::error::*; +use crate::s3::error::*; use crate::s3::xml as s3_xml; use crate::signature::verify_signed_content; diff --git a/src/api/error.rs b/src/api/s3/error.rs similarity index 100% rename from src/api/error.rs rename to src/api/s3/error.rs diff --git a/src/api/s3/get.rs b/src/api/s3/get.rs index 794bd4e9..7fa1a177 100644 --- a/src/api/s3/get.rs +++ b/src/api/s3/get.rs @@ -17,7 +17,7 @@ use garage_model::garage::Garage; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::error::*; +use crate::s3::error::*; const X_AMZ_MP_PARTS_COUNT: &str = "x-amz-mp-parts-count"; diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index d97aafe5..b4ba5bcd 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -16,7 +16,7 @@ use garage_model::s3::version_table::Version; use garage_table::{EmptyKey, EnumerationOrder}; use crate::encoding::*; -use crate::error::*; +use crate::s3::error::*; use crate::helpers::key_after_prefix; use crate::s3::put as s3_put; use crate::s3::xml as s3_xml; diff --git a/src/api/s3/mod.rs b/src/api/s3/mod.rs index 3f5c1915..7b56d4d8 100644 --- a/src/api/s3/mod.rs +++ b/src/api/s3/mod.rs @@ -1,4 +1,5 @@ pub mod api_server; +pub mod error; mod bucket; mod copy; diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 91648a19..343aa366 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -14,7 +14,7 @@ use serde::Deserialize; use garage_model::garage::Garage; -use crate::error::*; +use crate::s3::error::*; use crate::helpers::resolve_bucket; use crate::s3::put::{get_headers, save_stream}; use crate::s3::xml as s3_xml; diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index d50e32b0..660a8858 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -19,7 +19,7 @@ use garage_model::s3::block_ref_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; -use crate::error::*; +use crate::s3::error::*; use crate::s3::xml as s3_xml; use crate::signature::verify_signed_content; diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index e920e162..b12c63a7 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -3,7 +3,7 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; -use crate::error::{Error, OkOrBadRequest}; +use crate::s3::error::{Error, OkOrBadRequest}; use crate::helpers::Authorization; use crate::router_macros::{generateQueryParameters, router_match}; diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index 4fc7b7bb..b2582c4b 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use hyper::{Body, Request, Response, StatusCode}; use serde::{Deserialize, Serialize}; -use crate::error::*; +use crate::s3::error::*; use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; use crate::signature::verify_signed_content; diff --git a/src/api/s3/xml.rs b/src/api/s3/xml.rs index 75ec4559..111657a0 100644 --- a/src/api/s3/xml.rs +++ b/src/api/s3/xml.rs @@ -1,7 +1,7 @@ use quick_xml::se::to_string; use serde::{Deserialize, Serialize, Serializer}; -use crate::Error as ApiError; +use crate::s3::error::Error as ApiError; pub fn to_xml_with_header(x: &T) -> Result { let mut xml = r#""#.to_string(); diff --git a/src/api/signature/mod.rs b/src/api/signature/mod.rs index e3554080..4679747f 100644 --- a/src/api/signature/mod.rs +++ b/src/api/signature/mod.rs @@ -4,7 +4,7 @@ use sha2::Sha256; use garage_util::data::{sha256sum, Hash}; -use crate::error::*; +use crate::s3::error::*; pub mod payload; pub mod streaming; diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 52c4d401..47445bc7 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -15,7 +15,7 @@ use super::LONG_DATETIME; use super::{compute_scope, signing_hmac}; use crate::encoding::uri_encode; -use crate::error::*; +use crate::s3::error::*; pub async fn check_payload_signature( garage: &Garage, diff --git a/src/api/signature/streaming.rs b/src/api/signature/streaming.rs index 6c326c54..06a0512e 100644 --- a/src/api/signature/streaming.rs +++ b/src/api/signature/streaming.rs @@ -12,7 +12,7 @@ use garage_util::data::Hash; use super::{compute_scope, sha256sum, HmacSha256, LONG_DATETIME}; -use crate::error::*; +use crate::s3::error::*; pub fn parse_streaming_body( api_key: &Key, diff --git a/src/web/error.rs b/src/web/error.rs index 478731b5..bd8f17b5 100644 --- a/src/web/error.rs +++ b/src/web/error.rs @@ -9,7 +9,7 @@ use garage_api::generic_server::ApiError; pub enum Error { /// An error received from the API crate #[error(display = "API error: {}", _0)] - ApiError(garage_api::Error), + ApiError(garage_api::s3::error::Error), /// The file does not exist #[error(display = "Not found")] @@ -22,10 +22,10 @@ pub enum Error { impl From for Error where - garage_api::Error: From, + garage_api::s3::error::Error: From, { fn from(err: T) -> Self { - Error::ApiError(garage_api::Error::from(err)) + Error::ApiError(garage_api::s3::error::Error::from(err)) } } diff --git a/src/web/web_server.rs b/src/web/web_server.rs index e83bc4cb..dad98dfc 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -18,7 +18,7 @@ use opentelemetry::{ use crate::error::*; -use garage_api::error::{Error as ApiError, OkOrBadRequest, OkOrInternalError}; +use garage_api::s3::error::{Error as ApiError, OkOrBadRequest, OkOrInternalError}; use garage_api::helpers::{authority_to_host, host_to_bucket}; use garage_api::s3::cors::{add_cors_headers, find_matching_cors_rule, handle_options_for_bucket}; use garage_api::s3::get::{handle_get, handle_head}; -- 2.43.4 From 7a5d329e49cc7018cbfa14d37589f51860f66cf0 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 15:21:32 +0200 Subject: [PATCH 24/46] More error refactoring --- src/api/admin/bucket.rs | 4 ++-- src/api/admin/cluster.rs | 2 +- src/api/admin/error.rs | 5 ----- src/api/admin/key.rs | 2 +- src/api/admin/mod.rs | 11 ----------- src/api/common_error.rs | 5 +++++ src/api/helpers.rs | 29 +---------------------------- src/api/k2v/api_server.rs | 2 +- src/api/s3/api_server.rs | 2 +- src/api/s3/copy.rs | 4 ++-- src/api/s3/post_object.rs | 3 +-- src/model/helper/bucket.rs | 22 ++++++++++++++++++++++ 12 files changed, 37 insertions(+), 54 deletions(-) diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index db1fda0f..b226c015 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -18,7 +18,7 @@ use garage_model::s3::object_table::ObjectFilter; use crate::admin::error::*; use crate::admin::key::ApiBucketKeyPerm; -use crate::admin::parse_json_body; +use crate::helpers::parse_json_body; pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { let buckets = garage @@ -333,7 +333,7 @@ pub async fn handle_delete_bucket( ) .await?; if !objects.is_empty() { - return Err(Error::bad_request("Bucket is not empty")); + return Err(Error::BucketNotEmpty); } // --- done checking, now commit --- diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index db4d968d..91d99d8a 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -14,7 +14,7 @@ use garage_rpc::layout::*; use garage_model::garage::Garage; use crate::admin::error::*; -use crate::admin::parse_json_body; +use crate::helpers::parse_json_body; pub async fn handle_get_cluster_status(garage: &Arc) -> Result, Error> { let res = GetClusterStatusResponse { diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 1f49fed5..1d68dc69 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -40,10 +40,6 @@ pub enum Error { /// Bucket name is not valid according to AWS S3 specs #[error(display = "Invalid bucket name")] InvalidBucketName, - - /// The client sent a request for an action not supported by garage - #[error(display = "Unimplemented action: {}", _0)] - NotImplemented(String), } impl From for Error @@ -75,7 +71,6 @@ impl ApiError for Error { Error::NoSuchAccessKey | Error::NoSuchBucket => StatusCode::NOT_FOUND, Error::BucketNotEmpty | Error::BucketAlreadyExists => StatusCode::CONFLICT, Error::Forbidden(_) => StatusCode::FORBIDDEN, - Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED, Error::InvalidBucketName => StatusCode::BAD_REQUEST, } } diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index e5f25601..8060bf1a 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -12,7 +12,7 @@ use garage_model::garage::Garage; use garage_model::key_table::*; use crate::admin::error::*; -use crate::admin::parse_json_body; +use crate::helpers::parse_json_body; pub async fn handle_list_keys(garage: &Arc) -> Result, Error> { let res = garage diff --git a/src/api/admin/mod.rs b/src/api/admin/mod.rs index 73700e6e..c4857c10 100644 --- a/src/api/admin/mod.rs +++ b/src/api/admin/mod.rs @@ -5,14 +5,3 @@ mod router; mod bucket; mod cluster; mod key; - -use hyper::{Body, Request}; -use serde::Deserialize; - -use error::*; - -pub async fn parse_json_body Deserialize<'de>>(req: Request) -> Result { - let body = hyper::body::to_bytes(req.into_body()).await?; - let resp: T = serde_json::from_slice(&body).ok_or_bad_request("Invalid JSON")?; - Ok(resp) -} diff --git a/src/api/common_error.rs b/src/api/common_error.rs index eca27e6b..48106e03 100644 --- a/src/api/common_error.rs +++ b/src/api/common_error.rs @@ -38,6 +38,11 @@ impl CommonError { CommonError::BadRequest(_) => StatusCode::BAD_REQUEST, } } + + + pub fn bad_request(msg: M) -> Self { + CommonError::BadRequest(msg.to_string()) + } } /// Trait to map error to the Bad Request error code diff --git a/src/api/helpers.rs b/src/api/helpers.rs index 121fbd5a..599be3f7 100644 --- a/src/api/helpers.rs +++ b/src/api/helpers.rs @@ -2,12 +2,7 @@ use hyper::{Body, Request}; use idna::domain_to_unicode; use serde::Deserialize; -use garage_util::data::*; - -use garage_model::garage::Garage; -use garage_model::key_table::Key; - -use crate::s3::error::*; +use crate::common_error::{*, CommonError as Error}; /// What kind of authorization is required to perform a given action #[derive(Debug, Clone, PartialEq, Eq)] @@ -81,28 +76,6 @@ pub fn authority_to_host(authority: &str) -> Result { authority.map(|h| domain_to_unicode(h).0) } -#[allow(clippy::ptr_arg)] -pub async fn resolve_bucket( - garage: &Garage, - bucket_name: &String, - api_key: &Key, -) -> Result { - let api_key_params = api_key - .state - .as_option() - .ok_or_internal_error("Key should not be deleted at this point")?; - - if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) { - Ok(*bucket_id) - } else { - Ok(garage - .bucket_helper() - .resolve_global_bucket_name(bucket_name) - .await? - .ok_or(Error::NoSuchBucket)?) - } -} - /// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in /// the host header of the request /// diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index 9b4ad882..38ef8d45 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -100,7 +100,7 @@ impl ApiHandler for K2VApiServer { "k2v", )?; - let bucket_id = resolve_bucket(&garage, &bucket_name, &api_key).await?; + let bucket_id = garage.bucket_helper().resolve_bucket(&bucket_name, &api_key).await?; let bucket = garage .bucket_table .get(&EmptyKey, &bucket_id) diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 77ac3879..6b565fd0 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -149,7 +149,7 @@ impl ApiHandler for S3ApiServer { return handle_create_bucket(&garage, req, content_sha256, api_key, bucket_name).await; } - let bucket_id = resolve_bucket(&garage, &bucket_name, &api_key).await?; + let bucket_id = garage.bucket_helper().resolve_bucket(&bucket_name, &api_key).await?; let bucket = garage .bucket_table .get(&EmptyKey, &bucket_id) diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index abd90f4a..2468678e 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -19,7 +19,7 @@ use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; use crate::s3::error::*; -use crate::helpers::{parse_bucket_key, resolve_bucket}; +use crate::helpers::{parse_bucket_key}; use crate::s3::put::{decode_upload_id, get_headers}; use crate::s3::xml::{self as s3_xml, xmlns_tag}; @@ -413,7 +413,7 @@ async fn get_copy_source( let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?; let (source_bucket, source_key) = parse_bucket_key(©_source, None)?; - let source_bucket_id = resolve_bucket(garage, &source_bucket.to_string(), api_key).await?; + let source_bucket_id = garage.bucket_helper().resolve_bucket(&source_bucket.to_string(), api_key).await?; if !api_key.allow_read(&source_bucket_id) { return Err(Error::Forbidden(format!( diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 343aa366..c4b63452 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -15,7 +15,6 @@ use serde::Deserialize; use garage_model::garage::Garage; use crate::s3::error::*; -use crate::helpers::resolve_bucket; use crate::s3::put::{get_headers, save_stream}; use crate::s3::xml as s3_xml; use crate::signature::payload::{parse_date, verify_v4}; @@ -129,7 +128,7 @@ pub async fn handle_post_object( ) .await?; - let bucket_id = resolve_bucket(&garage, &bucket, &api_key).await?; + let bucket_id = garage.bucket_helper().resolve_bucket(&bucket, &api_key).await?; if !api_key.allow_write(&bucket_id) { return Err(Error::Forbidden( diff --git a/src/model/helper/bucket.rs b/src/model/helper/bucket.rs index 788bf3a6..2f1c6ae9 100644 --- a/src/model/helper/bucket.rs +++ b/src/model/helper/bucket.rs @@ -6,6 +6,7 @@ use garage_util::time::*; use crate::bucket_alias_table::*; use crate::bucket_table::*; +use crate::key_table::*; use crate::garage::Garage; use crate::helper::error::*; use crate::helper::key::KeyHelper; @@ -49,6 +50,27 @@ impl<'a> BucketHelper<'a> { } } + #[allow(clippy::ptr_arg)] + pub async fn resolve_bucket( + &self, + bucket_name: &String, + api_key: &Key, + ) -> Result { + let api_key_params = api_key + .state + .as_option() + .ok_or_message("Key should not be deleted at this point")?; + + if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) { + Ok(*bucket_id) + } else { + Ok(self. + resolve_global_bucket_name(bucket_name) + .await? + .ok_or_else(|| Error::NoSuchBucket(bucket_name.to_string()))?) + } + } + /// Returns a Bucket if it is present in bucket table, /// even if it is in deleted state. Querying a non-existing /// bucket ID returns an internal error. -- 2.43.4 From ec16d166f940f59098ae5cc0c0b3d8298f1bcc78 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 15:43:44 +0200 Subject: [PATCH 25/46] Separate error types for k2v and signature --- src/api/k2v/api_server.rs | 8 ++- src/api/k2v/batch.rs | 2 +- src/api/k2v/error.rs | 118 +++++++++++++++++++++++++++++++++ src/api/k2v/index.rs | 2 +- src/api/k2v/item.rs | 2 +- src/api/k2v/mod.rs | 1 + src/api/k2v/range.rs | 2 +- src/api/k2v/router.rs | 2 +- src/api/s3/api_server.rs | 3 +- src/api/s3/error.rs | 13 ++++ src/api/signature/error.rs | 54 +++++++++++++++ src/api/signature/mod.rs | 5 +- src/api/signature/payload.rs | 2 +- src/api/signature/streaming.rs | 2 +- 14 files changed, 203 insertions(+), 13 deletions(-) create mode 100644 src/api/k2v/error.rs create mode 100644 src/api/signature/error.rs diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index 38ef8d45..b70fcdff 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -12,7 +12,7 @@ use garage_util::error::Error as GarageError; use garage_model::garage::Garage; -use crate::s3::error::*; +use crate::k2v::error::*; use crate::generic_server::*; use crate::signature::payload::check_payload_signature; @@ -84,7 +84,8 @@ impl ApiHandler for K2VApiServer { // The OPTIONS method is procesed early, before we even check for an API key if let Endpoint::Options = endpoint { - return handle_options_s3api(garage, &req, Some(bucket_name)).await; + return Ok(handle_options_s3api(garage, &req, Some(bucket_name)).await + .ok_or_bad_request("Error handling OPTIONS")?); } let (api_key, mut content_sha256) = check_payload_signature(&garage, "k2v", &req).await?; @@ -126,7 +127,8 @@ impl ApiHandler for K2VApiServer { // are always preflighted, i.e. the browser should make // an OPTIONS call before to check it is allowed let matching_cors_rule = match *req.method() { - Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req)?, + Method::GET | Method::HEAD | Method::POST => find_matching_cors_rule(&bucket, &req) + .ok_or_internal_error("Error looking up CORS rule")?, _ => None, }; diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index dab3bfb2..8eae471c 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -12,7 +12,7 @@ use garage_model::garage::Garage; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::s3::error::*; +use crate::k2v::error::*; use crate::helpers::*; use crate::k2v::range::read_range; diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs new file mode 100644 index 00000000..6b9e81e6 --- /dev/null +++ b/src/api/k2v/error.rs @@ -0,0 +1,118 @@ +use err_derive::Error; +use hyper::header::HeaderValue; +use hyper::{Body, HeaderMap, StatusCode}; + +use garage_model::helper::error::Error as HelperError; + +use crate::common_error::CommonError; +pub use crate::common_error::{OkOrBadRequest, OkOrInternalError}; +use crate::generic_server::ApiError; +use crate::signature::error::Error as SignatureError; + +/// Errors of this crate +#[derive(Debug, Error)] +pub enum Error { + #[error(display = "{}", _0)] + /// Error from common error + CommonError(CommonError), + + // Category: cannot process + /// No proper api key was used, or the signature was invalid + #[error(display = "Forbidden: {}", _0)] + Forbidden(String), + + /// Authorization Header Malformed + #[error(display = "Authorization header malformed, expected scope: {}", _0)] + AuthorizationHeaderMalformed(String), + + /// The object requested don't exists + #[error(display = "Key not found")] + NoSuchKey, + + /// The bucket requested don't exists + #[error(display = "Bucket not found")] + NoSuchBucket, + + /// Some base64 encoded data was badly encoded + #[error(display = "Invalid base64: {}", _0)] + InvalidBase64(#[error(source)] base64::DecodeError), + + /// The client sent a header with invalid value + #[error(display = "Invalid header value: {}", _0)] + InvalidHeader(#[error(source)] hyper::header::ToStrError), + + /// The client asked for an invalid return format (invalid Accept header) + #[error(display = "Not acceptable: {}", _0)] + NotAcceptable(String), + + /// The request contained an invalid UTF-8 sequence in its path or in other parameters + #[error(display = "Invalid UTF-8: {}", _0)] + InvalidUtf8Str(#[error(source)] std::str::Utf8Error), +} + +impl From for Error +where + CommonError: From, +{ + fn from(err: T) -> Self { + Error::CommonError(CommonError::from(err)) + } +} + +impl From for Error { + fn from(err: HelperError) -> Self { + match err { + HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), + HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), + e => Self::CommonError(CommonError::BadRequest(format!("{}", e))), + } + } +} + +impl From for Error { + fn from(err: SignatureError) -> Self { + match err { + SignatureError::CommonError(c) => Self::CommonError(c), + SignatureError::AuthorizationHeaderMalformed(c) => Self::AuthorizationHeaderMalformed(c), + SignatureError::Forbidden(f) => Self::Forbidden(f), + SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), + SignatureError::InvalidHeader(h) => Self::InvalidHeader(h), + } + } +} + +impl Error { + //pub fn internal_error(msg: M) -> Self { + // Self::CommonError(CommonError::InternalError(GarageError::Message( + // msg.to_string(), + // ))) + //} + + pub fn bad_request(msg: M) -> Self { + Self::CommonError(CommonError::BadRequest(msg.to_string())) + } +} + +impl ApiError for Error { + /// Get the HTTP status code that best represents the meaning of the error for the client + fn http_status_code(&self) -> StatusCode { + match self { + Error::CommonError(c) => c.http_status_code(), + Error::NoSuchKey | Error::NoSuchBucket => StatusCode::NOT_FOUND, + Error::Forbidden(_) => StatusCode::FORBIDDEN, + Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE, + _ => StatusCode::BAD_REQUEST, + } + } + + fn add_http_headers(&self, _header_map: &mut HeaderMap) { + // nothing + } + + fn http_body(&self, garage_region: &str, path: &str) -> Body { + Body::from(format!( + "ERROR: {}\n\ngarage region: {}\npath: {}", + self, garage_region, path + )) + } +} diff --git a/src/api/k2v/index.rs b/src/api/k2v/index.rs index e587841c..d5db906d 100644 --- a/src/api/k2v/index.rs +++ b/src/api/k2v/index.rs @@ -12,7 +12,7 @@ use garage_table::util::*; use garage_model::garage::Garage; use garage_model::k2v::counter_table::{BYTES, CONFLICTS, ENTRIES, VALUES}; -use crate::s3::error::*; +use crate::k2v::error::*; use crate::k2v::range::read_range; pub async fn handle_read_index( diff --git a/src/api/k2v/item.rs b/src/api/k2v/item.rs index 95624d57..836d386f 100644 --- a/src/api/k2v/item.rs +++ b/src/api/k2v/item.rs @@ -10,7 +10,7 @@ use garage_model::garage::Garage; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::s3::error::*; +use crate::k2v::error::*; pub const X_GARAGE_CAUSALITY_TOKEN: &str = "X-Garage-Causality-Token"; diff --git a/src/api/k2v/mod.rs b/src/api/k2v/mod.rs index ee210ad5..b6a8c5cf 100644 --- a/src/api/k2v/mod.rs +++ b/src/api/k2v/mod.rs @@ -1,4 +1,5 @@ pub mod api_server; +mod error; mod router; mod batch; diff --git a/src/api/k2v/range.rs b/src/api/k2v/range.rs index cf6034b9..6aa5c90c 100644 --- a/src/api/k2v/range.rs +++ b/src/api/k2v/range.rs @@ -7,7 +7,7 @@ use std::sync::Arc; use garage_table::replication::TableShardedReplication; use garage_table::*; -use crate::s3::error::*; +use crate::k2v::error::*; use crate::helpers::key_after_prefix; /// Read range in a Garage table. diff --git a/src/api/k2v/router.rs b/src/api/k2v/router.rs index c509a4da..093fb9a7 100644 --- a/src/api/k2v/router.rs +++ b/src/api/k2v/router.rs @@ -1,4 +1,4 @@ -use crate::s3::error::*; +use crate::k2v::error::*; use std::borrow::Cow; diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 6b565fd0..4df9ee6d 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -119,7 +119,8 @@ impl ApiHandler for S3ApiServer { return handle_post_object(garage, req, bucket_name.unwrap()).await; } if let Endpoint::Options = endpoint { - return handle_options_s3api(garage, &req, bucket_name).await; + return handle_options_s3api(garage, &req, bucket_name).await + .map_err(Error::from); } let (api_key, mut content_sha256) = check_payload_signature(&garage, "s3", &req).await?; diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 3cb97019..a0c4703c 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -11,6 +11,7 @@ use crate::common_error::CommonError; pub use crate::common_error::{OkOrBadRequest, OkOrInternalError}; use crate::generic_server::ApiError; use crate::s3::xml as s3_xml; +use crate::signature::error::Error as SignatureError; /// Errors of this crate #[derive(Debug, Error)] @@ -134,6 +135,18 @@ impl From for Error { } } +impl From for Error { + fn from(err: SignatureError) -> Self { + match err { + SignatureError::CommonError(c) => Self::CommonError(c), + SignatureError::AuthorizationHeaderMalformed(c) => Self::AuthorizationHeaderMalformed(c), + SignatureError::Forbidden(f) => Self::Forbidden(f), + SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), + SignatureError::InvalidHeader(h) => Self::InvalidHeader(h), + } + } +} + impl From for Error { fn from(err: multer::Error) -> Self { Self::bad_request(err) diff --git a/src/api/signature/error.rs b/src/api/signature/error.rs new file mode 100644 index 00000000..69f3c6c5 --- /dev/null +++ b/src/api/signature/error.rs @@ -0,0 +1,54 @@ +use err_derive::Error; + +use garage_util::error::Error as GarageError; + +use crate::common_error::CommonError; +pub use crate::common_error::{OkOrBadRequest, OkOrInternalError}; + +/// Errors of this crate +#[derive(Debug, Error)] +pub enum Error { + #[error(display = "{}", _0)] + /// Error from common error + CommonError(CommonError), + + /// Authorization Header Malformed + #[error(display = "Authorization header malformed, expected scope: {}", _0)] + AuthorizationHeaderMalformed(String), + + /// No proper api key was used, or the signature was invalid + #[error(display = "Forbidden: {}", _0)] + Forbidden(String), + + // Category: bad request + /// The request contained an invalid UTF-8 sequence in its path or in other parameters + #[error(display = "Invalid UTF-8: {}", _0)] + InvalidUtf8Str(#[error(source)] std::str::Utf8Error), + + /// The client sent a header with invalid value + #[error(display = "Invalid header value: {}", _0)] + InvalidHeader(#[error(source)] hyper::header::ToStrError), +} + +impl From for Error +where + CommonError: From, +{ + fn from(err: T) -> Self { + Error::CommonError(CommonError::from(err)) + } +} + + +impl Error { + pub fn internal_error(msg: M) -> Self { + Self::CommonError(CommonError::InternalError(GarageError::Message( + msg.to_string(), + ))) + } + + pub fn bad_request(msg: M) -> Self { + Self::CommonError(CommonError::BadRequest(msg.to_string())) + } +} + diff --git a/src/api/signature/mod.rs b/src/api/signature/mod.rs index 4679747f..dd5b590c 100644 --- a/src/api/signature/mod.rs +++ b/src/api/signature/mod.rs @@ -4,11 +4,12 @@ use sha2::Sha256; use garage_util::data::{sha256sum, Hash}; -use crate::s3::error::*; - +pub mod error; pub mod payload; pub mod streaming; +use error::*; + pub const SHORT_DATE: &str = "%Y%m%d"; pub const LONG_DATETIME: &str = "%Y%m%dT%H%M%SZ"; diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 47445bc7..155a6f94 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -15,7 +15,7 @@ use super::LONG_DATETIME; use super::{compute_scope, signing_hmac}; use crate::encoding::uri_encode; -use crate::s3::error::*; +use crate::signature::error::*; pub async fn check_payload_signature( garage: &Garage, diff --git a/src/api/signature/streaming.rs b/src/api/signature/streaming.rs index 06a0512e..c8358c4f 100644 --- a/src/api/signature/streaming.rs +++ b/src/api/signature/streaming.rs @@ -12,7 +12,7 @@ use garage_util::data::Hash; use super::{compute_scope, sha256sum, HmacSha256, LONG_DATETIME}; -use crate::s3::error::*; +use crate::signature::error::*; pub fn parse_streaming_body( api_key: &Key, -- 2.43.4 From ea325d78d36d19f59a0849ace1f4567e2b095bd7 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 19:18:51 +0200 Subject: [PATCH 26/46] More error refactoring --- src/api/admin/api_server.rs | 8 +-- src/api/admin/bucket.rs | 5 +- src/api/admin/error.rs | 43 +++------------ src/api/common_error.rs | 67 ++++++++++++++++++++++- src/api/helpers.rs | 2 +- src/api/k2v/api_server.rs | 28 +++++----- src/api/k2v/batch.rs | 2 +- src/api/k2v/error.rs | 40 +++++--------- src/api/k2v/range.rs | 2 +- src/api/s3/api_server.rs | 28 +++++----- src/api/s3/bucket.rs | 5 +- src/api/s3/copy.rs | 9 ++- src/api/s3/cors.rs | 32 ++++------- src/api/s3/error.rs | 103 +++++++++++------------------------ src/api/s3/list.rs | 12 +++- src/api/s3/post_object.rs | 15 +++-- src/api/s3/router.rs | 2 +- src/api/s3/website.rs | 23 +++----- src/api/signature/error.rs | 22 +------- src/api/signature/payload.rs | 4 +- src/model/helper/bucket.rs | 12 ++-- src/web/web_server.rs | 4 +- 22 files changed, 209 insertions(+), 259 deletions(-) diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index b2effa62..098a54aa 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -114,13 +114,9 @@ impl ApiHandler for AdminApiServer { if let Some(h) = expected_auth_header { match req.headers().get("Authorization") { - None => Err(Error::Forbidden( - "Authorization token must be provided".into(), - )), + None => Err(Error::forbidden("Authorization token must be provided")), Some(v) if v.to_str().map(|hv| hv == h).unwrap_or(false) => Ok(()), - _ => Err(Error::Forbidden( - "Invalid authorization token provided".into(), - )), + _ => Err(Error::forbidden("Invalid authorization token provided")), }?; } diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index b226c015..c5518e4e 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -18,6 +18,7 @@ use garage_model::s3::object_table::ObjectFilter; use crate::admin::error::*; use crate::admin::key::ApiBucketKeyPerm; +use crate::common_error::CommonError; use crate::helpers::parse_json_body; pub async fn handle_list_buckets(garage: &Arc) -> Result, Error> { @@ -233,7 +234,7 @@ pub async fn handle_create_bucket( if let Some(alias) = garage.bucket_alias_table.get(&EmptyKey, ga).await? { if alias.state.get().is_some() { - return Err(Error::BucketAlreadyExists); + return Err(CommonError::BucketAlreadyExists.into()); } } } @@ -333,7 +334,7 @@ pub async fn handle_delete_bucket( ) .await?; if !objects.is_empty() { - return Err(Error::BucketNotEmpty); + return Err(CommonError::BucketNotEmpty.into()); } // --- done checking, now commit --- diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 1d68dc69..bb35c16b 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -4,9 +4,9 @@ use hyper::{Body, HeaderMap, StatusCode}; use garage_model::helper::error::Error as HelperError; -use crate::generic_server::ApiError; use crate::common_error::CommonError; -pub use crate::common_error::{OkOrBadRequest, OkOrInternalError}; +pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; +use crate::generic_server::ApiError; /// Errors of this crate #[derive(Debug, Error)] @@ -16,30 +16,9 @@ pub enum Error { CommonError(CommonError), // Category: cannot process - /// No proper api key was used, or the signature was invalid - #[error(display = "Forbidden: {}", _0)] - Forbidden(String), - /// The API access key does not exist #[error(display = "Access key not found")] NoSuchAccessKey, - - /// The bucket requested don't exists - #[error(display = "Bucket not found")] - NoSuchBucket, - - /// Tried to create a bucket that already exist - #[error(display = "Bucket already exists")] - BucketAlreadyExists, - - /// Tried to delete a non-empty bucket - #[error(display = "Tried to delete a non-empty bucket")] - BucketNotEmpty, - - // Category: bad request - /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name")] - InvalidBucketName, } impl From for Error @@ -51,14 +30,16 @@ where } } +impl CommonErrorDerivative for Error {} + impl From for Error { fn from(err: HelperError) -> Self { match err { HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(_) => Self::InvalidBucketName, + HelperError::InvalidBucketName(_) => Self::CommonError(CommonError::InvalidBucketName), + HelperError::NoSuchBucket(_) => Self::CommonError(CommonError::NoSuchBucket), HelperError::NoSuchAccessKey(_) => Self::NoSuchAccessKey, - HelperError::NoSuchBucket(_) => Self::NoSuchBucket, } } } @@ -68,10 +49,7 @@ impl ApiError for Error { fn http_status_code(&self) -> StatusCode { match self { Error::CommonError(c) => c.http_status_code(), - Error::NoSuchAccessKey | Error::NoSuchBucket => StatusCode::NOT_FOUND, - Error::BucketNotEmpty | Error::BucketAlreadyExists => StatusCode::CONFLICT, - Error::Forbidden(_) => StatusCode::FORBIDDEN, - Error::InvalidBucketName => StatusCode::BAD_REQUEST, + Error::NoSuchAccessKey => StatusCode::NOT_FOUND, } } @@ -80,15 +58,10 @@ impl ApiError for Error { } fn http_body(&self, garage_region: &str, path: &str) -> Body { + // TODO nice json error Body::from(format!( "ERROR: {}\n\ngarage region: {}\npath: {}", self, garage_region, path )) } } - -impl Error { - pub fn bad_request(msg: M) -> Self { - Self::CommonError(CommonError::BadRequest(msg.to_string())) - } -} diff --git a/src/api/common_error.rs b/src/api/common_error.rs index 48106e03..b6dbf059 100644 --- a/src/api/common_error.rs +++ b/src/api/common_error.rs @@ -6,7 +6,7 @@ use garage_util::error::Error as GarageError; /// Errors of this crate #[derive(Debug, Error)] pub enum CommonError { - // Category: internal error + // ---- INTERNAL ERRORS ---- /// Error related to deeper parts of Garage #[error(display = "Internal error: {}", _0)] InternalError(#[error(source)] GarageError), @@ -19,9 +19,34 @@ pub enum CommonError { #[error(display = "Internal error (HTTP error): {}", _0)] Http(#[error(source)] http::Error), - /// The client sent an invalid request + // ---- GENERIC CLIENT ERRORS ---- + /// Proper authentication was not provided + #[error(display = "Forbidden: {}", _0)] + Forbidden(String), + + /// Generic bad request response with custom message #[error(display = "Bad request: {}", _0)] BadRequest(String), + + // ---- SPECIFIC ERROR CONDITIONS ---- + // These have to be error codes referenced in the S3 spec here: + // https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList + /// The bucket requested don't exists + #[error(display = "Bucket not found")] + NoSuchBucket, + + /// Tried to create a bucket that already exist + #[error(display = "Bucket already exists")] + BucketAlreadyExists, + + /// Tried to delete a non-empty bucket + #[error(display = "Tried to delete a non-empty bucket")] + BucketNotEmpty, + + // Category: bad request + /// Bucket name is not valid according to AWS S3 specs + #[error(display = "Invalid bucket name")] + InvalidBucketName, } impl CommonError { @@ -36,15 +61,53 @@ impl CommonError { StatusCode::INTERNAL_SERVER_ERROR } CommonError::BadRequest(_) => StatusCode::BAD_REQUEST, + CommonError::Forbidden(_) => StatusCode::FORBIDDEN, + CommonError::NoSuchBucket => StatusCode::NOT_FOUND, + CommonError::BucketNotEmpty | CommonError::BucketAlreadyExists => StatusCode::CONFLICT, + CommonError::InvalidBucketName => StatusCode::BAD_REQUEST, } } + pub fn aws_code(&self) -> &'static str { + match self { + CommonError::Forbidden(_) => "AccessDenied", + CommonError::InternalError( + GarageError::Timeout + | GarageError::RemoteError(_) + | GarageError::Quorum(_, _, _, _), + ) => "ServiceUnavailable", + CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_) => { + "InternalError" + } + CommonError::BadRequest(_) => "InvalidRequest", + CommonError::NoSuchBucket => "NoSuchBucket", + CommonError::BucketAlreadyExists => "BucketAlreadyExists", + CommonError::BucketNotEmpty => "BucketNotEmpty", + CommonError::InvalidBucketName => "InvalidBucketName", + } + } pub fn bad_request(msg: M) -> Self { CommonError::BadRequest(msg.to_string()) } } +pub trait CommonErrorDerivative: From { + fn internal_error(msg: M) -> Self { + Self::from(CommonError::InternalError(GarageError::Message( + msg.to_string(), + ))) + } + + fn bad_request(msg: M) -> Self { + Self::from(CommonError::BadRequest(msg.to_string())) + } + + fn forbidden(msg: M) -> Self { + Self::from(CommonError::Forbidden(msg.to_string())) + } +} + /// Trait to map error to the Bad Request error code pub trait OkOrBadRequest { type S; diff --git a/src/api/helpers.rs b/src/api/helpers.rs index 599be3f7..aa350e3c 100644 --- a/src/api/helpers.rs +++ b/src/api/helpers.rs @@ -2,7 +2,7 @@ use hyper::{Body, Request}; use idna::domain_to_unicode; use serde::Deserialize; -use crate::common_error::{*, CommonError as Error}; +use crate::common_error::{CommonError as Error, *}; /// What kind of authorization is required to perform a given action #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/src/api/k2v/api_server.rs b/src/api/k2v/api_server.rs index b70fcdff..eb0fbdd7 100644 --- a/src/api/k2v/api_server.rs +++ b/src/api/k2v/api_server.rs @@ -7,13 +7,12 @@ use hyper::{Body, Method, Request, Response}; use opentelemetry::{trace::SpanRef, KeyValue}; -use garage_table::util::*; use garage_util::error::Error as GarageError; use garage_model::garage::Garage; -use crate::k2v::error::*; use crate::generic_server::*; +use crate::k2v::error::*; use crate::signature::payload::check_payload_signature; use crate::signature::streaming::*; @@ -84,14 +83,14 @@ impl ApiHandler for K2VApiServer { // The OPTIONS method is procesed early, before we even check for an API key if let Endpoint::Options = endpoint { - return Ok(handle_options_s3api(garage, &req, Some(bucket_name)).await + return Ok(handle_options_s3api(garage, &req, Some(bucket_name)) + .await .ok_or_bad_request("Error handling OPTIONS")?); } let (api_key, mut content_sha256) = check_payload_signature(&garage, "k2v", &req).await?; - let api_key = api_key.ok_or_else(|| { - Error::Forbidden("Garage does not support anonymous access yet".to_string()) - })?; + let api_key = api_key + .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; let req = parse_streaming_body( &api_key, @@ -101,13 +100,14 @@ impl ApiHandler for K2VApiServer { "k2v", )?; - let bucket_id = garage.bucket_helper().resolve_bucket(&bucket_name, &api_key).await?; + let bucket_id = garage + .bucket_helper() + .resolve_bucket(&bucket_name, &api_key) + .await?; let bucket = garage - .bucket_table - .get(&EmptyKey, &bucket_id) - .await? - .filter(|b| !b.state.is_deleted()) - .ok_or(Error::NoSuchBucket)?; + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; let allowed = match endpoint.authorization_type() { Authorization::Read => api_key.allow_read(&bucket_id), @@ -117,9 +117,7 @@ impl ApiHandler for K2VApiServer { }; if !allowed { - return Err(Error::Forbidden( - "Operation is not allowed for this key.".to_string(), - )); + return Err(Error::forbidden("Operation is not allowed for this key.")); } // Look up what CORS rule might apply to response. diff --git a/src/api/k2v/batch.rs b/src/api/k2v/batch.rs index 8eae471c..db9901cf 100644 --- a/src/api/k2v/batch.rs +++ b/src/api/k2v/batch.rs @@ -12,8 +12,8 @@ use garage_model::garage::Garage; use garage_model::k2v::causality::*; use garage_model::k2v::item_table::*; -use crate::k2v::error::*; use crate::helpers::*; +use crate::k2v::error::*; use crate::k2v::range::read_range; pub async fn handle_insert_batch( diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index 6b9e81e6..4d8c1154 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -5,7 +5,7 @@ use hyper::{Body, HeaderMap, StatusCode}; use garage_model::helper::error::Error as HelperError; use crate::common_error::CommonError; -pub use crate::common_error::{OkOrBadRequest, OkOrInternalError}; +pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; use crate::generic_server::ApiError; use crate::signature::error::Error as SignatureError; @@ -17,10 +17,6 @@ pub enum Error { CommonError(CommonError), // Category: cannot process - /// No proper api key was used, or the signature was invalid - #[error(display = "Forbidden: {}", _0)] - Forbidden(String), - /// Authorization Header Malformed #[error(display = "Authorization header malformed, expected scope: {}", _0)] AuthorizationHeaderMalformed(String), @@ -29,10 +25,6 @@ pub enum Error { #[error(display = "Key not found")] NoSuchKey, - /// The bucket requested don't exists - #[error(display = "Bucket not found")] - NoSuchBucket, - /// Some base64 encoded data was badly encoded #[error(display = "Invalid base64: {}", _0)] InvalidBase64(#[error(source)] base64::DecodeError), @@ -59,11 +51,15 @@ where } } +impl CommonErrorDerivative for Error {} + impl From for Error { fn from(err: HelperError) -> Self { match err { HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), + HelperError::InvalidBucketName(_) => Self::CommonError(CommonError::InvalidBucketName), + HelperError::NoSuchBucket(_) => Self::CommonError(CommonError::NoSuchBucket), e => Self::CommonError(CommonError::BadRequest(format!("{}", e))), } } @@ -73,35 +69,26 @@ impl From for Error { fn from(err: SignatureError) -> Self { match err { SignatureError::CommonError(c) => Self::CommonError(c), - SignatureError::AuthorizationHeaderMalformed(c) => Self::AuthorizationHeaderMalformed(c), - SignatureError::Forbidden(f) => Self::Forbidden(f), + SignatureError::AuthorizationHeaderMalformed(c) => { + Self::AuthorizationHeaderMalformed(c) + } SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), SignatureError::InvalidHeader(h) => Self::InvalidHeader(h), } } } -impl Error { - //pub fn internal_error(msg: M) -> Self { - // Self::CommonError(CommonError::InternalError(GarageError::Message( - // msg.to_string(), - // ))) - //} - - pub fn bad_request(msg: M) -> Self { - Self::CommonError(CommonError::BadRequest(msg.to_string())) - } -} - impl ApiError for Error { /// Get the HTTP status code that best represents the meaning of the error for the client fn http_status_code(&self) -> StatusCode { match self { Error::CommonError(c) => c.http_status_code(), - Error::NoSuchKey | Error::NoSuchBucket => StatusCode::NOT_FOUND, - Error::Forbidden(_) => StatusCode::FORBIDDEN, + Error::NoSuchKey => StatusCode::NOT_FOUND, Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE, - _ => StatusCode::BAD_REQUEST, + Error::AuthorizationHeaderMalformed(_) + | Error::InvalidBase64(_) + | Error::InvalidHeader(_) + | Error::InvalidUtf8Str(_) => StatusCode::BAD_REQUEST, } } @@ -110,6 +97,7 @@ impl ApiError for Error { } fn http_body(&self, garage_region: &str, path: &str) -> Body { + // TODO nice json error Body::from(format!( "ERROR: {}\n\ngarage region: {}\npath: {}", self, garage_region, path diff --git a/src/api/k2v/range.rs b/src/api/k2v/range.rs index 6aa5c90c..1f7dc4cd 100644 --- a/src/api/k2v/range.rs +++ b/src/api/k2v/range.rs @@ -7,8 +7,8 @@ use std::sync::Arc; use garage_table::replication::TableShardedReplication; use garage_table::*; -use crate::k2v::error::*; use crate::helpers::key_after_prefix; +use crate::k2v::error::*; /// Read range in a Garage table. /// Returns (entries, more?, nextStart) diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 4df9ee6d..87d0f288 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -8,14 +8,13 @@ use hyper::{Body, Method, Request, Response}; use opentelemetry::{trace::SpanRef, KeyValue}; -use garage_table::util::*; use garage_util::error::Error as GarageError; use garage_model::garage::Garage; use garage_model::key_table::Key; -use crate::s3::error::*; use crate::generic_server::*; +use crate::s3::error::*; use crate::signature::payload::check_payload_signature; use crate::signature::streaming::*; @@ -119,14 +118,14 @@ impl ApiHandler for S3ApiServer { return handle_post_object(garage, req, bucket_name.unwrap()).await; } if let Endpoint::Options = endpoint { - return handle_options_s3api(garage, &req, bucket_name).await + return handle_options_s3api(garage, &req, bucket_name) + .await .map_err(Error::from); } let (api_key, mut content_sha256) = check_payload_signature(&garage, "s3", &req).await?; - let api_key = api_key.ok_or_else(|| { - Error::Forbidden("Garage does not support anonymous access yet".to_string()) - })?; + let api_key = api_key + .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))?; let req = parse_streaming_body( &api_key, @@ -150,13 +149,14 @@ impl ApiHandler for S3ApiServer { return handle_create_bucket(&garage, req, content_sha256, api_key, bucket_name).await; } - let bucket_id = garage.bucket_helper().resolve_bucket(&bucket_name, &api_key).await?; + let bucket_id = garage + .bucket_helper() + .resolve_bucket(&bucket_name, &api_key) + .await?; let bucket = garage - .bucket_table - .get(&EmptyKey, &bucket_id) - .await? - .filter(|b| !b.state.is_deleted()) - .ok_or(Error::NoSuchBucket)?; + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; let allowed = match endpoint.authorization_type() { Authorization::Read => api_key.allow_read(&bucket_id), @@ -166,9 +166,7 @@ impl ApiHandler for S3ApiServer { }; if !allowed { - return Err(Error::Forbidden( - "Operation is not allowed for this key.".to_string(), - )); + return Err(Error::forbidden("Operation is not allowed for this key.")); } // Look up what CORS rule might apply to response. diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index d4a6b0cb..1304cc07 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -14,6 +14,7 @@ use garage_util::crdt::*; use garage_util::data::*; use garage_util::time::*; +use crate::common_error::CommonError; use crate::s3::error::*; use crate::s3::xml as s3_xml; use crate::signature::verify_signed_content; @@ -158,7 +159,7 @@ pub async fn handle_create_bucket( // otherwise return a forbidden error. let kp = api_key.bucket_permissions(&bucket_id); if !(kp.allow_write || kp.allow_owner) { - return Err(Error::BucketAlreadyExists); + return Err(CommonError::BucketAlreadyExists.into()); } } else { // Create the bucket! @@ -239,7 +240,7 @@ pub async fn handle_delete_bucket( ) .await?; if !objects.is_empty() { - return Err(Error::BucketNotEmpty); + return Err(CommonError::BucketNotEmpty.into()); } // --- done checking, now commit --- diff --git a/src/api/s3/copy.rs b/src/api/s3/copy.rs index 2468678e..0fc16993 100644 --- a/src/api/s3/copy.rs +++ b/src/api/s3/copy.rs @@ -18,8 +18,8 @@ use garage_model::s3::block_ref_table::*; use garage_model::s3::object_table::*; use garage_model::s3::version_table::*; +use crate::helpers::parse_bucket_key; use crate::s3::error::*; -use crate::helpers::{parse_bucket_key}; use crate::s3::put::{decode_upload_id, get_headers}; use crate::s3::xml::{self as s3_xml, xmlns_tag}; @@ -413,10 +413,13 @@ async fn get_copy_source( let copy_source = percent_encoding::percent_decode_str(copy_source).decode_utf8()?; let (source_bucket, source_key) = parse_bucket_key(©_source, None)?; - let source_bucket_id = garage.bucket_helper().resolve_bucket(&source_bucket.to_string(), api_key).await?; + let source_bucket_id = garage + .bucket_helper() + .resolve_bucket(&source_bucket.to_string(), api_key) + .await?; if !api_key.allow_read(&source_bucket_id) { - return Err(Error::Forbidden(format!( + return Err(Error::forbidden(format!( "Reading from bucket {} not allowed for this key", source_bucket ))); diff --git a/src/api/s3/cors.rs b/src/api/s3/cors.rs index 1ad4f2f8..c7273464 100644 --- a/src/api/s3/cors.rs +++ b/src/api/s3/cors.rs @@ -15,7 +15,6 @@ use crate::signature::verify_signed_content; use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule}; use garage_model::garage::Garage; -use garage_table::*; use garage_util::data::*; pub async fn handle_get_cors(bucket: &Bucket) -> Result, Error> { @@ -48,14 +47,11 @@ pub async fn handle_delete_cors( bucket_id: Uuid, ) -> Result, Error> { let mut bucket = garage - .bucket_table - .get(&EmptyKey, &bucket_id) - .await? - .ok_or(Error::NoSuchBucket)?; + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; + let param = bucket.params_mut().unwrap(); param.cors_config.update(None); garage.bucket_table.insert(&bucket).await?; @@ -78,14 +74,11 @@ pub async fn handle_put_cors( } let mut bucket = garage - .bucket_table - .get(&EmptyKey, &bucket_id) - .await? - .ok_or(Error::NoSuchBucket)?; + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; + let param = bucket.params_mut().unwrap(); let conf: CorsConfiguration = from_reader(&body as &[u8])?; conf.validate()?; @@ -119,12 +112,7 @@ pub async fn handle_options_s3api( let helper = garage.bucket_helper(); let bucket_id = helper.resolve_global_bucket_name(&bn).await?; if let Some(id) = bucket_id { - let bucket = garage - .bucket_table - .get(&EmptyKey, &id) - .await? - .filter(|b| !b.state.is_deleted()) - .ok_or(Error::NoSuchBucket)?; + let bucket = garage.bucket_helper().get_existing_bucket(id).await?; handle_options_for_bucket(req, &bucket) } else { // If there is a bucket name in the request, but that name @@ -185,7 +173,7 @@ pub fn handle_options_for_bucket( } } - Err(Error::Forbidden("This CORS request is not allowed.".into())) + Err(Error::forbidden("This CORS request is not allowed.")) } pub fn find_matching_cors_rule<'a>( diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index a0c4703c..4edff3a1 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -5,10 +5,9 @@ use hyper::header::HeaderValue; use hyper::{Body, HeaderMap, StatusCode}; use garage_model::helper::error::Error as HelperError; -use garage_util::error::Error as GarageError; use crate::common_error::CommonError; -pub use crate::common_error::{OkOrBadRequest, OkOrInternalError}; +pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; use crate::generic_server::ApiError; use crate::s3::xml as s3_xml; use crate::signature::error::Error as SignatureError; @@ -21,10 +20,6 @@ pub enum Error { CommonError(CommonError), // Category: cannot process - /// No proper api key was used, or the signature was invalid - #[error(display = "Forbidden: {}", _0)] - Forbidden(String), - /// Authorization Header Malformed #[error(display = "Authorization header malformed, expected scope: {}", _0)] AuthorizationHeaderMalformed(String), @@ -33,22 +28,10 @@ pub enum Error { #[error(display = "Key not found")] NoSuchKey, - /// The bucket requested don't exists - #[error(display = "Bucket not found")] - NoSuchBucket, - /// The multipart upload requested don't exists #[error(display = "Upload not found")] NoSuchUpload, - /// Tried to create a bucket that already exist - #[error(display = "Bucket already exists")] - BucketAlreadyExists, - - /// Tried to delete a non-empty bucket - #[error(display = "Tried to delete a non-empty bucket")] - BucketNotEmpty, - /// Precondition failed (e.g. x-amz-copy-source-if-match) #[error(display = "At least one of the preconditions you specified did not hold")] PreconditionFailed, @@ -75,14 +58,6 @@ pub enum Error { #[error(display = "Invalid UTF-8: {}", _0)] InvalidUtf8String(#[error(source)] std::string::FromUtf8Error), - /// Some base64 encoded data was badly encoded - #[error(display = "Invalid base64: {}", _0)] - InvalidBase64(#[error(source)] base64::DecodeError), - - /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name")] - InvalidBucketName, - /// The client sent invalid XML data #[error(display = "Invalid XML: {}", _0)] InvalidXml(String), @@ -95,10 +70,6 @@ pub enum Error { #[error(display = "Invalid HTTP range: {:?}", _0)] InvalidRange(#[error(from)] (http_range::HttpRangeParseError, u64)), - /// The client asked for an invalid return format (invalid Accept header) - #[error(display = "Not acceptable: {}", _0)] - NotAcceptable(String), - /// The client sent a request for an action not supported by garage #[error(display = "Unimplemented action: {}", _0)] NotImplemented(String), @@ -113,6 +84,20 @@ where } } +impl CommonErrorDerivative for Error {} + +impl From for Error { + fn from(err: HelperError) -> Self { + match err { + HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), + HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), + HelperError::InvalidBucketName(_) => Self::CommonError(CommonError::InvalidBucketName), + HelperError::NoSuchBucket(_) => Self::CommonError(CommonError::NoSuchBucket), + e => Self::bad_request(format!("{}", e)), + } + } +} + impl From for Error { fn from(err: roxmltree::Error) -> Self { Self::InvalidXml(format!("{}", err)) @@ -125,22 +110,13 @@ impl From for Error { } } -impl From for Error { - fn from(err: HelperError) -> Self { - match err { - HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), - e => Self::CommonError(CommonError::BadRequest(format!("{}", e))), - } - } -} - impl From for Error { fn from(err: SignatureError) -> Self { match err { SignatureError::CommonError(c) => Self::CommonError(c), - SignatureError::AuthorizationHeaderMalformed(c) => Self::AuthorizationHeaderMalformed(c), - SignatureError::Forbidden(f) => Self::Forbidden(f), + SignatureError::AuthorizationHeaderMalformed(c) => { + Self::AuthorizationHeaderMalformed(c) + } SignatureError::InvalidUtf8Str(i) => Self::InvalidUtf8Str(i), SignatureError::InvalidHeader(h) => Self::InvalidHeader(h), } @@ -156,39 +132,22 @@ impl From for Error { impl Error { pub fn aws_code(&self) -> &'static str { match self { + Error::CommonError(c) => c.aws_code(), Error::NoSuchKey => "NoSuchKey", - Error::NoSuchBucket => "NoSuchBucket", Error::NoSuchUpload => "NoSuchUpload", - Error::BucketAlreadyExists => "BucketAlreadyExists", - Error::BucketNotEmpty => "BucketNotEmpty", Error::PreconditionFailed => "PreconditionFailed", Error::InvalidPart => "InvalidPart", Error::InvalidPartOrder => "InvalidPartOrder", Error::EntityTooSmall => "EntityTooSmall", - Error::Forbidden(_) => "AccessDenied", Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed", Error::NotImplemented(_) => "NotImplemented", - Error::CommonError(CommonError::InternalError( - GarageError::Timeout - | GarageError::RemoteError(_) - | GarageError::Quorum(_, _, _, _), - )) => "ServiceUnavailable", - Error::CommonError( - CommonError::InternalError(_) | CommonError::Hyper(_) | CommonError::Http(_), - ) => "InternalError", - _ => "InvalidRequest", + Error::InvalidXml(_) => "MalformedXML", + Error::InvalidRange(_) => "InvalidRange", + Error::InvalidUtf8Str(_) | Error::InvalidUtf8String(_) | Error::InvalidHeader(_) => { + "InvalidRequest" + } } } - - pub fn internal_error(msg: M) -> Self { - Self::CommonError(CommonError::InternalError(GarageError::Message( - msg.to_string(), - ))) - } - - pub fn bad_request(msg: M) -> Self { - Self::CommonError(CommonError::BadRequest(msg.to_string())) - } } impl ApiError for Error { @@ -196,14 +155,18 @@ impl ApiError for Error { fn http_status_code(&self) -> StatusCode { match self { Error::CommonError(c) => c.http_status_code(), - Error::NoSuchKey | Error::NoSuchBucket | Error::NoSuchUpload => StatusCode::NOT_FOUND, - Error::BucketNotEmpty | Error::BucketAlreadyExists => StatusCode::CONFLICT, + Error::NoSuchKey | Error::NoSuchUpload => StatusCode::NOT_FOUND, Error::PreconditionFailed => StatusCode::PRECONDITION_FAILED, - Error::Forbidden(_) => StatusCode::FORBIDDEN, - Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE, Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE, Error::NotImplemented(_) => StatusCode::NOT_IMPLEMENTED, - _ => StatusCode::BAD_REQUEST, + Error::AuthorizationHeaderMalformed(_) + | Error::InvalidPart + | Error::InvalidPartOrder + | Error::EntityTooSmall + | Error::InvalidXml(_) + | Error::InvalidUtf8Str(_) + | Error::InvalidUtf8String(_) + | Error::InvalidHeader(_) => StatusCode::BAD_REQUEST, } } diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index b4ba5bcd..12f6149d 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -16,8 +16,8 @@ use garage_model::s3::version_table::Version; use garage_table::{EmptyKey, EnumerationOrder}; use crate::encoding::*; -use crate::s3::error::*; use crate::helpers::key_after_prefix; +use crate::s3::error::*; use crate::s3::put as s3_put; use crate::s3::xml as s3_xml; @@ -582,11 +582,17 @@ impl ListObjectsQuery { // representing the key to start with. (Some(token), _) => match &token[..1] { "[" => Ok(RangeBegin::IncludingKey { - key: String::from_utf8(base64::decode(token[1..].as_bytes())?)?, + key: String::from_utf8( + base64::decode(token[1..].as_bytes()) + .ok_or_bad_request("Invalid continuation token")?, + )?, fallback_key: None, }), "]" => Ok(RangeBegin::AfterKey { - key: String::from_utf8(base64::decode(token[1..].as_bytes())?)?, + key: String::from_utf8( + base64::decode(token[1..].as_bytes()) + .ok_or_bad_request("Invalid continuation token")?, + )?, }), _ => Err(Error::bad_request("Invalid continuation token".to_string())), }, diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index c4b63452..302ebe01 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -89,9 +89,7 @@ pub async fn handle_post_object( .to_str()?; let credential = params .get("x-amz-credential") - .ok_or_else(|| { - Error::Forbidden("Garage does not support anonymous access yet".to_string()) - })? + .ok_or_else(|| Error::forbidden("Garage does not support anonymous access yet"))? .to_str()?; let policy = params .get("policy") @@ -128,15 +126,16 @@ pub async fn handle_post_object( ) .await?; - let bucket_id = garage.bucket_helper().resolve_bucket(&bucket, &api_key).await?; + let bucket_id = garage + .bucket_helper() + .resolve_bucket(&bucket, &api_key) + .await?; if !api_key.allow_write(&bucket_id) { - return Err(Error::Forbidden( - "Operation is not allowed for this key.".to_string(), - )); + return Err(Error::forbidden("Operation is not allowed for this key.")); } - let decoded_policy = base64::decode(&policy)?; + let decoded_policy = base64::decode(&policy).ok_or_bad_request("Invalid policy")?; let decoded_policy: Policy = serde_json::from_slice(&decoded_policy).ok_or_bad_request("Invalid policy")?; diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index b12c63a7..0e769558 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -3,9 +3,9 @@ use std::borrow::Cow; use hyper::header::HeaderValue; use hyper::{HeaderMap, Method, Request}; -use crate::s3::error::{Error, OkOrBadRequest}; use crate::helpers::Authorization; use crate::router_macros::{generateQueryParameters, router_match}; +use crate::s3::error::*; router_match! {@func diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index b2582c4b..133c8327 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -10,7 +10,6 @@ use crate::signature::verify_signed_content; use garage_model::bucket_table::*; use garage_model::garage::Garage; -use garage_table::*; use garage_util::data::*; pub async fn handle_get_website(bucket: &Bucket) -> Result, Error> { @@ -47,14 +46,11 @@ pub async fn handle_delete_website( bucket_id: Uuid, ) -> Result, Error> { let mut bucket = garage - .bucket_table - .get(&EmptyKey, &bucket_id) - .await? - .ok_or(Error::NoSuchBucket)?; + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; + let param = bucket.params_mut().unwrap(); param.website_config.update(None); garage.bucket_table.insert(&bucket).await?; @@ -77,14 +73,11 @@ pub async fn handle_put_website( } let mut bucket = garage - .bucket_table - .get(&EmptyKey, &bucket_id) - .await? - .ok_or(Error::NoSuchBucket)?; + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; - let param = bucket - .params_mut() - .ok_or_internal_error("Bucket should not be deleted at this point")?; + let param = bucket.params_mut().unwrap(); let conf: WebsiteConfiguration = from_reader(&body as &[u8])?; conf.validate()?; diff --git a/src/api/signature/error.rs b/src/api/signature/error.rs index 69f3c6c5..3ef5cdcd 100644 --- a/src/api/signature/error.rs +++ b/src/api/signature/error.rs @@ -1,9 +1,7 @@ use err_derive::Error; -use garage_util::error::Error as GarageError; - use crate::common_error::CommonError; -pub use crate::common_error::{OkOrBadRequest, OkOrInternalError}; +pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; /// Errors of this crate #[derive(Debug, Error)] @@ -16,10 +14,6 @@ pub enum Error { #[error(display = "Authorization header malformed, expected scope: {}", _0)] AuthorizationHeaderMalformed(String), - /// No proper api key was used, or the signature was invalid - #[error(display = "Forbidden: {}", _0)] - Forbidden(String), - // Category: bad request /// The request contained an invalid UTF-8 sequence in its path or in other parameters #[error(display = "Invalid UTF-8: {}", _0)] @@ -39,16 +33,4 @@ where } } - -impl Error { - pub fn internal_error(msg: M) -> Self { - Self::CommonError(CommonError::InternalError(GarageError::Message( - msg.to_string(), - ))) - } - - pub fn bad_request(msg: M) -> Self { - Self::CommonError(CommonError::BadRequest(msg.to_string())) - } -} - +impl CommonErrorDerivative for Error {} diff --git a/src/api/signature/payload.rs b/src/api/signature/payload.rs index 155a6f94..4c7934e5 100644 --- a/src/api/signature/payload.rs +++ b/src/api/signature/payload.rs @@ -303,7 +303,7 @@ pub async fn verify_v4( .get(&EmptyKey, &key_id) .await? .filter(|k| !k.state.is_deleted()) - .ok_or_else(|| Error::Forbidden(format!("No such key: {}", &key_id)))?; + .ok_or_else(|| Error::forbidden(format!("No such key: {}", &key_id)))?; let key_p = key.params().unwrap(); let mut hmac = signing_hmac( @@ -316,7 +316,7 @@ pub async fn verify_v4( hmac.update(payload); let our_signature = hex::encode(hmac.finalize().into_bytes()); if signature != our_signature { - return Err(Error::Forbidden("Invalid signature".to_string())); + return Err(Error::forbidden("Invalid signature".to_string())); } Ok(key) diff --git a/src/model/helper/bucket.rs b/src/model/helper/bucket.rs index 2f1c6ae9..734cb40e 100644 --- a/src/model/helper/bucket.rs +++ b/src/model/helper/bucket.rs @@ -6,10 +6,10 @@ use garage_util::time::*; use crate::bucket_alias_table::*; use crate::bucket_table::*; -use crate::key_table::*; use crate::garage::Garage; use crate::helper::error::*; use crate::helper::key::KeyHelper; +use crate::key_table::*; use crate::permission::BucketKeyPerm; pub struct BucketHelper<'a>(pub(crate) &'a Garage); @@ -51,11 +51,7 @@ impl<'a> BucketHelper<'a> { } #[allow(clippy::ptr_arg)] - pub async fn resolve_bucket( - &self, - bucket_name: &String, - api_key: &Key, - ) -> Result { + pub async fn resolve_bucket(&self, bucket_name: &String, api_key: &Key) -> Result { let api_key_params = api_key .state .as_option() @@ -64,8 +60,8 @@ impl<'a> BucketHelper<'a> { if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) { Ok(*bucket_id) } else { - Ok(self. - resolve_global_bucket_name(bucket_name) + Ok(self + .resolve_global_bucket_name(bucket_name) .await? .ok_or_else(|| Error::NoSuchBucket(bucket_name.to_string()))?) } diff --git a/src/web/web_server.rs b/src/web/web_server.rs index dad98dfc..c30d8957 100644 --- a/src/web/web_server.rs +++ b/src/web/web_server.rs @@ -18,9 +18,11 @@ use opentelemetry::{ use crate::error::*; -use garage_api::s3::error::{Error as ApiError, OkOrBadRequest, OkOrInternalError}; use garage_api::helpers::{authority_to_host, host_to_bucket}; use garage_api::s3::cors::{add_cors_headers, find_matching_cors_rule, handle_options_for_bucket}; +use garage_api::s3::error::{ + CommonErrorDerivative, Error as ApiError, OkOrBadRequest, OkOrInternalError, +}; use garage_api::s3::get::{handle_get, handle_head}; use garage_model::garage::Garage; -- 2.43.4 From 5a535788fc0a69950bbfdc6f189597c5e37a6e3b Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 19:28:23 +0200 Subject: [PATCH 27/46] Json body for custom errors --- src/api/admin/error.rs | 30 +++++++++++++++++++++++++----- src/api/helpers.rs | 10 +++++++++- src/api/k2v/error.rs | 38 +++++++++++++++++++++++++++++++++----- 3 files changed, 67 insertions(+), 11 deletions(-) diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index bb35c16b..38dfe5b6 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -7,6 +7,7 @@ use garage_model::helper::error::Error as HelperError; use crate::common_error::CommonError; pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; use crate::generic_server::ApiError; +use crate::helpers::CustomApiErrorBody; /// Errors of this crate #[derive(Debug, Error)] @@ -44,6 +45,15 @@ impl From for Error { } } +impl Error { + fn code(&self) -> &'static str { + match self { + Error::CommonError(c) => c.aws_code(), + Error::NoSuchAccessKey => "NoSuchAccessKey", + } + } +} + impl ApiError for Error { /// Get the HTTP status code that best represents the meaning of the error for the client fn http_status_code(&self) -> StatusCode { @@ -58,10 +68,20 @@ impl ApiError for Error { } fn http_body(&self, garage_region: &str, path: &str) -> Body { - // TODO nice json error - Body::from(format!( - "ERROR: {}\n\ngarage region: {}\npath: {}", - self, garage_region, path - )) + let error = CustomApiErrorBody { + code: self.code().to_string(), + message: format!("{}", self), + path: path.to_string(), + region: garage_region.to_string(), + }; + Body::from(serde_json::to_string_pretty(&error).unwrap_or_else(|_| { + r#" +{ + "code": "InternalError", + "message": "JSON encoding of error failed" +} + "# + .into() + })) } } diff --git a/src/api/helpers.rs b/src/api/helpers.rs index aa350e3c..9fb12dbe 100644 --- a/src/api/helpers.rs +++ b/src/api/helpers.rs @@ -1,6 +1,6 @@ use hyper::{Body, Request}; use idna::domain_to_unicode; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use crate::common_error::{CommonError as Error, *}; @@ -279,3 +279,11 @@ mod tests { ); } } + +#[derive(Serialize)] +pub(crate) struct CustomApiErrorBody { + pub(crate) code: String, + pub(crate) message: String, + pub(crate) region: String, + pub(crate) path: String, +} diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index 4d8c1154..85d5de9d 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -7,6 +7,7 @@ use garage_model::helper::error::Error as HelperError; use crate::common_error::CommonError; pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; use crate::generic_server::ApiError; +use crate::helpers::CustomApiErrorBody; use crate::signature::error::Error as SignatureError; /// Errors of this crate @@ -78,6 +79,23 @@ impl From for Error { } } +impl Error { + /// This returns a keyword for the corresponding error. + /// Here, these keywords are not necessarily those from AWS S3, + /// as we are building a custom API + fn code(&self) -> &'static str { + match self { + Error::CommonError(c) => c.aws_code(), + Error::NoSuchKey => "NoSuchKey", + Error::NotAcceptable(_) => "NotAcceptable", + Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed", + Error::InvalidBase64(_) => "InvalidBase64", + Error::InvalidHeader(_) => "InvalidHeaderValue", + Error::InvalidUtf8Str(_) => "InvalidUtf8String", + } + } +} + impl ApiError for Error { /// Get the HTTP status code that best represents the meaning of the error for the client fn http_status_code(&self) -> StatusCode { @@ -97,10 +115,20 @@ impl ApiError for Error { } fn http_body(&self, garage_region: &str, path: &str) -> Body { - // TODO nice json error - Body::from(format!( - "ERROR: {}\n\ngarage region: {}\npath: {}", - self, garage_region, path - )) + let error = CustomApiErrorBody { + code: self.code().to_string(), + message: format!("{}", self), + path: path.to_string(), + region: garage_region.to_string(), + }; + Body::from(serde_json::to_string_pretty(&error).unwrap_or_else(|_| { + r#" +{ + "code": "InternalError", + "message": "JSON encoding of error failed" +} + "# + .into() + })) } } -- 2.43.4 From 8033bdb0b4577133cff7d4d90a811ed8f3e13365 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 19:36:17 +0200 Subject: [PATCH 28/46] More precisions in errors & small refactoring --- src/api/admin/bucket.rs | 9 ++++----- src/api/admin/error.rs | 16 +++++++++------- src/api/admin/key.rs | 25 ++++++------------------- src/api/common_error.rs | 16 ++++++++-------- src/api/k2v/error.rs | 6 ++++-- src/api/s3/error.rs | 6 ++++-- 6 files changed, 35 insertions(+), 43 deletions(-) diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index c5518e4e..00450319 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -248,11 +248,10 @@ pub async fn handle_create_bucket( } let key = garage - .key_table - .get(&EmptyKey, &la.access_key_id) - .await? - .ok_or(Error::NoSuchAccessKey)?; - let state = key.state.as_option().ok_or(Error::NoSuchAccessKey)?; + .key_helper() + .get_existing_key(&la.access_key_id) + .await?; + let state = key.state.as_option().unwrap(); if matches!(state.local_aliases.get(&la.alias), Some(_)) { return Err(Error::bad_request("Local alias already exists")); } diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 38dfe5b6..cd7e6af7 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -18,8 +18,8 @@ pub enum Error { // Category: cannot process /// The API access key does not exist - #[error(display = "Access key not found")] - NoSuchAccessKey, + #[error(display = "Access key not found: {}", _0)] + NoSuchAccessKey(String), } impl From for Error @@ -38,9 +38,11 @@ impl From for Error { match err { HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(_) => Self::CommonError(CommonError::InvalidBucketName), - HelperError::NoSuchBucket(_) => Self::CommonError(CommonError::NoSuchBucket), - HelperError::NoSuchAccessKey(_) => Self::NoSuchAccessKey, + HelperError::InvalidBucketName(n) => { + Self::CommonError(CommonError::InvalidBucketName(n)) + } + HelperError::NoSuchBucket(n) => Self::CommonError(CommonError::NoSuchBucket(n)), + HelperError::NoSuchAccessKey(n) => Self::NoSuchAccessKey(n), } } } @@ -49,7 +51,7 @@ impl Error { fn code(&self) -> &'static str { match self { Error::CommonError(c) => c.aws_code(), - Error::NoSuchAccessKey => "NoSuchAccessKey", + Error::NoSuchAccessKey(_) => "NoSuchAccessKey", } } } @@ -59,7 +61,7 @@ impl ApiError for Error { fn http_status_code(&self) -> StatusCode { match self { Error::CommonError(c) => c.http_status_code(), - Error::NoSuchAccessKey => StatusCode::NOT_FOUND, + Error::NoSuchAccessKey(_) => StatusCode::NOT_FOUND, } } diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 8060bf1a..1e910d52 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -50,17 +50,12 @@ pub async fn handle_get_key_info( search: Option, ) -> Result, Error> { let key = if let Some(id) = id { - garage - .key_table - .get(&EmptyKey, &id) - .await? - .ok_or(Error::NoSuchAccessKey)? + garage.key_helper().get_existing_key(&id).await? } else if let Some(search) = search { garage .key_helper() .get_existing_matching_key(&search) - .await - .map_err(|_| Error::NoSuchAccessKey)? + .await? } else { unreachable!(); }; @@ -92,13 +87,9 @@ pub async fn handle_update_key( ) -> Result, Error> { let req = parse_json_body::(req).await?; - let mut key = garage - .key_table - .get(&EmptyKey, &id) - .await? - .ok_or(Error::NoSuchAccessKey)?; + let mut key = garage.key_helper().get_existing_key(&id).await?; - let key_state = key.state.as_option_mut().ok_or(Error::NoSuchAccessKey)?; + let key_state = key.state.as_option_mut().unwrap(); if let Some(new_name) = req.name { key_state.name.update(new_name); @@ -127,13 +118,9 @@ struct UpdateKeyRequest { } pub async fn handle_delete_key(garage: &Arc, id: String) -> Result, Error> { - let mut key = garage - .key_table - .get(&EmptyKey, &id) - .await? - .ok_or(Error::NoSuchAccessKey)?; + let mut key = garage.key_helper().get_existing_key(&id).await?; - key.state.as_option().ok_or(Error::NoSuchAccessKey)?; + key.state.as_option().unwrap(); garage.key_helper().delete_key(&mut key).await?; diff --git a/src/api/common_error.rs b/src/api/common_error.rs index b6dbf059..20f9f266 100644 --- a/src/api/common_error.rs +++ b/src/api/common_error.rs @@ -32,8 +32,8 @@ pub enum CommonError { // These have to be error codes referenced in the S3 spec here: // https://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html#ErrorCodeList /// The bucket requested don't exists - #[error(display = "Bucket not found")] - NoSuchBucket, + #[error(display = "Bucket not found: {}", _0)] + NoSuchBucket(String), /// Tried to create a bucket that already exist #[error(display = "Bucket already exists")] @@ -45,8 +45,8 @@ pub enum CommonError { // Category: bad request /// Bucket name is not valid according to AWS S3 specs - #[error(display = "Invalid bucket name")] - InvalidBucketName, + #[error(display = "Invalid bucket name: {}", _0)] + InvalidBucketName(String), } impl CommonError { @@ -62,9 +62,9 @@ impl CommonError { } CommonError::BadRequest(_) => StatusCode::BAD_REQUEST, CommonError::Forbidden(_) => StatusCode::FORBIDDEN, - CommonError::NoSuchBucket => StatusCode::NOT_FOUND, + CommonError::NoSuchBucket(_) => StatusCode::NOT_FOUND, CommonError::BucketNotEmpty | CommonError::BucketAlreadyExists => StatusCode::CONFLICT, - CommonError::InvalidBucketName => StatusCode::BAD_REQUEST, + CommonError::InvalidBucketName(_) => StatusCode::BAD_REQUEST, } } @@ -80,10 +80,10 @@ impl CommonError { "InternalError" } CommonError::BadRequest(_) => "InvalidRequest", - CommonError::NoSuchBucket => "NoSuchBucket", + CommonError::NoSuchBucket(_) => "NoSuchBucket", CommonError::BucketAlreadyExists => "BucketAlreadyExists", CommonError::BucketNotEmpty => "BucketNotEmpty", - CommonError::InvalidBucketName => "InvalidBucketName", + CommonError::InvalidBucketName(_) => "InvalidBucketName", } } diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index 85d5de9d..dbd9a5f5 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -59,8 +59,10 @@ impl From for Error { match err { HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(_) => Self::CommonError(CommonError::InvalidBucketName), - HelperError::NoSuchBucket(_) => Self::CommonError(CommonError::NoSuchBucket), + HelperError::InvalidBucketName(n) => { + Self::CommonError(CommonError::InvalidBucketName(n)) + } + HelperError::NoSuchBucket(n) => Self::CommonError(CommonError::NoSuchBucket(n)), e => Self::CommonError(CommonError::BadRequest(format!("{}", e))), } } diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 4edff3a1..f2096bea 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -91,8 +91,10 @@ impl From for Error { match err { HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(_) => Self::CommonError(CommonError::InvalidBucketName), - HelperError::NoSuchBucket(_) => Self::CommonError(CommonError::NoSuchBucket), + HelperError::InvalidBucketName(n) => { + Self::CommonError(CommonError::InvalidBucketName(n)) + } + HelperError::NoSuchBucket(n) => Self::CommonError(CommonError::NoSuchBucket(n)), e => Self::bad_request(format!("{}", e)), } } -- 2.43.4 From d7736cb614564d5bf5d501d4cf473ea98889d239 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 19:43:40 +0200 Subject: [PATCH 29/46] Revert useless thing --- src/api/s3/api_server.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/api/s3/api_server.rs b/src/api/s3/api_server.rs index 87d0f288..ecc417ab 100644 --- a/src/api/s3/api_server.rs +++ b/src/api/s3/api_server.rs @@ -118,9 +118,7 @@ impl ApiHandler for S3ApiServer { return handle_post_object(garage, req, bucket_name.unwrap()).await; } if let Endpoint::Options = endpoint { - return handle_options_s3api(garage, &req, bucket_name) - .await - .map_err(Error::from); + return handle_options_s3api(garage, &req, bucket_name).await; } let (api_key, mut content_sha256) = check_payload_signature(&garage, "s3", &req).await?; -- 2.43.4 From ec50ffac42f90496495b054fad568a8553cffb64 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 13 May 2022 19:49:04 +0200 Subject: [PATCH 30/46] Remove useless string conversions --- src/api/k2v/router.rs | 4 ++-- src/api/s3/list.rs | 2 +- src/api/s3/post_object.rs | 22 +++++++++------------- src/api/s3/put.rs | 8 +++----- src/api/s3/router.rs | 2 +- src/api/s3/website.rs | 14 +++++++------- 6 files changed, 23 insertions(+), 29 deletions(-) diff --git a/src/api/k2v/router.rs b/src/api/k2v/router.rs index 093fb9a7..50e6965b 100644 --- a/src/api/k2v/router.rs +++ b/src/api/k2v/router.rs @@ -62,7 +62,7 @@ impl Endpoint { .unwrap_or((path.to_owned(), "")); if bucket.is_empty() { - return Err(Error::bad_request("Missing bucket name".to_owned())); + return Err(Error::bad_request("Missing bucket name")); } if *req.method() == Method::OPTIONS { @@ -83,7 +83,7 @@ impl Endpoint { Method::PUT => Self::from_put(partition_key, &mut query)?, Method::DELETE => Self::from_delete(partition_key, &mut query)?, _ if req.method() == method_search => Self::from_search(partition_key, &mut query)?, - _ => return Err(Error::bad_request("Unknown method".to_owned())), + _ => return Err(Error::bad_request("Unknown method")), }; if let Some(message) = query.nonempty_message() { diff --git a/src/api/s3/list.rs b/src/api/s3/list.rs index 12f6149d..e5f486c8 100644 --- a/src/api/s3/list.rs +++ b/src/api/s3/list.rs @@ -594,7 +594,7 @@ impl ListObjectsQuery { .ok_or_bad_request("Invalid continuation token")?, )?, }), - _ => Err(Error::bad_request("Invalid continuation token".to_string())), + _ => Err(Error::bad_request("Invalid continuation token")), }, // StartAfter has defined semantics in the spec: diff --git a/src/api/s3/post_object.rs b/src/api/s3/post_object.rs index 302ebe01..dc640f43 100644 --- a/src/api/s3/post_object.rs +++ b/src/api/s3/post_object.rs @@ -47,9 +47,7 @@ pub async fn handle_post_object( let field = if let Some(field) = multipart.next_field().await? { field } else { - return Err(Error::bad_request( - "Request did not contain a file".to_owned(), - )); + return Err(Error::bad_request("Request did not contain a file")); }; let name: HeaderName = if let Some(Ok(name)) = field.name().map(TryInto::try_into) { name @@ -66,7 +64,7 @@ pub async fn handle_post_object( "acl" => { if params.insert("x-amz-acl", content).is_some() { return Err(Error::bad_request( - "Field 'acl' provided more than one time".to_string(), + "Field 'acl' provided more than one time", )); } } @@ -143,9 +141,7 @@ pub async fn handle_post_object( .ok_or_bad_request("Invalid expiration date")? .into(); if Utc::now() - expiration > Duration::zero() { - return Err(Error::bad_request( - "Expiration date is in the paste".to_string(), - )); + return Err(Error::bad_request("Expiration date is in the paste")); } let mut conditions = decoded_policy.into_conditions()?; @@ -324,7 +320,7 @@ impl Policy { match condition { PolicyCondition::Equal(map) => { if map.len() != 1 { - return Err(Error::bad_request("Invalid policy item".to_owned())); + return Err(Error::bad_request("Invalid policy item")); } let (mut k, v) = map.into_iter().next().expect("size was verified"); k.make_ascii_lowercase(); @@ -332,7 +328,7 @@ impl Policy { } PolicyCondition::OtherOp([cond, mut key, value]) => { if key.remove(0) != '$' { - return Err(Error::bad_request("Invalid policy item".to_owned())); + return Err(Error::bad_request("Invalid policy item")); } key.make_ascii_lowercase(); match cond.as_str() { @@ -345,7 +341,7 @@ impl Policy { .or_default() .push(Operation::StartsWith(value)); } - _ => return Err(Error::bad_request("Invalid policy item".to_owned())), + _ => return Err(Error::bad_request("Invalid policy item")), } } PolicyCondition::SizeRange(key, min, max) => { @@ -353,7 +349,7 @@ impl Policy { length.0 = length.0.max(min); length.1 = length.1.min(max); } else { - return Err(Error::bad_request("Invalid policy item".to_owned())); + return Err(Error::bad_request("Invalid policy item")); } } } @@ -419,14 +415,14 @@ where // optimization to fail early when we know before the end it's too long if self.length.end() < &self.read { return Poll::Ready(Some(Err(Error::bad_request( - "File size does not match policy".to_owned(), + "File size does not match policy", )))); } } Poll::Ready(None) => { if !self.length.contains(&self.read) { return Poll::Ready(Some(Err(Error::bad_request( - "File size does not match policy".to_owned(), + "File size does not match policy", )))); } } diff --git a/src/api/s3/put.rs b/src/api/s3/put.rs index 660a8858..8b06ef3f 100644 --- a/src/api/s3/put.rs +++ b/src/api/s3/put.rs @@ -184,7 +184,7 @@ fn ensure_checksum_matches( if let Some(expected_sha256) = content_sha256 { if expected_sha256 != data_sha256sum { return Err(Error::bad_request( - "Unable to validate x-amz-content-sha256".to_string(), + "Unable to validate x-amz-content-sha256", )); } else { trace!("Successfully validated x-amz-content-sha256"); @@ -192,9 +192,7 @@ fn ensure_checksum_matches( } if let Some(expected_md5) = content_md5 { if expected_md5.trim_matches('"') != base64::encode(data_md5sum) { - return Err(Error::bad_request( - "Unable to validate content-md5".to_string(), - )); + return Err(Error::bad_request("Unable to validate content-md5")); } else { trace!("Successfully validated content-md5"); } @@ -513,7 +511,7 @@ pub async fn handle_complete_multipart_upload( let version = version.ok_or(Error::NoSuchKey)?; if version.blocks.is_empty() { - return Err(Error::bad_request("No data was uploaded".to_string())); + return Err(Error::bad_request("No data was uploaded")); } let headers = match object_version.state { diff --git a/src/api/s3/router.rs b/src/api/s3/router.rs index 0e769558..44f581ff 100644 --- a/src/api/s3/router.rs +++ b/src/api/s3/router.rs @@ -342,7 +342,7 @@ impl Endpoint { Method::POST => Self::from_post(key, &mut query)?, Method::PUT => Self::from_put(key, &mut query, req.headers())?, Method::DELETE => Self::from_delete(key, &mut query)?, - _ => return Err(Error::bad_request("Unknown method".to_owned())), + _ => return Err(Error::bad_request("Unknown method")), }; if let Some(message) = query.nonempty_message() { diff --git a/src/api/s3/website.rs b/src/api/s3/website.rs index 133c8327..77738971 100644 --- a/src/api/s3/website.rs +++ b/src/api/s3/website.rs @@ -170,7 +170,7 @@ impl WebsiteConfiguration { || self.routing_rules.is_some()) { return Err(Error::bad_request( - "Bad XML: can't have RedirectAllRequestsTo and other fields".to_owned(), + "Bad XML: can't have RedirectAllRequestsTo and other fields", )); } if let Some(ref ed) = self.error_document { @@ -216,7 +216,7 @@ impl Key { pub fn validate(&self) -> Result<(), Error> { if self.key.0.is_empty() { Err(Error::bad_request( - "Bad XML: error document specified but empty".to_owned(), + "Bad XML: error document specified but empty", )) } else { Ok(()) @@ -228,7 +228,7 @@ impl Suffix { pub fn validate(&self) -> Result<(), Error> { if self.suffix.0.is_empty() | self.suffix.0.contains('/') { Err(Error::bad_request( - "Bad XML: index document is empty or contains /".to_owned(), + "Bad XML: index document is empty or contains /", )) } else { Ok(()) @@ -240,7 +240,7 @@ impl Target { pub fn validate(&self) -> Result<(), Error> { if let Some(ref protocol) = self.protocol { if protocol.0 != "http" && protocol.0 != "https" { - return Err(Error::bad_request("Bad XML: invalid protocol".to_owned())); + return Err(Error::bad_request("Bad XML: invalid protocol")); } } Ok(()) @@ -263,18 +263,18 @@ impl Redirect { if self.replace_prefix.is_some() { if self.replace_full.is_some() { return Err(Error::bad_request( - "Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set".to_owned(), + "Bad XML: both ReplaceKeyPrefixWith and ReplaceKeyWith are set", )); } if !has_prefix { return Err(Error::bad_request( - "Bad XML: ReplaceKeyPrefixWith is set, but KeyPrefixEquals isn't".to_owned(), + "Bad XML: ReplaceKeyPrefixWith is set, but KeyPrefixEquals isn't", )); } } if let Some(ref protocol) = self.protocol { if protocol.0 != "http" && protocol.0 != "https" { - return Err(Error::bad_request("Bad XML: invalid protocol".to_owned())); + return Err(Error::bad_request("Bad XML: invalid protocol")); } } // TODO there are probably more invalide cases, but which ones? -- 2.43.4 From 8ff95f09c9edf218b6302470cd57868147316e59 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 17 May 2022 16:42:13 +0200 Subject: [PATCH 31/46] Return website config in GetBucketInfo, use serde(rename_all) --- src/api/admin/bucket.rs | 109 ++++++++++++++++++++++----------------- src/api/admin/cluster.rs | 4 +- src/api/admin/key.rs | 9 ++-- 3 files changed, 67 insertions(+), 55 deletions(-) diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 00450319..dacbd427 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -67,17 +67,16 @@ pub async fn handle_list_buckets(garage: &Arc) -> Result, } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] struct ListBucketResultItem { id: String, - #[serde(rename = "globalAliases")] global_aliases: Vec, - #[serde(rename = "localAliases")] local_aliases: Vec, } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] struct BucketLocalAlias { - #[serde(rename = "accessKeyId")] access_key_id: String, alias: String, } @@ -156,42 +155,50 @@ async fn bucket_info_results( let state = bucket.state.as_option().unwrap(); - let res = GetBucketInfoResult { - id: hex::encode(&bucket.id), - global_aliases: state - .aliases - .items() - .iter() - .filter(|(_, _, a)| *a) - .map(|(n, _, _)| n.to_string()) - .collect::>(), - keys: relevant_keys - .into_iter() - .map(|(_, key)| { - let p = key.state.as_option().unwrap(); - GetBucketInfoKey { - access_key_id: key.key_id, - name: p.name.get().to_string(), - permissions: p - .authorized_buckets - .get(&bucket.id) - .map(|p| ApiBucketKeyPerm { - read: p.allow_read, - write: p.allow_write, - owner: p.allow_owner, - }) - .unwrap_or_default(), - bucket_local_aliases: p - .local_aliases - .items() - .iter() - .filter(|(_, _, b)| *b == Some(bucket.id)) - .map(|(n, _, _)| n.to_string()) - .collect::>(), + let res = + GetBucketInfoResult { + id: hex::encode(&bucket.id), + global_aliases: state + .aliases + .items() + .iter() + .filter(|(_, _, a)| *a) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + website_access: state.website_config.get().is_some(), + website_config: state.website_config.get().clone().map(|wsc| { + GetBucketInfoWebsiteResult { + index_document: wsc.index_document, + error_document: wsc.error_document, } - }) - .collect::>(), - }; + }), + keys: relevant_keys + .into_iter() + .map(|(_, key)| { + let p = key.state.as_option().unwrap(); + GetBucketInfoKey { + access_key_id: key.key_id, + name: p.name.get().to_string(), + permissions: p + .authorized_buckets + .get(&bucket.id) + .map(|p| ApiBucketKeyPerm { + read: p.allow_read, + write: p.allow_write, + owner: p.allow_owner, + }) + .unwrap_or_default(), + bucket_local_aliases: p + .local_aliases + .items() + .iter() + .filter(|(_, _, b)| *b == Some(bucket.id)) + .map(|(n, _, _)| n.to_string()) + .collect::>(), + } + }) + .collect::>(), + }; let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; Ok(Response::builder() @@ -200,21 +207,29 @@ async fn bucket_info_results( } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] struct GetBucketInfoResult { id: String, - #[serde(rename = "globalAliases")] global_aliases: Vec, + website_access: bool, + #[serde(default)] + website_config: Option, keys: Vec, } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct GetBucketInfoWebsiteResult { + index_document: String, + error_document: Option, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] struct GetBucketInfoKey { - #[serde(rename = "accessKeyId")] access_key_id: String, - #[serde(rename = "name")] name: String, permissions: ApiBucketKeyPerm, - #[serde(rename = "bucketLocalAliases")] bucket_local_aliases: Vec, } @@ -293,19 +308,18 @@ pub async fn handle_create_bucket( } #[derive(Deserialize)] +#[serde(rename_all = "camelCase")] struct CreateBucketRequest { - #[serde(rename = "globalAlias")] global_alias: Option, - #[serde(rename = "localAlias")] local_alias: Option, } #[derive(Deserialize)] +#[serde(rename_all = "camelCase")] struct CreateBucketLocalAlias { - #[serde(rename = "accessKeyId")] access_key_id: String, alias: String, - #[serde(rename = "allPermissions", default)] + #[serde(default)] all_permissions: bool, } @@ -420,10 +434,9 @@ pub async fn handle_bucket_change_key_perm( } #[derive(Deserialize)] +#[serde(rename_all = "camelCase")] struct BucketKeyPermChangeRequest { - #[serde(rename = "bucketId")] bucket_id: String, - #[serde(rename = "accessKeyId")] access_key_id: String, permissions: ApiBucketKeyPerm, } diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 91d99d8a..44ad4a37 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -76,19 +76,19 @@ fn get_cluster_layout(garage: &Arc) -> GetClusterLayoutResponse { } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] struct GetClusterStatusResponse { node: String, garage_version: &'static str, - #[serde(rename = "knownNodes")] known_nodes: HashMap, layout: GetClusterLayoutResponse, } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] struct GetClusterLayoutResponse { version: u64, roles: HashMap>, - #[serde(rename = "stagedRoleChanges")] staged_role_changes: HashMap>, } diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index 1e910d52..be37088b 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -203,28 +203,27 @@ async fn key_info_results(garage: &Arc, key: Key) -> Result, } #[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] struct KeyPerm { - #[serde(rename = "createBucket", default)] + #[serde(default)] create_bucket: bool, } #[derive(Serialize)] +#[serde(rename_all = "camelCase")] struct KeyInfoBucketResult { id: String, - #[serde(rename = "globalAliases")] global_aliases: Vec, - #[serde(rename = "localAliases")] local_aliases: Vec, permissions: ApiBucketKeyPerm, } -- 2.43.4 From 8b1338ef2fcd36214c3ad90f5d3585377f28ec86 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 17 May 2022 16:48:23 +0200 Subject: [PATCH 32/46] Fix error code --- src/api/admin/bucket.rs | 2 +- src/api/admin/error.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index dacbd427..2c21edee 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -95,7 +95,7 @@ pub async fn handle_get_bucket_info( .bucket_helper() .resolve_global_bucket_name(&ga) .await? - .ok_or_bad_request("Bucket not found")?, + .ok_or_else(|| HelperError::NoSuchBucket(ga.to_string()))?, _ => { return Err(Error::bad_request( "Either id or globalAlias must be provided (but not both)", diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index cd7e6af7..592440a5 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -2,7 +2,7 @@ use err_derive::Error; use hyper::header::HeaderValue; use hyper::{Body, HeaderMap, StatusCode}; -use garage_model::helper::error::Error as HelperError; +pub use garage_model::helper::error::Error as HelperError; use crate::common_error::CommonError; pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInternalError}; -- 2.43.4 From e92c52eb6522a140cdced40bc047149dc638bfa4 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 17 May 2022 17:02:38 +0200 Subject: [PATCH 33/46] refactor --- src/api/admin/bucket.rs | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 2c21edee..2124f2c2 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -103,18 +103,18 @@ pub async fn handle_get_bucket_info( } }; + bucket_info_results(garage, bucket_id).await +} + +async fn bucket_info_results( + garage: &Arc, + bucket_id: Uuid, +) -> Result, Error> { let bucket = garage .bucket_helper() .get_existing_bucket(bucket_id) .await?; - bucket_info_results(garage, bucket).await -} - -async fn bucket_info_results( - garage: &Arc, - bucket: Bucket, -) -> Result, Error> { let mut relevant_keys = HashMap::new(); for (k, _) in bucket .state @@ -299,12 +299,7 @@ pub async fn handle_create_bucket( } } - let bucket = garage - .bucket_table - .get(&EmptyKey, &bucket.id) - .await? - .ok_or_internal_error("Bucket should now exist but doesn't")?; - bucket_info_results(garage, bucket).await + bucket_info_results(garage, bucket.id).await } #[derive(Deserialize)] @@ -425,12 +420,7 @@ pub async fn handle_bucket_change_key_perm( .set_bucket_key_permissions(bucket.id, &key.key_id, perm) .await?; - let bucket = garage - .bucket_table - .get(&EmptyKey, &bucket.id) - .await? - .ok_or_internal_error("Bucket should now exist but doesn't")?; - bucket_info_results(garage, bucket).await + bucket_info_results(garage, bucket.id).await } #[derive(Deserialize)] -- 2.43.4 From 2ce3513c108a53bdcc5a838704867a4499295d85 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 17 May 2022 17:16:29 +0200 Subject: [PATCH 34/46] Specify and implement {Global,Local}{Alias,Unalias}Bucket --- doc/drafts/admin-api.md | 19 +++++++++ src/api/admin/api_server.rs | 17 ++++++++ src/api/admin/bucket.rs | 78 +++++++++++++++++++++++++++++++++---- src/api/admin/router.rs | 28 ++++++++++++- 4 files changed, 133 insertions(+), 9 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 5dc3f127..14c4ec39 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -501,3 +501,22 @@ Request body format: Flags in `permissions` which have the value `true` will be deactivated. Other flags will remain unchanged. + +## Operations on bucket aliases + +### GlobalAliasBucket `PUT /bucket/alias/global?id=&alias=` + +Empty body. Creates a global alias for a bucket. + +### GlobalUnaliasBucket `DELETE /bucket/alias/global?id=&alias=` + +Removes a global alias for a bucket. + +### LocalAliasBucket `PUT /bucket/alias/local?id=&accessKeyId=&alias=` + +Empty body. Creates a local alias for a bucket in the namespace of a specific access key. + +### LocalUnaliasBucket `DELETE /bucket/alias/local?id=&accessKeyId&alias=` + +Removes a local alias for a bucket in the namespace of a specific access key. + diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 098a54aa..a51d66e5 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -151,6 +151,23 @@ impl ApiHandler for AdminApiServer { Endpoint::BucketDenyKey => { handle_bucket_change_key_perm(&self.garage, req, false).await } + // Bucket aliasing + Endpoint::GlobalAliasBucket { id, alias } => { + handle_global_alias_bucket(&self.garage, id, alias).await + } + Endpoint::GlobalUnaliasBucket { id, alias } => { + handle_global_unalias_bucket(&self.garage, id, alias).await + } + Endpoint::LocalAliasBucket { + id, + access_key_id, + alias, + } => handle_local_alias_bucket(&self.garage, id, access_key_id, alias).await, + Endpoint::LocalUnaliasBucket { + id, + access_key_id, + alias, + } => handle_local_unalias_bucket(&self.garage, id, access_key_id, alias).await, } } } diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 2124f2c2..cc37089a 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -87,10 +87,7 @@ pub async fn handle_get_bucket_info( global_alias: Option, ) -> Result, Error> { let bucket_id = match (id, global_alias) { - (Some(id), None) => { - let id_hex = hex::decode(&id).ok_or_bad_request("Invalid bucket id")?; - Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")? - } + (Some(id), None) => parse_bucket_id(&id)?, (None, Some(ga)) => garage .bucket_helper() .resolve_global_bucket_name(&ga) @@ -324,8 +321,7 @@ pub async fn handle_delete_bucket( ) -> Result, Error> { let helper = garage.bucket_helper(); - let id_hex = hex::decode(&id).ok_or_bad_request("Invalid bucket id")?; - let bucket_id = Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")?; + let bucket_id = parse_bucket_id(&id)?; let mut bucket = helper.get_existing_bucket(bucket_id).await?; let state = bucket.state.as_option().unwrap(); @@ -385,8 +381,7 @@ pub async fn handle_bucket_change_key_perm( ) -> Result, Error> { let req = parse_json_body::(req).await?; - let id_hex = hex::decode(&req.bucket_id).ok_or_bad_request("Invalid bucket id")?; - let bucket_id = Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")?; + let bucket_id = parse_bucket_id(&req.bucket_id)?; let bucket = garage .bucket_helper() @@ -430,3 +425,70 @@ struct BucketKeyPermChangeRequest { access_key_id: String, permissions: ApiBucketKeyPerm, } + +pub async fn handle_global_alias_bucket( + garage: &Arc, + bucket_id: String, + alias: String, +) -> Result, Error> { + let bucket_id = parse_bucket_id(&bucket_id)?; + + garage + .bucket_helper() + .set_global_bucket_alias(bucket_id, &alias) + .await?; + + bucket_info_results(garage, bucket_id).await +} + +pub async fn handle_global_unalias_bucket( + garage: &Arc, + bucket_id: String, + alias: String, +) -> Result, Error> { + let bucket_id = parse_bucket_id(&bucket_id)?; + + garage + .bucket_helper() + .unset_global_bucket_alias(bucket_id, &alias) + .await?; + + bucket_info_results(garage, bucket_id).await +} + +pub async fn handle_local_alias_bucket( + garage: &Arc, + bucket_id: String, + access_key_id: String, + alias: String, +) -> Result, Error> { + let bucket_id = parse_bucket_id(&bucket_id)?; + + garage + .bucket_helper() + .set_local_bucket_alias(bucket_id, &access_key_id, &alias) + .await?; + + bucket_info_results(garage, bucket_id).await +} + +pub async fn handle_local_unalias_bucket( + garage: &Arc, + bucket_id: String, + access_key_id: String, + alias: String, +) -> Result, Error> { + let bucket_id = parse_bucket_id(&bucket_id)?; + + garage + .bucket_helper() + .unset_local_bucket_alias(bucket_id, &access_key_id, &alias) + .await?; + + bucket_info_results(garage, bucket_id).await +} + +fn parse_bucket_id(id: &str) -> Result { + let id_hex = hex::decode(&id).ok_or_bad_request("Invalid bucket id")?; + Ok(Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")?) +} diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index 2a5098bf..6961becb 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -49,6 +49,25 @@ pub enum Endpoint { // Bucket-Key Permissions BucketAllowKey, BucketDenyKey, + // Bucket aliases + GlobalAliasBucket { + id: String, + alias: String, + }, + GlobalUnaliasBucket { + id: String, + alias: String, + }, + LocalAliasBucket { + id: String, + access_key_id: String, + alias: String, + }, + LocalUnaliasBucket { + id: String, + access_key_id: String, + alias: String, + }, }} impl Endpoint { @@ -87,6 +106,11 @@ impl Endpoint { // Bucket-key permissions POST "/bucket/allow" => BucketAllowKey, POST "/bucket/deny" => BucketDenyKey, + // Bucket aliases + PUT "/bucket/alias/global" => GlobalAliasBucket (query::id, query::alias), + DELETE "/bucket/alias/global" => GlobalUnaliasBucket (query::id, query::alias), + PUT "/bucket/alias/local" => LocalAliasBucket (query::id, query::access_key_id, query::alias), + DELETE "/bucket/alias/local" => LocalUnaliasBucket (query::id, query::access_key_id, query::alias), ]); if let Some(message) = query.nonempty_message() { @@ -107,5 +131,7 @@ impl Endpoint { generateQueryParameters! { "id" => id, "search" => search, - "globalAlias" => global_alias + "globalAlias" => global_alias, + "alias" => alias, + "accessKeyId" => access_key_id } -- 2.43.4 From 5072dbd2282736b3254627c26cfcf897505330e6 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 17 May 2022 17:44:00 +0200 Subject: [PATCH 35/46] Add PutBucketWebsite and DeleteBucketWebsite to admin api --- doc/drafts/admin-api.md | 21 +++++++++++++ src/api/admin/api_server.rs | 6 ++++ src/api/admin/bucket.rs | 59 +++++++++++++++++++++++++++++++++++++ src/api/admin/router.rs | 8 +++++ 4 files changed, 94 insertions(+) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 14c4ec39..e8ed087d 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -457,6 +457,26 @@ Deletes a storage bucket. A bucket cannot be deleted if it is not empty. Warning: this will delete all aliases associated with the bucket! +### PutBucketWebsite `PUT /bucket/website?id=` + +Sets the website configuration for a bucket (this also enables website access for this bucket). + +Request body format: + +```json +{ + "indexDocument": "index.html", + "errorDocument": "404.html", +} +``` + +The field `errorDocument` is optional, if no error document is set a generic error message is displayed when errors happen. + + +### DeleteBucketWebsite `DELETE /bucket/website?id=` + +Deletes the website configuration for a bucket (disables website access for this bucket). + ## Operations on permissions for keys on buckets @@ -502,6 +522,7 @@ Request body format: Flags in `permissions` which have the value `true` will be deactivated. Other flags will remain unchanged. + ## Operations on bucket aliases ### GlobalAliasBucket `PUT /bucket/alias/global?id=&alias=` diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index a51d66e5..6f568024 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -144,6 +144,12 @@ impl ApiHandler for AdminApiServer { } Endpoint::CreateBucket => handle_create_bucket(&self.garage, req).await, Endpoint::DeleteBucket { id } => handle_delete_bucket(&self.garage, id).await, + Endpoint::PutBucketWebsite { id } => { + handle_put_bucket_website(&self.garage, id, req).await + } + Endpoint::DeleteBucketWebsite { id } => { + handle_delete_bucket_website(&self.garage, id).await + } // Bucket-key permissions Endpoint::BucketAllowKey => { handle_bucket_change_key_perm(&self.garage, req, true).await diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index cc37089a..3ad2c735 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -374,6 +374,61 @@ pub async fn handle_delete_bucket( .body(Body::empty())?) } +// ---- BUCKET WEBSITE CONFIGURATION ---- + +pub async fn handle_put_bucket_website( + garage: &Arc, + id: String, + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; + let bucket_id = parse_bucket_id(&id)?; + + let mut bucket = garage + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; + + let state = bucket.state.as_option_mut().unwrap(); + state.website_config.update(Some(WebsiteConfig { + index_document: req.index_document, + error_document: req.error_document, + })); + + garage.bucket_table.insert(&bucket).await?; + + bucket_info_results(garage, bucket_id).await +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct PutBucketWebsiteRequest { + index_document: String, + #[serde(default)] + error_document: Option, +} + +pub async fn handle_delete_bucket_website( + garage: &Arc, + id: String, +) -> Result, Error> { + let bucket_id = parse_bucket_id(&id)?; + + let mut bucket = garage + .bucket_helper() + .get_existing_bucket(bucket_id) + .await?; + + let state = bucket.state.as_option_mut().unwrap(); + state.website_config.update(None); + + garage.bucket_table.insert(&bucket).await?; + + bucket_info_results(garage, bucket_id).await +} + +// ---- BUCKET/KEY PERMISSIONS ---- + pub async fn handle_bucket_change_key_perm( garage: &Arc, req: Request, @@ -426,6 +481,8 @@ struct BucketKeyPermChangeRequest { permissions: ApiBucketKeyPerm, } +// ---- BUCKET ALIASES ---- + pub async fn handle_global_alias_bucket( garage: &Arc, bucket_id: String, @@ -488,6 +545,8 @@ pub async fn handle_local_unalias_bucket( bucket_info_results(garage, bucket_id).await } +// ---- HELPER ---- + fn parse_bucket_id(id: &str) -> Result { let id_hex = hex::decode(&id).ok_or_bad_request("Invalid bucket id")?; Ok(Uuid::try_from(&id_hex).ok_or_bad_request("Invalid bucket id")?) diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index 6961becb..ae9e6681 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -46,6 +46,12 @@ pub enum Endpoint { DeleteBucket { id: String, }, + PutBucketWebsite { + id: String, + }, + DeleteBucketWebsite { + id: String, + }, // Bucket-Key Permissions BucketAllowKey, BucketDenyKey, @@ -103,6 +109,8 @@ impl Endpoint { GET "/bucket" => ListBuckets, POST "/bucket" => CreateBucket, DELETE "/bucket" if id => DeleteBucket (query::id), + PUT "/bucket/website" if id => PutBucketWebsite (query::id), + DELETE "/bucket/website" if id => DeleteBucketWebsite (query::id), // Bucket-key permissions POST "/bucket/allow" => BucketAllowKey, POST "/bucket/deny" => BucketDenyKey, -- 2.43.4 From 440a577563c313f7ce6fe928ff9a18ac1ad0deb3 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 17 May 2022 17:51:38 +0200 Subject: [PATCH 36/46] Prefix all APIs with `v0/` --- doc/drafts/admin-api.md | 48 ++++++++++++++++++++--------------------- src/api/admin/router.rs | 48 ++++++++++++++++++++--------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index e8ed087d..cbd73e15 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -15,7 +15,7 @@ Returns internal Garage metrics in Prometheus format. ## Cluster operations -### GetClusterStatus `GET /status` +### GetClusterStatus `GET /v0/status` Returns the cluster's current status in JSON, including: @@ -94,7 +94,7 @@ Example response body: } ``` -### GetClusterLayout `GET /layout` +### GetClusterLayout `GET /v0/layout` Returns the cluster's current layout in JSON, including: @@ -143,7 +143,7 @@ Example response body: } ``` -### UpdateClusterLayout `POST /layout` +### UpdateClusterLayout `POST /v0/layout` Send modifications to the cluster layout. These modifications will be included in the staged role changes, visible in subsequent calls @@ -174,7 +174,7 @@ Contrary to the CLI that may update only a subset of the fields values must be specified. -### ApplyClusterLayout `POST /layout/apply` +### ApplyClusterLayout `POST /v0/layout/apply` Applies to the cluster the layout changes currently registered as staged layout changes. @@ -191,7 +191,7 @@ Similarly to the CLI, the body must include the version of the new layout that will be created, which MUST be 1 + the value of the currently existing layout in the cluster. -### RevertClusterLayout `POST /layout/revert` +### RevertClusterLayout `POST /v0/layout/revert` Clears all of the staged layout changes. @@ -212,7 +212,7 @@ existing layout in the cluster. ## Access key operations -### ListKeys `GET /key` +### ListKeys `GET /v0/key` Returns all API access keys in the cluster. @@ -231,7 +231,7 @@ Example response: ] ``` -### CreateKey `POST /key` +### CreateKey `POST /v0/key` Creates a new API access key. @@ -243,8 +243,8 @@ Request body format: } ``` -### GetKeyInfo `GET /key?id=` -### GetKeyInfo `GET /key?search=` +### GetKeyInfo `GET /v0/key?id=` +### GetKeyInfo `GET /v0/key?search=` Returns information about the requested API access key. @@ -315,11 +315,11 @@ Example response: } ``` -### DeleteKey `DELETE /key?id=` +### DeleteKey `DELETE /v0/key?id=` Deletes an API access key. -### UpdateKey `POST /key?id=` +### UpdateKey `POST /v0/key?id=` Updates information about the specified API access key. @@ -342,7 +342,7 @@ The possible flags in `allow` and `deny` are: `createBucket`. ## Bucket operations -### ListBuckets `GET /bucket` +### ListBuckets `GET /v0/bucket` Returns all storage buckets in the cluster. @@ -384,8 +384,8 @@ Example response: ] ``` -### GetBucketInfo `GET /bucket?id=` -### GetBucketInfo `GET /bucket?globalAlias=` +### GetBucketInfo `GET /v0/bucket?id=` +### GetBucketInfo `GET /v0/bucket?globalAlias=` Returns information about the requested storage bucket. @@ -418,7 +418,7 @@ Example response: } ``` -### CreateBucket `POST /bucket` +### CreateBucket `POST /v0/bucket` Creates a new storage bucket. @@ -451,13 +451,13 @@ OR Creates a new bucket, either with a global alias, a local one, or no alias at all. -### DeleteBucket `DELETE /bucket?id=` +### DeleteBucket `DELETE /v0/bucket?id=` Deletes a storage bucket. A bucket cannot be deleted if it is not empty. Warning: this will delete all aliases associated with the bucket! -### PutBucketWebsite `PUT /bucket/website?id=` +### PutBucketWebsite `PUT /v0/bucket/website?id=` Sets the website configuration for a bucket (this also enables website access for this bucket). @@ -473,14 +473,14 @@ Request body format: The field `errorDocument` is optional, if no error document is set a generic error message is displayed when errors happen. -### DeleteBucketWebsite `DELETE /bucket/website?id=` +### DeleteBucketWebsite `DELETE /v0/bucket/website?id=` Deletes the website configuration for a bucket (disables website access for this bucket). ## Operations on permissions for keys on buckets -### BucketAllowKey `POST /bucket/allow` +### BucketAllowKey `POST /v0/bucket/allow` Allows a key to do read/write/owner operations on a bucket. @@ -501,7 +501,7 @@ Request body format: Flags in `permissions` which have the value `true` will be activated. Other flags will remain unchanged. -### BucketDenyKey `POST /bucket/deny` +### BucketDenyKey `POST /v0/bucket/deny` Denies a key from doing read/write/owner operations on a bucket. @@ -525,19 +525,19 @@ Other flags will remain unchanged. ## Operations on bucket aliases -### GlobalAliasBucket `PUT /bucket/alias/global?id=&alias=` +### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=&alias=` Empty body. Creates a global alias for a bucket. -### GlobalUnaliasBucket `DELETE /bucket/alias/global?id=&alias=` +### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=&alias=` Removes a global alias for a bucket. -### LocalAliasBucket `PUT /bucket/alias/local?id=&accessKeyId=&alias=` +### LocalAliasBucket `PUT /v0/bucket/alias/local?id=&accessKeyId=&alias=` Empty body. Creates a local alias for a bucket in the namespace of a specific access key. -### LocalUnaliasBucket `DELETE /bucket/alias/local?id=&accessKeyId&alias=` +### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=&accessKeyId&alias=` Removes a local alias for a bucket in the namespace of a specific access key. diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index ae9e6681..909ef102 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -90,35 +90,35 @@ impl Endpoint { let res = router_match!(@gen_path_parser (req.method(), path, query) [ OPTIONS _ => Options, GET "/metrics" => Metrics, - GET "/status" => GetClusterStatus, + GET "/v0/status" => GetClusterStatus, // Layout endpoints - GET "/layout" => GetClusterLayout, - POST "/layout" => UpdateClusterLayout, - POST "/layout/apply" => ApplyClusterLayout, - POST "/layout/revert" => RevertClusterLayout, + GET "/v0/layout" => GetClusterLayout, + POST "/v0/layout" => UpdateClusterLayout, + POST "/v0/layout/apply" => ApplyClusterLayout, + POST "/v0/layout/revert" => RevertClusterLayout, // API key endpoints - GET "/key" if id => GetKeyInfo (query_opt::id, query_opt::search), - GET "/key" if search => GetKeyInfo (query_opt::id, query_opt::search), - POST "/key" if id => UpdateKey (query::id), - POST "/key" => CreateKey, - DELETE "/key" if id => DeleteKey (query::id), - GET "/key" => ListKeys, + GET "/v0/key" if id => GetKeyInfo (query_opt::id, query_opt::search), + GET "/v0/key" if search => GetKeyInfo (query_opt::id, query_opt::search), + POST "/v0/key" if id => UpdateKey (query::id), + POST "/v0/key" => CreateKey, + DELETE "/v0/key" if id => DeleteKey (query::id), + GET "/v0/key" => ListKeys, // Bucket endpoints - GET "/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias), - GET "/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias), - GET "/bucket" => ListBuckets, - POST "/bucket" => CreateBucket, - DELETE "/bucket" if id => DeleteBucket (query::id), - PUT "/bucket/website" if id => PutBucketWebsite (query::id), - DELETE "/bucket/website" if id => DeleteBucketWebsite (query::id), + GET "/v0/bucket" if id => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/v0/bucket" if global_alias => GetBucketInfo (query_opt::id, query_opt::global_alias), + GET "/v0/bucket" => ListBuckets, + POST "/v0/bucket" => CreateBucket, + DELETE "/v0/bucket" if id => DeleteBucket (query::id), + PUT "/v0/bucket/website" if id => PutBucketWebsite (query::id), + DELETE "/v0/bucket/website" if id => DeleteBucketWebsite (query::id), // Bucket-key permissions - POST "/bucket/allow" => BucketAllowKey, - POST "/bucket/deny" => BucketDenyKey, + POST "/v0/bucket/allow" => BucketAllowKey, + POST "/v0/bucket/deny" => BucketDenyKey, // Bucket aliases - PUT "/bucket/alias/global" => GlobalAliasBucket (query::id, query::alias), - DELETE "/bucket/alias/global" => GlobalUnaliasBucket (query::id, query::alias), - PUT "/bucket/alias/local" => LocalAliasBucket (query::id, query::access_key_id, query::alias), - DELETE "/bucket/alias/local" => LocalUnaliasBucket (query::id, query::access_key_id, query::alias), + PUT "/v0/bucket/alias/global" => GlobalAliasBucket (query::id, query::alias), + DELETE "/v0/bucket/alias/global" => GlobalUnaliasBucket (query::id, query::alias), + PUT "/v0/bucket/alias/local" => LocalAliasBucket (query::id, query::access_key_id, query::alias), + DELETE "/v0/bucket/alias/local" => LocalUnaliasBucket (query::id, query::access_key_id, query::alias), ]); if let Some(message) = query.nonempty_message() { -- 2.43.4 From 70383b4363c9d9d1ddebcb29c60f833022e97d24 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 17 May 2022 18:43:47 +0200 Subject: [PATCH 37/46] Implement ConnectClusterNodes --- doc/drafts/admin-api.md | 30 ++++++++++++++++++++++++++++++ src/api/admin/api_server.rs | 1 + src/api/admin/cluster.rs | 33 +++++++++++++++++++++++++++++++++ src/api/admin/router.rs | 2 ++ src/rpc/system.rs | 12 ++++++++---- 5 files changed, 74 insertions(+), 4 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index cbd73e15..2e1ffa82 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -94,6 +94,36 @@ Example response body: } ``` +### ConnectClusterNodes `POST /v0/connect` + +Instructs this Garage node to connect to other Garage nodes at specified addresses. + +Example request body: + +```json +[ + "ec79480e0ce52ae26fd00c9da684e4fa56658d9c64cdcecb094e936de0bfe71f@10.0.0.11:3901", + "4a6ae5a1d0d33bf895f5bb4f0a418b7dc94c47c0dd2eb108d1158f3c8f60b0ff@10.0.0.12:3901" +] +``` + +The format of the string for a node to connect to is: `@:`, same as in the `garage node connect` CLI call. + +Example response: + +```json +[ + { + "success": true, + "error": null, + }, + { + "success": false, + "error": "Handshake error", + } +] +``` + ### GetClusterLayout `GET /v0/layout` Returns the cluster's current layout in JSON, including: diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 6f568024..61b0d24f 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -124,6 +124,7 @@ impl ApiHandler for AdminApiServer { Endpoint::Options => self.handle_options(&req), Endpoint::Metrics => self.handle_metrics(), Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await, + Endpoint::ConnectClusterNodes => handle_connect_cluster_nodes(&self.garage, req).await, // Layout Endpoint::GetClusterLayout => handle_get_cluster_layout(&self.garage).await, Endpoint::UpdateClusterLayout => handle_update_cluster_layout(&self.garage, req).await, diff --git a/src/api/admin/cluster.rs b/src/api/admin/cluster.rs index 44ad4a37..3401be42 100644 --- a/src/api/admin/cluster.rs +++ b/src/api/admin/cluster.rs @@ -45,6 +45,33 @@ pub async fn handle_get_cluster_status(garage: &Arc) -> Result, + req: Request, +) -> Result, Error> { + let req = parse_json_body::>(req).await?; + + let res = futures::future::join_all(req.iter().map(|node| garage.system.connect(node))) + .await + .into_iter() + .map(|r| match r { + Ok(()) => ConnectClusterNodesResponse { + success: true, + error: None, + }, + Err(e) => ConnectClusterNodesResponse { + success: false, + error: Some(format!("{}", e)), + }, + }) + .collect::>(); + + let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; + Ok(Response::builder() + .status(StatusCode::OK) + .body(Body::from(resp_json))?) +} + pub async fn handle_get_cluster_layout(garage: &Arc) -> Result, Error> { let res = get_cluster_layout(garage); let resp_json = serde_json::to_string_pretty(&res).map_err(GarageError::from)?; @@ -84,6 +111,12 @@ struct GetClusterStatusResponse { layout: GetClusterLayoutResponse, } +#[derive(Serialize)] +struct ConnectClusterNodesResponse { + success: bool, + error: Option, +} + #[derive(Serialize)] #[serde(rename_all = "camelCase")] struct GetClusterLayoutResponse { diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index 909ef102..41e7ed73 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -18,6 +18,7 @@ pub enum Endpoint { Options, Metrics, GetClusterStatus, + ConnectClusterNodes, // Layout GetClusterLayout, UpdateClusterLayout, @@ -91,6 +92,7 @@ impl Endpoint { OPTIONS _ => Options, GET "/metrics" => Metrics, GET "/v0/status" => GetClusterStatus, + POST "/v0/connect" => ConnectClusterNodes, // Layout endpoints GET "/v0/layout" => GetClusterLayout, POST "/v0/layout" => UpdateClusterLayout, diff --git a/src/rpc/system.rs b/src/rpc/system.rs index eb2f2e42..78d538ec 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -383,10 +383,14 @@ impl System { } } } - return Err(Error::Message(format!( - "Could not connect to specified peers. Errors: {:?}", - errors - ))); + if errors.len() == 1 { + return Err(Error::Message(errors[0].1.to_string())); + } else { + return Err(Error::Message(format!( + "Could not connect to specified peers. Errors: {:?}", + errors + ))); + } } // ---- INTERNALS ---- -- 2.43.4 From dcfa408887b588e9f9eee608c1358a5a81330e9e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 17 May 2022 19:02:13 +0200 Subject: [PATCH 38/46] Implement ImportKey --- doc/drafts/admin-api.md | 14 ++++++++++++++ src/api/admin/api_server.rs | 1 + src/api/admin/error.rs | 9 +++++++++ src/api/admin/key.rs | 25 +++++++++++++++++++++++++ src/api/admin/router.rs | 2 ++ src/rpc/system.rs | 7 ++----- 6 files changed, 53 insertions(+), 5 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 2e1ffa82..d6b8ed5d 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -273,6 +273,20 @@ Request body format: } ``` +### ImportKey `POST /v0/key/import` + +Imports an existing API key. + +Request body format: + +```json +{ + "accessKeyId": "GK31c2f218a2e44f485b94239e", + "secretAccessKey": "b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835", + "name": "NameOfMyKey" +} +``` + ### GetKeyInfo `GET /v0/key?id=` ### GetKeyInfo `GET /v0/key?search=` diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index 61b0d24f..a0af9bd9 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -136,6 +136,7 @@ impl ApiHandler for AdminApiServer { handle_get_key_info(&self.garage, id, search).await } Endpoint::CreateKey => handle_create_key(&self.garage, req).await, + Endpoint::ImportKey => handle_import_key(&self.garage, req).await, Endpoint::UpdateKey { id } => handle_update_key(&self.garage, id, req).await, Endpoint::DeleteKey { id } => handle_delete_key(&self.garage, id).await, // Buckets diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index 592440a5..b4475268 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -20,6 +20,13 @@ pub enum Error { /// The API access key does not exist #[error(display = "Access key not found: {}", _0)] NoSuchAccessKey(String), + + /// In Import key, the key already exists + #[error( + display = "Key {} already exists in data store. Even if it is deleted, we can't let you create a new key with the same ID. Sorry.", + _0 + )] + KeyAlreadyExists(String), } impl From for Error @@ -52,6 +59,7 @@ impl Error { match self { Error::CommonError(c) => c.aws_code(), Error::NoSuchAccessKey(_) => "NoSuchAccessKey", + Error::KeyAlreadyExists(_) => "KeyAlreadyExists", } } } @@ -62,6 +70,7 @@ impl ApiError for Error { match self { Error::CommonError(c) => c.http_status_code(), Error::NoSuchAccessKey(_) => StatusCode::NOT_FOUND, + Error::KeyAlreadyExists(_) => StatusCode::CONFLICT, } } diff --git a/src/api/admin/key.rs b/src/api/admin/key.rs index be37088b..f30b5dbb 100644 --- a/src/api/admin/key.rs +++ b/src/api/admin/key.rs @@ -80,6 +80,31 @@ struct CreateKeyRequest { name: String, } +pub async fn handle_import_key( + garage: &Arc, + req: Request, +) -> Result, Error> { + let req = parse_json_body::(req).await?; + + let prev_key = garage.key_table.get(&EmptyKey, &req.access_key_id).await?; + if prev_key.is_some() { + return Err(Error::KeyAlreadyExists(req.access_key_id.to_string())); + } + + let imported_key = Key::import(&req.access_key_id, &req.secret_access_key, &req.name); + garage.key_table.insert(&imported_key).await?; + + key_info_results(garage, imported_key).await +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ImportKeyRequest { + access_key_id: String, + secret_access_key: String, + name: String, +} + pub async fn handle_update_key( garage: &Arc, id: String, diff --git a/src/api/admin/router.rs b/src/api/admin/router.rs index 41e7ed73..93639873 100644 --- a/src/api/admin/router.rs +++ b/src/api/admin/router.rs @@ -27,6 +27,7 @@ pub enum Endpoint { // Keys ListKeys, CreateKey, + ImportKey, GetKeyInfo { id: Option, search: Option, @@ -103,6 +104,7 @@ impl Endpoint { GET "/v0/key" if search => GetKeyInfo (query_opt::id, query_opt::search), POST "/v0/key" if id => UpdateKey (query::id), POST "/v0/key" => CreateKey, + POST "/v0/key/import" => ImportKey, DELETE "/v0/key" if id => DeleteKey (query::id), GET "/v0/key" => ListKeys, // Bucket endpoints diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 78d538ec..1d7c3ea4 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -384,12 +384,9 @@ impl System { } } if errors.len() == 1 { - return Err(Error::Message(errors[0].1.to_string())); + Err(Error::Message(errors[0].1.to_string())) } else { - return Err(Error::Message(format!( - "Could not connect to specified peers. Errors: {:?}", - errors - ))); + Err(Error::Message(format!("{:?}", errors))) } } -- 2.43.4 From 926b3c0fadcd10b4ef75c7d8995170ac3d16ca89 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 18 May 2022 00:27:57 +0200 Subject: [PATCH 39/46] Rename error varian for Clippy --- src/api/admin/error.rs | 16 ++++++++-------- src/api/k2v/error.rs | 20 ++++++++++---------- src/api/s3/error.rs | 18 +++++++++--------- src/api/signature/error.rs | 4 ++-- 4 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index b4475268..cc51807f 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -14,7 +14,7 @@ use crate::helpers::CustomApiErrorBody; pub enum Error { #[error(display = "{}", _0)] /// Error from common error - CommonError(CommonError), + Common(CommonError), // Category: cannot process /// The API access key does not exist @@ -34,7 +34,7 @@ where CommonError: From, { fn from(err: T) -> Self { - Error::CommonError(CommonError::from(err)) + Error::Common(CommonError::from(err)) } } @@ -43,12 +43,12 @@ impl CommonErrorDerivative for Error {} impl From for Error { fn from(err: HelperError) -> Self { match err { - HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), + HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), + HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), HelperError::InvalidBucketName(n) => { - Self::CommonError(CommonError::InvalidBucketName(n)) + Self::Common(CommonError::InvalidBucketName(n)) } - HelperError::NoSuchBucket(n) => Self::CommonError(CommonError::NoSuchBucket(n)), + HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), HelperError::NoSuchAccessKey(n) => Self::NoSuchAccessKey(n), } } @@ -57,7 +57,7 @@ impl From for Error { impl Error { fn code(&self) -> &'static str { match self { - Error::CommonError(c) => c.aws_code(), + Error::Common(c) => c.aws_code(), Error::NoSuchAccessKey(_) => "NoSuchAccessKey", Error::KeyAlreadyExists(_) => "KeyAlreadyExists", } @@ -68,7 +68,7 @@ impl ApiError for Error { /// Get the HTTP status code that best represents the meaning of the error for the client fn http_status_code(&self) -> StatusCode { match self { - Error::CommonError(c) => c.http_status_code(), + Error::Common(c) => c.http_status_code(), Error::NoSuchAccessKey(_) => StatusCode::NOT_FOUND, Error::KeyAlreadyExists(_) => StatusCode::CONFLICT, } diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index dbd9a5f5..9263d9be 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -15,7 +15,7 @@ use crate::signature::error::Error as SignatureError; pub enum Error { #[error(display = "{}", _0)] /// Error from common error - CommonError(CommonError), + Common(CommonError), // Category: cannot process /// Authorization Header Malformed @@ -48,7 +48,7 @@ where CommonError: From, { fn from(err: T) -> Self { - Error::CommonError(CommonError::from(err)) + Error::Common(CommonError::from(err)) } } @@ -57,13 +57,13 @@ impl CommonErrorDerivative for Error {} impl From for Error { fn from(err: HelperError) -> Self { match err { - HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), + HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), + HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), HelperError::InvalidBucketName(n) => { - Self::CommonError(CommonError::InvalidBucketName(n)) + Self::Common(CommonError::InvalidBucketName(n)) } - HelperError::NoSuchBucket(n) => Self::CommonError(CommonError::NoSuchBucket(n)), - e => Self::CommonError(CommonError::BadRequest(format!("{}", e))), + HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), + e => Self::Common(CommonError::BadRequest(format!("{}", e))), } } } @@ -71,7 +71,7 @@ impl From for Error { impl From for Error { fn from(err: SignatureError) -> Self { match err { - SignatureError::CommonError(c) => Self::CommonError(c), + SignatureError::Common(c) => Self::Common(c), SignatureError::AuthorizationHeaderMalformed(c) => { Self::AuthorizationHeaderMalformed(c) } @@ -87,7 +87,7 @@ impl Error { /// as we are building a custom API fn code(&self) -> &'static str { match self { - Error::CommonError(c) => c.aws_code(), + Error::Common(c) => c.aws_code(), Error::NoSuchKey => "NoSuchKey", Error::NotAcceptable(_) => "NotAcceptable", Error::AuthorizationHeaderMalformed(_) => "AuthorizationHeaderMalformed", @@ -102,7 +102,7 @@ impl ApiError for Error { /// Get the HTTP status code that best represents the meaning of the error for the client fn http_status_code(&self) -> StatusCode { match self { - Error::CommonError(c) => c.http_status_code(), + Error::Common(c) => c.http_status_code(), Error::NoSuchKey => StatusCode::NOT_FOUND, Error::NotAcceptable(_) => StatusCode::NOT_ACCEPTABLE, Error::AuthorizationHeaderMalformed(_) diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index f2096bea..60e2915c 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -17,7 +17,7 @@ use crate::signature::error::Error as SignatureError; pub enum Error { #[error(display = "{}", _0)] /// Error from common error - CommonError(CommonError), + Common(CommonError), // Category: cannot process /// Authorization Header Malformed @@ -80,7 +80,7 @@ where CommonError: From, { fn from(err: T) -> Self { - Error::CommonError(CommonError::from(err)) + Error::Common(CommonError::from(err)) } } @@ -89,12 +89,12 @@ impl CommonErrorDerivative for Error {} impl From for Error { fn from(err: HelperError) -> Self { match err { - HelperError::Internal(i) => Self::CommonError(CommonError::InternalError(i)), - HelperError::BadRequest(b) => Self::CommonError(CommonError::BadRequest(b)), + HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), + HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), HelperError::InvalidBucketName(n) => { - Self::CommonError(CommonError::InvalidBucketName(n)) + Self::Common(CommonError::InvalidBucketName(n)) } - HelperError::NoSuchBucket(n) => Self::CommonError(CommonError::NoSuchBucket(n)), + HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), e => Self::bad_request(format!("{}", e)), } } @@ -115,7 +115,7 @@ impl From for Error { impl From for Error { fn from(err: SignatureError) -> Self { match err { - SignatureError::CommonError(c) => Self::CommonError(c), + SignatureError::Common(c) => Self::Common(c), SignatureError::AuthorizationHeaderMalformed(c) => { Self::AuthorizationHeaderMalformed(c) } @@ -134,7 +134,7 @@ impl From for Error { impl Error { pub fn aws_code(&self) -> &'static str { match self { - Error::CommonError(c) => c.aws_code(), + Error::Common(c) => c.aws_code(), Error::NoSuchKey => "NoSuchKey", Error::NoSuchUpload => "NoSuchUpload", Error::PreconditionFailed => "PreconditionFailed", @@ -156,7 +156,7 @@ impl ApiError for Error { /// Get the HTTP status code that best represents the meaning of the error for the client fn http_status_code(&self) -> StatusCode { match self { - Error::CommonError(c) => c.http_status_code(), + Error::Common(c) => c.http_status_code(), Error::NoSuchKey | Error::NoSuchUpload => StatusCode::NOT_FOUND, Error::PreconditionFailed => StatusCode::PRECONDITION_FAILED, Error::InvalidRange(_) => StatusCode::RANGE_NOT_SATISFIABLE, diff --git a/src/api/signature/error.rs b/src/api/signature/error.rs index 3ef5cdcd..f5a067bd 100644 --- a/src/api/signature/error.rs +++ b/src/api/signature/error.rs @@ -8,7 +8,7 @@ pub use crate::common_error::{CommonErrorDerivative, OkOrBadRequest, OkOrInterna pub enum Error { #[error(display = "{}", _0)] /// Error from common error - CommonError(CommonError), + Common(CommonError), /// Authorization Header Malformed #[error(display = "Authorization header malformed, expected scope: {}", _0)] @@ -29,7 +29,7 @@ where CommonError: From, { fn from(err: T) -> Self { - Error::CommonError(CommonError::from(err)) + Error::Common(CommonError::from(err)) } } -- 2.43.4 From 30e393b439e733107755517a68ebe681dd64a2d5 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 18 May 2022 00:32:51 +0200 Subject: [PATCH 40/46] Fix fmt --- src/api/admin/error.rs | 4 +--- src/api/k2v/error.rs | 4 +--- src/api/s3/error.rs | 4 +--- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/api/admin/error.rs b/src/api/admin/error.rs index cc51807f..c4613cb3 100644 --- a/src/api/admin/error.rs +++ b/src/api/admin/error.rs @@ -45,9 +45,7 @@ impl From for Error { match err { HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(n) => { - Self::Common(CommonError::InvalidBucketName(n)) - } + HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)), HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), HelperError::NoSuchAccessKey(n) => Self::NoSuchAccessKey(n), } diff --git a/src/api/k2v/error.rs b/src/api/k2v/error.rs index 9263d9be..4c55d8b5 100644 --- a/src/api/k2v/error.rs +++ b/src/api/k2v/error.rs @@ -59,9 +59,7 @@ impl From for Error { match err { HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(n) => { - Self::Common(CommonError::InvalidBucketName(n)) - } + HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)), HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), e => Self::Common(CommonError::BadRequest(format!("{}", e))), } diff --git a/src/api/s3/error.rs b/src/api/s3/error.rs index 60e2915c..ac632540 100644 --- a/src/api/s3/error.rs +++ b/src/api/s3/error.rs @@ -91,9 +91,7 @@ impl From for Error { match err { HelperError::Internal(i) => Self::Common(CommonError::InternalError(i)), HelperError::BadRequest(b) => Self::Common(CommonError::BadRequest(b)), - HelperError::InvalidBucketName(n) => { - Self::Common(CommonError::InvalidBucketName(n)) - } + HelperError::InvalidBucketName(n) => Self::Common(CommonError::InvalidBucketName(n)), HelperError::NoSuchBucket(n) => Self::Common(CommonError::NoSuchBucket(n)), e => Self::bad_request(format!("{}", e)), } -- 2.43.4 From 5367f8adb2aab70a5478c43b93de7051a93d831b Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 18 May 2022 10:09:51 +0200 Subject: [PATCH 41/46] Refactor bucket emptiness check and add k2v check --- src/api/admin/bucket.rs | 13 +---------- src/api/s3/bucket.rs | 13 +---------- src/garage/admin.rs | 14 +----------- src/model/helper/bucket.rs | 47 +++++++++++++++++++++++++++++++++++++- 4 files changed, 49 insertions(+), 38 deletions(-) diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 3ad2c735..30dc3436 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -14,7 +14,6 @@ use garage_model::bucket_alias_table::*; use garage_model::bucket_table::*; use garage_model::garage::Garage; use garage_model::permission::*; -use garage_model::s3::object_table::ObjectFilter; use crate::admin::error::*; use crate::admin::key::ApiBucketKeyPerm; @@ -327,17 +326,7 @@ pub async fn handle_delete_bucket( let state = bucket.state.as_option().unwrap(); // Check bucket is empty - let objects = garage - .object_table - .get_range( - &bucket_id, - None, - Some(ObjectFilter::IsData), - 10, - EnumerationOrder::Forward, - ) - .await?; - if !objects.is_empty() { + if !helper.is_bucket_empty(bucket_id).await? { return Err(CommonError::BucketNotEmpty.into()); } diff --git a/src/api/s3/bucket.rs b/src/api/s3/bucket.rs index 1304cc07..2071fe55 100644 --- a/src/api/s3/bucket.rs +++ b/src/api/s3/bucket.rs @@ -8,7 +8,6 @@ use garage_model::bucket_table::Bucket; use garage_model::garage::Garage; use garage_model::key_table::Key; use garage_model::permission::BucketKeyPerm; -use garage_model::s3::object_table::ObjectFilter; use garage_table::util::*; use garage_util::crdt::*; use garage_util::data::*; @@ -229,17 +228,7 @@ pub async fn handle_delete_bucket( // Delete bucket // Check bucket is empty - let objects = garage - .object_table - .get_range( - &bucket_id, - None, - Some(ObjectFilter::IsData), - 10, - EnumerationOrder::Forward, - ) - .await?; - if !objects.is_empty() { + if !garage.bucket_helper().is_bucket_empty(bucket_id).await? { return Err(CommonError::BucketNotEmpty.into()); } diff --git a/src/garage/admin.rs b/src/garage/admin.rs index c1ba297b..bc1f494a 100644 --- a/src/garage/admin.rs +++ b/src/garage/admin.rs @@ -22,7 +22,6 @@ use garage_model::helper::error::{Error, OkOrBadRequest}; use garage_model::key_table::*; use garage_model::migrate::Migrate; use garage_model::permission::*; -use garage_model::s3::object_table::ObjectFilter; use crate::cli::*; use crate::repair::Repair; @@ -213,18 +212,7 @@ impl AdminRpcHandler { } // Check bucket is empty - let objects = self - .garage - .object_table - .get_range( - &bucket_id, - None, - Some(ObjectFilter::IsData), - 10, - EnumerationOrder::Forward, - ) - .await?; - if !objects.is_empty() { + if !helper.is_bucket_empty(bucket_id).await? { return Err(Error::BadRequest(format!( "Bucket {} is not empty", query.name diff --git a/src/model/helper/bucket.rs b/src/model/helper/bucket.rs index 734cb40e..130ba5be 100644 --- a/src/model/helper/bucket.rs +++ b/src/model/helper/bucket.rs @@ -1,9 +1,10 @@ -use garage_table::util::*; use garage_util::crdt::*; use garage_util::data::*; use garage_util::error::{Error as GarageError, OkOrMessage}; use garage_util::time::*; +use garage_table::util::*; + use crate::bucket_alias_table::*; use crate::bucket_table::*; use crate::garage::Garage; @@ -11,6 +12,7 @@ use crate::helper::error::*; use crate::helper::key::KeyHelper; use crate::key_table::*; use crate::permission::BucketKeyPerm; +use crate::s3::object_table::ObjectFilter; pub struct BucketHelper<'a>(pub(crate) &'a Garage); @@ -427,4 +429,47 @@ impl<'a> BucketHelper<'a> { Ok(()) } + + pub async fn is_bucket_empty(&self, bucket_id: Uuid) -> Result { + let objects = self + .0 + .object_table + .get_range( + &bucket_id, + None, + Some(ObjectFilter::IsData), + 10, + EnumerationOrder::Forward, + ) + .await?; + if !objects.is_empty() { + return Ok(false); + } + + #[cfg(feature = "k2v")] + { + use garage_rpc::ring::Ring; + use std::sync::Arc; + + let ring: Arc = self.0.system.ring.borrow().clone(); + let k2vindexes = self + .0 + .k2v + .counter_table + .table + .get_range( + &bucket_id, + None, + Some((DeletedFilter::NotDeleted, ring.layout.node_id_vec.clone())), + 10, + EnumerationOrder::Forward, + ) + .await?; + if !k2vindexes.is_empty() { + return Ok(false); + } + } + + Ok(true) + } } -- 2.43.4 From d768f559da43032b257fc759c3b22ca29e1bbe49 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 23 May 2022 12:06:00 +0200 Subject: [PATCH 42/46] Update documentation with warning --- doc/drafts/admin-api.md | 79 +++++++++++++++++++++++------------------ 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index d6b8ed5d..9aa3cdbb 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -1,21 +1,30 @@ -# Access control +# Specification of Garage's administration API + + +**WARNING.** At this point, there is no comittement to stability of the APIs described in this document. +We will bump the version numbers prefixed to each API endpoint at each time the syntax +or semantics change, meaning that code that relies on these endpoint will break +when changes are introduced. + + +## Access control The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section: - `metrics_token`: the token for accessing the Metrics endpoint (if this token is not set in the config file, the Metrics endpoint can be accessed without access control); - `admin_token`: the token for accessing all of the other administration endpoints (if this token is not set in the config file, these endpoints can be accessed without access control). -# Administration API endpoints +## Administration API endpoints -## Metrics-related endpoints +### Metrics-related endpoints -### Metrics `GET /metrics` +#### Metrics `GET /metrics` Returns internal Garage metrics in Prometheus format. -## Cluster operations +### Cluster operations -### GetClusterStatus `GET /v0/status` +#### GetClusterStatus `GET /v0/status` Returns the cluster's current status in JSON, including: @@ -94,7 +103,7 @@ Example response body: } ``` -### ConnectClusterNodes `POST /v0/connect` +#### ConnectClusterNodes `POST /v0/connect` Instructs this Garage node to connect to other Garage nodes at specified addresses. @@ -124,7 +133,7 @@ Example response: ] ``` -### GetClusterLayout `GET /v0/layout` +#### GetClusterLayout `GET /v0/layout` Returns the cluster's current layout in JSON, including: @@ -173,7 +182,7 @@ Example response body: } ``` -### UpdateClusterLayout `POST /v0/layout` +#### UpdateClusterLayout `POST /v0/layout` Send modifications to the cluster layout. These modifications will be included in the staged role changes, visible in subsequent calls @@ -204,7 +213,7 @@ Contrary to the CLI that may update only a subset of the fields values must be specified. -### ApplyClusterLayout `POST /v0/layout/apply` +#### ApplyClusterLayout `POST /v0/layout/apply` Applies to the cluster the layout changes currently registered as staged layout changes. @@ -221,7 +230,7 @@ Similarly to the CLI, the body must include the version of the new layout that will be created, which MUST be 1 + the value of the currently existing layout in the cluster. -### RevertClusterLayout `POST /v0/layout/revert` +#### RevertClusterLayout `POST /v0/layout/revert` Clears all of the staged layout changes. @@ -240,9 +249,9 @@ version number, which MUST be 1 + the value of the currently existing layout in the cluster. -## Access key operations +### Access key operations -### ListKeys `GET /v0/key` +#### ListKeys `GET /v0/key` Returns all API access keys in the cluster. @@ -261,7 +270,7 @@ Example response: ] ``` -### CreateKey `POST /v0/key` +#### CreateKey `POST /v0/key` Creates a new API access key. @@ -273,7 +282,7 @@ Request body format: } ``` -### ImportKey `POST /v0/key/import` +#### ImportKey `POST /v0/key/import` Imports an existing API key. @@ -287,8 +296,8 @@ Request body format: } ``` -### GetKeyInfo `GET /v0/key?id=` -### GetKeyInfo `GET /v0/key?search=` +#### GetKeyInfo `GET /v0/key?id=` +#### GetKeyInfo `GET /v0/key?search=` Returns information about the requested API access key. @@ -359,11 +368,11 @@ Example response: } ``` -### DeleteKey `DELETE /v0/key?id=` +#### DeleteKey `DELETE /v0/key?id=` Deletes an API access key. -### UpdateKey `POST /v0/key?id=` +#### UpdateKey `POST /v0/key?id=` Updates information about the specified API access key. @@ -384,9 +393,9 @@ If they are present, the corresponding modifications are applied to the key, oth The possible flags in `allow` and `deny` are: `createBucket`. -## Bucket operations +### Bucket operations -### ListBuckets `GET /v0/bucket` +#### ListBuckets `GET /v0/bucket` Returns all storage buckets in the cluster. @@ -428,8 +437,8 @@ Example response: ] ``` -### GetBucketInfo `GET /v0/bucket?id=` -### GetBucketInfo `GET /v0/bucket?globalAlias=` +#### GetBucketInfo `GET /v0/bucket?id=` +#### GetBucketInfo `GET /v0/bucket?globalAlias=` Returns information about the requested storage bucket. @@ -462,7 +471,7 @@ Example response: } ``` -### CreateBucket `POST /v0/bucket` +#### CreateBucket `POST /v0/bucket` Creates a new storage bucket. @@ -495,13 +504,13 @@ OR Creates a new bucket, either with a global alias, a local one, or no alias at all. -### DeleteBucket `DELETE /v0/bucket?id=` +#### DeleteBucket `DELETE /v0/bucket?id=` Deletes a storage bucket. A bucket cannot be deleted if it is not empty. Warning: this will delete all aliases associated with the bucket! -### PutBucketWebsite `PUT /v0/bucket/website?id=` +#### PutBucketWebsite `PUT /v0/bucket/website?id=` Sets the website configuration for a bucket (this also enables website access for this bucket). @@ -517,14 +526,14 @@ Request body format: The field `errorDocument` is optional, if no error document is set a generic error message is displayed when errors happen. -### DeleteBucketWebsite `DELETE /v0/bucket/website?id=` +#### DeleteBucketWebsite `DELETE /v0/bucket/website?id=` Deletes the website configuration for a bucket (disables website access for this bucket). -## Operations on permissions for keys on buckets +### Operations on permissions for keys on buckets -### BucketAllowKey `POST /v0/bucket/allow` +#### BucketAllowKey `POST /v0/bucket/allow` Allows a key to do read/write/owner operations on a bucket. @@ -545,7 +554,7 @@ Request body format: Flags in `permissions` which have the value `true` will be activated. Other flags will remain unchanged. -### BucketDenyKey `POST /v0/bucket/deny` +#### BucketDenyKey `POST /v0/bucket/deny` Denies a key from doing read/write/owner operations on a bucket. @@ -567,21 +576,21 @@ Flags in `permissions` which have the value `true` will be deactivated. Other flags will remain unchanged. -## Operations on bucket aliases +### Operations on bucket aliases -### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=&alias=` +#### GlobalAliasBucket `PUT /v0/bucket/alias/global?id=&alias=` Empty body. Creates a global alias for a bucket. -### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=&alias=` +#### GlobalUnaliasBucket `DELETE /v0/bucket/alias/global?id=&alias=` Removes a global alias for a bucket. -### LocalAliasBucket `PUT /v0/bucket/alias/local?id=&accessKeyId=&alias=` +#### LocalAliasBucket `PUT /v0/bucket/alias/local?id=&accessKeyId=&alias=` Empty body. Creates a local alias for a bucket in the namespace of a specific access key. -### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=&accessKeyId&alias=` +#### LocalUnaliasBucket `DELETE /v0/bucket/alias/local?id=&accessKeyId&alias=` Removes a local alias for a bucket in the namespace of a specific access key. -- 2.43.4 From 1c88ee9bc50a8226fe0c7cf11533ddcf4b183885 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 23 May 2022 16:40:10 +0200 Subject: [PATCH 43/46] Make authorization token mandatory for admin API --- src/api/admin/api_server.rs | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/api/admin/api_server.rs b/src/api/admin/api_server.rs index a0af9bd9..57e3e5cf 100644 --- a/src/api/admin/api_server.rs +++ b/src/api/admin/api_server.rs @@ -107,17 +107,27 @@ impl ApiHandler for AdminApiServer { req: Request, endpoint: Endpoint, ) -> Result, Error> { - let expected_auth_header = match endpoint.authorization_type() { - Authorization::MetricsToken => self.metrics_token.as_ref(), - Authorization::AdminToken => self.admin_token.as_ref(), - }; + let expected_auth_header = + match endpoint.authorization_type() { + Authorization::MetricsToken => self.metrics_token.as_ref(), + Authorization::AdminToken => match &self.admin_token { + None => return Err(Error::forbidden( + "Admin token isn't configured, admin API access is disabled for security.", + )), + Some(t) => Some(t), + }, + }; if let Some(h) = expected_auth_header { match req.headers().get("Authorization") { - None => Err(Error::forbidden("Authorization token must be provided")), - Some(v) if v.to_str().map(|hv| hv == h).unwrap_or(false) => Ok(()), - _ => Err(Error::forbidden("Invalid authorization token provided")), - }?; + None => return Err(Error::forbidden("Authorization token must be provided")), + Some(v) => { + let authorized = v.to_str().map(|hv| hv.trim() == h).unwrap_or(false); + if !authorized { + return Err(Error::forbidden("Invalid authorization token provided")); + } + } + } } match endpoint { -- 2.43.4 From 2f250a83e14e7fa10810e2bb9925cc125baf670c Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 23 May 2022 17:39:26 +0200 Subject: [PATCH 44/46] fix doc --- doc/drafts/admin-api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index 9aa3cdbb..e6ce6336 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -12,7 +12,7 @@ when changes are introduced. The admin API uses two different tokens for acces control, that are specified in the config file's `[admin]` section: - `metrics_token`: the token for accessing the Metrics endpoint (if this token is not set in the config file, the Metrics endpoint can be accessed without access control); -- `admin_token`: the token for accessing all of the other administration endpoints (if this token is not set in the config file, these endpoints can be accessed without access control). +- `admin_token`: the token for accessing all of the other administration endpoints (if this token is not set in the config file, access to these endpoints is disabled entirely). ## Administration API endpoints -- 2.43.4 From 0b43a7135153428f036be2056e197041797b0997 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 24 May 2022 11:52:33 +0200 Subject: [PATCH 45/46] Fix some docs and change syntax of CreateBucket permissions --- doc/drafts/admin-api.md | 15 +++++++++++---- src/api/admin/bucket.rs | 13 ++++++++++--- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/doc/drafts/admin-api.md b/doc/drafts/admin-api.md index e6ce6336..b35a87f1 100644 --- a/doc/drafts/admin-api.md +++ b/doc/drafts/admin-api.md @@ -124,11 +124,11 @@ Example response: [ { "success": true, - "error": null, + "error": null }, { "success": false, - "error": "Handshake error", + "error": "Handshake error" } ] ``` @@ -490,7 +490,11 @@ OR "localAlias": { "accessKeyId": "GK31c2f218a2e44f485b94239e", "alias": "NameOfMyBucket", - "allPermissions": true + "allow": { + "read": true, + "write": true, + "owner": false + } } } ``` @@ -504,6 +508,9 @@ OR Creates a new bucket, either with a global alias, a local one, or no alias at all. +Technically, you can also specify both `globalAlias` and `localAlias` and that would create +two aliases, but I don't see why you would want to do that. + #### DeleteBucket `DELETE /v0/bucket?id=` Deletes a storage bucket. A bucket cannot be deleted if it is not empty. @@ -519,7 +526,7 @@ Request body format: ```json { "indexDocument": "index.html", - "errorDocument": "404.html", + "errorDocument": "404.html" } ``` diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 30dc3436..283e54c6 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use garage_util::crdt::*; use garage_util::data::*; use garage_util::error::Error as GarageError; +use garage_util::time::*; use garage_table::*; @@ -283,13 +284,19 @@ pub async fn handle_create_bucket( .bucket_helper() .set_local_bucket_alias(bucket.id, &la.access_key_id, &la.alias) .await?; - if la.all_permissions { + + if la.allow.read || la.allow.write || la.allow.owner { garage .bucket_helper() .set_bucket_key_permissions( bucket.id, &la.access_key_id, - BucketKeyPerm::ALL_PERMISSIONS, + BucketKeyPerm{ + timestamp: now_msec(), + allow_read: la.allow.read, + allow_write: la.allow.write, + allow_owner: la.allow.owner, + } ) .await?; } @@ -311,7 +318,7 @@ struct CreateBucketLocalAlias { access_key_id: String, alias: String, #[serde(default)] - all_permissions: bool, + allow: ApiBucketKeyPerm, } pub async fn handle_delete_bucket( -- 2.43.4 From be59cafd47652a995658bdb478f13e3307b39884 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 24 May 2022 11:53:57 +0200 Subject: [PATCH 46/46] cargo fmt --- src/api/admin/bucket.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/api/admin/bucket.rs b/src/api/admin/bucket.rs index 283e54c6..849d28ac 100644 --- a/src/api/admin/bucket.rs +++ b/src/api/admin/bucket.rs @@ -291,12 +291,12 @@ pub async fn handle_create_bucket( .set_bucket_key_permissions( bucket.id, &la.access_key_id, - BucketKeyPerm{ + BucketKeyPerm { timestamp: now_msec(), allow_read: la.allow.read, allow_write: la.allow.write, allow_owner: la.allow.owner, - } + }, ) .await?; } -- 2.43.4