K2V #293
12 changed files with 703 additions and 577 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -876,6 +876,7 @@ dependencies = [
|
|||
name = "garage_api"
|
||||
version = "0.7.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64",
|
||||
"bytes 1.1.0",
|
||||
"chrono",
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ garage_table = { version = "0.7.0", path = "../table" }
|
|||
garage_block = { version = "0.7.0", path = "../block" }
|
||||
garage_util = { version = "0.7.0", path = "../util" }
|
||||
|
||||
async-trait = "0.1.7"
|
||||
base64 = "0.13"
|
||||
bytes = "1.0"
|
||||
chrono = "0.4"
|
||||
|
|
|
|||
209
src/api/generic_server.rs
Normal file
209
src/api/generic_server.rs
Normal file
|
|
@ -0,0 +1,209 @@
|
|||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use chrono::{DateTime, NaiveDateTime, Utc};
|
||||
use futures::future::Future;
|
||||
use futures::prelude::*;
|
||||
use hyper::header;
|
||||
use hyper::server::conn::AddrStream;
|
||||
use hyper::service::{make_service_fn, service_fn};
|
||||
use hyper::{Body, Method, Request, Response, Server};
|
||||
|
||||
use opentelemetry::{
|
||||
global,
|
||||
metrics::{Counter, ValueRecorder},
|
||||
trace::{FutureExt, SpanRef, TraceContextExt, Tracer},
|
||||
Context, KeyValue,
|
||||
};
|
||||
|
||||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::metrics::{gen_trace_id, RecordDuration};
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::key_table::Key;
|
||||
|
||||
use garage_table::util::*;
|
||||
|
||||
use crate::error::*;
|
||||
use crate::signature::compute_scope;
|
||||
use crate::signature::payload::check_payload_signature;
|
||||
use crate::signature::streaming::SignedPayloadStream;
|
||||
use crate::signature::LONG_DATETIME;
|
||||
|
||||
pub(crate) trait ApiEndpoint: Send + Sync + 'static {
|
||||
fn name(&self) -> &'static str;
|
||||
fn add_span_attributes<'a>(&self, span: SpanRef<'a>);
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
pub(crate) trait ApiHandler: Send + Sync + 'static {
|
||||
const API_NAME: &'static str;
|
||||
const API_NAME_DISPLAY: &'static str;
|
||||
|
||||
type Endpoint: ApiEndpoint;
|
||||
|
||||
fn parse_endpoint(&self, r: &Request<Body>) -> Result<Self::Endpoint, Error>;
|
||||
async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
endpoint: Self::Endpoint,
|
||||
) -> Result<Response<Body>, Error>;
|
||||
}
|
||||
|
||||
pub(crate) struct ApiServer<A: ApiHandler> {
|
||||
s3_region: String,
|
||||
api_handler: A,
|
||||
|
||||
// Metrics
|
||||
request_counter: Counter<u64>,
|
||||
error_counter: Counter<u64>,
|
||||
request_duration: ValueRecorder<f64>,
|
||||
}
|
||||
|
||||
impl<A: ApiHandler> ApiServer<A> {
|
||||
pub fn new(s3_region: String, api_handler: A) -> Arc<Self> {
|
||||
let meter = global::meter("garage/api");
|
||||
Arc::new(Self {
|
||||
s3_region,
|
||||
api_handler,
|
||||
request_counter: meter
|
||||
.u64_counter(format!("api.{}.request_counter", A::API_NAME))
|
||||
.with_description(format!(
|
||||
"Number of API calls to the various {} API endpoints",
|
||||
A::API_NAME_DISPLAY
|
||||
))
|
||||
.init(),
|
||||
error_counter: meter
|
||||
.u64_counter(format!("api.{}.error_counter", A::API_NAME))
|
||||
.with_description(format!(
|
||||
"Number of API calls to the various {} API endpoints that resulted in errors",
|
||||
A::API_NAME_DISPLAY
|
||||
))
|
||||
.init(),
|
||||
request_duration: meter
|
||||
.f64_value_recorder(format!("api.{}.request_duration", A::API_NAME))
|
||||
.with_description(format!(
|
||||
"Duration of API calls to the various {} API endpoints",
|
||||
A::API_NAME_DISPLAY
|
||||
))
|
||||
.init(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn run_server(
|
||||
self: Arc<Self>,
|
||||
bind_addr: SocketAddr,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
let service = make_service_fn(|conn: &AddrStream| {
|
||||
let this = self.clone();
|
||||
|
||||
let client_addr = conn.remote_addr();
|
||||
async move {
|
||||
Ok::<_, GarageError>(service_fn(move |req: Request<Body>| {
|
||||
let this = this.clone();
|
||||
|
||||
this.handler(req, client_addr)
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
let server = Server::bind(&bind_addr).serve(service);
|
||||
|
||||
let graceful = server.with_graceful_shutdown(shutdown_signal);
|
||||
info!("API server listening on http://{}", bind_addr);
|
||||
|
||||
graceful.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handler(
|
||||
self: Arc<Self>,
|
||||
req: Request<Body>,
|
||||
addr: SocketAddr,
|
||||
) -> Result<Response<Body>, GarageError> {
|
||||
let uri = req.uri().clone();
|
||||
info!("{} {} {}", addr, req.method(), uri);
|
||||
debug!("{:?}", req);
|
||||
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let span = tracer
|
||||
.span_builder("S3 API call (unknown)")
|
||||
.with_trace_id(gen_trace_id())
|
||||
.with_attributes(vec![
|
||||
KeyValue::new("method", format!("{}", req.method())),
|
||||
KeyValue::new("uri", req.uri().to_string()),
|
||||
])
|
||||
.start(&tracer);
|
||||
|
||||
let res = self
|
||||
.handler_stage2(req)
|
||||
.with_context(Context::current_with_span(span))
|
||||
.await;
|
||||
|
||||
match res {
|
||||
Ok(x) => {
|
||||
debug!("{} {:?}", x.status(), x.headers());
|
||||
Ok(x)
|
||||
}
|
||||
Err(e) => {
|
||||
let body: Body = Body::from(e.aws_xml(&self.s3_region, uri.path()));
|
||||
let mut http_error_builder = Response::builder()
|
||||
.status(e.http_status_code())
|
||||
.header("Content-Type", "application/xml");
|
||||
|
||||
if let Some(header_map) = http_error_builder.headers_mut() {
|
||||
e.add_headers(header_map)
|
||||
}
|
||||
|
||||
let http_error = http_error_builder.body(body)?;
|
||||
|
||||
if e.http_status_code().is_server_error() {
|
||||
warn!("Response: error {}, {}", e.http_status_code(), e);
|
||||
} else {
|
||||
info!("Response: error {}, {}", e.http_status_code(), e);
|
||||
}
|
||||
Ok(http_error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handler_stage2(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
|
||||
let endpoint = self.api_handler.parse_endpoint(&req)?;
|
||||
debug!("Endpoint: {}", endpoint.name());
|
||||
|
||||
let current_context = Context::current();
|
||||
let current_span = current_context.span();
|
||||
current_span.update_name::<String>(format!("S3 API {}", endpoint.name()));
|
||||
current_span.set_attribute(KeyValue::new("endpoint", endpoint.name()));
|
||||
endpoint.add_span_attributes(current_span);
|
||||
|
||||
let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())];
|
||||
|
||||
let res = self
|
||||
.api_handler
|
||||
.handle(req, endpoint)
|
||||
.record_duration(&self.request_duration, &metrics_tags[..])
|
||||
.await;
|
||||
|
||||
self.request_counter.add(1, &metrics_tags[..]);
|
||||
|
||||
let status_code = match &res {
|
||||
Ok(r) => r.status(),
|
||||
Err(e) => e.http_status_code(),
|
||||
};
|
||||
if status_code.is_client_error() || status_code.is_server_error() {
|
||||
self.error_counter.add(
|
||||
1,
|
||||
&[
|
||||
metrics_tags[0].clone(),
|
||||
KeyValue::new("status_code", status_code.as_str().to_string()),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
res
|
||||
}
|
||||
}
|
||||
|
|
@ -1,6 +1,12 @@
|
|||
use crate::Error;
|
||||
use idna::domain_to_unicode;
|
||||
|
||||
use garage_util::data::*;
|
||||
|
||||
use garage_model::garage::Garage;
|
||||
use garage_model::key_table::Key;
|
||||
|
||||
use crate::error::*;
|
||||
|
||||
/// Host to bucket
|
||||
///
|
||||
/// Convert a host, like "bucket.garage-site.tld" to the corresponding bucket "bucket",
|
||||
|
|
@ -60,10 +66,116 @@ pub fn authority_to_host(authority: &str) -> Result<String, Error> {
|
|||
authority.map(|h| domain_to_unicode(h).0)
|
||||
}
|
||||
|
||||
#[allow(clippy::ptr_arg)]
|
||||
pub async fn resolve_bucket(
|
||||
garage: &Garage,
|
||||
bucket_name: &String,
|
||||
api_key: &Key,
|
||||
) -> Result<Uuid, Error> {
|
||||
let api_key_params = api_key
|
||||
.state
|
||||
.as_option()
|
||||
.ok_or_internal_error("Key should not be deleted at this point")?;
|
||||
|
||||
if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) {
|
||||
Ok(*bucket_id)
|
||||
} else {
|
||||
Ok(garage
|
||||
.bucket_helper()
|
||||
.resolve_global_bucket_name(bucket_name)
|
||||
.await?
|
||||
.ok_or(Error::NoSuchBucket)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in
|
||||
/// the host header of the request
|
||||
///
|
||||
/// S3 internally manages only buckets and keys. This function splits
|
||||
/// an HTTP path to get the corresponding bucket name and key.
|
||||
pub fn parse_bucket_key<'a>(
|
||||
path: &'a str,
|
||||
host_bucket: Option<&'a str>,
|
||||
) -> Result<(&'a str, Option<&'a str>), Error> {
|
||||
let path = path.trim_start_matches('/');
|
||||
|
||||
if let Some(bucket) = host_bucket {
|
||||
if !path.is_empty() {
|
||||
return Ok((bucket, Some(path)));
|
||||
} else {
|
||||
return Ok((bucket, None));
|
||||
}
|
||||
}
|
||||
|
||||
let (bucket, key) = match path.find('/') {
|
||||
Some(i) => {
|
||||
let key = &path[i + 1..];
|
||||
if !key.is_empty() {
|
||||
(&path[..i], Some(key))
|
||||
} else {
|
||||
(&path[..i], None)
|
||||
}
|
||||
}
|
||||
None => (path, None),
|
||||
};
|
||||
if bucket.is_empty() {
|
||||
return Err(Error::BadRequest("No bucket specified".to_string()));
|
||||
}
|
||||
Ok((bucket, key))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_containing_a_key() -> Result<(), Error> {
|
||||
let (bucket, key) = parse_bucket_key("/my_bucket/a/super/file.jpg", None)?;
|
||||
assert_eq!(bucket, "my_bucket");
|
||||
assert_eq!(key.expect("key must be set"), "a/super/file.jpg");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_containing_no_key() -> Result<(), Error> {
|
||||
let (bucket, key) = parse_bucket_key("/my_bucket/", None)?;
|
||||
assert_eq!(bucket, "my_bucket");
|
||||
assert!(key.is_none());
|
||||
let (bucket, key) = parse_bucket_key("/my_bucket", None)?;
|
||||
assert_eq!(bucket, "my_bucket");
|
||||
assert!(key.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_containing_no_bucket() {
|
||||
let parsed = parse_bucket_key("", None);
|
||||
assert!(parsed.is_err());
|
||||
let parsed = parse_bucket_key("/", None);
|
||||
assert!(parsed.is_err());
|
||||
let parsed = parse_bucket_key("////", None);
|
||||
assert!(parsed.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_with_vhost_and_key() -> Result<(), Error> {
|
||||
let (bucket, key) = parse_bucket_key("/a/super/file.jpg", Some("my-bucket"))?;
|
||||
assert_eq!(bucket, "my-bucket");
|
||||
assert_eq!(key.expect("key must be set"), "a/super/file.jpg");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_with_vhost_no_key() -> Result<(), Error> {
|
||||
let (bucket, key) = parse_bucket_key("", Some("my-bucket"))?;
|
||||
assert_eq!(bucket, "my-bucket");
|
||||
assert!(key.is_none());
|
||||
let (bucket, key) = parse_bucket_key("/", Some("my-bucket"))?;
|
||||
assert_eq!(bucket, "my-bucket");
|
||||
assert!(key.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authority_to_host_with_port() -> Result<(), Error> {
|
||||
let domain = authority_to_host("[::1]:3902")?;
|
||||
|
|
|
|||
2
src/api/k2v/mod.rs
Normal file
2
src/api/k2v/mod.rs
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
mod api_server;
|
||||
pub use api_server::run_api_server;
|
||||
|
|
@ -7,9 +7,9 @@ pub use error::Error;
|
|||
pub mod helpers;
|
||||
|
||||
mod encoding;
|
||||
mod generic_server;
|
||||
|
||||
/// This mode is public only to help testing. Don't expect stability here
|
||||
pub mod signature;
|
||||
|
||||
pub mod s3;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, NaiveDateTime, Utc};
|
||||
use futures::future::Future;
|
||||
use futures::prelude::*;
|
||||
|
|
@ -12,11 +13,10 @@ use hyper::{Body, Method, Request, Response, Server};
|
|||
use opentelemetry::{
|
||||
global,
|
||||
metrics::{Counter, ValueRecorder},
|
||||
trace::{FutureExt, TraceContextExt, Tracer},
|
||||
trace::{FutureExt, SpanRef, TraceContextExt, Tracer},
|
||||
Context, KeyValue,
|
||||
};
|
||||
|
||||
use garage_util::data::*;
|
||||
use garage_util::error::Error as GarageError;
|
||||
use garage_util::metrics::{gen_trace_id, RecordDuration};
|
||||
|
||||
|
|
@ -26,6 +26,7 @@ use garage_model::key_table::Key;
|
|||
use garage_table::util::*;
|
||||
|
||||
use crate::error::*;
|
||||
use crate::generic_server::*;
|
||||
use crate::signature::compute_scope;
|
||||
use crate::signature::payload::check_payload_signature;
|
||||
use crate::signature::streaming::SignedPayloadStream;
|
||||
|
|
@ -43,123 +44,51 @@ use crate::s3::put::*;
|
|||
use crate::s3::router::{Authorization, Endpoint};
|
||||
use crate::s3::website::*;
|
||||
|
||||
struct ApiMetrics {
|
||||
request_counter: Counter<u64>,
|
||||
error_counter: Counter<u64>,
|
||||
request_duration: ValueRecorder<f64>,
|
||||
pub struct S3ApiServer {
|
||||
garage: Arc<Garage>,
|
||||
}
|
||||
|
||||
impl ApiMetrics {
|
||||
fn new() -> Self {
|
||||
let meter = global::meter("garage/api");
|
||||
Self {
|
||||
request_counter: meter
|
||||
.u64_counter("api.request_counter")
|
||||
.with_description("Number of API calls to the various S3 API endpoints")
|
||||
.init(),
|
||||
error_counter: meter
|
||||
.u64_counter("api.error_counter")
|
||||
.with_description(
|
||||
"Number of API calls to the various S3 API endpoints that resulted in errors",
|
||||
)
|
||||
.init(),
|
||||
request_duration: meter
|
||||
.f64_value_recorder("api.request_duration")
|
||||
.with_description("Duration of API calls to the various S3 API endpoints")
|
||||
.init(),
|
||||
}
|
||||
}
|
||||
pub(crate) struct S3ApiEndpoint {
|
||||
bucket_name: Option<String>,
|
||||
endpoint: Endpoint,
|
||||
}
|
||||
|
||||
/// Run the S3 API server
|
||||
pub async fn run_api_server(
|
||||
impl S3ApiServer {
|
||||
pub async fn run(
|
||||
garage: Arc<Garage>,
|
||||
shutdown_signal: impl Future<Output = ()>,
|
||||
) -> Result<(), GarageError> {
|
||||
let addr = &garage.config.s3_api.api_bind_addr;
|
||||
) -> Result<(), GarageError> {
|
||||
let addr = garage.config.s3_api.api_bind_addr;
|
||||
|
||||
let metrics = Arc::new(ApiMetrics::new());
|
||||
|
||||
let service = make_service_fn(|conn: &AddrStream| {
|
||||
let garage = garage.clone();
|
||||
let metrics = metrics.clone();
|
||||
|
||||
let client_addr = conn.remote_addr();
|
||||
async move {
|
||||
Ok::<_, GarageError>(service_fn(move |req: Request<Body>| {
|
||||
let garage = garage.clone();
|
||||
let metrics = metrics.clone();
|
||||
|
||||
handler(garage, metrics, req, client_addr)
|
||||
}))
|
||||
}
|
||||
});
|
||||
|
||||
let server = Server::bind(addr).serve(service);
|
||||
|
||||
let graceful = server.with_graceful_shutdown(shutdown_signal);
|
||||
info!("API server listening on http://{}", addr);
|
||||
|
||||
graceful.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn handler(
|
||||
garage: Arc<Garage>,
|
||||
metrics: Arc<ApiMetrics>,
|
||||
req: Request<Body>,
|
||||
addr: SocketAddr,
|
||||
) -> Result<Response<Body>, GarageError> {
|
||||
let uri = req.uri().clone();
|
||||
info!("{} {} {}", addr, req.method(), uri);
|
||||
debug!("{:?}", req);
|
||||
|
||||
let tracer = opentelemetry::global::tracer("garage");
|
||||
let span = tracer
|
||||
.span_builder("S3 API call (unknown)")
|
||||
.with_trace_id(gen_trace_id())
|
||||
.with_attributes(vec![
|
||||
KeyValue::new("method", format!("{}", req.method())),
|
||||
KeyValue::new("uri", req.uri().to_string()),
|
||||
])
|
||||
.start(&tracer);
|
||||
|
||||
let res = handler_stage2(garage.clone(), metrics, req)
|
||||
.with_context(Context::current_with_span(span))
|
||||
.await;
|
||||
|
||||
match res {
|
||||
Ok(x) => {
|
||||
debug!("{} {:?}", x.status(), x.headers());
|
||||
Ok(x)
|
||||
}
|
||||
Err(e) => {
|
||||
let body: Body = Body::from(e.aws_xml(&garage.config.s3_api.s3_region, uri.path()));
|
||||
let mut http_error_builder = Response::builder()
|
||||
.status(e.http_status_code())
|
||||
.header("Content-Type", "application/xml");
|
||||
|
||||
if let Some(header_map) = http_error_builder.headers_mut() {
|
||||
e.add_headers(header_map)
|
||||
ApiServer::new(
|
||||
garage.config.s3_api.s3_region.clone(),
|
||||
S3ApiServer { garage },
|
||||
)
|
||||
.run_server(addr, shutdown_signal)
|
||||
.await
|
||||
}
|
||||
|
||||
let http_error = http_error_builder.body(body)?;
|
||||
|
||||
if e.http_status_code().is_server_error() {
|
||||
warn!("Response: error {}, {}", e.http_status_code(), e);
|
||||
} else {
|
||||
info!("Response: error {}, {}", e.http_status_code(), e);
|
||||
}
|
||||
Ok(http_error)
|
||||
async fn handle_request_without_bucket(
|
||||
&self,
|
||||
_req: Request<Body>,
|
||||
api_key: Key,
|
||||
endpoint: Endpoint,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
match endpoint {
|
||||
Endpoint::ListBuckets => handle_list_buckets(&self.garage, &api_key).await,
|
||||
endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handler_stage2(
|
||||
garage: Arc<Garage>,
|
||||
metrics: Arc<ApiMetrics>,
|
||||
req: Request<Body>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
#[async_trait]
|
||||
impl ApiHandler for S3ApiServer {
|
||||
const API_NAME: &'static str = "s3";
|
||||
const API_NAME_DISPLAY: &'static str = "S3";
|
||||
|
||||
type Endpoint = S3ApiEndpoint;
|
||||
|
||||
fn parse_endpoint(&self, req: &Request<Body>) -> Result<S3ApiEndpoint, Error> {
|
||||
let authority = req
|
||||
.headers()
|
||||
.get(header::HOST)
|
||||
|
|
@ -168,56 +97,34 @@ async fn handler_stage2(
|
|||
|
||||
let host = authority_to_host(authority)?;
|
||||
|
||||
let bucket_name = garage
|
||||
let bucket_name = self
|
||||
.garage
|
||||
.config
|
||||
.s3_api
|
||||
.root_domain
|
||||
.as_ref()
|
||||
.and_then(|root_domain| host_to_bucket(&host, root_domain));
|
||||
|
||||
let (endpoint, bucket_name) = Endpoint::from_request(&req, bucket_name.map(ToOwned::to_owned))?;
|
||||
debug!("Endpoint: {:?}", endpoint);
|
||||
let (endpoint, bucket_name) =
|
||||
Endpoint::from_request(&req, bucket_name.map(ToOwned::to_owned))?;
|
||||
|
||||
let current_context = Context::current();
|
||||
let current_span = current_context.span();
|
||||
current_span.update_name::<String>(format!("S3 API {}", endpoint.name()));
|
||||
current_span.set_attribute(KeyValue::new("endpoint", endpoint.name()));
|
||||
current_span.set_attribute(KeyValue::new(
|
||||
"bucket",
|
||||
bucket_name.clone().unwrap_or_default(),
|
||||
));
|
||||
|
||||
let metrics_tags = &[KeyValue::new("api_endpoint", endpoint.name())];
|
||||
|
||||
let res = handler_stage3(garage, req, endpoint, bucket_name)
|
||||
.record_duration(&metrics.request_duration, &metrics_tags[..])
|
||||
.await;
|
||||
|
||||
metrics.request_counter.add(1, &metrics_tags[..]);
|
||||
|
||||
let status_code = match &res {
|
||||
Ok(r) => r.status(),
|
||||
Err(e) => e.http_status_code(),
|
||||
};
|
||||
if status_code.is_client_error() || status_code.is_server_error() {
|
||||
metrics.error_counter.add(
|
||||
1,
|
||||
&[
|
||||
metrics_tags[0].clone(),
|
||||
KeyValue::new("status_code", status_code.as_str().to_string()),
|
||||
],
|
||||
);
|
||||
Ok(S3ApiEndpoint {
|
||||
bucket_name,
|
||||
endpoint,
|
||||
})
|
||||
}
|
||||
|
||||
res
|
||||
}
|
||||
|
||||
async fn handler_stage3(
|
||||
garage: Arc<Garage>,
|
||||
async fn handle(
|
||||
&self,
|
||||
req: Request<Body>,
|
||||
endpoint: Endpoint,
|
||||
bucket_name: Option<String>,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
endpoint: S3ApiEndpoint,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
let S3ApiEndpoint {
|
||||
bucket_name,
|
||||
endpoint,
|
||||
} = endpoint;
|
||||
let garage = self.garage.clone();
|
||||
|
||||
// Some endpoints are processed early, before we even check for an API key
|
||||
if let Endpoint::PostObject = endpoint {
|
||||
return handle_post_object(garage, req, bucket_name.unwrap()).await;
|
||||
|
|
@ -278,7 +185,11 @@ async fn handler_stage3(
|
|||
};
|
||||
|
||||
let bucket_name = match bucket_name {
|
||||
None => return handle_request_without_bucket(garage, req, api_key, endpoint).await,
|
||||
None => {
|
||||
return self
|
||||
.handle_request_without_bucket(req, api_key, endpoint)
|
||||
.await
|
||||
}
|
||||
Some(bucket) => bucket.to_string(),
|
||||
};
|
||||
|
||||
|
|
@ -340,7 +251,9 @@ async fn handler_stage3(
|
|||
)
|
||||
.await
|
||||
}
|
||||
Endpoint::CopyObject { key } => handle_copy(garage, &api_key, &req, bucket_id, &key).await,
|
||||
Endpoint::CopyObject { key } => {
|
||||
handle_copy(garage, &api_key, &req, bucket_id, &key).await
|
||||
}
|
||||
Endpoint::UploadPartCopy {
|
||||
key,
|
||||
part_number,
|
||||
|
|
@ -505,7 +418,9 @@ async fn handler_stage3(
|
|||
}
|
||||
Endpoint::DeleteBucketWebsite {} => handle_delete_website(garage, bucket_id).await,
|
||||
Endpoint::GetBucketCors {} => handle_get_cors(&bucket).await,
|
||||
Endpoint::PutBucketCors {} => handle_put_cors(garage, bucket_id, req, content_sha256).await,
|
||||
Endpoint::PutBucketCors {} => {
|
||||
handle_put_cors(garage, bucket_id, req, content_sha256).await
|
||||
}
|
||||
Endpoint::DeleteBucketCors {} => handle_delete_cors(garage, bucket_id).await,
|
||||
endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
|
||||
};
|
||||
|
|
@ -519,127 +434,18 @@ async fn handler_stage3(
|
|||
}
|
||||
|
||||
Ok(resp_ok)
|
||||
}
|
||||
|
||||
async fn handle_request_without_bucket(
|
||||
garage: Arc<Garage>,
|
||||
_req: Request<Body>,
|
||||
api_key: Key,
|
||||
endpoint: Endpoint,
|
||||
) -> Result<Response<Body>, Error> {
|
||||
match endpoint {
|
||||
Endpoint::ListBuckets => handle_list_buckets(&garage, &api_key).await,
|
||||
endpoint => Err(Error::NotImplemented(endpoint.name().to_owned())),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::ptr_arg)]
|
||||
pub async fn resolve_bucket(
|
||||
garage: &Garage,
|
||||
bucket_name: &String,
|
||||
api_key: &Key,
|
||||
) -> Result<Uuid, Error> {
|
||||
let api_key_params = api_key
|
||||
.state
|
||||
.as_option()
|
||||
.ok_or_internal_error("Key should not be deleted at this point")?;
|
||||
impl ApiEndpoint for S3ApiEndpoint {
|
||||
fn name(&self) -> &'static str {
|
||||
self.endpoint.name()
|
||||
}
|
||||
|
||||
if let Some(Some(bucket_id)) = api_key_params.local_aliases.get(bucket_name) {
|
||||
Ok(*bucket_id)
|
||||
} else {
|
||||
Ok(garage
|
||||
.bucket_helper()
|
||||
.resolve_global_bucket_name(bucket_name)
|
||||
.await?
|
||||
.ok_or(Error::NoSuchBucket)?)
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract the bucket name and the key name from an HTTP path and possibly a bucket provided in
|
||||
/// the host header of the request
|
||||
///
|
||||
/// S3 internally manages only buckets and keys. This function splits
|
||||
/// an HTTP path to get the corresponding bucket name and key.
|
||||
pub fn parse_bucket_key<'a>(
|
||||
path: &'a str,
|
||||
host_bucket: Option<&'a str>,
|
||||
) -> Result<(&'a str, Option<&'a str>), Error> {
|
||||
let path = path.trim_start_matches('/');
|
||||
|
||||
if let Some(bucket) = host_bucket {
|
||||
if !path.is_empty() {
|
||||
return Ok((bucket, Some(path)));
|
||||
} else {
|
||||
return Ok((bucket, None));
|
||||
}
|
||||
}
|
||||
|
||||
let (bucket, key) = match path.find('/') {
|
||||
Some(i) => {
|
||||
let key = &path[i + 1..];
|
||||
if !key.is_empty() {
|
||||
(&path[..i], Some(key))
|
||||
} else {
|
||||
(&path[..i], None)
|
||||
}
|
||||
}
|
||||
None => (path, None),
|
||||
};
|
||||
if bucket.is_empty() {
|
||||
return Err(Error::BadRequest("No bucket specified".to_string()));
|
||||
}
|
||||
Ok((bucket, key))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_containing_a_key() -> Result<(), Error> {
|
||||
let (bucket, key) = parse_bucket_key("/my_bucket/a/super/file.jpg", None)?;
|
||||
assert_eq!(bucket, "my_bucket");
|
||||
assert_eq!(key.expect("key must be set"), "a/super/file.jpg");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_containing_no_key() -> Result<(), Error> {
|
||||
let (bucket, key) = parse_bucket_key("/my_bucket/", None)?;
|
||||
assert_eq!(bucket, "my_bucket");
|
||||
assert!(key.is_none());
|
||||
let (bucket, key) = parse_bucket_key("/my_bucket", None)?;
|
||||
assert_eq!(bucket, "my_bucket");
|
||||
assert!(key.is_none());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_containing_no_bucket() {
|
||||
let parsed = parse_bucket_key("", None);
|
||||
assert!(parsed.is_err());
|
||||
let parsed = parse_bucket_key("/", None);
|
||||
assert!(parsed.is_err());
|
||||
let parsed = parse_bucket_key("////", None);
|
||||
assert!(parsed.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_with_vhost_and_key() -> Result<(), Error> {
|
||||
let (bucket, key) = parse_bucket_key("/a/super/file.jpg", Some("my-bucket"))?;
|
||||
assert_eq!(bucket, "my-bucket");
|
||||
assert_eq!(key.expect("key must be set"), "a/super/file.jpg");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_bucket_with_vhost_no_key() -> Result<(), Error> {
|
||||
let (bucket, key) = parse_bucket_key("", Some("my-bucket"))?;
|
||||
assert_eq!(bucket, "my-bucket");
|
||||
assert!(key.is_none());
|
||||
let (bucket, key) = parse_bucket_key("/", Some("my-bucket"))?;
|
||||
assert_eq!(bucket, "my-bucket");
|
||||
assert!(key.is_none());
|
||||
Ok(())
|
||||
fn add_span_attributes<'a>(&self, span: SpanRef<'a>) {
|
||||
span.set_attribute(KeyValue::new(
|
||||
"bucket",
|
||||
self.bucket_name.clone().unwrap_or_default(),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ use garage_model::s3::block_ref_table::*;
|
|||
use garage_model::s3::object_table::*;
|
||||
use garage_model::s3::version_table::*;
|
||||
|
||||
use crate::s3::api_server::{parse_bucket_key, resolve_bucket};
|
||||
use crate::error::*;
|
||||
use crate::helpers::{parse_bucket_key, resolve_bucket};
|
||||
use crate::s3::put::{decode_upload_id, get_headers};
|
||||
use crate::s3::xml::{self as s3_xml, xmlns_tag};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
mod api_server;
|
||||
pub use api_server::run_api_server;
|
||||
pub mod api_server;
|
||||
|
||||
mod bucket;
|
||||
mod copy;
|
||||
|
|
@ -9,6 +8,7 @@ pub mod get;
|
|||
mod list;
|
||||
mod post_object;
|
||||
mod put;
|
||||
mod router;
|
||||
mod website;
|
||||
|
||||
mod router;
|
||||
pub mod xml;
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ use serde::Deserialize;
|
|||
|
||||
use garage_model::garage::Garage;
|
||||
|
||||
use crate::s3::api_server::resolve_bucket;
|
||||
use crate::error::*;
|
||||
use crate::helpers::resolve_bucket;
|
||||
use crate::s3::put::{get_headers, save_stream};
|
||||
use crate::s3::xml as s3_xml;
|
||||
use crate::signature::payload::{parse_date, verify_v4};
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ use garage_util::error::Error;
|
|||
|
||||
use garage_admin::metrics::*;
|
||||
use garage_admin::tracing_setup::*;
|
||||
use garage_api::s3::run_api_server as run_s3_api_server;
|
||||
use garage_api::s3::api_server::S3ApiServer;
|
||||
use garage_model::garage::Garage;
|
||||
use garage_web::run_web_server;
|
||||
|
||||
|
|
@ -57,7 +57,7 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
|
|||
AdminRpcHandler::new(garage.clone());
|
||||
|
||||
info!("Initializing S3 API server...");
|
||||
let s3_api_server = tokio::spawn(run_s3_api_server(
|
||||
let s3_api_server = tokio::spawn(S3ApiServer::run(
|
||||
garage.clone(),
|
||||
wait_from(watch_cancel.clone()),
|
||||
));
|
||||
|
|
|
|||
|
|
@ -13,10 +13,10 @@ use garage_table::replication::TableFullReplication;
|
|||
use garage_table::replication::TableShardedReplication;
|
||||
use garage_table::*;
|
||||
|
||||
use crate::k2v::item_table::*;
|
||||
use crate::s3::block_ref_table::*;
|
||||
use crate::s3::object_table::*;
|
||||
use crate::s3::version_table::*;
|
||||
use crate::k2v::item_table::*;
|
||||
|
||||
use crate::bucket_alias_table::*;
|
||||
use crate::bucket_table::*;
|
||||
|
|
@ -151,13 +151,8 @@ impl Garage {
|
|||
);
|
||||
|
||||
// ---- K2V tables ----
|
||||
let k2v_item_table = Table::new(
|
||||
K2VItemTable{},
|
||||
meta_rep_param.clone(),
|
||||
system.clone(),
|
||||
&db,
|
||||
);
|
||||
|
||||
let k2v_item_table =
|
||||
Table::new(K2VItemTable {}, meta_rep_param.clone(), system.clone(), &db);
|
||||
|
||||
info!("Initialize Garage...");
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue