Deuxfleurs/garage
48
56
Fork 73
Code Issues 99 Pull requests 13 Releases 30 Wiki Activity

K2V #293

Merged
lx merged 68 commits from k2v into main 2022-05-10 11:16:58 +00:00
Conversation 0 Commits 68 Files changed 83 +79 -59
18 changed files with 79 additions and 59 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 6c990b2bf2 - Show all commits

2
src/api/error.rs
View file

@ -7,7 +7,7 @@ use hyper::{HeaderMap, StatusCode};
use garage_model::helper::error::Error as HelperError; use garage_model::helper::error::Error as HelperError;
use garage_util::error::Error as GarageError; use garage_util::error::Error as GarageError;
use crate::s3_xml; use crate::s3::xml as s3_xml;
/// Errors of this crate /// Errors of this crate
#[derive(Debug, Error)] #[derive(Debug, Error)]

18
src/api/lib.rs
View file

@ -4,24 +4,12 @@ extern crate tracing;
pub mod error; pub mod error;
pub use error::Error; pub use error::Error;
pub mod helpers;
mod encoding; mod encoding;
mod api_server;
pub use api_server::run_api_server;
/// This mode is public only to help testing. Don't expect stability here /// This mode is public only to help testing. Don't expect stability here
pub mod signature; pub mod signature;
pub mod helpers; pub mod s3;
mod s3_bucket;
mod s3_copy;
pub mod s3_cors;
mod s3_delete;
pub mod s3_get;
mod s3_list;
mod s3_post_object;
mod s3_put;
mod s3_router;
mod s3_website;
mod s3_xml;

20
src/api/api_server.rs → src/api/s3/api_server.rs
View file

@ -32,16 +32,16 @@ use crate::signature::streaming::SignedPayloadStream;
use crate::signature::LONG_DATETIME; use crate::signature::LONG_DATETIME;
use crate::helpers::*; use crate::helpers::*;
use crate::s3_bucket::*; use crate::s3::bucket::*;
use crate::s3_copy::*; use crate::s3::copy::*;
use crate::s3_cors::*; use crate::s3::cors::*;
use crate::s3_delete::*; use crate::s3::delete::*;
use crate::s3_get::*; use crate::s3::get::*;
use crate::s3_list::*; use crate::s3::list::*;
use crate::s3_post_object::handle_post_object; use crate::s3::post_object::handle_post_object;
use crate::s3_put::*; use crate::s3::put::*;
use crate::s3_router::{Authorization, Endpoint}; use crate::s3::router::{Authorization, Endpoint};
use crate::s3_website::*; use crate::s3::website::*;
struct ApiMetrics { struct ApiMetrics {
request_counter: Counter<u64>, request_counter: Counter<u64>,

2
src/api/s3_bucket.rs → src/api/s3/bucket.rs
View file

@ -15,7 +15,7 @@ use garage_util::data::*;
use garage_util::time::*; use garage_util::time::*;
use crate::error::*; use crate::error::*;
use crate::s3_xml; use crate::s3::xml as s3_xml;
use crate::signature::verify_signed_content; use crate::signature::verify_signed_content;
pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>, Error> { pub fn handle_get_bucket_location(garage: Arc<Garage>) -> Result<Response<Body>, Error> {

6
src/api/s3_copy.rs → src/api/s3/copy.rs
View file

@ -17,10 +17,10 @@ use garage_model::s3::block_ref_table::*;
use garage_model::s3::object_table::*; use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*; use garage_model::s3::version_table::*;
use crate::api_server::{parse_bucket_key, resolve_bucket}; use crate::s3::api_server::{parse_bucket_key, resolve_bucket};
use crate::error::*; use crate::error::*;
use crate::s3_put::{decode_upload_id, get_headers}; use crate::s3::put::{decode_upload_id, get_headers};
use crate::s3_xml::{self, xmlns_tag}; use crate::s3::xml::{self as s3_xml, xmlns_tag};
pub async fn handle_copy( pub async fn handle_copy(
garage: Arc<Garage>, garage: Arc<Garage>,

2
src/api/s3_cors.rs → src/api/s3/cors.rs
View file

@ -10,7 +10,7 @@ use hyper::{header::HeaderName, Body, Method, Request, Response, StatusCode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::error::*; use crate::error::*;
use crate::s3_xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
use crate::signature::verify_signed_content; use crate::signature::verify_signed_content;
use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule}; use garage_model::bucket_table::{Bucket, CorsRule as GarageCorsRule};

2
src/api/s3_delete.rs → src/api/s3/delete.rs
View file

@ -9,7 +9,7 @@ use garage_model::garage::Garage;
use garage_model::s3::object_table::*; use garage_model::s3::object_table::*;
use crate::error::*; use crate::error::*;
use crate::s3_xml; use crate::s3::xml as s3_xml;
use crate::signature::verify_signed_content; use crate::signature::verify_signed_content;
async fn handle_delete_internal( async fn handle_delete_internal(

0
src/api/s3_get.rs → src/api/s3/get.rs
View file

4
src/api/s3_list.rs → src/api/s3/list.rs
View file

@ -17,8 +17,8 @@ use garage_table::EmptyKey;
use crate::encoding::*; use crate::encoding::*;
use crate::error::*; use crate::error::*;
use crate::s3_put; use crate::s3::put as s3_put;
use crate::s3_xml; use crate::s3::xml as s3_xml;
const DUMMY_NAME: &str = "Dummy Key"; const DUMMY_NAME: &str = "Dummy Key";
const DUMMY_KEY: &str = "GKDummyKey"; const DUMMY_KEY: &str = "GKDummyKey";

14
src/api/s3/mod.rs Normal file
View file

@ -0,0 +1,14 @@
mod api_server;
pub use api_server::run_api_server;
mod bucket;
mod copy;
pub mod cors;
mod delete;
pub mod get;
mod list;
mod post_object;
mod put;
mod router;
mod website;
pub mod xml;

6
src/api/s3_post_object.rs → src/api/s3/post_object.rs
View file

@ -14,10 +14,10 @@ use serde::Deserialize;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use crate::api_server::resolve_bucket; use crate::s3::api_server::resolve_bucket;
use crate::error::*; use crate::error::*;
use crate::s3_put::{get_headers, save_stream}; use crate::s3::put::{get_headers, save_stream};
use crate::s3_xml; use crate::s3::xml as s3_xml;
use crate::signature::payload::{parse_date, verify_v4}; use crate::signature::payload::{parse_date, verify_v4};
pub async fn handle_post_object( pub async fn handle_post_object(

2
src/api/s3_put.rs → src/api/s3/put.rs
View file

@ -20,7 +20,7 @@ use garage_model::s3::object_table::*;
use garage_model::s3::version_table::*; use garage_model::s3::version_table::*;
use crate::error::*; use crate::error::*;
use crate::s3_xml; use crate::s3::xml as s3_xml;
use crate::signature::verify_signed_content; use crate::signature::verify_signed_content;
pub async fn handle_put( pub async fn handle_put(

0
src/api/s3_router.rs → src/api/s3/router.rs
View file

2
src/api/s3_website.rs → src/api/s3/website.rs
View file

@ -5,7 +5,7 @@ use hyper::{Body, Request, Response, StatusCode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use crate::error::*; use crate::error::*;
use crate::s3_xml::{to_xml_with_header, xmlns_tag, IntValue, Value}; use crate::s3::xml::{to_xml_with_header, xmlns_tag, IntValue, Value};
use crate::signature::verify_signed_content; use crate::signature::verify_signed_content;
use garage_model::bucket_table::*; use garage_model::bucket_table::*;

0
src/api/s3_xml.rs → src/api/s3/xml.rs
View file

10
src/garage/server.rs
View file

@ -8,7 +8,7 @@ use garage_util::error::Error;
use garage_admin::metrics::*; use garage_admin::metrics::*;
use garage_admin::tracing_setup::*; use garage_admin::tracing_setup::*;
use garage_api::run_api_server; use garage_api::s3::run_api_server as run_s3_api_server;
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_web::run_web_server; use garage_web::run_web_server;
@ -56,8 +56,8 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
info!("Create admin RPC handler..."); info!("Create admin RPC handler...");
AdminRpcHandler::new(garage.clone()); AdminRpcHandler::new(garage.clone());
info!("Initializing API server..."); info!("Initializing S3 API server...");
let api_server = tokio::spawn(run_api_server( let s3_api_server = tokio::spawn(run_s3_api_server(
garage.clone(), garage.clone(),
wait_from(watch_cancel.clone()), wait_from(watch_cancel.clone()),
)); ));
@ -80,8 +80,8 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
// Stuff runs // Stuff runs
// When a cancel signal is sent, stuff stops // When a cancel signal is sent, stuff stops
if let Err(e) = api_server.await? { if let Err(e) = s3_api_server.await? {
warn!("API server exited with error: {}", e); warn!("S3 API server exited with error: {}", e);
} }
if let Err(e) = web_server.await? { if let Err(e) = web_server.await? {
warn!("Web server exited with error: {}", e); warn!("Web server exited with error: {}", e);

44
src/model/garage.rs
View file

@ -16,6 +16,7 @@ use garage_table::*;
use crate::s3::block_ref_table::*; use crate::s3::block_ref_table::*;
use crate::s3::object_table::*; use crate::s3::object_table::*;
use crate::s3::version_table::*; use crate::s3::version_table::*;
use crate::k2v::item_table::*;
use crate::bucket_alias_table::*; use crate::bucket_alias_table::*;
use crate::bucket_table::*; use crate::bucket_table::*;
@ -36,16 +37,22 @@ pub struct Garage {
/// The block manager /// The block manager
pub block_manager: Arc<BlockManager>, pub block_manager: Arc<BlockManager>,
/// Table containing informations about buckets /// Table containing buckets
pub bucket_table: Arc<Table<BucketTable, TableFullReplication>>, pub bucket_table: Arc<Table<BucketTable, TableFullReplication>>,
/// Table containing informations about bucket aliases /// Table containing bucket aliases
pub bucket_alias_table: Arc<Table<BucketAliasTable, TableFullReplication>>, pub bucket_alias_table: Arc<Table<BucketAliasTable, TableFullReplication>>,
/// Table containing informations about api keys /// Table containing api keys
pub key_table: Arc<Table<KeyTable, TableFullReplication>>, pub key_table: Arc<Table<KeyTable, TableFullReplication>>,
/// Table containing S3 objects
pub object_table: Arc<Table<ObjectTable, TableShardedReplication>>, pub object_table: Arc<Table<ObjectTable, TableShardedReplication>>,
/// Table containing S3 object versions
pub version_table: Arc<Table<VersionTable, TableShardedReplication>>, pub version_table: Arc<Table<VersionTable, TableShardedReplication>>,
/// Table containing S3 block references (not blocks themselves)
pub block_ref_table: Arc<Table<BlockRefTable, TableShardedReplication>>, pub block_ref_table: Arc<Table<BlockRefTable, TableShardedReplication>>,
/// Table containing K2V items
pub k2v_item_table: Arc<Table<K2VItemTable, TableShardedReplication>>,
} }
impl Garage { impl Garage {
@ -96,6 +103,21 @@ impl Garage {
system.clone(), system.clone(),
); );
// ---- admin tables ----
info!("Initialize bucket_table...");
let bucket_table = Table::new(BucketTable, control_rep_param.clone(), system.clone(), &db);
info!("Initialize bucket_alias_table...");
let bucket_alias_table = Table::new(
BucketAliasTable,
control_rep_param.clone(),
system.clone(),
&db,
);
info!("Initialize key_table_table...");
let key_table = Table::new(KeyTable, control_rep_param, system.clone(), &db);
// ---- S3 tables ----
info!("Initialize block_ref_table..."); info!("Initialize block_ref_table...");
let block_ref_table = Table::new( let block_ref_table = Table::new(
BlockRefTable { BlockRefTable {
@ -123,24 +145,19 @@ impl Garage {
background: background.clone(), background: background.clone(),
version_table: version_table.clone(), version_table: version_table.clone(),
}, },
meta_rep_param, meta_rep_param.clone(),
system.clone(), system.clone(),
&db, &db,
); );
info!("Initialize bucket_table..."); // ---- K2V tables ----
let bucket_table = Table::new(BucketTable, control_rep_param.clone(), system.clone(), &db); let k2v_item_table = Table::new(
K2VItemTable{},
info!("Initialize bucket_alias_table..."); meta_rep_param.clone(),
let bucket_alias_table = Table::new(
BucketAliasTable,
control_rep_param.clone(),
system.clone(), system.clone(),
&db, &db,
); );
info!("Initialize key_table_table...");
let key_table = Table::new(KeyTable, control_rep_param, system.clone(), &db);
info!("Initialize Garage..."); info!("Initialize Garage...");
@ -156,6 +173,7 @@ impl Garage {
object_table, object_table,
version_table, version_table,
block_ref_table, block_ref_table,
k2v_item_table,
}) })
} }

4
src/web/web_server.rs
View file

@ -20,8 +20,8 @@ use crate::error::*;
use garage_api::error::{Error as ApiError, OkOrBadRequest, OkOrInternalError}; use garage_api::error::{Error as ApiError, OkOrBadRequest, OkOrInternalError};
use garage_api::helpers::{authority_to_host, host_to_bucket}; use garage_api::helpers::{authority_to_host, host_to_bucket};
use garage_api::s3_cors::{add_cors_headers, find_matching_cors_rule, handle_options_for_bucket}; use garage_api::s3::cors::{add_cors_headers, find_matching_cors_rule, handle_options_for_bucket};
use garage_api::s3_get::{handle_get, handle_head}; use garage_api::s3::get::{handle_get, handle_head};
use garage_model::garage::Garage; use garage_model::garage::Garage;