1
0
Fork 0
forked from Deuxfleurs/garage

Some work in actually storing things

This commit is contained in:
Alex 2020-04-08 22:00:41 +02:00
parent d50edcdb4f
commit bacc76a057
11 changed files with 302 additions and 110 deletions

46
Cargo.lock generated
View file

@ -5,6 +5,16 @@ name = "arc-swap"
version = "0.4.5" version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]]
name = "async-trait"
version = "0.1.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "autocfg" name = "autocfg"
version = "1.0.0" version = "1.0.0"
@ -270,6 +280,7 @@ dependencies = [
name = "garage" name = "garage"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"async-trait 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)",
"bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)",
"err-derive 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "err-derive 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -280,12 +291,12 @@ dependencies = [
"hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)",
"http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)",
"hyper 0.13.4 (registry+https://github.com/rust-lang/crates.io-index)", "hyper 0.13.4 (registry+https://github.com/rust-lang/crates.io-index)",
"kv 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)",
"rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)",
"rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)", "rmp-serde 0.14.3 (registry+https://github.com/rust-lang/crates.io-index)",
"serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)", "serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)",
"serde_bytes 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", "serde_bytes 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)",
"sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)",
"sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)",
"structopt 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", "structopt 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
@ -425,17 +436,6 @@ dependencies = [
"winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "kv"
version = "0.20.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"serde 1.0.106 (registry+https://github.com/rust-lang/crates.io-index)",
"sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)",
"thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
"toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "lazy_static" name = "lazy_static"
version = "1.4.0" version = "1.4.0"
@ -895,24 +895,6 @@ dependencies = [
"unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)",
] ]
[[package]]
name = "thiserror"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"thiserror-impl 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]]
name = "thiserror-impl"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
dependencies = [
"proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)",
"quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)",
"syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)",
]
[[package]] [[package]]
name = "time" name = "time"
version = "0.1.42" version = "0.1.42"
@ -1066,6 +1048,7 @@ dependencies = [
[metadata] [metadata]
"checksum arc-swap 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825" "checksum arc-swap 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825"
"checksum async-trait 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "da71fef07bc806586090247e971229289f64c210a278ee5ae419314eb386b31d"
"checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
"checksum bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf" "checksum bincode 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf"
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
@ -1112,7 +1095,6 @@ dependencies = [
"checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
"checksum itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e" "checksum itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e"
"checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
"checksum kv 0.20.2 (registry+https://github.com/rust-lang/crates.io-index)" = "cde1d1ce05b9169af359041cf834075ca4478f5917e6198d9454773e98aae143"
"checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
"checksum libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)" = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0" "checksum libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)" = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0"
"checksum lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" "checksum lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b"
@ -1166,8 +1148,6 @@ dependencies = [
"checksum syn-mid 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" "checksum syn-mid 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a"
"checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545"
"checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
"checksum thiserror 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "f0570dc61221295909abdb95c739f2e74325e14293b2026b0a7e195091ec54ae"
"checksum thiserror-impl 1.0.14 (registry+https://github.com/rust-lang/crates.io-index)" = "227362df41d566be41a28f64401e07a043157c21c14b9785a0d8e256f940a8fd"
"checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" "checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f"
"checksum tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "ee5a0dd887e37d37390c13ff8ac830f992307fe30a1fff0ab8427af67211ba28" "checksum tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "ee5a0dd887e37d37390c13ff8ac830f992307fe30a1fff0ab8427af67211ba28"
"checksum tokio-macros 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" "checksum tokio-macros 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389"

View file

@ -10,7 +10,7 @@ edition = "2018"
bytes = "0.4" bytes = "0.4"
http = "0.2" http = "0.2"
hyper = "0.13" hyper = "0.13"
kv = "0.20" sled = "0.31"
futures = "0.3" futures = "0.3"
futures-core = "0.3" futures-core = "0.3"
futures-channel = "0.3" futures-channel = "0.3"
@ -26,3 +26,4 @@ structopt = { version = "0.3", default-features = false }
rand = "0.7" rand = "0.7"
hex = "0.3" hex = "0.3"
sha2 = "0.8" sha2 = "0.8"
async-trait = "0.1.30"

View file

@ -9,21 +9,21 @@ use hyper::{Body, Method, Request, Response, Server, StatusCode};
use futures::future::Future; use futures::future::Future;
use crate::error::Error; use crate::error::Error;
use crate::membership::System;
use crate::data::*; use crate::data::*;
use crate::proto::*; use crate::proto::*;
use crate::rpc_client::*; use crate::rpc_client::*;
use crate::server::Garage;
pub async fn run_api_server(sys: Arc<System>, shutdown_signal: impl Future<Output=()>) -> Result<(), hyper::Error> { pub async fn run_api_server(garage: Arc<Garage>, shutdown_signal: impl Future<Output=()>) -> Result<(), hyper::Error> {
let addr = ([0, 0, 0, 0], sys.config.api_port).into(); let addr = ([0, 0, 0, 0], garage.system.config.api_port).into();
let service = make_service_fn(|conn: &AddrStream| { let service = make_service_fn(|conn: &AddrStream| {
let sys = sys.clone(); let garage = garage.clone();
let client_addr = conn.remote_addr(); let client_addr = conn.remote_addr();
async move { async move {
Ok::<_, Error>(service_fn(move |req: Request<Body>| { Ok::<_, Error>(service_fn(move |req: Request<Body>| {
let sys = sys.clone(); let garage = garage.clone();
handler(sys, req, client_addr) handler(garage, req, client_addr)
})) }))
} }
}); });
@ -36,8 +36,8 @@ pub async fn run_api_server(sys: Arc<System>, shutdown_signal: impl Future<Outpu
graceful.await graceful.await
} }
async fn handler(sys: Arc<System>, req: Request<Body>, addr: SocketAddr) -> Result<Response<Body>, Error> { async fn handler(garage: Arc<Garage>, req: Request<Body>, addr: SocketAddr) -> Result<Response<Body>, Error> {
match handler_inner(sys, req, addr).await { match handler_inner(garage, req, addr).await {
Ok(x) => Ok(x), Ok(x) => Ok(x),
Err(Error::BadRequest(e)) => { Err(Error::BadRequest(e)) => {
let mut bad_request = Response::new(Body::from(format!("{}\n", e))); let mut bad_request = Response::new(Body::from(format!("{}\n", e)));
@ -53,7 +53,7 @@ async fn handler(sys: Arc<System>, req: Request<Body>, addr: SocketAddr) -> Resu
} }
} }
async fn handler_inner(sys: Arc<System>, req: Request<Body>, addr: SocketAddr) -> Result<Response<Body>, Error> { async fn handler_inner(garage: Arc<Garage>, req: Request<Body>, addr: SocketAddr) -> Result<Response<Body>, Error> {
eprintln!("{} {} {}", addr, req.method(), req.uri()); eprintln!("{} {} {}", addr, req.method(), req.uri());
let bucket = req.headers() let bucket = req.headers()
@ -75,7 +75,7 @@ async fn handler_inner(sys: Arc<System>, req: Request<Body>, addr: SocketAddr) -
.map(|x| x.to_str()) .map(|x| x.to_str())
.unwrap_or(Ok("blob"))? .unwrap_or(Ok("blob"))?
.to_string(); .to_string();
let version_uuid = handle_put(sys, &mime_type, &bucket, &key, req.into_body()).await?; let version_uuid = handle_put(garage, &mime_type, &bucket, &key, req.into_body()).await?;
Ok(Response::new(Body::from( Ok(Response::new(Body::from(
format!("Version UUID: {:?}", version_uuid), format!("Version UUID: {:?}", version_uuid),
))) )))
@ -84,22 +84,24 @@ async fn handler_inner(sys: Arc<System>, req: Request<Body>, addr: SocketAddr) -
} }
} }
async fn handle_put(sys: Arc<System>, async fn handle_put(garage: Arc<Garage>,
mime_type: &str, mime_type: &str,
bucket: &str, key: &str, body: Body) bucket: &str, key: &str, body: Body)
-> Result<UUID, Error> -> Result<UUID, Error>
{ {
let version_uuid = gen_uuid(); let version_uuid = gen_uuid();
let mut chunker = BodyChunker::new(body, sys.config.block_size); let mut chunker = BodyChunker::new(body, garage.system.config.block_size);
let first_block = match chunker.next().await? { let first_block = match chunker.next().await? {
Some(x) => x, Some(x) => x,
None => return Err(Error::BadRequest(format!("Empty body"))), None => return Err(Error::BadRequest(format!("Empty body"))),
}; };
let mut version = VersionMeta{ let version_key = VersionMetaKey{
bucket: bucket.to_string(), bucket: bucket.to_string(),
key: key.to_string(), key: key.to_string(),
};
let mut version_value = VersionMetaValue {
timestamp: now_msec(), timestamp: now_msec(),
uuid: version_uuid.clone(), uuid: version_uuid.clone(),
mime_type: mime_type.to_string(), mime_type: mime_type.to_string(),
@ -107,27 +109,17 @@ async fn handle_put(sys: Arc<System>,
is_complete: false, is_complete: false,
data: VersionData::DeleteMarker, data: VersionData::DeleteMarker,
}; };
let version_who = sys.members.read().await
.walk_ring(&version_uuid, sys.config.meta_replication_factor);
if first_block.len() < INLINE_THRESHOLD { if first_block.len() < INLINE_THRESHOLD {
version.data = VersionData::Inline(first_block); version_value.data = VersionData::Inline(first_block);
version.is_complete = true; version_value.is_complete = true;
rpc_try_call_many(sys.clone(), garage.version_table.insert(&version_key, &version_value).await?;
&version_who[..],
&Message::AdvertiseVersion(version),
(sys.config.meta_replication_factor+1)/2,
DEFAULT_TIMEOUT).await?;
return Ok(version_uuid) return Ok(version_uuid)
} }
let first_block_hash = hash(&first_block[..]); let first_block_hash = hash(&first_block[..]);
version.data = VersionData::FirstBlock(first_block_hash); version_value.data = VersionData::FirstBlock(first_block_hash);
rpc_try_call_many(sys.clone(), garage.version_table.insert(&version_key, &version_value).await?;
&version_who[..],
&Message::AdvertiseVersion(version.clone()),
(sys.config.meta_replication_factor+1)/2,
DEFAULT_TIMEOUT).await?;
let block_meta = BlockMeta{ let block_meta = BlockMeta{
version_uuid: version_uuid.clone(), version_uuid: version_uuid.clone(),
@ -135,7 +127,7 @@ async fn handle_put(sys: Arc<System>,
hash: hash(&first_block[..]), hash: hash(&first_block[..]),
}; };
let mut next_offset = first_block.len(); let mut next_offset = first_block.len();
let mut put_curr_block = put_block(sys.clone(), block_meta, first_block); let mut put_curr_block = put_block(garage.clone(), block_meta, first_block);
loop { loop {
let (_, next_block) = futures::try_join!(put_curr_block, chunker.next())?; let (_, next_block) = futures::try_join!(put_curr_block, chunker.next())?;
if let Some(block) = next_block { if let Some(block) = next_block {
@ -145,7 +137,7 @@ async fn handle_put(sys: Arc<System>,
hash: hash(&block[..]), hash: hash(&block[..]),
}; };
next_offset += block.len(); next_offset += block.len();
put_curr_block = put_block(sys.clone(), block_meta, block); put_curr_block = put_block(garage.clone(), block_meta, block);
} else { } else {
break; break;
} }
@ -153,25 +145,21 @@ async fn handle_put(sys: Arc<System>,
// TODO: if at any step we have an error, we should undo everything we did // TODO: if at any step we have an error, we should undo everything we did
version.is_complete = true; version_value.is_complete = true;
rpc_try_call_many(sys.clone(), garage.version_table.insert(&version_key, &version_value).await?;
&version_who[..],
&Message::AdvertiseVersion(version),
(sys.config.meta_replication_factor+1)/2,
DEFAULT_TIMEOUT).await?;
Ok(version_uuid) Ok(version_uuid)
} }
async fn put_block(sys: Arc<System>, meta: BlockMeta, data: Vec<u8>) -> Result<(), Error> { async fn put_block(garage: Arc<Garage>, meta: BlockMeta, data: Vec<u8>) -> Result<(), Error> {
let who = sys.members.read().await let who = garage.system.members.read().await
.walk_ring(&meta.hash, sys.config.meta_replication_factor); .walk_ring(&meta.hash, garage.system.config.meta_replication_factor);
rpc_try_call_many(sys.clone(), rpc_try_call_many(garage.system.clone(),
&who[..], &who[..],
&Message::PutBlock(PutBlockMessage{ &Message::PutBlock(PutBlockMessage{
meta, meta,
data, data,
}), }),
(sys.config.meta_replication_factor+1)/2, (garage.system.config.meta_replication_factor+1)/2,
DEFAULT_TIMEOUT).await?; DEFAULT_TIMEOUT).await?;
Ok(()) Ok(())
} }

View file

@ -106,7 +106,7 @@ pub struct NetworkConfigEntry {
// Data management // Data management
pub const INLINE_THRESHOLD: usize = 2048; pub const INLINE_THRESHOLD: usize = 3072;
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]
pub struct SplitpointMeta { pub struct SplitpointMeta {
@ -118,27 +118,7 @@ pub struct SplitpointMeta {
pub deleted: bool, pub deleted: bool,
} }
#[derive(Clone, Debug, Serialize, Deserialize)] pub use crate::version_table::*;
pub struct VersionMeta {
pub bucket: String,
pub key: String,
pub timestamp: u64,
pub uuid: UUID,
pub mime_type: String,
pub size: u64,
pub is_complete: bool,
pub data: VersionData,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum VersionData {
DeleteMarker,
Inline(#[serde(with="serde_bytes")] Vec<u8>),
FirstBlock(Hash),
}
#[derive(Clone, Debug, Serialize, Deserialize)] #[derive(Clone, Debug, Serialize, Deserialize)]
pub struct BlockMeta { pub struct BlockMeta {

View file

@ -1,7 +1,12 @@
mod error; mod error;
mod data; mod data;
mod proto; mod proto;
mod membership; mod membership;
mod table;
mod version_table;
mod server; mod server;
mod rpc_server; mod rpc_server;
mod rpc_client; mod rpc_client;

View file

@ -124,6 +124,11 @@ impl Members {
i - 1 i - 1
} }
}; };
self.walk_ring_from_pos(start, n)
}
fn walk_ring_from_pos(&self, start: usize, n: usize) -> Vec<UUID> {
let mut ret = vec![]; let mut ret = vec![];
let mut datacenters = vec![]; let mut datacenters = vec![];

View file

@ -18,7 +18,8 @@ pub enum Message {
AdvertiseConfig(NetworkConfig), AdvertiseConfig(NetworkConfig),
PutBlock(PutBlockMessage), PutBlock(PutBlockMessage),
AdvertiseVersion(VersionMeta),
TableRPC(String, #[serde(with = "serde_bytes")] Vec<u8>),
} }
#[derive(Debug, Serialize, Deserialize)] #[derive(Debug, Serialize, Deserialize)]

View file

@ -1,6 +1,7 @@
use std::net::SocketAddr; use std::net::SocketAddr;
use std::sync::Arc; use std::sync::Arc;
use futures_util::future::FutureExt;
use bytes::IntoBuf; use bytes::IntoBuf;
use hyper::service::{make_service_fn, service_fn}; use hyper::service::{make_service_fn, service_fn};
use hyper::server::conn::AddrStream; use hyper::server::conn::AddrStream;
@ -9,7 +10,7 @@ use futures::future::Future;
use crate::error::Error; use crate::error::Error;
use crate::proto::Message; use crate::proto::Message;
use crate::membership::System; use crate::server::Garage;
fn err_to_msg(x: Result<Message, Error>) -> Message { fn err_to_msg(x: Result<Message, Error>) -> Message {
match x { match x {
@ -18,7 +19,7 @@ fn err_to_msg(x: Result<Message, Error>) -> Message {
} }
} }
async fn handler(sys: Arc<System>, req: Request<Body>, addr: SocketAddr) -> Result<Response<Body>, Error> { async fn handler(garage: Arc<Garage>, req: Request<Body>, addr: SocketAddr) -> Result<Response<Body>, Error> {
if req.method() != &Method::POST { if req.method() != &Method::POST {
let mut bad_request = Response::default(); let mut bad_request = Response::default();
*bad_request.status_mut() = StatusCode::BAD_REQUEST; *bad_request.status_mut() = StatusCode::BAD_REQUEST;
@ -30,12 +31,21 @@ async fn handler(sys: Arc<System>, req: Request<Body>, addr: SocketAddr) -> Resu
eprintln!("RPC from {}: {:?}", addr, msg); eprintln!("RPC from {}: {:?}", addr, msg);
let sys = garage.system.clone();
let resp = err_to_msg(match &msg { let resp = err_to_msg(match &msg {
Message::Ping(ping) => sys.handle_ping(&addr, ping).await, Message::Ping(ping) => sys.handle_ping(&addr, ping).await,
Message::PullStatus => sys.handle_pull_status().await, Message::PullStatus => sys.handle_pull_status().await,
Message::PullConfig => sys.handle_pull_config().await, Message::PullConfig => sys.handle_pull_config().await,
Message::AdvertiseNodesUp(adv) => sys.handle_advertise_nodes_up(adv).await, Message::AdvertiseNodesUp(adv) => sys.handle_advertise_nodes_up(adv).await,
Message::AdvertiseConfig(adv) => sys.handle_advertise_config(adv).await, Message::AdvertiseConfig(adv) => sys.handle_advertise_config(adv).await,
Message::TableRPC(table, msg) => {
if let Some(rpc_handler) = garage.table_rpc_handlers.get(table) {
rpc_handler.handle(&msg[..]).await
.map(|rep| Message::TableRPC(table.to_string(), rep))
} else {
Ok(Message::Error(format!("Unknown table: {}", table)))
}
}
_ => Ok(Message::Error(format!("Unexpected message: {:?}", msg))), _ => Ok(Message::Error(format!("Unexpected message: {:?}", msg))),
}); });
@ -46,16 +56,16 @@ async fn handler(sys: Arc<System>, req: Request<Body>, addr: SocketAddr) -> Resu
} }
pub async fn run_rpc_server(sys: Arc<System>, shutdown_signal: impl Future<Output=()>) -> Result<(), hyper::Error> { pub async fn run_rpc_server(garage: Arc<Garage>, shutdown_signal: impl Future<Output=()>) -> Result<(), hyper::Error> {
let bind_addr = ([0, 0, 0, 0], sys.config.rpc_port).into(); let bind_addr = ([0, 0, 0, 0], garage.system.config.rpc_port).into();
let service = make_service_fn(|conn: &AddrStream| { let service = make_service_fn(|conn: &AddrStream| {
let client_addr = conn.remote_addr(); let client_addr = conn.remote_addr();
let sys = sys.clone(); let garage = garage.clone();
async move { async move {
Ok::<_, Error>(service_fn(move |req: Request<Body>| { Ok::<_, Error>(service_fn(move |req: Request<Body>| {
let sys = sys.clone(); let garage = garage.clone();
handler(sys, req, client_addr) handler(garage, req, client_addr)
})) }))
} }
}); });

View file

@ -1,3 +1,4 @@
use std::collections::HashMap;
use std::io::{Read, Write}; use std::io::{Read, Write};
use std::sync::Arc; use std::sync::Arc;
use std::net::SocketAddr; use std::net::SocketAddr;
@ -6,10 +7,51 @@ use futures::channel::oneshot;
use serde::Deserialize; use serde::Deserialize;
use crate::data::*; use crate::data::*;
use crate::proto::*;
use crate::error::Error; use crate::error::Error;
use crate::membership::System; use crate::membership::System;
use crate::api_server; use crate::api_server;
use crate::rpc_server; use crate::rpc_server;
use crate::table::*;
pub struct Garage {
pub db: sled::Db,
pub system: Arc<System>,
pub table_rpc_handlers: HashMap<String, Box<dyn TableRpcHandler + Sync + Send>>,
pub version_table: Arc<Table<VersionTable>>,
}
impl Garage {
pub fn new(config: Config, id: UUID, db: sled::Db) -> Self {
let system = Arc::new(System::new(config, id));
let meta_rep_param = TableReplicationParams{
replication_factor: system.config.meta_replication_factor,
write_quorum: (system.config.meta_replication_factor+1)/2,
read_quorum: (system.config.meta_replication_factor+1)/2,
timeout: DEFAULT_TIMEOUT,
};
let version_table = Arc::new(Table::new(
system.clone(),
&db,
"version".to_string(),
meta_rep_param.clone()));
let mut garage = Self{
db,
system: system.clone(),
table_rpc_handlers: HashMap::new(),
version_table,
};
garage.table_rpc_handlers.insert(
garage.version_table.name.clone(),
garage.version_table.clone().rpc_handler());
garage
}
}
fn default_block_size() -> usize { fn default_block_size() -> usize {
1048576 1048576
@ -88,20 +130,25 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
let config = read_config(config_file) let config = read_config(config_file)
.expect("Unable to read config file"); .expect("Unable to read config file");
let mut db_path = config.metadata_dir.clone();
db_path.push("garage_metadata");
let db = sled::open(db_path)
.expect("Unable to open DB");
let id = gen_node_id(&config.metadata_dir) let id = gen_node_id(&config.metadata_dir)
.expect("Unable to read or generate node ID"); .expect("Unable to read or generate node ID");
println!("Node ID: {}", hex::encode(&id)); println!("Node ID: {}", hex::encode(&id));
let sys = Arc::new(System::new(config, id)); let garage = Arc::new(Garage::new(config, id, db));
let (tx1, rx1) = oneshot::channel(); let (tx1, rx1) = oneshot::channel();
let (tx2, rx2) = oneshot::channel(); let (tx2, rx2) = oneshot::channel();
let rpc_server = rpc_server::run_rpc_server(sys.clone(), wait_from(rx1)); let rpc_server = rpc_server::run_rpc_server(garage.clone(), wait_from(rx1));
let api_server = api_server::run_api_server(sys.clone(), wait_from(rx2)); let api_server = api_server::run_api_server(garage.clone(), wait_from(rx2));
tokio::spawn(shutdown_signal(vec![tx1, tx2])); tokio::spawn(shutdown_signal(vec![tx1, tx2]));
tokio::spawn(sys.bootstrap()); tokio::spawn(garage.system.clone().bootstrap());
futures::try_join!(rpc_server, api_server)?; futures::try_join!(rpc_server, api_server)?;
Ok(()) Ok(())

116
src/table.rs Normal file
View file

@ -0,0 +1,116 @@
use std::marker::PhantomData;
use std::time::Duration;
use std::sync::Arc;
use serde::{Serialize, Deserialize};
use async_trait::async_trait;
use crate::error::Error;
use crate::proto::*;
use crate::data::*;
use crate::membership::System;
use crate::rpc_client::*;
pub struct Table<F: TableFormat> {
phantom: PhantomData<F>,
pub name: String,
pub system: Arc<System>,
pub store: sled::Tree,
pub partitions: Vec<Partition>,
pub param: TableReplicationParams,
}
#[derive(Clone)]
pub struct TableReplicationParams {
pub replication_factor: usize,
pub read_quorum: usize,
pub write_quorum: usize,
pub timeout: Duration,
}
#[async_trait]
pub trait TableRpcHandler {
async fn handle(&self, rpc: &[u8]) -> Result<Vec<u8>, Error>;
}
struct TableRpcHandlerAdapter<F: TableFormat> {
table: Arc<Table<F>>,
}
#[async_trait]
impl<F: TableFormat + 'static> TableRpcHandler for TableRpcHandlerAdapter<F> {
async fn handle(&self, rpc: &[u8]) -> Result<Vec<u8>, Error> {
let msg = rmp_serde::decode::from_read_ref::<_, TableRPC<F>>(rpc)?;
let rep = self.table.handle(msg).await?;
Ok(rmp_serde::encode::to_vec_named(&rep)?)
}
}
#[derive(Debug, Serialize, Deserialize)]
pub enum TableRPC<F: TableFormat> {
Update(F::K, F::V),
}
pub struct Partition {
pub begin: Hash,
pub end: Hash,
pub other_nodes: Vec<UUID>,
}
pub trait KeyHash {
fn hash(&self) -> Hash;
}
pub trait ValueMerge {
fn merge(&mut self, other: &Self);
}
#[async_trait]
pub trait TableFormat: Send + Sync {
type K: Clone + Serialize + for<'de> Deserialize<'de> + KeyHash + Send + Sync;
type V: Clone + Serialize + for<'de> Deserialize<'de> + ValueMerge + Send + Sync;
async fn updated(&self, key: &Self::K, old: Option<&Self::V>, new: &Self::V);
}
impl<F: TableFormat + 'static> Table<F> {
pub fn new(system: Arc<System>, db: &sled::Db, name: String, param: TableReplicationParams) -> Self {
let store = db.open_tree(&name)
.expect("Unable to open DB tree");
Self{
phantom: PhantomData::default(),
name,
system,
store,
partitions: Vec::new(),
param,
}
}
pub fn rpc_handler(self: Arc<Self>) -> Box<dyn TableRpcHandler + Send + Sync> {
Box::new(TableRpcHandlerAdapter::<F>{ table: self })
}
pub async fn insert(&self, k: &F::K, v: &F::V) -> Result<(), Error> {
unimplemented!();
let hash = k.hash();
let who = self.system.members.read().await
.walk_ring(&hash, self.param.replication_factor);
let msg = rmp_serde::encode::to_vec_named(&TableRPC::<F>::Update(k.clone(), v.clone()))?;
rpc_try_call_many(self.system.clone(),
&who[..],
&Message::TableRPC(self.name.to_string(), msg),
self.param.write_quorum,
self.param.timeout).await?;
Ok(())
}
async fn handle(&self, msg: TableRPC<F>) -> Result<TableRPC<F>, Error> {
unimplemented!()
}
}

59
src/version_table.rs Normal file
View file

@ -0,0 +1,59 @@
use std::sync::Arc;
use serde::{Serialize, Deserialize};
use async_trait::async_trait;
use crate::data::*;
use crate::table::*;
use crate::membership::System;
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct VersionMetaKey {
pub bucket: String,
pub key: String,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct VersionMetaValue {
pub timestamp: u64,
pub uuid: UUID,
pub mime_type: String,
pub size: u64,
pub is_complete: bool,
pub data: VersionData,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub enum VersionData {
DeleteMarker,
Inline(#[serde(with="serde_bytes")] Vec<u8>),
FirstBlock(Hash),
}
pub struct VersionTable {
system: Arc<System>,
}
impl KeyHash for VersionMetaKey {
fn hash(&self) -> Hash {
hash(self.bucket.as_bytes())
}
}
impl ValueMerge for VersionMetaValue {
fn merge(&mut self, other: &Self) {
unimplemented!()
}
}
#[async_trait]
impl TableFormat for VersionTable {
type K = VersionMetaKey;
type V = VersionMetaValue;
async fn updated(&self, key: &Self::K, old: Option<&Self::V>, new: &Self::V) {
unimplemented!()
}
}