Reorganize code

This commit is contained in:
Alex 2020-04-23 17:05:46 +00:00
parent 01a8acdeec
commit c9c6b0dbd4
24 changed files with 254 additions and 208 deletions

View file

@ -5,15 +5,18 @@ use tokio::sync::watch;
use crate::data::*;
use crate::error::Error;
use crate::rpc_client::*;
use crate::rpc_server::*;
use crate::server::Garage;
use crate::table::*;
use crate::*;
use crate::block_ref_table::*;
use crate::bucket_table::*;
use crate::version_table::*;
use crate::table::*;
use crate::rpc::rpc_client::*;
use crate::rpc::rpc_server::*;
use crate::store::block_ref_table::*;
use crate::store::bucket_table::*;
use crate::store::version_table::*;
use crate::*;
pub const ADMIN_RPC_TIMEOUT: Duration = Duration::from_secs(30);
pub const ADMIN_RPC_PATH: &str = "_admin";

View file

@ -11,14 +11,16 @@ use hyper::{Body, Method, Request, Response, Server, StatusCode};
use crate::data::*;
use crate::error::Error;
use crate::http_util::*;
use crate::server::Garage;
use crate::table::EmptyKey;
use crate::block::INLINE_THRESHOLD;
use crate::block_ref_table::*;
use crate::object_table::*;
use crate::server::Garage;
use crate::version_table::*;
use crate::store::block::INLINE_THRESHOLD;
use crate::store::block_ref_table::*;
use crate::store::object_table::*;
use crate::store::version_table::*;
use crate::api::http_util::*;
type BodyType = Box<dyn HttpBody<Data = Bytes, Error = Error> + Send + Unpin>;
@ -26,7 +28,7 @@ pub async fn run_api_server(
garage: Arc<Garage>,
shutdown_signal: impl Future<Output = ()>,
) -> Result<(), Error> {
let addr = &garage.system.config.api_bind_addr;
let addr = &garage.config.api_bind_addr;
let service = make_service_fn(|conn: &AddrStream| {
let garage = garage.clone();
@ -111,7 +113,7 @@ async fn handle_put(
) -> Result<UUID, Error> {
let version_uuid = gen_uuid();
let mut chunker = BodyChunker::new(body, garage.system.config.block_size);
let mut chunker = BodyChunker::new(body, garage.config.block_size);
let first_block = match chunker.next().await? {
Some(x) => x,
None => return Err(Error::BadRequest(format!("Empty body"))),

2
src/api/mod.rs Normal file
View file

@ -0,0 +1,2 @@
pub mod api_server;
pub mod http_util;

66
src/config.rs Normal file
View file

@ -0,0 +1,66 @@
use std::io::Read;
use std::net::SocketAddr;
use std::path::PathBuf;
use serde::Deserialize;
use crate::error::Error;
#[derive(Deserialize, Debug, Clone)]
pub struct Config {
pub metadata_dir: PathBuf,
pub data_dir: PathBuf,
pub api_bind_addr: SocketAddr,
pub rpc_bind_addr: SocketAddr,
pub bootstrap_peers: Vec<SocketAddr>,
#[serde(default = "default_max_concurrent_rpc_requests")]
pub max_concurrent_rpc_requests: usize,
#[serde(default = "default_block_size")]
pub block_size: usize,
#[serde(default = "default_replication_factor")]
pub meta_replication_factor: usize,
#[serde(default = "default_epidemic_factor")]
pub meta_epidemic_factor: usize,
#[serde(default = "default_replication_factor")]
pub data_replication_factor: usize,
pub rpc_tls: Option<TlsConfig>,
}
fn default_max_concurrent_rpc_requests() -> usize {
12
}
fn default_block_size() -> usize {
1048576
}
fn default_replication_factor() -> usize {
3
}
fn default_epidemic_factor() -> usize {
3
}
#[derive(Deserialize, Debug, Clone)]
pub struct TlsConfig {
pub ca_cert: String,
pub node_cert: String,
pub node_key: String,
}
pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
let mut file = std::fs::OpenOptions::new()
.read(true)
.open(config_file.as_path())?;
let mut config = String::new();
file.read_to_string(&mut config)?;
Ok(toml::from_str(&config)?)
}

View file

@ -3,7 +3,7 @@ use hyper::StatusCode;
use std::io;
use crate::data::Hash;
use crate::rpc_client::RPCError;
use crate::rpc::rpc_client::RPCError;
#[derive(Debug, Error)]
pub enum Error {

View file

@ -3,29 +3,18 @@
#[macro_use]
extern crate log;
mod background;
mod config;
mod data;
mod error;
mod background;
mod membership;
mod api;
mod rpc;
mod store;
mod table;
mod table_fullcopy;
mod table_sharded;
mod table_sync;
mod block;
mod block_ref_table;
mod bucket_table;
mod object_table;
mod version_table;
mod admin_rpc;
mod api_server;
mod http_util;
mod rpc_client;
mod rpc_server;
mod server;
mod tls_util;
use std::collections::HashSet;
use std::net::SocketAddr;
@ -36,11 +25,12 @@ use std::time::Duration;
use serde::{Deserialize, Serialize};
use structopt::StructOpt;
use config::TlsConfig;
use data::*;
use error::Error;
use membership::*;
use rpc_client::*;
use server::TlsConfig;
use rpc::membership::*;
use rpc::rpc_client::*;
use admin_rpc::*;

View file

@ -1,7 +1,7 @@
use std::collections::HashMap;
use std::hash::Hash as StdHash;
use std::hash::Hasher;
use std::io::Read;
use std::io::{Read, Write};
use std::net::{IpAddr, SocketAddr};
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
@ -20,9 +20,9 @@ use tokio::sync::Mutex;
use crate::background::BackgroundRunner;
use crate::data::*;
use crate::error::Error;
use crate::rpc_client::*;
use crate::rpc_server::*;
use crate::server::Config;
use crate::rpc::rpc_client::*;
use crate::rpc::rpc_server::*;
const PING_INTERVAL: Duration = Duration::from_secs(10);
const PING_TIMEOUT: Duration = Duration::from_secs(2);
@ -78,8 +78,9 @@ pub struct NetworkConfigEntry {
}
pub struct System {
pub config: Config,
pub id: UUID,
pub data_dir: PathBuf,
pub rpc_local_port: u16,
pub state_info: StateInfo,
@ -251,6 +252,29 @@ impl Ring {
}
}
fn gen_node_id(metadata_dir: &PathBuf) -> Result<UUID, Error> {
let mut id_file = metadata_dir.clone();
id_file.push("node_id");
if id_file.as_path().exists() {
let mut f = std::fs::File::open(id_file.as_path())?;
let mut d = vec![];
f.read_to_end(&mut d)?;
if d.len() != 32 {
return Err(Error::Message(format!("Corrupt node_id file")));
}
let mut id = [0u8; 32];
id.copy_from_slice(&d[..]);
Ok(id.into())
} else {
let id = gen_uuid();
let mut f = std::fs::File::create(id_file.as_path())?;
f.write_all(id.as_slice())?;
Ok(id)
}
}
fn read_network_config(metadata_dir: &PathBuf) -> Result<NetworkConfig, Error> {
let mut path = metadata_dir.clone();
path.push("network_config");
@ -270,12 +294,15 @@ fn read_network_config(metadata_dir: &PathBuf) -> Result<NetworkConfig, Error> {
impl System {
pub fn new(
config: Config,
id: UUID,
data_dir: PathBuf,
rpc_http_client: Arc<RpcHttpClient>,
background: Arc<BackgroundRunner>,
rpc_server: &mut RpcServer,
) -> Arc<Self> {
let net_config = match read_network_config(&config.metadata_dir) {
let id = gen_node_id(&data_dir).expect("Unable to read or generate node ID");
info!("Node ID: {}", hex::encode(&id));
let net_config = match read_network_config(&data_dir) {
Ok(x) => x,
Err(e) => {
info!(
@ -309,11 +336,6 @@ impl System {
ring.rebuild_ring();
let (update_ring, ring) = watch::channel(Arc::new(ring));
let rpc_http_client = Arc::new(
RpcHttpClient::new(config.max_concurrent_rpc_requests, &config.rpc_tls)
.expect("Could not create RPC client"),
);
let rpc_path = MEMBERSHIP_RPC_PATH.to_string();
let rpc_client = RpcClient::new(
RpcAddrClient::<Message>::new(rpc_http_client.clone(), rpc_path.clone()),
@ -322,8 +344,9 @@ impl System {
);
let sys = Arc::new(System {
config,
id,
data_dir,
rpc_local_port: rpc_server.bind_addr.port(),
state_info,
rpc_http_client,
rpc_client,
@ -363,7 +386,7 @@ impl System {
}
async fn save_network_config(self: Arc<Self>) -> Result<(), Error> {
let mut path = self.config.metadata_dir.clone();
let mut path = self.data_dir.clone();
path.push("network_config");
let ring = self.ring.borrow().clone();
@ -379,7 +402,7 @@ impl System {
let ring = self.ring.borrow().clone();
Message::Ping(PingMessage {
id: self.id,
rpc_port: self.config.rpc_bind_addr.port(),
rpc_port: self.rpc_local_port,
status_hash: status.hash,
config_version: ring.config.version,
state_info: self.state_info.clone(),
@ -397,13 +420,8 @@ impl System {
self.rpc_client.call_many(&to[..], msg, timeout).await;
}
pub async fn bootstrap(self: Arc<Self>) {
let bootstrap_peers = self
.config
.bootstrap_peers
.iter()
.map(|ip| (*ip, None))
.collect::<Vec<_>>();
pub async fn bootstrap(self: Arc<Self>, peers: &[SocketAddr]) {
let bootstrap_peers = peers.iter().map(|ip| (*ip, None)).collect::<Vec<_>>();
self.clone().ping_nodes(bootstrap_peers).await;
self.clone()
@ -557,7 +575,7 @@ impl System {
for node in adv.iter() {
if node.id == self.id {
// learn our own ip address
let self_addr = SocketAddr::new(node.addr.ip(), self.config.rpc_bind_addr.port());
let self_addr = SocketAddr::new(node.addr.ip(), self.rpc_local_port);
let old_self = status.nodes.insert(
node.id,
Arc::new(StatusEntry {

4
src/rpc/mod.rs Normal file
View file

@ -0,0 +1,4 @@
pub mod membership;
pub mod rpc_client;
pub mod rpc_server;
pub mod tls_util;

View file

@ -20,10 +20,12 @@ use tokio::sync::{watch, Semaphore};
use crate::background::BackgroundRunner;
use crate::data::*;
use crate::error::Error;
use crate::membership::Status;
use crate::rpc_server::RpcMessage;
use crate::server::TlsConfig;
use crate::tls_util;
use crate::rpc::membership::Status;
use crate::rpc::rpc_server::RpcMessage;
use crate::rpc::tls_util;
use crate::config::TlsConfig;
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);

View file

@ -16,10 +16,11 @@ use tokio::net::{TcpListener, TcpStream};
use tokio_rustls::server::TlsStream;
use tokio_rustls::TlsAcceptor;
use crate::config::TlsConfig;
use crate::data::*;
use crate::error::Error;
use crate::server::TlsConfig;
use crate::tls_util;
use crate::rpc::tls_util;
pub trait RpcMessage: Serialize + for<'de> Deserialize<'de> + Send + Sync {}

View file

@ -1,79 +1,34 @@
use std::io::{Read, Write};
use std::net::SocketAddr;
use std::path::PathBuf;
use std::sync::Arc;
use futures_util::future::*;
use serde::Deserialize;
use tokio::sync::watch;
use crate::background::*;
use crate::data::*;
use crate::config::*;
use crate::error::Error;
use crate::membership::System;
use crate::rpc_server::RpcServer;
use crate::table::*;
use crate::table_fullcopy::*;
use crate::table_sharded::*;
use crate::block::*;
use crate::block_ref_table::*;
use crate::bucket_table::*;
use crate::object_table::*;
use crate::version_table::*;
use crate::rpc::membership::System;
use crate::rpc::rpc_client::RpcHttpClient;
use crate::rpc::rpc_server::RpcServer;
use crate::table::table_fullcopy::*;
use crate::table::table_sharded::*;
use crate::table::*;
use crate::store::block::*;
use crate::store::block_ref_table::*;
use crate::store::bucket_table::*;
use crate::store::object_table::*;
use crate::store::version_table::*;
use crate::api::api_server;
use crate::admin_rpc::*;
use crate::api_server;
#[derive(Deserialize, Debug, Clone)]
pub struct Config {
pub metadata_dir: PathBuf,
pub data_dir: PathBuf,
pub api_bind_addr: SocketAddr,
pub rpc_bind_addr: SocketAddr,
pub bootstrap_peers: Vec<SocketAddr>,
#[serde(default = "default_max_concurrent_rpc_requests")]
pub max_concurrent_rpc_requests: usize,
#[serde(default = "default_block_size")]
pub block_size: usize,
#[serde(default = "default_replication_factor")]
pub meta_replication_factor: usize,
#[serde(default = "default_epidemic_factor")]
pub meta_epidemic_factor: usize,
#[serde(default = "default_replication_factor")]
pub data_replication_factor: usize,
pub rpc_tls: Option<TlsConfig>,
}
fn default_max_concurrent_rpc_requests() -> usize {
12
}
fn default_block_size() -> usize {
1048576
}
fn default_replication_factor() -> usize {
3
}
fn default_epidemic_factor() -> usize {
3
}
#[derive(Deserialize, Debug, Clone)]
pub struct TlsConfig {
pub ca_cert: String,
pub node_cert: String,
pub node_key: String,
}
pub struct Garage {
pub config: Config,
pub db: sled::Db,
pub background: Arc<BackgroundRunner>,
pub system: Arc<System>,
@ -88,33 +43,46 @@ pub struct Garage {
impl Garage {
pub async fn new(
config: Config,
id: UUID,
db: sled::Db,
background: Arc<BackgroundRunner>,
rpc_server: &mut RpcServer,
) -> Arc<Self> {
info!("Initialize membership management system...");
let system = System::new(config.clone(), id, background.clone(), rpc_server);
info!("Initialize block manager...");
let block_manager =
BlockManager::new(&db, config.data_dir.clone(), system.clone(), rpc_server);
let rpc_http_client = Arc::new(
RpcHttpClient::new(config.max_concurrent_rpc_requests, &config.rpc_tls)
.expect("Could not create RPC client"),
);
let system = System::new(
config.metadata_dir.clone(),
rpc_http_client,
background.clone(),
rpc_server,
);
let data_rep_param = TableShardedReplication {
replication_factor: system.config.data_replication_factor,
write_quorum: (system.config.data_replication_factor + 1) / 2,
replication_factor: config.data_replication_factor,
write_quorum: (config.data_replication_factor + 1) / 2,
read_quorum: 1,
};
let meta_rep_param = TableShardedReplication {
replication_factor: system.config.meta_replication_factor,
write_quorum: (system.config.meta_replication_factor + 1) / 2,
read_quorum: (system.config.meta_replication_factor + 1) / 2,
replication_factor: config.meta_replication_factor,
write_quorum: (config.meta_replication_factor + 1) / 2,
read_quorum: (config.meta_replication_factor + 1) / 2,
};
let control_rep_param = TableFullReplication::new(
system.config.meta_epidemic_factor,
(system.config.meta_epidemic_factor + 1) / 2,
config.meta_epidemic_factor,
(config.meta_epidemic_factor + 1) / 2,
);
info!("Initialize block manager...");
let block_manager = BlockManager::new(
&db,
config.data_dir.clone(),
data_rep_param.clone(),
system.clone(),
rpc_server,
);
info!("Initialize block_ref_table...");
@ -172,6 +140,7 @@ impl Garage {
info!("Initialize Garage...");
let garage = Arc::new(Self {
config,
db,
system: system.clone(),
block_manager,
@ -193,40 +162,6 @@ impl Garage {
}
}
fn read_config(config_file: PathBuf) -> Result<Config, Error> {
let mut file = std::fs::OpenOptions::new()
.read(true)
.open(config_file.as_path())?;
let mut config = String::new();
file.read_to_string(&mut config)?;
Ok(toml::from_str(&config)?)
}
fn gen_node_id(metadata_dir: &PathBuf) -> Result<UUID, Error> {
let mut id_file = metadata_dir.clone();
id_file.push("node_id");
if id_file.as_path().exists() {
let mut f = std::fs::File::open(id_file.as_path())?;
let mut d = vec![];
f.read_to_end(&mut d)?;
if d.len() != 32 {
return Err(Error::Message(format!("Corrupt node_id file")));
}
let mut id = [0u8; 32];
id.copy_from_slice(&d[..]);
Ok(id.into())
} else {
let id = gen_uuid();
let mut f = std::fs::File::create(id_file.as_path())?;
f.write_all(id.as_slice())?;
Ok(id)
}
}
async fn shutdown_signal(send_cancel: watch::Sender<bool>) -> Result<(), Error> {
// Wait for the CTRL+C signal
tokio::signal::ctrl_c()
@ -249,9 +184,6 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
info!("Loading configuration...");
let config = read_config(config_file).expect("Unable to read config file");
let id = gen_node_id(&config.metadata_dir).expect("Unable to read or generate node ID");
info!("Node ID: {}", hex::encode(&id));
info!("Opening database...");
let mut db_path = config.metadata_dir.clone();
db_path.push("db");
@ -264,17 +196,21 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
let (send_cancel, watch_cancel) = watch::channel(false);
let background = BackgroundRunner::new(16, watch_cancel.clone());
let garage = Garage::new(config, id, db, background.clone(), &mut rpc_server).await;
let garage = Garage::new(config, db, background.clone(), &mut rpc_server).await;
info!("Initializing RPC and API servers...");
let run_rpc_server = Arc::new(rpc_server).run(wait_from(watch_cancel.clone()));
let api_server = api_server::run_api_server(garage.clone(), wait_from(watch_cancel.clone()));
futures::try_join!(
garage.system.clone().bootstrap().map(|rv| {
info!("Bootstrap done");
Ok(rv)
}),
garage
.system
.clone()
.bootstrap(&garage.config.bootstrap_peers[..])
.map(|rv| {
info!("Bootstrap done");
Ok(rv)
}),
run_rpc_server.map(|rv| {
info!("RPC server exited");
rv

View file

@ -14,11 +14,16 @@ use tokio::sync::{watch, Mutex, Notify};
use crate::data;
use crate::data::*;
use crate::error::Error;
use crate::membership::System;
use crate::rpc_client::*;
use crate::rpc_server::*;
use crate::block_ref_table::*;
use crate::rpc::membership::System;
use crate::rpc::rpc_client::*;
use crate::rpc::rpc_server::*;
use crate::table::table_sharded::TableShardedReplication;
use crate::table::TableReplication;
use crate::store::block_ref_table::*;
use crate::server::Garage;
pub const INLINE_THRESHOLD: usize = 3072;
@ -47,6 +52,7 @@ pub struct PutBlockMessage {
impl RpcMessage for Message {}
pub struct BlockManager {
pub replication: TableShardedReplication,
pub data_dir: PathBuf,
pub data_dir_lock: Mutex<()>,
@ -64,6 +70,7 @@ impl BlockManager {
pub fn new(
db: &sled::Db,
data_dir: PathBuf,
replication: TableShardedReplication,
system: Arc<System>,
rpc_server: &mut RpcServer,
) -> Arc<Self> {
@ -80,6 +87,7 @@ impl BlockManager {
let rpc_client = system.rpc_client::<Message>(rpc_path);
let block_manager = Arc::new(Self {
replication,
data_dir,
data_dir_lock: Mutex::new(()),
rc,
@ -302,8 +310,8 @@ impl BlockManager {
.await?;
let needed_by_others = !active_refs.is_empty();
if needed_by_others {
let ring = garage.system.ring.borrow().clone();
let who = ring.walk_ring(&hash, garage.system.config.data_replication_factor);
let ring = self.system.ring.borrow().clone();
let who = self.replication.replication_nodes(&hash, &ring);
let msg = Arc::new(Message::NeedBlockQuery(*hash));
let who_needs_fut = who.iter().map(|to| {
self.rpc_client
@ -361,8 +369,7 @@ impl BlockManager {
}
pub async fn rpc_get_block(&self, hash: &Hash) -> Result<Vec<u8>, Error> {
let ring = self.system.ring.borrow().clone();
let who = ring.walk_ring(&hash, self.system.config.data_replication_factor);
let who = self.replication.read_nodes(&hash, &self.system);
let resps = self
.rpc_client
.try_call_many(
@ -386,13 +393,12 @@ impl BlockManager {
}
pub async fn rpc_put_block(&self, hash: Hash, data: Vec<u8>) -> Result<(), Error> {
let ring = self.system.ring.borrow().clone();
let who = ring.walk_ring(&hash, self.system.config.data_replication_factor);
let who = self.replication.write_nodes(&hash, &self.system);
self.rpc_client
.try_call_many(
&who[..],
Message::PutBlock(PutBlockMessage { hash, data }),
RequestStrategy::with_quorum((self.system.config.data_replication_factor + 1) / 2)
RequestStrategy::with_quorum(self.replication.write_quorum())
.with_timeout(BLOCK_RW_TIMEOUT),
)
.await?;

View file

@ -5,9 +5,10 @@ use std::sync::Arc;
use crate::background::*;
use crate::data::*;
use crate::error::Error;
use crate::table::*;
use crate::block::*;
use crate::store::block::*;
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct BlockRef {

5
src/store/mod.rs Normal file
View file

@ -0,0 +1,5 @@
pub mod block;
pub mod block_ref_table;
pub mod bucket_table;
pub mod object_table;
pub mod version_table;

View file

@ -5,10 +5,11 @@ use std::sync::Arc;
use crate::background::BackgroundRunner;
use crate::data::*;
use crate::error::Error;
use crate::table::*;
use crate::table_sharded::*;
use crate::version_table::*;
use crate::table::table_sharded::*;
use crate::table::*;
use crate::store::version_table::*;
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct Object {

View file

@ -5,10 +5,11 @@ use std::sync::Arc;
use crate::background::BackgroundRunner;
use crate::data::*;
use crate::error::Error;
use crate::table::*;
use crate::table_sharded::*;
use crate::block_ref_table::*;
use crate::table::table_sharded::*;
use crate::table::*;
use crate::store::block_ref_table::*;
#[derive(PartialEq, Clone, Debug, Serialize, Deserialize)]
pub struct Version {

6
src/table/mod.rs Normal file
View file

@ -0,0 +1,6 @@
pub mod table;
pub mod table_fullcopy;
pub mod table_sharded;
pub mod table_sync;
pub use table::*;

View file

@ -10,10 +10,12 @@ use serde_bytes::ByteBuf;
use crate::data::*;
use crate::error::Error;
use crate::membership::{Ring, System};
use crate::rpc_client::*;
use crate::rpc_server::*;
use crate::table_sync::*;
use crate::rpc::membership::{Ring, System};
use crate::rpc::rpc_client::*;
use crate::rpc::rpc_server::*;
use crate::table::table_sync::*;
const TABLE_RPC_TIMEOUT: Duration = Duration::from_secs(10);

View file

@ -2,7 +2,7 @@ use arc_swap::ArcSwapOption;
use std::sync::Arc;
use crate::data::*;
use crate::membership::{Ring, System};
use crate::rpc::membership::{Ring, System};
use crate::table::*;
#[derive(Clone)]

View file

@ -1,5 +1,5 @@
use crate::data::*;
use crate::membership::{Ring, System};
use crate::rpc::membership::{Ring, System};
use crate::table::*;
#[derive(Clone)]

View file

@ -14,7 +14,7 @@ use tokio::sync::{mpsc, watch};
use crate::data::*;
use crate::error::Error;
use crate::membership::Ring;
use crate::rpc::membership::Ring;
use crate::table::*;
const MAX_DEPTH: usize = 16;