diff --git a/script/dev-bucket.sh b/script/dev-bucket.sh index 05d8c105..438d934d 100755 --- a/script/dev-bucket.sh +++ b/script/dev-bucket.sh @@ -9,11 +9,11 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/" NIX_RELEASE="${REPO_FOLDER}/result/bin/" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" -garage bucket create eprouvette -KEY_INFO=`garage key new --name opérateur` +garage -c /tmp/config.1.toml bucket create eprouvette +KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur) ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'` SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'` -garage bucket allow eprouvette --read --write --key $ACCESS_KEY +garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --key $ACCESS_KEY echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3 echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3." diff --git a/script/dev-cluster.sh b/script/dev-cluster.sh index 0afdd97c..d56fa6e3 100755 --- a/script/dev-cluster.sh +++ b/script/dev-cluster.sh @@ -17,6 +17,10 @@ MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m" WHICH_GARAGE=$(which garage || exit 1) echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n" +NETWORK_SECRET="$(openssl rand -hex 32)" + + +# <<<<<<<<< BEGIN FOR LOOP ON NODES for count in $(seq 1 3); do CONF_PATH="/tmp/config.$count.toml" LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m" @@ -26,13 +30,11 @@ block_size = 1048576 # objects are split in blocks of maximum this number of b metadata_dir = "/tmp/garage-meta-$count" data_dir = "/tmp/garage-data-$count" rpc_bind_addr = "0.0.0.0:$((3900+$count))" # the port other Garage nodes will use to talk to this node -bootstrap_peers = [ - "127.0.0.1:3901", - "127.0.0.1:3902", - "127.0.0.1:3903" -] +rpc_public_addr = "127.0.0.1:$((3900+$count))" +bootstrap_peers = [] max_concurrent_rpc_requests = 12 replication_mode = "3" +rpc_secret = "$NETWORK_SECRET" [s3_api] api_bind_addr = "0.0.0.0:$((3910+$count))" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part. @@ -61,11 +63,21 @@ if [ -z "$SKIP_HTTPS" ]; then socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 & fi -(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) & +(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) & +done +# >>>>>>>>>>>>>>>> END FOR LOOP ON NODES + +sleep 5 +# Establish connections between nodes +for count in $(seq 1 3); do + NODE=$(garage -c /tmp/config.$count.toml node-id -q) + for count2 in $(seq 1 3); do + garage -c /tmp/config.$count2.toml node connect $NODE + done done RETRY=120 -until garage status 2>&1|grep -q Healthy ; do +until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do (( RETRY-- )) if (( RETRY <= 0 )); then echo -en "${MAIN_LABEL} Garage did not start" @@ -74,6 +86,7 @@ until garage status 2>&1|grep -q Healthy ; do echo -en "${MAIN_LABEL} cluster starting...\n" sleep 1 done + echo -en "${MAIN_LABEL} cluster started\n" wait diff --git a/script/dev-configure.sh b/script/dev-configure.sh index fdae959b..72fb67ea 100755 --- a/script/dev-configure.sh +++ b/script/dev-configure.sh @@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" sleep 5 RETRY=120 -until garage status 2>&1|grep -q Healthy ; do +until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do (( RETRY-- )) if (( RETRY <= 0 )); then echo "garage did not start in time, failing." @@ -21,10 +21,10 @@ until garage status 2>&1|grep -q Healthy ; do sleep 1 done -garage status \ +garage -c /tmp/config.1.toml status \ | grep UNCONFIGURED \ | grep -Po '^[0-9a-f]+' \ | while read id; do - garage node configure -z dc1 -c 1 $id + garage -c /tmp/config.1.toml node configure -z dc1 -c 1 $id done diff --git a/script/test-smoke.sh b/script/test-smoke.sh index 335d55e9..ce9c032a 100755 --- a/script/test-smoke.sh +++ b/script/test-smoke.sh @@ -21,9 +21,9 @@ ${SCRIPT_FOLDER}/dev-configure.sh ${SCRIPT_FOLDER}/dev-bucket.sh which garage -garage status -garage key list -garage bucket list +garage -c /tmp/config.1.toml status +garage -c /tmp/config.1.toml key list +garage -c /tmp/config.1.toml bucket list dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline storage (< INLINE_THRESHOLD = 3072 bytes) dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked diff --git a/src/garage/cli.rs b/src/garage/cli.rs index 940a5a85..67606b97 100644 --- a/src/garage/cli.rs +++ b/src/garage/cli.rs @@ -1,5 +1,4 @@ use std::collections::HashSet; -use std::path::PathBuf; use serde::{Deserialize, Serialize}; use structopt::StructOpt; @@ -21,7 +20,12 @@ use crate::admin_rpc::*; pub enum Command { /// Run Garage server #[structopt(name = "server")] - Server(ServerOpt), + Server, + + /// Print identifier (public key) of this garage node. + /// Generates a new keypair if necessary. + #[structopt(name = "node-id")] + NodeId(NodeIdOpt), /// Get network status #[structopt(name = "status")] @@ -48,13 +52,6 @@ pub enum Command { Stats(StatsOpt), } -#[derive(StructOpt, Debug)] -pub struct ServerOpt { - /// Configuration file - #[structopt(short = "c", long = "config", default_value = "./config.toml")] - pub config_file: PathBuf, -} - #[derive(StructOpt, Debug)] pub enum NodeOperation { /// Connect to Garage node that is currently isolated from the system @@ -70,6 +67,13 @@ pub enum NodeOperation { Remove(RemoveNodeOpt), } +#[derive(StructOpt, Debug)] +pub struct NodeIdOpt { + /// Do not print usage instructions to stderr + #[structopt(short = "q", long = "quiet")] + pub(crate) quiet: bool, +} + #[derive(StructOpt, Debug)] pub struct ConnectNodeOpt { /// Node public key and address, in the format: @@ -384,7 +388,8 @@ pub async fn cmd_status(rpc_cli: &Endpoint, rpc_host: NodeID) -> .any(|(id, _)| !status_keys.contains(id)); if failure_case_1 || failure_case_2 { println!("\nFailed nodes:"); - let mut failed_nodes = vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()]; + let mut failed_nodes = + vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()]; for adv in status.iter().filter(|adv| !adv.is_up) { if let Some(cfg) = config.members.get(&adv.id) { failed_nodes.push(format!( @@ -421,14 +426,15 @@ pub async fn cmd_connect( rpc_host: NodeID, args: ConnectNodeOpt, ) -> Result<(), Error> { - match rpc_cli.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL).await?? { + match rpc_cli + .call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL) + .await?? + { SystemRpc::Ok => { println!("Success."); Ok(()) } - r => { - Err(Error::BadRpc(format!("Unexpected response: {:?}", r))) - } + r => Err(Error::BadRpc(format!("Unexpected response: {:?}", r))), } } @@ -654,4 +660,3 @@ pub fn find_matching_node( Ok(candidates[0]) } } - diff --git a/src/garage/main.rs b/src/garage/main.rs index 543860ca..f7ef19cd 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -9,9 +9,11 @@ mod cli; mod repair; mod server; +use std::path::PathBuf; + use structopt::StructOpt; -use netapp::util::parse_peer_addr; +use netapp::util::parse_and_resolve_peer_addr; use netapp::NetworkKey; use garage_util::error::Error; @@ -34,6 +36,10 @@ struct Opt { #[structopt(short = "s", long = "rpc-secret")] pub rpc_secret: Option, + /// Configuration file (garage.toml) + #[structopt(short = "c", long = "config", default_value = "/etc/garage.toml")] + pub config_file: PathBuf, + #[structopt(subcommand)] cmd: Command, } @@ -45,16 +51,18 @@ async fn main() { let opt = Opt::from_args(); - let res = if let Command::Server(server_opt) = opt.cmd { - // Abort on panic (same behavior as in Go) - std::panic::set_hook(Box::new(|panic_info| { - error!("{}", panic_info.to_string()); - std::process::abort(); - })); + let res = match opt.cmd { + Command::Server => { + // Abort on panic (same behavior as in Go) + std::panic::set_hook(Box::new(|panic_info| { + error!("{}", panic_info.to_string()); + std::process::abort(); + })); - server::run_server(server_opt.config_file).await - } else { - cli_command(opt).await + server::run_server(opt.config_file).await + } + Command::NodeId(node_id_opt) => node_id_command(opt.config_file, node_id_opt.quiet), + _ => cli_command(opt).await, }; if let Err(e) = res { @@ -63,16 +71,42 @@ async fn main() { } async fn cli_command(opt: Opt) -> Result<(), Error> { - let net_key_hex_str = &opt.rpc_secret.expect("No RPC secret provided"); + let config = if opt.rpc_secret.is_none() || opt.rpc_host.is_none() { + Some(garage_util::config::read_config(opt.config_file.clone()) + .map_err(|e| Error::Message(format!("Unable to read configuration file {}: {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy(), e)))?) + } else { + None + }; + + // Find and parse network RPC secret + let net_key_hex_str = opt + .rpc_secret + .as_ref() + .or_else(|| config.as_ref().map(|c| &c.rpc_secret)) + .expect("No RPC secret provided"); let network_key = NetworkKey::from_slice( &hex::decode(net_key_hex_str).expect("Invalid RPC secret key (bad hex)")[..], ) .expect("Invalid RPC secret provided (wrong length)"); + + // Generate a temporary keypair for our RPC client let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair(); let netapp = NetApp::new(network_key, sk); - let (id, addr) = - parse_peer_addr(&opt.rpc_host.expect("No RPC host provided")).expect("Invalid RPC host"); + + // Find and parse the address of the target host + let (id, addr) = if let Some(h) = opt.rpc_host { + let (id, addrs) = parse_and_resolve_peer_addr(&h).expect("Invalid RPC host"); + (id, addrs[0]) + } else if let Some(a) = config.as_ref().map(|c| c.rpc_public_addr).flatten() { + let node_key = garage_rpc::system::gen_node_key(&config.unwrap().metadata_dir) + .map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?; + (node_key.public_key(), a) + } else { + return Err(Error::Message("No RPC host provided".into())); + }; + + // Connect to target host netapp.clone().try_connect(addr, id).await?; let system_rpc_endpoint = netapp.endpoint::(SYSTEM_RPC_PATH.into()); @@ -80,3 +114,58 @@ async fn cli_command(opt: Opt) -> Result<(), Error> { cli_cmd(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await } + +fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> { + let config = garage_util::config::read_config(config_file.clone()).map_err(|e| { + Error::Message(format!( + "Unable to read configuration file {}: {}", + config_file.to_string_lossy(), + e + )) + })?; + + let node_key = garage_rpc::system::gen_node_key(&config.metadata_dir) + .map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?; + + let idstr = if let Some(addr) = config.rpc_public_addr { + let idstr = format!("{}@{}", hex::encode(&node_key.public_key()), addr); + println!("{}", idstr); + idstr + } else { + let idstr = hex::encode(&node_key.public_key()); + println!("{}", idstr); + + if !quiet { + eprintln!("WARNING: I don't know the public address to reach this node."); + eprintln!("In all of the instructions below, replace 127.0.0.1:3901 by the appropriate address and port."); + } + + format!("{}@127.0.0.1:3901", idstr) + }; + + if !quiet { + eprintln!(""); + eprintln!( + "To instruct a node to connect to this node, run the following command on that node:" + ); + eprintln!(" garage [-c ] node connect {}", idstr); + eprintln!(""); + eprintln!("Or instruct them to connect from here by running:"); + eprintln!( + " garage -c {} -h node connect {}", + config_file.to_string_lossy(), + idstr + ); + eprintln!( + "where is their own node identifier in the format: @:" + ); + eprintln!(""); + eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:"); + eprintln!(" bootstrap_peers = ["); + eprintln!(" \"{}\",", idstr); + eprintln!(" ..."); + eprintln!(" ]"); + } + + Ok(()) +} diff --git a/src/rpc/rpc_helper.rs b/src/rpc/rpc_helper.rs index 9f735ab4..90b3cf15 100644 --- a/src/rpc/rpc_helper.rs +++ b/src/rpc/rpc_helper.rs @@ -14,8 +14,8 @@ pub use netapp::proto::*; pub use netapp::{NetApp, NodeID}; use garage_util::background::BackgroundRunner; -use garage_util::error::Error; use garage_util::data::Uuid; +use garage_util::error::Error; const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); diff --git a/src/rpc/system.rs b/src/rpc/system.rs index 886811b1..51abede3 100644 --- a/src/rpc/system.rs +++ b/src/rpc/system.rs @@ -18,8 +18,8 @@ use tokio::sync::Mutex; use netapp::endpoint::{Endpoint, EndpointHandler}; use netapp::peering::fullmesh::FullMeshPeeringStrategy; use netapp::proto::*; -use netapp::{NetApp, NetworkKey, NodeID, NodeKey}; use netapp::util::parse_and_resolve_peer_addr; +use netapp::{NetApp, NetworkKey, NodeID, NodeKey}; use garage_util::background::BackgroundRunner; use garage_util::data::Uuid; @@ -110,7 +110,7 @@ pub struct KnownNodeInfo { pub status: NodeStatus, } -fn gen_node_key(metadata_dir: &Path) -> Result { +pub fn gen_node_key(metadata_dir: &Path) -> Result { let mut key_file = metadata_dir.to_path_buf(); key_file.push("node_key"); if key_file.as_path().exists() { @@ -246,8 +246,12 @@ impl System { } async fn handle_connect(&self, node: &str) -> Result { - let (pubkey, addrs) = parse_and_resolve_peer_addr(node) - .ok_or_else(|| Error::Message(format!("Unable to parse or resolve node specification: {}", node)))?; + let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| { + Error::Message(format!( + "Unable to parse or resolve node specification: {}", + node + )) + })?; let mut errors = vec![]; for ip in addrs.iter() { match self.netapp.clone().try_connect(*ip, pubkey).await { @@ -257,7 +261,10 @@ impl System { } } } - return Err(Error::Message(format!("Could not connect to specified peers. Errors: {:?}", errors))); + return Err(Error::Message(format!( + "Could not connect to specified peers. Errors: {:?}", + errors + ))); } fn handle_pull_config(&self) -> SystemRpc { @@ -267,23 +274,25 @@ impl System { fn handle_get_known_nodes(&self) -> SystemRpc { let node_status = self.node_status.read().unwrap(); - let known_nodes = - self.fullmesh - .get_peer_list() - .iter() - .map(|n| KnownNodeInfo { - id: n.id.into(), - addr: n.addr, - is_up: n.is_up(), - status: node_status.get(&n.id.into()).cloned().map(|(_, st)| st).unwrap_or( - NodeStatus { - hostname: "?".to_string(), - replication_factor: 0, - config_version: 0, - }, - ), - }) - .collect::>(); + let known_nodes = self + .fullmesh + .get_peer_list() + .iter() + .map(|n| KnownNodeInfo { + id: n.id.into(), + addr: n.addr, + is_up: n.is_up(), + status: node_status + .get(&n.id.into()) + .cloned() + .map(|(_, st)| st) + .unwrap_or(NodeStatus { + hostname: "?".to_string(), + replication_factor: 0, + config_version: 0, + }), + }) + .collect::>(); SystemRpc::ReturnKnownNodes(known_nodes) } @@ -330,14 +339,14 @@ impl System { drop(update_ring); let self2 = self.clone(); - let adv2 = adv.clone(); + let adv = adv.clone(); self.background.spawn_cancellable(async move { self2 .rpc .broadcast( &self2.system_endpoint, - SystemRpc::AdvertiseConfig(adv2), - RequestStrategy::with_priority(PRIO_NORMAL), + SystemRpc::AdvertiseConfig(adv), + RequestStrategy::with_priority(PRIO_HIGH), ) .await; Ok(()) diff --git a/src/table/replication/fullcopy.rs b/src/table/replication/fullcopy.rs index ae6851fb..8f01fbdd 100644 --- a/src/table/replication/fullcopy.rs +++ b/src/table/replication/fullcopy.rs @@ -28,11 +28,7 @@ impl TableReplication for TableFullReplication { fn write_nodes(&self, _hash: &Hash) -> Vec { let ring = self.system.ring.borrow(); - ring.config - .members - .keys() - .cloned() - .collect::>() + ring.config.members.keys().cloned().collect::>() } fn write_quorum(&self) -> usize { let nmembers = self.system.ring.borrow().config.members.len(); diff --git a/src/util/config.rs b/src/util/config.rs index fe0a6fa8..95c5cfc0 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -6,8 +6,8 @@ use std::path::PathBuf; use serde::de::Error as SerdeError; use serde::{de, Deserialize}; -use netapp::NodeID; use netapp::util::parse_and_resolve_peer_addr; +use netapp::NodeID; use crate::error::Error; @@ -117,8 +117,9 @@ where let mut ret = vec![]; for peer in >::deserialize(deserializer)? { - let (pubkey, addrs) = parse_and_resolve_peer_addr(peer) - .ok_or_else(|| D::Error::custom(format!("Unable to parse or resolve peer: {}", peer)))?; + let (pubkey, addrs) = parse_and_resolve_peer_addr(peer).ok_or_else(|| { + D::Error::custom(format!("Unable to parse or resolve peer: {}", peer)) + })?; for ip in addrs { ret.push((pubkey.clone(), ip)); }