Adapt tests to new syntax with public keys

script/dev-bucket.sh

@@ -9,11 +9,11 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
 NIX_RELEASE="${REPO_FOLDER}/result/bin/"
 PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
 
-garage bucket create eprouvette
+garage -c /tmp/config.1.toml bucket create eprouvette
-KEY_INFO=`garage key new --name opérateur`
+KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur)
 ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
 SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
-garage bucket allow eprouvette --read --write --key $ACCESS_KEY
+garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --key $ACCESS_KEY
 echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
 
 echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."

script/dev-cluster.sh

@@ -17,6 +17,10 @@ MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
 WHICH_GARAGE=$(which garage || exit 1)
 echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n"
 
+NETWORK_SECRET="$(openssl rand -hex 32)"
+
+
+# <<<<<<<<< BEGIN FOR LOOP ON NODES
 for count in $(seq 1 3); do
 CONF_PATH="/tmp/config.$count.toml"
 LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
@@ -26,13 +30,11 @@ block_size = 1048576			# objects are split in blocks of maximum this number of b
 metadata_dir = "/tmp/garage-meta-$count"
 data_dir = "/tmp/garage-data-$count"
 rpc_bind_addr = "0.0.0.0:$((3900+$count))"		# the port other Garage nodes will use to talk to this node
-bootstrap_peers = [
+rpc_public_addr = "127.0.0.1:$((3900+$count))"
-  "127.0.0.1:3901",
+bootstrap_peers = []
-  "127.0.0.1:3902",
-  "127.0.0.1:3903"
-]
 max_concurrent_rpc_requests = 12
 replication_mode = "3"
+rpc_secret = "$NETWORK_SECRET"
 
 [s3_api]
 api_bind_addr = "0.0.0.0:$((3910+$count))"	# the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
@@ -61,11 +63,21 @@ if [ -z "$SKIP_HTTPS" ]; then
   socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
 fi
 
-(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
+(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
+done
+# >>>>>>>>>>>>>>>> END FOR LOOP ON NODES
+
+sleep 5
+# Establish connections between nodes
+for count in $(seq 1 3); do
+	NODE=$(garage -c /tmp/config.$count.toml node-id -q)
+	for count2 in $(seq 1 3); do
+		garage -c /tmp/config.$count2.toml node connect $NODE
+	done
 done
 
 RETRY=120
-until garage status 2>&1|grep -q Healthy ; do 
+until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do 
   (( RETRY-- ))
   if (( RETRY <= 0 )); then 
     echo -en "${MAIN_LABEL} Garage did not start"
@@ -74,6 +86,7 @@ until garage status 2>&1|grep -q Healthy ; do
 	echo -en "${MAIN_LABEL} cluster starting...\n"
 	sleep 1
 done
+
 echo -en "${MAIN_LABEL} cluster started\n"
 
 wait

script/dev-configure.sh

@@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
 
 sleep 5
 RETRY=120
-until garage status 2>&1|grep -q Healthy ; do 
+until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do 
   (( RETRY-- ))
   if (( RETRY <= 0 )); then
     echo "garage did not start in time, failing."
@@ -21,10 +21,10 @@ until garage status 2>&1|grep -q Healthy ; do
 	sleep 1
 done
 
-garage status \
+garage -c /tmp/config.1.toml status \
 	| grep UNCONFIGURED \
 	| grep -Po '^[0-9a-f]+' \
 	| while read id; do 
-	  garage node configure -z dc1 -c 1 $id
+	  garage -c /tmp/config.1.toml node configure -z dc1 -c 1 $id
 	done
 

script/test-smoke.sh

@@ -21,9 +21,9 @@ ${SCRIPT_FOLDER}/dev-configure.sh
 ${SCRIPT_FOLDER}/dev-bucket.sh
 
 which garage
-garage status
+garage -c /tmp/config.1.toml status
-garage key list
+garage -c /tmp/config.1.toml key list
-garage bucket list
+garage -c /tmp/config.1.toml bucket list
 
 dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline storage (< INLINE_THRESHOLD = 3072 bytes)
 dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked

src/garage/cli.rs

@@ -1,5 +1,4 @@
 use std::collections::HashSet;
-use std::path::PathBuf;
 
 use serde::{Deserialize, Serialize};
 use structopt::StructOpt;
@@ -21,7 +20,12 @@ use crate::admin_rpc::*;
 pub enum Command {
 	/// Run Garage server
 	#[structopt(name = "server")]
-	Server(ServerOpt),
+	Server,
+
+	/// Print identifier (public key) of this garage node.
+	/// Generates a new keypair if necessary.
+	#[structopt(name = "node-id")]
+	NodeId(NodeIdOpt),
 
 	/// Get network status
 	#[structopt(name = "status")]
@@ -48,13 +52,6 @@ pub enum Command {
 	Stats(StatsOpt),
 }
 
-#[derive(StructOpt, Debug)]
-pub struct ServerOpt {
-	/// Configuration file
-	#[structopt(short = "c", long = "config", default_value = "./config.toml")]
-	pub config_file: PathBuf,
-}
-
 #[derive(StructOpt, Debug)]
 pub enum NodeOperation {
 	/// Connect to Garage node that is currently isolated from the system
@@ -70,6 +67,13 @@ pub enum NodeOperation {
 	Remove(RemoveNodeOpt),
 }
 
+#[derive(StructOpt, Debug)]
+pub struct NodeIdOpt {
+	/// Do not print usage instructions to stderr
+	#[structopt(short = "q", long = "quiet")]
+	pub(crate) quiet: bool,
+}
+
 #[derive(StructOpt, Debug)]
 pub struct ConnectNodeOpt {
 	/// Node public key and address, in the format:
@@ -384,7 +388,8 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
 		.any(|(id, _)| !status_keys.contains(id));
 	if failure_case_1 || failure_case_2 {
 		println!("\nFailed nodes:");
-		let mut failed_nodes = vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()];
+		let mut failed_nodes =
+			vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()];
 		for adv in status.iter().filter(|adv| !adv.is_up) {
 			if let Some(cfg) = config.members.get(&adv.id) {
 				failed_nodes.push(format!(
@@ -421,14 +426,15 @@ pub async fn cmd_connect(
 	rpc_host: NodeID,
 	args: ConnectNodeOpt,
 ) -> Result<(), Error> {
-	match rpc_cli.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL).await?? {
+	match rpc_cli
+		.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL)
+		.await??
+	{
 		SystemRpc::Ok => {
 			println!("Success.");
 			Ok(())
 		}
-		r => {
+		r => Err(Error::BadRpc(format!("Unexpected response: {:?}", r))),
-			Err(Error::BadRpc(format!("Unexpected response: {:?}", r)))
-		}
 	}
 }
 
@@ -654,4 +660,3 @@ pub fn find_matching_node(
 		Ok(candidates[0])
 	}
 }
-

src/garage/main.rs

@@ -9,9 +9,11 @@ mod cli;
 mod repair;
 mod server;
 
+use std::path::PathBuf;
+
 use structopt::StructOpt;
 
-use netapp::util::parse_peer_addr;
+use netapp::util::parse_and_resolve_peer_addr;
 use netapp::NetworkKey;
 
 use garage_util::error::Error;
@@ -34,6 +36,10 @@ struct Opt {
 	#[structopt(short = "s", long = "rpc-secret")]
 	pub rpc_secret: Option<String>,
 
+	/// Configuration file (garage.toml)
+	#[structopt(short = "c", long = "config", default_value = "/etc/garage.toml")]
+	pub config_file: PathBuf,
+
 	#[structopt(subcommand)]
 	cmd: Command,
 }
@@ -45,16 +51,18 @@ async fn main() {
 
 	let opt = Opt::from_args();
 
-	let res = if let Command::Server(server_opt) = opt.cmd {
+	let res = match opt.cmd {
+		Command::Server => {
 			// Abort on panic (same behavior as in Go)
 			std::panic::set_hook(Box::new(|panic_info| {
 				error!("{}", panic_info.to_string());
 				std::process::abort();
 			}));
 
-		server::run_server(server_opt.config_file).await
+			server::run_server(opt.config_file).await
-	} else {
+		}
-		cli_command(opt).await
+		Command::NodeId(node_id_opt) => node_id_command(opt.config_file, node_id_opt.quiet),
+		_ => cli_command(opt).await,
 	};
 
 	if let Err(e) = res {
@@ -63,16 +71,42 @@ async fn main() {
 }
 
 async fn cli_command(opt: Opt) -> Result<(), Error> {
-	let net_key_hex_str = &opt.rpc_secret.expect("No RPC secret provided");
+	let config = if opt.rpc_secret.is_none() || opt.rpc_host.is_none() {
+		Some(garage_util::config::read_config(opt.config_file.clone())
+			.map_err(|e| Error::Message(format!("Unable to read configuration file {}: {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy(), e)))?)
+	} else {
+		None
+	};
+
+	// Find and parse network RPC secret
+	let net_key_hex_str = opt
+		.rpc_secret
+		.as_ref()
+		.or_else(|| config.as_ref().map(|c| &c.rpc_secret))
+		.expect("No RPC secret provided");
 	let network_key = NetworkKey::from_slice(
 		&hex::decode(net_key_hex_str).expect("Invalid RPC secret key (bad hex)")[..],
 	)
 	.expect("Invalid RPC secret provided (wrong length)");
+
+	// Generate a temporary keypair for our RPC client
 	let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair();
 
 	let netapp = NetApp::new(network_key, sk);
-	let (id, addr) =
+
-		parse_peer_addr(&opt.rpc_host.expect("No RPC host provided")).expect("Invalid RPC host");
+	// Find and parse the address of the target host
+	let (id, addr) = if let Some(h) = opt.rpc_host {
+		let (id, addrs) = parse_and_resolve_peer_addr(&h).expect("Invalid RPC host");
+		(id, addrs[0])
+	} else if let Some(a) = config.as_ref().map(|c| c.rpc_public_addr).flatten() {
+		let node_key = garage_rpc::system::gen_node_key(&config.unwrap().metadata_dir)
+			.map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
+		(node_key.public_key(), a)
+	} else {
+		return Err(Error::Message("No RPC host provided".into()));
+	};
+
+	// Connect to target host
 	netapp.clone().try_connect(addr, id).await?;
 
 	let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
@@ -80,3 +114,58 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 
 	cli_cmd(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await
 }
+
+fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
+	let config = garage_util::config::read_config(config_file.clone()).map_err(|e| {
+		Error::Message(format!(
+			"Unable to read configuration file {}: {}",
+			config_file.to_string_lossy(),
+			e
+		))
+	})?;
+
+	let node_key = garage_rpc::system::gen_node_key(&config.metadata_dir)
+		.map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
+
+	let idstr = if let Some(addr) = config.rpc_public_addr {
+		let idstr = format!("{}@{}", hex::encode(&node_key.public_key()), addr);
+		println!("{}", idstr);
+		idstr
+	} else {
+		let idstr = hex::encode(&node_key.public_key());
+		println!("{}", idstr);
+
+		if !quiet {
+			eprintln!("WARNING: I don't know the public address to reach this node.");
+			eprintln!("In all of the instructions below, replace 127.0.0.1:3901 by the appropriate address and port.");
+		}
+
+		format!("{}@127.0.0.1:3901", idstr)
+	};
+
+	if !quiet {
+		eprintln!("");
+		eprintln!(
+			"To instruct a node to connect to this node, run the following command on that node:"
+		);
+		eprintln!("    garage [-c <config file path>] node connect {}", idstr);
+		eprintln!("");
+		eprintln!("Or instruct them to connect from here by running:");
+		eprintln!(
+			"    garage -c {} -h <remote node> node connect {}",
+			config_file.to_string_lossy(),
+			idstr
+		);
+		eprintln!(
+			"where <remote_node> is their own node identifier in the format: <pubkey>@<ip>:<port>"
+		);
+		eprintln!("");
+		eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:");
+		eprintln!("    bootstrap_peers = [");
+		eprintln!("        \"{}\",", idstr);
+		eprintln!("        ...");
+		eprintln!("    ]");
+	}
+
+	Ok(())
+}

src/rpc/rpc_helper.rs

@@ -14,8 +14,8 @@ pub use netapp::proto::*;
 pub use netapp::{NetApp, NodeID};
 
 use garage_util::background::BackgroundRunner;
-use garage_util::error::Error;
 use garage_util::data::Uuid;
+use garage_util::error::Error;
 
 const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
 

src/rpc/system.rs

@@ -18,8 +18,8 @@ use tokio::sync::Mutex;
 use netapp::endpoint::{Endpoint, EndpointHandler};
 use netapp::peering::fullmesh::FullMeshPeeringStrategy;
 use netapp::proto::*;
-use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
 use netapp::util::parse_and_resolve_peer_addr;
+use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
 
 use garage_util::background::BackgroundRunner;
 use garage_util::data::Uuid;
@@ -110,7 +110,7 @@ pub struct KnownNodeInfo {
 	pub status: NodeStatus,
 }
 
-fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
+pub fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
 	let mut key_file = metadata_dir.to_path_buf();
 	key_file.push("node_key");
 	if key_file.as_path().exists() {
@@ -246,8 +246,12 @@ impl System {
 	}
 
 	async fn handle_connect(&self, node: &str) -> Result<SystemRpc, Error> {
-		let (pubkey, addrs) = parse_and_resolve_peer_addr(node)
+		let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| {
-			.ok_or_else(|| Error::Message(format!("Unable to parse or resolve node specification: {}", node)))?;
+			Error::Message(format!(
+				"Unable to parse or resolve node specification: {}",
+				node
+			))
+		})?;
 		let mut errors = vec![];
 		for ip in addrs.iter() {
 			match self.netapp.clone().try_connect(*ip, pubkey).await {
@@ -257,7 +261,10 @@ impl System {
 				}
 			}
 		}
-		return Err(Error::Message(format!("Could not connect to specified peers. Errors: {:?}", errors)));
+		return Err(Error::Message(format!(
+			"Could not connect to specified peers. Errors: {:?}",
+			errors
+		)));
 	}
 
 	fn handle_pull_config(&self) -> SystemRpc {
@@ -267,21 +274,23 @@ impl System {
 
 	fn handle_get_known_nodes(&self) -> SystemRpc {
 		let node_status = self.node_status.read().unwrap();
-		let known_nodes =
+		let known_nodes = self
-			self.fullmesh
+			.fullmesh
 			.get_peer_list()
 			.iter()
 			.map(|n| KnownNodeInfo {
 				id: n.id.into(),
 				addr: n.addr,
 				is_up: n.is_up(),
-					status: node_status.get(&n.id.into()).cloned().map(|(_, st)| st).unwrap_or(
+				status: node_status
-						NodeStatus {
+					.get(&n.id.into())
+					.cloned()
+					.map(|(_, st)| st)
+					.unwrap_or(NodeStatus {
 						hostname: "?".to_string(),
 						replication_factor: 0,
 						config_version: 0,
-						},
+					}),
-					),
 			})
 			.collect::<Vec<_>>();
 		SystemRpc::ReturnKnownNodes(known_nodes)
@@ -330,14 +339,14 @@ impl System {
 			drop(update_ring);
 
 			let self2 = self.clone();
-			let adv2 = adv.clone();
+			let adv = adv.clone();
 			self.background.spawn_cancellable(async move {
 				self2
 					.rpc
 					.broadcast(
 						&self2.system_endpoint,
-						SystemRpc::AdvertiseConfig(adv2),
+						SystemRpc::AdvertiseConfig(adv),
-						RequestStrategy::with_priority(PRIO_NORMAL),
+						RequestStrategy::with_priority(PRIO_HIGH),
 					)
 					.await;
 				Ok(())

src/table/replication/fullcopy.rs

@@ -28,11 +28,7 @@ impl TableReplication for TableFullReplication {
 
 	fn write_nodes(&self, _hash: &Hash) -> Vec<Uuid> {
 		let ring = self.system.ring.borrow();
-		ring.config
+		ring.config.members.keys().cloned().collect::<Vec<_>>()
-			.members
-			.keys()
-			.cloned()
-			.collect::<Vec<_>>()
 	}
 	fn write_quorum(&self) -> usize {
 		let nmembers = self.system.ring.borrow().config.members.len();

src/util/config.rs

@@ -6,8 +6,8 @@ use std::path::PathBuf;
 use serde::de::Error as SerdeError;
 use serde::{de, Deserialize};
 
-use netapp::NodeID;
 use netapp::util::parse_and_resolve_peer_addr;
+use netapp::NodeID;
 
 use crate::error::Error;
 
@@ -117,8 +117,9 @@ where
 	let mut ret = vec![];
 
 	for peer in <Vec<&str>>::deserialize(deserializer)? {
-		let (pubkey, addrs) = parse_and_resolve_peer_addr(peer)
+		let (pubkey, addrs) = parse_and_resolve_peer_addr(peer).ok_or_else(|| {
-			.ok_or_else(|| D::Error::custom(format!("Unable to parse or resolve peer: {}", peer)))?;
+			D::Error::custom(format!("Unable to parse or resolve peer: {}", peer))
+		})?;
 		for ip in addrs {
 			ret.push((pubkey.clone(), ip));
 		}