forked from Deuxfleurs/garage
Adapt tests to new syntax with public keys
This commit is contained in:
parent
65070f3c05
commit
a8ae78af0a
10 changed files with 191 additions and 78 deletions
|
|
@ -9,11 +9,11 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
|||
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
|
||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
||||
|
||||
garage bucket create eprouvette
|
||||
KEY_INFO=`garage key new --name opérateur`
|
||||
garage -c /tmp/config.1.toml bucket create eprouvette
|
||||
KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur)
|
||||
ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
|
||||
SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
|
||||
garage bucket allow eprouvette --read --write --key $ACCESS_KEY
|
||||
garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --key $ACCESS_KEY
|
||||
echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
|
||||
|
||||
echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."
|
||||
|
|
|
|||
|
|
@ -17,6 +17,10 @@ MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
|
|||
WHICH_GARAGE=$(which garage || exit 1)
|
||||
echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n"
|
||||
|
||||
NETWORK_SECRET="$(openssl rand -hex 32)"
|
||||
|
||||
|
||||
# <<<<<<<<< BEGIN FOR LOOP ON NODES
|
||||
for count in $(seq 1 3); do
|
||||
CONF_PATH="/tmp/config.$count.toml"
|
||||
LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
|
||||
|
|
@ -26,13 +30,11 @@ block_size = 1048576 # objects are split in blocks of maximum this number of b
|
|||
metadata_dir = "/tmp/garage-meta-$count"
|
||||
data_dir = "/tmp/garage-data-$count"
|
||||
rpc_bind_addr = "0.0.0.0:$((3900+$count))" # the port other Garage nodes will use to talk to this node
|
||||
bootstrap_peers = [
|
||||
"127.0.0.1:3901",
|
||||
"127.0.0.1:3902",
|
||||
"127.0.0.1:3903"
|
||||
]
|
||||
rpc_public_addr = "127.0.0.1:$((3900+$count))"
|
||||
bootstrap_peers = []
|
||||
max_concurrent_rpc_requests = 12
|
||||
replication_mode = "3"
|
||||
rpc_secret = "$NETWORK_SECRET"
|
||||
|
||||
[s3_api]
|
||||
api_bind_addr = "0.0.0.0:$((3910+$count))" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
|
||||
|
|
@ -61,11 +63,21 @@ if [ -z "$SKIP_HTTPS" ]; then
|
|||
socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
|
||||
fi
|
||||
|
||||
(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
|
||||
(garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
|
||||
done
|
||||
# >>>>>>>>>>>>>>>> END FOR LOOP ON NODES
|
||||
|
||||
sleep 5
|
||||
# Establish connections between nodes
|
||||
for count in $(seq 1 3); do
|
||||
NODE=$(garage -c /tmp/config.$count.toml node-id -q)
|
||||
for count2 in $(seq 1 3); do
|
||||
garage -c /tmp/config.$count2.toml node connect $NODE
|
||||
done
|
||||
done
|
||||
|
||||
RETRY=120
|
||||
until garage status 2>&1|grep -q Healthy ; do
|
||||
until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
|
||||
(( RETRY-- ))
|
||||
if (( RETRY <= 0 )); then
|
||||
echo -en "${MAIN_LABEL} Garage did not start"
|
||||
|
|
@ -74,6 +86,7 @@ until garage status 2>&1|grep -q Healthy ; do
|
|||
echo -en "${MAIN_LABEL} cluster starting...\n"
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo -en "${MAIN_LABEL} cluster started\n"
|
||||
|
||||
wait
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
|||
|
||||
sleep 5
|
||||
RETRY=120
|
||||
until garage status 2>&1|grep -q Healthy ; do
|
||||
until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
|
||||
(( RETRY-- ))
|
||||
if (( RETRY <= 0 )); then
|
||||
echo "garage did not start in time, failing."
|
||||
|
|
@ -21,10 +21,10 @@ until garage status 2>&1|grep -q Healthy ; do
|
|||
sleep 1
|
||||
done
|
||||
|
||||
garage status \
|
||||
garage -c /tmp/config.1.toml status \
|
||||
| grep UNCONFIGURED \
|
||||
| grep -Po '^[0-9a-f]+' \
|
||||
| while read id; do
|
||||
garage node configure -z dc1 -c 1 $id
|
||||
garage -c /tmp/config.1.toml node configure -z dc1 -c 1 $id
|
||||
done
|
||||
|
||||
|
|
|
|||
|
|
@ -21,9 +21,9 @@ ${SCRIPT_FOLDER}/dev-configure.sh
|
|||
${SCRIPT_FOLDER}/dev-bucket.sh
|
||||
|
||||
which garage
|
||||
garage status
|
||||
garage key list
|
||||
garage bucket list
|
||||
garage -c /tmp/config.1.toml status
|
||||
garage -c /tmp/config.1.toml key list
|
||||
garage -c /tmp/config.1.toml bucket list
|
||||
|
||||
dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline storage (< INLINE_THRESHOLD = 3072 bytes)
|
||||
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
use std::collections::HashSet;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use structopt::StructOpt;
|
||||
|
|
@ -21,7 +20,12 @@ use crate::admin_rpc::*;
|
|||
pub enum Command {
|
||||
/// Run Garage server
|
||||
#[structopt(name = "server")]
|
||||
Server(ServerOpt),
|
||||
Server,
|
||||
|
||||
/// Print identifier (public key) of this garage node.
|
||||
/// Generates a new keypair if necessary.
|
||||
#[structopt(name = "node-id")]
|
||||
NodeId(NodeIdOpt),
|
||||
|
||||
/// Get network status
|
||||
#[structopt(name = "status")]
|
||||
|
|
@ -48,13 +52,6 @@ pub enum Command {
|
|||
Stats(StatsOpt),
|
||||
}
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub struct ServerOpt {
|
||||
/// Configuration file
|
||||
#[structopt(short = "c", long = "config", default_value = "./config.toml")]
|
||||
pub config_file: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub enum NodeOperation {
|
||||
/// Connect to Garage node that is currently isolated from the system
|
||||
|
|
@ -70,6 +67,13 @@ pub enum NodeOperation {
|
|||
Remove(RemoveNodeOpt),
|
||||
}
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub struct NodeIdOpt {
|
||||
/// Do not print usage instructions to stderr
|
||||
#[structopt(short = "q", long = "quiet")]
|
||||
pub(crate) quiet: bool,
|
||||
}
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub struct ConnectNodeOpt {
|
||||
/// Node public key and address, in the format:
|
||||
|
|
@ -384,7 +388,8 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
|
|||
.any(|(id, _)| !status_keys.contains(id));
|
||||
if failure_case_1 || failure_case_2 {
|
||||
println!("\nFailed nodes:");
|
||||
let mut failed_nodes = vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()];
|
||||
let mut failed_nodes =
|
||||
vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()];
|
||||
for adv in status.iter().filter(|adv| !adv.is_up) {
|
||||
if let Some(cfg) = config.members.get(&adv.id) {
|
||||
failed_nodes.push(format!(
|
||||
|
|
@ -421,14 +426,15 @@ pub async fn cmd_connect(
|
|||
rpc_host: NodeID,
|
||||
args: ConnectNodeOpt,
|
||||
) -> Result<(), Error> {
|
||||
match rpc_cli.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL).await?? {
|
||||
match rpc_cli
|
||||
.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL)
|
||||
.await??
|
||||
{
|
||||
SystemRpc::Ok => {
|
||||
println!("Success.");
|
||||
Ok(())
|
||||
}
|
||||
r => {
|
||||
Err(Error::BadRpc(format!("Unexpected response: {:?}", r)))
|
||||
}
|
||||
r => Err(Error::BadRpc(format!("Unexpected response: {:?}", r))),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -654,4 +660,3 @@ pub fn find_matching_node(
|
|||
Ok(candidates[0])
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,9 +9,11 @@ mod cli;
|
|||
mod repair;
|
||||
mod server;
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use structopt::StructOpt;
|
||||
|
||||
use netapp::util::parse_peer_addr;
|
||||
use netapp::util::parse_and_resolve_peer_addr;
|
||||
use netapp::NetworkKey;
|
||||
|
||||
use garage_util::error::Error;
|
||||
|
|
@ -34,6 +36,10 @@ struct Opt {
|
|||
#[structopt(short = "s", long = "rpc-secret")]
|
||||
pub rpc_secret: Option<String>,
|
||||
|
||||
/// Configuration file (garage.toml)
|
||||
#[structopt(short = "c", long = "config", default_value = "/etc/garage.toml")]
|
||||
pub config_file: PathBuf,
|
||||
|
||||
#[structopt(subcommand)]
|
||||
cmd: Command,
|
||||
}
|
||||
|
|
@ -45,16 +51,18 @@ async fn main() {
|
|||
|
||||
let opt = Opt::from_args();
|
||||
|
||||
let res = if let Command::Server(server_opt) = opt.cmd {
|
||||
let res = match opt.cmd {
|
||||
Command::Server => {
|
||||
// Abort on panic (same behavior as in Go)
|
||||
std::panic::set_hook(Box::new(|panic_info| {
|
||||
error!("{}", panic_info.to_string());
|
||||
std::process::abort();
|
||||
}));
|
||||
|
||||
server::run_server(server_opt.config_file).await
|
||||
} else {
|
||||
cli_command(opt).await
|
||||
server::run_server(opt.config_file).await
|
||||
}
|
||||
Command::NodeId(node_id_opt) => node_id_command(opt.config_file, node_id_opt.quiet),
|
||||
_ => cli_command(opt).await,
|
||||
};
|
||||
|
||||
if let Err(e) = res {
|
||||
|
|
@ -63,16 +71,42 @@ async fn main() {
|
|||
}
|
||||
|
||||
async fn cli_command(opt: Opt) -> Result<(), Error> {
|
||||
let net_key_hex_str = &opt.rpc_secret.expect("No RPC secret provided");
|
||||
let config = if opt.rpc_secret.is_none() || opt.rpc_host.is_none() {
|
||||
Some(garage_util::config::read_config(opt.config_file.clone())
|
||||
.map_err(|e| Error::Message(format!("Unable to read configuration file {}: {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy(), e)))?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Find and parse network RPC secret
|
||||
let net_key_hex_str = opt
|
||||
.rpc_secret
|
||||
.as_ref()
|
||||
.or_else(|| config.as_ref().map(|c| &c.rpc_secret))
|
||||
.expect("No RPC secret provided");
|
||||
let network_key = NetworkKey::from_slice(
|
||||
&hex::decode(net_key_hex_str).expect("Invalid RPC secret key (bad hex)")[..],
|
||||
)
|
||||
.expect("Invalid RPC secret provided (wrong length)");
|
||||
|
||||
// Generate a temporary keypair for our RPC client
|
||||
let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair();
|
||||
|
||||
let netapp = NetApp::new(network_key, sk);
|
||||
let (id, addr) =
|
||||
parse_peer_addr(&opt.rpc_host.expect("No RPC host provided")).expect("Invalid RPC host");
|
||||
|
||||
// Find and parse the address of the target host
|
||||
let (id, addr) = if let Some(h) = opt.rpc_host {
|
||||
let (id, addrs) = parse_and_resolve_peer_addr(&h).expect("Invalid RPC host");
|
||||
(id, addrs[0])
|
||||
} else if let Some(a) = config.as_ref().map(|c| c.rpc_public_addr).flatten() {
|
||||
let node_key = garage_rpc::system::gen_node_key(&config.unwrap().metadata_dir)
|
||||
.map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
|
||||
(node_key.public_key(), a)
|
||||
} else {
|
||||
return Err(Error::Message("No RPC host provided".into()));
|
||||
};
|
||||
|
||||
// Connect to target host
|
||||
netapp.clone().try_connect(addr, id).await?;
|
||||
|
||||
let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
|
||||
|
|
@ -80,3 +114,58 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
|
|||
|
||||
cli_cmd(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await
|
||||
}
|
||||
|
||||
fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
|
||||
let config = garage_util::config::read_config(config_file.clone()).map_err(|e| {
|
||||
Error::Message(format!(
|
||||
"Unable to read configuration file {}: {}",
|
||||
config_file.to_string_lossy(),
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
let node_key = garage_rpc::system::gen_node_key(&config.metadata_dir)
|
||||
.map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
|
||||
|
||||
let idstr = if let Some(addr) = config.rpc_public_addr {
|
||||
let idstr = format!("{}@{}", hex::encode(&node_key.public_key()), addr);
|
||||
println!("{}", idstr);
|
||||
idstr
|
||||
} else {
|
||||
let idstr = hex::encode(&node_key.public_key());
|
||||
println!("{}", idstr);
|
||||
|
||||
if !quiet {
|
||||
eprintln!("WARNING: I don't know the public address to reach this node.");
|
||||
eprintln!("In all of the instructions below, replace 127.0.0.1:3901 by the appropriate address and port.");
|
||||
}
|
||||
|
||||
format!("{}@127.0.0.1:3901", idstr)
|
||||
};
|
||||
|
||||
if !quiet {
|
||||
eprintln!("");
|
||||
eprintln!(
|
||||
"To instruct a node to connect to this node, run the following command on that node:"
|
||||
);
|
||||
eprintln!(" garage [-c <config file path>] node connect {}", idstr);
|
||||
eprintln!("");
|
||||
eprintln!("Or instruct them to connect from here by running:");
|
||||
eprintln!(
|
||||
" garage -c {} -h <remote node> node connect {}",
|
||||
config_file.to_string_lossy(),
|
||||
idstr
|
||||
);
|
||||
eprintln!(
|
||||
"where <remote_node> is their own node identifier in the format: <pubkey>@<ip>:<port>"
|
||||
);
|
||||
eprintln!("");
|
||||
eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:");
|
||||
eprintln!(" bootstrap_peers = [");
|
||||
eprintln!(" \"{}\",", idstr);
|
||||
eprintln!(" ...");
|
||||
eprintln!(" ]");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,8 +14,8 @@ pub use netapp::proto::*;
|
|||
pub use netapp::{NetApp, NodeID};
|
||||
|
||||
use garage_util::background::BackgroundRunner;
|
||||
use garage_util::error::Error;
|
||||
use garage_util::data::Uuid;
|
||||
use garage_util::error::Error;
|
||||
|
||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
|
|
|
|||
|
|
@ -18,8 +18,8 @@ use tokio::sync::Mutex;
|
|||
use netapp::endpoint::{Endpoint, EndpointHandler};
|
||||
use netapp::peering::fullmesh::FullMeshPeeringStrategy;
|
||||
use netapp::proto::*;
|
||||
use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
|
||||
use netapp::util::parse_and_resolve_peer_addr;
|
||||
use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
|
||||
|
||||
use garage_util::background::BackgroundRunner;
|
||||
use garage_util::data::Uuid;
|
||||
|
|
@ -110,7 +110,7 @@ pub struct KnownNodeInfo {
|
|||
pub status: NodeStatus,
|
||||
}
|
||||
|
||||
fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
|
||||
pub fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
|
||||
let mut key_file = metadata_dir.to_path_buf();
|
||||
key_file.push("node_key");
|
||||
if key_file.as_path().exists() {
|
||||
|
|
@ -246,8 +246,12 @@ impl System {
|
|||
}
|
||||
|
||||
async fn handle_connect(&self, node: &str) -> Result<SystemRpc, Error> {
|
||||
let (pubkey, addrs) = parse_and_resolve_peer_addr(node)
|
||||
.ok_or_else(|| Error::Message(format!("Unable to parse or resolve node specification: {}", node)))?;
|
||||
let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| {
|
||||
Error::Message(format!(
|
||||
"Unable to parse or resolve node specification: {}",
|
||||
node
|
||||
))
|
||||
})?;
|
||||
let mut errors = vec![];
|
||||
for ip in addrs.iter() {
|
||||
match self.netapp.clone().try_connect(*ip, pubkey).await {
|
||||
|
|
@ -257,7 +261,10 @@ impl System {
|
|||
}
|
||||
}
|
||||
}
|
||||
return Err(Error::Message(format!("Could not connect to specified peers. Errors: {:?}", errors)));
|
||||
return Err(Error::Message(format!(
|
||||
"Could not connect to specified peers. Errors: {:?}",
|
||||
errors
|
||||
)));
|
||||
}
|
||||
|
||||
fn handle_pull_config(&self) -> SystemRpc {
|
||||
|
|
@ -267,21 +274,23 @@ impl System {
|
|||
|
||||
fn handle_get_known_nodes(&self) -> SystemRpc {
|
||||
let node_status = self.node_status.read().unwrap();
|
||||
let known_nodes =
|
||||
self.fullmesh
|
||||
let known_nodes = self
|
||||
.fullmesh
|
||||
.get_peer_list()
|
||||
.iter()
|
||||
.map(|n| KnownNodeInfo {
|
||||
id: n.id.into(),
|
||||
addr: n.addr,
|
||||
is_up: n.is_up(),
|
||||
status: node_status.get(&n.id.into()).cloned().map(|(_, st)| st).unwrap_or(
|
||||
NodeStatus {
|
||||
status: node_status
|
||||
.get(&n.id.into())
|
||||
.cloned()
|
||||
.map(|(_, st)| st)
|
||||
.unwrap_or(NodeStatus {
|
||||
hostname: "?".to_string(),
|
||||
replication_factor: 0,
|
||||
config_version: 0,
|
||||
},
|
||||
),
|
||||
}),
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
SystemRpc::ReturnKnownNodes(known_nodes)
|
||||
|
|
@ -330,14 +339,14 @@ impl System {
|
|||
drop(update_ring);
|
||||
|
||||
let self2 = self.clone();
|
||||
let adv2 = adv.clone();
|
||||
let adv = adv.clone();
|
||||
self.background.spawn_cancellable(async move {
|
||||
self2
|
||||
.rpc
|
||||
.broadcast(
|
||||
&self2.system_endpoint,
|
||||
SystemRpc::AdvertiseConfig(adv2),
|
||||
RequestStrategy::with_priority(PRIO_NORMAL),
|
||||
SystemRpc::AdvertiseConfig(adv),
|
||||
RequestStrategy::with_priority(PRIO_HIGH),
|
||||
)
|
||||
.await;
|
||||
Ok(())
|
||||
|
|
|
|||
|
|
@ -28,11 +28,7 @@ impl TableReplication for TableFullReplication {
|
|||
|
||||
fn write_nodes(&self, _hash: &Hash) -> Vec<Uuid> {
|
||||
let ring = self.system.ring.borrow();
|
||||
ring.config
|
||||
.members
|
||||
.keys()
|
||||
.cloned()
|
||||
.collect::<Vec<_>>()
|
||||
ring.config.members.keys().cloned().collect::<Vec<_>>()
|
||||
}
|
||||
fn write_quorum(&self) -> usize {
|
||||
let nmembers = self.system.ring.borrow().config.members.len();
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ use std::path::PathBuf;
|
|||
use serde::de::Error as SerdeError;
|
||||
use serde::{de, Deserialize};
|
||||
|
||||
use netapp::NodeID;
|
||||
use netapp::util::parse_and_resolve_peer_addr;
|
||||
use netapp::NodeID;
|
||||
|
||||
use crate::error::Error;
|
||||
|
||||
|
|
@ -117,8 +117,9 @@ where
|
|||
let mut ret = vec![];
|
||||
|
||||
for peer in <Vec<&str>>::deserialize(deserializer)? {
|
||||
let (pubkey, addrs) = parse_and_resolve_peer_addr(peer)
|
||||
.ok_or_else(|| D::Error::custom(format!("Unable to parse or resolve peer: {}", peer)))?;
|
||||
let (pubkey, addrs) = parse_and_resolve_peer_addr(peer).ok_or_else(|| {
|
||||
D::Error::custom(format!("Unable to parse or resolve peer: {}", peer))
|
||||
})?;
|
||||
for ip in addrs {
|
||||
ret.push((pubkey.clone(), ip));
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue