Adapt tests to new syntax with public keys

This commit is contained in:
Alex 2021-10-19 16:16:10 +02:00
parent 65070f3c05
commit a8ae78af0a
No known key found for this signature in database
GPG key ID: EDABF9711E244EB1
10 changed files with 191 additions and 78 deletions

View file

@ -9,11 +9,11 @@ GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
NIX_RELEASE="${REPO_FOLDER}/result/bin/" NIX_RELEASE="${REPO_FOLDER}/result/bin/"
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH" PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
garage bucket create eprouvette garage -c /tmp/config.1.toml bucket create eprouvette
KEY_INFO=`garage key new --name opérateur` KEY_INFO=$(garage -c /tmp/config.1.toml key new --name opérateur)
ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'` ACCESS_KEY=`echo $KEY_INFO|grep -Po 'GK[a-f0-9]+'`
SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'` SECRET_KEY=`echo $KEY_INFO|grep -Po 'Secret key: [a-f0-9]+'|grep -Po '[a-f0-9]+$'`
garage bucket allow eprouvette --read --write --key $ACCESS_KEY garage -c /tmp/config.1.toml bucket allow eprouvette --read --write --key $ACCESS_KEY
echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3 echo "$ACCESS_KEY $SECRET_KEY" > /tmp/garage.s3
echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3." echo "Bucket s3://eprouvette created. Credentials stored in /tmp/garage.s3."

View file

@ -17,6 +17,10 @@ MAIN_LABEL="\e[${FANCYCOLORS[0]}[main]\e[49m"
WHICH_GARAGE=$(which garage || exit 1) WHICH_GARAGE=$(which garage || exit 1)
echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n" echo -en "${MAIN_LABEL} Found garage at: ${WHICH_GARAGE}\n"
NETWORK_SECRET="$(openssl rand -hex 32)"
# <<<<<<<<< BEGIN FOR LOOP ON NODES
for count in $(seq 1 3); do for count in $(seq 1 3); do
CONF_PATH="/tmp/config.$count.toml" CONF_PATH="/tmp/config.$count.toml"
LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m" LABEL="\e[${FANCYCOLORS[$count]}[$count]\e[49m"
@ -26,13 +30,11 @@ block_size = 1048576 # objects are split in blocks of maximum this number of b
metadata_dir = "/tmp/garage-meta-$count" metadata_dir = "/tmp/garage-meta-$count"
data_dir = "/tmp/garage-data-$count" data_dir = "/tmp/garage-data-$count"
rpc_bind_addr = "0.0.0.0:$((3900+$count))" # the port other Garage nodes will use to talk to this node rpc_bind_addr = "0.0.0.0:$((3900+$count))" # the port other Garage nodes will use to talk to this node
bootstrap_peers = [ rpc_public_addr = "127.0.0.1:$((3900+$count))"
"127.0.0.1:3901", bootstrap_peers = []
"127.0.0.1:3902",
"127.0.0.1:3903"
]
max_concurrent_rpc_requests = 12 max_concurrent_rpc_requests = 12
replication_mode = "3" replication_mode = "3"
rpc_secret = "$NETWORK_SECRET"
[s3_api] [s3_api]
api_bind_addr = "0.0.0.0:$((3910+$count))" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part. api_bind_addr = "0.0.0.0:$((3910+$count))" # the S3 API port, HTTP without TLS. Add a reverse proxy for the TLS part.
@ -61,11 +63,21 @@ if [ -z "$SKIP_HTTPS" ]; then
socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 & socat openssl-listen:4443,reuseaddr,fork,cert=/tmp/garagessl/test.pem,verify=0 tcp4-connect:localhost:3911 &
fi fi
(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) & (garage -c /tmp/config.$count.toml server 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
done
# >>>>>>>>>>>>>>>> END FOR LOOP ON NODES
sleep 5
# Establish connections between nodes
for count in $(seq 1 3); do
NODE=$(garage -c /tmp/config.$count.toml node-id -q)
for count2 in $(seq 1 3); do
garage -c /tmp/config.$count2.toml node connect $NODE
done
done done
RETRY=120 RETRY=120
until garage status 2>&1|grep -q Healthy ; do until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
(( RETRY-- )) (( RETRY-- ))
if (( RETRY <= 0 )); then if (( RETRY <= 0 )); then
echo -en "${MAIN_LABEL} Garage did not start" echo -en "${MAIN_LABEL} Garage did not start"
@ -74,6 +86,7 @@ until garage status 2>&1|grep -q Healthy ; do
echo -en "${MAIN_LABEL} cluster starting...\n" echo -en "${MAIN_LABEL} cluster starting...\n"
sleep 1 sleep 1
done done
echo -en "${MAIN_LABEL} cluster started\n" echo -en "${MAIN_LABEL} cluster started\n"
wait wait

View file

@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
sleep 5 sleep 5
RETRY=120 RETRY=120
until garage status 2>&1|grep -q Healthy ; do until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
(( RETRY-- )) (( RETRY-- ))
if (( RETRY <= 0 )); then if (( RETRY <= 0 )); then
echo "garage did not start in time, failing." echo "garage did not start in time, failing."
@ -21,10 +21,10 @@ until garage status 2>&1|grep -q Healthy ; do
sleep 1 sleep 1
done done
garage status \ garage -c /tmp/config.1.toml status \
| grep UNCONFIGURED \ | grep UNCONFIGURED \
| grep -Po '^[0-9a-f]+' \ | grep -Po '^[0-9a-f]+' \
| while read id; do | while read id; do
garage node configure -z dc1 -c 1 $id garage -c /tmp/config.1.toml node configure -z dc1 -c 1 $id
done done

View file

@ -21,9 +21,9 @@ ${SCRIPT_FOLDER}/dev-configure.sh
${SCRIPT_FOLDER}/dev-bucket.sh ${SCRIPT_FOLDER}/dev-bucket.sh
which garage which garage
garage status garage -c /tmp/config.1.toml status
garage key list garage -c /tmp/config.1.toml key list
garage bucket list garage -c /tmp/config.1.toml bucket list
dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline storage (< INLINE_THRESHOLD = 3072 bytes) dd if=/dev/urandom of=/tmp/garage.1.rnd bs=1k count=2 # No multipart, inline storage (< INLINE_THRESHOLD = 3072 bytes)
dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked dd if=/dev/urandom of=/tmp/garage.2.rnd bs=1M count=5 # No multipart but file will be chunked

View file

@ -1,5 +1,4 @@
use std::collections::HashSet; use std::collections::HashSet;
use std::path::PathBuf;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use structopt::StructOpt; use structopt::StructOpt;
@ -21,7 +20,12 @@ use crate::admin_rpc::*;
pub enum Command { pub enum Command {
/// Run Garage server /// Run Garage server
#[structopt(name = "server")] #[structopt(name = "server")]
Server(ServerOpt), Server,
/// Print identifier (public key) of this garage node.
/// Generates a new keypair if necessary.
#[structopt(name = "node-id")]
NodeId(NodeIdOpt),
/// Get network status /// Get network status
#[structopt(name = "status")] #[structopt(name = "status")]
@ -48,13 +52,6 @@ pub enum Command {
Stats(StatsOpt), Stats(StatsOpt),
} }
#[derive(StructOpt, Debug)]
pub struct ServerOpt {
/// Configuration file
#[structopt(short = "c", long = "config", default_value = "./config.toml")]
pub config_file: PathBuf,
}
#[derive(StructOpt, Debug)] #[derive(StructOpt, Debug)]
pub enum NodeOperation { pub enum NodeOperation {
/// Connect to Garage node that is currently isolated from the system /// Connect to Garage node that is currently isolated from the system
@ -70,6 +67,13 @@ pub enum NodeOperation {
Remove(RemoveNodeOpt), Remove(RemoveNodeOpt),
} }
#[derive(StructOpt, Debug)]
pub struct NodeIdOpt {
/// Do not print usage instructions to stderr
#[structopt(short = "q", long = "quiet")]
pub(crate) quiet: bool,
}
#[derive(StructOpt, Debug)] #[derive(StructOpt, Debug)]
pub struct ConnectNodeOpt { pub struct ConnectNodeOpt {
/// Node public key and address, in the format: /// Node public key and address, in the format:
@ -384,7 +388,8 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
.any(|(id, _)| !status_keys.contains(id)); .any(|(id, _)| !status_keys.contains(id));
if failure_case_1 || failure_case_2 { if failure_case_1 || failure_case_2 {
println!("\nFailed nodes:"); println!("\nFailed nodes:");
let mut failed_nodes = vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()]; let mut failed_nodes =
vec!["ID\tHostname\tAddress\tTag\tZone\tCapacity\tLast seen".to_string()];
for adv in status.iter().filter(|adv| !adv.is_up) { for adv in status.iter().filter(|adv| !adv.is_up) {
if let Some(cfg) = config.members.get(&adv.id) { if let Some(cfg) = config.members.get(&adv.id) {
failed_nodes.push(format!( failed_nodes.push(format!(
@ -421,14 +426,15 @@ pub async fn cmd_connect(
rpc_host: NodeID, rpc_host: NodeID,
args: ConnectNodeOpt, args: ConnectNodeOpt,
) -> Result<(), Error> { ) -> Result<(), Error> {
match rpc_cli.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL).await?? { match rpc_cli
.call(&rpc_host, &SystemRpc::Connect(args.node), PRIO_NORMAL)
.await??
{
SystemRpc::Ok => { SystemRpc::Ok => {
println!("Success."); println!("Success.");
Ok(()) Ok(())
} }
r => { r => Err(Error::BadRpc(format!("Unexpected response: {:?}", r))),
Err(Error::BadRpc(format!("Unexpected response: {:?}", r)))
}
} }
} }
@ -654,4 +660,3 @@ pub fn find_matching_node(
Ok(candidates[0]) Ok(candidates[0])
} }
} }

View file

@ -9,9 +9,11 @@ mod cli;
mod repair; mod repair;
mod server; mod server;
use std::path::PathBuf;
use structopt::StructOpt; use structopt::StructOpt;
use netapp::util::parse_peer_addr; use netapp::util::parse_and_resolve_peer_addr;
use netapp::NetworkKey; use netapp::NetworkKey;
use garage_util::error::Error; use garage_util::error::Error;
@ -34,6 +36,10 @@ struct Opt {
#[structopt(short = "s", long = "rpc-secret")] #[structopt(short = "s", long = "rpc-secret")]
pub rpc_secret: Option<String>, pub rpc_secret: Option<String>,
/// Configuration file (garage.toml)
#[structopt(short = "c", long = "config", default_value = "/etc/garage.toml")]
pub config_file: PathBuf,
#[structopt(subcommand)] #[structopt(subcommand)]
cmd: Command, cmd: Command,
} }
@ -45,16 +51,18 @@ async fn main() {
let opt = Opt::from_args(); let opt = Opt::from_args();
let res = if let Command::Server(server_opt) = opt.cmd { let res = match opt.cmd {
Command::Server => {
// Abort on panic (same behavior as in Go) // Abort on panic (same behavior as in Go)
std::panic::set_hook(Box::new(|panic_info| { std::panic::set_hook(Box::new(|panic_info| {
error!("{}", panic_info.to_string()); error!("{}", panic_info.to_string());
std::process::abort(); std::process::abort();
})); }));
server::run_server(server_opt.config_file).await server::run_server(opt.config_file).await
} else { }
cli_command(opt).await Command::NodeId(node_id_opt) => node_id_command(opt.config_file, node_id_opt.quiet),
_ => cli_command(opt).await,
}; };
if let Err(e) = res { if let Err(e) = res {
@ -63,16 +71,42 @@ async fn main() {
} }
async fn cli_command(opt: Opt) -> Result<(), Error> { async fn cli_command(opt: Opt) -> Result<(), Error> {
let net_key_hex_str = &opt.rpc_secret.expect("No RPC secret provided"); let config = if opt.rpc_secret.is_none() || opt.rpc_host.is_none() {
Some(garage_util::config::read_config(opt.config_file.clone())
.map_err(|e| Error::Message(format!("Unable to read configuration file {}: {}. Configuration file is needed because -h or -s is not provided on the command line.", opt.config_file.to_string_lossy(), e)))?)
} else {
None
};
// Find and parse network RPC secret
let net_key_hex_str = opt
.rpc_secret
.as_ref()
.or_else(|| config.as_ref().map(|c| &c.rpc_secret))
.expect("No RPC secret provided");
let network_key = NetworkKey::from_slice( let network_key = NetworkKey::from_slice(
&hex::decode(net_key_hex_str).expect("Invalid RPC secret key (bad hex)")[..], &hex::decode(net_key_hex_str).expect("Invalid RPC secret key (bad hex)")[..],
) )
.expect("Invalid RPC secret provided (wrong length)"); .expect("Invalid RPC secret provided (wrong length)");
// Generate a temporary keypair for our RPC client
let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair(); let (_pk, sk) = sodiumoxide::crypto::sign::ed25519::gen_keypair();
let netapp = NetApp::new(network_key, sk); let netapp = NetApp::new(network_key, sk);
let (id, addr) =
parse_peer_addr(&opt.rpc_host.expect("No RPC host provided")).expect("Invalid RPC host"); // Find and parse the address of the target host
let (id, addr) = if let Some(h) = opt.rpc_host {
let (id, addrs) = parse_and_resolve_peer_addr(&h).expect("Invalid RPC host");
(id, addrs[0])
} else if let Some(a) = config.as_ref().map(|c| c.rpc_public_addr).flatten() {
let node_key = garage_rpc::system::gen_node_key(&config.unwrap().metadata_dir)
.map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
(node_key.public_key(), a)
} else {
return Err(Error::Message("No RPC host provided".into()));
};
// Connect to target host
netapp.clone().try_connect(addr, id).await?; netapp.clone().try_connect(addr, id).await?;
let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into()); let system_rpc_endpoint = netapp.endpoint::<SystemRpc, ()>(SYSTEM_RPC_PATH.into());
@ -80,3 +114,58 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
cli_cmd(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await cli_cmd(opt.cmd, &system_rpc_endpoint, &admin_rpc_endpoint, id).await
} }
fn node_id_command(config_file: PathBuf, quiet: bool) -> Result<(), Error> {
let config = garage_util::config::read_config(config_file.clone()).map_err(|e| {
Error::Message(format!(
"Unable to read configuration file {}: {}",
config_file.to_string_lossy(),
e
))
})?;
let node_key = garage_rpc::system::gen_node_key(&config.metadata_dir)
.map_err(|e| Error::Message(format!("Unable to read or generate node key: {}", e)))?;
let idstr = if let Some(addr) = config.rpc_public_addr {
let idstr = format!("{}@{}", hex::encode(&node_key.public_key()), addr);
println!("{}", idstr);
idstr
} else {
let idstr = hex::encode(&node_key.public_key());
println!("{}", idstr);
if !quiet {
eprintln!("WARNING: I don't know the public address to reach this node.");
eprintln!("In all of the instructions below, replace 127.0.0.1:3901 by the appropriate address and port.");
}
format!("{}@127.0.0.1:3901", idstr)
};
if !quiet {
eprintln!("");
eprintln!(
"To instruct a node to connect to this node, run the following command on that node:"
);
eprintln!(" garage [-c <config file path>] node connect {}", idstr);
eprintln!("");
eprintln!("Or instruct them to connect from here by running:");
eprintln!(
" garage -c {} -h <remote node> node connect {}",
config_file.to_string_lossy(),
idstr
);
eprintln!(
"where <remote_node> is their own node identifier in the format: <pubkey>@<ip>:<port>"
);
eprintln!("");
eprintln!("This node identifier can also be added as a bootstrap node in other node's garage.toml files:");
eprintln!(" bootstrap_peers = [");
eprintln!(" \"{}\",", idstr);
eprintln!(" ...");
eprintln!(" ]");
}
Ok(())
}

View file

@ -14,8 +14,8 @@ pub use netapp::proto::*;
pub use netapp::{NetApp, NodeID}; pub use netapp::{NetApp, NodeID};
use garage_util::background::BackgroundRunner; use garage_util::background::BackgroundRunner;
use garage_util::error::Error;
use garage_util::data::Uuid; use garage_util::data::Uuid;
use garage_util::error::Error;
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10); const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);

View file

@ -18,8 +18,8 @@ use tokio::sync::Mutex;
use netapp::endpoint::{Endpoint, EndpointHandler}; use netapp::endpoint::{Endpoint, EndpointHandler};
use netapp::peering::fullmesh::FullMeshPeeringStrategy; use netapp::peering::fullmesh::FullMeshPeeringStrategy;
use netapp::proto::*; use netapp::proto::*;
use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
use netapp::util::parse_and_resolve_peer_addr; use netapp::util::parse_and_resolve_peer_addr;
use netapp::{NetApp, NetworkKey, NodeID, NodeKey};
use garage_util::background::BackgroundRunner; use garage_util::background::BackgroundRunner;
use garage_util::data::Uuid; use garage_util::data::Uuid;
@ -110,7 +110,7 @@ pub struct KnownNodeInfo {
pub status: NodeStatus, pub status: NodeStatus,
} }
fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> { pub fn gen_node_key(metadata_dir: &Path) -> Result<NodeKey, Error> {
let mut key_file = metadata_dir.to_path_buf(); let mut key_file = metadata_dir.to_path_buf();
key_file.push("node_key"); key_file.push("node_key");
if key_file.as_path().exists() { if key_file.as_path().exists() {
@ -246,8 +246,12 @@ impl System {
} }
async fn handle_connect(&self, node: &str) -> Result<SystemRpc, Error> { async fn handle_connect(&self, node: &str) -> Result<SystemRpc, Error> {
let (pubkey, addrs) = parse_and_resolve_peer_addr(node) let (pubkey, addrs) = parse_and_resolve_peer_addr(node).ok_or_else(|| {
.ok_or_else(|| Error::Message(format!("Unable to parse or resolve node specification: {}", node)))?; Error::Message(format!(
"Unable to parse or resolve node specification: {}",
node
))
})?;
let mut errors = vec![]; let mut errors = vec![];
for ip in addrs.iter() { for ip in addrs.iter() {
match self.netapp.clone().try_connect(*ip, pubkey).await { match self.netapp.clone().try_connect(*ip, pubkey).await {
@ -257,7 +261,10 @@ impl System {
} }
} }
} }
return Err(Error::Message(format!("Could not connect to specified peers. Errors: {:?}", errors))); return Err(Error::Message(format!(
"Could not connect to specified peers. Errors: {:?}",
errors
)));
} }
fn handle_pull_config(&self) -> SystemRpc { fn handle_pull_config(&self) -> SystemRpc {
@ -267,21 +274,23 @@ impl System {
fn handle_get_known_nodes(&self) -> SystemRpc { fn handle_get_known_nodes(&self) -> SystemRpc {
let node_status = self.node_status.read().unwrap(); let node_status = self.node_status.read().unwrap();
let known_nodes = let known_nodes = self
self.fullmesh .fullmesh
.get_peer_list() .get_peer_list()
.iter() .iter()
.map(|n| KnownNodeInfo { .map(|n| KnownNodeInfo {
id: n.id.into(), id: n.id.into(),
addr: n.addr, addr: n.addr,
is_up: n.is_up(), is_up: n.is_up(),
status: node_status.get(&n.id.into()).cloned().map(|(_, st)| st).unwrap_or( status: node_status
NodeStatus { .get(&n.id.into())
.cloned()
.map(|(_, st)| st)
.unwrap_or(NodeStatus {
hostname: "?".to_string(), hostname: "?".to_string(),
replication_factor: 0, replication_factor: 0,
config_version: 0, config_version: 0,
}, }),
),
}) })
.collect::<Vec<_>>(); .collect::<Vec<_>>();
SystemRpc::ReturnKnownNodes(known_nodes) SystemRpc::ReturnKnownNodes(known_nodes)
@ -330,14 +339,14 @@ impl System {
drop(update_ring); drop(update_ring);
let self2 = self.clone(); let self2 = self.clone();
let adv2 = adv.clone(); let adv = adv.clone();
self.background.spawn_cancellable(async move { self.background.spawn_cancellable(async move {
self2 self2
.rpc .rpc
.broadcast( .broadcast(
&self2.system_endpoint, &self2.system_endpoint,
SystemRpc::AdvertiseConfig(adv2), SystemRpc::AdvertiseConfig(adv),
RequestStrategy::with_priority(PRIO_NORMAL), RequestStrategy::with_priority(PRIO_HIGH),
) )
.await; .await;
Ok(()) Ok(())

View file

@ -28,11 +28,7 @@ impl TableReplication for TableFullReplication {
fn write_nodes(&self, _hash: &Hash) -> Vec<Uuid> { fn write_nodes(&self, _hash: &Hash) -> Vec<Uuid> {
let ring = self.system.ring.borrow(); let ring = self.system.ring.borrow();
ring.config ring.config.members.keys().cloned().collect::<Vec<_>>()
.members
.keys()
.cloned()
.collect::<Vec<_>>()
} }
fn write_quorum(&self) -> usize { fn write_quorum(&self) -> usize {
let nmembers = self.system.ring.borrow().config.members.len(); let nmembers = self.system.ring.borrow().config.members.len();

View file

@ -6,8 +6,8 @@ use std::path::PathBuf;
use serde::de::Error as SerdeError; use serde::de::Error as SerdeError;
use serde::{de, Deserialize}; use serde::{de, Deserialize};
use netapp::NodeID;
use netapp::util::parse_and_resolve_peer_addr; use netapp::util::parse_and_resolve_peer_addr;
use netapp::NodeID;
use crate::error::Error; use crate::error::Error;
@ -117,8 +117,9 @@ where
let mut ret = vec![]; let mut ret = vec![];
for peer in <Vec<&str>>::deserialize(deserializer)? { for peer in <Vec<&str>>::deserialize(deserializer)? {
let (pubkey, addrs) = parse_and_resolve_peer_addr(peer) let (pubkey, addrs) = parse_and_resolve_peer_addr(peer).ok_or_else(|| {
.ok_or_else(|| D::Error::custom(format!("Unable to parse or resolve peer: {}", peer)))?; D::Error::custom(format!("Unable to parse or resolve peer: {}", peer))
})?;
for ip in addrs { for ip in addrs {
ret.push((pubkey.clone(), ip)); ret.push((pubkey.clone(), ip));
} }