feature: Register consul services with agent API #567

Merged
lx merged 10 commits from unrob/garage:roberto/consul-agent-registration into main 2023-06-02 14:35:00 +00:00
8 changed files with 194 additions and 340 deletions
Showing only changes of commit fd7dbea5b8 - Show all commits

View file

@ -35,24 +35,19 @@ bootstrap_peers = [
[consul_discovery] [consul_discovery]
mode = "node"
unrob marked this conversation as resolved Outdated
Outdated
Review

Is this the old name of consul_http_api ?

I'd suggest calling this parameter api, as it is already under the [consul_discovery] section so there is no need to specify that it is related to Consul.

For all parameters: take care of making sure the names in the docs are consistent with the actual code

Is this the old name of `consul_http_api` ? I'd suggest calling this parameter `api`, as it is already under the `[consul_discovery]` section so there is no need to specify that it is related to Consul. For all parameters: take care of making sure the names in the docs are consistent with the actual code
consul_http_addr = "http://127.0.0.1:8500" consul_http_addr = "http://127.0.0.1:8500"
service_name = "garage-daemon" service_name = "garage-daemon"
ca_cert = "/etc/consul/consul-ca.crt" ca_cert = "/etc/consul/consul-ca.crt"
client_cert = "/etc/consul/consul-client.crt" client_cert = "/etc/consul/consul-client.crt"
client_key = "/etc/consul/consul-key.crt" client_key = "/etc/consul/consul-key.crt"
# for `service` mode, unset client_cert and client_key, and optionally enable `consul_http_token`
unrob marked this conversation as resolved Outdated
Outdated
Review

I think you meant for agent API mode instead?

I think you meant for `agent` API mode instead?
# consul_http_token = "abcdef-01234-56789"
unrob marked this conversation as resolved Outdated
Outdated
Review

Similarly, I'd suggest calling this simply token

Similarly, I'd suggest calling this simply `token`
tls_skip_verify = false tls_skip_verify = false
[consul_service_discovery]
consul_http_addr = "https://127.0.0.1:8501"
consul_http_token = "abcdef-01234-56789"
service_name = "garage"
ca_cert = "/etc/consul/consul-ca.crt"
tls_skip_verify = false
# tags to add to the published service
tags = [ "dns-enabled" ] tags = [ "dns-enabled" ]
# additional service meta to send along registration
meta = { dns-acl = "allow trusted" } meta = { dns-acl = "allow trusted" }
[kubernetes_discovery] [kubernetes_discovery]
namespace = "garage" namespace = "garage"
service_name = "garage-daemon" service_name = "garage-daemon"
@ -323,6 +318,12 @@ Garage supports discovering other nodes of the cluster using Consul. For this
to work correctly, nodes need to know their IP address by which they can be to work correctly, nodes need to know their IP address by which they can be
reached by other nodes of the cluster, which should be set in `rpc_public_addr`. reached by other nodes of the cluster, which should be set in `rpc_public_addr`.
### `mode`
Two modes of service discovery are supported: `node` and `service`. `node`, the default will register a service using
the `/v1/catalog` endpoints and mTLS (if `client_cert` and `client_key` are provided). `service` mode uses the
`v1/agent` endpoints instead, where an optional `consul_http_token` may be provided.
### `consul_http_addr` and `service_name` ### `consul_http_addr` and `service_name`
The `consul_http_addr` parameter should be set to the full HTTP(S) address of the Consul server. The `consul_http_addr` parameter should be set to the full HTTP(S) address of the Consul server.
unrob marked this conversation as resolved Outdated
Outdated
Review

token instead of consul_http_token

`token` instead of `consul_http_token`
@ -334,7 +335,8 @@ RPC ports are announced.
### `client_cert`, `client_key` ### `client_cert`, `client_key`
TLS client certificate and client key to use when communicating with Consul over TLS. Both are mandatory when doing so. `node` mode only. TLS client certificate and client key to use when communicating with Consul over TLS.
Both are mandatory when doing so.
### `ca_cert` ### `ca_cert`
@ -345,6 +347,29 @@ TLS CA certificate to use when communicating with Consul over TLS.
Skip server hostname verification in TLS handshake. Skip server hostname verification in TLS handshake.
`ca_cert` is ignored when this is set. `ca_cert` is ignored when this is set.
### `consul_http_token`
`service` mode only. Uses the provided token for communication with Consul. The policy assigned to this token
should at least have these rules:
```hcl
// the `service_name` specified above
service "garage" {
policy = "write"
}
service_prefix "" {
policy = "read"
}
node_prefix "" {
policy = "read"
}
```
### `tags` and `meta`
Additional list of tags and map of service meta to add during service registration.
## The `[kubernetes_discovery]` section ## The `[kubernetes_discovery]` section

View file

@ -88,7 +88,6 @@ sqlite = [ "garage_model/sqlite" ]
# Automatic registration and discovery via Consul API # Automatic registration and discovery via Consul API
consul-discovery = [ "garage_rpc/consul-discovery" ] consul-discovery = [ "garage_rpc/consul-discovery" ]
consul-service-discovery = [ "garage_rpc/consul-service-discovery" ]
# Automatic registration and discovery via Kubernetes API # Automatic registration and discovery via Kubernetes API
kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ] kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ]
# Prometheus exporter (/metrics endpoint). # Prometheus exporter (/metrics endpoint).

View file

@ -95,8 +95,6 @@ async fn main() {
"sqlite", "sqlite",
#[cfg(feature = "consul-discovery")] #[cfg(feature = "consul-discovery")]
"consul-discovery", "consul-discovery",
#[cfg(feature = "consul-service-discovery")]
"consul-service-discovery",
#[cfg(feature = "kubernetes-discovery")] #[cfg(feature = "kubernetes-discovery")]
"kubernetes-discovery", "kubernetes-discovery",
#[cfg(feature = "metrics")] #[cfg(feature = "metrics")]

View file

@ -9,6 +9,9 @@ use serde::{Deserialize, Serialize};
use netapp::NodeID; use netapp::NodeID;
use garage_util::config::ConsulDiscoveryConfig; use garage_util::config::ConsulDiscoveryConfig;
use garage_util::config::ConsulDiscoveryMode;
const META_PREFIX: &str = "fr-deuxfleurs-garagehq";
#[derive(Deserialize, Clone, Debug)] #[derive(Deserialize, Clone, Debug)]
struct ConsulQueryEntry { struct ConsulQueryEntry {
@ -18,6 +21,8 @@ struct ConsulQueryEntry {
service_port: u16, service_port: u16,
#[serde(rename = "NodeMeta")] #[serde(rename = "NodeMeta")]
node_meta: HashMap<String, String>, node_meta: HashMap<String, String>,
#[serde(rename = "ServiceMeta")]
service_meta: HashMap<String, String>,
} }
#[derive(Serialize, Clone, Debug)] #[derive(Serialize, Clone, Debug)]
@ -29,14 +34,30 @@ struct ConsulPublishEntry {
#[serde(rename = "NodeMeta")] #[serde(rename = "NodeMeta")]
node_meta: HashMap<String, String>, node_meta: HashMap<String, String>,
#[serde(rename = "Service")] #[serde(rename = "Service")]
service: ConsulPublishService, service: ConsulPublishCatalogService,
}
#[derive(Serialize, Clone, Debug)]
struct ConsulPublishCatalogService {
#[serde(rename = "ID")]
service_id: String,
#[serde(rename = "Service")]
service_name: String,
#[serde(rename = "Tags")]
tags: Vec<String>,
#[serde(rename = "Meta")]
service_meta: HashMap<String, String>,
unrob marked this conversation as resolved Outdated
Outdated
Review

It's called service_meta here but meta in the other definition below. Let's just call it meta everywhere

It's called `service_meta` here but `meta` in the other definition below. Let's just call it `meta` everywhere
#[serde(rename = "Address")]
address: IpAddr,
#[serde(rename = "Port")]
port: u16,
} }
#[derive(Serialize, Clone, Debug)] #[derive(Serialize, Clone, Debug)]
struct ConsulPublishService { struct ConsulPublishService {
#[serde(rename = "ID")] #[serde(rename = "ID")]
service_id: String, service_id: String,
#[serde(rename = "Service")] #[serde(rename = "Name")]
service_name: String, service_name: String,
#[serde(rename = "Tags")] #[serde(rename = "Tags")]
tags: Vec<String>, tags: Vec<String>,
@ -44,10 +65,11 @@ struct ConsulPublishService {
address: IpAddr, address: IpAddr,
#[serde(rename = "Port")] #[serde(rename = "Port")]
port: u16, port: u16,
#[serde(rename = "Meta")]
meta: HashMap<String, String>,
} }
// ---- // ----
pub struct ConsulDiscovery { pub struct ConsulDiscovery {
config: ConsulDiscoveryConfig, config: ConsulDiscoveryConfig,
client: reqwest::Client, client: reqwest::Client,
@ -55,7 +77,20 @@ pub struct ConsulDiscovery {
impl ConsulDiscovery { impl ConsulDiscovery {
pub fn new(config: ConsulDiscoveryConfig) -> Result<Self, ConsulError> { pub fn new(config: ConsulDiscoveryConfig) -> Result<Self, ConsulError> {
let client = match (&config.client_cert, &config.client_key) { let mut builder: reqwest::ClientBuilder = reqwest::Client::builder();
unrob marked this conversation as resolved Outdated
Outdated
Review

add .use_rustls_tls() here and remove all other instances below (we are using rustls in all cases)

add `.use_rustls_tls()` here and remove all other instances below (we are using rustls in all cases)
builder = builder.danger_accept_invalid_certs(config.tls_skip_verify);
let client: reqwest::Client = match &config.mode {
ConsulDiscoveryMode::Node => {
if let Some(ca_cert) = &config.ca_cert {
let mut ca_cert_buf = vec![];
File::open(ca_cert)?.read_to_end(&mut ca_cert_buf)?;
builder = builder.use_rustls_tls();
builder = builder
.add_root_certificate(reqwest::Certificate::from_pem(&ca_cert_buf[..])?);
}
match (&config.client_cert, &config.client_key) {
(Some(client_cert), Some(client_key)) => { (Some(client_cert), Some(client_key)) => {
let mut client_cert_buf = vec![]; let mut client_cert_buf = vec![];
File::open(client_cert)?.read_to_end(&mut client_cert_buf)?; File::open(client_cert)?.read_to_end(&mut client_cert_buf)?;
@ -67,30 +102,35 @@ impl ConsulDiscovery {
&[&client_cert_buf[..], &client_key_buf[..]].concat()[..], &[&client_cert_buf[..], &client_key_buf[..]].concat()[..],
)?; )?;
if config.tls_skip_verify { builder = builder.use_rustls_tls();
reqwest::Client::builder() builder = builder.identity(identity);
.use_rustls_tls() }
.danger_accept_invalid_certs(true) (None, None) => {}
.identity(identity) _ => return Err(ConsulError::InvalidTLSConfig),
.build()? }
} else if let Some(ca_cert) = &config.ca_cert {
builder.build()?
unrob marked this conversation as resolved Outdated
Outdated
Review

refactoring: move builder.build()? after the match

This means that there is no let client = match, just a match block that mutates builder

and after the match block there is let client = builder.build()?;

refactoring: move `builder.build()?` after the match This means that there is no `let client = match`, just a match block that mutates `builder` and after the match block there is `let client = builder.build()?;`
}
ConsulDiscoveryMode::Service => {
if let Some(ca_cert) = &config.ca_cert {
let mut ca_cert_buf = vec![]; let mut ca_cert_buf = vec![];
File::open(ca_cert)?.read_to_end(&mut ca_cert_buf)?; File::open(ca_cert)?.read_to_end(&mut ca_cert_buf)?;
builder = builder
.add_root_certificate(reqwest::Certificate::from_pem(&ca_cert_buf[..])?);
builder = builder.use_rustls_tls();
}
reqwest::Client::builder() if let Some(token) = &config.consul_http_token {
.use_rustls_tls() let mut headers = reqwest::header::HeaderMap::new();
.add_root_certificate(reqwest::Certificate::from_pem(&ca_cert_buf[..])?) headers.insert(
.identity(identity) "x-consul-token",
.build()? reqwest::header::HeaderValue::from_str(&token)?,
} else { );
reqwest::Client::builder() builder = builder.default_headers(headers);
.use_rustls_tls()
.identity(identity)
.build()?
} }
builder.build()?
} }
(None, None) => reqwest::Client::new(),
_ => return Err(ConsulError::InvalidTLSConfig),
}; };
Ok(Self { client, config }) Ok(Self { client, config })
@ -110,9 +150,12 @@ impl ConsulDiscovery {
let mut ret = vec![]; let mut ret = vec![];
for ent in entries { for ent in entries {
let ip = ent.address.parse::<IpAddr>().ok(); let ip = ent.address.parse::<IpAddr>().ok();
let pubkey = ent let pubkey = match &self.config.mode {
.node_meta ConsulDiscoveryMode::Node => ent.node_meta.get("pubkey"),
.get("pubkey") ConsulDiscoveryMode::Service => {
ent.service_meta.get(&format!("{}-pubkey", META_PREFIX))
}
}
.and_then(|k| hex::decode(k).ok()) .and_then(|k| hex::decode(k).ok())
.and_then(|k| NodeID::from_slice(&k[..])); .and_then(|k| NodeID::from_slice(&k[..]));
if let (Some(ip), Some(pubkey)) = (ip, pubkey) { if let (Some(ip), Some(pubkey)) = (ip, pubkey) {
@ -138,29 +181,63 @@ impl ConsulDiscovery {
rpc_public_addr: SocketAddr, rpc_public_addr: SocketAddr,
) -> Result<(), ConsulError> { ) -> Result<(), ConsulError> {
let node = format!("garage:{}", hex::encode(&node_id[..8])); let node = format!("garage:{}", hex::encode(&node_id[..8]));
let tags = [
vec!["advertised-by-garage".into(), hostname.into()],
unrob marked this conversation as resolved Outdated
Outdated
Review

this can also be simplified by using META_PREFIX in all cases

this can also be simplified by using META_PREFIX in all cases
self.config.tags.clone(),
]
.concat();
let advertisement = ConsulPublishEntry { let meta_prefix: String = match &self.config.mode {
ConsulDiscoveryMode::Node => "".to_string(),
ConsulDiscoveryMode::Service => format!("{}-", META_PREFIX),
};
let mut meta = HashMap::from([
(format!("{}pubkey", meta_prefix), hex::encode(node_id)),
unrob marked this conversation as resolved Outdated
Outdated
Review

I'd replace 187-195 by the following:

let mut meta = self.config.meta.clone().unwrap_or_default();
meta.insert(format!("{}-pubkey", META_PREFIX), hex::encode(node_id));
meta.insert(format!("{}-hostname", META_PREFIX), hostname.to_string());
I'd replace 187-195 by the following: ```rust let mut meta = self.config.meta.clone().unwrap_or_default(); meta.insert(format!("{}-pubkey", META_PREFIX), hex::encode(node_id)); meta.insert(format!("{}-hostname", META_PREFIX), hostname.to_string()); ```
(format!("{}hostname", meta_prefix), hostname.to_string()),
]);
if let Some(global_meta) = &self.config.meta {
for (key, value) in global_meta.into_iter() {
meta.insert(key.clone(), value.clone());
}
}
let url = format!(
"{}/v1/{}",
self.config.consul_http_addr,
(match &self.config.mode {
ConsulDiscoveryMode::Node => "catalog/register",
ConsulDiscoveryMode::Service => "agent/service/register?replace-existing-checks",
})
);
let req = self.client.put(&url);
let http = (match &self.config.mode {
ConsulDiscoveryMode::Node => req.json(&ConsulPublishEntry {
node: node.clone(), node: node.clone(),
address: rpc_public_addr.ip(), address: rpc_public_addr.ip(),
node_meta: [ node_meta: meta.clone(),
("pubkey".to_string(), hex::encode(node_id)), service: ConsulPublishCatalogService {
("hostname".to_string(), hostname.to_string()),
]
.iter()
.cloned()
.collect(),
service: ConsulPublishService {
service_id: node.clone(), service_id: node.clone(),
service_name: self.config.service_name.clone(), service_name: self.config.service_name.clone(),
tags: vec!["advertised-by-garage".into(), hostname.into()], tags,
service_meta: meta.clone(),
address: rpc_public_addr.ip(), address: rpc_public_addr.ip(),
port: rpc_public_addr.port(), port: rpc_public_addr.port(),
}, },
}; }),
ConsulDiscoveryMode::Service => req.json(&ConsulPublishService {
unrob marked this conversation as resolved Outdated
Outdated
Review

I don't like the big (...) around the match. Can you let the result of the match to a separate variable req_json, and then do let http = req_json.send().await?;

I don't like the big `(`...`)` around the match. Can you `let` the result of the match to a separate variable `req_json`, and then do `let http = req_json.send().await?;`
Outdated
Review

Went with an untagged enum here, since I couldn't think of a type for the advertisement structs; thought I could use serde::Serialize but that didn't work out and I'm still reading up to understand why.

Went with an untagged enum here, since I couldn't think of a type for the advertisement structs; thought I could use `serde::Serialize` but that didn't work out and I'm still reading up to understand why.
let url = format!("{}/v1/catalog/register", self.config.consul_http_addr); service_id: node.clone(),
service_name: self.config.service_name.clone(),
let http = self.client.put(&url).json(&advertisement).send().await?; tags,
meta,
address: rpc_public_addr.ip(),
port: rpc_public_addr.port(),
}),
})
.send()
.await?;
http.error_for_status()?; http.error_for_status()?;
Ok(()) Ok(())
@ -176,4 +253,6 @@ pub enum ConsulError {
Reqwest(#[error(source)] reqwest::Error), Reqwest(#[error(source)] reqwest::Error),
#[error(display = "Invalid Consul TLS configuration")] #[error(display = "Invalid Consul TLS configuration")]
InvalidTLSConfig, InvalidTLSConfig,
#[error(display = "Token error: {}", _0)]
Token(#[error(source)] reqwest::header::InvalidHeaderValue),
} }

View file

@ -1,174 +0,0 @@
use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::net::{IpAddr, SocketAddr};
use err_derive::Error;
use serde::{Deserialize, Serialize};
use netapp::NodeID;
use garage_util::config::ConsulServiceConfig;
const META_PREFIX: &str = "fr-deuxfleurs-garagehq";
#[derive(Deserialize, Clone, Debug)]
struct ConsulQueryEntry {
#[serde(rename = "Address")]
address: String,
#[serde(rename = "ServicePort")]
service_port: u16,
#[serde(rename = "ServiceMeta")]
service_meta: HashMap<String, String>,
}
#[derive(Serialize, Clone, Debug)]
struct ConsulPublishService {
#[serde(rename = "ID")]
service_id: String,
#[serde(rename = "Name")]
service_name: String,
#[serde(rename = "Tags")]
tags: Vec<String>,
#[serde(rename = "Address")]
address: IpAddr,
#[serde(rename = "Port")]
port: u16,
#[serde(rename = "Meta")]
meta: HashMap<String, String>,
}
// ----
pub struct ConsulServiceDiscovery {
config: ConsulServiceConfig,
client: reqwest::Client,
}
impl ConsulServiceDiscovery {
pub fn new(config: ConsulServiceConfig) -> Result<Self, ConsulError> {
let mut builder: reqwest::ClientBuilder = match &config.ca_cert {
Some(client_ca) => {
let mut ca_cert_buf = vec![];
File::open(client_ca)?.read_to_end(&mut ca_cert_buf)?;
let req: reqwest::ClientBuilder = reqwest::Client::builder()
.add_root_certificate(reqwest::Certificate::from_pem(&ca_cert_buf[..])?)
.use_rustls_tls();
if config.tls_skip_verify {
req.danger_accept_invalid_certs(true)
} else {
req
}
}
None => reqwest::Client::builder(),
};
if let Some(token) = &config.consul_http_token {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
"x-consul-token",
reqwest::header::HeaderValue::from_str(&token)?,
);
builder = builder.default_headers(headers);
}
let client = builder.build()?;
Ok(Self { client, config })
}
// ---- READING FROM CONSUL CATALOG ----
pub async fn get_consul_services(&self) -> Result<Vec<(NodeID, SocketAddr)>, ConsulError> {
let url = format!(
"{}/v1/catalog/service/{}",
self.config.consul_http_addr, self.config.service_name
);
let req = self.client.get(&url);
let http = req.send().await?;
let entries: Vec<ConsulQueryEntry> = http.json().await?;
let mut ret = vec![];
for ent in entries {
let ip = ent.address.parse::<IpAddr>().ok();
let pubkey = ent
.service_meta
.get(&format!("{}-pubkey", META_PREFIX))
.and_then(|k| hex::decode(k).ok())
.and_then(|k| NodeID::from_slice(&k[..]));
if let (Some(ip), Some(pubkey)) = (ip, pubkey) {
ret.push((pubkey, SocketAddr::new(ip, ent.service_port)));
} else {
warn!(
"Could not process node spec from Consul: {:?} (invalid IP or public key)",
ent
);
}
}
debug!("Got nodes from Consul: {:?}", ret);
Ok(ret)
}
// ---- PUBLISHING TO CONSUL CATALOG ----
pub async fn publish_consul_service(
&self,
node_id: NodeID,
hostname: &str,
rpc_public_addr: SocketAddr,
) -> Result<(), ConsulError> {
let node = format!("garage:{}", hex::encode(&node_id[..8]));
let tags = [
vec!["advertised-by-garage".into(), hostname.into()],
self.config.tags.clone(),
]
.concat();
let mut meta = HashMap::from([
(format!("{}-pubkey", META_PREFIX), hex::encode(node_id)),
(format!("{}-hostname", META_PREFIX), hostname.to_string()),
]);
if let Some(global_meta) = &self.config.meta {
for (key, value) in global_meta.into_iter() {
meta.insert(key.clone(), value.clone());
}
}
let advertisement: ConsulPublishService = ConsulPublishService {
service_id: node.clone(),
service_name: self.config.service_name.clone(),
tags,
meta,
address: rpc_public_addr.ip(),
port: rpc_public_addr.port(),
};
let url = format!(
"{}/v1/agent/service/register?replace-existing-checks",
self.config.consul_http_addr
);
let req = self.client.put(&url);
let http = req.json(&advertisement).send().await?;
http.error_for_status()?;
Ok(())
}
}
/// Regroup all Consul discovery errors
#[derive(Debug, Error)]
pub enum ConsulError {
#[error(display = "IO error: {}", _0)]
Io(#[error(source)] std::io::Error),
#[error(display = "HTTP error: {}", _0)]
Reqwest(#[error(source)] reqwest::Error),
#[error(display = "Invalid HTTP header error: {}", _0)]
HeaderValue(#[error(source)] reqwest::header::InvalidHeaderValue),
}

View file

@ -8,8 +8,6 @@ mod system_metrics;
#[cfg(feature = "consul-discovery")] #[cfg(feature = "consul-discovery")]
mod consul; mod consul;
#[cfg(feature = "consul-service-discovery")]
mod consul_services;
#[cfg(feature = "kubernetes-discovery")] #[cfg(feature = "kubernetes-discovery")]
mod kubernetes; mod kubernetes;

View file

@ -32,8 +32,6 @@ use garage_util::time::*;
#[cfg(feature = "consul-discovery")] #[cfg(feature = "consul-discovery")]
use crate::consul::ConsulDiscovery; use crate::consul::ConsulDiscovery;
#[cfg(feature = "consul-service-discovery")]
use crate::consul_services::ConsulServiceDiscovery;
#[cfg(feature = "kubernetes-discovery")] #[cfg(feature = "kubernetes-discovery")]
use crate::kubernetes::*; use crate::kubernetes::*;
use crate::layout::*; use crate::layout::*;
@ -100,18 +98,12 @@ pub struct System {
system_endpoint: Arc<Endpoint<SystemRpc, System>>, system_endpoint: Arc<Endpoint<SystemRpc, System>>,
rpc_listen_addr: SocketAddr, rpc_listen_addr: SocketAddr,
#[cfg(any( #[cfg(any(feature = "consul-discovery", feature = "kubernetes-discovery"))]
feature = "consul-discovery",
feature = "consul-service-discovery",
feature = "kubernetes-discovery"
))]
rpc_public_addr: Option<SocketAddr>, rpc_public_addr: Option<SocketAddr>,
bootstrap_peers: Vec<String>, bootstrap_peers: Vec<String>,
#[cfg(feature = "consul-discovery")] #[cfg(feature = "consul-discovery")]
consul_discovery: Option<ConsulDiscovery>, consul_discovery: Option<ConsulDiscovery>,
#[cfg(feature = "consul-service-discovery")]
consul_service_discovery: Option<ConsulServiceDiscovery>,
#[cfg(feature = "kubernetes-discovery")] #[cfg(feature = "kubernetes-discovery")]
kubernetes_discovery: Option<KubernetesDiscoveryConfig>, kubernetes_discovery: Option<KubernetesDiscoveryConfig>,
@ -354,19 +346,6 @@ impl System {
warn!("Consul discovery is not enabled in this build."); warn!("Consul discovery is not enabled in this build.");
} }
#[cfg(feature = "consul-service-discovery")]
let consul_service_discovery = match &config.consul_service_discovery {
Some(cfg) => Some(
ConsulServiceDiscovery::new(cfg.clone())
.ok_or_message("Invalid Consul service discovery configuration")?,
),
None => None,
};
#[cfg(not(feature = "consul-service-discovery"))]
if config.consul_service_discovery.is_some() {
warn!("Consul service discovery is not enabled in this build.");
}
#[cfg(not(feature = "kubernetes-discovery"))] #[cfg(not(feature = "kubernetes-discovery"))]
if config.kubernetes_discovery.is_some() { if config.kubernetes_discovery.is_some() {
warn!("Kubernetes discovery is not enabled in this build."); warn!("Kubernetes discovery is not enabled in this build.");
@ -390,17 +369,11 @@ impl System {
replication_mode, replication_mode,
replication_factor, replication_factor,
rpc_listen_addr: config.rpc_bind_addr, rpc_listen_addr: config.rpc_bind_addr,
#[cfg(any( #[cfg(any(feature = "consul-discovery", feature = "kubernetes-discovery"))]
feature = "consul-discovery",
feature = "consul-service-discovery",
feature = "kubernetes-discovery"
))]
rpc_public_addr, rpc_public_addr,
bootstrap_peers: config.bootstrap_peers.clone(), bootstrap_peers: config.bootstrap_peers.clone(),
#[cfg(feature = "consul-discovery")] #[cfg(feature = "consul-discovery")]
consul_discovery, consul_discovery,
#[cfg(feature = "consul-service-discovery")]
consul_service_discovery,
#[cfg(feature = "kubernetes-discovery")] #[cfg(feature = "kubernetes-discovery")]
kubernetes_discovery: config.kubernetes_discovery.clone(), kubernetes_discovery: config.kubernetes_discovery.clone(),
metrics, metrics,
@ -582,33 +555,6 @@ impl System {
} }
} }
#[cfg(feature = "consul-service-discovery")]
async fn advertise_to_consul(self: Arc<Self>) {
let c = match &self.consul_service_discovery {
Some(c) => c,
_ => return,
};
let rpc_public_addr = match self.rpc_public_addr {
Some(addr) => addr,
None => {
warn!("Not advertising to Consul because rpc_public_addr is not defined in config file and could not be autodetected.");
return;
}
};
if let Err(e) = c
.publish_consul_service(
self.netapp.id,
&self.local_status.load_full().hostname,
rpc_public_addr,
)
.await
{
error!("Error while publishing Consul service: {}", e);
}
}
#[cfg(feature = "kubernetes-discovery")] #[cfg(feature = "kubernetes-discovery")]
async fn advertise_to_kubernetes(self: Arc<Self>) { async fn advertise_to_kubernetes(self: Arc<Self>) {
let k = match &self.kubernetes_discovery { let k = match &self.kubernetes_discovery {
@ -798,7 +744,7 @@ impl System {
ping_list.extend(peers.0.iter().map(|(id, addr)| ((*id).into(), *addr))) ping_list.extend(peers.0.iter().map(|(id, addr)| ((*id).into(), *addr)))
} }
// Fetch peer list from Consul Nodes // Fetch peer list from Consul
#[cfg(feature = "consul-discovery")] #[cfg(feature = "consul-discovery")]
if let Some(c) = &self.consul_discovery { if let Some(c) = &self.consul_discovery {
match c.get_consul_nodes().await { match c.get_consul_nodes().await {
@ -811,19 +757,6 @@ impl System {
} }
} }
// Fetch peer list from Consul Services
#[cfg(feature = "consul-service-discovery")]
if let Some(c) = &self.consul_service_discovery {
match c.get_consul_services().await {
Ok(node_list) => {
ping_list.extend(node_list);
}
Err(e) => {
warn!("Could not retrieve service list from Consul: {}", e);
}
}
}
// Fetch peer list from Kubernetes // Fetch peer list from Kubernetes
#[cfg(feature = "kubernetes-discovery")] #[cfg(feature = "kubernetes-discovery")]
if let Some(k) = &self.kubernetes_discovery { if let Some(k) = &self.kubernetes_discovery {
@ -863,9 +796,6 @@ impl System {
#[cfg(feature = "consul-discovery")] #[cfg(feature = "consul-discovery")]
tokio::spawn(self.clone().advertise_to_consul()); tokio::spawn(self.clone().advertise_to_consul());
#[cfg(feature = "consul-service-discovery")]
tokio::spawn(self.clone().advertise_to_consul());
#[cfg(feature = "kubernetes-discovery")] #[cfg(feature = "kubernetes-discovery")]
tokio::spawn(self.clone().advertise_to_kubernetes()); tokio::spawn(self.clone().advertise_to_kubernetes());

View file

@ -56,9 +56,6 @@ pub struct Config {
/// Configuration for automatic node discovery through Consul /// Configuration for automatic node discovery through Consul
#[serde(default)] #[serde(default)]
pub consul_discovery: Option<ConsulDiscoveryConfig>, pub consul_discovery: Option<ConsulDiscoveryConfig>,
/// Configuration for automatic node discovery through Consul
#[serde(default)]
pub consul_service_discovery: Option<ConsulServiceConfig>,
/// Configuration for automatic node discovery through Kubernetes /// Configuration for automatic node discovery through Kubernetes
#[serde(default)] #[serde(default)]
pub kubernetes_discovery: Option<KubernetesDiscoveryConfig>, pub kubernetes_discovery: Option<KubernetesDiscoveryConfig>,
@ -138,8 +135,23 @@ pub struct AdminConfig {
pub trace_sink: Option<String>, pub trace_sink: Option<String>,
} }
#[derive(Deserialize, Debug, Clone)]
pub enum ConsulDiscoveryMode {
#[serde(rename_all = "lowercase")]
unrob marked this conversation as resolved Outdated
Outdated
Review

I think the rename_all line has to go above the pub enum ConsulDiscoveryAPI line?

I think the `rename_all` line has to go above the `pub enum ConsulDiscoveryAPI` line?
Node,
Service,
}
impl ConsulDiscoveryMode {
unrob marked this conversation as resolved Outdated
Outdated
Review

This whole impl can be avoided by deriving Default on the enum. The default variant has to be marked with a #[default] tag.

This whole impl can be avoided by deriving `Default` on the enum. The default variant has to be marked with a `#[default]` tag.
fn default() -> Self {
ConsulDiscoveryMode::Node
}
}
#[derive(Deserialize, Debug, Clone)] #[derive(Deserialize, Debug, Clone)]
pub struct ConsulDiscoveryConfig { pub struct ConsulDiscoveryConfig {
/// Mode of consul operation: either `node` (the default) or `service`
#[serde(default = "ConsulDiscoveryMode::default")]
pub mode: ConsulDiscoveryMode,
/// Consul http or https address to connect to to discover more peers /// Consul http or https address to connect to to discover more peers
pub consul_http_addr: String, pub consul_http_addr: String,
/// Consul service name to use /// Consul service name to use
@ -150,30 +162,17 @@ pub struct ConsulDiscoveryConfig {
pub client_cert: Option<String>, pub client_cert: Option<String>,
/// Client TLS key to use when connecting to Consul /// Client TLS key to use when connecting to Consul
pub client_key: Option<String>, pub client_key: Option<String>,
/// /// Token to use for connecting to consul
pub consul_http_token: Option<String>,
/// Skip TLS hostname verification /// Skip TLS hostname verification
#[serde(default)] #[serde(default)]
pub tls_skip_verify: bool, pub tls_skip_verify: bool,
} /// Additional tags to add to the service
#[derive(Deserialize, Debug, Clone)]
pub struct ConsulServiceConfig {
/// Consul http or https address to connect to to discover more peers
pub consul_http_addr: String,
/// Token to use for connecting to consul
pub consul_http_token: Option<String>,
/// Consul service name to use
pub service_name: String,
/// CA TLS certificate to use when connecting to Consul
pub ca_cert: Option<String>,
// Additional tags to add to the service
#[serde(default)] #[serde(default)]
pub tags: Vec<String>, pub tags: Vec<String>,
// Additional service metadata to add /// Additional service metadata to add
#[serde(default)] #[serde(default)]
pub meta: Option<std::collections::HashMap<String, String>>, pub meta: Option<std::collections::HashMap<String, String>>,
/// Skip TLS hostname verification
#[serde(default)]
pub tls_skip_verify: bool,
} }
#[derive(Deserialize, Debug, Clone)] #[derive(Deserialize, Debug, Clone)]