feature: Register consul services with agent API #567
|
@ -318,16 +318,16 @@ Garage supports discovering other nodes of the cluster using Consul. For this
|
||||||
to work correctly, nodes need to know their IP address by which they can be
|
to work correctly, nodes need to know their IP address by which they can be
|
||||||
reached by other nodes of the cluster, which should be set in `rpc_public_addr`.
|
reached by other nodes of the cluster, which should be set in `rpc_public_addr`.
|
||||||
|
|
||||||
### `mode`
|
|
||||||
|
|
||||||
Two modes of service discovery are supported: `node` and `service`. `node`, the default will register a service using
|
|
||||||
the `/v1/catalog` endpoints and mTLS (if `client_cert` and `client_key` are provided). `service` mode uses the
|
|
||||||
`v1/agent` endpoints instead, where an optional `consul_http_token` may be provided.
|
|
||||||
|
|
||||||
### `consul_http_addr` and `service_name`
|
### `consul_http_addr` and `service_name`
|
||||||
|
|
||||||
The `consul_http_addr` parameter should be set to the full HTTP(S) address of the Consul server.
|
The `consul_http_addr` parameter should be set to the full HTTP(S) address of the Consul server.
|
||||||
|
|
||||||
|
### `consul_http_api`
|
||||||
|
|
||||||
|
Two APIs for service registration are supported: `catalog` and `agent`. `catalog`, the default, will register a service using
|
||||||
|
the `/v1/catalog` endpoints and mTLS (if `client_cert` and `client_key` are provided). The `agent` API uses the
|
||||||
|
`v1/agent` endpoints instead, where an optional `consul_http_token` may be provided.
|
||||||
unrob marked this conversation as resolved
Outdated
|
|||||||
|
|
||||||
### `service_name`
|
### `service_name`
|
||||||
|
|
||||||
`service_name` should be set to the service name under which Garage's
|
`service_name` should be set to the service name under which Garage's
|
||||||
|
@ -335,8 +335,8 @@ RPC ports are announced.
|
||||||
|
|
||||||
### `client_cert`, `client_key`
|
### `client_cert`, `client_key`
|
||||||
|
|
||||||
`node` mode only. TLS client certificate and client key to use when communicating with Consul over TLS.
|
TLS client certificate and client key to use when communicating with Consul over TLS. Both are mandatory when doing so.
|
||||||
Both are mandatory when doing so.
|
Only available when `consul_http_api = "catalog"`.
|
||||||
|
|
||||||
### `ca_cert`
|
### `ca_cert`
|
||||||
|
|
||||||
|
@ -349,8 +349,8 @@ Skip server hostname verification in TLS handshake.
|
||||||
|
|
||||||
### `consul_http_token`
|
### `consul_http_token`
|
||||||
|
|
||||||
`service` mode only. Uses the provided token for communication with Consul. The policy assigned to this token
|
Uses the provided token for communication with Consul. Only available when `consul_http_api = "agent"`.
|
||||||
should at least have these rules:
|
The policy assigned to this token should at least have these rules:
|
||||||
|
|
||||||
```hcl
|
```hcl
|
||||||
// the `service_name` specified above
|
// the `service_name` specified above
|
||||||
|
|
|
@ -8,8 +8,8 @@ use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use netapp::NodeID;
|
use netapp::NodeID;
|
||||||
|
|
||||||
|
use garage_util::config::ConsulDiscoveryAPI;
|
||||||
use garage_util::config::ConsulDiscoveryConfig;
|
use garage_util::config::ConsulDiscoveryConfig;
|
||||||
use garage_util::config::ConsulDiscoveryMode;
|
|
||||||
|
|
||||||
const META_PREFIX: &str = "fr-deuxfleurs-garagehq";
|
const META_PREFIX: &str = "fr-deuxfleurs-garagehq";
|
||||||
|
|
||||||
|
@ -78,18 +78,18 @@ pub struct ConsulDiscovery {
|
||||||
impl ConsulDiscovery {
|
impl ConsulDiscovery {
|
||||||
pub fn new(config: ConsulDiscoveryConfig) -> Result<Self, ConsulError> {
|
pub fn new(config: ConsulDiscoveryConfig) -> Result<Self, ConsulError> {
|
||||||
let mut builder: reqwest::ClientBuilder = reqwest::Client::builder();
|
let mut builder: reqwest::ClientBuilder = reqwest::Client::builder();
|
||||||
unrob marked this conversation as resolved
Outdated
lx
commented
add add `.use_rustls_tls()` here and remove all other instances below (we are using rustls in all cases)
|
|||||||
builder = builder.danger_accept_invalid_certs(config.tls_skip_verify);
|
if config.tls_skip_verify {
|
||||||
|
builder = builder.danger_accept_invalid_certs(true);
|
||||||
let client: reqwest::Client = match &config.mode {
|
} else if let Some(ca_cert) = &config.ca_cert {
|
||||||
ConsulDiscoveryMode::Node => {
|
|
||||||
if let Some(ca_cert) = &config.ca_cert {
|
|
||||||
let mut ca_cert_buf = vec![];
|
let mut ca_cert_buf = vec![];
|
||||||
File::open(ca_cert)?.read_to_end(&mut ca_cert_buf)?;
|
File::open(ca_cert)?.read_to_end(&mut ca_cert_buf)?;
|
||||||
builder = builder.use_rustls_tls();
|
builder = builder.use_rustls_tls();
|
||||||
builder = builder
|
builder =
|
||||||
.add_root_certificate(reqwest::Certificate::from_pem(&ca_cert_buf[..])?);
|
builder.add_root_certificate(reqwest::Certificate::from_pem(&ca_cert_buf[..])?);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let client: reqwest::Client = match &config.consul_http_api {
|
||||||
|
ConsulDiscoveryAPI::Catalog => {
|
||||||
match (&config.client_cert, &config.client_key) {
|
match (&config.client_cert, &config.client_key) {
|
||||||
(Some(client_cert), Some(client_key)) => {
|
(Some(client_cert), Some(client_key)) => {
|
||||||
let mut client_cert_buf = vec![];
|
let mut client_cert_buf = vec![];
|
||||||
|
@ -111,15 +111,7 @@ impl ConsulDiscovery {
|
||||||
|
|
||||||
builder.build()?
|
builder.build()?
|
||||||
unrob marked this conversation as resolved
Outdated
lx
commented
refactoring: move This means that there is no and after the match block there is refactoring: move `builder.build()?` after the match
This means that there is no `let client = match`, just a match block that mutates `builder`
and after the match block there is `let client = builder.build()?;`
|
|||||||
}
|
}
|
||||||
ConsulDiscoveryMode::Service => {
|
ConsulDiscoveryAPI::Agent => {
|
||||||
if let Some(ca_cert) = &config.ca_cert {
|
|
||||||
let mut ca_cert_buf = vec![];
|
|
||||||
File::open(ca_cert)?.read_to_end(&mut ca_cert_buf)?;
|
|
||||||
builder = builder
|
|
||||||
.add_root_certificate(reqwest::Certificate::from_pem(&ca_cert_buf[..])?);
|
|
||||||
builder = builder.use_rustls_tls();
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(token) = &config.consul_http_token {
|
if let Some(token) = &config.consul_http_token {
|
||||||
let mut headers = reqwest::header::HeaderMap::new();
|
let mut headers = reqwest::header::HeaderMap::new();
|
||||||
headers.insert(
|
headers.insert(
|
||||||
|
@ -150,9 +142,9 @@ impl ConsulDiscovery {
|
||||||
let mut ret = vec![];
|
let mut ret = vec![];
|
||||||
for ent in entries {
|
for ent in entries {
|
||||||
let ip = ent.address.parse::<IpAddr>().ok();
|
let ip = ent.address.parse::<IpAddr>().ok();
|
||||||
let pubkey = match &self.config.mode {
|
let pubkey = match &self.config.consul_http_api {
|
||||||
ConsulDiscoveryMode::Node => ent.node_meta.get("pubkey"),
|
ConsulDiscoveryAPI::Catalog => ent.node_meta.get("pubkey"),
|
||||||
ConsulDiscoveryMode::Service => {
|
ConsulDiscoveryAPI::Agent => {
|
||||||
ent.service_meta.get(&format!("{}-pubkey", META_PREFIX))
|
ent.service_meta.get(&format!("{}-pubkey", META_PREFIX))
|
||||||
unrob marked this conversation as resolved
Outdated
lx
commented
I think we should also add the I think we should also add the `META_PREFIX` to the old code that uses the Catalog API, it's a good practice to have and it will simplify the code. It will make Consul discovery incompatible between versions but that's fine, we are still beta.
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -187,9 +179,9 @@ impl ConsulDiscovery {
|
||||||
]
|
]
|
||||||
.concat();
|
.concat();
|
||||||
|
|
||||||
let meta_prefix: String = match &self.config.mode {
|
let meta_prefix: String = match &self.config.consul_http_api {
|
||||||
ConsulDiscoveryMode::Node => "".to_string(),
|
ConsulDiscoveryAPI::Catalog => "".to_string(),
|
||||||
ConsulDiscoveryMode::Service => format!("{}-", META_PREFIX),
|
ConsulDiscoveryAPI::Agent => format!("{}-", META_PREFIX),
|
||||||
};
|
};
|
||||||
unrob marked this conversation as resolved
Outdated
lx
commented
this can also be simplified by using META_PREFIX in all cases this can also be simplified by using META_PREFIX in all cases
|
|||||||
|
|
||||||
let mut meta = HashMap::from([
|
let mut meta = HashMap::from([
|
||||||
|
@ -206,15 +198,15 @@ impl ConsulDiscovery {
|
||||||
let url = format!(
|
let url = format!(
|
||||||
"{}/v1/{}",
|
"{}/v1/{}",
|
||||||
self.config.consul_http_addr,
|
self.config.consul_http_addr,
|
||||||
(match &self.config.mode {
|
(match &self.config.consul_http_api {
|
||||||
ConsulDiscoveryMode::Node => "catalog/register",
|
ConsulDiscoveryAPI::Catalog => "catalog/register",
|
||||||
ConsulDiscoveryMode::Service => "agent/service/register?replace-existing-checks",
|
ConsulDiscoveryAPI::Agent => "agent/service/register?replace-existing-checks",
|
||||||
})
|
})
|
||||||
);
|
);
|
||||||
|
|
||||||
let req = self.client.put(&url);
|
let req = self.client.put(&url);
|
||||||
let http = (match &self.config.mode {
|
let http = (match &self.config.consul_http_api {
|
||||||
ConsulDiscoveryMode::Node => req.json(&ConsulPublishEntry {
|
ConsulDiscoveryAPI::Catalog => req.json(&ConsulPublishEntry {
|
||||||
node: node.clone(),
|
node: node.clone(),
|
||||||
address: rpc_public_addr.ip(),
|
address: rpc_public_addr.ip(),
|
||||||
node_meta: meta.clone(),
|
node_meta: meta.clone(),
|
||||||
|
@ -227,7 +219,7 @@ impl ConsulDiscovery {
|
||||||
port: rpc_public_addr.port(),
|
port: rpc_public_addr.port(),
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
ConsulDiscoveryMode::Service => req.json(&ConsulPublishService {
|
ConsulDiscoveryAPI::Agent => req.json(&ConsulPublishService {
|
||||||
service_id: node.clone(),
|
service_id: node.clone(),
|
||||||
service_name: self.config.service_name.clone(),
|
service_name: self.config.service_name.clone(),
|
||||||
tags,
|
tags,
|
||||||
|
|
|
@ -136,22 +136,22 @@ pub struct AdminConfig {
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
pub enum ConsulDiscoveryMode {
|
pub enum ConsulDiscoveryAPI {
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
unrob marked this conversation as resolved
Outdated
lx
commented
I think the I think the `rename_all` line has to go above the `pub enum ConsulDiscoveryAPI` line?
|
|||||||
Node,
|
Catalog,
|
||||||
Service,
|
Agent,
|
||||||
}
|
}
|
||||||
impl ConsulDiscoveryMode {
|
impl ConsulDiscoveryAPI {
|
||||||
unrob marked this conversation as resolved
Outdated
lx
commented
This whole impl can be avoided by deriving This whole impl can be avoided by deriving `Default` on the enum. The default variant has to be marked with a `#[default]` tag.
|
|||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
ConsulDiscoveryMode::Node
|
ConsulDiscoveryAPI::Catalog
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize, Debug, Clone)]
|
#[derive(Deserialize, Debug, Clone)]
|
||||||
pub struct ConsulDiscoveryConfig {
|
pub struct ConsulDiscoveryConfig {
|
||||||
/// Mode of consul operation: either `node` (the default) or `service`
|
/// The consul api to use when registering: either `catalog` (the default) or `agent`
|
||||||
#[serde(default = "ConsulDiscoveryMode::default")]
|
#[serde(default = "ConsulDiscoveryAPI::default")]
|
||||||
pub mode: ConsulDiscoveryMode,
|
pub consul_http_api: ConsulDiscoveryAPI,
|
||||||
/// Consul http or https address to connect to to discover more peers
|
/// Consul http or https address to connect to to discover more peers
|
||||||
pub consul_http_addr: String,
|
pub consul_http_addr: String,
|
||||||
/// Consul service name to use
|
/// Consul service name to use
|
||||||
|
|
token
instead ofconsul_http_token