From ca449ebff452345229953b6ba406acefb157385b Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 30 Nov 2023 15:35:32 +0100 Subject: [PATCH 1/3] some comments --- src/cert_store.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/cert_store.rs b/src/cert_store.rs index 0ced178..e25395d 100644 --- a/src/cert_store.rs +++ b/src/cert_store.rs @@ -70,6 +70,9 @@ impl CertStore { loop { let mut domains: HashSet = HashSet::new(); + // Collect domains that need a TLS certificate + // either from the proxy configuration (eagerly) + // or on reaction to a user request (lazily) select! { res = rx_proxy_config.changed() => { if res.is_err() { @@ -78,6 +81,8 @@ impl CertStore { let proxy_config: Arc = rx_proxy_config.borrow().clone(); for ent in proxy_config.entries.iter() { + // Eagerly generate certificates for domains that + // are not patterns if let HostDescription::Hostname(domain) = &ent.url_prefix.host { if let Some((host, _port)) = domain.split_once(':') { domains.insert(host.to_string()); @@ -85,6 +90,9 @@ impl CertStore { domains.insert(domain.clone()); } } + + // @TODO Register a map of + // UrlPrefix -> OnDemandTlsAskCheckUrl } } need_cert = rx_need_cert.recv() => { @@ -100,12 +108,17 @@ impl CertStore { } } + // Now that we have our list of domains to check, + // actually do something for dom in domains.iter() { + // Exclude from the list domains that were checked less than 60 + // seconds ago match t_last_check.get(dom) { Some(t) if Instant::now() - *t < Duration::from_secs(60) => continue, _ => t_last_check.insert(dom.to_string(), Instant::now()), }; + // Actual Let's Encrypt calls are done here (in sister function) debug!("Checking cert for domain: {}", dom); if let Err(e) = self.check_cert(dom).await { warn!("({}) Could not get certificate: {}", dom, e); @@ -186,6 +199,15 @@ impl CertStore { Ok(()) } + /// Check certificate ensure that the certificate is in the memory store + /// and that it does not need to be renewed. + /// + /// If it's not in the memory store, it tries to load it from Consul, + /// if it's not in Consul, it calls Let's Encrypt. + /// + /// If the certificate is outdated in the memory store, it tries to load + /// a more recent version in Consul, if the Consul version is also outdated, + /// it tries to renew it pub async fn check_cert(self: &Arc, domain: &str) -> Result<()> { // First, try locally. { @@ -226,6 +248,7 @@ impl CertStore { self.renew_cert(domain).await } + /// This is the place where certificates are generated or renewed pub async fn renew_cert(self: &Arc, domain: &str) -> Result<()> { info!("({}) Renewing certificate", domain); From 753903ee021e7aac7c0914883ae8d547148b4d95 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 30 Nov 2023 16:53:04 +0100 Subject: [PATCH 2/3] implement feature --- src/cert_store.rs | 169 +++++++++++++++++++++++++++++++++----------- src/proxy_config.rs | 23 +++--- 2 files changed, 143 insertions(+), 49 deletions(-) diff --git a/src/cert_store.rs b/src/cert_store.rs index e25395d..7c6b4b3 100644 --- a/src/cert_store.rs +++ b/src/cert_store.rs @@ -67,58 +67,73 @@ impl CertStore { let mut t_last_check: HashMap = HashMap::new(); - loop { - let mut domains: HashSet = HashSet::new(); + // Collect data from proxy config + let mut static_domains: HashSet = HashSet::new(); + let mut on_demand_checks: Vec<(glob::Pattern, Option)> = vec![]; - // Collect domains that need a TLS certificate - // either from the proxy configuration (eagerly) - // or on reaction to a user request (lazily) - select! { + loop { + // Collect domains that need a TLS certificate + // either from the proxy configuration (eagerly) + // or on reaction to a user request (lazily) + let domains = select! { res = rx_proxy_config.changed() => { if res.is_err() { bail!("rx_proxy_config closed"); } + on_demand_checks.clear(); let proxy_config: Arc = rx_proxy_config.borrow().clone(); for ent in proxy_config.entries.iter() { - // Eagerly generate certificates for domains that - // are not patterns - if let HostDescription::Hostname(domain) = &ent.url_prefix.host { + // Eagerly generate certificates for domains that + // are not patterns + match &ent.url_prefix.host { + HostDescription::Hostname(domain) => { if let Some((host, _port)) = domain.split_once(':') { - domains.insert(host.to_string()); + static_domains.insert(host.to_string()); + //domains.insert(host.to_string()); } else { - domains.insert(domain.clone()); + static_domains.insert(domain.clone()); + //domains.insert(domain.clone()); } - } - - // @TODO Register a map of - // UrlPrefix -> OnDemandTlsAskCheckUrl + }, + HostDescription::Pattern(pattern) => { + on_demand_checks.push((pattern.clone(), ent.on_demand_tls_ask.clone())); + } + } } + + // only static_domains are refreshed + static_domains.clone() } need_cert = rx_need_cert.recv() => { match need_cert { Some(dom) => { - domains.insert(dom); + let mut candidates: HashSet = HashSet::new(); + + // collect certificates as much as possible + candidates.insert(dom); while let Ok(dom2) = rx_need_cert.try_recv() { - domains.insert(dom2); + candidates.insert(dom2); } + + self.domain_validation(candidates, &static_domains, on_demand_checks.as_slice()).await } None => bail!("rx_need_cert closed"), - }; + } } - } + }; - // Now that we have our list of domains to check, - // actually do something + // Now that we have our list of domains to check, + // actually do something for dom in domains.iter() { - // Exclude from the list domains that were checked less than 60 - // seconds ago + // Exclude from the list domains that were checked less than 60 + // seconds ago match t_last_check.get(dom) { Some(t) if Instant::now() - *t < Duration::from_secs(60) => continue, _ => t_last_check.insert(dom.to_string(), Instant::now()), }; - // Actual Let's Encrypt calls are done here (in sister function) + // Actual Let's Encrypt calls are done here (in sister function) debug!("Checking cert for domain: {}", dom); if let Err(e) = self.check_cert(dom).await { warn!("({}) Could not get certificate: {}", dom, e); @@ -127,6 +142,73 @@ impl CertStore { } } + async fn domain_validation( + &self, + candidates: HashSet, + static_domains: &HashSet, + checks: &[(glob::Pattern, Option)], + ) -> HashSet { + let mut domains: HashSet = HashSet::new(); + + // Filter certificates... + for candidate in candidates.into_iter() { + // Disallow obvious wrong domains... + if !candidate.contains('.') || candidate.ends_with(".local") { + warn!("Probably not a publicly accessible domain, skipping (a self-signed certificate will be used)"); + continue; + } + + // Try to register domain as a static domain + if static_domains.contains(&candidate) { + trace!("domain {} validated as static domain", candidate); + domains.insert(candidate); + continue; + } + + // It's not a static domain, maybe an on-demand domain? + for (pattern, maybe_check_url) in checks.iter() { + // check glob pattern + if pattern.matches(&candidate) { + // if no check url is set, accept domain as long as it matches the pattern + let check_url = match maybe_check_url { + None => { + trace!( + "domain {} validated on glob pattern {} only", + candidate, + pattern + ); + domains.insert(candidate); + break; + } + Some(url) => url, + }; + + // if a check url is set, call it + match self.on_demand_tls_ask(check_url, &candidate).await { + Ok(()) => { + trace!( + "domain {} validated on glob pattern {} and on check url {}", + candidate, + pattern, + check_url + ); + domains.insert(candidate); + break; + } + Err(e) => { + warn!("domain {} validation refused on glob pattern {} and on check url {} with error: {}", candidate, pattern, check_url, e); + } + } + } + } + // Avoid DDoSing a backend + tokio::time::sleep(Duration::from_secs(2)).await; + } + + return domains; + } + + /// This function is also in charge of the refresh of the domain names fn get_cert_for_https(self: &Arc, domain: &str) -> Result> { // Check if domain is authorized if !self @@ -199,15 +281,15 @@ impl CertStore { Ok(()) } - /// Check certificate ensure that the certificate is in the memory store - /// and that it does not need to be renewed. - /// - /// If it's not in the memory store, it tries to load it from Consul, - /// if it's not in Consul, it calls Let's Encrypt. - /// - /// If the certificate is outdated in the memory store, it tries to load - /// a more recent version in Consul, if the Consul version is also outdated, - /// it tries to renew it + /// Check certificate ensure that the certificate is in the memory store + /// and that it does not need to be renewed. + /// + /// If it's not in the memory store, it tries to load it from Consul, + /// if it's not in Consul, it calls Let's Encrypt. + /// + /// If the certificate is outdated in the memory store, it tries to load + /// a more recent version in Consul, if the Consul version is also outdated, + /// it tries to renew it pub async fn check_cert(self: &Arc, domain: &str) -> Result<()> { // First, try locally. { @@ -248,16 +330,10 @@ impl CertStore { self.renew_cert(domain).await } - /// This is the place where certificates are generated or renewed + /// This is the place where certificates are generated or renewed pub async fn renew_cert(self: &Arc, domain: &str) -> Result<()> { info!("({}) Renewing certificate", domain); - // Basic sanity check (we could add more kinds of checks here) - // This is just to help avoid getting rate-limited against ACME server - if !domain.contains('.') || domain.ends_with(".local") { - bail!("Probably not a publicly accessible domain, skipping (a self-signed certificate will be used)"); - } - // ---- Acquire lock ---- // the lock is acquired for half an hour, // so that in case of an error we won't retry before @@ -373,6 +449,19 @@ impl CertStore { Ok(()) } + async fn on_demand_tls_ask(&self, check_url: &str, domain: &str) -> Result<()> { + let httpcli = reqwest::Client::new(); + let chall_url = format!("{}?domain={}", check_url, domain); + info!("({}) On-demand TLS check", domain); + + let httpresp = httpcli.get(&chall_url).send().await?; + if httpresp.status() != reqwest::StatusCode::OK { + bail!("{} is not authorized for on-demand TLS", domain); + } + + Ok(()) + } + async fn check_domain_accessibility(&self, domain: &str, session: &str) -> Result<()> { // Returns Ok(()) only if domain is a correct domain name that // redirects to this server diff --git a/src/proxy_config.rs b/src/proxy_config.rs index 8381de2..7690f8a 100644 --- a/src/proxy_config.rs +++ b/src/proxy_config.rs @@ -108,6 +108,10 @@ pub struct ProxyEntry { /// when matching this rule pub redirects: Vec<(UrlPrefix, UrlPrefix, u16)>, + /// Wether or not the domain must be validated before asking a certificate + /// to let's encrypt (only for Glob patterns) + pub on_demand_tls_ask: Option, + /// Number of calls in progress, used to deprioritize slow back-ends pub calls_in_progress: atomic::AtomicI64, /// Time of last call, used for round-robin selection @@ -142,14 +146,14 @@ impl ProxyEntry { let mut add_headers = vec![]; let mut redirects = vec![]; + let mut on_demand_tls_ask: Option = None; for mid in middleware.into_iter() { + // LocalLb and GlobalLb are handled in the parent function match mid { ConfigTag::AddHeader(k, v) => add_headers.push((k.to_string(), v.clone())), ConfigTag::AddRedirect(m, r, c) => redirects.push(((*m).clone(), (*r).clone(), *c)), - ConfigTag::LocalLb | ConfigTag::GlobalLb => { - /* handled in parent fx */ - () - } + ConfigTag::OnDemandTlsAsk(url) => on_demand_tls_ask = Some(url.to_string()), + ConfigTag::LocalLb | ConfigTag::GlobalLb => (), }; } @@ -166,6 +170,7 @@ impl ProxyEntry { flags, add_headers, redirects, + on_demand_tls_ask, // internal last_call: atomic::AtomicI64::from(0), calls_in_progress: atomic::AtomicI64::from(0), @@ -247,6 +252,7 @@ enum MatchTag { enum ConfigTag<'a> { AddHeader(&'a str, String), AddRedirect(UrlPrefix, UrlPrefix, u16), + OnDemandTlsAsk(&'a str), GlobalLb, LocalLb, } @@ -321,6 +327,9 @@ fn parse_tricot_tags(tag: &str) -> Option { p_match, p_replace, http_code, ))) } + ["tricot-on-demand-tls-ask", url, ..] => { + Some(ParsedTag::Middleware(ConfigTag::OnDemandTlsAsk(url))) + } ["tricot-global-lb", ..] => Some(ParsedTag::Middleware(ConfigTag::GlobalLb)), ["tricot-local-lb", ..] => Some(ParsedTag::Middleware(ConfigTag::LocalLb)), _ => None, @@ -369,13 +378,9 @@ fn parse_consul_service( // some legacy processing that would need a refactor later for mid in collected_middleware.iter() { match mid { - ConfigTag::AddHeader(_, _) | ConfigTag::AddRedirect(_, _, _) => - /* not handled here */ - { - () - } ConfigTag::GlobalLb => flags.global_lb = true, ConfigTag::LocalLb => flags.site_lb = true, + _ => (), }; } From b9b035034ff7dee1089d8c629296391fe0539515 Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 30 Nov 2023 17:34:07 +0100 Subject: [PATCH 3/3] centralize all the checks in the same place --- src/cert_store.rs | 102 +++++++++++++++++++++++----------------------- 1 file changed, 52 insertions(+), 50 deletions(-) diff --git a/src/cert_store.rs b/src/cert_store.rs index 7c6b4b3..edbd0a1 100644 --- a/src/cert_store.rs +++ b/src/cert_store.rs @@ -22,12 +22,19 @@ pub struct CertStore { consul: Consul, node_name: String, letsencrypt_email: String, + certs: RwLock>>, self_signed_certs: RwLock>>, + rx_proxy_config: watch::Receiver>, tx_need_cert: mpsc::UnboundedSender, } +struct ProcessedDomains { + static_domains: HashSet, + on_demand_domains: Vec<(glob::Pattern, Option)>, +} + impl CertStore { pub fn new( consul: Consul, @@ -41,10 +48,10 @@ impl CertStore { let cert_store = Arc::new(Self { consul, node_name, + letsencrypt_email, certs: RwLock::new(HashMap::new()), self_signed_certs: RwLock::new(HashMap::new()), rx_proxy_config, - letsencrypt_email, tx_need_cert: tx, }); @@ -66,57 +73,55 @@ impl CertStore { let mut rx_proxy_config = self.rx_proxy_config.clone(); let mut t_last_check: HashMap = HashMap::new(); - - // Collect data from proxy config - let mut static_domains: HashSet = HashSet::new(); - let mut on_demand_checks: Vec<(glob::Pattern, Option)> = vec![]; + let mut proc_domains: Option = None; loop { - // Collect domains that need a TLS certificate - // either from the proxy configuration (eagerly) - // or on reaction to a user request (lazily) let domains = select! { + // Refresh some internal states, schedule static_domains for renew res = rx_proxy_config.changed() => { if res.is_err() { bail!("rx_proxy_config closed"); } - on_demand_checks.clear(); + let mut static_domains: HashSet = HashSet::new(); + let mut on_demand_domains: Vec<(glob::Pattern, Option)> = vec![]; + let proxy_config: Arc = rx_proxy_config.borrow().clone(); + for ent in proxy_config.entries.iter() { - // Eagerly generate certificates for domains that - // are not patterns - match &ent.url_prefix.host { - HostDescription::Hostname(domain) => { - if let Some((host, _port)) = domain.split_once(':') { - static_domains.insert(host.to_string()); - //domains.insert(host.to_string()); - } else { - static_domains.insert(domain.clone()); - //domains.insert(domain.clone()); - } + // Eagerly generate certificates for domains that + // are not patterns + match &ent.url_prefix.host { + HostDescription::Hostname(domain) => { + if let Some((host, _port)) = domain.split_once(':') { + static_domains.insert(host.to_string()); + } else { + static_domains.insert(domain.clone()); + } }, - HostDescription::Pattern(pattern) => { - on_demand_checks.push((pattern.clone(), ent.on_demand_tls_ask.clone())); - } - } + HostDescription::Pattern(pattern) => { + on_demand_domains.push((pattern.clone(), ent.on_demand_tls_ask.clone())); + }, + } } - // only static_domains are refreshed - static_domains.clone() + // only static_domains are refreshed + proc_domains = Some(ProcessedDomains { static_domains: static_domains.clone(), on_demand_domains }); + self.domain_validation(static_domains, proc_domains.as_ref()).await } + // renew static and on-demand domains need_cert = rx_need_cert.recv() => { match need_cert { Some(dom) => { - let mut candidates: HashSet = HashSet::new(); + let mut candidates: HashSet = HashSet::new(); - // collect certificates as much as possible + // collect certificates as much as possible candidates.insert(dom); while let Ok(dom2) = rx_need_cert.try_recv() { candidates.insert(dom2); } - self.domain_validation(candidates, &static_domains, on_demand_checks.as_slice()).await + self.domain_validation(candidates, proc_domains.as_ref()).await } None => bail!("rx_need_cert closed"), } @@ -145,28 +150,36 @@ impl CertStore { async fn domain_validation( &self, candidates: HashSet, - static_domains: &HashSet, - checks: &[(glob::Pattern, Option)], + maybe_proc_domains: Option<&ProcessedDomains>, ) -> HashSet { let mut domains: HashSet = HashSet::new(); + // Handle initialization + let proc_domains = match maybe_proc_domains { + None => { + warn!("Proxy config is not yet loaded, refusing all certificate generation"); + return domains; + } + Some(proc) => proc, + }; + // Filter certificates... - for candidate in candidates.into_iter() { + 'outer: for candidate in candidates.into_iter() { // Disallow obvious wrong domains... if !candidate.contains('.') || candidate.ends_with(".local") { - warn!("Probably not a publicly accessible domain, skipping (a self-signed certificate will be used)"); + warn!("{} is probably not a publicly accessible domain, skipping (a self-signed certificate will be used)", candidate); continue; } // Try to register domain as a static domain - if static_domains.contains(&candidate) { + if proc_domains.static_domains.contains(&candidate) { trace!("domain {} validated as static domain", candidate); domains.insert(candidate); continue; } // It's not a static domain, maybe an on-demand domain? - for (pattern, maybe_check_url) in checks.iter() { + for (pattern, maybe_check_url) in proc_domains.on_demand_domains.iter() { // check glob pattern if pattern.matches(&candidate) { // if no check url is set, accept domain as long as it matches the pattern @@ -178,12 +191,14 @@ impl CertStore { pattern ); domains.insert(candidate); - break; + continue 'outer; } Some(url) => url, }; // if a check url is set, call it + // -- avoid DDoSing a backend + tokio::time::sleep(Duration::from_secs(2)).await; match self.on_demand_tls_ask(check_url, &candidate).await { Ok(()) => { trace!( @@ -193,7 +208,7 @@ impl CertStore { check_url ); domains.insert(candidate); - break; + continue 'outer; } Err(e) => { warn!("domain {} validation refused on glob pattern {} and on check url {} with error: {}", candidate, pattern, check_url, e); @@ -201,8 +216,6 @@ impl CertStore { } } } - // Avoid DDoSing a backend - tokio::time::sleep(Duration::from_secs(2)).await; } return domains; @@ -210,17 +223,6 @@ impl CertStore { /// This function is also in charge of the refresh of the domain names fn get_cert_for_https(self: &Arc, domain: &str) -> Result> { - // Check if domain is authorized - if !self - .rx_proxy_config - .borrow() - .entries - .iter() - .any(|ent| ent.url_prefix.host.matches(domain)) - { - bail!("Domain {} should not have a TLS certificate.", domain); - } - // Check in local memory if it exists if let Some(cert) = self.certs.read().unwrap().get(domain) { if cert.is_old() {