Retry DNS updates when they fail
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Alex 2023-01-11 22:35:56 +01:00
parent 7c8dcd7aa7
commit 5f946d485c
Signed by: lx
GPG key ID: 0E496D15096376BE
2 changed files with 25 additions and 6 deletions

View file

@ -24,7 +24,7 @@ pub struct DnsConfig {
pub entries: HashMap<DnsEntryKey, DnsEntryValue>,
}
#[derive(Debug, Hash, PartialEq, Eq)]
#[derive(Clone, Debug, Hash, PartialEq, Eq)]
pub struct DnsEntryKey {
pub dns_path: String,
pub record_type: DnsRecordType,
@ -35,7 +35,7 @@ pub struct DnsEntryValue {
pub targets: HashSet<String>,
}
#[derive(Debug, Hash, PartialEq, Eq)]
#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
#[allow(clippy::upper_case_acronyms)]
pub enum DnsRecordType {
A,

View file

@ -1,3 +1,4 @@
use std::collections::HashSet;
use std::net::{Ipv4Addr, Ipv6Addr};
use std::sync::Arc;
use std::time::Duration;
@ -10,6 +11,8 @@ use tracing::*;
use crate::dns_config::*;
use crate::DomainProvider;
const RETRY_DELAY: Duration = Duration::from_secs(600); // 10 minutes
pub async fn dns_updater_task(
mut rx_dns_config: watch::Receiver<Arc<DnsConfig>>,
providers: Vec<DomainProvider>,
@ -30,6 +33,8 @@ pub async fn dns_updater_task(
info!("DNS updater starting");
let mut config = Arc::new(DnsConfig::new());
let mut failures = HashSet::new();
while !*must_exit.borrow() {
select!(
c = rx_dns_config.changed() => {
@ -37,19 +42,31 @@ pub async fn dns_updater_task(
break;
}
}
_ = tokio::time::sleep(RETRY_DELAY) => {
if failures.is_empty() {
continue;
}
}
_ = must_exit.changed() => continue,
);
// Always lag 15 seconds behind actual updates,
// to avoid sending too many at once and hitting rate limits
// TODO: retry regularly rate limits are hit
tokio::time::sleep(Duration::from_secs(15)).await;
let new_config: Arc<DnsConfig> = rx_dns_config.borrow_and_update().clone();
let mut new_failures = HashSet::new();
for (key, value) in new_config.entries.iter() {
// Skip entries that haven't changed
if config.entries.get(key) == Some(value) {
if failures.contains(key) {
info!(
record = key.to_string(),
target = value.to_string(),
"retrying after failure"
);
} else if config.entries.get(key) == Some(value) {
// Skip entries that haven't changed, and that were
// successfully updated on the previous iteration
continue;
}
@ -75,8 +92,9 @@ pub async fn dns_updater_task(
record = key.to_string(),
target = value.to_string(),
error = e.to_string(),
"unable to update record"
"unable to update record, will retry later"
);
new_failures.insert(key.clone());
}
} else {
error!(
@ -87,6 +105,7 @@ pub async fn dns_updater_task(
}
config = new_config;
failures = new_failures;
}
}