Retry DNS updates when they fail

src/dns_config.rs

@@ -24,7 +24,7 @@ pub struct DnsConfig {
     pub entries: HashMap<DnsEntryKey, DnsEntryValue>,
 }
 
-#[derive(Debug, Hash, PartialEq, Eq)]
+#[derive(Clone, Debug, Hash, PartialEq, Eq)]
 pub struct DnsEntryKey {
     pub dns_path: String,
     pub record_type: DnsRecordType,
@@ -35,7 +35,7 @@ pub struct DnsEntryValue {
     pub targets: HashSet<String>,
 }
 
-#[derive(Debug, Hash, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)]
 #[allow(clippy::upper_case_acronyms)]
 pub enum DnsRecordType {
     A,

src/dns_updater.rs

@@ -1,3 +1,4 @@
+use std::collections::HashSet;
 use std::net::{Ipv4Addr, Ipv6Addr};
 use std::sync::Arc;
 use std::time::Duration;
@@ -10,6 +11,8 @@ use tracing::*;
 use crate::dns_config::*;
 use crate::DomainProvider;
 
+const RETRY_DELAY: Duration = Duration::from_secs(600); // 10 minutes
+
 pub async fn dns_updater_task(
     mut rx_dns_config: watch::Receiver<Arc<DnsConfig>>,
     providers: Vec<DomainProvider>,
@@ -30,6 +33,8 @@ pub async fn dns_updater_task(
     info!("DNS updater starting");
 
     let mut config = Arc::new(DnsConfig::new());
+    let mut failures = HashSet::new();
+
     while !*must_exit.borrow() {
         select!(
             c = rx_dns_config.changed() => {
@@ -37,19 +42,31 @@ pub async fn dns_updater_task(
                     break;
                 }
             }
+            _ = tokio::time::sleep(RETRY_DELAY) => {
+                if failures.is_empty() {
+                    continue;
+                }
+            }
             _ = must_exit.changed() => continue,
         );
 
         // Always lag 15 seconds behind actual updates,
         // to avoid sending too many at once and hitting rate limits
-        // TODO: retry regularly rate limits are hit
         tokio::time::sleep(Duration::from_secs(15)).await;
 
         let new_config: Arc<DnsConfig> = rx_dns_config.borrow_and_update().clone();
+        let mut new_failures = HashSet::new();
 
         for (key, value) in new_config.entries.iter() {
-            // Skip entries that haven't changed
-            if config.entries.get(key) == Some(value) {
+            if failures.contains(key) {
+                info!(
+                    record = key.to_string(),
+                    target = value.to_string(),
+                    "retrying after failure"
+                );
+            } else if config.entries.get(key) == Some(value) {
+                // Skip entries that haven't changed, and that were
+                // successfully updated on the previous iteration
                 continue;
             }
 
@@ -75,8 +92,9 @@ pub async fn dns_updater_task(
                         record = key.to_string(),
                         target = value.to_string(),
                         error = e.to_string(),
-                        "unable to update record"
+                        "unable to update record, will retry later"
                     );
+                    new_failures.insert(key.clone());
                 }
             } else {
                 error!(
@@ -87,6 +105,7 @@ pub async fn dns_updater_task(
         }
 
         config = new_config;
+        failures = new_failures;
     }
 }