Better retry mechanism

src/consul.rs

@@ -27,6 +27,10 @@ impl Consul {
     };
   }
 
+  pub fn watch_node_reset(&mut self) -> () {
+    self.idx = None;
+  }
+
   pub async fn watch_node(&mut self, host: &str) -> Result<CatalogNode> {
     let url = match self.idx {
       Some(i) => format!("{}/v1/catalog/node/{}?index={}", self.url, host, i),

src/consul_actor.rs

@@ -31,7 +31,7 @@ pub struct ConsulActor {
 fn retry_to_time(retries: u32, max_time: Duration) -> Duration {
   // 1.2^x seems to be a good value to exponentially increase time at a good pace
   // eg. 1.2^32 = 341 seconds ~= 5 minutes - ie. after 32 retries we wait 5 minutes
-  return Duration::from_secs(cmp::max(max_time.as_secs(), 1.2f64.powf(retries as f64) as u64))
+  return Duration::from_secs(cmp::min(max_time.as_secs(), 1.2f64.powf(retries as f64) as u64))
 }
 
 fn from_catalog_to_open_ports(catalog: &consul::CatalogNode) -> messages::OpenPorts {
@@ -71,6 +71,7 @@ impl ConsulActor {
       let catalog = match self.consul.watch_node(&self.node).await {
         Ok(c) => c,
         Err(e) => {
+          self.consul.watch_node_reset();
           self.retries = cmp::min(u32::MAX - 1, self.retries) + 1;
           let will_retry_in = retry_to_time(self.retries, Duration::from_secs(600));
           error!("Failed to query consul. Will retry in {}s. {}", will_retry_in.as_secs(), e);
@@ -78,7 +79,7 @@ impl ConsulActor {
           continue;
         }
       };
-
+      self.retries = 0;
       info!("{:#?}", from_catalog_to_open_ports(&catalog));
     }
   }