forked from Deuxfleurs/diplonat
Better retry mechanism
This commit is contained in:
parent
28b661aa47
commit
ec777652c1
2 changed files with 7 additions and 2 deletions
|
@ -27,6 +27,10 @@ impl Consul {
|
|||
};
|
||||
}
|
||||
|
||||
pub fn watch_node_reset(&mut self) -> () {
|
||||
self.idx = None;
|
||||
}
|
||||
|
||||
pub async fn watch_node(&mut self, host: &str) -> Result<CatalogNode> {
|
||||
let url = match self.idx {
|
||||
Some(i) => format!("{}/v1/catalog/node/{}?index={}", self.url, host, i),
|
||||
|
|
|
@ -31,7 +31,7 @@ pub struct ConsulActor {
|
|||
fn retry_to_time(retries: u32, max_time: Duration) -> Duration {
|
||||
// 1.2^x seems to be a good value to exponentially increase time at a good pace
|
||||
// eg. 1.2^32 = 341 seconds ~= 5 minutes - ie. after 32 retries we wait 5 minutes
|
||||
return Duration::from_secs(cmp::max(max_time.as_secs(), 1.2f64.powf(retries as f64) as u64))
|
||||
return Duration::from_secs(cmp::min(max_time.as_secs(), 1.2f64.powf(retries as f64) as u64))
|
||||
}
|
||||
|
||||
fn from_catalog_to_open_ports(catalog: &consul::CatalogNode) -> messages::OpenPorts {
|
||||
|
@ -71,6 +71,7 @@ impl ConsulActor {
|
|||
let catalog = match self.consul.watch_node(&self.node).await {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
self.consul.watch_node_reset();
|
||||
self.retries = cmp::min(u32::MAX - 1, self.retries) + 1;
|
||||
let will_retry_in = retry_to_time(self.retries, Duration::from_secs(600));
|
||||
error!("Failed to query consul. Will retry in {}s. {}", will_retry_in.as_secs(), e);
|
||||
|
@ -78,7 +79,7 @@ impl ConsulActor {
|
|||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
self.retries = 0;
|
||||
info!("{:#?}", from_catalog_to_open_ports(&catalog));
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue