Add semaphore to limit RAM used by buffered outgoing requests
This commit is contained in:
parent
8c4f418fe8
commit
6f13d083ab
3 changed files with 34 additions and 7 deletions
|
@ -7,6 +7,7 @@ use futures::stream::futures_unordered::FuturesUnordered;
|
||||||
use futures::stream::StreamExt;
|
use futures::stream::StreamExt;
|
||||||
use futures_util::future::FutureExt;
|
use futures_util::future::FutureExt;
|
||||||
use tokio::select;
|
use tokio::select;
|
||||||
|
use tokio::sync::Semaphore;
|
||||||
|
|
||||||
pub use netapp::endpoint::{Endpoint, EndpointHandler, Message as Rpc};
|
pub use netapp::endpoint::{Endpoint, EndpointHandler, Message as Rpc};
|
||||||
use netapp::peering::fullmesh::FullMeshPeeringStrategy;
|
use netapp::peering::fullmesh::FullMeshPeeringStrategy;
|
||||||
|
@ -14,11 +15,16 @@ pub use netapp::proto::*;
|
||||||
pub use netapp::{NetApp, NodeID};
|
pub use netapp::{NetApp, NodeID};
|
||||||
|
|
||||||
use garage_util::background::BackgroundRunner;
|
use garage_util::background::BackgroundRunner;
|
||||||
use garage_util::data::Uuid;
|
use garage_util::data::*;
|
||||||
use garage_util::error::Error;
|
use garage_util::error::Error;
|
||||||
|
|
||||||
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
|
const DEFAULT_TIMEOUT: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
|
// Try to never have more than 200MB of outgoing requests
|
||||||
|
// buffered at the same time. Other requests are queued until
|
||||||
|
// space is freed.
|
||||||
|
const REQUEST_BUFFER_SIZE: usize = 200 * 1024 * 1024;
|
||||||
|
|
||||||
/// Strategy to apply when making RPC
|
/// Strategy to apply when making RPC
|
||||||
#[derive(Copy, Clone)]
|
#[derive(Copy, Clone)]
|
||||||
pub struct RequestStrategy {
|
pub struct RequestStrategy {
|
||||||
|
@ -64,9 +70,21 @@ impl RequestStrategy {
|
||||||
pub struct RpcHelper {
|
pub struct RpcHelper {
|
||||||
pub(crate) fullmesh: Arc<FullMeshPeeringStrategy>,
|
pub(crate) fullmesh: Arc<FullMeshPeeringStrategy>,
|
||||||
pub(crate) background: Arc<BackgroundRunner>,
|
pub(crate) background: Arc<BackgroundRunner>,
|
||||||
|
request_buffer_semaphore: Arc<Semaphore>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RpcHelper {
|
impl RpcHelper {
|
||||||
|
pub(crate) fn new(
|
||||||
|
fullmesh: Arc<FullMeshPeeringStrategy>,
|
||||||
|
background: Arc<BackgroundRunner>,
|
||||||
|
) -> Self {
|
||||||
|
Self {
|
||||||
|
fullmesh,
|
||||||
|
background,
|
||||||
|
request_buffer_semaphore: Arc::new(Semaphore::new(REQUEST_BUFFER_SIZE)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn call<M, H, S>(
|
pub async fn call<M, H, S>(
|
||||||
&self,
|
&self,
|
||||||
endpoint: &Endpoint<M, H>,
|
endpoint: &Endpoint<M, H>,
|
||||||
|
@ -92,10 +110,19 @@ impl RpcHelper {
|
||||||
M: Rpc<Response = Result<S, Error>>,
|
M: Rpc<Response = Result<S, Error>>,
|
||||||
H: EndpointHandler<M>,
|
H: EndpointHandler<M>,
|
||||||
{
|
{
|
||||||
|
let msg_size = rmp_to_vec_all_named(&msg)?.len() as u32;
|
||||||
|
let permit = self.request_buffer_semaphore.acquire_many(msg_size).await?;
|
||||||
|
|
||||||
let node_id = to.into();
|
let node_id = to.into();
|
||||||
select! {
|
select! {
|
||||||
res = endpoint.call(&node_id, &msg, strat.rs_priority) => Ok(res??),
|
res = endpoint.call(&node_id, &msg, strat.rs_priority) => {
|
||||||
_ = tokio::time::sleep(strat.rs_timeout) => Err(Error::Timeout),
|
drop(permit);
|
||||||
|
Ok(res??)
|
||||||
|
}
|
||||||
|
_ = tokio::time::sleep(strat.rs_timeout) => {
|
||||||
|
drop(permit);
|
||||||
|
Err(Error::Timeout)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -235,10 +235,7 @@ impl System {
|
||||||
node_status: RwLock::new(HashMap::new()),
|
node_status: RwLock::new(HashMap::new()),
|
||||||
netapp: netapp.clone(),
|
netapp: netapp.clone(),
|
||||||
fullmesh: fullmesh.clone(),
|
fullmesh: fullmesh.clone(),
|
||||||
rpc: RpcHelper {
|
rpc: RpcHelper::new(fullmesh, background.clone()),
|
||||||
fullmesh,
|
|
||||||
background: background.clone(),
|
|
||||||
},
|
|
||||||
system_endpoint,
|
system_endpoint,
|
||||||
replication_factor,
|
replication_factor,
|
||||||
rpc_listen_addr: config.rpc_bind_addr,
|
rpc_listen_addr: config.rpc_bind_addr,
|
||||||
|
|
|
@ -41,6 +41,9 @@ pub enum Error {
|
||||||
#[error(display = "Tokio join error: {}", _0)]
|
#[error(display = "Tokio join error: {}", _0)]
|
||||||
TokioJoin(#[error(source)] tokio::task::JoinError),
|
TokioJoin(#[error(source)] tokio::task::JoinError),
|
||||||
|
|
||||||
|
#[error(display = "Tokio semaphore acquire error: {}", _0)]
|
||||||
|
TokioSemAcquire(#[error(source)] tokio::sync::AcquireError),
|
||||||
|
|
||||||
#[error(display = "Remote error: {}", _0)]
|
#[error(display = "Remote error: {}", _0)]
|
||||||
RemoteError(String),
|
RemoteError(String),
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue