From e8214cb1807d3145907c7ed9e077fa45ada4aeea Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 22 Apr 2020 16:51:52 +0000 Subject: [PATCH] Better concurrency: Use Notify instead of stupid sleep in background worker Use Semaphore to limit concurrent requests in rpc_client Make more background tasks cancellable --- Cargo.lock | 171 +++++++++++++++++++++++----------------------- src/background.rs | 9 ++- src/main.rs | 2 +- src/membership.rs | 6 +- src/rpc_client.rs | 54 +++++++++++---- src/rpc_server.rs | 7 +- src/server.rs | 6 ++ src/table.rs | 10 +-- src/table_sync.rs | 3 +- test_write.sh | 5 +- 10 files changed, 153 insertions(+), 120 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2440009d..e8adba6d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10,7 +10,7 @@ dependencies = [ [[package]] name = "arc-swap" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -28,8 +28,8 @@ name = "atty" version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hermit-abi 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "hermit-abi 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -98,7 +98,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "cc" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -178,10 +178,10 @@ dependencies = [ [[package]] name = "err-derive" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro-error 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-error 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "rustversion 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -204,7 +204,7 @@ name = "fs2" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -316,17 +316,17 @@ dependencies = [ name = "garage" version = "0.1.0" dependencies = [ - "arc-swap 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "arc-swap 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", "async-trait 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", - "err-derive 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "err-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "gethostname 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.13.4 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)", "hyper-rustls 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "pretty_env_logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -338,8 +338,8 @@ dependencies = [ "serde_json 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", "sha2 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)", - "structopt 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-rustls 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)", "webpki 0.21.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -350,7 +350,7 @@ name = "generic-array" version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)", + "typenum 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -358,7 +358,7 @@ name = "gethostname" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -368,7 +368,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "wasi 0.9.0+wasi-snapshot-preview1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -386,7 +386,7 @@ dependencies = [ "indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -400,10 +400,10 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -445,7 +445,7 @@ dependencies = [ [[package]] name = "hyper" -version = "0.13.4" +version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -459,9 +459,9 @@ dependencies = [ "itoa 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "pin-project 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "want 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -473,10 +473,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", "futures-util 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.13.4 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-rustls 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "webpki 0.21.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -494,7 +494,7 @@ name = "iovec" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -526,12 +526,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.68" +version = "0.2.69" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "lock_api" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "scopeguard 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -573,7 +573,7 @@ dependencies = [ "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", "net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", @@ -587,7 +587,7 @@ version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -608,7 +608,7 @@ version = "0.2.33" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -622,11 +622,11 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "hermit-abi 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "hermit-abi 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -636,37 +636,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "parking_lot" -version = "0.10.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot_core 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot_core 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "parking_lot_core" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", "cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", - "smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "pin-project" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "pin-project-internal 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-internal 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "pin-project-internal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -700,10 +700,10 @@ dependencies = [ [[package]] name = "proc-macro-error" -version = "0.4.12" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "proc-macro-error-attr 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-error-attr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", @@ -712,7 +712,7 @@ dependencies = [ [[package]] name = "proc-macro-error-attr" -version = "0.4.12" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", @@ -759,7 +759,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "rand_chacha 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "rand_core 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand_hc 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -816,9 +816,9 @@ name = "ring" version = "0.16.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)", "untrusted 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", "web-sys 0.3.37 (registry+https://github.com/rust-lang/crates.io-index)", @@ -937,8 +937,8 @@ name = "signal-hook-registry" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "arc-swap 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "arc-swap 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -956,14 +956,14 @@ dependencies = [ "crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", "fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", "fxhash 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", - "parking_lot 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "parking_lot 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "smallvec" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -973,21 +973,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "structopt" -version = "0.3.12" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "structopt-derive 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "structopt-derive 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "structopt-derive" -version = "0.4.5" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "proc-macro-error 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)", + "proc-macro-error 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", "proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)", "quote 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", "syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1050,17 +1050,16 @@ dependencies = [ [[package]] name = "time" -version = "0.1.42" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.56 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "tokio" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1068,11 +1067,11 @@ dependencies = [ "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)", "memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)", "mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)", "mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "pin-project-lite 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "signal-hook-registry 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1097,7 +1096,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "futures-core 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "rustls 0.17.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", "webpki 0.21.2 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -1111,7 +1110,7 @@ dependencies = [ "futures-sink 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "pin-project-lite 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", - "tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)", + "tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -1134,7 +1133,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "typenum" -version = "1.11.2" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -1291,7 +1290,7 @@ dependencies = [ [metadata] "checksum aho-corasick 0.7.10 (registry+https://github.com/rust-lang/crates.io-index)" = "8716408b8bc624ed7f65d223ddb9ac2d044c0547b6fa4b0d554f3a9540496ada" -"checksum arc-swap 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "d663a8e9a99154b5fb793032533f6328da35e23aac63d5c152279aa8ba356825" +"checksum arc-swap 0.4.6 (registry+https://github.com/rust-lang/crates.io-index)" = "b585a98a234c46fc563103e9278c9391fde1f4e6850334da895d27edb9580f62" "checksum async-trait 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)" = "da71fef07bc806586090247e971229289f64c210a278ee5ae419314eb386b31d" "checksum atty 0.2.14 (registry+https://github.com/rust-lang/crates.io-index)" = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" @@ -1304,7 +1303,7 @@ dependencies = [ "checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" "checksum bytes 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)" = "130aac562c0dd69c56b3b1cc8ffd2e17be31d0b6c25b61c96b76231aa23e39e1" -"checksum cc 1.0.50 (registry+https://github.com/rust-lang/crates.io-index)" = "95e28fa049fda1c330bcf9d723be7663a899c4679724b34c81e9f5a326aab8cd" +"checksum cc 1.0.51 (registry+https://github.com/rust-lang/crates.io-index)" = "9c9384ca4b90c0ea47e19a5c996d6643a3e73dedf9b89c65efb67587e34da1bb" "checksum cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" "checksum clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5067f5bb2d80ef5d68b4c87db81601f0b75bca627bc2ef76b141d7b846a3c6d9" "checksum cloudabi 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" @@ -1313,7 +1312,7 @@ dependencies = [ "checksum crossbeam-utils 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" "checksum digest 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" "checksum env_logger 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -"checksum err-derive 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "82f46c91bbed409ee74495549acbfcc7fae856e712e1df15afe75d0775eedc6c" +"checksum err-derive 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "22deed3a8124cff5fa835713fa105621e43bbdc46690c3a6b68328a012d350d4" "checksum fake-simd 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum fs2 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)" = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" @@ -1334,13 +1333,13 @@ dependencies = [ "checksum getrandom 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "7abc8dd8451921606d809ba32e95b6111925cd2906060d2dcc29c070220503eb" "checksum h2 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)" = "377038bf3c89d18d6ca1431e7a5027194fbd724ca10592b9487ede5e8e144f42" "checksum heck 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" -"checksum hermit-abi 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "725cf19794cf90aa94e65050cb4191ff5d8fa87a498383774c47b332e3af952e" +"checksum hermit-abi 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "8a0d737e0f947a1864e93d33fdef4af8445a00d1ed8dc0c8ddb73139ea6abf15" "checksum hex 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" "checksum http 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" "checksum http-body 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" "checksum httparse 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" "checksum humantime 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -"checksum hyper 0.13.4 (registry+https://github.com/rust-lang/crates.io-index)" = "ed6081100e960d9d74734659ffc9cc91daf1c0fc7aceb8eaa94ee1a3f5046f2e" +"checksum hyper 0.13.5 (registry+https://github.com/rust-lang/crates.io-index)" = "96816e1d921eca64d208a85aab4f7798455a8e34229ee5a88c935bdee1b78b14" "checksum hyper-rustls 0.20.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ac965ea399ec3a25ac7d13b8affd4b8f39325cca00858ddf5eb29b79e6b14b08" "checksum indexmap 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "076f042c5b7b98f31d205f1249267e12a6518c1481e9dae9764af19b707d2292" "checksum iovec 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" @@ -1348,8 +1347,8 @@ dependencies = [ "checksum js-sys 0.3.37 (registry+https://github.com/rust-lang/crates.io-index)" = "6a27d435371a2fa5b6d2b028a74bbdb1234f308da363226a2854ca3ff8ba7055" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" "checksum lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" -"checksum libc 0.2.68 (registry+https://github.com/rust-lang/crates.io-index)" = "dea0c0405123bba743ee3f91f49b1c7cfb684eef0da0a50110f758ccf24cdff0" -"checksum lock_api 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "79b2de95ecb4691949fea4716ca53cdbcfccb2c612e19644a8bad05edcf9f47b" +"checksum libc 0.2.69 (registry+https://github.com/rust-lang/crates.io-index)" = "99e85c08494b21a9054e7fe1374a732aeadaff3980b6990b94bfd3a70f690005" +"checksum lock_api 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" "checksum log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "14b6052be84e6b71ab17edffc2eeabf5c2c3ae1fdb464aae35ac50c67a44e1f7" "checksum maybe-uninit 2.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" "checksum memchr 2.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" @@ -1359,18 +1358,18 @@ dependencies = [ "checksum miow 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" "checksum net2 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "42550d9fb7b6684a6d404d9fa7250c2eb2646df731d1c06afc06dcee9e1bcf88" "checksum num-traits 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "c62be47e61d1842b9170f0fdeec8eba98e60e90e5446449a0545e5152acd7096" -"checksum num_cpus 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "46203554f085ff89c235cd12f7075f3233af9b11ed7c9e16dfe2560d03313ce6" +"checksum num_cpus 1.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" "checksum opaque-debug 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" -"checksum parking_lot 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "92e98c49ab0b7ce5b222f2cc9193fc4efe11c6d0bd4f648e374684a6857b1cfc" -"checksum parking_lot_core 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7582838484df45743c8434fbff785e8edf260c28748353d44bc0da32e0ceabf1" -"checksum pin-project 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "7804a463a8d9572f13453c516a5faea534a2403d7ced2f0c7e100eeff072772c" -"checksum pin-project-internal 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)" = "385322a45f2ecf3410c68d2a549a4a2685e8051d0f278e39743ff4e451cb9b3f" +"checksum parking_lot 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" +"checksum parking_lot_core 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d58c7c768d4ba344e3e8d72518ac13e259d7c7ade24167003b8488e10b6740a3" +"checksum pin-project 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)" = "6f6a7f5eee6292c559c793430c55c00aea9d3b3d1905e855806ca4d7253426a2" +"checksum pin-project-internal 0.4.9 (registry+https://github.com/rust-lang/crates.io-index)" = "8988430ce790d8682672117bc06dda364c0be32d3abd738234f19f3240bad99a" "checksum pin-project-lite 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "237844750cfbb86f67afe27eee600dfbbcb6188d734139b534cbfbf4f96792ae" "checksum pin-utils 0.1.0-alpha.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5894c618ce612a3fa23881b152b608bafb8c56cfc22f434a3ba3120b40f7b587" "checksum ppv-lite86 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "74490b50b9fbe561ac330df47c08f3f33073d2d00c150f719147d7c54522fa1b" "checksum pretty_env_logger 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "926d36b9553851b8b0005f1275891b392ee4d2d833852c417ed025477350fb9d" -"checksum proc-macro-error 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "18f33027081eba0a6d8aba6d1b1c3a3be58cbb12106341c2d5759fcd9b5277e7" -"checksum proc-macro-error-attr 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "8a5b4b77fdb63c1eca72173d68d24501c54ab1269409f6b672c85deb18af69de" +"checksum proc-macro-error 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98e9e4b82e0ef281812565ea4751049f1bdcdfccda7d3f459f2e138a40c08678" +"checksum proc-macro-error-attr 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4f5444ead4e9935abd7f27dc51f7e852a0569ac888096d5ec2499470794e2e53" "checksum proc-macro-hack 0.5.15 (registry+https://github.com/rust-lang/crates.io-index)" = "0d659fe7c6d27f25e9d80a1a094c223f5246f6a6596453e09d7229bf42750b63" "checksum proc-macro-nested 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8e946095f9d3ed29ec38de908c22f95d9ac008e424c7bcae54c75a79c527c694" "checksum proc-macro2 1.0.10 (registry+https://github.com/rust-lang/crates.io-index)" = "df246d292ff63439fea9bc8c0a270bed0e390d5ebd4db4ba15aba81111b5abe3" @@ -1399,25 +1398,25 @@ dependencies = [ "checksum signal-hook-registry 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "94f478ede9f64724c5d173d7bb56099ec3e2d9fc2774aac65d34b8b890405f41" "checksum slab 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" "checksum sled 0.31.0 (registry+https://github.com/rust-lang/crates.io-index)" = "8fb6824dde66ad33bf20c6e8476f5b82b871bc8bc3c129a10ea2f7dae5060fa3" -"checksum smallvec 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "5c2fb2ec9bcd216a5b0d0ccf31ab17b5ed1d627960edff65bbe95d3ce221cefc" +"checksum smallvec 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "05720e22615919e4734f6a99ceae50d00226c3c5aca406e102ebc33298214e0a" "checksum spin 0.5.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -"checksum structopt 0.3.12 (registry+https://github.com/rust-lang/crates.io-index)" = "c8faa2719539bbe9d77869bfb15d4ee769f99525e707931452c97b693b3f159d" -"checksum structopt-derive 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3f88b8e18c69496aad6f9ddf4630dd7d585bcaf765786cb415b9aec2fe5a0430" +"checksum structopt 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "863246aaf5ddd0d6928dfeb1a9ca65f505599e4e1b399935ef7e75107516b4ef" +"checksum structopt-derive 0.4.7 (registry+https://github.com/rust-lang/crates.io-index)" = "d239ca4b13aee7a2142e6795cbd69e457665ff8037aed33b3effdc430d2f927a" "checksum syn 1.0.17 (registry+https://github.com/rust-lang/crates.io-index)" = "0df0eb663f387145cab623dea85b09c2c5b4b0aef44e945d928e682fce71bb03" "checksum syn-mid 0.5.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7be3539f6c128a931cf19dcee741c1af532c7fd387baa739c03dd2e96479338a" "checksum synstructure 0.12.3 (registry+https://github.com/rust-lang/crates.io-index)" = "67656ea1dc1b41b1451851562ea232ec2e5a80242139f7e679ceccfb5d61f545" "checksum termcolor 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)" = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" "checksum textwrap 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" "checksum thread_local 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -"checksum time 0.1.42 (registry+https://github.com/rust-lang/crates.io-index)" = "db8dcfca086c1143c9270ac42a2bbd8a7ee477b78ac8e45b19abfb0cbede4b6f" -"checksum tokio 0.2.16 (registry+https://github.com/rust-lang/crates.io-index)" = "ee5a0dd887e37d37390c13ff8ac830f992307fe30a1fff0ab8427af67211ba28" +"checksum time 0.1.43 (registry+https://github.com/rust-lang/crates.io-index)" = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +"checksum tokio 0.2.18 (registry+https://github.com/rust-lang/crates.io-index)" = "34ef16d072d2b6dc8b4a56c70f5c5ced1a37752116f8e7c1e80c659aa7cb6713" "checksum tokio-macros 0.2.5 (registry+https://github.com/rust-lang/crates.io-index)" = "f0c3acc6aa564495a0f2e1d59fab677cd7f81a19994cfc7f3ad0e64301560389" "checksum tokio-rustls 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4adb8b3e5f86b707f1b54e7c15b6de52617a823608ccda98a15d3a24222f265a" "checksum tokio-util 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)" = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" "checksum toml 0.5.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ffc92d160b1eef40665be3a05630d003936a3bc7da7421277846c2613e92c71a" "checksum tower-service 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" "checksum try-lock 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e604eb7b43c06650e854be16a2a03155743d3752dd1c943f6829e26b7a36e382" -"checksum typenum 1.11.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6d2783fe2d6b8c1101136184eb41be8b1ad379e4657050b8aaff0c79ee7575f9" +"checksum typenum 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)" = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" "checksum unicode-segmentation 1.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" "checksum unicode-width 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479" "checksum unicode-xid 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c" diff --git a/src/background.rs b/src/background.rs index 1f04e49b3..f0dbdcb50 100644 --- a/src/background.rs +++ b/src/background.rs @@ -3,9 +3,8 @@ use std::pin::Pin; use futures::future::join_all; use std::sync::Arc; -use std::time::Duration; use tokio::sync::Mutex; -use tokio::sync::{mpsc, watch}; +use tokio::sync::{mpsc, watch, Notify}; use crate::error::Error; @@ -18,6 +17,7 @@ pub struct BackgroundRunner { queue_in: mpsc::UnboundedSender<(Job, bool)>, queue_out: Mutex>, + job_notify: Notify, workers: Mutex>>, } @@ -30,6 +30,7 @@ impl BackgroundRunner { stop_signal, queue_in, queue_out: Mutex::new(queue_out), + job_notify: Notify::new(), workers: Mutex::new(Vec::new()), }) } @@ -58,6 +59,7 @@ impl BackgroundRunner { { let boxed: Job = Box::pin(job); let _: Result<_, _> = self.queue_in.clone().send((boxed, false)); + self.job_notify.notify(); } pub fn spawn_cancellable(&self, job: T) @@ -66,6 +68,7 @@ impl BackgroundRunner { { let boxed: Job = Box::pin(job); let _: Result<_, _> = self.queue_in.clone().send((boxed, true)); + self.job_notify.notify(); } pub async fn spawn_worker(&self, name: String, worker: F) @@ -97,7 +100,7 @@ impl BackgroundRunner { info!("Background runner {} exiting", i); return; } - tokio::time::delay_for(Duration::from_secs(1)).await; + self.job_notify.notified().await; } } } diff --git a/src/main.rs b/src/main.rs index e0ae7db7b..8985f181d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -238,7 +238,7 @@ async fn main() { }; let rpc_http_cli = - Arc::new(RpcHttpClient::new(&tls_config).expect("Could not create RPC client")); + Arc::new(RpcHttpClient::new(8, &tls_config).expect("Could not create RPC client")); let membership_rpc_cli = RpcAddrClient::new(rpc_http_cli.clone(), MEMBERSHIP_RPC_PATH.to_string()); let admin_rpc_cli = RpcAddrClient::new(rpc_http_cli.clone(), ADMIN_RPC_PATH.to_string()); diff --git a/src/membership.rs b/src/membership.rs index 78c1dbe3d..f9ffa3b40 100644 --- a/src/membership.rs +++ b/src/membership.rs @@ -296,8 +296,10 @@ impl System { ring.rebuild_ring(); let (update_ring, ring) = watch::channel(Arc::new(ring)); - let rpc_http_client = - Arc::new(RpcHttpClient::new(&config.rpc_tls).expect("Could not create RPC client")); + let rpc_http_client = Arc::new( + RpcHttpClient::new(config.max_concurrent_requests, &config.rpc_tls) + .expect("Could not create RPC client"), + ); let rpc_path = MEMBERSHIP_RPC_PATH.to_string(); let rpc_client = RpcClient::new( diff --git a/src/rpc_client.rs b/src/rpc_client.rs index 4a0652644..a5c44a2ec 100644 --- a/src/rpc_client.rs +++ b/src/rpc_client.rs @@ -10,7 +10,7 @@ use futures::stream::StreamExt; use futures_util::future::FutureExt; use hyper::client::{Client, HttpConnector}; use hyper::{Body, Method, Request}; -use tokio::sync::watch; +use tokio::sync::{watch, Semaphore}; use crate::background::BackgroundRunner; use crate::data::*; @@ -117,7 +117,7 @@ impl RpcClient { if results.len() >= stop_after { // Continue requests in background // TODO: make this optionnal (only usefull for write requests) - self.clone().background.spawn(async move { + self.clone().background.spawn_cancellable(async move { resp_stream.collect::>().await; Ok(()) }); @@ -164,14 +164,22 @@ impl RpcAddrClient { } } -pub enum RpcHttpClient { +pub struct RpcHttpClient { + request_limiter: Semaphore, + method: ClientMethod, +} + +enum ClientMethod { HTTP(Client), HTTPS(Client, hyper::Body>), } impl RpcHttpClient { - pub fn new(tls_config: &Option) -> Result { - if let Some(cf) = tls_config { + pub fn new( + max_concurrent_requests: usize, + tls_config: &Option, + ) -> Result { + let method = if let Some(cf) = tls_config { let ca_certs = tls_util::load_certs(&cf.ca_cert)?; let node_certs = tls_util::load_certs(&cf.node_cert)?; let node_key = tls_util::load_private_key(&cf.node_key)?; @@ -187,10 +195,14 @@ impl RpcHttpClient { let connector = tls_util::HttpsConnectorFixedDnsname::::new(config, "garage"); - Ok(RpcHttpClient::HTTPS(Client::builder().build(connector))) + ClientMethod::HTTPS(Client::builder().build(connector)) } else { - Ok(RpcHttpClient::HTTP(Client::new())) - } + ClientMethod::HTTP(Client::new()) + }; + Ok(RpcHttpClient { + method, + request_limiter: Semaphore::new(max_concurrent_requests), + }) } async fn call( @@ -204,9 +216,9 @@ impl RpcHttpClient { MB: Borrow, M: RpcMessage, { - let uri = match self { - RpcHttpClient::HTTP(_) => format!("http://{}/{}", to_addr, path), - RpcHttpClient::HTTPS(_) => format!("https://{}/{}", to_addr, path), + let uri = match self.method { + ClientMethod::HTTP(_) => format!("http://{}/{}", to_addr, path), + ClientMethod::HTTPS(_) => format!("https://{}/{}", to_addr, path), }; let req = Request::builder() @@ -214,12 +226,22 @@ impl RpcHttpClient { .uri(uri) .body(Body::from(rmp_to_vec_all_named(msg.borrow())?))?; - let resp_fut = match self { - RpcHttpClient::HTTP(client) => client.request(req).fuse(), - RpcHttpClient::HTTPS(client) => client.request(req).fuse(), + let resp_fut = match &self.method { + ClientMethod::HTTP(client) => client.request(req).fuse(), + ClientMethod::HTTPS(client) => client.request(req).fuse(), }; + + let slot = self.request_limiter.acquire().await; let resp = tokio::time::timeout(timeout, resp_fut) - .await? + .await + .map_err(|e| { + debug!( + "RPC timeout to {}: {}", + to_addr, + debug_serialize(msg.borrow()) + ); + e + })? .map_err(|e| { warn!( "RPC HTTP client error when connecting to {}: {}", @@ -227,6 +249,7 @@ impl RpcHttpClient { ); e })?; + drop(slot); let status = resp.status(); let body = hyper::body::to_bytes(resp.into_body()).await?; @@ -240,3 +263,4 @@ impl RpcHttpClient { } } } + diff --git a/src/rpc_server.rs b/src/rpc_server.rs index 938eb5125..51661a669 100644 --- a/src/rpc_server.rs +++ b/src/rpc_server.rs @@ -52,9 +52,10 @@ where Ok(resp) => { let resp_bytes = rmp_to_vec_all_named::>(&Ok(resp))?; trace!( - "]RPC:{},ok ({} ms)", + "]RPC:{},ok ({} ms), request: {}", name, - (Instant::now() - begin_time).as_millis() + (Instant::now() - begin_time).as_millis(), + req_str, ); Ok(Response::new(Body::from(resp_bytes))) } @@ -68,7 +69,7 @@ where name, e, (Instant::now() - begin_time).as_millis(), - req_str + req_str, ); Ok(err_response) } diff --git a/src/server.rs b/src/server.rs index 542c8675f..7b6f22401 100644 --- a/src/server.rs +++ b/src/server.rs @@ -35,6 +35,9 @@ pub struct Config { pub bootstrap_peers: Vec, + #[serde(default = "default_max_concurrent_requests")] + pub max_concurrent_requests: usize, + #[serde(default = "default_block_size")] pub block_size: usize, @@ -50,6 +53,9 @@ pub struct Config { pub rpc_tls: Option, } +fn default_max_concurrent_requests() -> usize { + 12 +} fn default_block_size() -> usize { 1048576 } diff --git a/src/table.rs b/src/table.rs index 3a21dfc70..bc375a968 100644 --- a/src/table.rs +++ b/src/table.rs @@ -269,7 +269,7 @@ where let ent2 = ret_entry.clone(); self.system .background - .spawn(async move { self2.repair_on_read(&who[..], ent2).await }); + .spawn_cancellable(async move { self2.repair_on_read(&who[..], ent2).await }); } } Ok(ret) @@ -324,7 +324,7 @@ where } if !to_repair.is_empty() { let self2 = self.clone(); - self.system.background.spawn(async move { + self.system.background.spawn_cancellable(async move { for (_, v) in to_repair.iter_mut() { self2.repair_on_read(&who[..], v.take().unwrap()).await?; } @@ -472,7 +472,7 @@ where self.instance.updated(old_entry, Some(new_entry)).await?; self.system .background - .spawn(syncer.clone().invalidate(tree_key)); + .spawn_cancellable(syncer.clone().invalidate(tree_key)); } } @@ -480,7 +480,7 @@ where let self2 = self.clone(); self.system .background - .spawn(async move { self2.insert_many(&epidemic_propagate[..]).await }); + .spawn_cancellable(async move { self2.insert_many(&epidemic_propagate[..]).await }); } Ok(()) @@ -500,7 +500,7 @@ where self.instance.updated(Some(old_entry), None).await?; self.system .background - .spawn(syncer.clone().invalidate(key.to_vec())); + .spawn_cancellable(syncer.clone().invalidate(key.to_vec())); count += 1; } } diff --git a/src/table_sync.rs b/src/table_sync.rs index 6442841da..603c7aa6a 100644 --- a/src/table_sync.rs +++ b/src/table_sync.rs @@ -132,10 +132,9 @@ where .await; let s3 = syncer.clone(); - table.system.background.spawn(async move { + tokio::spawn(async move { tokio::time::delay_for(Duration::from_secs(20)).await; s3.add_full_scan().await; - Ok(()) }); syncer diff --git a/test_write.sh b/test_write.sh index 2c06594d8..12a3cb500 100755 --- a/test_write.sh +++ b/test_write.sh @@ -1,8 +1,7 @@ #!/bin/bash for FILE in $(find target/debug/deps); do - echo - echo $FILE - curl -v localhost:3900/$FILE -X PUT -H 'Host: garage' -H 'Content-Type: application/blob' --data-binary "@$FILE" + echo -n "$FILE " + curl localhost:3900/$FILE -X PUT -H 'Host: garage' -H 'Content-Type: application/blob' --data-binary "@$FILE" || echo "ERROR" done