Deuxfleurs/garage
48
56
Fork 73
Code Issues 100 Pull requests 13 Releases 30 Wiki Activity

WIP migrate to tokio 1
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse source
This commit is contained in:
Alex 2021-03-15 22:36:41 +01:00
parent 4d4117f2b4
commit 0cd5b2ae19
21 changed files with 233 additions and 336 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

278
Cargo.lock generated
View file

@ -17,6 +17,12 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8" checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8"
[[package]]
name = "arc-swap"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73"
[[package]] [[package]]
name = "arrayvec" name = "arrayvec"
version = "0.3.25" version = "0.3.25"
@ -46,7 +52,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [ dependencies = [
"hermit-abi", "hermit-abi",
"libc", "libc",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -55,12 +61,6 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "base64"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.13.0" version = "0.13.0"
@ -142,12 +142,6 @@ dependencies = [
"iovec", "iovec",
] ]
[[package]]
name = "bytes"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.0.1" version = "1.0.1"
@ -182,7 +176,7 @@ dependencies = [
"num-integer", "num-integer",
"num-traits", "num-traits",
"time", "time",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -347,7 +341,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [ dependencies = [
"libc", "libc",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -356,22 +350,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
dependencies = [
"bitflags",
"fuchsia-zircon-sys",
]
[[package]]
name = "fuchsia-zircon-sys"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.12" version = "0.3.12"
@ -460,7 +438,7 @@ dependencies = [
"futures-sink", "futures-sink",
"futures-task", "futures-task",
"memchr", "memchr",
"pin-project-lite 0.2.4", "pin-project-lite",
"pin-utils", "pin-utils",
"proc-macro-hack", "proc-macro-hack",
"proc-macro-nested", "proc-macro-nested",
@ -507,7 +485,7 @@ dependencies = [
name = "garage_api" name = "garage_api"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"base64 0.13.0", "base64",
"bytes 0.4.12", "bytes 0.4.12",
"chrono", "chrono",
"crypto-mac 0.7.0", "crypto-mac 0.7.0",
@ -537,7 +515,7 @@ dependencies = [
name = "garage_model" name = "garage_model"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap 0.4.8",
"async-trait", "async-trait",
"bytes 0.4.12", "bytes 0.4.12",
"futures", "futures",
@ -560,7 +538,7 @@ dependencies = [
name = "garage_rpc" name = "garage_rpc"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap 0.4.8",
"bytes 0.4.12", "bytes 0.4.12",
"futures", "futures",
"futures-util", "futures-util",
@ -579,6 +557,7 @@ dependencies = [
"sha2", "sha2",
"tokio", "tokio",
"tokio-rustls", "tokio-rustls",
"tokio-stream",
"webpki", "webpki",
] ]
@ -607,6 +586,7 @@ dependencies = [
name = "garage_util" name = "garage_util"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"arc-swap 1.2.0",
"blake2", "blake2",
"chrono", "chrono",
"err-derive", "err-derive",
@ -664,7 +644,7 @@ dependencies = [
"libc", "libc",
"log", "log",
"rustversion", "rustversion",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -693,7 +673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028" checksum = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028"
dependencies = [ dependencies = [
"libc", "libc",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -731,11 +711,11 @@ dependencies = [
[[package]] [[package]]
name = "h2" name = "h2"
version = "0.2.7" version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78"
dependencies = [ dependencies = [
"bytes 0.5.6", "bytes 1.0.1",
"fnv", "fnv",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -746,7 +726,6 @@ dependencies = [
"tokio", "tokio",
"tokio-util", "tokio-util",
"tracing", "tracing",
"tracing-futures",
] ]
[[package]] [[package]]
@ -812,11 +791,11 @@ dependencies = [
[[package]] [[package]]
name = "http-body" name = "http-body"
version = "0.3.1" version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994"
dependencies = [ dependencies = [
"bytes 0.5.6", "bytes 1.0.1",
"http", "http",
] ]
@ -849,11 +828,11 @@ dependencies = [
[[package]] [[package]]
name = "hyper" name = "hyper"
version = "0.13.9" version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7"
dependencies = [ dependencies = [
"bytes 0.5.6", "bytes 1.0.1",
"futures-channel", "futures-channel",
"futures-core", "futures-core",
"futures-util", "futures-util",
@ -863,7 +842,7 @@ dependencies = [
"httparse", "httparse",
"httpdate", "httpdate",
"itoa", "itoa",
"pin-project 1.0.4", "pin-project",
"socket2", "socket2",
"tokio", "tokio",
"tower-service", "tower-service",
@ -873,11 +852,10 @@ dependencies = [
[[package]] [[package]]
name = "hyper-rustls" name = "hyper-rustls"
version = "0.20.0" version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac965ea399ec3a25ac7d13b8affd4b8f39325cca00858ddf5eb29b79e6b14b08" checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
dependencies = [ dependencies = [
"bytes 0.5.6",
"futures-util", "futures-util",
"hyper", "hyper",
"log", "log",
@ -947,16 +925,6 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "kernel32-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
dependencies = [
"winapi 0.2.8",
"winapi-build",
]
[[package]] [[package]]
name = "lazy_static" name = "lazy_static"
version = "1.4.0" version = "1.4.0"
@ -965,9 +933,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.82" version = "0.2.89"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" checksum = "538c092e5586f4cdd7dd8078c4a79220e3e168880218124dcbce860f0ea938c6"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
@ -1032,55 +1000,25 @@ dependencies = [
[[package]] [[package]]
name = "mio" name = "mio"
version = "0.6.23" version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" checksum = "2182a122f3b7f3f5329cb1972cee089ba2459a0a80a56935e6e674f096f8d839"
dependencies = [ dependencies = [
"cfg-if 0.1.10",
"fuchsia-zircon",
"fuchsia-zircon-sys",
"iovec",
"kernel32-sys",
"libc", "libc",
"log", "log",
"miow", "miow",
"net2", "ntapi",
"slab", "winapi",
"winapi 0.2.8",
]
[[package]]
name = "mio-uds"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0"
dependencies = [
"iovec",
"libc",
"mio",
] ]
[[package]] [[package]]
name = "miow" name = "miow"
version = "0.2.2" version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897"
dependencies = [ dependencies = [
"kernel32-sys", "socket2",
"net2", "winapi",
"winapi 0.2.8",
"ws2_32-sys",
]
[[package]]
name = "net2"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae"
dependencies = [
"cfg-if 0.1.10",
"libc",
"winapi 0.3.9",
] ]
[[package]] [[package]]
@ -1089,6 +1027,15 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
[[package]]
name = "ntapi"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "num-integer" name = "num-integer"
version = "0.1.44" version = "0.1.44"
@ -1164,7 +1111,7 @@ dependencies = [
"libc", "libc",
"redox_syscall", "redox_syscall",
"smallvec", "smallvec",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1173,33 +1120,13 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pin-project"
version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15"
dependencies = [
"pin-project-internal 0.4.27",
]
[[package]] [[package]]
name = "pin-project" name = "pin-project"
version = "1.0.4" version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2"
dependencies = [ dependencies = [
"pin-project-internal 1.0.4", "pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
dependencies = [
"proc-macro2",
"quote",
"syn",
] ]
[[package]] [[package]]
@ -1213,12 +1140,6 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "pin-project-lite"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b"
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.4" version = "0.2.4"
@ -1317,7 +1238,7 @@ dependencies = [
"libc", "libc",
"rand_core 0.3.1", "rand_core 0.3.1",
"rdrand", "rdrand",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1424,7 +1345,7 @@ dependencies = [
"spin", "spin",
"untrusted", "untrusted",
"web-sys", "web-sys",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1459,11 +1380,11 @@ dependencies = [
[[package]] [[package]]
name = "rustls" name = "rustls"
version = "0.17.0" version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b"
dependencies = [ dependencies = [
"base64 0.11.0", "base64",
"log", "log",
"ring", "ring",
"sct", "sct",
@ -1607,7 +1528,7 @@ checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"libc", "libc",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1710,7 +1631,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
dependencies = [ dependencies = [
"libc", "libc",
"wasi 0.10.0+wasi-snapshot-preview1", "wasi 0.10.0+wasi-snapshot-preview1",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1730,32 +1651,28 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "0.2.24" version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" checksum = "8d56477f6ed99e10225f38f9f75f872f29b8b8bd8c0b946f63345bb144e9eeda"
dependencies = [ dependencies = [
"bytes 0.5.6", "autocfg",
"fnv", "bytes 1.0.1",
"futures-core",
"iovec",
"lazy_static",
"libc", "libc",
"memchr", "memchr",
"mio", "mio",
"mio-uds",
"num_cpus", "num_cpus",
"pin-project-lite 0.1.11", "once_cell",
"pin-project-lite",
"signal-hook-registry", "signal-hook-registry",
"slab",
"tokio-macros", "tokio-macros",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
name = "tokio-macros" name = "tokio-macros"
version = "0.2.6" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -1764,27 +1681,37 @@ dependencies = [
[[package]] [[package]]
name = "tokio-rustls" name = "tokio-rustls"
version = "0.13.1" version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
dependencies = [ dependencies = [
"futures-core",
"rustls", "rustls",
"tokio", "tokio",
"webpki", "webpki",
] ]
[[package]] [[package]]
name = "tokio-util" name = "tokio-stream"
version = "0.3.1" version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" checksum = "c535f53c0cfa1acace62995a8994fc9cc1f12d202420da96ff306ee24d576469"
dependencies = [ dependencies = [
"bytes 0.5.6", "futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec31e5cc6b46e653cf57762f36f71d5e6386391d88a72fd6db4508f8f676fb29"
dependencies = [
"bytes 1.0.1",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
"log", "log",
"pin-project-lite 0.1.11", "pin-project-lite",
"tokio", "tokio",
] ]
@ -1810,8 +1737,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"log", "pin-project-lite",
"pin-project-lite 0.2.4",
"tracing-core", "tracing-core",
] ]
@ -1824,16 +1750,6 @@ dependencies = [
"lazy_static", "lazy_static",
] ]
[[package]]
name = "tracing-futures"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c"
dependencies = [
"pin-project 0.4.27",
"tracing",
]
[[package]] [[package]]
name = "try-lock" name = "try-lock"
version = "0.2.3" version = "0.2.3"
@ -2002,12 +1918,6 @@ dependencies = [
"untrusted", "untrusted",
] ]
[[package]]
name = "winapi"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"
@ -2018,12 +1928,6 @@ dependencies = [
"winapi-x86_64-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu",
] ]
[[package]]
name = "winapi-build"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
[[package]] [[package]]
name = "winapi-i686-pc-windows-gnu" name = "winapi-i686-pc-windows-gnu"
version = "0.4.0" version = "0.4.0"
@ -2036,7 +1940,7 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [ dependencies = [
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -2045,16 +1949,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "ws2_32-sys"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
dependencies = [
"winapi 0.2.8",
"winapi-build",
]
[[package]] [[package]]
name = "xmlparser" name = "xmlparser"
version = "0.13.3" version = "0.13.3"

4
src/api/Cargo.toml
View file

@ -31,10 +31,10 @@ rand = "0.7"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
http = "0.2" http = "0.2"
hyper = "^0.13.6" hyper = "0.14"
url = "2.1" url = "2.1"
httpdate = "0.3" httpdate = "0.3"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"

2
src/garage/Cargo.toml
View file

@ -38,4 +38,4 @@ serde = { version = "1.0", default-features = false, features = ["derive", "rc"]
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }

6
src/garage/server.rs
View file

@ -21,13 +21,13 @@ async fn shutdown_signal(send_cancel: watch::Sender<bool>) -> Result<(), Error>
.await .await
.expect("failed to install CTRL+C signal handler"); .expect("failed to install CTRL+C signal handler");
info!("Received CTRL+C, shutting down."); info!("Received CTRL+C, shutting down.");
send_cancel.broadcast(true)?; send_cancel.send(true)?;
Ok(()) Ok(())
} }
async fn wait_from(mut chan: watch::Receiver<bool>) -> () { async fn wait_from(mut chan: watch::Receiver<bool>) -> () {
while let Some(exit_now) = chan.recv().await { while !*chan.borrow() {
if exit_now { if chan.changed().await.is_err() {
return; return;
} }
} }

3
src/model/Cargo.toml
View file

@ -33,5 +33,4 @@ serde_bytes = "0.11"
async-trait = "0.1.30" async-trait = "0.1.30"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }

23
src/model/block.rs
View file

@ -5,10 +5,9 @@ use std::time::Duration;
use arc_swap::ArcSwapOption; use arc_swap::ArcSwapOption;
use futures::future::*; use futures::future::*;
use futures::select; use futures::select;
use futures::stream::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::fs; use tokio::fs;
use tokio::prelude::*; use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::sync::{watch, Mutex, Notify}; use tokio::sync::{watch, Mutex, Notify};
use garage_util::data::*; use garage_util::data::*;
@ -134,7 +133,7 @@ impl BlockManager {
let bm2 = self.clone(); let bm2 = self.clone();
let background = self.system.background.clone(); let background = self.system.background.clone();
tokio::spawn(async move { tokio::spawn(async move {
tokio::time::delay_for(Duration::from_secs(10 * (i + 1))).await; tokio::time::sleep(Duration::from_secs(10 * (i + 1))).await;
background.spawn_worker(format!("block resync worker {}", i), move |must_exit| { background.spawn_worker(format!("block resync worker {}", i), move |must_exit| {
bm2.resync_loop(must_exit) bm2.resync_loop(must_exit)
}); });
@ -251,7 +250,7 @@ impl BlockManager {
let mut key = u64::to_be_bytes(when).to_vec(); let mut key = u64::to_be_bytes(when).to_vec();
key.extend(hash.as_ref()); key.extend(hash.as_ref());
self.resync_queue.insert(key, hash.as_ref())?; self.resync_queue.insert(key, hash.as_ref())?;
self.resync_notify.notify(); self.resync_notify.notify_waiters();
Ok(()) Ok(())
} }
@ -262,7 +261,7 @@ impl BlockManager {
while !*must_exit.borrow() { while !*must_exit.borrow() {
if let Err(e) = self.resync_iter(&mut must_exit).await { if let Err(e) = self.resync_iter(&mut must_exit).await {
warn!("Error in block resync loop: {}", e); warn!("Error in block resync loop: {}", e);
tokio::time::delay_for(Duration::from_secs(10)).await; tokio::time::sleep(Duration::from_secs(10)).await;
} }
} }
} }
@ -283,17 +282,17 @@ impl BlockManager {
self.resync_queue.remove(&time_bytes)?; self.resync_queue.remove(&time_bytes)?;
res?; // propagate error to delay main loop res?; // propagate error to delay main loop
} else { } else {
let delay = tokio::time::delay_for(Duration::from_millis(time_msec - now)); let delay = tokio::time::sleep(Duration::from_millis(time_msec - now));
select! { select! {
_ = delay.fuse() => (), _ = delay.fuse() => (),
_ = self.resync_notify.notified().fuse() => (), _ = self.resync_notify.notified().fuse() => (),
_ = must_exit.recv().fuse() => (), _ = must_exit.changed().fuse() => (),
} }
} }
} else { } else {
select! { select! {
_ = self.resync_notify.notified().fuse() => (), _ = self.resync_notify.notified().fuse() => (),
_ = must_exit.recv().fuse() => (), _ = must_exit.changed().fuse() => (),
} }
} }
Ok(()) Ok(())
@ -467,8 +466,12 @@ impl BlockManager {
// so that we can offload them if necessary and then delete them locally. // so that we can offload them if necessary and then delete them locally.
async move { async move {
let mut ls_data_dir = fs::read_dir(path).await?; let mut ls_data_dir = fs::read_dir(path).await?;
while let Some(data_dir_ent) = ls_data_dir.next().await { loop {
let data_dir_ent = data_dir_ent?; let data_dir_ent = ls_data_dir.next_entry().await?;
let data_dir_ent = match data_dir_ent {
Some(x) => x,
None => break,
};
let name = data_dir_ent.file_name(); let name = data_dir_ent.file_name();
let name = match name.into_string() { let name = match name.into_string() {
Ok(x) => x, Ok(x) => x,

3
src/model/object_table.rs
View file

@ -195,7 +195,8 @@ impl TableSchema for ObjectTable {
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) { fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
let version_table = self.version_table.clone(); let version_table = self.version_table.clone();
self.background.spawn(async move { // TODO not cancellable
self.background.spawn_cancellable(async move {
if let (Some(old_v), Some(new_v)) = (old, new) { if let (Some(old_v), Some(new_v)) = (old, new) {
// Propagate deletion of old versions // Propagate deletion of old versions
for v in old_v.versions.iter() { for v in old_v.versions.iter() {

3
src/model/version_table.rs
View file

@ -110,7 +110,8 @@ impl TableSchema for VersionTable {
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) { fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
let block_ref_table = self.block_ref_table.clone(); let block_ref_table = self.block_ref_table.clone();
self.background.spawn(async move { // TODO not cancellable
self.background.spawn_cancellable(async move {
if let (Some(old_v), Some(new_v)) = (old, new) { if let (Some(old_v), Some(new_v)) = (old, new) {
// Propagate deletion of version blocks // Propagate deletion of version blocks
if new_v.deleted.get() && !old_v.deleted.get() { if new_v.deleted.get() && !old_v.deleted.get() {

11
src/rpc/Cargo.toml
View file

@ -29,13 +29,14 @@ serde_json = "1.0"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
tokio-stream = {version = "0.1", features = ["net"] }
http = "0.2" http = "0.2"
hyper = "0.13" hyper = { version = "0.14", features = ["full"] }
rustls = "0.17" rustls = "0.19"
tokio-rustls = "0.13" tokio-rustls = "0.22"
hyper-rustls = { version = "0.20", default-features = false } hyper-rustls = { version = "0.22", default-features = false }
webpki = "0.21" webpki = "0.21"

30
src/rpc/membership.rs
View file

@ -11,9 +11,9 @@ use futures::future::join_all;
use futures::select; use futures::select;
use futures_util::future::*; use futures_util::future::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::prelude::*;
use tokio::sync::watch; use tokio::sync::watch;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use tokio::io::AsyncWriteExt;
use garage_util::background::BackgroundRunner; use garage_util::background::BackgroundRunner;
use garage_util::data::*; use garage_util::data::*;
@ -395,7 +395,7 @@ impl System {
if has_changes { if has_changes {
status.recalculate_hash(); status.recalculate_hash();
} }
if let Err(e) = update_locked.0.broadcast(Arc::new(status)) { if let Err(e) = update_locked.0.send(Arc::new(status)) {
error!("In ping_nodes: could not save status update ({})", e); error!("In ping_nodes: could not save status update ({})", e);
} }
drop(update_locked); drop(update_locked);
@ -421,7 +421,7 @@ impl System {
let status_hash = status.hash; let status_hash = status.hash;
let config_version = self.ring.borrow().config.version; let config_version = self.ring.borrow().config.version;
update_locked.0.broadcast(Arc::new(status))?; update_locked.0.send(Arc::new(status))?;
drop(update_locked); drop(update_locked);
if is_new || status_hash != ping.status_hash { if is_new || status_hash != ping.status_hash {
@ -503,7 +503,7 @@ impl System {
if has_changed { if has_changed {
status.recalculate_hash(); status.recalculate_hash();
} }
update_lock.0.broadcast(Arc::new(status))?; update_lock.0.send(Arc::new(status))?;
drop(update_lock); drop(update_lock);
if to_ping.len() > 0 { if to_ping.len() > 0 {
@ -523,7 +523,7 @@ impl System {
if adv.version > ring.config.version { if adv.version > ring.config.version {
let ring = Ring::new(adv.clone()); let ring = Ring::new(adv.clone());
update_lock.1.broadcast(Arc::new(ring))?; update_lock.1.send(Arc::new(ring))?;
drop(update_lock); drop(update_lock);
self.background.spawn_cancellable( self.background.spawn_cancellable(
@ -531,7 +531,7 @@ impl System {
.broadcast(Message::AdvertiseConfig(adv.clone()), PING_TIMEOUT) .broadcast(Message::AdvertiseConfig(adv.clone()), PING_TIMEOUT)
.map(Ok), .map(Ok),
); );
self.background.spawn(self.clone().save_network_config()); self.background.spawn(self.clone().save_network_config()).await;
} }
Ok(Message::Ok) Ok(Message::Ok)
@ -539,7 +539,7 @@ impl System {
async fn ping_loop(self: Arc<Self>, mut stop_signal: watch::Receiver<bool>) { async fn ping_loop(self: Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
loop { loop {
let restart_at = tokio::time::delay_for(PING_INTERVAL); let restart_at = tokio::time::sleep(PING_INTERVAL);
let status = self.status.borrow().clone(); let status = self.status.borrow().clone();
let ping_addrs = status let ping_addrs = status
@ -553,10 +553,9 @@ impl System {
select! { select! {
_ = restart_at.fuse() => (), _ = restart_at.fuse() => (),
must_exit = stop_signal.recv().fuse() => { _ = stop_signal.changed().fuse() => {
match must_exit { if *stop_signal.borrow() {
None | Some(true) => return, return;
_ => (),
} }
} }
} }
@ -570,7 +569,7 @@ impl System {
consul_service_name: String, consul_service_name: String,
) { ) {
loop { loop {
let restart_at = tokio::time::delay_for(CONSUL_INTERVAL); let restart_at = tokio::time::sleep(CONSUL_INTERVAL);
match get_consul_nodes(&consul_host, &consul_service_name).await { match get_consul_nodes(&consul_host, &consul_service_name).await {
Ok(mut node_list) => { Ok(mut node_list) => {
@ -584,10 +583,9 @@ impl System {
select! { select! {
_ = restart_at.fuse() => (), _ = restart_at.fuse() => (),
must_exit = stop_signal.recv().fuse() => { _ = stop_signal.changed().fuse() => {
match must_exit { if *stop_signal.borrow() {
None | Some(true) => return, return;
_ => (),
} }
} }
} }

2
src/rpc/rpc_client.rs
View file

@ -198,7 +198,7 @@ impl<M: RpcMessage + 'static> RpcClient<M> {
let wait_finished_fut = tokio::spawn(async move { let wait_finished_fut = tokio::spawn(async move {
resp_stream.collect::<Vec<_>>().await; resp_stream.collect::<Vec<_>>().await;
}); });
self.background.spawn(wait_finished_fut.map(|_| Ok(()))); self.background.spawn(wait_finished_fut.map(|_| Ok(()))).await;
} }
Ok(results) Ok(results)

5
src/rpc/rpc_server.rs
View file

@ -13,6 +13,7 @@ use hyper::service::{make_service_fn, service_fn};
use hyper::{Body, Method, Request, Response, Server, StatusCode}; use hyper::{Body, Method, Request, Response, Server, StatusCode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::net::{TcpListener, TcpStream}; use tokio::net::{TcpListener, TcpStream};
use tokio_stream::wrappers::TcpListenerStream;
use tokio_rustls::server::TlsStream; use tokio_rustls::server::TlsStream;
use tokio_rustls::TlsAcceptor; use tokio_rustls::TlsAcceptor;
@ -171,8 +172,8 @@ impl RpcServer {
config.set_single_cert([&node_certs[..], &ca_certs[..]].concat(), node_key)?; config.set_single_cert([&node_certs[..], &ca_certs[..]].concat(), node_key)?;
let tls_acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); let tls_acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
let mut listener = TcpListener::bind(&self.bind_addr).await?; let listener = TcpListener::bind(&self.bind_addr).await?;
let incoming = listener.incoming().filter_map(|socket| async { let incoming = TcpListenerStream::new(listener).filter_map(|socket| async {
match socket { match socket {
Ok(stream) => match tls_acceptor.clone().accept(stream).await { Ok(stream) => match tls_acceptor.clone().accept(stream).await {
Ok(x) => Some(Ok::<_, hyper::Error>(x)), Ok(x) => Some(Ok::<_, hyper::Error>(x)),

2
src/table/Cargo.toml
View file

@ -31,5 +31,5 @@ serde_bytes = "0.11"
async-trait = "0.1.30" async-trait = "0.1.30"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }

6
src/table/data.rs
View file

@ -159,7 +159,7 @@ where
if let Some((old_entry, new_entry, new_bytes_hash)) = changed { if let Some((old_entry, new_entry, new_bytes_hash)) = changed {
let is_tombstone = new_entry.is_tombstone(); let is_tombstone = new_entry.is_tombstone();
self.instance.updated(old_entry, Some(new_entry)); self.instance.updated(old_entry, Some(new_entry));
self.merkle_updater.todo_notify.notify(); self.merkle_updater.todo_notify.notify_one();
if is_tombstone { if is_tombstone {
self.gc_todo.insert(&tree_key, new_bytes_hash.as_slice())?; self.gc_todo.insert(&tree_key, new_bytes_hash.as_slice())?;
} }
@ -184,7 +184,7 @@ where
if removed { if removed {
let old_entry = self.decode_entry(v)?; let old_entry = self.decode_entry(v)?;
self.instance.updated(Some(old_entry), None); self.instance.updated(Some(old_entry), None);
self.merkle_updater.todo_notify.notify(); self.merkle_updater.todo_notify.notify_one();
} }
Ok(removed) Ok(removed)
} }
@ -209,7 +209,7 @@ where
if let Some(old_v) = removed { if let Some(old_v) = removed {
let old_entry = self.decode_entry(&old_v[..])?; let old_entry = self.decode_entry(&old_v[..])?;
self.instance.updated(Some(old_entry), None); self.instance.updated(Some(old_entry), None);
self.merkle_updater.todo_notify.notify(); self.merkle_updater.todo_notify.notify_one();
Ok(true) Ok(true)
} else { } else {
Ok(false) Ok(false)

4
src/table/gc.rs
View file

@ -85,8 +85,8 @@ where
} }
} }
select! { select! {
_ = tokio::time::delay_for(Duration::from_secs(10)).fuse() => (), _ = tokio::time::sleep(Duration::from_secs(10)).fuse() => (),
_ = must_exit.recv().fuse() => (), _ = must_exit.changed().fuse() => (),
} }
} }
} }

4
src/table/merkle.rs
View file

@ -121,13 +121,13 @@ impl MerkleUpdater {
"({}) Error while iterating on Merkle todo tree: {}", "({}) Error while iterating on Merkle todo tree: {}",
self.table_name, e self.table_name, e
); );
tokio::time::delay_for(Duration::from_secs(10)).await; tokio::time::sleep(Duration::from_secs(10)).await;
} }
} }
} else { } else {
select! { select! {
_ = self.todo_notify.notified().fuse() => (), _ = self.todo_notify.notified().fuse() => (),
_ = must_exit.recv().fuse() => (), _ = must_exit.changed().fuse() => (),
} }
} }
} }

35
src/table/sync.rs
View file

@ -3,7 +3,7 @@ use std::convert::TryInto;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use futures::{pin_mut, select}; use futures::{select};
use futures_util::future::*; use futures_util::future::*;
use futures_util::stream::*; use futures_util::stream::*;
use rand::Rng; use rand::Rng;
@ -110,7 +110,7 @@ where
let s3 = syncer.clone(); let s3 = syncer.clone();
tokio::spawn(async move { tokio::spawn(async move {
tokio::time::delay_for(Duration::from_secs(20)).await; tokio::time::sleep(Duration::from_secs(20)).await;
s3.add_full_sync(); s3.add_full_sync();
}); });
@ -142,23 +142,16 @@ where
let mut nothing_to_do_since = Some(Instant::now()); let mut nothing_to_do_since = Some(Instant::now());
while !*must_exit.borrow() { while !*must_exit.borrow() {
let s_ring_recv = ring_recv.recv().fuse();
let s_busy = busy_rx.recv().fuse();
let s_must_exit = must_exit.recv().fuse();
let s_timeout = tokio::time::delay_for(Duration::from_secs(1)).fuse();
pin_mut!(s_ring_recv, s_busy, s_must_exit, s_timeout);
select! { select! {
new_ring_r = s_ring_recv => { _ = ring_recv.changed().fuse() => {
if let Some(new_ring) = new_ring_r { let new_ring = ring_recv.borrow();
if !Arc::ptr_eq(&new_ring, &prev_ring) { if !Arc::ptr_eq(&new_ring, &prev_ring) {
debug!("({}) Ring changed, adding full sync to syncer todo list", self.data.name); debug!("({}) Ring changed, adding full sync to syncer todo list", self.data.name);
self.add_full_sync(); self.add_full_sync();
prev_ring = new_ring; prev_ring = new_ring.clone();
}
} }
} }
busy_opt = s_busy => { busy_opt = busy_rx.recv().fuse() => {
if let Some(busy) = busy_opt { if let Some(busy) = busy_opt {
if busy { if busy {
nothing_to_do_since = None; nothing_to_do_since = None;
@ -169,12 +162,8 @@ where
} }
} }
} }
must_exit_v = s_must_exit => { _ = must_exit.changed().fuse() => (),
if must_exit_v.unwrap_or(false) { _ = tokio::time::sleep(Duration::from_secs(1)).fuse() => {
break;
}
}
_ = s_timeout => {
if nothing_to_do_since.map(|t| Instant::now() - t >= ANTI_ENTROPY_INTERVAL).unwrap_or(false) { if nothing_to_do_since.map(|t| Instant::now() - t >= ANTI_ENTROPY_INTERVAL).unwrap_or(false) {
nothing_to_do_since = None; nothing_to_do_since = None;
debug!("({}) Interval passed, adding full sync to syncer todo list", self.data.name); debug!("({}) Interval passed, adding full sync to syncer todo list", self.data.name);
@ -213,7 +202,7 @@ where
} }
} else { } else {
busy_tx.send(false).unwrap(); busy_tx.send(false).unwrap();
tokio::time::delay_for(Duration::from_secs(1)).await; tokio::time::sleep(Duration::from_secs(1)).await;
} }
} }
} }

7
src/util/Cargo.toml
View file

@ -28,14 +28,15 @@ rmp-serde = "0.14.3"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_json = "1.0" serde_json = "1.0"
chrono = "0.4" chrono = "0.4"
arc-swap = "1.2"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
http = "0.2" http = "0.2"
hyper = "0.13" hyper = "0.14"
rustls = "0.17" rustls = "0.19"
webpki = "0.21" webpki = "0.21"
roxmltree = "0.11" roxmltree = "0.11"

131
src/util/background.rs
View file

@ -2,11 +2,9 @@ use core::future::Future;
use std::pin::Pin; use std::pin::Pin;
use std::sync::Mutex; use std::sync::Mutex;
use futures::future::join_all; use arc_swap::ArcSwapOption;
use futures::select;
use futures_util::future::*;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::{mpsc, watch, Notify}; use tokio::sync::{mpsc, watch};
use crate::error::Error; use crate::error::Error;
@ -14,12 +12,9 @@ type JobOutput = Result<(), Error>;
type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>; type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>;
pub struct BackgroundRunner { pub struct BackgroundRunner {
n_runners: usize,
pub stop_signal: watch::Receiver<bool>, pub stop_signal: watch::Receiver<bool>,
queue_in: mpsc::UnboundedSender<(Job, bool)>, queue_in: ArcSwapOption<mpsc::UnboundedSender<(Job, bool)>>,
queue_out: Mutex<mpsc::UnboundedReceiver<(Job, bool)>>,
job_notify: Notify,
workers: Mutex<Vec<tokio::task::JoinHandle<()>>>, workers: Mutex<Vec<tokio::task::JoinHandle<()>>>,
} }
@ -27,50 +22,91 @@ pub struct BackgroundRunner {
impl BackgroundRunner { impl BackgroundRunner {
pub fn new(n_runners: usize, stop_signal: watch::Receiver<bool>) -> Arc<Self> { pub fn new(n_runners: usize, stop_signal: watch::Receiver<bool>) -> Arc<Self> {
let (queue_in, queue_out) = mpsc::unbounded_channel(); let (queue_in, queue_out) = mpsc::unbounded_channel();
let mut workers = vec![];
let queue_out = Arc::new(tokio::sync::Mutex::new(queue_out));
for i in 0..n_runners {
let queue_out = queue_out.clone();
let stop_signal = stop_signal.clone();
workers.push(tokio::spawn(async move {
while let Some((job, cancellable)) = queue_out.lock().await.recv().await {
if cancellable && *stop_signal.borrow() {
continue;
}
if let Err(e) = job.await {
error!("Job failed: {}", e)
}
}
info!("Worker {} exiting", i);
}));
}
Arc::new(Self { Arc::new(Self {
n_runners,
stop_signal, stop_signal,
queue_in, queue_in: ArcSwapOption::new(Some(Arc::new(queue_in))),
queue_out: Mutex::new(queue_out), workers: Mutex::new(workers),
job_notify: Notify::new(),
workers: Mutex::new(Vec::new()),
}) })
} }
pub async fn run(self: Arc<Self>) { pub async fn run(self: Arc<Self>) {
let mut workers = self.workers.lock().unwrap();
for i in 0..self.n_runners {
workers.push(tokio::spawn(self.clone().runner(i)));
}
drop(workers);
let mut stop_signal = self.stop_signal.clone(); let mut stop_signal = self.stop_signal.clone();
while let Some(exit_now) = stop_signal.recv().await {
loop {
let exit_now = match stop_signal.changed().await {
Ok(()) => *stop_signal.borrow(),
Err(e) => {
error!("Watch .changed() error: {}", e);
true
}
};
if exit_now { if exit_now {
let mut workers = self.workers.lock().unwrap(); break;
let workers_vec = workers.drain(..).collect::<Vec<_>>(); }
join_all(workers_vec).await; }
return;
info!("Closing background job queue_in...");
drop(self.queue_in.swap(None));
info!("Waiting for all workers to terminate...");
while let Some(task) = self.workers.lock().unwrap().pop() {
if let Err(e) = task.await {
warn!("Error awaiting task: {}", e);
} }
} }
} }
pub fn spawn<T>(&self, job: T) // Spawn a task to be run in background
pub async fn spawn<T>(&self, job: T)
where where
T: Future<Output = JobOutput> + Send + 'static, T: Future<Output = JobOutput> + Send + 'static,
{ {
let boxed: Job = Box::pin(job); match self.queue_in.load().as_ref() {
let _: Result<_, _> = self.queue_in.clone().send((boxed, false)); Some(chan) => {
self.job_notify.notify(); let boxed: Job = Box::pin(job);
chan.send((boxed, false)).map_err(|_| "send error").unwrap();
}
None => {
warn!("Doing background job now because we are exiting...");
if let Err(e) = job.await {
warn!("Task failed: {}", e);
}
}
}
} }
pub fn spawn_cancellable<T>(&self, job: T) pub fn spawn_cancellable<T>(&self, job: T)
where where
T: Future<Output = JobOutput> + Send + 'static, T: Future<Output = JobOutput> + Send + 'static,
{ {
let boxed: Job = Box::pin(job); match self.queue_in.load().as_ref() {
let _: Result<_, _> = self.queue_in.clone().send((boxed, true)); Some(chan) => {
self.job_notify.notify(); let boxed: Job = Box::pin(job);
chan.send((boxed, false)).map_err(|_| "send error").unwrap();
}
None => (), // drop job if we are exiting
}
} }
pub fn spawn_worker<F, T>(&self, name: String, worker: F) pub fn spawn_worker<F, T>(&self, name: String, worker: F)
@ -85,37 +121,4 @@ impl BackgroundRunner {
info!("Worker exited: {}", name); info!("Worker exited: {}", name);
})); }));
} }
async fn runner(self: Arc<Self>, i: usize) {
let mut stop_signal = self.stop_signal.clone();
loop {
let must_exit: bool = *stop_signal.borrow();
if let Some(job) = self.dequeue_job(must_exit) {
if let Err(e) = job.await {
error!("Job failed: {}", e)
}
} else {
if must_exit {
info!("Background runner {} exiting", i);
return;
}
select! {
_ = self.job_notify.notified().fuse() => (),
_ = stop_signal.recv().fuse() => (),
}
}
}
}
fn dequeue_job(&self, must_exit: bool) -> Option<Job> {
let mut queue = self.queue_out.lock().unwrap();
while let Ok((job, cancellable)) = queue.try_recv() {
if cancellable && must_exit {
continue;
} else {
return Some(job);
}
}
None
}
} }

8
src/util/error.rs
View file

@ -8,16 +8,22 @@ use crate::data::*;
pub enum RPCError { pub enum RPCError {
#[error(display = "Node is down: {:?}.", _0)] #[error(display = "Node is down: {:?}.", _0)]
NodeDown(UUID), NodeDown(UUID),
#[error(display = "Timeout: {}", _0)] #[error(display = "Timeout: {}", _0)]
Timeout(#[error(source)] tokio::time::Elapsed), Timeout(#[error(source)] tokio::time::error::Elapsed),
#[error(display = "HTTP error: {}", _0)] #[error(display = "HTTP error: {}", _0)]
HTTP(#[error(source)] http::Error), HTTP(#[error(source)] http::Error),
#[error(display = "Hyper error: {}", _0)] #[error(display = "Hyper error: {}", _0)]
Hyper(#[error(source)] hyper::Error), Hyper(#[error(source)] hyper::Error),
#[error(display = "Messagepack encode error: {}", _0)] #[error(display = "Messagepack encode error: {}", _0)]
RMPEncode(#[error(source)] rmp_serde::encode::Error), RMPEncode(#[error(source)] rmp_serde::encode::Error),
#[error(display = "Messagepack decode error: {}", _0)] #[error(display = "Messagepack decode error: {}", _0)]
RMPDecode(#[error(source)] rmp_serde::decode::Error), RMPDecode(#[error(source)] rmp_serde::decode::Error),
#[error(display = "Too many errors: {:?}", _0)] #[error(display = "Too many errors: {:?}", _0)]
TooManyErrors(Vec<String>), TooManyErrors(Vec<String>),
} }

2
src/web/Cargo.toml
View file

@ -22,7 +22,7 @@ err-derive = "0.2.3"
log = "0.4" log = "0.4"
futures = "0.3" futures = "0.3"
http = "0.2" http = "0.2"
hyper = "0.13" hyper = "0.14"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"
roxmltree = "0.11" roxmltree = "0.11"
idna = "0.2" idna = "0.2"