Deuxfleurs/garage
44
56
Fork 74
Code Issues 100 Pull requests 14 Releases 30 Wiki Activity

WIP add content defined chunking #42

Closed
trinity-1686a wants to merge 42 commits from content-defined-chunking into master
pull from: content-defined-chunking
merge into: Deuxfleurs:master
Conversation 3 Commits 42 Files changed 60 +233 -336
21 changed files with 233 additions and 336 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 0cd5b2ae19 - Show all commits

278
Cargo.lock generated
View file

@ -17,6 +17,12 @@ version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8" checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8"
[[package]]
name = "arc-swap"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73"
[[package]] [[package]]
name = "arrayvec" name = "arrayvec"
version = "0.3.25" version = "0.3.25"
@ -46,7 +52,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [ dependencies = [
"hermit-abi", "hermit-abi",
"libc", "libc",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -55,12 +61,6 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "base64"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.13.0" version = "0.13.0"
@ -142,12 +142,6 @@ dependencies = [
"iovec", "iovec",
] ]
[[package]]
name = "bytes"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.0.1" version = "1.0.1"
@ -182,7 +176,7 @@ dependencies = [
"num-integer", "num-integer",
"num-traits", "num-traits",
"time", "time",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -347,7 +341,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [ dependencies = [
"libc", "libc",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -356,22 +350,6 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
dependencies = [
"bitflags",
"fuchsia-zircon-sys",
]
[[package]]
name = "fuchsia-zircon-sys"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
[[package]] [[package]]
name = "futures" name = "futures"
version = "0.3.12" version = "0.3.12"
@ -460,7 +438,7 @@ dependencies = [
"futures-sink", "futures-sink",
"futures-task", "futures-task",
"memchr", "memchr",
"pin-project-lite 0.2.4", "pin-project-lite",
"pin-utils", "pin-utils",
"proc-macro-hack", "proc-macro-hack",
"proc-macro-nested", "proc-macro-nested",
@ -507,7 +485,7 @@ dependencies = [
name = "garage_api" name = "garage_api"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"base64 0.13.0", "base64",
"bytes 0.4.12", "bytes 0.4.12",
"chrono", "chrono",
"crypto-mac 0.7.0", "crypto-mac 0.7.0",
@ -537,7 +515,7 @@ dependencies = [
name = "garage_model" name = "garage_model"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap 0.4.8",
"async-trait", "async-trait",
"bytes 0.4.12", "bytes 0.4.12",
"futures", "futures",
@ -560,7 +538,7 @@ dependencies = [
name = "garage_rpc" name = "garage_rpc"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"arc-swap", "arc-swap 0.4.8",
"bytes 0.4.12", "bytes 0.4.12",
"futures", "futures",
"futures-util", "futures-util",
@ -579,6 +557,7 @@ dependencies = [
"sha2", "sha2",
"tokio", "tokio",
"tokio-rustls", "tokio-rustls",
"tokio-stream",
"webpki", "webpki",
] ]
@ -607,6 +586,7 @@ dependencies = [
name = "garage_util" name = "garage_util"
version = "0.1.1" version = "0.1.1"
dependencies = [ dependencies = [
"arc-swap 1.2.0",
"blake2", "blake2",
"chrono", "chrono",
"err-derive", "err-derive",
@ -664,7 +644,7 @@ dependencies = [
"libc", "libc",
"log", "log",
"rustversion", "rustversion",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -693,7 +673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028" checksum = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028"
dependencies = [ dependencies = [
"libc", "libc",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -731,11 +711,11 @@ dependencies = [
[[package]] [[package]]
name = "h2" name = "h2"
version = "0.2.7" version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78"
dependencies = [ dependencies = [
"bytes 0.5.6", "bytes 1.0.1",
"fnv", "fnv",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -746,7 +726,6 @@ dependencies = [
"tokio", "tokio",
"tokio-util", "tokio-util",
"tracing", "tracing",
"tracing-futures",
] ]
[[package]] [[package]]
@ -812,11 +791,11 @@ dependencies = [
[[package]] [[package]]
name = "http-body" name = "http-body"
version = "0.3.1" version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994"
dependencies = [ dependencies = [
"bytes 0.5.6", "bytes 1.0.1",
"http", "http",
] ]
@ -849,11 +828,11 @@ dependencies = [
[[package]] [[package]]
name = "hyper" name = "hyper"
version = "0.13.9" version = "0.14.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7"
dependencies = [ dependencies = [
"bytes 0.5.6", "bytes 1.0.1",
"futures-channel", "futures-channel",
"futures-core", "futures-core",
"futures-util", "futures-util",
@ -863,7 +842,7 @@ dependencies = [
"httparse", "httparse",
"httpdate", "httpdate",
"itoa", "itoa",
"pin-project 1.0.4", "pin-project",
"socket2", "socket2",
"tokio", "tokio",
"tower-service", "tower-service",
@ -873,11 +852,10 @@ dependencies = [
[[package]] [[package]]
name = "hyper-rustls" name = "hyper-rustls"
version = "0.20.0" version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac965ea399ec3a25ac7d13b8affd4b8f39325cca00858ddf5eb29b79e6b14b08" checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
dependencies = [ dependencies = [
"bytes 0.5.6",
"futures-util", "futures-util",
"hyper", "hyper",
"log", "log",
@ -947,16 +925,6 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "kernel32-sys"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
dependencies = [
"winapi 0.2.8",
"winapi-build",
]
[[package]] [[package]]
name = "lazy_static" name = "lazy_static"
version = "1.4.0" version = "1.4.0"
@ -965,9 +933,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]] [[package]]
name = "libc" name = "libc"
version = "0.2.82" version = "0.2.89"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" checksum = "538c092e5586f4cdd7dd8078c4a79220e3e168880218124dcbce860f0ea938c6"
[[package]] [[package]]
name = "lock_api" name = "lock_api"
@ -1032,55 +1000,25 @@ dependencies = [
[[package]] [[package]]
name = "mio" name = "mio"
version = "0.6.23" version = "0.7.10"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" checksum = "2182a122f3b7f3f5329cb1972cee089ba2459a0a80a56935e6e674f096f8d839"
dependencies = [ dependencies = [
"cfg-if 0.1.10",
"fuchsia-zircon",
"fuchsia-zircon-sys",
"iovec",
"kernel32-sys",
"libc", "libc",
"log", "log",
"miow", "miow",
"net2", "ntapi",
"slab", "winapi",
"winapi 0.2.8",
]
[[package]]
name = "mio-uds"
version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0"
dependencies = [
"iovec",
"libc",
"mio",
] ]
[[package]] [[package]]
name = "miow" name = "miow"
version = "0.2.2" version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897"
dependencies = [ dependencies = [
"kernel32-sys", "socket2",
"net2", "winapi",
"winapi 0.2.8",
"ws2_32-sys",
]
[[package]]
name = "net2"
version = "0.2.37"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae"
dependencies = [
"cfg-if 0.1.10",
"libc",
"winapi 0.3.9",
] ]
[[package]] [[package]]
@ -1089,6 +1027,15 @@ version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
[[package]]
name = "ntapi"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
dependencies = [
"winapi",
]
[[package]] [[package]]
name = "num-integer" name = "num-integer"
version = "0.1.44" version = "0.1.44"
@ -1164,7 +1111,7 @@ dependencies = [
"libc", "libc",
"redox_syscall", "redox_syscall",
"smallvec", "smallvec",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1173,33 +1120,13 @@ version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
name = "pin-project"
version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15"
dependencies = [
"pin-project-internal 0.4.27",
]
[[package]] [[package]]
name = "pin-project" name = "pin-project"
version = "1.0.4" version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2"
dependencies = [ dependencies = [
"pin-project-internal 1.0.4", "pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "0.4.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
dependencies = [
"proc-macro2",
"quote",
"syn",
] ]
[[package]] [[package]]
@ -1213,12 +1140,6 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "pin-project-lite"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b"
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.4" version = "0.2.4"
@ -1317,7 +1238,7 @@ dependencies = [
"libc", "libc",
"rand_core 0.3.1", "rand_core 0.3.1",
"rdrand", "rdrand",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1424,7 +1345,7 @@ dependencies = [
"spin", "spin",
"untrusted", "untrusted",
"web-sys", "web-sys",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1459,11 +1380,11 @@ dependencies = [
[[package]] [[package]]
name = "rustls" name = "rustls"
version = "0.17.0" version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b"
dependencies = [ dependencies = [
"base64 0.11.0", "base64",
"log", "log",
"ring", "ring",
"sct", "sct",
@ -1607,7 +1528,7 @@ checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"libc", "libc",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1710,7 +1631,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
dependencies = [ dependencies = [
"libc", "libc",
"wasi 0.10.0+wasi-snapshot-preview1", "wasi 0.10.0+wasi-snapshot-preview1",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -1730,32 +1651,28 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "0.2.24" version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48" checksum = "8d56477f6ed99e10225f38f9f75f872f29b8b8bd8c0b946f63345bb144e9eeda"
dependencies = [ dependencies = [
"bytes 0.5.6", "autocfg",
"fnv", "bytes 1.0.1",
"futures-core",
"iovec",
"lazy_static",
"libc", "libc",
"memchr", "memchr",
"mio", "mio",
"mio-uds",
"num_cpus", "num_cpus",
"pin-project-lite 0.1.11", "once_cell",
"pin-project-lite",
"signal-hook-registry", "signal-hook-registry",
"slab",
"tokio-macros", "tokio-macros",
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
name = "tokio-macros" name = "tokio-macros"
version = "0.2.6" version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a" checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -1764,27 +1681,37 @@ dependencies = [
[[package]] [[package]]
name = "tokio-rustls" name = "tokio-rustls"
version = "0.13.1" version = "0.22.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4" checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
dependencies = [ dependencies = [
"futures-core",
"rustls", "rustls",
"tokio", "tokio",
"webpki", "webpki",
] ]
[[package]] [[package]]
name = "tokio-util" name = "tokio-stream"
version = "0.3.1" version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" checksum = "c535f53c0cfa1acace62995a8994fc9cc1f12d202420da96ff306ee24d576469"
dependencies = [ dependencies = [
"bytes 0.5.6", "futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec31e5cc6b46e653cf57762f36f71d5e6386391d88a72fd6db4508f8f676fb29"
dependencies = [
"bytes 1.0.1",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
"log", "log",
"pin-project-lite 0.1.11", "pin-project-lite",
"tokio", "tokio",
] ]
@ -1810,8 +1737,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
dependencies = [ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
"log", "pin-project-lite",
"pin-project-lite 0.2.4",
"tracing-core", "tracing-core",
] ]
@ -1824,16 +1750,6 @@ dependencies = [
"lazy_static", "lazy_static",
] ]
[[package]]
name = "tracing-futures"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c"
dependencies = [
"pin-project 0.4.27",
"tracing",
]
[[package]] [[package]]
name = "try-lock" name = "try-lock"
version = "0.2.3" version = "0.2.3"
@ -2002,12 +1918,6 @@ dependencies = [
"untrusted", "untrusted",
] ]
[[package]]
name = "winapi"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
[[package]] [[package]]
name = "winapi" name = "winapi"
version = "0.3.9" version = "0.3.9"
@ -2018,12 +1928,6 @@ dependencies = [
"winapi-x86_64-pc-windows-gnu", "winapi-x86_64-pc-windows-gnu",
] ]
[[package]]
name = "winapi-build"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
[[package]] [[package]]
name = "winapi-i686-pc-windows-gnu" name = "winapi-i686-pc-windows-gnu"
version = "0.4.0" version = "0.4.0"
@ -2036,7 +1940,7 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
dependencies = [ dependencies = [
"winapi 0.3.9", "winapi",
] ]
[[package]] [[package]]
@ -2045,16 +1949,6 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "ws2_32-sys"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
dependencies = [
"winapi 0.2.8",
"winapi-build",
]
[[package]] [[package]]
name = "xmlparser" name = "xmlparser"
version = "0.13.3" version = "0.13.3"

4
src/api/Cargo.toml
View file

@ -31,10 +31,10 @@ rand = "0.7"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
http = "0.2" http = "0.2"
hyper = "^0.13.6" hyper = "0.14"
url = "2.1" url = "2.1"
httpdate = "0.3" httpdate = "0.3"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"

2
src/garage/Cargo.toml
View file

@ -38,4 +38,4 @@ serde = { version = "1.0", default-features = false, features = ["derive", "rc"]
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }

6
src/garage/server.rs
View file

@ -21,13 +21,13 @@ async fn shutdown_signal(send_cancel: watch::Sender<bool>) -> Result<(), Error>
.await .await
.expect("failed to install CTRL+C signal handler"); .expect("failed to install CTRL+C signal handler");
info!("Received CTRL+C, shutting down."); info!("Received CTRL+C, shutting down.");
send_cancel.broadcast(true)?; send_cancel.send(true)?;
Ok(()) Ok(())
} }
async fn wait_from(mut chan: watch::Receiver<bool>) -> () { async fn wait_from(mut chan: watch::Receiver<bool>) -> () {
while let Some(exit_now) = chan.recv().await { while !*chan.borrow() {
if exit_now { if chan.changed().await.is_err() {
return; return;
} }
} }

3
src/model/Cargo.toml
View file

@ -33,5 +33,4 @@ serde_bytes = "0.11"
async-trait = "0.1.30" async-trait = "0.1.30"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }

23
src/model/block.rs
View file

@ -5,10 +5,9 @@ use std::time::Duration;
use arc_swap::ArcSwapOption; use arc_swap::ArcSwapOption;
use futures::future::*; use futures::future::*;
use futures::select; use futures::select;
use futures::stream::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::fs; use tokio::fs;
use tokio::prelude::*; use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::sync::{watch, Mutex, Notify}; use tokio::sync::{watch, Mutex, Notify};
use garage_util::data::*; use garage_util::data::*;
@ -134,7 +133,7 @@ impl BlockManager {
let bm2 = self.clone(); let bm2 = self.clone();
let background = self.system.background.clone(); let background = self.system.background.clone();
tokio::spawn(async move { tokio::spawn(async move {
tokio::time::delay_for(Duration::from_secs(10 * (i + 1))).await; tokio::time::sleep(Duration::from_secs(10 * (i + 1))).await;
background.spawn_worker(format!("block resync worker {}", i), move |must_exit| { background.spawn_worker(format!("block resync worker {}", i), move |must_exit| {
bm2.resync_loop(must_exit) bm2.resync_loop(must_exit)
}); });
@ -251,7 +250,7 @@ impl BlockManager {
let mut key = u64::to_be_bytes(when).to_vec(); let mut key = u64::to_be_bytes(when).to_vec();
key.extend(hash.as_ref()); key.extend(hash.as_ref());
self.resync_queue.insert(key, hash.as_ref())?; self.resync_queue.insert(key, hash.as_ref())?;
self.resync_notify.notify(); self.resync_notify.notify_waiters();
Ok(()) Ok(())
} }
@ -262,7 +261,7 @@ impl BlockManager {
while !*must_exit.borrow() { while !*must_exit.borrow() {
if let Err(e) = self.resync_iter(&mut must_exit).await { if let Err(e) = self.resync_iter(&mut must_exit).await {
warn!("Error in block resync loop: {}", e); warn!("Error in block resync loop: {}", e);
tokio::time::delay_for(Duration::from_secs(10)).await; tokio::time::sleep(Duration::from_secs(10)).await;
} }
} }
} }
@ -283,17 +282,17 @@ impl BlockManager {
self.resync_queue.remove(&time_bytes)?; self.resync_queue.remove(&time_bytes)?;
res?; // propagate error to delay main loop res?; // propagate error to delay main loop
} else { } else {
let delay = tokio::time::delay_for(Duration::from_millis(time_msec - now)); let delay = tokio::time::sleep(Duration::from_millis(time_msec - now));
select! { select! {
_ = delay.fuse() => (), _ = delay.fuse() => (),
_ = self.resync_notify.notified().fuse() => (), _ = self.resync_notify.notified().fuse() => (),
_ = must_exit.recv().fuse() => (), _ = must_exit.changed().fuse() => (),
} }
} }
} else { } else {
select! { select! {
_ = self.resync_notify.notified().fuse() => (), _ = self.resync_notify.notified().fuse() => (),
_ = must_exit.recv().fuse() => (), _ = must_exit.changed().fuse() => (),
} }
} }
Ok(()) Ok(())
@ -467,8 +466,12 @@ impl BlockManager {
// so that we can offload them if necessary and then delete them locally. // so that we can offload them if necessary and then delete them locally.
async move { async move {
let mut ls_data_dir = fs::read_dir(path).await?; let mut ls_data_dir = fs::read_dir(path).await?;
while let Some(data_dir_ent) = ls_data_dir.next().await { loop {
let data_dir_ent = data_dir_ent?; let data_dir_ent = ls_data_dir.next_entry().await?;
let data_dir_ent = match data_dir_ent {
Some(x) => x,
None => break,
};
let name = data_dir_ent.file_name(); let name = data_dir_ent.file_name();
let name = match name.into_string() { let name = match name.into_string() {
Ok(x) => x, Ok(x) => x,

3
src/model/object_table.rs
View file

@ -195,7 +195,8 @@ impl TableSchema for ObjectTable {
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) { fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
let version_table = self.version_table.clone(); let version_table = self.version_table.clone();
self.background.spawn(async move { // TODO not cancellable
self.background.spawn_cancellable(async move {
if let (Some(old_v), Some(new_v)) = (old, new) { if let (Some(old_v), Some(new_v)) = (old, new) {
// Propagate deletion of old versions // Propagate deletion of old versions
for v in old_v.versions.iter() { for v in old_v.versions.iter() {

3
src/model/version_table.rs
View file

@ -110,7 +110,8 @@ impl TableSchema for VersionTable {
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) { fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
let block_ref_table = self.block_ref_table.clone(); let block_ref_table = self.block_ref_table.clone();
self.background.spawn(async move { // TODO not cancellable
self.background.spawn_cancellable(async move {
if let (Some(old_v), Some(new_v)) = (old, new) { if let (Some(old_v), Some(new_v)) = (old, new) {
// Propagate deletion of version blocks // Propagate deletion of version blocks
if new_v.deleted.get() && !old_v.deleted.get() { if new_v.deleted.get() && !old_v.deleted.get() {

11
src/rpc/Cargo.toml
View file

@ -29,13 +29,14 @@ serde_json = "1.0"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
tokio-stream = {version = "0.1", features = ["net"] }
http = "0.2" http = "0.2"
hyper = "0.13" hyper = { version = "0.14", features = ["full"] }
rustls = "0.17" rustls = "0.19"
tokio-rustls = "0.13" tokio-rustls = "0.22"
hyper-rustls = { version = "0.20", default-features = false } hyper-rustls = { version = "0.22", default-features = false }
webpki = "0.21" webpki = "0.21"

30
src/rpc/membership.rs
View file

@ -11,9 +11,9 @@ use futures::future::join_all;
use futures::select; use futures::select;
use futures_util::future::*; use futures_util::future::*;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::prelude::*;
use tokio::sync::watch; use tokio::sync::watch;
use tokio::sync::Mutex; use tokio::sync::Mutex;
use tokio::io::AsyncWriteExt;
use garage_util::background::BackgroundRunner; use garage_util::background::BackgroundRunner;
use garage_util::data::*; use garage_util::data::*;
@ -395,7 +395,7 @@ impl System {
if has_changes { if has_changes {
status.recalculate_hash(); status.recalculate_hash();
} }
if let Err(e) = update_locked.0.broadcast(Arc::new(status)) { if let Err(e) = update_locked.0.send(Arc::new(status)) {
error!("In ping_nodes: could not save status update ({})", e); error!("In ping_nodes: could not save status update ({})", e);
} }
drop(update_locked); drop(update_locked);
@ -421,7 +421,7 @@ impl System {
let status_hash = status.hash; let status_hash = status.hash;
let config_version = self.ring.borrow().config.version; let config_version = self.ring.borrow().config.version;
update_locked.0.broadcast(Arc::new(status))?; update_locked.0.send(Arc::new(status))?;
drop(update_locked); drop(update_locked);
if is_new || status_hash != ping.status_hash { if is_new || status_hash != ping.status_hash {
@ -503,7 +503,7 @@ impl System {
if has_changed { if has_changed {
status.recalculate_hash(); status.recalculate_hash();
} }
update_lock.0.broadcast(Arc::new(status))?; update_lock.0.send(Arc::new(status))?;
drop(update_lock); drop(update_lock);
if to_ping.len() > 0 { if to_ping.len() > 0 {
@ -523,7 +523,7 @@ impl System {
if adv.version > ring.config.version { if adv.version > ring.config.version {
let ring = Ring::new(adv.clone()); let ring = Ring::new(adv.clone());
update_lock.1.broadcast(Arc::new(ring))?; update_lock.1.send(Arc::new(ring))?;
drop(update_lock); drop(update_lock);
self.background.spawn_cancellable( self.background.spawn_cancellable(
@ -531,7 +531,7 @@ impl System {
.broadcast(Message::AdvertiseConfig(adv.clone()), PING_TIMEOUT) .broadcast(Message::AdvertiseConfig(adv.clone()), PING_TIMEOUT)
.map(Ok), .map(Ok),
); );
self.background.spawn(self.clone().save_network_config()); self.background.spawn(self.clone().save_network_config()).await;
} }
Ok(Message::Ok) Ok(Message::Ok)
@ -539,7 +539,7 @@ impl System {
async fn ping_loop(self: Arc<Self>, mut stop_signal: watch::Receiver<bool>) { async fn ping_loop(self: Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
loop { loop {
let restart_at = tokio::time::delay_for(PING_INTERVAL); let restart_at = tokio::time::sleep(PING_INTERVAL);
let status = self.status.borrow().clone(); let status = self.status.borrow().clone();
let ping_addrs = status let ping_addrs = status
@ -553,10 +553,9 @@ impl System {
select! { select! {
_ = restart_at.fuse() => (), _ = restart_at.fuse() => (),
must_exit = stop_signal.recv().fuse() => { _ = stop_signal.changed().fuse() => {
match must_exit { if *stop_signal.borrow() {
None | Some(true) => return, return;
_ => (),
} }
} }
} }
@ -570,7 +569,7 @@ impl System {
consul_service_name: String, consul_service_name: String,
) { ) {
loop { loop {
let restart_at = tokio::time::delay_for(CONSUL_INTERVAL); let restart_at = tokio::time::sleep(CONSUL_INTERVAL);
match get_consul_nodes(&consul_host, &consul_service_name).await { match get_consul_nodes(&consul_host, &consul_service_name).await {
Ok(mut node_list) => { Ok(mut node_list) => {
@ -584,10 +583,9 @@ impl System {
select! { select! {
_ = restart_at.fuse() => (), _ = restart_at.fuse() => (),
must_exit = stop_signal.recv().fuse() => { _ = stop_signal.changed().fuse() => {
match must_exit { if *stop_signal.borrow() {
None | Some(true) => return, return;
_ => (),
} }
} }
} }

2
src/rpc/rpc_client.rs
View file

@ -198,7 +198,7 @@ impl<M: RpcMessage + 'static> RpcClient<M> {
let wait_finished_fut = tokio::spawn(async move { let wait_finished_fut = tokio::spawn(async move {
resp_stream.collect::<Vec<_>>().await; resp_stream.collect::<Vec<_>>().await;
}); });
self.background.spawn(wait_finished_fut.map(|_| Ok(()))); self.background.spawn(wait_finished_fut.map(|_| Ok(()))).await;
} }
Ok(results) Ok(results)

5
src/rpc/rpc_server.rs
View file

@ -13,6 +13,7 @@ use hyper::service::{make_service_fn, service_fn};
use hyper::{Body, Method, Request, Response, Server, StatusCode}; use hyper::{Body, Method, Request, Response, Server, StatusCode};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tokio::net::{TcpListener, TcpStream}; use tokio::net::{TcpListener, TcpStream};
use tokio_stream::wrappers::TcpListenerStream;
use tokio_rustls::server::TlsStream; use tokio_rustls::server::TlsStream;
use tokio_rustls::TlsAcceptor; use tokio_rustls::TlsAcceptor;
@ -171,8 +172,8 @@ impl RpcServer {
config.set_single_cert([&node_certs[..], &ca_certs[..]].concat(), node_key)?; config.set_single_cert([&node_certs[..], &ca_certs[..]].concat(), node_key)?;
let tls_acceptor = Arc::new(TlsAcceptor::from(Arc::new(config))); let tls_acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
let mut listener = TcpListener::bind(&self.bind_addr).await?; let listener = TcpListener::bind(&self.bind_addr).await?;
let incoming = listener.incoming().filter_map(|socket| async { let incoming = TcpListenerStream::new(listener).filter_map(|socket| async {
match socket { match socket {
Ok(stream) => match tls_acceptor.clone().accept(stream).await { Ok(stream) => match tls_acceptor.clone().accept(stream).await {
Ok(x) => Some(Ok::<_, hyper::Error>(x)), Ok(x) => Some(Ok::<_, hyper::Error>(x)),

2
src/table/Cargo.toml
View file

@ -31,5 +31,5 @@ serde_bytes = "0.11"
async-trait = "0.1.30" async-trait = "0.1.30"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }

6
src/table/data.rs
View file

@ -159,7 +159,7 @@ where
if let Some((old_entry, new_entry, new_bytes_hash)) = changed { if let Some((old_entry, new_entry, new_bytes_hash)) = changed {
let is_tombstone = new_entry.is_tombstone(); let is_tombstone = new_entry.is_tombstone();
self.instance.updated(old_entry, Some(new_entry)); self.instance.updated(old_entry, Some(new_entry));
self.merkle_updater.todo_notify.notify(); self.merkle_updater.todo_notify.notify_one();
if is_tombstone { if is_tombstone {
self.gc_todo.insert(&tree_key, new_bytes_hash.as_slice())?; self.gc_todo.insert(&tree_key, new_bytes_hash.as_slice())?;
} }
@ -184,7 +184,7 @@ where
if removed { if removed {
let old_entry = self.decode_entry(v)?; let old_entry = self.decode_entry(v)?;
self.instance.updated(Some(old_entry), None); self.instance.updated(Some(old_entry), None);
self.merkle_updater.todo_notify.notify(); self.merkle_updater.todo_notify.notify_one();
} }
Ok(removed) Ok(removed)
} }
@ -209,7 +209,7 @@ where
if let Some(old_v) = removed { if let Some(old_v) = removed {
let old_entry = self.decode_entry(&old_v[..])?; let old_entry = self.decode_entry(&old_v[..])?;
self.instance.updated(Some(old_entry), None); self.instance.updated(Some(old_entry), None);
self.merkle_updater.todo_notify.notify(); self.merkle_updater.todo_notify.notify_one();
Ok(true) Ok(true)
} else { } else {
Ok(false) Ok(false)

4
src/table/gc.rs
View file

@ -85,8 +85,8 @@ where
} }
} }
select! { select! {
_ = tokio::time::delay_for(Duration::from_secs(10)).fuse() => (), _ = tokio::time::sleep(Duration::from_secs(10)).fuse() => (),
_ = must_exit.recv().fuse() => (), _ = must_exit.changed().fuse() => (),
} }
} }
} }

4
src/table/merkle.rs
View file

@ -121,13 +121,13 @@ impl MerkleUpdater {
"({}) Error while iterating on Merkle todo tree: {}", "({}) Error while iterating on Merkle todo tree: {}",
self.table_name, e self.table_name, e
); );
tokio::time::delay_for(Duration::from_secs(10)).await; tokio::time::sleep(Duration::from_secs(10)).await;
} }
} }
} else { } else {
select! { select! {
_ = self.todo_notify.notified().fuse() => (), _ = self.todo_notify.notified().fuse() => (),
_ = must_exit.recv().fuse() => (), _ = must_exit.changed().fuse() => (),
} }
} }
} }

35
src/table/sync.rs
View file

@ -3,7 +3,7 @@ use std::convert::TryInto;
use std::sync::{Arc, Mutex}; use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant}; use std::time::{Duration, Instant};
use futures::{pin_mut, select}; use futures::{select};
use futures_util::future::*; use futures_util::future::*;
use futures_util::stream::*; use futures_util::stream::*;
use rand::Rng; use rand::Rng;
@ -110,7 +110,7 @@ where
let s3 = syncer.clone(); let s3 = syncer.clone();
tokio::spawn(async move { tokio::spawn(async move {
tokio::time::delay_for(Duration::from_secs(20)).await; tokio::time::sleep(Duration::from_secs(20)).await;
s3.add_full_sync(); s3.add_full_sync();
}); });
@ -142,23 +142,16 @@ where
let mut nothing_to_do_since = Some(Instant::now()); let mut nothing_to_do_since = Some(Instant::now());
while !*must_exit.borrow() { while !*must_exit.borrow() {
let s_ring_recv = ring_recv.recv().fuse();
let s_busy = busy_rx.recv().fuse();
let s_must_exit = must_exit.recv().fuse();
let s_timeout = tokio::time::delay_for(Duration::from_secs(1)).fuse();
pin_mut!(s_ring_recv, s_busy, s_must_exit, s_timeout);
select! { select! {
new_ring_r = s_ring_recv => { _ = ring_recv.changed().fuse() => {
if let Some(new_ring) = new_ring_r { let new_ring = ring_recv.borrow();
if !Arc::ptr_eq(&new_ring, &prev_ring) { if !Arc::ptr_eq(&new_ring, &prev_ring) {
debug!("({}) Ring changed, adding full sync to syncer todo list", self.data.name); debug!("({}) Ring changed, adding full sync to syncer todo list", self.data.name);
self.add_full_sync(); self.add_full_sync();
prev_ring = new_ring; prev_ring = new_ring.clone();
}
} }
} }
busy_opt = s_busy => { busy_opt = busy_rx.recv().fuse() => {
if let Some(busy) = busy_opt { if let Some(busy) = busy_opt {
if busy { if busy {
nothing_to_do_since = None; nothing_to_do_since = None;
@ -169,12 +162,8 @@ where
} }
} }
} }
must_exit_v = s_must_exit => { _ = must_exit.changed().fuse() => (),
if must_exit_v.unwrap_or(false) { _ = tokio::time::sleep(Duration::from_secs(1)).fuse() => {
break;
}
}
_ = s_timeout => {
if nothing_to_do_since.map(|t| Instant::now() - t >= ANTI_ENTROPY_INTERVAL).unwrap_or(false) { if nothing_to_do_since.map(|t| Instant::now() - t >= ANTI_ENTROPY_INTERVAL).unwrap_or(false) {
nothing_to_do_since = None; nothing_to_do_since = None;
debug!("({}) Interval passed, adding full sync to syncer todo list", self.data.name); debug!("({}) Interval passed, adding full sync to syncer todo list", self.data.name);
@ -213,7 +202,7 @@ where
} }
} else { } else {
busy_tx.send(false).unwrap(); busy_tx.send(false).unwrap();
tokio::time::delay_for(Duration::from_secs(1)).await; tokio::time::sleep(Duration::from_secs(1)).await;
} }
} }
} }

7
src/util/Cargo.toml
View file

@ -28,14 +28,15 @@ rmp-serde = "0.14.3"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_json = "1.0" serde_json = "1.0"
chrono = "0.4" chrono = "0.4"
arc-swap = "1.2"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] } tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
http = "0.2" http = "0.2"
hyper = "0.13" hyper = "0.14"
rustls = "0.17" rustls = "0.19"
webpki = "0.21" webpki = "0.21"
roxmltree = "0.11" roxmltree = "0.11"

131
src/util/background.rs
View file

@ -2,11 +2,9 @@ use core::future::Future;
use std::pin::Pin; use std::pin::Pin;
use std::sync::Mutex; use std::sync::Mutex;
use futures::future::join_all; use arc_swap::ArcSwapOption;
use futures::select;
use futures_util::future::*;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::{mpsc, watch, Notify}; use tokio::sync::{mpsc, watch};
use crate::error::Error; use crate::error::Error;
@ -14,12 +12,9 @@ type JobOutput = Result<(), Error>;
type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>; type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>;
pub struct BackgroundRunner { pub struct BackgroundRunner {
n_runners: usize,
pub stop_signal: watch::Receiver<bool>, pub stop_signal: watch::Receiver<bool>,
queue_in: mpsc::UnboundedSender<(Job, bool)>, queue_in: ArcSwapOption<mpsc::UnboundedSender<(Job, bool)>>,
queue_out: Mutex<mpsc::UnboundedReceiver<(Job, bool)>>,
job_notify: Notify,
workers: Mutex<Vec<tokio::task::JoinHandle<()>>>, workers: Mutex<Vec<tokio::task::JoinHandle<()>>>,
} }
@ -27,50 +22,91 @@ pub struct BackgroundRunner {
impl BackgroundRunner { impl BackgroundRunner {
pub fn new(n_runners: usize, stop_signal: watch::Receiver<bool>) -> Arc<Self> { pub fn new(n_runners: usize, stop_signal: watch::Receiver<bool>) -> Arc<Self> {
let (queue_in, queue_out) = mpsc::unbounded_channel(); let (queue_in, queue_out) = mpsc::unbounded_channel();
let mut workers = vec![];
let queue_out = Arc::new(tokio::sync::Mutex::new(queue_out));
for i in 0..n_runners {
let queue_out = queue_out.clone();
let stop_signal = stop_signal.clone();
workers.push(tokio::spawn(async move {
while let Some((job, cancellable)) = queue_out.lock().await.recv().await {
if cancellable && *stop_signal.borrow() {
continue;
}
if let Err(e) = job.await {
error!("Job failed: {}", e)
}
}
info!("Worker {} exiting", i);
}));
}
Arc::new(Self { Arc::new(Self {
n_runners,
stop_signal, stop_signal,
queue_in, queue_in: ArcSwapOption::new(Some(Arc::new(queue_in))),
queue_out: Mutex::new(queue_out), workers: Mutex::new(workers),
job_notify: Notify::new(),
workers: Mutex::new(Vec::new()),
}) })
} }
pub async fn run(self: Arc<Self>) { pub async fn run(self: Arc<Self>) {
let mut workers = self.workers.lock().unwrap();
for i in 0..self.n_runners {
workers.push(tokio::spawn(self.clone().runner(i)));
}
drop(workers);
let mut stop_signal = self.stop_signal.clone(); let mut stop_signal = self.stop_signal.clone();
while let Some(exit_now) = stop_signal.recv().await {
loop {
let exit_now = match stop_signal.changed().await {
Ok(()) => *stop_signal.borrow(),
Err(e) => {
error!("Watch .changed() error: {}", e);
true
}
};
if exit_now { if exit_now {
let mut workers = self.workers.lock().unwrap(); break;
let workers_vec = workers.drain(..).collect::<Vec<_>>(); }
join_all(workers_vec).await; }
return;
info!("Closing background job queue_in...");
drop(self.queue_in.swap(None));
info!("Waiting for all workers to terminate...");
while let Some(task) = self.workers.lock().unwrap().pop() {
if let Err(e) = task.await {
warn!("Error awaiting task: {}", e);
} }
} }
} }
pub fn spawn<T>(&self, job: T) // Spawn a task to be run in background
pub async fn spawn<T>(&self, job: T)
where where
T: Future<Output = JobOutput> + Send + 'static, T: Future<Output = JobOutput> + Send + 'static,
{ {
let boxed: Job = Box::pin(job); match self.queue_in.load().as_ref() {
let _: Result<_, _> = self.queue_in.clone().send((boxed, false)); Some(chan) => {
self.job_notify.notify(); let boxed: Job = Box::pin(job);
chan.send((boxed, false)).map_err(|_| "send error").unwrap();
}
None => {
warn!("Doing background job now because we are exiting...");
if let Err(e) = job.await {
warn!("Task failed: {}", e);
}
}
}
} }
pub fn spawn_cancellable<T>(&self, job: T) pub fn spawn_cancellable<T>(&self, job: T)
where where
T: Future<Output = JobOutput> + Send + 'static, T: Future<Output = JobOutput> + Send + 'static,
{ {
let boxed: Job = Box::pin(job); match self.queue_in.load().as_ref() {
let _: Result<_, _> = self.queue_in.clone().send((boxed, true)); Some(chan) => {
self.job_notify.notify(); let boxed: Job = Box::pin(job);
chan.send((boxed, false)).map_err(|_| "send error").unwrap();
}
None => (), // drop job if we are exiting
}
} }
pub fn spawn_worker<F, T>(&self, name: String, worker: F) pub fn spawn_worker<F, T>(&self, name: String, worker: F)
@ -85,37 +121,4 @@ impl BackgroundRunner {
info!("Worker exited: {}", name); info!("Worker exited: {}", name);
})); }));
} }
async fn runner(self: Arc<Self>, i: usize) {
let mut stop_signal = self.stop_signal.clone();
loop {
let must_exit: bool = *stop_signal.borrow();
if let Some(job) = self.dequeue_job(must_exit) {
if let Err(e) = job.await {
error!("Job failed: {}", e)
}
} else {
if must_exit {
info!("Background runner {} exiting", i);
return;
}
select! {
_ = self.job_notify.notified().fuse() => (),
_ = stop_signal.recv().fuse() => (),
}
}
}
}
fn dequeue_job(&self, must_exit: bool) -> Option<Job> {
let mut queue = self.queue_out.lock().unwrap();
while let Ok((job, cancellable)) = queue.try_recv() {
if cancellable && must_exit {
continue;
} else {
return Some(job);
}
}
None
}
} }

8
src/util/error.rs
View file

@ -8,16 +8,22 @@ use crate::data::*;
pub enum RPCError { pub enum RPCError {
#[error(display = "Node is down: {:?}.", _0)] #[error(display = "Node is down: {:?}.", _0)]
NodeDown(UUID), NodeDown(UUID),
#[error(display = "Timeout: {}", _0)] #[error(display = "Timeout: {}", _0)]
Timeout(#[error(source)] tokio::time::Elapsed), Timeout(#[error(source)] tokio::time::error::Elapsed),
#[error(display = "HTTP error: {}", _0)] #[error(display = "HTTP error: {}", _0)]
HTTP(#[error(source)] http::Error), HTTP(#[error(source)] http::Error),
#[error(display = "Hyper error: {}", _0)] #[error(display = "Hyper error: {}", _0)]
Hyper(#[error(source)] hyper::Error), Hyper(#[error(source)] hyper::Error),
#[error(display = "Messagepack encode error: {}", _0)] #[error(display = "Messagepack encode error: {}", _0)]
RMPEncode(#[error(source)] rmp_serde::encode::Error), RMPEncode(#[error(source)] rmp_serde::encode::Error),
#[error(display = "Messagepack decode error: {}", _0)] #[error(display = "Messagepack decode error: {}", _0)]
RMPDecode(#[error(source)] rmp_serde::decode::Error), RMPDecode(#[error(source)] rmp_serde::decode::Error),
#[error(display = "Too many errors: {:?}", _0)] #[error(display = "Too many errors: {:?}", _0)]
TooManyErrors(Vec<String>), TooManyErrors(Vec<String>),
} }

2
src/web/Cargo.toml
View file

@ -22,7 +22,7 @@ err-derive = "0.2.3"
log = "0.4" log = "0.4"
futures = "0.3" futures = "0.3"
http = "0.2" http = "0.2"
hyper = "0.13" hyper = "0.14"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"
roxmltree = "0.11" roxmltree = "0.11"
idna = "0.2" idna = "0.2"