WIP add content defined chunking #42
21 changed files with 233 additions and 336 deletions
278
Cargo.lock
generated
278
Cargo.lock
generated
|
|
@ -17,6 +17,12 @@ version = "0.4.8"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8"
|
checksum = "dabe5a181f83789739c194cbe5a897dde195078fac08568d09221fd6137a7ba8"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "arc-swap"
|
||||||
|
version = "1.2.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "d4d7d63395147b81a9e570bcc6243aaf71c017bd666d4909cfef0085bdda8d73"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "arrayvec"
|
name = "arrayvec"
|
||||||
version = "0.3.25"
|
version = "0.3.25"
|
||||||
|
|
@ -46,7 +52,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"hermit-abi",
|
"hermit-abi",
|
||||||
"libc",
|
"libc",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -55,12 +61,6 @@ version = "1.0.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
|
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "base64"
|
|
||||||
version = "0.11.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "base64"
|
name = "base64"
|
||||||
version = "0.13.0"
|
version = "0.13.0"
|
||||||
|
|
@ -142,12 +142,6 @@ dependencies = [
|
||||||
"iovec",
|
"iovec",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "bytes"
|
|
||||||
version = "0.5.6"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bytes"
|
name = "bytes"
|
||||||
version = "1.0.1"
|
version = "1.0.1"
|
||||||
|
|
@ -182,7 +176,7 @@ dependencies = [
|
||||||
"num-integer",
|
"num-integer",
|
||||||
"num-traits",
|
"num-traits",
|
||||||
"time",
|
"time",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -347,7 +341,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -356,22 +350,6 @@ version = "0.1.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
|
checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fuchsia-zircon"
|
|
||||||
version = "0.3.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82"
|
|
||||||
dependencies = [
|
|
||||||
"bitflags",
|
|
||||||
"fuchsia-zircon-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "fuchsia-zircon-sys"
|
|
||||||
version = "0.3.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "futures"
|
name = "futures"
|
||||||
version = "0.3.12"
|
version = "0.3.12"
|
||||||
|
|
@ -460,7 +438,7 @@ dependencies = [
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
"futures-task",
|
"futures-task",
|
||||||
"memchr",
|
"memchr",
|
||||||
"pin-project-lite 0.2.4",
|
"pin-project-lite",
|
||||||
"pin-utils",
|
"pin-utils",
|
||||||
"proc-macro-hack",
|
"proc-macro-hack",
|
||||||
"proc-macro-nested",
|
"proc-macro-nested",
|
||||||
|
|
@ -507,7 +485,7 @@ dependencies = [
|
||||||
name = "garage_api"
|
name = "garage_api"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64 0.13.0",
|
"base64",
|
||||||
"bytes 0.4.12",
|
"bytes 0.4.12",
|
||||||
"chrono",
|
"chrono",
|
||||||
"crypto-mac 0.7.0",
|
"crypto-mac 0.7.0",
|
||||||
|
|
@ -537,7 +515,7 @@ dependencies = [
|
||||||
name = "garage_model"
|
name = "garage_model"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap 0.4.8",
|
||||||
"async-trait",
|
"async-trait",
|
||||||
"bytes 0.4.12",
|
"bytes 0.4.12",
|
||||||
"futures",
|
"futures",
|
||||||
|
|
@ -560,7 +538,7 @@ dependencies = [
|
||||||
name = "garage_rpc"
|
name = "garage_rpc"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"arc-swap",
|
"arc-swap 0.4.8",
|
||||||
"bytes 0.4.12",
|
"bytes 0.4.12",
|
||||||
"futures",
|
"futures",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
|
|
@ -579,6 +557,7 @@ dependencies = [
|
||||||
"sha2",
|
"sha2",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-rustls",
|
"tokio-rustls",
|
||||||
|
"tokio-stream",
|
||||||
"webpki",
|
"webpki",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -607,6 +586,7 @@ dependencies = [
|
||||||
name = "garage_util"
|
name = "garage_util"
|
||||||
version = "0.1.1"
|
version = "0.1.1"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
|
"arc-swap 1.2.0",
|
||||||
"blake2",
|
"blake2",
|
||||||
"chrono",
|
"chrono",
|
||||||
"err-derive",
|
"err-derive",
|
||||||
|
|
@ -664,7 +644,7 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"log",
|
"log",
|
||||||
"rustversion",
|
"rustversion",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -693,7 +673,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028"
|
checksum = "e692e296bfac1d2533ef168d0b60ff5897b8b70a4009276834014dd8924cc028"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -731,11 +711,11 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "h2"
|
name = "h2"
|
||||||
version = "0.2.7"
|
version = "0.3.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535"
|
checksum = "d832b01df74254fe364568d6ddc294443f61cbec82816b60904303af87efae78"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 0.5.6",
|
"bytes 1.0.1",
|
||||||
"fnv",
|
"fnv",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
|
|
@ -746,7 +726,6 @@ dependencies = [
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-util",
|
"tokio-util",
|
||||||
"tracing",
|
"tracing",
|
||||||
"tracing-futures",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -812,11 +791,11 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "http-body"
|
name = "http-body"
|
||||||
version = "0.3.1"
|
version = "0.4.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b"
|
checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 0.5.6",
|
"bytes 1.0.1",
|
||||||
"http",
|
"http",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -849,11 +828,11 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hyper"
|
name = "hyper"
|
||||||
version = "0.13.9"
|
version = "0.14.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf"
|
checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 0.5.6",
|
"bytes 1.0.1",
|
||||||
"futures-channel",
|
"futures-channel",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-util",
|
"futures-util",
|
||||||
|
|
@ -863,7 +842,7 @@ dependencies = [
|
||||||
"httparse",
|
"httparse",
|
||||||
"httpdate",
|
"httpdate",
|
||||||
"itoa",
|
"itoa",
|
||||||
"pin-project 1.0.4",
|
"pin-project",
|
||||||
"socket2",
|
"socket2",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tower-service",
|
"tower-service",
|
||||||
|
|
@ -873,11 +852,10 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "hyper-rustls"
|
name = "hyper-rustls"
|
||||||
version = "0.20.0"
|
version = "0.22.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ac965ea399ec3a25ac7d13b8affd4b8f39325cca00858ddf5eb29b79e6b14b08"
|
checksum = "5f9f7a97316d44c0af9b0301e65010573a853a9fc97046d7331d7f6bc0fd5a64"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 0.5.6",
|
|
||||||
"futures-util",
|
"futures-util",
|
||||||
"hyper",
|
"hyper",
|
||||||
"log",
|
"log",
|
||||||
|
|
@ -947,16 +925,6 @@ dependencies = [
|
||||||
"wasm-bindgen",
|
"wasm-bindgen",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "kernel32-sys"
|
|
||||||
version = "0.2.2"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d"
|
|
||||||
dependencies = [
|
|
||||||
"winapi 0.2.8",
|
|
||||||
"winapi-build",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lazy_static"
|
name = "lazy_static"
|
||||||
version = "1.4.0"
|
version = "1.4.0"
|
||||||
|
|
@ -965,9 +933,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libc"
|
name = "libc"
|
||||||
version = "0.2.82"
|
version = "0.2.89"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
|
checksum = "538c092e5586f4cdd7dd8078c4a79220e3e168880218124dcbce860f0ea938c6"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "lock_api"
|
name = "lock_api"
|
||||||
|
|
@ -1032,55 +1000,25 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "mio"
|
name = "mio"
|
||||||
version = "0.6.23"
|
version = "0.7.10"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4"
|
checksum = "2182a122f3b7f3f5329cb1972cee089ba2459a0a80a56935e6e674f096f8d839"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if 0.1.10",
|
|
||||||
"fuchsia-zircon",
|
|
||||||
"fuchsia-zircon-sys",
|
|
||||||
"iovec",
|
|
||||||
"kernel32-sys",
|
|
||||||
"libc",
|
"libc",
|
||||||
"log",
|
"log",
|
||||||
"miow",
|
"miow",
|
||||||
"net2",
|
"ntapi",
|
||||||
"slab",
|
"winapi",
|
||||||
"winapi 0.2.8",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "mio-uds"
|
|
||||||
version = "0.6.8"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0"
|
|
||||||
dependencies = [
|
|
||||||
"iovec",
|
|
||||||
"libc",
|
|
||||||
"mio",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "miow"
|
name = "miow"
|
||||||
version = "0.2.2"
|
version = "0.3.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d"
|
checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"kernel32-sys",
|
"socket2",
|
||||||
"net2",
|
"winapi",
|
||||||
"winapi 0.2.8",
|
|
||||||
"ws2_32-sys",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "net2"
|
|
||||||
version = "0.2.37"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "391630d12b68002ae1e25e8f974306474966550ad82dac6886fb8910c19568ae"
|
|
||||||
dependencies = [
|
|
||||||
"cfg-if 0.1.10",
|
|
||||||
"libc",
|
|
||||||
"winapi 0.3.9",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1089,6 +1027,15 @@ version = "0.1.14"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ntapi"
|
||||||
|
version = "0.3.6"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
|
||||||
|
dependencies = [
|
||||||
|
"winapi",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "num-integer"
|
name = "num-integer"
|
||||||
version = "0.1.44"
|
version = "0.1.44"
|
||||||
|
|
@ -1164,7 +1111,7 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"redox_syscall",
|
"redox_syscall",
|
||||||
"smallvec",
|
"smallvec",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1173,33 +1120,13 @@ version = "2.1.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pin-project"
|
|
||||||
version = "0.4.27"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15"
|
|
||||||
dependencies = [
|
|
||||||
"pin-project-internal 0.4.27",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pin-project"
|
name = "pin-project"
|
||||||
version = "1.0.4"
|
version = "1.0.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2"
|
checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"pin-project-internal 1.0.4",
|
"pin-project-internal",
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pin-project-internal"
|
|
||||||
version = "0.4.27"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895"
|
|
||||||
dependencies = [
|
|
||||||
"proc-macro2",
|
|
||||||
"quote",
|
|
||||||
"syn",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1213,12 +1140,6 @@ dependencies = [
|
||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "pin-project-lite"
|
|
||||||
version = "0.1.11"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "pin-project-lite"
|
name = "pin-project-lite"
|
||||||
version = "0.2.4"
|
version = "0.2.4"
|
||||||
|
|
@ -1317,7 +1238,7 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"rand_core 0.3.1",
|
"rand_core 0.3.1",
|
||||||
"rdrand",
|
"rdrand",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1424,7 +1345,7 @@ dependencies = [
|
||||||
"spin",
|
"spin",
|
||||||
"untrusted",
|
"untrusted",
|
||||||
"web-sys",
|
"web-sys",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1459,11 +1380,11 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls"
|
name = "rustls"
|
||||||
version = "0.17.0"
|
version = "0.19.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1"
|
checksum = "064fd21ff87c6e87ed4506e68beb42459caa4a0e2eb144932e6776768556980b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"base64 0.11.0",
|
"base64",
|
||||||
"log",
|
"log",
|
||||||
"ring",
|
"ring",
|
||||||
"sct",
|
"sct",
|
||||||
|
|
@ -1607,7 +1528,7 @@ checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if 1.0.0",
|
"cfg-if 1.0.0",
|
||||||
"libc",
|
"libc",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1710,7 +1631,7 @@ checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
"wasi 0.10.0+wasi-snapshot-preview1",
|
"wasi 0.10.0+wasi-snapshot-preview1",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -1730,32 +1651,28 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio"
|
name = "tokio"
|
||||||
version = "0.2.24"
|
version = "1.3.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "099837d3464c16a808060bb3f02263b412f6fafcb5d01c533d309985fbeebe48"
|
checksum = "8d56477f6ed99e10225f38f9f75f872f29b8b8bd8c0b946f63345bb144e9eeda"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 0.5.6",
|
"autocfg",
|
||||||
"fnv",
|
"bytes 1.0.1",
|
||||||
"futures-core",
|
|
||||||
"iovec",
|
|
||||||
"lazy_static",
|
|
||||||
"libc",
|
"libc",
|
||||||
"memchr",
|
"memchr",
|
||||||
"mio",
|
"mio",
|
||||||
"mio-uds",
|
|
||||||
"num_cpus",
|
"num_cpus",
|
||||||
"pin-project-lite 0.1.11",
|
"once_cell",
|
||||||
|
"pin-project-lite",
|
||||||
"signal-hook-registry",
|
"signal-hook-registry",
|
||||||
"slab",
|
|
||||||
"tokio-macros",
|
"tokio-macros",
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-macros"
|
name = "tokio-macros"
|
||||||
version = "0.2.6"
|
version = "1.1.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e44da00bfc73a25f814cd8d7e57a68a5c31b74b3152a0a1d1f590c97ed06265a"
|
checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"proc-macro2",
|
"proc-macro2",
|
||||||
"quote",
|
"quote",
|
||||||
|
|
@ -1764,27 +1681,37 @@ dependencies = [
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-rustls"
|
name = "tokio-rustls"
|
||||||
version = "0.13.1"
|
version = "0.22.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4"
|
checksum = "bc6844de72e57df1980054b38be3a9f4702aba4858be64dd700181a8a6d0e1b6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"futures-core",
|
|
||||||
"rustls",
|
"rustls",
|
||||||
"tokio",
|
"tokio",
|
||||||
"webpki",
|
"webpki",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "tokio-util"
|
name = "tokio-stream"
|
||||||
version = "0.3.1"
|
version = "0.1.4"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499"
|
checksum = "c535f53c0cfa1acace62995a8994fc9cc1f12d202420da96ff306ee24d576469"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bytes 0.5.6",
|
"futures-core",
|
||||||
|
"pin-project-lite",
|
||||||
|
"tokio",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "tokio-util"
|
||||||
|
version = "0.6.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "ec31e5cc6b46e653cf57762f36f71d5e6386391d88a72fd6db4508f8f676fb29"
|
||||||
|
dependencies = [
|
||||||
|
"bytes 1.0.1",
|
||||||
"futures-core",
|
"futures-core",
|
||||||
"futures-sink",
|
"futures-sink",
|
||||||
"log",
|
"log",
|
||||||
"pin-project-lite 0.1.11",
|
"pin-project-lite",
|
||||||
"tokio",
|
"tokio",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -1810,8 +1737,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
|
checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cfg-if 1.0.0",
|
"cfg-if 1.0.0",
|
||||||
"log",
|
"pin-project-lite",
|
||||||
"pin-project-lite 0.2.4",
|
|
||||||
"tracing-core",
|
"tracing-core",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
@ -1824,16 +1750,6 @@ dependencies = [
|
||||||
"lazy_static",
|
"lazy_static",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "tracing-futures"
|
|
||||||
version = "0.2.4"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c"
|
|
||||||
dependencies = [
|
|
||||||
"pin-project 0.4.27",
|
|
||||||
"tracing",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "try-lock"
|
name = "try-lock"
|
||||||
version = "0.2.3"
|
version = "0.2.3"
|
||||||
|
|
@ -2002,12 +1918,6 @@ dependencies = [
|
||||||
"untrusted",
|
"untrusted",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "winapi"
|
|
||||||
version = "0.2.8"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "winapi"
|
name = "winapi"
|
||||||
version = "0.3.9"
|
version = "0.3.9"
|
||||||
|
|
@ -2018,12 +1928,6 @@ dependencies = [
|
||||||
"winapi-x86_64-pc-windows-gnu",
|
"winapi-x86_64-pc-windows-gnu",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "winapi-build"
|
|
||||||
version = "0.1.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "winapi-i686-pc-windows-gnu"
|
name = "winapi-i686-pc-windows-gnu"
|
||||||
version = "0.4.0"
|
version = "0.4.0"
|
||||||
|
|
@ -2036,7 +1940,7 @@ version = "0.1.5"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
|
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"winapi 0.3.9",
|
"winapi",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
|
|
@ -2045,16 +1949,6 @@ version = "0.4.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "ws2_32-sys"
|
|
||||||
version = "0.2.1"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e"
|
|
||||||
dependencies = [
|
|
||||||
"winapi 0.2.8",
|
|
||||||
"winapi-build",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "xmlparser"
|
name = "xmlparser"
|
||||||
version = "0.13.3"
|
version = "0.13.3"
|
||||||
|
|
|
||||||
|
|
@ -31,10 +31,10 @@ rand = "0.7"
|
||||||
|
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
|
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||||
|
|
||||||
http = "0.2"
|
http = "0.2"
|
||||||
hyper = "^0.13.6"
|
hyper = "0.14"
|
||||||
url = "2.1"
|
url = "2.1"
|
||||||
httpdate = "0.3"
|
httpdate = "0.3"
|
||||||
percent-encoding = "2.1.0"
|
percent-encoding = "2.1.0"
|
||||||
|
|
|
||||||
|
|
@ -38,4 +38,4 @@ serde = { version = "1.0", default-features = false, features = ["derive", "rc"]
|
||||||
|
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
|
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||||
|
|
|
||||||
|
|
@ -21,13 +21,13 @@ async fn shutdown_signal(send_cancel: watch::Sender<bool>) -> Result<(), Error>
|
||||||
.await
|
.await
|
||||||
.expect("failed to install CTRL+C signal handler");
|
.expect("failed to install CTRL+C signal handler");
|
||||||
info!("Received CTRL+C, shutting down.");
|
info!("Received CTRL+C, shutting down.");
|
||||||
send_cancel.broadcast(true)?;
|
send_cancel.send(true)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn wait_from(mut chan: watch::Receiver<bool>) -> () {
|
async fn wait_from(mut chan: watch::Receiver<bool>) -> () {
|
||||||
while let Some(exit_now) = chan.recv().await {
|
while !*chan.borrow() {
|
||||||
if exit_now {
|
if chan.changed().await.is_err() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -33,5 +33,4 @@ serde_bytes = "0.11"
|
||||||
async-trait = "0.1.30"
|
async-trait = "0.1.30"
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
|
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -5,10 +5,9 @@ use std::time::Duration;
|
||||||
use arc_swap::ArcSwapOption;
|
use arc_swap::ArcSwapOption;
|
||||||
use futures::future::*;
|
use futures::future::*;
|
||||||
use futures::select;
|
use futures::select;
|
||||||
use futures::stream::*;
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::fs;
|
use tokio::fs;
|
||||||
use tokio::prelude::*;
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||||
use tokio::sync::{watch, Mutex, Notify};
|
use tokio::sync::{watch, Mutex, Notify};
|
||||||
|
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
|
|
@ -134,7 +133,7 @@ impl BlockManager {
|
||||||
let bm2 = self.clone();
|
let bm2 = self.clone();
|
||||||
let background = self.system.background.clone();
|
let background = self.system.background.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
tokio::time::delay_for(Duration::from_secs(10 * (i + 1))).await;
|
tokio::time::sleep(Duration::from_secs(10 * (i + 1))).await;
|
||||||
background.spawn_worker(format!("block resync worker {}", i), move |must_exit| {
|
background.spawn_worker(format!("block resync worker {}", i), move |must_exit| {
|
||||||
bm2.resync_loop(must_exit)
|
bm2.resync_loop(must_exit)
|
||||||
});
|
});
|
||||||
|
|
@ -251,7 +250,7 @@ impl BlockManager {
|
||||||
let mut key = u64::to_be_bytes(when).to_vec();
|
let mut key = u64::to_be_bytes(when).to_vec();
|
||||||
key.extend(hash.as_ref());
|
key.extend(hash.as_ref());
|
||||||
self.resync_queue.insert(key, hash.as_ref())?;
|
self.resync_queue.insert(key, hash.as_ref())?;
|
||||||
self.resync_notify.notify();
|
self.resync_notify.notify_waiters();
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -262,7 +261,7 @@ impl BlockManager {
|
||||||
while !*must_exit.borrow() {
|
while !*must_exit.borrow() {
|
||||||
if let Err(e) = self.resync_iter(&mut must_exit).await {
|
if let Err(e) = self.resync_iter(&mut must_exit).await {
|
||||||
warn!("Error in block resync loop: {}", e);
|
warn!("Error in block resync loop: {}", e);
|
||||||
tokio::time::delay_for(Duration::from_secs(10)).await;
|
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -283,17 +282,17 @@ impl BlockManager {
|
||||||
self.resync_queue.remove(&time_bytes)?;
|
self.resync_queue.remove(&time_bytes)?;
|
||||||
res?; // propagate error to delay main loop
|
res?; // propagate error to delay main loop
|
||||||
} else {
|
} else {
|
||||||
let delay = tokio::time::delay_for(Duration::from_millis(time_msec - now));
|
let delay = tokio::time::sleep(Duration::from_millis(time_msec - now));
|
||||||
select! {
|
select! {
|
||||||
_ = delay.fuse() => (),
|
_ = delay.fuse() => (),
|
||||||
_ = self.resync_notify.notified().fuse() => (),
|
_ = self.resync_notify.notified().fuse() => (),
|
||||||
_ = must_exit.recv().fuse() => (),
|
_ = must_exit.changed().fuse() => (),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
select! {
|
select! {
|
||||||
_ = self.resync_notify.notified().fuse() => (),
|
_ = self.resync_notify.notified().fuse() => (),
|
||||||
_ = must_exit.recv().fuse() => (),
|
_ = must_exit.changed().fuse() => (),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
|
|
@ -467,8 +466,12 @@ impl BlockManager {
|
||||||
// so that we can offload them if necessary and then delete them locally.
|
// so that we can offload them if necessary and then delete them locally.
|
||||||
async move {
|
async move {
|
||||||
let mut ls_data_dir = fs::read_dir(path).await?;
|
let mut ls_data_dir = fs::read_dir(path).await?;
|
||||||
while let Some(data_dir_ent) = ls_data_dir.next().await {
|
loop {
|
||||||
let data_dir_ent = data_dir_ent?;
|
let data_dir_ent = ls_data_dir.next_entry().await?;
|
||||||
|
let data_dir_ent = match data_dir_ent {
|
||||||
|
Some(x) => x,
|
||||||
|
None => break,
|
||||||
|
};
|
||||||
let name = data_dir_ent.file_name();
|
let name = data_dir_ent.file_name();
|
||||||
let name = match name.into_string() {
|
let name = match name.into_string() {
|
||||||
Ok(x) => x,
|
Ok(x) => x,
|
||||||
|
|
|
||||||
|
|
@ -195,7 +195,8 @@ impl TableSchema for ObjectTable {
|
||||||
|
|
||||||
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
|
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
|
||||||
let version_table = self.version_table.clone();
|
let version_table = self.version_table.clone();
|
||||||
self.background.spawn(async move {
|
// TODO not cancellable
|
||||||
|
self.background.spawn_cancellable(async move {
|
||||||
if let (Some(old_v), Some(new_v)) = (old, new) {
|
if let (Some(old_v), Some(new_v)) = (old, new) {
|
||||||
// Propagate deletion of old versions
|
// Propagate deletion of old versions
|
||||||
for v in old_v.versions.iter() {
|
for v in old_v.versions.iter() {
|
||||||
|
|
|
||||||
|
|
@ -110,7 +110,8 @@ impl TableSchema for VersionTable {
|
||||||
|
|
||||||
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
|
fn updated(&self, old: Option<Self::E>, new: Option<Self::E>) {
|
||||||
let block_ref_table = self.block_ref_table.clone();
|
let block_ref_table = self.block_ref_table.clone();
|
||||||
self.background.spawn(async move {
|
// TODO not cancellable
|
||||||
|
self.background.spawn_cancellable(async move {
|
||||||
if let (Some(old_v), Some(new_v)) = (old, new) {
|
if let (Some(old_v), Some(new_v)) = (old, new) {
|
||||||
// Propagate deletion of version blocks
|
// Propagate deletion of version blocks
|
||||||
if new_v.deleted.get() && !old_v.deleted.get() {
|
if new_v.deleted.get() && !old_v.deleted.get() {
|
||||||
|
|
|
||||||
|
|
@ -29,13 +29,14 @@ serde_json = "1.0"
|
||||||
|
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
|
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||||
|
tokio-stream = {version = "0.1", features = ["net"] }
|
||||||
|
|
||||||
http = "0.2"
|
http = "0.2"
|
||||||
hyper = "0.13"
|
hyper = { version = "0.14", features = ["full"] }
|
||||||
rustls = "0.17"
|
rustls = "0.19"
|
||||||
tokio-rustls = "0.13"
|
tokio-rustls = "0.22"
|
||||||
hyper-rustls = { version = "0.20", default-features = false }
|
hyper-rustls = { version = "0.22", default-features = false }
|
||||||
webpki = "0.21"
|
webpki = "0.21"
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -11,9 +11,9 @@ use futures::future::join_all;
|
||||||
use futures::select;
|
use futures::select;
|
||||||
use futures_util::future::*;
|
use futures_util::future::*;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::prelude::*;
|
|
||||||
use tokio::sync::watch;
|
use tokio::sync::watch;
|
||||||
use tokio::sync::Mutex;
|
use tokio::sync::Mutex;
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
|
||||||
use garage_util::background::BackgroundRunner;
|
use garage_util::background::BackgroundRunner;
|
||||||
use garage_util::data::*;
|
use garage_util::data::*;
|
||||||
|
|
@ -395,7 +395,7 @@ impl System {
|
||||||
if has_changes {
|
if has_changes {
|
||||||
status.recalculate_hash();
|
status.recalculate_hash();
|
||||||
}
|
}
|
||||||
if let Err(e) = update_locked.0.broadcast(Arc::new(status)) {
|
if let Err(e) = update_locked.0.send(Arc::new(status)) {
|
||||||
error!("In ping_nodes: could not save status update ({})", e);
|
error!("In ping_nodes: could not save status update ({})", e);
|
||||||
}
|
}
|
||||||
drop(update_locked);
|
drop(update_locked);
|
||||||
|
|
@ -421,7 +421,7 @@ impl System {
|
||||||
let status_hash = status.hash;
|
let status_hash = status.hash;
|
||||||
let config_version = self.ring.borrow().config.version;
|
let config_version = self.ring.borrow().config.version;
|
||||||
|
|
||||||
update_locked.0.broadcast(Arc::new(status))?;
|
update_locked.0.send(Arc::new(status))?;
|
||||||
drop(update_locked);
|
drop(update_locked);
|
||||||
|
|
||||||
if is_new || status_hash != ping.status_hash {
|
if is_new || status_hash != ping.status_hash {
|
||||||
|
|
@ -503,7 +503,7 @@ impl System {
|
||||||
if has_changed {
|
if has_changed {
|
||||||
status.recalculate_hash();
|
status.recalculate_hash();
|
||||||
}
|
}
|
||||||
update_lock.0.broadcast(Arc::new(status))?;
|
update_lock.0.send(Arc::new(status))?;
|
||||||
drop(update_lock);
|
drop(update_lock);
|
||||||
|
|
||||||
if to_ping.len() > 0 {
|
if to_ping.len() > 0 {
|
||||||
|
|
@ -523,7 +523,7 @@ impl System {
|
||||||
|
|
||||||
if adv.version > ring.config.version {
|
if adv.version > ring.config.version {
|
||||||
let ring = Ring::new(adv.clone());
|
let ring = Ring::new(adv.clone());
|
||||||
update_lock.1.broadcast(Arc::new(ring))?;
|
update_lock.1.send(Arc::new(ring))?;
|
||||||
drop(update_lock);
|
drop(update_lock);
|
||||||
|
|
||||||
self.background.spawn_cancellable(
|
self.background.spawn_cancellable(
|
||||||
|
|
@ -531,7 +531,7 @@ impl System {
|
||||||
.broadcast(Message::AdvertiseConfig(adv.clone()), PING_TIMEOUT)
|
.broadcast(Message::AdvertiseConfig(adv.clone()), PING_TIMEOUT)
|
||||||
.map(Ok),
|
.map(Ok),
|
||||||
);
|
);
|
||||||
self.background.spawn(self.clone().save_network_config());
|
self.background.spawn(self.clone().save_network_config()).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(Message::Ok)
|
Ok(Message::Ok)
|
||||||
|
|
@ -539,7 +539,7 @@ impl System {
|
||||||
|
|
||||||
async fn ping_loop(self: Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
|
async fn ping_loop(self: Arc<Self>, mut stop_signal: watch::Receiver<bool>) {
|
||||||
loop {
|
loop {
|
||||||
let restart_at = tokio::time::delay_for(PING_INTERVAL);
|
let restart_at = tokio::time::sleep(PING_INTERVAL);
|
||||||
|
|
||||||
let status = self.status.borrow().clone();
|
let status = self.status.borrow().clone();
|
||||||
let ping_addrs = status
|
let ping_addrs = status
|
||||||
|
|
@ -553,10 +553,9 @@ impl System {
|
||||||
|
|
||||||
select! {
|
select! {
|
||||||
_ = restart_at.fuse() => (),
|
_ = restart_at.fuse() => (),
|
||||||
must_exit = stop_signal.recv().fuse() => {
|
_ = stop_signal.changed().fuse() => {
|
||||||
match must_exit {
|
if *stop_signal.borrow() {
|
||||||
None | Some(true) => return,
|
return;
|
||||||
_ => (),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -570,7 +569,7 @@ impl System {
|
||||||
consul_service_name: String,
|
consul_service_name: String,
|
||||||
) {
|
) {
|
||||||
loop {
|
loop {
|
||||||
let restart_at = tokio::time::delay_for(CONSUL_INTERVAL);
|
let restart_at = tokio::time::sleep(CONSUL_INTERVAL);
|
||||||
|
|
||||||
match get_consul_nodes(&consul_host, &consul_service_name).await {
|
match get_consul_nodes(&consul_host, &consul_service_name).await {
|
||||||
Ok(mut node_list) => {
|
Ok(mut node_list) => {
|
||||||
|
|
@ -584,10 +583,9 @@ impl System {
|
||||||
|
|
||||||
select! {
|
select! {
|
||||||
_ = restart_at.fuse() => (),
|
_ = restart_at.fuse() => (),
|
||||||
must_exit = stop_signal.recv().fuse() => {
|
_ = stop_signal.changed().fuse() => {
|
||||||
match must_exit {
|
if *stop_signal.borrow() {
|
||||||
None | Some(true) => return,
|
return;
|
||||||
_ => (),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -198,7 +198,7 @@ impl<M: RpcMessage + 'static> RpcClient<M> {
|
||||||
let wait_finished_fut = tokio::spawn(async move {
|
let wait_finished_fut = tokio::spawn(async move {
|
||||||
resp_stream.collect::<Vec<_>>().await;
|
resp_stream.collect::<Vec<_>>().await;
|
||||||
});
|
});
|
||||||
self.background.spawn(wait_finished_fut.map(|_| Ok(())));
|
self.background.spawn(wait_finished_fut.map(|_| Ok(()))).await;
|
||||||
}
|
}
|
||||||
|
|
||||||
Ok(results)
|
Ok(results)
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,7 @@ use hyper::service::{make_service_fn, service_fn};
|
||||||
use hyper::{Body, Method, Request, Response, Server, StatusCode};
|
use hyper::{Body, Method, Request, Response, Server, StatusCode};
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
use tokio::net::{TcpListener, TcpStream};
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
|
use tokio_stream::wrappers::TcpListenerStream;
|
||||||
use tokio_rustls::server::TlsStream;
|
use tokio_rustls::server::TlsStream;
|
||||||
use tokio_rustls::TlsAcceptor;
|
use tokio_rustls::TlsAcceptor;
|
||||||
|
|
||||||
|
|
@ -171,8 +172,8 @@ impl RpcServer {
|
||||||
config.set_single_cert([&node_certs[..], &ca_certs[..]].concat(), node_key)?;
|
config.set_single_cert([&node_certs[..], &ca_certs[..]].concat(), node_key)?;
|
||||||
let tls_acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
|
let tls_acceptor = Arc::new(TlsAcceptor::from(Arc::new(config)));
|
||||||
|
|
||||||
let mut listener = TcpListener::bind(&self.bind_addr).await?;
|
let listener = TcpListener::bind(&self.bind_addr).await?;
|
||||||
let incoming = listener.incoming().filter_map(|socket| async {
|
let incoming = TcpListenerStream::new(listener).filter_map(|socket| async {
|
||||||
match socket {
|
match socket {
|
||||||
Ok(stream) => match tls_acceptor.clone().accept(stream).await {
|
Ok(stream) => match tls_acceptor.clone().accept(stream).await {
|
||||||
Ok(x) => Some(Ok::<_, hyper::Error>(x)),
|
Ok(x) => Some(Ok::<_, hyper::Error>(x)),
|
||||||
|
|
|
||||||
|
|
@ -31,5 +31,5 @@ serde_bytes = "0.11"
|
||||||
async-trait = "0.1.30"
|
async-trait = "0.1.30"
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
|
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -159,7 +159,7 @@ where
|
||||||
if let Some((old_entry, new_entry, new_bytes_hash)) = changed {
|
if let Some((old_entry, new_entry, new_bytes_hash)) = changed {
|
||||||
let is_tombstone = new_entry.is_tombstone();
|
let is_tombstone = new_entry.is_tombstone();
|
||||||
self.instance.updated(old_entry, Some(new_entry));
|
self.instance.updated(old_entry, Some(new_entry));
|
||||||
self.merkle_updater.todo_notify.notify();
|
self.merkle_updater.todo_notify.notify_one();
|
||||||
if is_tombstone {
|
if is_tombstone {
|
||||||
self.gc_todo.insert(&tree_key, new_bytes_hash.as_slice())?;
|
self.gc_todo.insert(&tree_key, new_bytes_hash.as_slice())?;
|
||||||
}
|
}
|
||||||
|
|
@ -184,7 +184,7 @@ where
|
||||||
if removed {
|
if removed {
|
||||||
let old_entry = self.decode_entry(v)?;
|
let old_entry = self.decode_entry(v)?;
|
||||||
self.instance.updated(Some(old_entry), None);
|
self.instance.updated(Some(old_entry), None);
|
||||||
self.merkle_updater.todo_notify.notify();
|
self.merkle_updater.todo_notify.notify_one();
|
||||||
}
|
}
|
||||||
Ok(removed)
|
Ok(removed)
|
||||||
}
|
}
|
||||||
|
|
@ -209,7 +209,7 @@ where
|
||||||
if let Some(old_v) = removed {
|
if let Some(old_v) = removed {
|
||||||
let old_entry = self.decode_entry(&old_v[..])?;
|
let old_entry = self.decode_entry(&old_v[..])?;
|
||||||
self.instance.updated(Some(old_entry), None);
|
self.instance.updated(Some(old_entry), None);
|
||||||
self.merkle_updater.todo_notify.notify();
|
self.merkle_updater.todo_notify.notify_one();
|
||||||
Ok(true)
|
Ok(true)
|
||||||
} else {
|
} else {
|
||||||
Ok(false)
|
Ok(false)
|
||||||
|
|
|
||||||
|
|
@ -85,8 +85,8 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
select! {
|
select! {
|
||||||
_ = tokio::time::delay_for(Duration::from_secs(10)).fuse() => (),
|
_ = tokio::time::sleep(Duration::from_secs(10)).fuse() => (),
|
||||||
_ = must_exit.recv().fuse() => (),
|
_ = must_exit.changed().fuse() => (),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -121,13 +121,13 @@ impl MerkleUpdater {
|
||||||
"({}) Error while iterating on Merkle todo tree: {}",
|
"({}) Error while iterating on Merkle todo tree: {}",
|
||||||
self.table_name, e
|
self.table_name, e
|
||||||
);
|
);
|
||||||
tokio::time::delay_for(Duration::from_secs(10)).await;
|
tokio::time::sleep(Duration::from_secs(10)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
select! {
|
select! {
|
||||||
_ = self.todo_notify.notified().fuse() => (),
|
_ = self.todo_notify.notified().fuse() => (),
|
||||||
_ = must_exit.recv().fuse() => (),
|
_ = must_exit.changed().fuse() => (),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ use std::convert::TryInto;
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use futures::{pin_mut, select};
|
use futures::{select};
|
||||||
use futures_util::future::*;
|
use futures_util::future::*;
|
||||||
use futures_util::stream::*;
|
use futures_util::stream::*;
|
||||||
use rand::Rng;
|
use rand::Rng;
|
||||||
|
|
@ -110,7 +110,7 @@ where
|
||||||
|
|
||||||
let s3 = syncer.clone();
|
let s3 = syncer.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
tokio::time::delay_for(Duration::from_secs(20)).await;
|
tokio::time::sleep(Duration::from_secs(20)).await;
|
||||||
s3.add_full_sync();
|
s3.add_full_sync();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -142,23 +142,16 @@ where
|
||||||
let mut nothing_to_do_since = Some(Instant::now());
|
let mut nothing_to_do_since = Some(Instant::now());
|
||||||
|
|
||||||
while !*must_exit.borrow() {
|
while !*must_exit.borrow() {
|
||||||
let s_ring_recv = ring_recv.recv().fuse();
|
|
||||||
let s_busy = busy_rx.recv().fuse();
|
|
||||||
let s_must_exit = must_exit.recv().fuse();
|
|
||||||
let s_timeout = tokio::time::delay_for(Duration::from_secs(1)).fuse();
|
|
||||||
pin_mut!(s_ring_recv, s_busy, s_must_exit, s_timeout);
|
|
||||||
|
|
||||||
select! {
|
select! {
|
||||||
new_ring_r = s_ring_recv => {
|
_ = ring_recv.changed().fuse() => {
|
||||||
if let Some(new_ring) = new_ring_r {
|
let new_ring = ring_recv.borrow();
|
||||||
if !Arc::ptr_eq(&new_ring, &prev_ring) {
|
if !Arc::ptr_eq(&new_ring, &prev_ring) {
|
||||||
debug!("({}) Ring changed, adding full sync to syncer todo list", self.data.name);
|
debug!("({}) Ring changed, adding full sync to syncer todo list", self.data.name);
|
||||||
self.add_full_sync();
|
self.add_full_sync();
|
||||||
prev_ring = new_ring;
|
prev_ring = new_ring.clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
busy_opt = busy_rx.recv().fuse() => {
|
||||||
busy_opt = s_busy => {
|
|
||||||
if let Some(busy) = busy_opt {
|
if let Some(busy) = busy_opt {
|
||||||
if busy {
|
if busy {
|
||||||
nothing_to_do_since = None;
|
nothing_to_do_since = None;
|
||||||
|
|
@ -169,12 +162,8 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
must_exit_v = s_must_exit => {
|
_ = must_exit.changed().fuse() => (),
|
||||||
if must_exit_v.unwrap_or(false) {
|
_ = tokio::time::sleep(Duration::from_secs(1)).fuse() => {
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ = s_timeout => {
|
|
||||||
if nothing_to_do_since.map(|t| Instant::now() - t >= ANTI_ENTROPY_INTERVAL).unwrap_or(false) {
|
if nothing_to_do_since.map(|t| Instant::now() - t >= ANTI_ENTROPY_INTERVAL).unwrap_or(false) {
|
||||||
nothing_to_do_since = None;
|
nothing_to_do_since = None;
|
||||||
debug!("({}) Interval passed, adding full sync to syncer todo list", self.data.name);
|
debug!("({}) Interval passed, adding full sync to syncer todo list", self.data.name);
|
||||||
|
|
@ -213,7 +202,7 @@ where
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
busy_tx.send(false).unwrap();
|
busy_tx.send(false).unwrap();
|
||||||
tokio::time::delay_for(Duration::from_secs(1)).await;
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -28,14 +28,15 @@ rmp-serde = "0.14.3"
|
||||||
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
|
||||||
serde_json = "1.0"
|
serde_json = "1.0"
|
||||||
chrono = "0.4"
|
chrono = "0.4"
|
||||||
|
arc-swap = "1.2"
|
||||||
|
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
futures-util = "0.3"
|
futures-util = "0.3"
|
||||||
tokio = { version = "0.2", default-features = false, features = ["rt-core", "rt-threaded", "io-driver", "net", "tcp", "time", "macros", "sync", "signal", "fs"] }
|
tokio = { version = "1.0", default-features = false, features = ["rt", "rt-multi-thread", "io-util", "net", "time", "macros", "sync", "signal", "fs"] }
|
||||||
|
|
||||||
http = "0.2"
|
http = "0.2"
|
||||||
hyper = "0.13"
|
hyper = "0.14"
|
||||||
rustls = "0.17"
|
rustls = "0.19"
|
||||||
webpki = "0.21"
|
webpki = "0.21"
|
||||||
|
|
||||||
roxmltree = "0.11"
|
roxmltree = "0.11"
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,9 @@ use core::future::Future;
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
|
|
||||||
use futures::future::join_all;
|
use arc_swap::ArcSwapOption;
|
||||||
use futures::select;
|
|
||||||
use futures_util::future::*;
|
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio::sync::{mpsc, watch, Notify};
|
use tokio::sync::{mpsc, watch};
|
||||||
|
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
|
|
||||||
|
|
@ -14,12 +12,9 @@ type JobOutput = Result<(), Error>;
|
||||||
type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>;
|
type Job = Pin<Box<dyn Future<Output = JobOutput> + Send>>;
|
||||||
|
|
||||||
pub struct BackgroundRunner {
|
pub struct BackgroundRunner {
|
||||||
n_runners: usize,
|
|
||||||
pub stop_signal: watch::Receiver<bool>,
|
pub stop_signal: watch::Receiver<bool>,
|
||||||
|
|
||||||
queue_in: mpsc::UnboundedSender<(Job, bool)>,
|
queue_in: ArcSwapOption<mpsc::UnboundedSender<(Job, bool)>>,
|
||||||
queue_out: Mutex<mpsc::UnboundedReceiver<(Job, bool)>>,
|
|
||||||
job_notify: Notify,
|
|
||||||
|
|
||||||
workers: Mutex<Vec<tokio::task::JoinHandle<()>>>,
|
workers: Mutex<Vec<tokio::task::JoinHandle<()>>>,
|
||||||
}
|
}
|
||||||
|
|
@ -27,50 +22,91 @@ pub struct BackgroundRunner {
|
||||||
impl BackgroundRunner {
|
impl BackgroundRunner {
|
||||||
pub fn new(n_runners: usize, stop_signal: watch::Receiver<bool>) -> Arc<Self> {
|
pub fn new(n_runners: usize, stop_signal: watch::Receiver<bool>) -> Arc<Self> {
|
||||||
let (queue_in, queue_out) = mpsc::unbounded_channel();
|
let (queue_in, queue_out) = mpsc::unbounded_channel();
|
||||||
|
|
||||||
|
let mut workers = vec![];
|
||||||
|
let queue_out = Arc::new(tokio::sync::Mutex::new(queue_out));
|
||||||
|
|
||||||
|
for i in 0..n_runners {
|
||||||
|
let queue_out = queue_out.clone();
|
||||||
|
let stop_signal = stop_signal.clone();
|
||||||
|
|
||||||
|
workers.push(tokio::spawn(async move {
|
||||||
|
while let Some((job, cancellable)) = queue_out.lock().await.recv().await {
|
||||||
|
if cancellable && *stop_signal.borrow() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
if let Err(e) = job.await {
|
||||||
|
error!("Job failed: {}", e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
info!("Worker {} exiting", i);
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
Arc::new(Self {
|
Arc::new(Self {
|
||||||
n_runners,
|
|
||||||
stop_signal,
|
stop_signal,
|
||||||
queue_in,
|
queue_in: ArcSwapOption::new(Some(Arc::new(queue_in))),
|
||||||
queue_out: Mutex::new(queue_out),
|
workers: Mutex::new(workers),
|
||||||
job_notify: Notify::new(),
|
|
||||||
workers: Mutex::new(Vec::new()),
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn run(self: Arc<Self>) {
|
pub async fn run(self: Arc<Self>) {
|
||||||
let mut workers = self.workers.lock().unwrap();
|
|
||||||
for i in 0..self.n_runners {
|
|
||||||
workers.push(tokio::spawn(self.clone().runner(i)));
|
|
||||||
}
|
|
||||||
drop(workers);
|
|
||||||
|
|
||||||
let mut stop_signal = self.stop_signal.clone();
|
let mut stop_signal = self.stop_signal.clone();
|
||||||
while let Some(exit_now) = stop_signal.recv().await {
|
|
||||||
|
loop {
|
||||||
|
let exit_now = match stop_signal.changed().await {
|
||||||
|
Ok(()) => *stop_signal.borrow(),
|
||||||
|
Err(e) => {
|
||||||
|
error!("Watch .changed() error: {}", e);
|
||||||
|
true
|
||||||
|
}
|
||||||
|
};
|
||||||
if exit_now {
|
if exit_now {
|
||||||
let mut workers = self.workers.lock().unwrap();
|
break;
|
||||||
let workers_vec = workers.drain(..).collect::<Vec<_>>();
|
}
|
||||||
join_all(workers_vec).await;
|
}
|
||||||
return;
|
|
||||||
|
info!("Closing background job queue_in...");
|
||||||
|
drop(self.queue_in.swap(None));
|
||||||
|
|
||||||
|
info!("Waiting for all workers to terminate...");
|
||||||
|
while let Some(task) = self.workers.lock().unwrap().pop() {
|
||||||
|
if let Err(e) = task.await {
|
||||||
|
warn!("Error awaiting task: {}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn spawn<T>(&self, job: T)
|
// Spawn a task to be run in background
|
||||||
|
pub async fn spawn<T>(&self, job: T)
|
||||||
where
|
where
|
||||||
T: Future<Output = JobOutput> + Send + 'static,
|
T: Future<Output = JobOutput> + Send + 'static,
|
||||||
{
|
{
|
||||||
|
match self.queue_in.load().as_ref() {
|
||||||
|
Some(chan) => {
|
||||||
let boxed: Job = Box::pin(job);
|
let boxed: Job = Box::pin(job);
|
||||||
let _: Result<_, _> = self.queue_in.clone().send((boxed, false));
|
chan.send((boxed, false)).map_err(|_| "send error").unwrap();
|
||||||
self.job_notify.notify();
|
}
|
||||||
|
None => {
|
||||||
|
warn!("Doing background job now because we are exiting...");
|
||||||
|
if let Err(e) = job.await {
|
||||||
|
warn!("Task failed: {}", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn spawn_cancellable<T>(&self, job: T)
|
pub fn spawn_cancellable<T>(&self, job: T)
|
||||||
where
|
where
|
||||||
T: Future<Output = JobOutput> + Send + 'static,
|
T: Future<Output = JobOutput> + Send + 'static,
|
||||||
{
|
{
|
||||||
|
match self.queue_in.load().as_ref() {
|
||||||
|
Some(chan) => {
|
||||||
let boxed: Job = Box::pin(job);
|
let boxed: Job = Box::pin(job);
|
||||||
let _: Result<_, _> = self.queue_in.clone().send((boxed, true));
|
chan.send((boxed, false)).map_err(|_| "send error").unwrap();
|
||||||
self.job_notify.notify();
|
}
|
||||||
|
None => (), // drop job if we are exiting
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn spawn_worker<F, T>(&self, name: String, worker: F)
|
pub fn spawn_worker<F, T>(&self, name: String, worker: F)
|
||||||
|
|
@ -85,37 +121,4 @@ impl BackgroundRunner {
|
||||||
info!("Worker exited: {}", name);
|
info!("Worker exited: {}", name);
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn runner(self: Arc<Self>, i: usize) {
|
|
||||||
let mut stop_signal = self.stop_signal.clone();
|
|
||||||
loop {
|
|
||||||
let must_exit: bool = *stop_signal.borrow();
|
|
||||||
if let Some(job) = self.dequeue_job(must_exit) {
|
|
||||||
if let Err(e) = job.await {
|
|
||||||
error!("Job failed: {}", e)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
if must_exit {
|
|
||||||
info!("Background runner {} exiting", i);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
select! {
|
|
||||||
_ = self.job_notify.notified().fuse() => (),
|
|
||||||
_ = stop_signal.recv().fuse() => (),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn dequeue_job(&self, must_exit: bool) -> Option<Job> {
|
|
||||||
let mut queue = self.queue_out.lock().unwrap();
|
|
||||||
while let Ok((job, cancellable)) = queue.try_recv() {
|
|
||||||
if cancellable && must_exit {
|
|
||||||
continue;
|
|
||||||
} else {
|
|
||||||
return Some(job);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
None
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -8,16 +8,22 @@ use crate::data::*;
|
||||||
pub enum RPCError {
|
pub enum RPCError {
|
||||||
#[error(display = "Node is down: {:?}.", _0)]
|
#[error(display = "Node is down: {:?}.", _0)]
|
||||||
NodeDown(UUID),
|
NodeDown(UUID),
|
||||||
|
|
||||||
#[error(display = "Timeout: {}", _0)]
|
#[error(display = "Timeout: {}", _0)]
|
||||||
Timeout(#[error(source)] tokio::time::Elapsed),
|
Timeout(#[error(source)] tokio::time::error::Elapsed),
|
||||||
|
|
||||||
#[error(display = "HTTP error: {}", _0)]
|
#[error(display = "HTTP error: {}", _0)]
|
||||||
HTTP(#[error(source)] http::Error),
|
HTTP(#[error(source)] http::Error),
|
||||||
|
|
||||||
#[error(display = "Hyper error: {}", _0)]
|
#[error(display = "Hyper error: {}", _0)]
|
||||||
Hyper(#[error(source)] hyper::Error),
|
Hyper(#[error(source)] hyper::Error),
|
||||||
|
|
||||||
#[error(display = "Messagepack encode error: {}", _0)]
|
#[error(display = "Messagepack encode error: {}", _0)]
|
||||||
RMPEncode(#[error(source)] rmp_serde::encode::Error),
|
RMPEncode(#[error(source)] rmp_serde::encode::Error),
|
||||||
|
|
||||||
#[error(display = "Messagepack decode error: {}", _0)]
|
#[error(display = "Messagepack decode error: {}", _0)]
|
||||||
RMPDecode(#[error(source)] rmp_serde::decode::Error),
|
RMPDecode(#[error(source)] rmp_serde::decode::Error),
|
||||||
|
|
||||||
#[error(display = "Too many errors: {:?}", _0)]
|
#[error(display = "Too many errors: {:?}", _0)]
|
||||||
TooManyErrors(Vec<String>),
|
TooManyErrors(Vec<String>),
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ err-derive = "0.2.3"
|
||||||
log = "0.4"
|
log = "0.4"
|
||||||
futures = "0.3"
|
futures = "0.3"
|
||||||
http = "0.2"
|
http = "0.2"
|
||||||
hyper = "0.13"
|
hyper = "0.14"
|
||||||
percent-encoding = "2.1.0"
|
percent-encoding = "2.1.0"
|
||||||
roxmltree = "0.11"
|
roxmltree = "0.11"
|
||||||
idna = "0.2"
|
idna = "0.2"
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue