Update sled & try to debug deadlock (but its in sled...)

This commit is contained in:
Alex 2021-02-23 21:27:28 +01:00
parent bf25c95fe2
commit 20e6e9fa20
12 changed files with 207 additions and 34 deletions

164
Cargo.lock generated
View file

@ -220,10 +220,24 @@ checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace"
dependencies = [ dependencies = [
"autocfg", "autocfg",
"cfg-if 0.1.10", "cfg-if 0.1.10",
"crossbeam-utils", "crossbeam-utils 0.7.2",
"lazy_static", "lazy_static",
"maybe-uninit", "maybe-uninit",
"memoffset", "memoffset 0.5.6",
"scopeguard",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d60ab4a8dba064f2fbb5aa270c28da5cf4bbd0e72dae1140a6b0353a779dbe00"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils 0.8.2",
"lazy_static",
"loom",
"memoffset 0.6.1",
"scopeguard", "scopeguard",
] ]
@ -238,6 +252,18 @@ dependencies = [
"lazy_static", "lazy_static",
] ]
[[package]]
name = "crossbeam-utils"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bae8f328835f8f5a6ceb6a7842a7f2d0c03692adb5c889347235d59194731fe3"
dependencies = [
"autocfg",
"cfg-if 1.0.0",
"lazy_static",
"loom",
]
[[package]] [[package]]
name = "crypto-mac" name = "crypto-mac"
version = "0.7.0" version = "0.7.0"
@ -503,7 +529,8 @@ dependencies = [
"rmp-serde", "rmp-serde",
"serde", "serde",
"sha2", "sha2",
"sled", "sled 0.31.0",
"sled 0.34.6",
"structopt", "structopt",
"tokio", "tokio",
"toml", "toml",
@ -559,7 +586,7 @@ dependencies = [
"serde", "serde",
"serde_bytes", "serde_bytes",
"sha2", "sha2",
"sled", "sled 0.34.6",
"tokio", "tokio",
] ]
@ -584,7 +611,7 @@ dependencies = [
"serde", "serde",
"serde_bytes", "serde_bytes",
"sha2", "sha2",
"sled", "sled 0.31.0",
"tokio", "tokio",
] ]
@ -660,7 +687,7 @@ dependencies = [
"rmp-serde", "rmp-serde",
"serde", "serde",
"serde_bytes", "serde_bytes",
"sled", "sled 0.34.6",
"tokio", "tokio",
] ]
@ -684,7 +711,7 @@ dependencies = [
"rmp-serde", "rmp-serde",
"serde", "serde",
"serde_bytes", "serde_bytes",
"sled", "sled 0.31.0",
"tokio", "tokio",
] ]
@ -708,7 +735,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"sha2", "sha2",
"sled", "sled 0.34.6",
"tokio", "tokio",
"toml", "toml",
"webpki", "webpki",
@ -734,7 +761,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"sha2", "sha2",
"sled", "sled 0.31.0",
"tokio", "tokio",
"toml", "toml",
"webpki", "webpki",
@ -764,6 +791,19 @@ version = "0.3.55"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2"
[[package]]
name = "generator"
version = "0.6.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9fed24fd1e18827652b4d55652899a1e9da8e54d91624dc3437a5bc3a9f9a9c"
dependencies = [
"cc",
"libc",
"log",
"rustversion",
"winapi 0.3.9",
]
[[package]] [[package]]
name = "generic-array" name = "generic-array"
version = "0.12.3" version = "0.12.3"
@ -983,6 +1023,15 @@ dependencies = [
"hashbrown", "hashbrown",
] ]
[[package]]
name = "instant"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec"
dependencies = [
"cfg-if 1.0.0",
]
[[package]] [[package]]
name = "iovec" name = "iovec"
version = "0.1.4" version = "0.1.4"
@ -1044,6 +1093,15 @@ dependencies = [
"scopeguard", "scopeguard",
] ]
[[package]]
name = "lock_api"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312"
dependencies = [
"scopeguard",
]
[[package]] [[package]]
name = "log" name = "log"
version = "0.4.13" version = "0.4.13"
@ -1053,6 +1111,17 @@ dependencies = [
"cfg-if 0.1.10", "cfg-if 0.1.10",
] ]
[[package]]
name = "loom"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d44c73b4636e497b4917eb21c33539efa3816741a2d3ff26c6316f1b529481a4"
dependencies = [
"cfg-if 1.0.0",
"generator",
"scoped-tls",
]
[[package]] [[package]]
name = "matches" name = "matches"
version = "0.1.8" version = "0.1.8"
@ -1091,6 +1160,15 @@ dependencies = [
"autocfg", "autocfg",
] ]
[[package]]
name = "memoffset"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "mio" name = "mio"
version = "0.6.23" version = "0.6.23"
@ -1209,8 +1287,19 @@ version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e"
dependencies = [ dependencies = [
"lock_api", "lock_api 0.3.4",
"parking_lot_core", "parking_lot_core 0.7.2",
]
[[package]]
name = "parking_lot"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb"
dependencies = [
"instant",
"lock_api 0.4.2",
"parking_lot_core 0.8.3",
] ]
[[package]] [[package]]
@ -1222,7 +1311,21 @@ dependencies = [
"cfg-if 0.1.10", "cfg-if 0.1.10",
"cloudabi", "cloudabi",
"libc", "libc",
"redox_syscall", "redox_syscall 0.1.57",
"smallvec",
"winapi 0.3.9",
]
[[package]]
name = "parking_lot_core"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018"
dependencies = [
"cfg-if 1.0.0",
"instant",
"libc",
"redox_syscall 0.2.5",
"smallvec", "smallvec",
"winapi 0.3.9", "winapi 0.3.9",
] ]
@ -1451,6 +1554,15 @@ version = "0.1.57"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
[[package]]
name = "redox_syscall"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9"
dependencies = [
"bitflags",
]
[[package]] [[package]]
name = "regex" name = "regex"
version = "1.4.3" version = "1.4.3"
@ -1539,6 +1651,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "scoped-tls"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
[[package]] [[package]]
name = "scopeguard" name = "scopeguard"
version = "1.1.0" version = "1.1.0"
@ -1635,13 +1753,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fb6824dde66ad33bf20c6e8476f5b82b871bc8bc3c129a10ea2f7dae5060fa3" checksum = "8fb6824dde66ad33bf20c6e8476f5b82b871bc8bc3c129a10ea2f7dae5060fa3"
dependencies = [ dependencies = [
"crc32fast", "crc32fast",
"crossbeam-epoch", "crossbeam-epoch 0.8.2",
"crossbeam-utils", "crossbeam-utils 0.7.2",
"fs2", "fs2",
"fxhash", "fxhash",
"libc", "libc",
"log", "log",
"parking_lot", "parking_lot 0.10.2",
]
[[package]]
name = "sled"
version = "0.34.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
dependencies = [
"crc32fast",
"crossbeam-epoch 0.9.2",
"crossbeam-utils 0.8.2",
"fs2",
"fxhash",
"libc",
"log",
"parking_lot 0.11.1",
] ]
[[package]] [[package]]

View file

@ -28,7 +28,8 @@ sha2 = "0.8"
log = "0.4" log = "0.4"
pretty_env_logger = "0.4" pretty_env_logger = "0.4"
sled = "0.31" sled = "0.34"
old_sled = { package = "sled", version = "0.31" }
structopt = { version = "0.3", default-features = false } structopt = { version = "0.3", default-features = false }
toml = "0.5" toml = "0.5"

View file

@ -40,7 +40,28 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> {
info!("Opening database..."); info!("Opening database...");
let mut db_path = config.metadata_dir.clone(); let mut db_path = config.metadata_dir.clone();
db_path.push("db"); db_path.push("db");
let db = sled::open(db_path).expect("Unable to open DB"); let db = match sled::open(&db_path) {
Ok(db) => db,
Err(e) => {
warn!("Old DB could not be openned ({}), attempting migration.", e);
let old = old_sled::open(&db_path).expect("Unable to open old DB for migration");
let mut new_path = config.metadata_dir.clone();
new_path.push("db2");
let new = sled::open(&new_path).expect("Unable to open new DB for migration");
new.import(old.export());
if old.checksum().expect("unable to compute old db checksum")
!= new.checksum().expect("unable to compute new db checksum")
{
panic!("db checksums don't match after migration");
}
drop(new);
drop(old);
std::fs::remove_dir_all(&db_path).expect("Cannot remove old DB folder");
std::fs::rename(new_path, &db_path)
.expect("Cannot move new DB folder to correct place");
sled::open(db_path).expect("Unable to open new DB after migration")
}
};
info!("Initialize RPC server..."); info!("Initialize RPC server...");
let mut rpc_server = RpcServer::new(config.rpc_bind_addr.clone(), config.rpc_tls.clone()); let mut rpc_server = RpcServer::new(config.rpc_bind_addr.clone(), config.rpc_tls.clone());

View file

@ -25,7 +25,7 @@ sha2 = "0.8"
arc-swap = "0.4" arc-swap = "0.4"
log = "0.4" log = "0.4"
sled = "0.31" sled = "0.34"
rmp-serde = "0.14.3" rmp-serde = "0.14.3"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }

View file

@ -3,7 +3,6 @@ use serde::{Deserialize, Serialize};
use garage_table::crdt::CRDT; use garage_table::crdt::CRDT;
use garage_table::*; use garage_table::*;
use crate::key_table::PermissionSet; use crate::key_table::PermissionSet;
// We import the same file but in its version 0.1.0. // We import the same file but in its version 0.1.0.

View file

@ -310,7 +310,9 @@ impl RpcHttpClient {
ClientMethod::HTTPS(client) => client.request(req).fuse(), ClientMethod::HTTPS(client) => client.request(req).fuse(),
}; };
trace!("({}) Acquiring request_limiter slot...", path);
let slot = self.request_limiter.acquire().await; let slot = self.request_limiter.acquire().await;
trace!("({}) Got slot, doing request to {}...", path, to_addr);
let resp = tokio::time::timeout(timeout, resp_fut) let resp = tokio::time::timeout(timeout, resp_fut)
.await .await
.map_err(|e| { .map_err(|e| {
@ -330,6 +332,7 @@ impl RpcHttpClient {
})?; })?;
let status = resp.status(); let status = resp.status();
trace!("({}) Request returned, got status {}", path, status);
let body = hyper::body::to_bytes(resp.into_body()).await?; let body = hyper::body::to_bytes(resp.into_body()).await?;
drop(slot); drop(slot);

View file

@ -48,6 +48,12 @@ where
let begin_time = Instant::now(); let begin_time = Instant::now();
let whole_body = hyper::body::to_bytes(req.into_body()).await?; let whole_body = hyper::body::to_bytes(req.into_body()).await?;
let msg = rmp_serde::decode::from_read::<_, M>(whole_body.into_buf())?; let msg = rmp_serde::decode::from_read::<_, M>(whole_body.into_buf())?;
trace!(
"Request message: {}",
serde_json::to_string(&msg).unwrap_or("<json error>".into())
);
match handler(msg, sockaddr).await { match handler(msg, sockaddr).await {
Ok(resp) => { Ok(resp) => {
let resp_bytes = rmp_to_vec_all_named::<Result<M, String>>(&Ok(resp))?; let resp_bytes = rmp_to_vec_all_named::<Result<M, String>>(&Ok(resp))?;
@ -112,7 +118,8 @@ impl RpcServer {
return Ok(bad_request); return Ok(bad_request);
} }
let path = &req.uri().path()[1..]; let path = &req.uri().path()[1..].to_string();
let handler = match self.handlers.get(path) { let handler = match self.handlers.get(path) {
Some(h) => h, Some(h) => h,
None => { None => {
@ -122,6 +129,8 @@ impl RpcServer {
} }
}; };
trace!("({}) Handling request", path);
let resp_waiter = tokio::spawn(handler(req, addr)); let resp_waiter = tokio::spawn(handler(req, addr));
match resp_waiter.await { match resp_waiter.await {
Err(err) => { Err(err) => {
@ -131,11 +140,15 @@ impl RpcServer {
Ok(ise) Ok(ise)
} }
Ok(Err(err)) => { Ok(Err(err)) => {
trace!("({}) Request handler failed: {}", path, err);
let mut bad_request = Response::new(Body::from(format!("{}", err))); let mut bad_request = Response::new(Body::from(format!("{}", err)));
*bad_request.status_mut() = StatusCode::BAD_REQUEST; *bad_request.status_mut() = StatusCode::BAD_REQUEST;
Ok(bad_request) Ok(bad_request)
} }
Ok(Ok(resp)) => Ok(resp), Ok(Ok(resp)) => {
trace!("({}) Request handler succeeded", path);
Ok(resp)
}
} }
} }

View file

@ -23,7 +23,7 @@ arc-swap = "0.4"
log = "0.4" log = "0.4"
hexdump = "0.1" hexdump = "0.1"
sled = "0.31" sled = "0.34"
rmp-serde = "0.14.3" rmp-serde = "0.14.3"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }

View file

@ -394,7 +394,7 @@ where
Some(prev_bytes) => { Some(prev_bytes) => {
let old_entry = self let old_entry = self
.decode_entry(&prev_bytes) .decode_entry(&prev_bytes)
.map_err(sled::ConflictableTransactionError::Abort)?; .map_err(sled::transaction::ConflictableTransactionError::Abort)?;
let mut new_entry = old_entry.clone(); let mut new_entry = old_entry.clone();
new_entry.merge(&update); new_entry.merge(&update);
(Some(old_entry), new_entry) (Some(old_entry), new_entry)
@ -404,7 +404,7 @@ where
let new_bytes = rmp_to_vec_all_named(&new_entry) let new_bytes = rmp_to_vec_all_named(&new_entry)
.map_err(Error::RMPEncode) .map_err(Error::RMPEncode)
.map_err(sled::ConflictableTransactionError::Abort)?; .map_err(sled::transaction::ConflictableTransactionError::Abort)?;
db.insert(tree_key.clone(), new_bytes)?; db.insert(tree_key.clone(), new_bytes)?;
Ok((old_entry, new_entry)) Ok((old_entry, new_entry))
})?; })?;
@ -429,11 +429,7 @@ where
Ok(()) Ok(())
} }
pub(crate) fn delete_if_equal( pub(crate) fn delete_if_equal(self: &Arc<Self>, k: &[u8], v: &[u8]) -> Result<bool, Error> {
self: &Arc<Self>,
k: &[u8],
v: &[u8],
) -> Result<bool, Error> {
let removed = self.store.transaction(|txn| { let removed = self.store.transaction(|txn| {
if let Some(cur_v) = self.store.get(k)? { if let Some(cur_v) = self.store.get(k)? {
if cur_v == v { if cur_v == v {

View file

@ -385,6 +385,7 @@ where
must_exit: &mut watch::Receiver<bool>, must_exit: &mut watch::Receiver<bool>,
) -> Result<RangeChecksum, Error> { ) -> Result<RangeChecksum, Error> {
assert!(range.level != 0); assert!(range.level != 0);
trace!("Call range_checksum {:?}", range);
if range.level == 1 { if range.level == 1 {
let mut children = vec![]; let mut children = vec![];
@ -400,6 +401,7 @@ where
.iter() .iter()
.all(|x| *x == 0u8) .all(|x| *x == 0u8)
{ {
trace!("range_checksum {:?} returning {} items", range, children.len());
return Ok(RangeChecksum { return Ok(RangeChecksum {
bounds: range.clone(), bounds: range.clone(),
children, children,
@ -414,6 +416,7 @@ where
}; };
children.push((item_range, blake2sum(&value[..]))); children.push((item_range, blake2sum(&value[..])));
} }
trace!("range_checksum {:?} returning {} items", range, children.len());
Ok(RangeChecksum { Ok(RangeChecksum {
bounds: range.clone(), bounds: range.clone(),
children, children,
@ -439,6 +442,7 @@ where
} }
if sub_ck.found_limit.is_none() || sub_ck.hash.is_none() { if sub_ck.found_limit.is_none() || sub_ck.hash.is_none() {
trace!("range_checksum {:?} returning {} items", range, children.len());
return Ok(RangeChecksum { return Ok(RangeChecksum {
bounds: range.clone(), bounds: range.clone(),
children, children,
@ -453,6 +457,7 @@ where
.iter() .iter()
.all(|x| *x == 0u8) .all(|x| *x == 0u8)
{ {
trace!("range_checksum {:?} returning {} items", range, children.len());
return Ok(RangeChecksum { return Ok(RangeChecksum {
bounds: range.clone(), bounds: range.clone(),
children, children,
@ -463,6 +468,7 @@ where
sub_range.begin = found_limit; sub_range.begin = found_limit;
} }
trace!("range_checksum {:?} exiting due to must_exit", range);
Err(Error::Message(format!("Exiting."))) Err(Error::Message(format!("Exiting.")))
} }
} }

View file

@ -21,7 +21,7 @@ err-derive = "0.2.3"
log = "0.4" log = "0.4"
fasthash = "0.4" fasthash = "0.4"
sled = "0.31" sled = "0.34"
toml = "0.5" toml = "0.5"
rmp-serde = "0.14.3" rmp-serde = "0.14.3"

View file

@ -73,11 +73,11 @@ pub enum Error {
Message(String), Message(String),
} }
impl From<sled::TransactionError<Error>> for Error { impl From<sled::transaction::TransactionError<Error>> for Error {
fn from(e: sled::TransactionError<Error>) -> Error { fn from(e: sled::transaction::TransactionError<Error>) -> Error {
match e { match e {
sled::TransactionError::Abort(x) => x, sled::transaction::TransactionError::Abort(x) => x,
sled::TransactionError::Storage(x) => Error::Sled(x), sled::transaction::TransactionError::Storage(x) => Error::Sled(x),
} }
} }
} }