From 20e6e9fa2035ac12946bf0dcd5b8049090955bde Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Tue, 23 Feb 2021 21:27:28 +0100 Subject: [PATCH] Update sled & try to debug deadlock (but its in sled...) --- Cargo.lock | 164 ++++++++++++++++++++++++++++++++++---- src/garage/Cargo.toml | 3 +- src/garage/server.rs | 23 +++++- src/model/Cargo.toml | 2 +- src/model/bucket_table.rs | 1 - src/rpc/rpc_client.rs | 3 + src/rpc/rpc_server.rs | 17 +++- src/table/Cargo.toml | 2 +- src/table/table.rs | 10 +-- src/table/table_sync.rs | 6 ++ src/util/Cargo.toml | 2 +- src/util/error.rs | 8 +- 12 files changed, 207 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 252ca27..a34dea2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -220,10 +220,24 @@ checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg", "cfg-if 0.1.10", - "crossbeam-utils", + "crossbeam-utils 0.7.2", "lazy_static", "maybe-uninit", - "memoffset", + "memoffset 0.5.6", + "scopeguard", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60ab4a8dba064f2fbb5aa270c28da5cf4bbd0e72dae1140a6b0353a779dbe00" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils 0.8.2", + "lazy_static", + "loom", + "memoffset 0.6.1", "scopeguard", ] @@ -238,6 +252,18 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae8f328835f8f5a6ceb6a7842a7f2d0c03692adb5c889347235d59194731fe3" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "lazy_static", + "loom", +] + [[package]] name = "crypto-mac" version = "0.7.0" @@ -503,7 +529,8 @@ dependencies = [ "rmp-serde", "serde", "sha2", - "sled", + "sled 0.31.0", + "sled 0.34.6", "structopt", "tokio", "toml", @@ -559,7 +586,7 @@ dependencies = [ "serde", "serde_bytes", "sha2", - "sled", + "sled 0.34.6", "tokio", ] @@ -584,7 +611,7 @@ dependencies = [ "serde", "serde_bytes", "sha2", - "sled", + "sled 0.31.0", "tokio", ] @@ -660,7 +687,7 @@ dependencies = [ "rmp-serde", "serde", "serde_bytes", - "sled", + "sled 0.34.6", "tokio", ] @@ -684,7 +711,7 @@ dependencies = [ "rmp-serde", "serde", "serde_bytes", - "sled", + "sled 0.31.0", "tokio", ] @@ -708,7 +735,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "sled", + "sled 0.34.6", "tokio", "toml", "webpki", @@ -734,7 +761,7 @@ dependencies = [ "serde", "serde_json", "sha2", - "sled", + "sled 0.31.0", "tokio", "toml", "webpki", @@ -764,6 +791,19 @@ version = "0.3.55" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f5f3913fa0bfe7ee1fd8248b6b9f42a5af4b9d65ec2dd2c3c26132b950ecfc2" +[[package]] +name = "generator" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fed24fd1e18827652b4d55652899a1e9da8e54d91624dc3437a5bc3a9f9a9c" +dependencies = [ + "cc", + "libc", + "log", + "rustversion", + "winapi 0.3.9", +] + [[package]] name = "generic-array" version = "0.12.3" @@ -983,6 +1023,15 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "instant" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "iovec" version = "0.1.4" @@ -1044,6 +1093,15 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lock_api" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd96ffd135b2fd7b973ac026d28085defbe8983df057ced3eb4f2130b0831312" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.13" @@ -1053,6 +1111,17 @@ dependencies = [ "cfg-if 0.1.10", ] +[[package]] +name = "loom" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d44c73b4636e497b4917eb21c33539efa3816741a2d3ff26c6316f1b529481a4" +dependencies = [ + "cfg-if 1.0.0", + "generator", + "scoped-tls", +] + [[package]] name = "matches" version = "0.1.8" @@ -1091,6 +1160,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" +dependencies = [ + "autocfg", +] + [[package]] name = "mio" version = "0.6.23" @@ -1209,8 +1287,19 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3a704eb390aafdc107b0e392f56a82b668e3a71366993b5340f5833fd62505e" dependencies = [ - "lock_api", - "parking_lot_core", + "lock_api 0.3.4", + "parking_lot_core 0.7.2", +] + +[[package]] +name = "parking_lot" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" +dependencies = [ + "instant", + "lock_api 0.4.2", + "parking_lot_core 0.8.3", ] [[package]] @@ -1222,7 +1311,21 @@ dependencies = [ "cfg-if 0.1.10", "cloudabi", "libc", - "redox_syscall", + "redox_syscall 0.1.57", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "parking_lot_core" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" +dependencies = [ + "cfg-if 1.0.0", + "instant", + "libc", + "redox_syscall 0.2.5", "smallvec", "winapi 0.3.9", ] @@ -1451,6 +1554,15 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "redox_syscall" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" +dependencies = [ + "bitflags", +] + [[package]] name = "regex" version = "1.4.3" @@ -1539,6 +1651,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "scoped-tls" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1635,13 +1753,29 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8fb6824dde66ad33bf20c6e8476f5b82b871bc8bc3c129a10ea2f7dae5060fa3" dependencies = [ "crc32fast", - "crossbeam-epoch", - "crossbeam-utils", + "crossbeam-epoch 0.8.2", + "crossbeam-utils 0.7.2", "fs2", "fxhash", "libc", "log", - "parking_lot", + "parking_lot 0.10.2", +] + +[[package]] +name = "sled" +version = "0.34.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc" +dependencies = [ + "crc32fast", + "crossbeam-epoch 0.9.2", + "crossbeam-utils 0.8.2", + "fs2", + "fxhash", + "libc", + "log", + "parking_lot 0.11.1", ] [[package]] diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 254645c..03bc472 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -28,7 +28,8 @@ sha2 = "0.8" log = "0.4" pretty_env_logger = "0.4" -sled = "0.31" +sled = "0.34" +old_sled = { package = "sled", version = "0.31" } structopt = { version = "0.3", default-features = false } toml = "0.5" diff --git a/src/garage/server.rs b/src/garage/server.rs index ec78c06..2e109f8 100644 --- a/src/garage/server.rs +++ b/src/garage/server.rs @@ -40,7 +40,28 @@ pub async fn run_server(config_file: PathBuf) -> Result<(), Error> { info!("Opening database..."); let mut db_path = config.metadata_dir.clone(); db_path.push("db"); - let db = sled::open(db_path).expect("Unable to open DB"); + let db = match sled::open(&db_path) { + Ok(db) => db, + Err(e) => { + warn!("Old DB could not be openned ({}), attempting migration.", e); + let old = old_sled::open(&db_path).expect("Unable to open old DB for migration"); + let mut new_path = config.metadata_dir.clone(); + new_path.push("db2"); + let new = sled::open(&new_path).expect("Unable to open new DB for migration"); + new.import(old.export()); + if old.checksum().expect("unable to compute old db checksum") + != new.checksum().expect("unable to compute new db checksum") + { + panic!("db checksums don't match after migration"); + } + drop(new); + drop(old); + std::fs::remove_dir_all(&db_path).expect("Cannot remove old DB folder"); + std::fs::rename(new_path, &db_path) + .expect("Cannot move new DB folder to correct place"); + sled::open(db_path).expect("Unable to open new DB after migration") + } + }; info!("Initialize RPC server..."); let mut rpc_server = RpcServer::new(config.rpc_bind_addr.clone(), config.rpc_tls.clone()); diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 2c779d0..48b75d2 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -25,7 +25,7 @@ sha2 = "0.8" arc-swap = "0.4" log = "0.4" -sled = "0.31" +sled = "0.34" rmp-serde = "0.14.3" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } diff --git a/src/model/bucket_table.rs b/src/model/bucket_table.rs index af42d55..2878aa3 100644 --- a/src/model/bucket_table.rs +++ b/src/model/bucket_table.rs @@ -3,7 +3,6 @@ use serde::{Deserialize, Serialize}; use garage_table::crdt::CRDT; use garage_table::*; - use crate::key_table::PermissionSet; // We import the same file but in its version 0.1.0. diff --git a/src/rpc/rpc_client.rs b/src/rpc/rpc_client.rs index 5183bb4..7038439 100644 --- a/src/rpc/rpc_client.rs +++ b/src/rpc/rpc_client.rs @@ -310,7 +310,9 @@ impl RpcHttpClient { ClientMethod::HTTPS(client) => client.request(req).fuse(), }; + trace!("({}) Acquiring request_limiter slot...", path); let slot = self.request_limiter.acquire().await; + trace!("({}) Got slot, doing request to {}...", path, to_addr); let resp = tokio::time::timeout(timeout, resp_fut) .await .map_err(|e| { @@ -330,6 +332,7 @@ impl RpcHttpClient { })?; let status = resp.status(); + trace!("({}) Request returned, got status {}", path, status); let body = hyper::body::to_bytes(resp.into_body()).await?; drop(slot); diff --git a/src/rpc/rpc_server.rs b/src/rpc/rpc_server.rs index 4113f15..1c6bc8d 100644 --- a/src/rpc/rpc_server.rs +++ b/src/rpc/rpc_server.rs @@ -48,6 +48,12 @@ where let begin_time = Instant::now(); let whole_body = hyper::body::to_bytes(req.into_body()).await?; let msg = rmp_serde::decode::from_read::<_, M>(whole_body.into_buf())?; + + trace!( + "Request message: {}", + serde_json::to_string(&msg).unwrap_or("".into()) + ); + match handler(msg, sockaddr).await { Ok(resp) => { let resp_bytes = rmp_to_vec_all_named::>(&Ok(resp))?; @@ -112,7 +118,8 @@ impl RpcServer { return Ok(bad_request); } - let path = &req.uri().path()[1..]; + let path = &req.uri().path()[1..].to_string(); + let handler = match self.handlers.get(path) { Some(h) => h, None => { @@ -122,6 +129,8 @@ impl RpcServer { } }; + trace!("({}) Handling request", path); + let resp_waiter = tokio::spawn(handler(req, addr)); match resp_waiter.await { Err(err) => { @@ -131,11 +140,15 @@ impl RpcServer { Ok(ise) } Ok(Err(err)) => { + trace!("({}) Request handler failed: {}", path, err); let mut bad_request = Response::new(Body::from(format!("{}", err))); *bad_request.status_mut() = StatusCode::BAD_REQUEST; Ok(bad_request) } - Ok(Ok(resp)) => Ok(resp), + Ok(Ok(resp)) => { + trace!("({}) Request handler succeeded", path); + Ok(resp) + } } } diff --git a/src/table/Cargo.toml b/src/table/Cargo.toml index 483c386..6485a54 100644 --- a/src/table/Cargo.toml +++ b/src/table/Cargo.toml @@ -23,7 +23,7 @@ arc-swap = "0.4" log = "0.4" hexdump = "0.1" -sled = "0.31" +sled = "0.34" rmp-serde = "0.14.3" serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } diff --git a/src/table/table.rs b/src/table/table.rs index 737ed58..8389c29 100644 --- a/src/table/table.rs +++ b/src/table/table.rs @@ -394,7 +394,7 @@ where Some(prev_bytes) => { let old_entry = self .decode_entry(&prev_bytes) - .map_err(sled::ConflictableTransactionError::Abort)?; + .map_err(sled::transaction::ConflictableTransactionError::Abort)?; let mut new_entry = old_entry.clone(); new_entry.merge(&update); (Some(old_entry), new_entry) @@ -404,7 +404,7 @@ where let new_bytes = rmp_to_vec_all_named(&new_entry) .map_err(Error::RMPEncode) - .map_err(sled::ConflictableTransactionError::Abort)?; + .map_err(sled::transaction::ConflictableTransactionError::Abort)?; db.insert(tree_key.clone(), new_bytes)?; Ok((old_entry, new_entry)) })?; @@ -429,11 +429,7 @@ where Ok(()) } - pub(crate) fn delete_if_equal( - self: &Arc, - k: &[u8], - v: &[u8], - ) -> Result { + pub(crate) fn delete_if_equal(self: &Arc, k: &[u8], v: &[u8]) -> Result { let removed = self.store.transaction(|txn| { if let Some(cur_v) = self.store.get(k)? { if cur_v == v { diff --git a/src/table/table_sync.rs b/src/table/table_sync.rs index b81dad8..5839127 100644 --- a/src/table/table_sync.rs +++ b/src/table/table_sync.rs @@ -385,6 +385,7 @@ where must_exit: &mut watch::Receiver, ) -> Result { assert!(range.level != 0); + trace!("Call range_checksum {:?}", range); if range.level == 1 { let mut children = vec![]; @@ -400,6 +401,7 @@ where .iter() .all(|x| *x == 0u8) { + trace!("range_checksum {:?} returning {} items", range, children.len()); return Ok(RangeChecksum { bounds: range.clone(), children, @@ -414,6 +416,7 @@ where }; children.push((item_range, blake2sum(&value[..]))); } + trace!("range_checksum {:?} returning {} items", range, children.len()); Ok(RangeChecksum { bounds: range.clone(), children, @@ -439,6 +442,7 @@ where } if sub_ck.found_limit.is_none() || sub_ck.hash.is_none() { + trace!("range_checksum {:?} returning {} items", range, children.len()); return Ok(RangeChecksum { bounds: range.clone(), children, @@ -453,6 +457,7 @@ where .iter() .all(|x| *x == 0u8) { + trace!("range_checksum {:?} returning {} items", range, children.len()); return Ok(RangeChecksum { bounds: range.clone(), children, @@ -463,6 +468,7 @@ where sub_range.begin = found_limit; } + trace!("range_checksum {:?} exiting due to must_exit", range); Err(Error::Message(format!("Exiting."))) } } diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 9311584..35130c9 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -21,7 +21,7 @@ err-derive = "0.2.3" log = "0.4" fasthash = "0.4" -sled = "0.31" +sled = "0.34" toml = "0.5" rmp-serde = "0.14.3" diff --git a/src/util/error.rs b/src/util/error.rs index e5dcf65..dbf71ac 100644 --- a/src/util/error.rs +++ b/src/util/error.rs @@ -73,11 +73,11 @@ pub enum Error { Message(String), } -impl From> for Error { - fn from(e: sled::TransactionError) -> Error { +impl From> for Error { + fn from(e: sled::transaction::TransactionError) -> Error { match e { - sled::TransactionError::Abort(x) => x, - sled::TransactionError::Storage(x) => Error::Sled(x), + sled::transaction::TransactionError::Abort(x) => x, + sled::transaction::TransactionError::Storage(x) => Error::Sled(x), } } }