From 51b9731a086b4e158cbfa2127bcbfd6cb6274578 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 11 Sep 2023 18:03:20 +0200 Subject: [PATCH 1/2] make lmdb's map_size configurable (fix #628) --- Cargo.lock | 1 + Cargo.nix | 3 ++- doc/book/reference-manual/configuration.md | 9 +++++++++ src/model/Cargo.toml | 1 + src/model/garage.rs | 14 +++++++++++++- src/util/config.rs | 4 ++++ 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d14701cf0..eab8e8530 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1340,6 +1340,7 @@ dependencies = [ "async-trait", "base64 0.21.3", "blake2", + "bytesize", "err-derive", "futures", "futures-util", diff --git a/Cargo.nix b/Cargo.nix index 9e458f888..600222091 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -33,7 +33,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "b958f9aca0ee3fb1f7b52b15508132d0a96480a7f43f83e0da6609c0fe1812ef"; + nixifiedLockHash = "c5e95ea3fbf4a23e07fe76a8c8886e4eb4a7c95b2d9ca8fa22fa4d8792b4d29f"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -1911,6 +1911,7 @@ in async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out; blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out; + bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index b916bb61e..c3b8ca002 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -17,6 +17,7 @@ block_size = 1048576 sled_cache_capacity = 134217728 sled_flush_every_ms = 2000 +lmdb_map_size = "10T" replication_mode = "3" @@ -160,6 +161,14 @@ Increase this if sled is thrashing your SSD, at the risk of losing more data in of a power outage (though this should not matter much as data is replicated on other nodes). The default value, 2000ms, should be appropriate for most use cases. +### `lmdb_map_size` + +This parameters can be used to set the map size used by LMDB, +which is the size of the virtual memory region used for mapping the database file. +The value of this parameter is the maximum size the metadata database can take. +This value is not bound by the physical RAM size of the machine running Garage. +If not specified, it defaults to 1GiB on 32-bit machines and 1TiB on 64-bit machines. + ### `replication_mode` Garage supports the following replication modes: diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index 1d3600a60..caf7d1b06 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -23,6 +23,7 @@ garage_util.workspace = true async-trait = "0.1.7" arc-swap = "1.0" blake2 = "0.10" +bytesize = "1.2" err-derive = "0.3" hex = "0.4" base64 = "0.21" diff --git a/src/model/garage.rs b/src/model/garage.rs index 3daa1b334..8fea6a2c7 100644 --- a/src/model/garage.rs +++ b/src/model/garage.rs @@ -121,11 +121,22 @@ impl Garage { // ---- LMDB DB ---- #[cfg(feature = "lmdb")] "lmdb" | "heed" => { + use std::convert::TryInto; db_path.push("db.lmdb"); info!("Opening LMDB database at: {}", db_path.display()); std::fs::create_dir_all(&db_path) .ok_or_message("Unable to create LMDB data directory")?; - let map_size = garage_db::lmdb_adapter::recommended_map_size(); + let map_size = match &config.lmdb_map_size { + None => garage_db::lmdb_adapter::recommended_map_size(), + Some(v) => { + let v: usize = v + .parse::() + .ok() + .and_then(|x| x.as_u64().try_into().ok()) + .ok_or_message("invalid value for `lmdb_map_size`")?; + v - (v % 4096) + } + }; use db::lmdb_adapter::heed; let mut env_builder = heed::EnvOpenOptions::new(); @@ -142,6 +153,7 @@ impl Garage { "OutOfMemory error while trying to open LMDB database. This can happen \ if your operating system is not allowing you to use sufficient virtual \ memory address space. Please check that no limit is set (ulimit -v). \ + You may also try to set a smaller `lmdb_map_size` configuration parameter. \ On 32-bit machines, you should probably switch to another database engine.".into())) } x => x.ok_or_message("Unable to open LMDB DB")?, diff --git a/src/util/config.rs b/src/util/config.rs index 1da95b2f1..070bd83eb 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -72,6 +72,10 @@ pub struct Config { #[serde(default = "default_sled_flush_every_ms")] pub sled_flush_every_ms: u64, + /// LMDB map size + #[serde(default)] + pub lmdb_map_size: Option, + // -- APIs /// Configuration for S3 api pub s3_api: S3ApiConfig, From f8b3883611578713ecb8bcacaf24ca8029e7b739 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Mon, 11 Sep 2023 18:34:59 +0200 Subject: [PATCH 2/2] config: make block_size and sled_cache_capacity expressable as strings --- Cargo.lock | 2 +- Cargo.nix | 4 +- doc/book/reference-manual/configuration.md | 8 +-- src/model/Cargo.toml | 1 - src/model/garage.rs | 16 ++---- src/util/Cargo.toml | 1 + src/util/config.rs | 65 +++++++++++++++++++--- 7 files changed, 69 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eab8e8530..837ce67d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1340,7 +1340,6 @@ dependencies = [ "async-trait", "base64 0.21.3", "blake2", - "bytesize", "err-derive", "futures", "futures-util", @@ -1422,6 +1421,7 @@ dependencies = [ "async-trait", "blake2", "bytes", + "bytesize", "chrono", "digest", "err-derive", diff --git a/Cargo.nix b/Cargo.nix index 600222091..e57dd49fe 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -33,7 +33,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "c5e95ea3fbf4a23e07fe76a8c8886e4eb4a7c95b2d9ca8fa22fa4d8792b4d29f"; + nixifiedLockHash = "3e3f41f614ab470ecb4b06c670cd6a84c443d799d01f1d48f1d251872099c468"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -1911,7 +1911,6 @@ in async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out; blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out; - bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out; futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out; @@ -2015,6 +2014,7 @@ in async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out; blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out; bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out; + bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out; chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out; digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out; err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out; diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index c3b8ca002..08c013f77 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -15,9 +15,9 @@ db_engine = "lmdb" block_size = 1048576 -sled_cache_capacity = 134217728 +sled_cache_capacity = "128MiB" sled_flush_every_ms = 2000 -lmdb_map_size = "10T" +lmdb_map_size = "1T" replication_mode = "3" @@ -134,8 +134,8 @@ and not just the path to the metadata directory. ### `block_size` Garage splits stored objects in consecutive chunks of size `block_size` -(except the last one which might be smaller). The default size is 1MB and -should work in most cases. We recommend increasing it to e.g. 10MB if +(except the last one which might be smaller). The default size is 1MiB and +should work in most cases. We recommend increasing it to e.g. 10MiB if you are using Garage to store large files and have fast network connections between all nodes (e.g. 1gbps). diff --git a/src/model/Cargo.toml b/src/model/Cargo.toml index caf7d1b06..1d3600a60 100644 --- a/src/model/Cargo.toml +++ b/src/model/Cargo.toml @@ -23,7 +23,6 @@ garage_util.workspace = true async-trait = "0.1.7" arc-swap = "1.0" blake2 = "0.10" -bytesize = "1.2" err-derive = "0.3" hex = "0.4" base64 = "0.21" diff --git a/src/model/garage.rs b/src/model/garage.rs index 8fea6a2c7..a432aa7a2 100644 --- a/src/model/garage.rs +++ b/src/model/garage.rs @@ -95,7 +95,7 @@ impl Garage { info!("Opening Sled database at: {}", db_path.display()); let db = db::sled_adapter::sled::Config::default() .path(&db_path) - .cache_capacity(config.sled_cache_capacity) + .cache_capacity(config.sled_cache_capacity as u64) .flush_every_ms(Some(config.sled_flush_every_ms)) .open() .ok_or_message("Unable to open sled DB")?; @@ -121,21 +121,13 @@ impl Garage { // ---- LMDB DB ---- #[cfg(feature = "lmdb")] "lmdb" | "heed" => { - use std::convert::TryInto; db_path.push("db.lmdb"); info!("Opening LMDB database at: {}", db_path.display()); std::fs::create_dir_all(&db_path) .ok_or_message("Unable to create LMDB data directory")?; - let map_size = match &config.lmdb_map_size { - None => garage_db::lmdb_adapter::recommended_map_size(), - Some(v) => { - let v: usize = v - .parse::() - .ok() - .and_then(|x| x.as_u64().try_into().ok()) - .ok_or_message("invalid value for `lmdb_map_size`")?; - v - (v % 4096) - } + let map_size = match config.lmdb_map_size { + v if v == usize::default() => garage_db::lmdb_adapter::recommended_map_size(), + v => v - (v % 4096), }; use db::lmdb_adapter::heed; diff --git a/src/util/Cargo.toml b/src/util/Cargo.toml index 00dae4d12..2efb02702 100644 --- a/src/util/Cargo.toml +++ b/src/util/Cargo.toml @@ -20,6 +20,7 @@ arc-swap = "1.0" async-trait = "0.1" blake2 = "0.10" bytes = "1.0" +bytesize = "1.2" digest = "0.10" err-derive = "0.3" hexdump = "0.1" diff --git a/src/util/config.rs b/src/util/config.rs index 070bd83eb..724f9404e 100644 --- a/src/util/config.rs +++ b/src/util/config.rs @@ -1,4 +1,5 @@ //! Contains type and functions related to Garage configuration file +use std::convert::TryFrom; use std::io::Read; use std::net::SocketAddr; use std::path::PathBuf; @@ -16,7 +17,10 @@ pub struct Config { pub data_dir: PathBuf, /// Size of data blocks to save to disk - #[serde(default = "default_block_size")] + #[serde( + deserialize_with = "deserialize_capacity", + default = "default_block_size" + )] pub block_size: usize, /// Replication mode. Supported values: @@ -66,15 +70,18 @@ pub struct Config { pub db_engine: String, /// Sled cache size, in bytes - #[serde(default = "default_sled_cache_capacity")] - pub sled_cache_capacity: u64, + #[serde( + deserialize_with = "deserialize_capacity", + default = "default_sled_cache_capacity" + )] + pub sled_cache_capacity: usize, /// Sled flush interval in milliseconds #[serde(default = "default_sled_flush_every_ms")] pub sled_flush_every_ms: u64, /// LMDB map size - #[serde(default)] - pub lmdb_map_size: Option, + #[serde(deserialize_with = "deserialize_capacity", default)] + pub lmdb_map_size: usize, // -- APIs /// Configuration for S3 api @@ -190,7 +197,7 @@ fn default_db_engine() -> String { "sled".into() } -fn default_sled_cache_capacity() -> u64 { +fn default_sled_cache_capacity() -> usize { 128 * 1024 * 1024 } fn default_sled_flush_every_ms() -> u64 { @@ -270,8 +277,6 @@ fn deserialize_compression<'de, D>(deserializer: D) -> Result, D::Er where D: de::Deserializer<'de>, { - use std::convert::TryFrom; - struct OptionVisitor; impl<'de> serde::de::Visitor<'de> for OptionVisitor { @@ -316,6 +321,50 @@ where deserializer.deserialize_any(OptionVisitor) } +fn deserialize_capacity<'de, D>(deserializer: D) -> Result +where + D: de::Deserializer<'de>, +{ + struct CapacityVisitor; + + impl<'de> serde::de::Visitor<'de> for CapacityVisitor { + type Value = usize; + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("int or ''") + } + + fn visit_str(self, value: &str) -> Result + where + E: de::Error, + { + value + .parse::() + .map(|x| x.as_u64()) + .map_err(|e| E::custom(format!("invalid capacity value: {}", e))) + .and_then(|v| { + usize::try_from(v) + .map_err(|_| E::custom("capacity value out of bound".to_owned())) + }) + } + + fn visit_i64(self, v: i64) -> Result + where + E: de::Error, + { + usize::try_from(v).map_err(|_| E::custom("capacity value out of bound".to_owned())) + } + + fn visit_u64(self, v: u64) -> Result + where + E: de::Error, + { + usize::try_from(v).map_err(|_| E::custom("capacity value out of bound".to_owned())) + } + } + + deserializer.deserialize_any(CapacityVisitor) +} + #[cfg(test)] mod tests { use crate::error::Error;