config: make block_size and sled_cache_capacity expressable as strings

Cargo.lock

@@ -1340,7 +1340,6 @@ dependencies = [
  "async-trait",
  "base64 0.21.3",
  "blake2",
- "bytesize",
  "err-derive",
  "futures",
  "futures-util",
@@ -1422,6 +1421,7 @@ dependencies = [
  "async-trait",
  "blake2",
  "bytes",
+ "bytesize",
  "chrono",
  "digest",
  "err-derive",

Cargo.nix

@@ -33,7 +33,7 @@ args@{
   ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "c5e95ea3fbf4a23e07fe76a8c8886e4eb4a7c95b2d9ca8fa22fa4d8792b4d29f";
+  nixifiedLockHash = "3e3f41f614ab470ecb4b06c670cd6a84c443d799d01f1d48f1d251872099c468";
   workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
   currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
   lockHashIgnored = if ignoreLockHash
@@ -1911,7 +1911,6 @@ in
       async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out;
       base64 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".base64."0.21.3" { inherit profileName; }).out;
       blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out;
-      bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
       futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.28" { inherit profileName; }).out;
       futures_util = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures-util."0.3.28" { inherit profileName; }).out;
@@ -2015,6 +2014,7 @@ in
       async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.73" { profileName = "__noProfile"; }).out;
       blake2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".blake2."0.10.6" { inherit profileName; }).out;
       bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytes."1.4.0" { inherit profileName; }).out;
+      bytesize = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".bytesize."1.3.0" { inherit profileName; }).out;
       chrono = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".chrono."0.4.26" { inherit profileName; }).out;
       digest = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".digest."0.10.7" { inherit profileName; }).out;
       err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;

doc/book/reference-manual/configuration.md

@@ -15,9 +15,9 @@ db_engine = "lmdb"
 
 block_size = 1048576
 
-sled_cache_capacity = 134217728
+sled_cache_capacity = "128MiB"
 sled_flush_every_ms = 2000
-lmdb_map_size = "10T"
+lmdb_map_size = "1T"
 
 replication_mode = "3"
 
@@ -134,8 +134,8 @@ and not just the path to the metadata directory.
 ### `block_size`
 
 Garage splits stored objects in consecutive chunks of size `block_size`
-(except the last one which might be smaller). The default size is 1MB and
-should work in most cases. We recommend increasing it to e.g. 10MB if
+(except the last one which might be smaller). The default size is 1MiB and
+should work in most cases. We recommend increasing it to e.g. 10MiB if
 you are using Garage to store large files and have fast network connections
 between all nodes (e.g. 1gbps).
 

src/model/Cargo.toml

@@ -23,7 +23,6 @@ garage_util.workspace = true
 async-trait = "0.1.7"
 arc-swap = "1.0"
 blake2 = "0.10"
-bytesize = "1.2"
 err-derive = "0.3"
 hex = "0.4"
 base64 = "0.21"

src/model/garage.rs

@@ -95,7 +95,7 @@ impl Garage {
 				info!("Opening Sled database at: {}", db_path.display());
 				let db = db::sled_adapter::sled::Config::default()
 					.path(&db_path)
-					.cache_capacity(config.sled_cache_capacity)
+					.cache_capacity(config.sled_cache_capacity as u64)
 					.flush_every_ms(Some(config.sled_flush_every_ms))
 					.open()
 					.ok_or_message("Unable to open sled DB")?;
@@ -121,21 +121,13 @@ impl Garage {
 			// ---- LMDB DB ----
 			#[cfg(feature = "lmdb")]
 			"lmdb" | "heed" => {
-				use std::convert::TryInto;
 				db_path.push("db.lmdb");
 				info!("Opening LMDB database at: {}", db_path.display());
 				std::fs::create_dir_all(&db_path)
 					.ok_or_message("Unable to create LMDB data directory")?;
-				let map_size = match &config.lmdb_map_size {
-					None => garage_db::lmdb_adapter::recommended_map_size(),
-					Some(v) => {
-						let v: usize = v
-							.parse::<bytesize::ByteSize>()
-							.ok()
-							.and_then(|x| x.as_u64().try_into().ok())
-							.ok_or_message("invalid value for `lmdb_map_size`")?;
-						v - (v % 4096)
-					}
+				let map_size = match config.lmdb_map_size {
+					v if v == usize::default() => garage_db::lmdb_adapter::recommended_map_size(),
+					v => v - (v % 4096),
 				};
 
 				use db::lmdb_adapter::heed;

src/util/Cargo.toml

@@ -20,6 +20,7 @@ arc-swap = "1.0"
 async-trait = "0.1"
 blake2 = "0.10"
 bytes = "1.0"
+bytesize = "1.2"
 digest = "0.10"
 err-derive = "0.3"
 hexdump = "0.1"

src/util/config.rs

@@ -1,4 +1,5 @@
 //! Contains type and functions related to Garage configuration file
+use std::convert::TryFrom;
 use std::io::Read;
 use std::net::SocketAddr;
 use std::path::PathBuf;
@@ -16,7 +17,10 @@ pub struct Config {
 	pub data_dir: PathBuf,
 
 	/// Size of data blocks to save to disk
-	#[serde(default = "default_block_size")]
+	#[serde(
+		deserialize_with = "deserialize_capacity",
+		default = "default_block_size"
+	)]
 	pub block_size: usize,
 
 	/// Replication mode. Supported values:
@@ -66,15 +70,18 @@ pub struct Config {
 	pub db_engine: String,
 
 	/// Sled cache size, in bytes
-	#[serde(default = "default_sled_cache_capacity")]
-	pub sled_cache_capacity: u64,
+	#[serde(
+		deserialize_with = "deserialize_capacity",
+		default = "default_sled_cache_capacity"
+	)]
+	pub sled_cache_capacity: usize,
 	/// Sled flush interval in milliseconds
 	#[serde(default = "default_sled_flush_every_ms")]
 	pub sled_flush_every_ms: u64,
 
 	/// LMDB map size
-	#[serde(default)]
-	pub lmdb_map_size: Option<String>,
+	#[serde(deserialize_with = "deserialize_capacity", default)]
+	pub lmdb_map_size: usize,
 
 	// -- APIs
 	/// Configuration for S3 api
@@ -190,7 +197,7 @@ fn default_db_engine() -> String {
 	"sled".into()
 }
 
-fn default_sled_cache_capacity() -> u64 {
+fn default_sled_cache_capacity() -> usize {
 	128 * 1024 * 1024
 }
 fn default_sled_flush_every_ms() -> u64 {
@@ -270,8 +277,6 @@ fn deserialize_compression<'de, D>(deserializer: D) -> Result<Option<i32>, D::Er
 where
 	D: de::Deserializer<'de>,
 {
-	use std::convert::TryFrom;
-
 	struct OptionVisitor;
 
 	impl<'de> serde::de::Visitor<'de> for OptionVisitor {
@@ -316,6 +321,50 @@ where
 	deserializer.deserialize_any(OptionVisitor)
 }
 
+fn deserialize_capacity<'de, D>(deserializer: D) -> Result<usize, D::Error>
+where
+	D: de::Deserializer<'de>,
+{
+	struct CapacityVisitor;
+
+	impl<'de> serde::de::Visitor<'de> for CapacityVisitor {
+		type Value = usize;
+		fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
+			formatter.write_str("int or '<capacity>'")
+		}
+
+		fn visit_str<E>(self, value: &str) -> Result<Self::Value, E>
+		where
+			E: de::Error,
+		{
+			value
+				.parse::<bytesize::ByteSize>()
+				.map(|x| x.as_u64())
+				.map_err(|e| E::custom(format!("invalid capacity value: {}", e)))
+				.and_then(|v| {
+					usize::try_from(v)
+						.map_err(|_| E::custom("capacity value out of bound".to_owned()))
+				})
+		}
+
+		fn visit_i64<E>(self, v: i64) -> Result<Self::Value, E>
+		where
+			E: de::Error,
+		{
+			usize::try_from(v).map_err(|_| E::custom("capacity value out of bound".to_owned()))
+		}
+
+		fn visit_u64<E>(self, v: u64) -> Result<Self::Value, E>
+		where
+			E: de::Error,
+		{
+			usize::try_from(v).map_err(|_| E::custom("capacity value out of bound".to_owned()))
+		}
+	}
+
+	deserializer.deserialize_any(CapacityVisitor)
+}
+
 #[cfg(test)]
 mod tests {
 	use crate::error::Error;