Garage v0.8.5

This minor release includes the following improvements and fixes:
 
 New features:
 
 - Configuration: make LMDB's `map_size` configurable and make `block_size` and `sled_cache_capacity` expressable as strings (such as `10M`) (#628, #630)
 
 - Add support for binding to Unix sockets for the S3, K2V, Admin and Web API servers (#640)
 
 - Move the `convert_db` command into the main Garage binary (#645)
 
 - Add support for specifying RPC secret and admin tokens as environment variables (#643)
 
 - Add `allow_world_readable_secrets` option to config file (#663, #685)
 
 Bug fixes:
 
 - Use `statvfs` instead of mount list to determine free space in metadata/data directories (#611, #631)
 
 - Add missing casts to fix 32-bit build (#632)
 
 - Fix error when none of the HTTP servers (S3/K2V/Admin/Web) is started and fix shutdown hang (#613, #633)
 
 - Add missing CORS headers to PostObject response (#609, #656)
 
 - Monitoring: finer histogram boundaries in Prometheus exported metrics (#531, #686)
 
 Other:
 
 - Documentation improvements (#641)
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEEwhSWp0+ubv79TiqUDkltFQljdr4FAmWmWvsACgkQDkltFQlj
 dr59rRAAiMGQpDUK0QqiCgrp1rcUhvtj3DsQEpT7F14Jo3I7bFDmONZolPbO8YAs
 VE4S4CBQogNH0lMQ6EvJYiBCxDWkxdVibKqDWOYJmUw3bZ6Ypn1eZIF0+Uf1TDI+
 C6CxYbyDQtqvm330K2Du2uOoGiIgm83b6jktK/0FtbAE2GWhtYmQwoelprAGH20i
 baaSfkZbBl8toUscakyhPVVSQ86BcVQ2jqL6Ofu4eQknjMRqCeAIQhMB2ikpiwBz
 hbTZ3x0EfJJqiHocfkTE3B3cPnDKuHDzxPRhLMB/olEpzoxaLJ2+tc0ziQdl06/F
 1c8nHM57L1IaDGKAkpcANnj3yVf3jfPqq9SEUNi+xSIWbvln91RvXU4RIB8hiZqa
 rqAHjDuys++3DoAUr/L4X233MWufVAEYT4B+jaPAv6ys35xhQwPAMJrA0OZEr+hE
 HQMPIG9uMDVjZ2QCgFYgC02kEqvxbsRSVnb0wjI7eoNOk0LKo154eJh1cOGd4Ibs
 yBTiIi1+Y7RCXNxcIHKlj5vMUHPBr2D8DVFj21kfZKUtMQ/8yScoiRC14ZR4J2xF
 IYe3aDm80l3tYgnPRVj4fOGiIPsqnZd4iazYKwj2cifB8tzYfyh5/9fv2aio8K5y
 0GAw4AoTtgg1hLMadbc3om7wy64IRaZzXjv59eYPEotZYdreVpM=
 =RVm8
 -----END PGP SIGNATURE-----
gpgsig -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCAAdFiEEwhSWp0+ubv79TiqUDkltFQljdr4FAmWmZKIACgkQDkltFQlj
 dr6wZRAA1XuOBax/7YsIix3ag0kjnwnGAx8wYaA+Jiojw2yv/+ePL6yGHcKA93lI
 SL8l5G06fTDgpbpfdVbgyRzGT2tmjrXvkygRWf2WMDZ9I+8WxUA2q8aWaEMiNmvd
 0cfzYi14TgX+O0wEbKeeqrXG0473/yThk5U1FNbdJd7rkJ4JzaOTthdk0LJLiEUG
 zQ/YIYx3FVFoVI0rdORb3HKzqYHjMAvpzNhEIeqkrpDEzplQ3jKvY+rYWQL3S9zE
 bHbkZPoT62OpJGMr04/1FUkB+ctsvUrM0CskruaSKWyD2M1xTo/Ug4jh5muVIcdJ
 hJis1/k5rV8JDTIkb6eAxKqfVzI+56yDxofT8rVF4JhvlzvXDLOa0uyDVyA8/6un
 ylWRzs2Mlj6/TbscmPjrdH8v2Lb0zjWxvXe2iYnHHfldWUlYuBtI6FZiG3uNjBCs
 7ns3xr4VOw13RM5auVkEQksIO6lru0kvH18GB3h6Msx67w2JUzl+PaNv8PdRtnmV
 0SfLUl1Nh8yT2h9qG6/3cDE9E1G/mjg8SgljoEe6ahs/BUZmLuTHTyBjf+P22ZbO
 DCITM3CwrV+y/aKnRdLvd6LOWFinUqMS8YvVSVqJh9vo9R+dt33LdBMdWjP4IYHF
 MbACe4FzeG3AXUcHB/mDCm7a2H2BFwzAovFy0SE639PfWBxNue0=
 =gzWq
 -----END PGP SIGNATURE-----

Merge tag 'v0.8.5' into sync-08-09

Garage v0.8.5

This minor release includes the following improvements and fixes:

New features:

- Configuration: make LMDB's `map_size` configurable and make `block_size` and `sled_cache_capacity` expressable as strings (such as `10M`) (#628, #630)

- Add support for binding to Unix sockets for the S3, K2V, Admin and Web API servers (#640)

- Move the `convert_db` command into the main Garage binary (#645)

- Add support for specifying RPC secret and admin tokens as environment variables (#643)

- Add `allow_world_readable_secrets` option to config file (#663, #685)

Bug fixes:

- Use `statvfs` instead of mount list to determine free space in metadata/data directories (#611, #631)

- Add missing casts to fix 32-bit build (#632)

- Fix error when none of the HTTP servers (S3/K2V/Admin/Web) is started and fix shutdown hang (#613, #633)

- Add missing CORS headers to PostObject response (#609, #656)

- Monitoring: finer histogram boundaries in Prometheus exported metrics (#531, #686)

Other:

- Documentation improvements (#641)

Cargo.lock

@@ -1227,6 +1227,7 @@ dependencies = [
  "hyper",
  "k2v-client",
  "kuska-sodiumoxide",
+ "mktemp",
  "netapp",
  "opentelemetry",
  "opentelemetry-otlp",

Cargo.nix

@@ -33,7 +33,7 @@ args@{
   ignoreLockHash,
 }:
 let
-  nixifiedLockHash = "1a87886681a3ef0b83c95addc26674a538b8a93d35bc80db8998e1fcd0821f6c";
+  nixifiedLockHash = "a8ba32879366acd517e5c5feb96e9b009e1dc1991b51fb3acbb6dba2e5c0d935";
   workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
   currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
   lockHashIgnored = if ignoreLockHash
@@ -1771,6 +1771,7 @@ in
       http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
       hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
       k2v_client = (rustPackages."unknown".k2v-client."0.0.4" { inherit profileName; }).out;
+      mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out;
       serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out;
       sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out;
       static_init = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.3" { inherit profileName; }).out;

doc/book/reference-manual/configuration.md

@@ -394,7 +394,7 @@ Compression is done synchronously, setting a value too high will add latency to
 This value can be different between nodes, compression is done by the node which receive the
 API call.
 
-#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET` (env) {#rpc_secret}
+#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET`, `GARAGE_RPC_SECRET_FILE` (env) {#rpc_secret}
 
 Garage uses a secret key, called an RPC secret, that is shared between all
 nodes of the cluster in order to identify these nodes and allow them to
@@ -406,6 +406,9 @@ Since Garage `v0.8.2`, the RPC secret can also be stored in a file whose path is
 given in the configuration variable `rpc_secret_file`, or specified as an
 environment variable `GARAGE_RPC_SECRET`.
 
+Since Garage `v0.8.5` and `v0.9.1`, you can also specify the path of a file
+storing the secret as the `GARAGE_RPC_SECRET_FILE` environment variable.
+
 #### `rpc_bind_addr` {#rpc_bind_addr}
 
 The address and port on which to bind for inter-cluster communcations
@@ -438,6 +441,17 @@ be obtained by running `garage node id` and then included directly in the
 key will be returned by `garage node id` and you will have to add the IP
 yourself.
 
+### `allow_world_readable_secrets`
+
+Garage checks the permissions of your secret files to make sure they're not
+world-readable. In some cases, the check might fail and consider your files as
+world-readable even if they're not, for instance when using Posix ACLs.
+
+Setting `allow_world_readable_secrets` to `true` bypass this
+permission verification.
+
+Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS`
+environment variable to `true` to bypass the permissions check.
 
 ### The `[consul_discovery]` section
 
@@ -583,7 +597,7 @@ See [administration API reference](@/documentation/reference-manual/admin-api.md
 Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons,
 the socket will have 0220 mode. Make sure to set user and group permissions accordingly.
 
-#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN` (env) {#admin_metrics_token}
+#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env) {#admin_metrics_token}
 
 The token for accessing the Metrics endpoint. If this token is not set, the
 Metrics endpoint can be accessed without access control.
@@ -593,8 +607,9 @@ You can use any random string for this value. We recommend generating a random t
 `metrics_token` was introduced in Garage `v0.7.2`.
 `metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`.
 
+`GARAGE_METRICS_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
 
-#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN` (env) {#admin_token}
+#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env) {#admin_token}
 
 The token for accessing all of the other administration endpoints.  If this
 token is not set, access to these endpoints is disabled entirely.
@@ -604,6 +619,7 @@ You can use any random string for this value. We recommend generating a random t
 `admin_token` was introduced in Garage `v0.7.2`.
 `admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`.
 
+`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
 
 #### `trace_sink` {#admin_trace_sink}
 

src/api/s3/post_object.rs

@@ -15,6 +15,7 @@ use serde::Deserialize;
 
 use garage_model::garage::Garage;
 
+use crate::s3::cors::*;
 use crate::s3::error::*;
 use crate::s3::put::{get_headers, save_stream};
 use crate::s3::xml as s3_xml;
@@ -242,7 +243,7 @@ pub async fn handle_post_object(
 
 	let etag = format!("\"{}\"", md5);
 
-	let resp = if let Some(mut target) = params
+	let mut resp = if let Some(mut target) = params
 		.get("success_action_redirect")
 		.and_then(|h| h.to_str().ok())
 		.and_then(|u| url::Url::parse(u).ok())
@@ -262,8 +263,7 @@ pub async fn handle_post_object(
 	} else {
 		let path = head
 			.uri
-			.into_parts()
+			.path_and_query()
-			.path_and_query
 			.map(|paq| paq.path().to_string())
 			.unwrap_or_else(|| "/".to_string());
 		let authority = head
@@ -308,6 +308,13 @@ pub async fn handle_post_object(
 		}
 	};
 
+	let matching_cors_rule =
+		find_matching_cors_rule(&bucket, &Request::from_parts(head, Body::empty()))?;
+	if let Some(rule) = matching_cors_rule {
+		add_cors_headers(&mut resp, rule)
+			.ok_or_internal_error("Invalid bucket CORS configuration")?;
+	}
+
 	Ok(resp)
 }
 

src/garage/Cargo.toml

@@ -67,6 +67,7 @@ chrono = "0.4"
 http = "0.2"
 hmac = "0.12"
 hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
+mktemp = "0.5"
 sha2 = "0.10"
 
 static_init = "1.0"

src/garage/main.rs

@@ -7,6 +7,7 @@ extern crate tracing;
 mod admin;
 mod cli;
 mod repair;
+mod secrets;
 mod server;
 #[cfg(feature = "telemetry-otlp")]
 mod tracing_setup;
@@ -28,7 +29,6 @@ use structopt::StructOpt;
 use netapp::util::parse_and_resolve_peer_addr;
 use netapp::NetworkKey;
 
-use garage_util::config::Config;
 use garage_util::error::*;
 
 use garage_rpc::system::*;
@@ -38,6 +38,7 @@ use garage_model::helper::error::Error as HelperError;
 
 use admin::*;
 use cli::*;
+use secrets::Secrets;
 
 #[derive(StructOpt, Debug)]
 #[structopt(
@@ -66,24 +67,6 @@ struct Opt {
 	cmd: Command,
 }
 
-#[derive(StructOpt, Debug)]
-pub struct Secrets {
-	/// RPC secret network key, used to replace rpc_secret in config.toml when running the
-	/// daemon or doing admin operations
-	#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
-	pub rpc_secret: Option<String>,
-
-	/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
-	/// running the Garage daemon
-	#[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
-	pub admin_token: Option<String>,
-
-	/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
-	/// running the Garage daemon
-	#[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
-	pub metrics_token: Option<String>,
-}
-
 #[tokio::main]
 async fn main() {
 	// Initialize version and features info
@@ -261,16 +244,3 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
 		Ok(x) => Ok(x),
 	}
 }
-
-fn fill_secrets(mut config: Config, secrets: Secrets) -> Config {
-	if secrets.rpc_secret.is_some() {
-		config.rpc_secret = secrets.rpc_secret;
-	}
-	if secrets.admin_token.is_some() {
-		config.admin.admin_token = secrets.admin_token;
-	}
-	if secrets.metrics_token.is_some() {
-		config.admin.metrics_token = secrets.metrics_token;
-	}
-	config
-}

src/garage/repair/offline.rs

@@ -6,7 +6,7 @@ use garage_util::error::*;
 use garage_model::garage::Garage;
 
 use crate::cli::structs::*;
-use crate::{fill_secrets, Secrets};
+use crate::secrets::{fill_secrets, Secrets};
 
 pub async fn offline_repair(
 	config_file: PathBuf,
@@ -20,7 +20,7 @@ pub async fn offline_repair(
 	}
 
 	info!("Loading configuration...");
-	let config = fill_secrets(read_config(config_file)?, secrets);
+	let config = fill_secrets(read_config(config_file)?, secrets)?;
 
 	info!("Initializing Garage main data store...");
 	let garage = Garage::new(config)?;

src/garage/secrets.rs

@@ -0,0 +1,318 @@
+use structopt::StructOpt;
+
+use garage_util::config::Config;
+use garage_util::error::Error;
+
+/// Structure for secret values or paths that are passed as CLI arguments or environment
+/// variables, instead of in the config file.
+#[derive(StructOpt, Debug, Default, Clone)]
+pub struct Secrets {
+	/// Skip permission check on files containing secrets
+	#[cfg(unix)]
+	#[structopt(
+		long = "allow-world-readable-secrets",
+		env = "GARAGE_ALLOW_WORLD_READABLE_SECRETS"
+	)]
+	pub allow_world_readable_secrets: Option<bool>,
+
+	/// RPC secret network key, used to replace rpc_secret in config.toml when running the
+	/// daemon or doing admin operations
+	#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
+	pub rpc_secret: Option<String>,
+
+	/// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret
+	/// when running the daemon or doing admin operations
+	#[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")]
+	pub rpc_secret_file: Option<String>,
+
+	/// Admin API authentication token, replaces admin.admin_token in config.toml when
+	/// running the Garage daemon
+	#[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
+	pub admin_token: Option<String>,
+
+	/// Admin API authentication token file path, replaces admin.admin_token in config.toml
+	/// and admin-token when running the Garage daemon
+	#[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")]
+	pub admin_token_file: Option<String>,
+
+	/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
+	/// running the Garage daemon
+	#[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
+	pub metrics_token: Option<String>,
+
+	/// Metrics API authentication token file path, replaces admin.metrics_token in config.toml
+	/// and metrics-token when running the Garage daemon
+	#[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")]
+	pub metrics_token_file: Option<String>,
+}
+
+/// Single function to fill all secrets in the Config struct from their correct source (value
+/// from config or CLI param or env variable or read from a file specified in config or CLI
+/// param or env variable)
+pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result<Config, Error> {
+	let allow_world_readable = secrets
+		.allow_world_readable_secrets
+		.unwrap_or(config.allow_world_readable_secrets);
+
+	fill_secret(
+		&mut config.rpc_secret,
+		&config.rpc_secret_file,
+		&secrets.rpc_secret,
+		&secrets.rpc_secret_file,
+		"rpc_secret",
+		allow_world_readable,
+	)?;
+
+	fill_secret(
+		&mut config.admin.admin_token,
+		&config.admin.admin_token_file,
+		&secrets.admin_token,
+		&secrets.admin_token_file,
+		"admin.admin_token",
+		allow_world_readable,
+	)?;
+	fill_secret(
+		&mut config.admin.metrics_token,
+		&config.admin.metrics_token_file,
+		&secrets.metrics_token,
+		&secrets.metrics_token_file,
+		"admin.metrics_token",
+		allow_world_readable,
+	)?;
+
+	Ok(config)
+}
+
+fn fill_secret(
+	config_secret: &mut Option<String>,
+	config_secret_file: &Option<String>,
+	cli_secret: &Option<String>,
+	cli_secret_file: &Option<String>,
+	name: &'static str,
+	allow_world_readable: bool,
+) -> Result<(), Error> {
+	let cli_value = match (&cli_secret, &cli_secret_file) {
+		(Some(_), Some(_)) => {
+			return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
+		}
+		(Some(secret), None) => Some(secret.to_string()),
+		(None, Some(file)) => Some(read_secret_file(file, allow_world_readable)?),
+		(None, None) => None,
+	};
+
+	if let Some(val) = cli_value {
+		if config_secret.is_some() || config_secret_file.is_some() {
+			debug!("Overriding secret `{}` using value specified using CLI argument or environnement variable.", name);
+		}
+
+		*config_secret = Some(val);
+	} else if let Some(file_path) = &config_secret_file {
+		if config_secret.is_some() {
+			return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
+		}
+
+		*config_secret = Some(read_secret_file(file_path, allow_world_readable)?);
+	}
+
+	Ok(())
+}
+
+fn read_secret_file(file_path: &String, allow_world_readable: bool) -> Result<String, Error> {
+	if !allow_world_readable {
+		#[cfg(unix)]
+		{
+			use std::os::unix::fs::MetadataExt;
+			let metadata = std::fs::metadata(file_path)?;
+			if metadata.mode() & 0o077 != 0 {
+				return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
+			}
+		}
+	}
+
+	let secret_buf = std::fs::read_to_string(file_path)?;
+
+	// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
+	//           also editors sometimes add a trailing newline
+	Ok(String::from(secret_buf.trim_end()))
+}
+
+#[cfg(test)]
+mod tests {
+	use std::fs::File;
+	use std::io::Write;
+
+	use garage_util::config::read_config;
+	use garage_util::error::Error;
+
+	use super::*;
+
+	#[test]
+	fn test_rpc_secret_file_works() -> Result<(), Error> {
+		let path_secret = mktemp::Temp::new_file()?;
+		let mut file_secret = File::create(path_secret.as_path())?;
+		writeln!(file_secret, "foo")?;
+		drop(file_secret);
+
+		let path_config = mktemp::Temp::new_file()?;
+		let mut file_config = File::create(path_config.as_path())?;
+		let path_secret_path = path_secret.as_path();
+		writeln!(
+			file_config,
+			r#"
+			metadata_dir = "/tmp/garage/meta"
+			data_dir = "/tmp/garage/data"
+			replication_mode = "3"
+			rpc_bind_addr = "[::]:3901"
+			rpc_secret_file = "{}"
+
+			[s3_api]
+			s3_region = "garage"
+			api_bind_addr = "[::]:3900"
+			"#,
+			path_secret_path.display()
+		)?;
+		drop(file_config);
+
+		// Second configuration file, same as previous one
+		// except it allows world-readable secrets.
+		let path_config_allow_world_readable = mktemp::Temp::new_file()?;
+		let mut file_config_allow_world_readable =
+			File::create(path_config_allow_world_readable.as_path())?;
+		writeln!(
+			file_config_allow_world_readable,
+			r#"
+			metadata_dir = "/tmp/garage/meta"
+			data_dir = "/tmp/garage/data"
+			replication_mode = "3"
+			rpc_bind_addr = "[::]:3901"
+			rpc_secret_file = "{}"
+			allow_world_readable_secrets = true
+
+			[s3_api]
+			s3_region = "garage"
+			api_bind_addr = "[::]:3900"
+			"#,
+			path_secret_path.display()
+		)?;
+		drop(file_config_allow_world_readable);
+
+		let config = read_config(path_config.to_path_buf())?;
+		let config = fill_secrets(config, Secrets::default())?;
+		assert_eq!("foo", config.rpc_secret.unwrap());
+
+		// ---- Check non world-readable secrets config ----
+		#[cfg(unix)]
+		{
+			let secrets_allow_world_readable = Secrets {
+				allow_world_readable_secrets: Some(true),
+				..Default::default()
+			};
+			let secrets_no_allow_world_readable = Secrets {
+				allow_world_readable_secrets: Some(false),
+				..Default::default()
+			};
+
+			use std::os::unix::fs::PermissionsExt;
+			let metadata = std::fs::metadata(&path_secret_path)?;
+			let mut perm = metadata.permissions();
+			perm.set_mode(0o660);
+			std::fs::set_permissions(&path_secret_path, perm)?;
+
+			// Config file that just specifies the path
+			let config = read_config(path_config.to_path_buf())?;
+			assert!(fill_secrets(config, Secrets::default()).is_err());
+
+			let config = read_config(path_config.to_path_buf())?;
+			assert!(fill_secrets(config, secrets_allow_world_readable.clone()).is_ok());
+
+			let config = read_config(path_config.to_path_buf())?;
+			assert!(fill_secrets(config, secrets_no_allow_world_readable.clone()).is_err());
+
+			// Config file that also specifies to allow world_readable_secrets
+			let config = read_config(path_config_allow_world_readable.to_path_buf())?;
+			assert!(fill_secrets(config, Secrets::default()).is_ok());
+
+			let config = read_config(path_config_allow_world_readable.to_path_buf())?;
+			assert!(fill_secrets(config, secrets_allow_world_readable).is_ok());
+
+			let config = read_config(path_config_allow_world_readable.to_path_buf())?;
+			assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err());
+		}
+
+		// ---- Check alternative secrets specified on CLI ----
+
+		let path_secret2 = mktemp::Temp::new_file()?;
+		let mut file_secret2 = File::create(path_secret2.as_path())?;
+		writeln!(file_secret2, "bar")?;
+		drop(file_secret2);
+
+		let config = read_config(path_config.to_path_buf())?;
+		let config = fill_secrets(
+			config,
+			Secrets {
+				rpc_secret: Some("baz".into()),
+				..Default::default()
+			},
+		)?;
+		assert_eq!(config.rpc_secret.as_deref(), Some("baz"));
+
+		let config = read_config(path_config.to_path_buf())?;
+		let config = fill_secrets(
+			config,
+			Secrets {
+				rpc_secret_file: Some(path_secret2.display().to_string()),
+				..Default::default()
+			},
+		)?;
+		assert_eq!(config.rpc_secret.as_deref(), Some("bar"));
+
+		let config = read_config(path_config.to_path_buf())?;
+		assert!(fill_secrets(
+			config,
+			Secrets {
+				rpc_secret: Some("baz".into()),
+				rpc_secret_file: Some(path_secret2.display().to_string()),
+				..Default::default()
+			}
+		)
+		.is_err());
+
+		drop(path_secret);
+		drop(path_secret2);
+		drop(path_config);
+		drop(path_config_allow_world_readable);
+
+		Ok(())
+	}
+
+	#[test]
+	fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> {
+		let path_config = mktemp::Temp::new_file()?;
+		let mut file_config = File::create(path_config.as_path())?;
+		writeln!(
+			file_config,
+			r#"
+			metadata_dir = "/tmp/garage/meta"
+			data_dir = "/tmp/garage/data"
+			replication_mode = "3"
+			rpc_bind_addr = "[::]:3901"
+			rpc_secret= "dummy"
+			rpc_secret_file = "dummy"
+
+			[s3_api]
+			s3_region = "garage"
+			api_bind_addr = "[::]:3900"
+			"#
+		)?;
+		let config = read_config(path_config.to_path_buf())?;
+		assert_eq!(
+			"only one of `rpc_secret` and `rpc_secret_file` can be set",
+			fill_secrets(config, Secrets::default())
+				.unwrap_err()
+				.to_string()
+		);
+		drop(path_config);
+		drop(file_config);
+		Ok(())
+	}
+}

src/garage/server.rs

@@ -15,9 +15,9 @@ use garage_web::WebServer;
 use garage_api::k2v::api_server::K2VApiServer;
 
 use crate::admin::*;
+use crate::secrets::{fill_secrets, Secrets};
 #[cfg(feature = "telemetry-otlp")]
 use crate::tracing_setup::*;
-use crate::{fill_secrets, Secrets};
 
 async fn wait_from(mut chan: watch::Receiver<bool>) {
 	while !*chan.borrow() {
@@ -29,12 +29,19 @@ async fn wait_from(mut chan: watch::Receiver<bool>) {
 
 pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Error> {
 	info!("Loading configuration...");
-	let config = fill_secrets(read_config(config_file)?, secrets);
+	let config = fill_secrets(read_config(config_file)?, secrets)?;
 
 	// ---- Initialize Garage internals ----
 
 	#[cfg(feature = "metrics")]
-	let metrics_exporter = opentelemetry_prometheus::exporter().init();
+	let metrics_exporter = opentelemetry_prometheus::exporter()
+		.with_default_summary_quantiles(vec![0.25, 0.5, 0.75, 0.9, 0.95, 0.99])
+		.with_default_histogram_boundaries(vec![
+			0.001, 0.0015, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.02, 0.03, 0.05, 0.07, 0.1,
+			0.15, 0.2, 0.3, 0.5, 0.7, 1., 1.5, 2., 3., 5., 7., 10., 15., 20., 30., 40., 50., 60.,
+			70., 100.,
+		])
+		.init();
 
 	info!("Initializing Garage main data store...");
 	let garage = Garage::new(config.clone())?;

src/util/config.rs

@@ -1,6 +1,5 @@
 //! Contains type and functions related to Garage configuration file
 use std::convert::TryFrom;
-use std::io::Read;
 use std::net::SocketAddr;
 use std::path::PathBuf;
 
@@ -45,11 +44,15 @@ pub struct Config {
 	)]
 	pub compression_level: Option<i32>,
 
+	/// Skip the permission check of secret files. Useful when
+	/// POSIX ACLs (or more complex chmods) are used.
+	#[serde(default)]
+	pub allow_world_readable_secrets: bool,
+
 	/// RPC secret key: 32 bytes hex encoded
 	pub rpc_secret: Option<String>,
 	/// Optional file where RPC secret key is read from
 	pub rpc_secret_file: Option<String>,
-
 	/// Address to bind for RPC
 	pub rpc_bind_addr: SocketAddr,
 	/// Public IP address of this node
@@ -221,6 +224,13 @@ pub struct KubernetesDiscoveryConfig {
 	pub skip_crd: bool,
 }
 
+/// Read and parse configuration
+pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
+	let config = std::fs::read_to_string(config_file)?;
+
+	Ok(toml::from_str(&config)?)
+}
+
 fn default_db_engine() -> String {
 	"lmdb".into()
 }
@@ -235,68 +245,6 @@ fn default_block_size() -> usize {
 	1048576
 }
 
-/// Read and parse configuration
-pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
-	let mut file = std::fs::OpenOptions::new()
-		.read(true)
-		.open(config_file.as_path())?;
-
-	let mut config = String::new();
-	file.read_to_string(&mut config)?;
-
-	let mut parsed_config: Config = toml::from_str(&config)?;
-
-	secret_from_file(
-		&mut parsed_config.rpc_secret,
-		&parsed_config.rpc_secret_file,
-		"rpc_secret",
-	)?;
-	secret_from_file(
-		&mut parsed_config.admin.metrics_token,
-		&parsed_config.admin.metrics_token_file,
-		"admin.metrics_token",
-	)?;
-	secret_from_file(
-		&mut parsed_config.admin.admin_token,
-		&parsed_config.admin.admin_token_file,
-		"admin.admin_token",
-	)?;
-
-	Ok(parsed_config)
-}
-
-fn secret_from_file(
-	secret: &mut Option<String>,
-	secret_file: &Option<String>,
-	name: &'static str,
-) -> Result<(), Error> {
-	match (&secret, &secret_file) {
-		(_, None) => {
-			// no-op
-		}
-		(Some(_), Some(_)) => {
-			return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
-		}
-		(None, Some(file_path)) => {
-			#[cfg(unix)]
-			if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") {
-				use std::os::unix::fs::MetadataExt;
-				let metadata = std::fs::metadata(file_path)?;
-				if metadata.mode() & 0o077 != 0 {
-					return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
-				}
-			}
-			let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?;
-			let mut secret_buf = String::new();
-			file.read_to_string(&mut secret_buf)?;
-			// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
-			//           also editors sometimes add a trailing newline
-			*secret = Some(String::from(secret_buf.trim_end()));
-		}
-	}
-	Ok(())
-}
-
 fn default_compression() -> Option<i32> {
 	Some(1)
 }
@@ -425,83 +373,4 @@ mod tests {
 
 		Ok(())
 	}
-
-	#[test]
-	fn test_rpc_secret_file_works() -> Result<(), Error> {
-		let path_secret = mktemp::Temp::new_file()?;
-		let mut file_secret = File::create(path_secret.as_path())?;
-		writeln!(file_secret, "foo")?;
-		drop(file_secret);
-
-		let path_config = mktemp::Temp::new_file()?;
-		let mut file_config = File::create(path_config.as_path())?;
-		let path_secret_path = path_secret.as_path();
-		writeln!(
-			file_config,
-			r#"
-			metadata_dir = "/tmp/garage/meta"
-			data_dir = "/tmp/garage/data"
-			replication_mode = "3"
-			rpc_bind_addr = "[::]:3901"
-			rpc_secret_file = "{}"
-
-			[s3_api]
-			s3_region = "garage"
-			api_bind_addr = "[::]:3900"
-			"#,
-			path_secret_path.display()
-		)?;
-		let config = super::read_config(path_config.to_path_buf())?;
-		assert_eq!("foo", config.rpc_secret.unwrap());
-
-		#[cfg(unix)]
-		{
-			use std::os::unix::fs::PermissionsExt;
-			let metadata = std::fs::metadata(&path_secret_path)?;
-			let mut perm = metadata.permissions();
-			perm.set_mode(0o660);
-			std::fs::set_permissions(&path_secret_path, perm)?;
-
-			std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "false");
-			assert!(super::read_config(path_config.to_path_buf()).is_err());
-
-			std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "true");
-			assert!(super::read_config(path_config.to_path_buf()).is_ok());
-		}
-
-		drop(path_config);
-		drop(path_secret);
-		drop(file_config);
-		Ok(())
-	}
-
-	#[test]
-	fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> {
-		let path_config = mktemp::Temp::new_file()?;
-		let mut file_config = File::create(path_config.as_path())?;
-		writeln!(
-			file_config,
-			r#"
-			metadata_dir = "/tmp/garage/meta"
-			data_dir = "/tmp/garage/data"
-			replication_mode = "3"
-			rpc_bind_addr = "[::]:3901"
-			rpc_secret= "dummy"
-			rpc_secret_file = "dummy"
-
-			[s3_api]
-			s3_region = "garage"
-			api_bind_addr = "[::]:3900"
-			"#
-		)?;
-		assert_eq!(
-			"only one of `rpc_secret` and `rpc_secret_file` can be set",
-			super::read_config(path_config.to_path_buf())
-				.unwrap_err()
-				.to_string()
-		);
-		drop(path_config);
-		drop(file_config);
-		Ok(())
-	}
 }