diff --git a/Cargo.lock b/Cargo.lock index c3dee8c7..c41e0564 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1350,6 +1350,7 @@ dependencies = [ "sha2", "static_init", "structopt", + "syslog-tracing", "timeago", "tokio", "toml", @@ -3985,6 +3986,17 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "syslog-tracing" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "340b1540dcdb6b066bc2966e7974f977ab1a38f21b2be189014ffb0cc2405768" +dependencies = [ + "libc", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "system-configuration" version = "0.5.1" diff --git a/Cargo.nix b/Cargo.nix index e5b975b8..bb852806 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -34,7 +34,7 @@ args@{ ignoreLockHash, }: let - nixifiedLockHash = "a49da9d5ef560672a34c1e004c0122e706a74fac512300f20858f136cd00582e"; + nixifiedLockHash = "1ef5e578c148e63bdc6491d497aba66b38dcf011779d417228906ce7b19d55f4"; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); lockHashIgnored = if ignoreLockHash @@ -1927,6 +1927,8 @@ in (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/opentelemetry-prometheus") "opentelemetry-prometheus") (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage/prometheus") "prometheus") (lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/sqlite") "sqlite") + (lib.optional (rootFeatures' ? "garage/syslog") "syslog") + (lib.optional (rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing") "syslog-tracing") (lib.optional (rootFeatures' ? "garage/system-libs") "system-libs") (lib.optional (rootFeatures' ? "garage/telemetry-otlp") "telemetry-otlp") ]; @@ -1960,6 +1962,7 @@ in serde_bytes = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_bytes."0.11.14" { inherit profileName; }).out; sha1 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha1."0.10.6" { inherit profileName; }).out; structopt = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".structopt."0.3.26" { inherit profileName; }).out; + ${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "syslog_tracing" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".syslog-tracing."0.3.0" { inherit profileName; }).out; timeago = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".timeago."0.4.2" { inherit profileName; }).out; tokio = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tokio."1.36.0" { inherit profileName; }).out; toml = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".toml."0.8.10" { inherit profileName; }).out; @@ -5675,6 +5678,18 @@ in }; }); + "registry+https://github.com/rust-lang/crates.io-index".syslog-tracing."0.3.0" = overridableMkRustCrate (profileName: rec { + name = "syslog-tracing"; + version = "0.3.0"; + registry = "registry+https://github.com/rust-lang/crates.io-index"; + src = fetchCratesIo { inherit name version; sha256 = "340b1540dcdb6b066bc2966e7974f977ab1a38f21b2be189014ffb0cc2405768"; }; + dependencies = { + ${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "libc" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.153" { inherit profileName; }).out; + ${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "tracing_core" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-core."0.1.32" { inherit profileName; }).out; + ${ if rootFeatures' ? "garage/syslog" || rootFeatures' ? "garage/syslog-tracing" then "tracing_subscriber" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".tracing-subscriber."0.3.18" { inherit profileName; }).out; + }; + }); + "registry+https://github.com/rust-lang/crates.io-index".system-configuration."0.5.1" = overridableMkRustCrate (profileName: rec { name = "system-configuration"; version = "0.5.1"; diff --git a/Cargo.toml b/Cargo.toml index 16d3df4c..c0aad2d0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,6 +76,7 @@ kuska-handshake = { version = "0.2.0", features = ["default", "async_std"] } clap = { version = "4.1", features = ["derive", "env"] } pretty_env_logger = "0.5" structopt = { version = "0.3", default-features = false } +syslog-tracing = "0.3" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/doc/book/cookbook/from-source.md b/doc/book/cookbook/from-source.md index f0e185a4..7105c999 100644 --- a/doc/book/cookbook/from-source.md +++ b/doc/book/cookbook/from-source.md @@ -90,5 +90,6 @@ The following feature flags are available in v0.8.0: | `kubernetes-discovery` | optional | Enable automatic registration and discovery
of cluster nodes through the Kubernetes API | | `metrics` | *by default* | Enable collection of metrics in Prometheus format on the admin API | | `telemetry-otlp` | optional | Enable collection of execution traces using OpenTelemetry | +| `syslog` | optional | Enable logging to Syslog | | `lmdb` | *by default* | Enable using LMDB to store Garage's metadata | | `sqlite` | *by default* | Enable using Sqlite3 to store Garage's metadata | diff --git a/doc/book/reference-manual/configuration.md b/doc/book/reference-manual/configuration.md index a21f945b..6fbe829f 100644 --- a/doc/book/reference-manual/configuration.md +++ b/doc/book/reference-manual/configuration.md @@ -31,6 +31,8 @@ rpc_bind_addr = "[::]:3901" rpc_bind_outgoing = false rpc_public_addr = "[fc00:1::1]:3901" +allow_world_readable_secrets = false + bootstrap_peers = [ "563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901", "86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332@[fc00:1::2]:3901", @@ -81,7 +83,10 @@ The following gives details about each available configuration option. ### Index +[Environment variables](#env_variables). + Top-level configuration options: +[`allow_world_readable_secrets`](#allow_world_readable_secrets), [`block_size`](#block_size), [`bootstrap_peers`](#bootstrap_peers), [`compression_level`](#compression_level), @@ -132,6 +137,23 @@ The `[admin]` section: [`admin_token`/`admin_token_file`](#admin_token), [`trace_sink`](#admin_trace_sink), +### Environment variables {#env_variables} + +The following configuration parameter must be specified as an environment +variable, it does not exist in the configuration file: + +- `GARAGE_LOG_TO_SYSLOG` (since v0.9.4): set this to `1` or `true` to make the + Garage daemon send its logs to `syslog` (using the libc `syslog` function) + instead of printing to stderr. + +The following environment variables can be used to override the corresponding +values in the configuration file: + +- [`GARAGE_ALLOW_WORLD_READABLE_SECRETS`](#allow_world_readable_secrets) +- [`GARAGE_RPC_SECRET` and `GARAGE_RPC_SECRET_FILE`](#rpc_secret) +- [`GARAGE_ADMIN_TOKEN` and `GARAGE_ADMIN_TOKEN_FILE`](#admin_token) +- [`GARAGE_METRICS_TOKEN` and `GARAGE_METRICS_TOKEN`](#admin_metrics_token) + ### Top-level configuration options @@ -502,7 +524,7 @@ be obtained by running `garage node id` and then included directly in the key will be returned by `garage node id` and you will have to add the IP yourself. -### `allow_world_readable_secrets` +### `allow_world_readable_secrets` or `GARAGE_ALLOW_WORLD_READABLE_SECRETS` (env) {#allow_world_readable_secrets} Garage checks the permissions of your secret files to make sure they're not world-readable. In some cases, the check might fail and consider your files as diff --git a/k2v_test.py b/k2v_test.py deleted file mode 100755 index 3219056e..00000000 --- a/k2v_test.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python - -import os -import requests -from datetime import datetime - -# let's talk to our AWS Elasticsearch cluster -#from requests_aws4auth import AWS4Auth -#auth = AWS4Auth('GK31c2f218a2e44f485b94239e', -# 'b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835', -# 'us-east-1', -# 's3') - -from aws_requests_auth.aws_auth import AWSRequestsAuth -auth = AWSRequestsAuth(aws_access_key='GK31c2f218a2e44f485b94239e', - aws_secret_access_key='b892c0665f0ada8a4755dae98baa3b133590e11dae3bcc1f9d769d67f16c3835', - aws_host='localhost:3812', - aws_region='us-east-1', - aws_service='k2v') - - -print("-- ReadIndex") -response = requests.get('http://localhost:3812/alex', - auth=auth) -print(response.headers) -print(response.text) - - -sort_keys = ["a", "b", "c", "d"] - -for sk in sort_keys: - print("-- (%s) Put initial (no CT)"%sk) - response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth, - data='{}: Hello, world!'.format(datetime.timestamp(datetime.now()))) - print(response.headers) - print(response.text) - - print("-- Get") - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - ct = response.headers["x-garage-causality-token"] - - print("-- ReadIndex") - response = requests.get('http://localhost:3812/alex', - auth=auth) - print(response.headers) - print(response.text) - - print("-- Put with CT") - response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth, - headers={'x-garage-causality-token': ct}, - data='{}: Good bye, world!'.format(datetime.timestamp(datetime.now()))) - print(response.headers) - print(response.text) - - print("-- Get") - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - - print("-- Put again with same CT (concurrent)") - response = requests.put('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth, - headers={'x-garage-causality-token': ct}, - data='{}: Concurrent value, oops'.format(datetime.timestamp(datetime.now()))) - print(response.headers) - print(response.text) - -for sk in sort_keys: - print("-- (%s) Get"%sk) - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - ct = response.headers["x-garage-causality-token"] - - print("-- Delete") - response = requests.delete('http://localhost:3812/alex/root?sort_key=%s'%sk, - headers={'x-garage-causality-token': ct}, - auth=auth) - print(response.headers) - print(response.text) - -print("-- ReadIndex") -response = requests.get('http://localhost:3812/alex', - auth=auth) -print(response.headers) -print(response.text) - -print("-- InsertBatch") -response = requests.post('http://localhost:3812/alex', - auth=auth, - data=''' -[ - {"pk": "root", "sk": "a", "ct": null, "v": "aW5pdGlhbCB0ZXN0Cg=="}, - {"pk": "root", "sk": "b", "ct": null, "v": "aW5pdGlhbCB0ZXN1Cg=="}, - {"pk": "root", "sk": "c", "ct": null, "v": "aW5pdGlhbCB0ZXN2Cg=="} -] -''') -print(response.headers) -print(response.text) - -print("-- ReadIndex") -response = requests.get('http://localhost:3812/alex', - auth=auth) -print(response.headers) -print(response.text) - -for sk in sort_keys: - print("-- (%s) Get"%sk) - response = requests.get('http://localhost:3812/alex/root?sort_key=%s'%sk, - auth=auth) - print(response.headers) - print(response.text) - ct = response.headers["x-garage-causality-token"] - -print("-- ReadBatch") -response = requests.post('http://localhost:3812/alex?search', - auth=auth, - data=''' -[ - {"partitionKey": "root"}, - {"partitionKey": "root", "tombstones": true}, - {"partitionKey": "root", "tombstones": true, "limit": 2}, - {"partitionKey": "root", "start": "c", "singleItem": true}, - {"partitionKey": "root", "start": "b", "end": "d", "tombstones": true} -] -''') -print(response.headers) -print(response.text) - - -print("-- DeleteBatch") -response = requests.post('http://localhost:3812/alex?delete', - auth=auth, - data=''' -[ - {"partitionKey": "root", "start": "b", "end": "c"} -] -''') -print(response.headers) -print(response.text) - -print("-- ReadBatch") -response = requests.post('http://localhost:3812/alex?search', - auth=auth, - data=''' -[ - {"partitionKey": "root"} -] -''') -print(response.headers) -print(response.text) diff --git a/nix/compile.nix b/nix/compile.nix index a31a2005..6a92e479 100644 --- a/nix/compile.nix +++ b/nix/compile.nix @@ -173,6 +173,7 @@ let "garage/kubernetes-discovery" "garage/metrics" "garage/telemetry-otlp" + "garage/syslog" ] else [ ])); diff --git a/src/block/layout.rs b/src/block/layout.rs index e8339405..e78f3f08 100644 --- a/src/block/layout.rs +++ b/src/block/layout.rs @@ -1,3 +1,4 @@ +use std::collections::HashMap; use std::path::PathBuf; use serde::{Deserialize, Serialize}; @@ -13,9 +14,12 @@ const DRIVE_NPART: usize = 1024; const HASH_DRIVE_BYTES: (usize, usize) = (2, 3); +const MARKER_FILE_NAME: &str = "garage-marker"; + #[derive(Serialize, Deserialize, Debug, Clone)] pub(crate) struct DataLayout { pub(crate) data_dirs: Vec, + markers: HashMap, /// Primary storage location (index in data_dirs) for each partition /// = the location where the data is supposed to be, blocks are always @@ -75,16 +79,17 @@ impl DataLayout { Ok(Self { data_dirs, + markers: HashMap::new(), part_prim, part_sec, }) } - pub(crate) fn update(&mut self, dirs: &DataDirEnum) -> Result<(), Error> { + pub(crate) fn update(self, dirs: &DataDirEnum) -> Result { // Make list of new data directories, exit if nothing changed let data_dirs = make_data_dirs(dirs)?; if data_dirs == self.data_dirs { - return Ok(()); + return Ok(self); } let total_cap = data_dirs.iter().filter_map(|x| x.capacity()).sum::(); @@ -214,11 +219,43 @@ impl DataLayout { } // Apply newly generated config - *self = Self { + Ok(Self { data_dirs, + markers: self.markers, part_prim, part_sec, - }; + }) + } + + pub(crate) fn check_markers(&mut self) -> Result<(), Error> { + let data_dirs = &self.data_dirs; + self.markers + .retain(|k, _| data_dirs.iter().any(|x| x.path == *k)); + + for dir in self.data_dirs.iter() { + let mut marker_path = dir.path.clone(); + marker_path.push(MARKER_FILE_NAME); + let existing_marker = std::fs::read_to_string(&marker_path).ok(); + match (existing_marker, self.markers.get(&dir.path)) { + (Some(m1), Some(m2)) => { + if m1 != *m2 { + return Err(Error::Message(format!("Mismatched content for marker file `{}` in data directory `{}`. If you moved data directories or changed their mountpoints, you should remove the `data_layout` file in Garage's metadata directory and restart Garage.", MARKER_FILE_NAME, dir.path.display()))); + } + } + (None, Some(_)) => { + return Err(Error::Message(format!("Could not find expected marker file `{}` in data directory `{}`, make sure this data directory is mounted correctly.", MARKER_FILE_NAME, dir.path.display()))); + } + (Some(mkr), None) => { + self.markers.insert(dir.path.clone(), mkr); + } + (None, None) => { + let mkr = hex::encode(garage_util::data::gen_uuid().as_slice()); + std::fs::write(&marker_path, &mkr)?; + self.markers.insert(dir.path.clone(), mkr); + } + } + } + Ok(()) } @@ -255,6 +292,7 @@ impl DataLayout { pub(crate) fn without_secondary_locations(&self) -> Self { Self { data_dirs: self.data_dirs.clone(), + markers: self.markers.clone(), part_prim: self.part_prim.clone(), part_sec: self.part_sec.iter().map(|_| vec![]).collect::>(), } @@ -322,14 +360,12 @@ fn make_data_dirs(dirs: &DataDirEnum) -> Result, Error> { fn dir_not_empty(path: &PathBuf) -> Result { for entry in std::fs::read_dir(&path)? { let dir = entry?; - if dir.file_type()?.is_dir() - && dir - .file_name() - .into_string() - .ok() - .and_then(|hex| hex::decode(&hex).ok()) - .is_some() - { + let ft = dir.file_type()?; + let name = dir.file_name().into_string().ok(); + if ft.is_file() && name.as_deref() == Some(MARKER_FILE_NAME) { + return Ok(true); + } + if ft.is_dir() && name.and_then(|hex| hex::decode(&hex).ok()).is_some() { return Ok(true); } } diff --git a/src/block/manager.rs b/src/block/manager.rs index 8ee33096..82db2cab 100644 --- a/src/block/manager.rs +++ b/src/block/manager.rs @@ -127,17 +127,15 @@ impl BlockManager { // Load or compute layout, i.e. assignment of data blocks to the different data directories let data_layout_persister: Persister = Persister::new(&system.metadata_dir, "data_layout"); - let data_layout = match data_layout_persister.load() { - Ok(mut layout) => { - layout - .update(&config.data_dir) - .ok_or_message("invalid data_dir config")?; - layout - } + let mut data_layout = match data_layout_persister.load() { + Ok(layout) => layout + .update(&config.data_dir) + .ok_or_message("invalid data_dir config")?, Err(_) => { DataLayout::initialize(&config.data_dir).ok_or_message("invalid data_dir config")? } }; + data_layout.check_markers()?; data_layout_persister .save(&data_layout) .expect("cannot save data_layout"); diff --git a/src/garage/Cargo.toml b/src/garage/Cargo.toml index 17da68f8..a4acbb1f 100644 --- a/src/garage/Cargo.toml +++ b/src/garage/Cargo.toml @@ -59,6 +59,7 @@ opentelemetry.workspace = true opentelemetry-prometheus = { workspace = true, optional = true } opentelemetry-otlp = { workspace = true, optional = true } prometheus = { workspace = true, optional = true } +syslog-tracing = { workspace = true, optional = true } [dev-dependencies] aws-config.workspace = true @@ -97,6 +98,8 @@ kubernetes-discovery = [ "garage_rpc/kubernetes-discovery" ] metrics = [ "garage_api/metrics", "opentelemetry-prometheus", "prometheus" ] # Exporter for the OpenTelemetry Collector. telemetry-otlp = [ "opentelemetry-otlp" ] +# Logging to syslog +syslog = [ "syslog-tracing" ] # NOTE: bundled-libs and system-libs should be treat as mutually exclusive; # exactly one of them should be enabled. diff --git a/src/garage/main.rs b/src/garage/main.rs index 5e9c061f..92fd4d0c 100644 --- a/src/garage/main.rs +++ b/src/garage/main.rs @@ -138,17 +138,8 @@ async fn main() { let opt = Opt::from_clap(&Opt::clap().version(version.as_str()).get_matches()); // Initialize logging as well as other libraries used in Garage - if std::env::var("RUST_LOG").is_err() { - let default_log = match &opt.cmd { - Command::Server => "netapp=info,garage=info", - _ => "netapp=warn,garage=warn", - }; - std::env::set_var("RUST_LOG", default_log) - } - tracing_subscriber::fmt() - .with_writer(std::io::stderr) - .with_env_filter(tracing_subscriber::filter::EnvFilter::from_default_env()) - .init(); + init_logging(&opt); + sodiumoxide::init().expect("Unable to init sodiumoxide"); let res = match opt.cmd { @@ -171,6 +162,58 @@ async fn main() { } } +fn init_logging(opt: &Opt) { + if std::env::var("RUST_LOG").is_err() { + let default_log = match &opt.cmd { + Command::Server => "netapp=info,garage=info", + _ => "netapp=warn,garage=warn", + }; + std::env::set_var("RUST_LOG", default_log) + } + + let env_filter = tracing_subscriber::filter::EnvFilter::from_default_env(); + + if std::env::var("GARAGE_LOG_TO_SYSLOG") + .map(|x| x == "1" || x == "true") + .unwrap_or(false) + { + #[cfg(feature = "syslog")] + { + use std::ffi::CStr; + use syslog_tracing::{Facility, Options, Syslog}; + + let syslog = Syslog::new( + CStr::from_bytes_with_nul(b"garage\0").unwrap(), + Options::LOG_PID | Options::LOG_PERROR, + Facility::Daemon, + ) + .expect("Unable to init syslog"); + + tracing_subscriber::fmt() + .with_writer(syslog) + .with_env_filter(env_filter) + .with_ansi(false) // disable ANSI escape sequences (colours) + .with_file(false) + .with_level(false) + .without_time() + .compact() + .init(); + + return; + } + #[cfg(not(feature = "syslog"))] + { + eprintln!("Syslog support is not enabled in this build."); + std::process::exit(1); + } + } + + tracing_subscriber::fmt() + .with_writer(std::io::stderr) + .with_env_filter(env_filter) + .init(); +} + async fn cli_command(opt: Opt) -> Result<(), Error> { let config = if (opt.secrets.rpc_secret.is_none() && opt.secrets.rpc_secret_file.is_none()) || opt.rpc_host.is_none() diff --git a/src/garage/tests/k2v/poll.rs b/src/garage/tests/k2v/poll.rs index 277f8bc8..7c06cea9 100644 --- a/src/garage/tests/k2v/poll.rs +++ b/src/garage/tests/k2v/poll.rs @@ -10,6 +10,7 @@ use crate::common; use crate::json_body; #[tokio::test] +#[ignore = "currently broken"] async fn test_poll_item() { let ctx = common::context(); let bucket = ctx.create_bucket("test-k2v-poll-item"); @@ -98,6 +99,7 @@ async fn test_poll_item() { } #[tokio::test] +#[ignore = "currently broken"] async fn test_poll_range() { let ctx = common::context(); let bucket = ctx.create_bucket("test-k2v-poll-range");