Merge branch 'main' into rustversion-label

This commit is contained in:
Alex 2023-03-13 16:14:13 +01:00
commit f37ec584b6
6 changed files with 125 additions and 41 deletions

25
Cargo.lock generated
View file

@ -1738,9 +1738,9 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146"
[[package]] [[package]]
name = "ipnetwork" name = "ipnetwork"
version = "0.18.0" version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4088d739b183546b239688ddbc79891831df421773df95e236daf7867866d355" checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e"
dependencies = [ dependencies = [
"serde", "serde",
] ]
@ -2214,6 +2214,12 @@ dependencies = [
"tokio-util 0.7.4", "tokio-util 0.7.4",
] ]
[[package]]
name = "no-std-net"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65"
[[package]] [[package]]
name = "nom" name = "nom"
version = "7.1.3" version = "7.1.3"
@ -2598,15 +2604,18 @@ checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
[[package]] [[package]]
name = "pnet_base" name = "pnet_base"
version = "0.28.0" version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25488cd551a753dcaaa6fffc9f69a7610a412dd8954425bf7ffad5f7d1156fb8" checksum = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e"
dependencies = [
"no-std-net",
]
[[package]] [[package]]
name = "pnet_datalink" name = "pnet_datalink"
version = "0.28.0" version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4d1f8ab1ef6c914cf51dc5dfe0be64088ea5f3b08bbf5a31abc70356d271198" checksum = "c302da22118d2793c312a35fb3da6846cb0fab6c3ad53fd67e37809b06cdafce"
dependencies = [ dependencies = [
"ipnetwork", "ipnetwork",
"libc", "libc",
@ -2617,9 +2626,9 @@ dependencies = [
[[package]] [[package]]
name = "pnet_sys" name = "pnet_sys"
version = "0.28.0" version = "0.33.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9a3f32b0df45515befd19eed04616f6b56a488da92afc61164ef455e955f07f" checksum = "faf7a58b2803d818a374be9278a1fe8f88fce14b936afbe225000cfcd9c73f16"
dependencies = [ dependencies = [
"libc", "libc",
"winapi", "winapi",

View file

@ -32,7 +32,7 @@ args@{
ignoreLockHash, ignoreLockHash,
}: }:
let let
nixifiedLockHash = "0ee5ebe74e19a2d2d3c41b80ed443c93f079f696681902dda4ca659be282d985"; nixifiedLockHash = "d36a8590fdf6eeb128a5a852d55945b595d5830291ad0aca95a21dcc1fab8681";
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc; workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock); currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
lockHashIgnored = if ignoreLockHash lockHashIgnored = if ignoreLockHash
@ -1783,7 +1783,7 @@ in
sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out; sodiumoxide = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".kuska-sodiumoxide."0.2.5-0" { inherit profileName; }).out;
netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out; netapp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".netapp."0.5.2" { inherit profileName; }).out;
opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out; opentelemetry = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".opentelemetry."0.17.0" { inherit profileName; }).out;
pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" { inherit profileName; }).out; pnet_datalink = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.33.0" { inherit profileName; }).out;
rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out; rand = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".rand."0.8.5" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.14" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/reqwest" then "reqwest" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".reqwest."0.11.14" { inherit profileName; }).out;
${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.11" { inherit profileName; }).out; ${ if rootFeatures' ? "garage/kubernetes-discovery" || rootFeatures' ? "garage_rpc/kubernetes-discovery" || rootFeatures' ? "garage_rpc/schemars" then "schemars" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".schemars."0.8.11" { inherit profileName; }).out;
@ -2423,11 +2423,11 @@ in
]; ];
}); });
"registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.18.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.20.0" = overridableMkRustCrate (profileName: rec {
name = "ipnetwork"; name = "ipnetwork";
version = "0.18.0"; version = "0.20.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "4088d739b183546b239688ddbc79891831df421773df95e236daf7867866d355"; }; src = fetchCratesIo { inherit name version; sha256 = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e"; };
features = builtins.concatLists [ features = builtins.concatLists [
[ "default" ] [ "default" ]
[ "serde" ] [ "serde" ]
@ -3105,6 +3105,16 @@ in
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".no-std-net."0.6.0" = overridableMkRustCrate (profileName: rec {
name = "no-std-net";
version = "0.6.0";
registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "43794a0ace135be66a25d3ae77d41b91615fb68ae937f904090203e81f755b65"; };
features = builtins.concatLists [
[ "std" ]
];
});
"registry+https://github.com/rust-lang/crates.io-index".nom."7.1.3" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".nom."7.1.3" = overridableMkRustCrate (profileName: rec {
name = "nom"; name = "nom";
version = "7.1.3"; version = "7.1.3";
@ -3639,32 +3649,42 @@ in
src = fetchCratesIo { inherit name version; sha256 = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"; }; src = fetchCratesIo { inherit name version; sha256 = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.28.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.33.0" = overridableMkRustCrate (profileName: rec {
name = "pnet_base"; name = "pnet_base";
version = "0.28.0"; version = "0.33.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "25488cd551a753dcaaa6fffc9f69a7610a412dd8954425bf7ffad5f7d1156fb8"; }; src = fetchCratesIo { inherit name version; sha256 = "872e46346144ebf35219ccaa64b1dffacd9c6f188cd7d012bd6977a2a838f42e"; };
features = builtins.concatLists [
[ "std" ]
];
dependencies = {
no_std_net = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".no-std-net."0.6.0" { inherit profileName; }).out;
};
}); });
"registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.28.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".pnet_datalink."0.33.0" = overridableMkRustCrate (profileName: rec {
name = "pnet_datalink"; name = "pnet_datalink";
version = "0.28.0"; version = "0.33.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "d4d1f8ab1ef6c914cf51dc5dfe0be64088ea5f3b08bbf5a31abc70356d271198"; }; src = fetchCratesIo { inherit name version; sha256 = "c302da22118d2793c312a35fb3da6846cb0fab6c3ad53fd67e37809b06cdafce"; };
features = builtins.concatLists [
[ "default" ]
[ "std" ]
];
dependencies = { dependencies = {
ipnetwork = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.18.0" { inherit profileName; }).out; ipnetwork = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".ipnetwork."0.20.0" { inherit profileName; }).out;
libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out;
pnet_base = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.28.0" { inherit profileName; }).out; pnet_base = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_base."0.33.0" { inherit profileName; }).out;
pnet_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.28.0" { inherit profileName; }).out; pnet_sys = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.33.0" { inherit profileName; }).out;
${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out;
}; };
}); });
"registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.28.0" = overridableMkRustCrate (profileName: rec { "registry+https://github.com/rust-lang/crates.io-index".pnet_sys."0.33.0" = overridableMkRustCrate (profileName: rec {
name = "pnet_sys"; name = "pnet_sys";
version = "0.28.0"; version = "0.33.0";
registry = "registry+https://github.com/rust-lang/crates.io-index"; registry = "registry+https://github.com/rust-lang/crates.io-index";
src = fetchCratesIo { inherit name version; sha256 = "d9a3f32b0df45515befd19eed04616f6b56a488da92afc61164ef455e955f07f"; }; src = fetchCratesIo { inherit name version; sha256 = "faf7a58b2803d818a374be9278a1fe8f88fce14b936afbe225000cfcd9c73f16"; };
dependencies = { dependencies = {
libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out; libc = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".libc."0.2.139" { inherit profileName; }).out;
${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out; ${ if hostPlatform.isWindows then "winapi" else null } = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".winapi."0.3.9" { inherit profileName; }).out;

View file

@ -12,6 +12,7 @@ These tools are particularly suitable for debug, backups, website deployments or
| [AWS CLI](#aws-cli) | ✅ | Recommended | | [AWS CLI](#aws-cli) | ✅ | Recommended |
| [rclone](#rclone) | ✅ | | | [rclone](#rclone) | ✅ | |
| [s3cmd](#s3cmd) | ✅ | | | [s3cmd](#s3cmd) | ✅ | |
| [s5cmd](#s5cmd) | ✅ | |
| [(Cyber)duck](#cyberduck) | ✅ | | | [(Cyber)duck](#cyberduck) | ✅ | |
| [WinSCP (libs3)](#winscp) | ✅ | CLI instructions only | | [WinSCP (libs3)](#winscp) | ✅ | CLI instructions only |
| [sftpgo](#sftpgo) | ✅ | | | [sftpgo](#sftpgo) | ✅ | |
@ -178,6 +179,26 @@ s3cmd put /tmp/hello.txt s3://my-bucket/
s3cmd get s3://my-bucket/hello.txt hello.txt s3cmd get s3://my-bucket/hello.txt hello.txt
``` ```
## `s5cmd`
Configure a credentials file as follows:
```bash
export AWS_ACCESS_KEY_ID=GK...
export AWS_SECRET_ACCESS_KEY=
export AWS_DEFAULT_REGION='garage'
export AWS_ENDPOINT='http://localhost:3900'
```
After adding these environment variables in your shell, `s5cmd` can be used
with:
```bash
s5cmd --endpoint-url=$AWS_ENDPOINT ls
```
See its usage output for other commands available.
## Cyberduck & duck {#cyberduck} ## Cyberduck & duck {#cyberduck}
Both Cyberduck (the GUI) and duck (the CLI) have a concept of "Connection Profiles" that contain some presets for a specific provider. Both Cyberduck (the GUI) and duck (the CLI) have a concept of "Connection Profiles" that contain some presets for a specific provider.

View file

@ -152,6 +152,7 @@ impl BlockManager {
tx_scrub_command: ArcSwapOption::new(None), tx_scrub_command: ArcSwapOption::new(None),
}); });
block_manager.endpoint.set_handler(block_manager.clone()); block_manager.endpoint.set_handler(block_manager.clone());
block_manager.scrub_persister.set_with(|_| ()).unwrap();
block_manager block_manager
} }

View file

@ -5,7 +5,6 @@ use std::time::Duration;
use async_trait::async_trait; use async_trait::async_trait;
use rand::Rng; use rand::Rng;
use serde::{Deserialize, Serialize};
use tokio::fs; use tokio::fs;
use tokio::select; use tokio::select;
use tokio::sync::mpsc; use tokio::sync::mpsc;
@ -162,6 +161,50 @@ impl Worker for RepairWorker {
// and whose parameter (esp. speed) can be controlled at runtime. // and whose parameter (esp. speed) can be controlled at runtime.
// ---- ---- ---- // ---- ---- ----
mod v081 {
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize)]
pub struct ScrubWorkerPersisted {
pub tranquility: u32,
pub(crate) time_last_complete_scrub: u64,
pub(crate) corruptions_detected: u64,
}
impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {}
}
mod v082 {
use serde::{Deserialize, Serialize};
use super::v081;
#[derive(Serialize, Deserialize)]
pub struct ScrubWorkerPersisted {
pub tranquility: u32,
pub(crate) time_last_complete_scrub: u64,
pub(crate) time_next_run_scrub: u64,
pub(crate) corruptions_detected: u64,
}
impl garage_util::migrate::Migrate for ScrubWorkerPersisted {
type Previous = v081::ScrubWorkerPersisted;
fn migrate(old: v081::ScrubWorkerPersisted) -> ScrubWorkerPersisted {
use crate::repair::randomize_next_scrub_run_time;
ScrubWorkerPersisted {
tranquility: old.tranquility,
time_last_complete_scrub: old.time_last_complete_scrub,
time_next_run_scrub: randomize_next_scrub_run_time(old.time_last_complete_scrub),
corruptions_detected: old.corruptions_detected,
}
}
}
}
pub use v082::*;
pub struct ScrubWorker { pub struct ScrubWorker {
manager: Arc<BlockManager>, manager: Arc<BlockManager>,
rx_cmd: mpsc::Receiver<ScrubWorkerCommand>, rx_cmd: mpsc::Receiver<ScrubWorkerCommand>,
@ -172,19 +215,11 @@ pub struct ScrubWorker {
persister: PersisterShared<ScrubWorkerPersisted>, persister: PersisterShared<ScrubWorkerPersisted>,
} }
#[derive(Serialize, Deserialize)] fn randomize_next_scrub_run_time(timestamp: u64) -> u64 {
pub struct ScrubWorkerPersisted {
pub tranquility: u32,
pub(crate) time_last_complete_scrub: u64,
pub(crate) time_next_run_scrub: u64,
pub(crate) corruptions_detected: u64,
}
fn randomize_next_scrub_run_time() -> u64 {
// Take SCRUB_INTERVAL and mix in a random interval of 10 days to attempt to // Take SCRUB_INTERVAL and mix in a random interval of 10 days to attempt to
// balance scrub load across different cluster nodes. // balance scrub load across different cluster nodes.
let next_run_timestamp = now_msec() let next_run_timestamp = timestamp
+ SCRUB_INTERVAL + SCRUB_INTERVAL
.saturating_add(Duration::from_secs( .saturating_add(Duration::from_secs(
rand::thread_rng().gen_range(0..3600 * 24 * 10), rand::thread_rng().gen_range(0..3600 * 24 * 10),
@ -194,12 +229,11 @@ fn randomize_next_scrub_run_time() -> u64 {
next_run_timestamp next_run_timestamp
} }
impl garage_util::migrate::InitialFormat for ScrubWorkerPersisted {}
impl Default for ScrubWorkerPersisted { impl Default for ScrubWorkerPersisted {
fn default() -> Self { fn default() -> Self {
ScrubWorkerPersisted { ScrubWorkerPersisted {
time_last_complete_scrub: 0, time_last_complete_scrub: 0,
time_next_run_scrub: randomize_next_scrub_run_time(), time_next_run_scrub: randomize_next_scrub_run_time(now_msec()),
tranquility: INITIAL_SCRUB_TRANQUILITY, tranquility: INITIAL_SCRUB_TRANQUILITY,
corruptions_detected: 0, corruptions_detected: 0,
} }
@ -361,7 +395,7 @@ impl Worker for ScrubWorker {
} else { } else {
self.persister.set_with(|p| { self.persister.set_with(|p| {
p.time_last_complete_scrub = now_msec(); p.time_last_complete_scrub = now_msec();
p.time_next_run_scrub = randomize_next_scrub_run_time(); p.time_next_run_scrub = randomize_next_scrub_run_time(now_msec());
})?; })?;
self.work = ScrubWorkerState::Finished; self.work = ScrubWorkerState::Finished;
self.tranquilizer.clear(); self.tranquilizer.clear();

View file

@ -37,8 +37,7 @@ k8s-openapi = { version = "0.16", features = ["v1_22"], optional = true }
schemars = { version = "0.8", optional = true } schemars = { version = "0.8", optional = true }
reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls-manual-roots", "json"] } reqwest = { version = "0.11", optional = true, default-features = false, features = ["rustls-tls-manual-roots", "json"] }
# newer version requires rust edition 2021 pnet_datalink = "0.33"
pnet_datalink = "0.28"
futures = "0.3" futures = "0.3"
futures-util = "0.3" futures-util = "0.3"