WIP: Windows support #925
22 changed files with 151 additions and 174 deletions
|
@ -16,7 +16,7 @@ steps:
|
|||
- name: build
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
|
||||
- nix-build --no-build-output --attr pkgs.amd64.debug --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
|
||||
|
||||
- name: unit + func tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
|
@ -24,7 +24,7 @@ steps:
|
|||
GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage
|
||||
GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
|
||||
- nix-build --no-build-output --attr pkgs.amd64.debug --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
|
||||
- nix-build --no-build-output --attr test.amd64
|
||||
- ./result/bin/garage_db-*
|
||||
- ./result/bin/garage_api-*
|
||||
|
@ -43,5 +43,5 @@ steps:
|
|||
- name: integration tests
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
commands:
|
||||
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
|
||||
- nix-build --no-build-output --attr pkgs.amd64.debug --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
|
||||
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
|
|
@ -9,11 +9,11 @@ depends_on:
|
|||
steps:
|
||||
- name: refresh-index
|
||||
image: nixpkgs/nix:nixos-22.05
|
||||
secrets:
|
||||
- source: garagehq_aws_access_key_id
|
||||
target: AWS_ACCESS_KEY_ID
|
||||
- source: garagehq_aws_secret_access_key
|
||||
target: AWS_SECRET_ACCESS_KEY
|
||||
environment:
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-shell --attr ci --run "refresh_index"
|
||||
|
|
|
@ -48,11 +48,10 @@ steps:
|
|||
image: nixpkgs/nix:nixos-22.05
|
||||
environment:
|
||||
TARGET: "${TARGET}"
|
||||
secrets:
|
||||
- source: garagehq_aws_access_key_id
|
||||
target: AWS_ACCESS_KEY_ID
|
||||
- source: garagehq_aws_secret_access_key
|
||||
target: AWS_SECRET_ACCESS_KEY
|
||||
AWS_ACCESS_KEY_ID:
|
||||
from_secret: garagehq_aws_access_key_id
|
||||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- nix-shell --attr ci --run "to_s3"
|
||||
|
||||
|
|
27
Cargo.nix
27
Cargo.nix
|
@ -25,6 +25,7 @@ args@{
|
|||
target ? null,
|
||||
codegenOpts ? null,
|
||||
profileOpts ? null,
|
||||
cargoUnstableFlags ? null,
|
||||
rustcLinkFlags ? null,
|
||||
rustcBuildFlags ? null,
|
||||
mkRustCrate,
|
||||
|
@ -51,7 +52,7 @@ else let
|
|||
rootFeatures' = expandFeatures rootFeatures;
|
||||
overridableMkRustCrate = f:
|
||||
let
|
||||
drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts rustcLinkFlags rustcBuildFlags; } // (f profileName)));
|
||||
drvs = genDrvsByProfile profilesByName ({ profile, profileName }: mkRustCrate ({ inherit release profile hostPlatformCpu hostPlatformFeatures target profileOpts codegenOpts cargoUnstableFlags rustcLinkFlags rustcBuildFlags; } // (f profileName)));
|
||||
in { compileMode ? null, profileName ? decideProfile compileMode release }:
|
||||
let drv = drvs.${profileName}; in if compileMode == null then drv else drv.override { inherit compileMode; };
|
||||
in
|
||||
|
@ -1756,7 +1757,7 @@ in
|
|||
name = "format_table";
|
||||
version = "0.1.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/format-table");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
});
|
||||
|
||||
"registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" = overridableMkRustCrate (profileName: rec {
|
||||
|
@ -1914,7 +1915,7 @@ in
|
|||
name = "garage";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/garage");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default") "bundled-libs")
|
||||
(lib.optional (rootFeatures' ? "garage/consul-discovery") "consul-discovery")
|
||||
|
@ -1992,7 +1993,7 @@ in
|
|||
name = "garage_api";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/api");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v") "k2v")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/metrics" || rootFeatures' ? "garage_api/metrics") "metrics")
|
||||
|
@ -2056,7 +2057,7 @@ in
|
|||
name = "garage_block";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/block");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/system-libs" || rootFeatures' ? "garage_block/system-libs") "system-libs")
|
||||
];
|
||||
|
@ -2089,7 +2090,7 @@ in
|
|||
name = "garage_db";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/db");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/bundled-libs" || rootFeatures' ? "garage/default" || rootFeatures' ? "garage_db/bundled-libs") "bundled-libs")
|
||||
(lib.optional (rootFeatures' ? "garage_db/default") "default")
|
||||
|
@ -2118,7 +2119,7 @@ in
|
|||
name = "garage_model";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/model");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage_model/default") "default")
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v") "k2v")
|
||||
|
@ -2157,7 +2158,7 @@ in
|
|||
name = "garage_net";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/net");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
features = builtins.concatLists [
|
||||
[ "default" ]
|
||||
(lib.optional (rootFeatures' ? "garage_net/opentelemetry" || rootFeatures' ? "garage_net/telemetry") "opentelemetry")
|
||||
|
@ -2194,7 +2195,7 @@ in
|
|||
name = "garage_rpc";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/rpc");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery") "consul-discovery")
|
||||
(lib.optional (rootFeatures' ? "garage/consul-discovery" || rootFeatures' ? "garage_rpc/consul-discovery" || rootFeatures' ? "garage_rpc/err-derive") "err-derive")
|
||||
|
@ -2243,7 +2244,7 @@ in
|
|||
name = "garage_table";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/table");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
dependencies = {
|
||||
arc_swap = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".arc-swap."1.6.0" { inherit profileName; }).out;
|
||||
async_trait = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".async-trait."0.1.77" { profileName = "__noProfile"; }).out;
|
||||
|
@ -2268,7 +2269,7 @@ in
|
|||
name = "garage_util";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/util");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "garage/default" || rootFeatures' ? "garage/k2v" || rootFeatures' ? "garage_api/k2v" || rootFeatures' ? "garage_model/k2v" || rootFeatures' ? "garage_util/k2v") "k2v")
|
||||
];
|
||||
|
@ -2312,7 +2313,7 @@ in
|
|||
name = "garage_web";
|
||||
version = "1.0.1";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/web");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
dependencies = {
|
||||
err_derive = (buildRustPackages."registry+https://github.com/rust-lang/crates.io-index".err-derive."0.3.1" { profileName = "__noProfile"; }).out;
|
||||
futures = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".futures."0.3.30" { inherit profileName; }).out;
|
||||
|
@ -3148,7 +3149,7 @@ in
|
|||
name = "k2v-client";
|
||||
version = "0.0.4";
|
||||
registry = "unknown";
|
||||
src = fetchCrateLocal (workspaceSrc + "/src/k2v-client");
|
||||
src = fetchCrateLocal workspaceSrc;
|
||||
features = builtins.concatLists [
|
||||
(lib.optional (rootFeatures' ? "k2v-client/clap" || rootFeatures' ? "k2v-client/cli") "clap")
|
||||
(lib.optional (rootFeatures' ? "k2v-client/cli") "cli")
|
||||
|
|
|
@ -45,11 +45,4 @@ in {
|
|||
];
|
||||
});
|
||||
};
|
||||
clippy = {
|
||||
amd64 = (compile {
|
||||
inherit system git_version pkgsSrc cargo2nixOverlay;
|
||||
target = "x86_64-unknown-linux-musl";
|
||||
compiler = "clippy";
|
||||
}).workspace.garage { compileMode = "build"; };
|
||||
};
|
||||
}
|
||||
|
|
|
@ -23,7 +23,7 @@ client = minio.Minio(
|
|||
"GKyourapikey",
|
||||
"abcd[...]1234",
|
||||
# Force the region, this is specific to garage
|
||||
region="region",
|
||||
region="garage",
|
||||
)
|
||||
```
|
||||
|
||||
|
|
|
@ -335,6 +335,7 @@ From the [official Mastodon documentation](https://docs.joinmastodon.org/admin/t
|
|||
|
||||
```bash
|
||||
$ RAILS_ENV=production bin/tootctl media remove --days 3
|
||||
$ RAILS_ENV=production bin/tootctl media remove --days 15 --prune-profiles
|
||||
$ RAILS_ENV=production bin/tootctl media remove-orphans
|
||||
$ RAILS_ENV=production bin/tootctl preview_cards remove --days 15
|
||||
```
|
||||
|
@ -353,8 +354,6 @@ Imports: 1.7 KB
|
|||
Settings: 0 Bytes
|
||||
```
|
||||
|
||||
Unfortunately, [old avatars and headers cannot currently be cleaned up](https://github.com/mastodon/mastodon/issues/9567).
|
||||
|
||||
### Migrating your data
|
||||
|
||||
Data migration should be done with an efficient S3 client.
|
||||
|
|
|
@ -50,3 +50,20 @@ locations. They use Garage themselves for the following tasks:
|
|||
|
||||
The Deuxfleurs Garage cluster is a multi-site cluster currently composed of
|
||||
9 nodes in 3 physical locations.
|
||||
|
||||
### Triplebit
|
||||
|
||||
[Triplebit](https://www.triplebit.org) is a non-profit hosting provider and
|
||||
ISP focused on improving access to privacy-related services. They use
|
||||
Garage themselves for the following tasks:
|
||||
|
||||
- Hosting of their homepage, [privacyguides.org](https://www.privacyguides.org/), and various other static sites
|
||||
|
||||
- As a Mastodon object storage backend for [mstdn.party](https://mstdn.party/) and [mstdn.plus](https://mstdn.plus/)
|
||||
|
||||
- As a PeerTube storage backend for [neat.tube](https://neat.tube/)
|
||||
|
||||
- As a [Matrix media backend](https://github.com/matrix-org/synapse-s3-storage-provider)
|
||||
|
||||
Triplebit's Garage cluster is a multi-site cluster currently composed of
|
||||
10 nodes in 3 physical locations.
|
||||
|
|
|
@ -199,7 +199,7 @@ For instance here you could write just `garage layout assign -z dc1 -c 1G 563e`.
|
|||
The layout then has to be applied to the cluster, using:
|
||||
|
||||
```bash
|
||||
garage layout apply
|
||||
garage layout apply --version 1
|
||||
```
|
||||
|
||||
|
||||
|
|
|
@ -16,6 +16,7 @@ data_dir = "/var/lib/garage/data"
|
|||
metadata_fsync = true
|
||||
data_fsync = false
|
||||
disable_scrub = false
|
||||
use_local_tz = false
|
||||
metadata_auto_snapshot_interval = "6h"
|
||||
|
||||
db_engine = "lmdb"
|
||||
|
@ -99,6 +100,7 @@ Top-level configuration options:
|
|||
[`data_fsync`](#data_fsync),
|
||||
[`db_engine`](#db_engine),
|
||||
[`disable_scrub`](#disable_scrub),
|
||||
[`use_local_tz`](#use_local_tz),
|
||||
[`lmdb_map_size`](#lmdb_map_size),
|
||||
[`metadata_auto_snapshot_interval`](#metadata_auto_snapshot_interval),
|
||||
[`metadata_dir`](#metadata_dir),
|
||||
|
@ -427,6 +429,13 @@ you should delete it from the data directory and then call `garage repair
|
|||
blocks` on the node to ensure that it re-obtains a copy from another node on
|
||||
the network.
|
||||
|
||||
#### `use_local_tz` {#use_local_tz}
|
||||
|
||||
By default, Garage runs the lifecycle worker every day at midnight in UTC. Set the
|
||||
`use_local_tz` configuration value to `true` if you want Garage to run the
|
||||
lifecycle worker at midnight in your local timezone. If you have multiple nodes,
|
||||
you should also ensure that each node has the same timezone configuration.
|
||||
|
||||
#### `block_size` {#block_size}
|
||||
|
||||
Garage splits stored objects in consecutive chunks of size `block_size`
|
||||
|
|
49
flake.lock
49
flake.lock
|
@ -12,17 +12,17 @@
|
|||
"rust-overlay": "rust-overlay"
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1666087781,
|
||||
"narHash": "sha256-trKVdjMZ8mNkGfLcY5LsJJGtdV3xJDZnMVrkFjErlcs=",
|
||||
"owner": "Alexis211",
|
||||
"lastModified": 1705129117,
|
||||
"narHash": "sha256-LgdDHibvimzYhxBK3kxCk2gAL7k4Hyigl5KI0X9cijA=",
|
||||
"owner": "cargo2nix",
|
||||
"repo": "cargo2nix",
|
||||
"rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
|
||||
"rev": "ae19a9e1f8f0880c088ea155ab66cee1fa001f59",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "Alexis211",
|
||||
"owner": "cargo2nix",
|
||||
"repo": "cargo2nix",
|
||||
"rev": "a7a61179b66054904ef6a195d8da736eaaa06c36",
|
||||
"rev": "ae19a9e1f8f0880c088ea155ab66cee1fa001f59",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
|
@ -58,33 +58,17 @@
|
|||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1724395761,
|
||||
"narHash": "sha256-zRkDV/nbrnp3Y8oCADf5ETl1sDrdmAW6/bBVJ8EbIdQ=",
|
||||
"lastModified": 1736692550,
|
||||
"narHash": "sha256-7tk8xH+g0sJkKLTJFOxphJxxOjMDFMWv24nXslaU2ro=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "ae815cee91b417be55d43781eb4b73ae1ecc396c",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
"nixpkgs_2": {
|
||||
"locked": {
|
||||
"lastModified": 1724681257,
|
||||
"narHash": "sha256-EJRuc5Qp7yfXko5ZNeEMYAs4DzAvkCyALuJ/tGllhN4=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "0239aeb2f82ea27ccd6b61582b8f7fb8750eeada",
|
||||
"rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "0239aeb2f82ea27ccd6b61582b8f7fb8750eeada",
|
||||
"rev": "7c4869c47090dd7f9f1bdfb49a22aea026996815",
|
||||
"type": "github"
|
||||
}
|
||||
},
|
||||
|
@ -96,19 +80,22 @@
|
|||
"cargo2nix",
|
||||
"flake-utils"
|
||||
],
|
||||
"nixpkgs": "nixpkgs_2"
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
},
|
||||
"rust-overlay": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
"nixpkgs": [
|
||||
"cargo2nix",
|
||||
"nixpkgs"
|
||||
]
|
||||
},
|
||||
"locked": {
|
||||
"lastModified": 1724638882,
|
||||
"narHash": "sha256-ap2jIQi/FuUHR6HCht6ASWhoz8EiB99XmI8Esot38VE=",
|
||||
"lastModified": 1736649126,
|
||||
"narHash": "sha256-XCw5sv/ePsroqiF3lJM6Y2X9EhPdHeE47gr3Q8b0UQw=",
|
||||
"owner": "oxalica",
|
||||
"repo": "rust-overlay",
|
||||
"rev": "19b70f147b9c67a759e35824b241f1ed92e46694",
|
||||
"rev": "162ab0edc2936508470199b2e8e6c444a2535019",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
|
|
13
flake.nix
13
flake.nix
|
@ -2,24 +2,27 @@
|
|||
description =
|
||||
"Garage, an S3-compatible distributed object store for self-hosted deployments";
|
||||
|
||||
# Nixpkgs 24.05 as of 2024-08-26 has rustc v1.77
|
||||
# Nixpkgs 24.11 as of 2025-01-12 has rustc v1.82
|
||||
inputs.nixpkgs.url =
|
||||
"github:NixOS/nixpkgs/0239aeb2f82ea27ccd6b61582b8f7fb8750eeada";
|
||||
"github:NixOS/nixpkgs/7c4869c47090dd7f9f1bdfb49a22aea026996815";
|
||||
|
||||
inputs.flake-compat.url = "github:nix-community/flake-compat";
|
||||
|
||||
inputs.cargo2nix = {
|
||||
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
|
||||
url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
|
||||
#url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
|
||||
|
||||
# As of 2023-04-25:
|
||||
# - my two patches were merged into unstable (one for clippy and one to "fix" feature detection)
|
||||
# - rustc v1.66
|
||||
# url = "github:cargo2nix/cargo2nix/8fb57a670f7993bfc24099c33eb9c5abb51f29a2";
|
||||
|
||||
# Rust overlay as of 2024-08-26
|
||||
# Mainline cargo2nix as of of 2025-01-12 (branch release-0.11.0)
|
||||
url = "github:cargo2nix/cargo2nix/ae19a9e1f8f0880c088ea155ab66cee1fa001f59";
|
||||
|
||||
# Rust overlay as of 2025-01-12
|
||||
inputs.rust-overlay.url =
|
||||
"github:oxalica/rust-overlay/19b70f147b9c67a759e35824b241f1ed92e46694";
|
||||
"github:oxalica/rust-overlay/162ab0edc2936508470199b2e8e6c444a2535019";
|
||||
|
||||
inputs.nixpkgs.follows = "nixpkgs";
|
||||
inputs.flake-compat.follows = "flake-compat";
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
{ system, target ? null, pkgsSrc, cargo2nixOverlay, compiler ? "rustc"
|
||||
{ system, target ? null, pkgsSrc, cargo2nixOverlay
|
||||
, release ? false, git_version ? null, features ? null, }:
|
||||
|
||||
let
|
||||
|
@ -66,24 +66,10 @@ let
|
|||
};
|
||||
|
||||
toolchainOptions = {
|
||||
rustVersion = "1.77.0";
|
||||
rustVersion = "1.78.0";
|
||||
extraRustComponents = [ "clippy" ];
|
||||
};
|
||||
|
||||
buildEnv = (drv:
|
||||
{
|
||||
rustc = drv.setBuildEnv;
|
||||
clippy = ''
|
||||
${drv.setBuildEnv or ""}
|
||||
echo
|
||||
echo --- BUILDING WITH CLIPPY ---
|
||||
echo
|
||||
|
||||
export NIX_RUST_BUILD_FLAGS="''${NIX_RUST_BUILD_FLAGS} --deny warnings"
|
||||
export RUSTC="''${CLIPPY_DRIVER}"
|
||||
'';
|
||||
}.${compiler});
|
||||
|
||||
/* Cargo2nix provides many overrides by default, you can take inspiration from them:
|
||||
https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix
|
||||
|
||||
|
@ -92,9 +78,7 @@ let
|
|||
*/
|
||||
packageOverrides = pkgs:
|
||||
pkgs.rustBuilder.overrides.all ++ [
|
||||
/* [1] We add some logic to compile our crates with clippy, it provides us many additional lints
|
||||
|
||||
[2] We need to alter Nix hardening to make static binaries: PIE,
|
||||
/* [1] We need to alter Nix hardening to make static binaries: PIE,
|
||||
Position Independent Executables seems to be supported only on amd64. Having
|
||||
this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms.
|
||||
Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets
|
||||
|
@ -102,11 +86,11 @@ let
|
|||
PIE is a feature used by ASLR, which helps mitigate security issues.
|
||||
Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh
|
||||
|
||||
[3] We want to inject the git version while keeping the build deterministic.
|
||||
[2] We want to inject the git version while keeping the build deterministic.
|
||||
As we do not want to consider the .git folder as part of the input source,
|
||||
we ask the user (the CI often) to pass the value to Nix.
|
||||
|
||||
[4] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library.
|
||||
[3] We don't want libsodium-sys and zstd-sys to try to use pkgconfig to build against a system library.
|
||||
However the features to do so get activated for some reason (due to a bug in cargo2nix?),
|
||||
so disable them manually here.
|
||||
*/
|
||||
|
@ -114,7 +98,7 @@ let
|
|||
name = "garage";
|
||||
overrideAttrs = drv:
|
||||
(if git_version != null then {
|
||||
# [3]
|
||||
# [2]
|
||||
preConfigure = ''
|
||||
${drv.preConfigure or ""}
|
||||
export GIT_VERSION="${git_version}"
|
||||
|
@ -122,8 +106,6 @@ let
|
|||
} else
|
||||
{ }) // {
|
||||
# [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
# [2]
|
||||
hardeningDisable = [ "pie" ];
|
||||
};
|
||||
|
||||
|
@ -139,74 +121,11 @@ let
|
|||
} else {};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_rpc";
|
||||
overrideAttrs = drv: { # [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_db";
|
||||
overrideAttrs = drv: { # [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_util";
|
||||
overrideAttrs = drv: { # [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_table";
|
||||
overrideAttrs = drv: { # [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_block";
|
||||
overrideAttrs = drv: { # [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_model";
|
||||
overrideAttrs = drv: { # [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_api";
|
||||
overrideAttrs = drv: { # [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_web";
|
||||
overrideAttrs = drv: { # [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "k2v-client";
|
||||
overrideAttrs = drv: { # [1]
|
||||
setBuildEnv = (buildEnv drv);
|
||||
};
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "libsodium-sys";
|
||||
|
||||
overrideArgs = old: {
|
||||
features = [ ]; # [4]
|
||||
features = [ ]; # [3]
|
||||
};
|
||||
|
||||
# libsodium-sys doesn't consider windows-gnu to be a "supported" build
|
||||
|
@ -223,7 +142,7 @@ let
|
|||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "zstd-sys";
|
||||
overrideArgs = old: {
|
||||
features = [ ]; # [4]
|
||||
features = [ ]; # [3]
|
||||
};
|
||||
})
|
||||
|
||||
|
|
|
@ -76,6 +76,9 @@ spec:
|
|||
- name: etc
|
||||
mountPath: /etc/garage.toml
|
||||
subPath: garage.toml
|
||||
{{- with .Values.extraVolumeMounts }}
|
||||
{{- toYaml . | nindent 12 }}
|
||||
{{- end }}
|
||||
# TODO
|
||||
# livenessProbe:
|
||||
# httpGet:
|
||||
|
@ -110,6 +113,9 @@ spec:
|
|||
- name: data
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- with .Values.extraVolumes }}
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
|
|
|
@ -218,6 +218,10 @@ affinity: {}
|
|||
|
||||
environment: {}
|
||||
|
||||
extraVolumes: {}
|
||||
|
||||
extraVolumeMounts: {}
|
||||
|
||||
monitoring:
|
||||
metrics:
|
||||
# If true, a service for monitoring is created with a prometheus.io/scrape annotation
|
||||
|
|
|
@ -115,7 +115,7 @@ in
|
|||
shellHook = ''
|
||||
function refresh_cache {
|
||||
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
|
||||
for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do
|
||||
for attr in pkgs.amd64.debug test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do
|
||||
echo "Updating cache for ''${attr}"
|
||||
nix copy -j8 \
|
||||
--to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \
|
||||
|
|
|
@ -279,7 +279,8 @@ impl DataLayout {
|
|||
u16::from_be_bytes([
|
||||
hash.as_slice()[HASH_DRIVE_BYTES.0],
|
||||
hash.as_slice()[HASH_DRIVE_BYTES.1],
|
||||
]) as usize % DRIVE_NPART
|
||||
]) as usize
|
||||
% DRIVE_NPART
|
||||
}
|
||||
|
||||
fn block_dir_from(&self, hash: &Hash, dir: &PathBuf) -> PathBuf {
|
||||
|
|
|
@ -141,7 +141,7 @@ impl Garage {
|
|||
)?)
|
||||
.ok()
|
||||
.and_then(|x| NetworkKey::from_slice(&x))
|
||||
.ok_or_message("Invalid RPC secret key: expected 32 bits of entropy, please check the documentation for requirements")?;
|
||||
.ok_or_message("Invalid RPC secret key: expected 32 bytes of random hex, please check the documentation for requirements")?;
|
||||
|
||||
let (replication_factor, consistency_mode) = parse_replication_mode(&config)?;
|
||||
|
||||
|
|
|
@ -279,7 +279,8 @@ impl<'a> LockedHelper<'a> {
|
|||
.local_aliases
|
||||
.get(alias_name)
|
||||
.cloned()
|
||||
.flatten() != Some(bucket_id)
|
||||
.flatten()
|
||||
!= Some(bucket_id)
|
||||
{
|
||||
return Err(GarageError::Message(format!(
|
||||
"Bucket {:?} does not have alias {} in namespace of key {}",
|
||||
|
|
|
@ -70,7 +70,7 @@ pub fn register_bg_vars(
|
|||
|
||||
impl LifecycleWorker {
|
||||
pub fn new(garage: Arc<Garage>, persister: PersisterShared<LifecycleWorkerPersisted>) -> Self {
|
||||
let today = today();
|
||||
let today = today(garage.config.use_local_tz);
|
||||
let last_completed = persister.get_with(|x| {
|
||||
x.last_completed
|
||||
.as_deref()
|
||||
|
@ -205,8 +205,9 @@ impl Worker for LifecycleWorker {
|
|||
async fn wait_for_work(&mut self) -> WorkerState {
|
||||
match &self.state {
|
||||
State::Completed(d) => {
|
||||
let use_local_tz = self.garage.config.use_local_tz;
|
||||
let next_day = d.succ_opt().expect("no next day");
|
||||
let next_start = midnight_ts(next_day);
|
||||
let next_start = midnight_ts(next_day, use_local_tz);
|
||||
loop {
|
||||
let now = now_msec();
|
||||
if now < next_start {
|
||||
|
@ -218,7 +219,7 @@ impl Worker for LifecycleWorker {
|
|||
break;
|
||||
}
|
||||
}
|
||||
self.state = State::start(std::cmp::max(next_day, today()));
|
||||
self.state = State::start(std::cmp::max(next_day, today(use_local_tz)));
|
||||
}
|
||||
State::Running { .. } => (),
|
||||
}
|
||||
|
@ -385,10 +386,16 @@ fn check_size_filter(version_data: &ObjectVersionData, filter: &LifecycleFilter)
|
|||
true
|
||||
}
|
||||
|
||||
fn midnight_ts(date: NaiveDate) -> u64 {
|
||||
date.and_hms_opt(0, 0, 0)
|
||||
.expect("midnight does not exist")
|
||||
.timestamp_millis() as u64
|
||||
fn midnight_ts(date: NaiveDate, use_local_tz: bool) -> u64 {
|
||||
let midnight = date.and_hms_opt(0, 0, 0).expect("midnight does not exist");
|
||||
if use_local_tz {
|
||||
return midnight
|
||||
.and_local_timezone(Local)
|
||||
.single()
|
||||
.expect("bad local midnight")
|
||||
.timestamp_millis() as u64;
|
||||
}
|
||||
midnight.timestamp_millis() as u64
|
||||
}
|
||||
|
||||
fn next_date(ts: u64) -> NaiveDate {
|
||||
|
@ -399,6 +406,9 @@ fn next_date(ts: u64) -> NaiveDate {
|
|||
.expect("no next day")
|
||||
}
|
||||
|
||||
fn today() -> NaiveDate {
|
||||
fn today(use_local_tz: bool) -> NaiveDate {
|
||||
if use_local_tz {
|
||||
return Local::now().naive_local().date();
|
||||
}
|
||||
Utc::now().naive_utc().date()
|
||||
}
|
||||
|
|
|
@ -818,6 +818,16 @@ impl NodeStatus {
|
|||
#[cfg(not(windows))]
|
||||
fn update_disk_usage(&mut self, meta_dir: &Path, data_dir: &DataDirEnum) {
|
||||
use nix::sys::statvfs::statvfs;
|
||||
|
||||
// The HashMap used below requires a filesystem identifier from statfs (instead of statvfs) on FreeBSD, as
|
||||
// FreeBSD's statvfs filesystem identifier is "not meaningful in this implementation" (man 3 statvfs).
|
||||
|
||||
#[cfg(target_os = "freebsd")]
|
||||
let get_filesystem_id = |path: &Path| match nix::sys::statfs::statfs(path) {
|
||||
Ok(fs) => Some(fs.filesystem_id()),
|
||||
Err(_) => None,
|
||||
};
|
||||
|
||||
let mount_avail = |path: &Path| match statvfs(path) {
|
||||
Ok(x) => {
|
||||
let avail = x.blocks_available() as u64 * x.fragment_size() as u64;
|
||||
|
@ -828,6 +838,7 @@ impl NodeStatus {
|
|||
};
|
||||
|
||||
self.meta_disk_avail = mount_avail(meta_dir).map(|(_, a, t)| (a, t));
|
||||
|
||||
self.data_disk_avail = match data_dir {
|
||||
DataDirEnum::Single(dir) => mount_avail(dir).map(|(_, a, t)| (a, t)),
|
||||
DataDirEnum::Multiple(dirs) => (|| {
|
||||
|
@ -838,12 +849,25 @@ impl NodeStatus {
|
|||
if dir.capacity.is_none() {
|
||||
continue;
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "freebsd"))]
|
||||
match mount_avail(&dir.path) {
|
||||
Some((fsid, avail, total)) => {
|
||||
mounts.insert(fsid, (avail, total));
|
||||
}
|
||||
None => return None,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "freebsd")]
|
||||
match get_filesystem_id(&dir.path) {
|
||||
Some(fsid) => match mount_avail(&dir.path) {
|
||||
Some((_, avail, total)) => {
|
||||
mounts.insert(fsid, (avail, total));
|
||||
}
|
||||
None => return None,
|
||||
},
|
||||
None => return None,
|
||||
}
|
||||
}
|
||||
Some(
|
||||
mounts
|
||||
|
|
|
@ -27,6 +27,10 @@ pub struct Config {
|
|||
#[serde(default)]
|
||||
pub disable_scrub: bool,
|
||||
|
||||
/// Use local timezone
|
||||
#[serde(default)]
|
||||
pub use_local_tz: bool,
|
||||
|
||||
/// Automatic snapshot interval for metadata
|
||||
#[serde(default)]
|
||||
pub metadata_auto_snapshot_interval: Option<String>,
|
||||
|
|
Loading…
Reference in a new issue