Compare commits
1 commit
4c5be79b80
...
b1ee3e54ba
| Author | SHA1 | Date | |
|---|---|---|---|
b1ee3e54ba
|
43 changed files with 414 additions and 3117 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -1227,7 +1227,6 @@ dependencies = [
|
|||
"hyper",
|
||||
"k2v-client",
|
||||
"kuska-sodiumoxide",
|
||||
"mktemp",
|
||||
"netapp",
|
||||
"opentelemetry",
|
||||
"opentelemetry-otlp",
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ args@{
|
|||
ignoreLockHash,
|
||||
}:
|
||||
let
|
||||
nixifiedLockHash = "a8ba32879366acd517e5c5feb96e9b009e1dc1991b51fb3acbb6dba2e5c0d935";
|
||||
nixifiedLockHash = "1a87886681a3ef0b83c95addc26674a538b8a93d35bc80db8998e1fcd0821f6c";
|
||||
workspaceSrc = if args.workspaceSrc == null then ./. else args.workspaceSrc;
|
||||
currentLockHash = builtins.hashFile "sha256" (workspaceSrc + /Cargo.lock);
|
||||
lockHashIgnored = if ignoreLockHash
|
||||
|
|
@ -1771,7 +1771,6 @@ in
|
|||
http = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".http."0.2.9" { inherit profileName; }).out;
|
||||
hyper = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".hyper."0.14.27" { inherit profileName; }).out;
|
||||
k2v_client = (rustPackages."unknown".k2v-client."0.0.4" { inherit profileName; }).out;
|
||||
mktemp = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".mktemp."0.5.0" { inherit profileName; }).out;
|
||||
serde_json = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".serde_json."1.0.105" { inherit profileName; }).out;
|
||||
sha2 = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".sha2."0.10.7" { inherit profileName; }).out;
|
||||
static_init = (rustPackages."registry+https://github.com/rust-lang/crates.io-index".static_init."1.0.3" { inherit profileName; }).out;
|
||||
|
|
|
|||
|
|
@ -1,24 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Garage Adminstration API v0</title>
|
||||
<!-- needed for adaptive design -->
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<link href="./css/redoc.css" rel="stylesheet">
|
||||
|
||||
<!--
|
||||
Redoc doesn't change outer page styles
|
||||
-->
|
||||
<style>
|
||||
body {
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<redoc spec-url='./garage-admin-v1.yml'></redoc>
|
||||
<script src="./redoc.standalone.js"> </script>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -37,84 +37,30 @@ import (
|
|||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
garage "git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-golang"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Initialization
|
||||
// Set Host and other parameters
|
||||
configuration := garage.NewConfiguration()
|
||||
configuration.Host = "127.0.0.1:3903"
|
||||
|
||||
|
||||
// We can now generate a client
|
||||
client := garage.NewAPIClient(configuration)
|
||||
|
||||
// Authentication is handled through the context pattern
|
||||
ctx := context.WithValue(context.Background(), garage.ContextAccessToken, "s3cr3t")
|
||||
|
||||
// Nodes
|
||||
fmt.Println("--- nodes ---")
|
||||
nodes, _, _ := client.NodesApi.GetNodes(ctx).Execute()
|
||||
fmt.Fprintf(os.Stdout, "First hostname: %v\n", nodes.KnownNodes[0].Hostname)
|
||||
capa := int64(1000000000)
|
||||
change := []garage.NodeRoleChange{
|
||||
garage.NodeRoleChange{NodeRoleUpdate: &garage.NodeRoleUpdate {
|
||||
Id: *nodes.KnownNodes[0].Id,
|
||||
Zone: "dc1",
|
||||
Capacity: *garage.NewNullableInt64(&capa),
|
||||
Tags: []string{ "fast", "amd64" },
|
||||
}},
|
||||
// Send a request
|
||||
resp, r, err := client.NodesApi.GetNodes(ctx).Execute()
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error when calling `NodesApi.GetNodes``: %v\n", err)
|
||||
fmt.Fprintf(os.Stderr, "Full HTTP response: %v\n", r)
|
||||
}
|
||||
staged, _, _ := client.LayoutApi.AddLayout(ctx).NodeRoleChange(change).Execute()
|
||||
msg, _, _ := client.LayoutApi.ApplyLayout(ctx).LayoutVersion(*garage.NewLayoutVersion(staged.Version + 1)).Execute()
|
||||
fmt.Printf(strings.Join(msg.Message, "\n")) // Layout configured
|
||||
|
||||
health, _, _ := client.NodesApi.GetHealth(ctx).Execute()
|
||||
fmt.Printf("Status: %s, nodes: %v/%v, storage: %v/%v, partitions: %v/%v\n", health.Status, health.ConnectedNodes, health.KnownNodes, health.StorageNodesOk, health.StorageNodes, health.PartitionsAllOk, health.Partitions)
|
||||
|
||||
// Key
|
||||
fmt.Println("\n--- key ---")
|
||||
key := "openapi-key"
|
||||
keyInfo, _, _ := client.KeyApi.AddKey(ctx).AddKeyRequest(garage.AddKeyRequest{Name: *garage.NewNullableString(&key) }).Execute()
|
||||
defer client.KeyApi.DeleteKey(ctx).Id(*keyInfo.AccessKeyId).Execute()
|
||||
fmt.Printf("AWS_ACCESS_KEY_ID=%s\nAWS_SECRET_ACCESS_KEY=%s\n", *keyInfo.AccessKeyId, *keyInfo.SecretAccessKey.Get())
|
||||
|
||||
id := *keyInfo.AccessKeyId
|
||||
canCreateBucket := true
|
||||
updateKeyRequest := *garage.NewUpdateKeyRequest()
|
||||
updateKeyRequest.SetName("openapi-key-updated")
|
||||
updateKeyRequest.SetAllow(garage.UpdateKeyRequestAllow { CreateBucket: &canCreateBucket })
|
||||
update, _, _ := client.KeyApi.UpdateKey(ctx).Id(id).UpdateKeyRequest(updateKeyRequest).Execute()
|
||||
fmt.Printf("Updated %v with key name %v\n", *update.AccessKeyId, *update.Name)
|
||||
|
||||
keyList, _, _ := client.KeyApi.ListKeys(ctx).Execute()
|
||||
fmt.Printf("Keys count: %v\n", len(keyList))
|
||||
|
||||
// Bucket
|
||||
fmt.Println("\n--- bucket ---")
|
||||
global_name := "global-ns-openapi-bucket"
|
||||
local_name := "local-ns-openapi-bucket"
|
||||
bucketInfo, _, _ := client.BucketApi.CreateBucket(ctx).CreateBucketRequest(garage.CreateBucketRequest{
|
||||
GlobalAlias: &global_name,
|
||||
LocalAlias: &garage.CreateBucketRequestLocalAlias {
|
||||
AccessKeyId: keyInfo.AccessKeyId,
|
||||
Alias: &local_name,
|
||||
},
|
||||
}).Execute()
|
||||
defer client.BucketApi.DeleteBucket(ctx).Id(*bucketInfo.Id).Execute()
|
||||
fmt.Printf("Bucket id: %s\n", *bucketInfo.Id)
|
||||
|
||||
updateBucketRequest := *garage.NewUpdateBucketRequest()
|
||||
website := garage.NewUpdateBucketRequestWebsiteAccess()
|
||||
website.SetEnabled(true)
|
||||
website.SetIndexDocument("index.html")
|
||||
website.SetErrorDocument("errors/4xx.html")
|
||||
updateBucketRequest.SetWebsiteAccess(*website)
|
||||
quotas := garage.NewUpdateBucketRequestQuotas()
|
||||
quotas.SetMaxSize(1000000000)
|
||||
quotas.SetMaxObjects(999999999)
|
||||
updateBucketRequest.SetQuotas(*quotas)
|
||||
updatedBucket, _, _ := client.BucketApi.UpdateBucket(ctx).Id(*bucketInfo.Id).UpdateBucketRequest(updateBucketRequest).Execute()
|
||||
fmt.Printf("Bucket %v website activation: %v\n", *updatedBucket.Id, *updatedBucket.WebsiteAccess)
|
||||
|
||||
bucketList, _, _ := client.BucketApi.ListBuckets(ctx).Execute()
|
||||
fmt.Printf("Bucket count: %v\n", len(bucketList))
|
||||
// Process the response
|
||||
fmt.Fprintf(os.Stdout, "Target hostname: %v\n", resp.KnownNodes[resp.Node].Hostname)
|
||||
}
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -31,9 +31,9 @@ npm install --save git+https://git.deuxfleurs.fr/garage-sdk/garage-admin-sdk-js.
|
|||
A short example:
|
||||
|
||||
```javascript
|
||||
const garage = require('garage_administration_api_v1garage_v0_9_0');
|
||||
const garage = require('garage_administration_api_v0garage_v0_8_0');
|
||||
|
||||
const api = new garage.ApiClient("http://127.0.0.1:3903/v1");
|
||||
const api = new garage.ApiClient("http://127.0.0.1:3903/v0");
|
||||
api.authentications['bearerAuth'].accessToken = "s3cr3t";
|
||||
|
||||
const [node, layout, key, bucket] = [
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ from garage_admin_sdk.apis import *
|
|||
from garage_admin_sdk.models import *
|
||||
|
||||
configuration = garage_admin_sdk.Configuration(
|
||||
host = "http://localhost:3903/v1",
|
||||
host = "http://localhost:3903/v0",
|
||||
access_token = "s3cr3t"
|
||||
)
|
||||
|
||||
|
|
@ -94,14 +94,13 @@ print(f"running garage {status.garage_version}, node_id {status.node}")
|
|||
|
||||
# Change layout of this node
|
||||
current = layout.get_layout()
|
||||
layout.add_layout([
|
||||
NodeRoleChange(
|
||||
id = status.node,
|
||||
layout.add_layout({
|
||||
status.node: NodeClusterInfo(
|
||||
zone = "dc1",
|
||||
capacity = 1000000000,
|
||||
capacity = 1,
|
||||
tags = [ "dev" ],
|
||||
)
|
||||
])
|
||||
})
|
||||
layout.apply_layout(LayoutVersion(
|
||||
version = current.version + 1
|
||||
))
|
||||
|
|
|
|||
|
|
@ -146,7 +146,7 @@ Keep the Key ID and the Secret key in a pad, they will be needed later.
|
|||
|
||||
We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
|
||||
```bash
|
||||
garage bucket create peertube-videos
|
||||
garage bucket create peertube-video
|
||||
garage bucket create peertube-playlist
|
||||
```
|
||||
|
||||
|
|
@ -216,7 +216,7 @@ object_storage:
|
|||
|
||||
# Same settings but for webtorrent videos
|
||||
videos:
|
||||
bucket_name: 'peertube-videos'
|
||||
bucket_name: 'peertube-video'
|
||||
prefix: ''
|
||||
# You must fill this field to make Peertube use our reverse proxy/website logic
|
||||
base_url: 'http://peertube-videos.web.garage.localhost'
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@ Our website serving logic is as follow:
|
|||
|
||||
Now we need to infer the URL of your website through your bucket name.
|
||||
Let assume:
|
||||
- we set `root_domain = ".web.example.com"` in `garage.toml` ([ref](@/documentation/reference-manual/configuration.md#web_root_domain))
|
||||
- we set `root_domain = ".web.example.com"` in `garage.toml` ([ref](@/documentation/reference-manual/configuration.md#root_domain))
|
||||
- our bucket name is `garagehq.deuxfleurs.fr`.
|
||||
|
||||
Our bucket will be served if the Host field matches one of these 2 values (the port is ignored):
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ An introduction to building cluster layouts can be found in the [production depl
|
|||
In Garage, all of the data that can be stored in a given cluster is divided
|
||||
into slices which we call *partitions*. Each partition is stored by
|
||||
one or several nodes in the cluster
|
||||
(see [`replication_mode`](@/documentation/reference-manual/configuration.md#replication_mode)).
|
||||
(see [`replication_mode`](@/documentation/reference-manual/configuration.md#replication-mode)).
|
||||
The layout determines the correspondence between these partition,
|
||||
which exist on a logical level, and actual storage nodes.
|
||||
|
||||
|
|
|
|||
|
|
@ -13,11 +13,8 @@ We will bump the version numbers prefixed to each API endpoint at each time the
|
|||
or semantics change, meaning that code that relies on these endpoint will break
|
||||
when changes are introduced.
|
||||
|
||||
Versions:
|
||||
- Before Garage 0.7.2 - no admin API
|
||||
- Garage 0.7.2 - admin APIv0
|
||||
- Garage 0.9.0 - admin APIv1, deprecate admin APIv0
|
||||
|
||||
The Garage administration API was introduced in version 0.7.2, this document
|
||||
does not apply to older versions of Garage.
|
||||
|
||||
|
||||
## Access control
|
||||
|
|
@ -134,9 +131,7 @@ $ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=exampl
|
|||
|
||||
### Cluster operations
|
||||
|
||||
These endpoints have a dedicated OpenAPI spec.
|
||||
- APIv1 - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v1.yml)
|
||||
- APIv0 (deprecated) - [HTML spec](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html) - [OpenAPI YAML](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml)
|
||||
These endpoints are defined on a dedicated [Redocly page](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.html). You can also download its [OpenAPI specification](https://garagehq.deuxfleurs.fr/api/garage-admin-v0.yml).
|
||||
|
||||
Requesting the API from the command line can be as simple as running:
|
||||
|
||||
|
|
|
|||
|
|
@ -8,8 +8,6 @@ weight = 20
|
|||
Here is an example `garage.toml` configuration file that illustrates all of the possible options:
|
||||
|
||||
```toml
|
||||
replication_mode = "3"
|
||||
|
||||
metadata_dir = "/var/lib/garage/meta"
|
||||
data_dir = "/var/lib/garage/data"
|
||||
metadata_fsync = true
|
||||
|
|
@ -23,6 +21,8 @@ sled_cache_capacity = "128MiB"
|
|||
sled_flush_every_ms = 2000
|
||||
lmdb_map_size = "1T"
|
||||
|
||||
replication_mode = "3"
|
||||
|
||||
compression_level = 1
|
||||
|
||||
rpc_secret = "4425f5c26c5e11581d3223904324dcb5b5d5dfb14e5e7f35e38c595424f5f1e6"
|
||||
|
|
@ -77,64 +77,157 @@ The following gives details about each available configuration option.
|
|||
|
||||
## Available configuration options
|
||||
|
||||
### Index
|
||||
### `metadata_dir`
|
||||
|
||||
Top-level configuration options:
|
||||
[`block_size`](#block_size),
|
||||
[`bootstrap_peers`](#bootstrap_peers),
|
||||
[`compression_level`](#compression_level),
|
||||
[`data_dir`](#metadata_dir),
|
||||
[`data_fsync`](#data_fsync),
|
||||
[`db_engine`](#db_engine),
|
||||
[`lmdb_map_size`](#lmdb_map_size),
|
||||
[`metadata_dir`](#metadata_dir),
|
||||
[`metadata_fsync`](#metadata_fsync),
|
||||
[`replication_mode`](#replication_mode),
|
||||
[`rpc_bind_addr`](#rpc_bind_addr),
|
||||
[`rpc_public_addr`](#rpc_public_addr),
|
||||
[`rpc_secret`](#rpc_secret),
|
||||
[`rpc_secret_file`](#rpc_secret),
|
||||
[`sled_cache_capacity`](#sled_cache_capacity),
|
||||
[`sled_flush_every_ms`](#sled_flush_every_ms).
|
||||
The directory in which Garage will store its metadata. This contains the node identifier,
|
||||
the network configuration and the peer list, the list of buckets and keys as well
|
||||
as the index of all objects, object version and object blocks.
|
||||
|
||||
The `[consul_discovery]` section:
|
||||
[`api`](#consul_api),
|
||||
[`ca_cert`](#consul_ca_cert),
|
||||
[`client_cert`](#consul_client_cert),
|
||||
[`client_key`](#consul_client_cert),
|
||||
[`consul_http_addr`](#consul_http_addr),
|
||||
[`meta`](#consul_tags),
|
||||
[`service_name`](#consul_service_name),
|
||||
[`tags`](#consul_tags),
|
||||
[`tls_skip_verify`](#consul_tls_skip_verify),
|
||||
[`token`](#consul_token).
|
||||
Store this folder on a fast SSD drive if possible to maximize Garage's performance.
|
||||
|
||||
The `[kubernetes_discovery]` section:
|
||||
[`namespace`](#kube_namespace),
|
||||
[`service_name`](#kube_service_name),
|
||||
[`skip_crd`](#kube_skip_crd).
|
||||
### `data_dir`
|
||||
|
||||
The `[s3_api]` section:
|
||||
[`api_bind_addr`](#s3_api_bind_addr),
|
||||
[`root_domain`](#s3_root_domain),
|
||||
[`s3_region`](#s3_region).
|
||||
The directory in which Garage will store the data blocks of objects.
|
||||
This folder can be placed on an HDD. The space available for `data_dir`
|
||||
should be counted to determine a node's capacity
|
||||
when [adding it to the cluster layout](@/documentation/cookbook/real-world.md).
|
||||
|
||||
The `[s3_web]` section:
|
||||
[`bind_addr`](#web_bind_addr),
|
||||
[`root_domain`](#web_root_domain).
|
||||
Since `v0.9.0`, Garage supports multiple data directories with the following syntax:
|
||||
|
||||
The `[admin]` section:
|
||||
[`api_bind_addr`](#admin_api_bind_addr),
|
||||
[`metrics_token`](#admin_metrics_token),
|
||||
[`metrics_token_file`](#admin_metrics_token),
|
||||
[`admin_token`](#admin_token),
|
||||
[`admin_token_file`](#admin_token),
|
||||
[`trace_sink`](#admin_trace_sink),
|
||||
```toml
|
||||
data_dir = [
|
||||
{ path = "/path/to/old_data", read_only = true },
|
||||
{ path = "/path/to/new_hdd1", capacity = "2T" },
|
||||
{ path = "/path/to/new_hdd2", capacity = "4T" },
|
||||
]
|
||||
```
|
||||
|
||||
See [the dedicated documentation page](@/documentation/operations/multi-hdd.md)
|
||||
on how to operate Garage in such a setup.
|
||||
|
||||
### Top-level configuration options
|
||||
### `db_engine` (since `v0.8.0`)
|
||||
|
||||
#### `replication_mode` {#replication_mode}
|
||||
By default, Garage uses the Sled embedded database library
|
||||
to store its metadata on-disk. Since `v0.8.0`, Garage can use alternative storage backends as follows:
|
||||
|
||||
| DB engine | `db_engine` value | Database path |
|
||||
| --------- | ----------------- | ------------- |
|
||||
| [Sled](https://sled.rs) | `"sled"` | `<metadata_dir>/db/` |
|
||||
| [LMDB](https://www.lmdb.tech) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
|
||||
| [Sqlite](https://sqlite.org) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
|
||||
|
||||
Performance characteristics of the different DB engines are as follows:
|
||||
|
||||
- Sled: the default database engine, which tends to produce
|
||||
large data files and also has performance issues, especially when the metadata folder
|
||||
is on a traditional HDD and not on SSD.
|
||||
- LMDB: the recommended alternative on 64-bit systems,
|
||||
much more space-efficiant and slightly faster. Note that the data format of LMDB is not portable
|
||||
between architectures, so for instance the Garage database of an x86-64
|
||||
node cannot be moved to an ARM64 node. Also note that, while LMDB can technically be used on 32-bit systems,
|
||||
this will limit your node to very small database sizes due to how LMDB works; it is therefore not recommended.
|
||||
- Sqlite: Garage supports Sqlite as a storage backend for metadata,
|
||||
however it may have issues and is also very slow in its current implementation,
|
||||
so it is not recommended to be used for now.
|
||||
|
||||
It is possible to convert Garage's metadata directory from one format to another with a small utility named `convert_db`,
|
||||
which can be downloaded at the following locations:
|
||||
[for amd64](https://garagehq.deuxfleurs.fr/_releases/convert_db/amd64/convert_db),
|
||||
[for i386](https://garagehq.deuxfleurs.fr/_releases/convert_db/i386/convert_db),
|
||||
[for arm64](https://garagehq.deuxfleurs.fr/_releases/convert_db/arm64/convert_db),
|
||||
[for arm](https://garagehq.deuxfleurs.fr/_releases/convert_db/arm/convert_db).
|
||||
The `convert_db` utility is used as folows:
|
||||
|
||||
```
|
||||
convert-db -a <input db engine> -i <input db path> \
|
||||
-b <output db engine> -o <output db path>
|
||||
```
|
||||
|
||||
Make sure to specify the full database path as presented in the table above,
|
||||
and not just the path to the metadata directory.
|
||||
|
||||
### `metadata_fsync`
|
||||
|
||||
Whether to enable synchronous mode for the database engine or not.
|
||||
This is disabled (`false`) by default.
|
||||
|
||||
This reduces the risk of metadata corruption in case of power failures,
|
||||
at the cost of a significant drop in write performance,
|
||||
as Garage will have to pause to sync data to disk much more often
|
||||
(several times for API calls such as PutObject).
|
||||
|
||||
Using this option reduces the risk of simultaneous metadata corruption on several
|
||||
cluster nodes, which could lead to data loss.
|
||||
|
||||
If multi-site replication is used, this option is most likely not necessary, as
|
||||
it is extremely unlikely that two nodes in different locations will have a
|
||||
power failure at the exact same time.
|
||||
|
||||
(Metadata corruption on a single node is not an issue, the corrupted data file
|
||||
can always be deleted and reconstructed from the other nodes in the cluster.)
|
||||
|
||||
Here is how this option impacts the different database engines:
|
||||
|
||||
| Database | `metadata_fsync = false` (default) | `metadata_fsync = true` |
|
||||
|----------|------------------------------------|-------------------------------|
|
||||
| Sled | default options | *unsupported* |
|
||||
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
|
||||
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
|
||||
|
||||
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
|
||||
|
||||
### `data_fsync`
|
||||
|
||||
Whether to `fsync` data blocks and their containing directory after they are
|
||||
saved to disk.
|
||||
This is disabled (`false`) by default.
|
||||
|
||||
This might reduce the risk that a data block is lost in rare
|
||||
situations such as simultaneous node losing power,
|
||||
at the cost of a moderate drop in write performance.
|
||||
|
||||
Similarly to `metatada_fsync`, this is likely not necessary
|
||||
if geographical replication is used.
|
||||
|
||||
### `block_size`
|
||||
|
||||
Garage splits stored objects in consecutive chunks of size `block_size`
|
||||
(except the last one which might be smaller). The default size is 1MiB and
|
||||
should work in most cases. We recommend increasing it to e.g. 10MiB if
|
||||
you are using Garage to store large files and have fast network connections
|
||||
between all nodes (e.g. 1gbps).
|
||||
|
||||
If you are interested in tuning this, feel free to do so (and remember to
|
||||
report your findings to us!). When this value is changed for a running Garage
|
||||
installation, only files newly uploaded will be affected. Previously uploaded
|
||||
files will remain available. This however means that chunks from existing files
|
||||
will not be deduplicated with chunks from newly uploaded files, meaning you
|
||||
might use more storage space that is optimally possible.
|
||||
|
||||
### `sled_cache_capacity`
|
||||
|
||||
This parameter can be used to tune the capacity of the cache used by
|
||||
[sled](https://sled.rs), the database Garage uses internally to store metadata.
|
||||
Tune this to fit the RAM you wish to make available to your Garage instance.
|
||||
This value has a conservative default (128MB) so that Garage doesn't use too much
|
||||
RAM by default, but feel free to increase this for higher performance.
|
||||
|
||||
### `sled_flush_every_ms`
|
||||
|
||||
This parameters can be used to tune the flushing interval of sled.
|
||||
Increase this if sled is thrashing your SSD, at the risk of losing more data in case
|
||||
of a power outage (though this should not matter much as data is replicated on other
|
||||
nodes). The default value, 2000ms, should be appropriate for most use cases.
|
||||
|
||||
### `lmdb_map_size`
|
||||
|
||||
This parameters can be used to set the map size used by LMDB,
|
||||
which is the size of the virtual memory region used for mapping the database file.
|
||||
The value of this parameter is the maximum size the metadata database can take.
|
||||
This value is not bound by the physical RAM size of the machine running Garage.
|
||||
If not specified, it defaults to 1GiB on 32-bit machines and 1TiB on 64-bit machines.
|
||||
|
||||
### `replication_mode`
|
||||
|
||||
Garage supports the following replication modes:
|
||||
|
||||
|
|
@ -217,160 +310,7 @@ to the cluster while rebalancing is in progress. In theory, no data should be
|
|||
lost as rebalancing is a routine operation for Garage, although we cannot
|
||||
guarantee you that everything will go right in such an extreme scenario.
|
||||
|
||||
#### `metadata_dir` {#metadata_dir}
|
||||
|
||||
The directory in which Garage will store its metadata. This contains the node identifier,
|
||||
the network configuration and the peer list, the list of buckets and keys as well
|
||||
as the index of all objects, object version and object blocks.
|
||||
|
||||
Store this folder on a fast SSD drive if possible to maximize Garage's performance.
|
||||
|
||||
#### `data_dir` {#data_dir}
|
||||
|
||||
The directory in which Garage will store the data blocks of objects.
|
||||
This folder can be placed on an HDD. The space available for `data_dir`
|
||||
should be counted to determine a node's capacity
|
||||
when [adding it to the cluster layout](@/documentation/cookbook/real-world.md).
|
||||
|
||||
Since `v0.9.0`, Garage supports multiple data directories with the following syntax:
|
||||
|
||||
```toml
|
||||
data_dir = [
|
||||
{ path = "/path/to/old_data", read_only = true },
|
||||
{ path = "/path/to/new_hdd1", capacity = "2T" },
|
||||
{ path = "/path/to/new_hdd2", capacity = "4T" },
|
||||
]
|
||||
```
|
||||
|
||||
See [the dedicated documentation page](@/documentation/operations/multi-hdd.md)
|
||||
on how to operate Garage in such a setup.
|
||||
|
||||
#### `db_engine` (since `v0.8.0`) {#db_engine}
|
||||
|
||||
Since `v0.8.0`, Garage can use alternative storage backends as follows:
|
||||
|
||||
| DB engine | `db_engine` value | Database path |
|
||||
| --------- | ----------------- | ------------- |
|
||||
| [LMDB](https://www.lmdb.tech) (default since `v0.9.0`) | `"lmdb"` | `<metadata_dir>/db.lmdb/` |
|
||||
| [Sled](https://sled.rs) (default up to `v0.8.0`) | `"sled"` | `<metadata_dir>/db/` |
|
||||
| [Sqlite](https://sqlite.org) | `"sqlite"` | `<metadata_dir>/db.sqlite` |
|
||||
|
||||
Sled was the only database engine up to Garage v0.7.0. Performance issues and
|
||||
API limitations of Sled prompted the addition of alternative engines in v0.8.0.
|
||||
Since v0.9.0, LMDB is the default engine instead of Sled, and Sled is
|
||||
deprecated. We plan to remove Sled in Garage v1.0.
|
||||
|
||||
Performance characteristics of the different DB engines are as follows:
|
||||
|
||||
- Sled: tends to produce large data files and also has performance issues,
|
||||
especially when the metadata folder is on a traditional HDD and not on SSD.
|
||||
|
||||
- LMDB: the recommended database engine on 64-bit systems, much more
|
||||
space-efficient and slightly faster. Note that the data format of LMDB is not
|
||||
portable between architectures, so for instance the Garage database of an
|
||||
x86-64 node cannot be moved to an ARM64 node. Also note that, while LMDB can
|
||||
technically be used on 32-bit systems, this will limit your node to very
|
||||
small database sizes due to how LMDB works; it is therefore not recommended.
|
||||
|
||||
- Sqlite: Garage supports Sqlite as an alternative storage backend for
|
||||
metadata, and although it has not been tested as much, it is expected to work
|
||||
satisfactorily. Since Garage v0.9.0, performance issues have largely been
|
||||
fixed by allowing for a no-fsync mode (see `metadata_fsync`). Sqlite does not
|
||||
have the database size limitation of LMDB on 32-bit systems.
|
||||
|
||||
It is possible to convert Garage's metadata directory from one format to another
|
||||
using the `garage convert-db` command, which should be used as follows:
|
||||
|
||||
```
|
||||
garage convert-db -a <input db engine> -i <input db path> \
|
||||
-b <output db engine> -o <output db path>
|
||||
```
|
||||
|
||||
Make sure to specify the full database path as presented in the table above
|
||||
(third colummn), and not just the path to the metadata directory.
|
||||
|
||||
#### `metadata_fsync` {#metadata_fsync}
|
||||
|
||||
Whether to enable synchronous mode for the database engine or not.
|
||||
This is disabled (`false`) by default.
|
||||
|
||||
This reduces the risk of metadata corruption in case of power failures,
|
||||
at the cost of a significant drop in write performance,
|
||||
as Garage will have to pause to sync data to disk much more often
|
||||
(several times for API calls such as PutObject).
|
||||
|
||||
Using this option reduces the risk of simultaneous metadata corruption on several
|
||||
cluster nodes, which could lead to data loss.
|
||||
|
||||
If multi-site replication is used, this option is most likely not necessary, as
|
||||
it is extremely unlikely that two nodes in different locations will have a
|
||||
power failure at the exact same time.
|
||||
|
||||
(Metadata corruption on a single node is not an issue, the corrupted data file
|
||||
can always be deleted and reconstructed from the other nodes in the cluster.)
|
||||
|
||||
Here is how this option impacts the different database engines:
|
||||
|
||||
| Database | `metadata_fsync = false` (default) | `metadata_fsync = true` |
|
||||
|----------|------------------------------------|-------------------------------|
|
||||
| Sled | default options | *unsupported* |
|
||||
| Sqlite | `PRAGMA synchronous = OFF` | `PRAGMA synchronous = NORMAL` |
|
||||
| LMDB | `MDB_NOMETASYNC` + `MDB_NOSYNC` | `MDB_NOMETASYNC` |
|
||||
|
||||
Note that the Sqlite database is always ran in `WAL` mode (`PRAGMA journal_mode = WAL`).
|
||||
|
||||
#### `data_fsync` {#data_fsync}
|
||||
|
||||
Whether to `fsync` data blocks and their containing directory after they are
|
||||
saved to disk.
|
||||
This is disabled (`false`) by default.
|
||||
|
||||
This might reduce the risk that a data block is lost in rare
|
||||
situations such as simultaneous node losing power,
|
||||
at the cost of a moderate drop in write performance.
|
||||
|
||||
Similarly to `metatada_fsync`, this is likely not necessary
|
||||
if geographical replication is used.
|
||||
|
||||
#### `block_size` {#block_size}
|
||||
|
||||
Garage splits stored objects in consecutive chunks of size `block_size`
|
||||
(except the last one which might be smaller). The default size is 1MiB and
|
||||
should work in most cases. We recommend increasing it to e.g. 10MiB if
|
||||
you are using Garage to store large files and have fast network connections
|
||||
between all nodes (e.g. 1gbps).
|
||||
|
||||
If you are interested in tuning this, feel free to do so (and remember to
|
||||
report your findings to us!). When this value is changed for a running Garage
|
||||
installation, only files newly uploaded will be affected. Previously uploaded
|
||||
files will remain available. This however means that chunks from existing files
|
||||
will not be deduplicated with chunks from newly uploaded files, meaning you
|
||||
might use more storage space that is optimally possible.
|
||||
|
||||
#### `sled_cache_capacity` {#sled_cache_capacity}
|
||||
|
||||
This parameter can be used to tune the capacity of the cache used by
|
||||
[sled](https://sled.rs), the database Garage uses internally to store metadata.
|
||||
Tune this to fit the RAM you wish to make available to your Garage instance.
|
||||
This value has a conservative default (128MB) so that Garage doesn't use too much
|
||||
RAM by default, but feel free to increase this for higher performance.
|
||||
|
||||
#### `sled_flush_every_ms` {#sled_flush_every_ms}
|
||||
|
||||
This parameters can be used to tune the flushing interval of sled.
|
||||
Increase this if sled is thrashing your SSD, at the risk of losing more data in case
|
||||
of a power outage (though this should not matter much as data is replicated on other
|
||||
nodes). The default value, 2000ms, should be appropriate for most use cases.
|
||||
|
||||
#### `lmdb_map_size` {#lmdb_map_size}
|
||||
|
||||
This parameters can be used to set the map size used by LMDB,
|
||||
which is the size of the virtual memory region used for mapping the database file.
|
||||
The value of this parameter is the maximum size the metadata database can take.
|
||||
This value is not bound by the physical RAM size of the machine running Garage.
|
||||
If not specified, it defaults to 1GiB on 32-bit machines and 1TiB on 64-bit machines.
|
||||
|
||||
#### `compression_level` {#compression_level}
|
||||
### `compression_level`
|
||||
|
||||
Zstd compression level to use for storing blocks.
|
||||
|
||||
|
|
@ -394,7 +334,7 @@ Compression is done synchronously, setting a value too high will add latency to
|
|||
This value can be different between nodes, compression is done by the node which receive the
|
||||
API call.
|
||||
|
||||
#### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET`, `GARAGE_RPC_SECRET_FILE` (env) {#rpc_secret}
|
||||
### `rpc_secret`, `rpc_secret_file` or `GARAGE_RPC_SECRET`, `GARAGE_RPC_SECRET_FILE` (env)
|
||||
|
||||
Garage uses a secret key, called an RPC secret, that is shared between all
|
||||
nodes of the cluster in order to identify these nodes and allow them to
|
||||
|
|
@ -406,10 +346,10 @@ Since Garage `v0.8.2`, the RPC secret can also be stored in a file whose path is
|
|||
given in the configuration variable `rpc_secret_file`, or specified as an
|
||||
environment variable `GARAGE_RPC_SECRET`.
|
||||
|
||||
Since Garage `v0.8.5` and `v0.9.1`, you can also specify the path of a file
|
||||
storing the secret as the `GARAGE_RPC_SECRET_FILE` environment variable.
|
||||
Since Garage `v0.9.0`, you can also specify the path of a file storing the secret
|
||||
as the `GARAGE_RPC_SECRET_FILE` environment variable.
|
||||
|
||||
#### `rpc_bind_addr` {#rpc_bind_addr}
|
||||
### `rpc_bind_addr`
|
||||
|
||||
The address and port on which to bind for inter-cluster communcations
|
||||
(reffered to as RPC for remote procedure calls).
|
||||
|
|
@ -418,14 +358,14 @@ the node, even in the case of a NAT: the NAT should be configured to forward the
|
|||
port number to the same internal port nubmer. This means that if you have several nodes running
|
||||
behind a NAT, they should each use a different RPC port number.
|
||||
|
||||
#### `rpc_public_addr` {#rpc_public_addr}
|
||||
### `rpc_public_addr`
|
||||
|
||||
The address and port that other nodes need to use to contact this node for
|
||||
RPC calls. **This parameter is optional but recommended.** In case you have
|
||||
a NAT that binds the RPC port to a port that is different on your public IP,
|
||||
this field might help making it work.
|
||||
|
||||
#### `bootstrap_peers` {#bootstrap_peers}
|
||||
### `bootstrap_peers`
|
||||
|
||||
A list of peer identifiers on which to contact other Garage peers of this cluster.
|
||||
These peer identifiers have the following syntax:
|
||||
|
|
@ -441,54 +381,43 @@ be obtained by running `garage node id` and then included directly in the
|
|||
key will be returned by `garage node id` and you will have to add the IP
|
||||
yourself.
|
||||
|
||||
### `allow_world_readable_secrets`
|
||||
|
||||
Garage checks the permissions of your secret files to make sure they're not
|
||||
world-readable. In some cases, the check might fail and consider your files as
|
||||
world-readable even if they're not, for instance when using Posix ACLs.
|
||||
|
||||
Setting `allow_world_readable_secrets` to `true` bypass this
|
||||
permission verification.
|
||||
|
||||
Alternatively, you can set the `GARAGE_ALLOW_WORLD_READABLE_SECRETS`
|
||||
environment variable to `true` to bypass the permissions check.
|
||||
|
||||
### The `[consul_discovery]` section
|
||||
## The `[consul_discovery]` section
|
||||
|
||||
Garage supports discovering other nodes of the cluster using Consul. For this
|
||||
to work correctly, nodes need to know their IP address by which they can be
|
||||
reached by other nodes of the cluster, which should be set in `rpc_public_addr`.
|
||||
|
||||
#### `consul_http_addr` {#consul_http_addr}
|
||||
### `consul_http_addr` and `service_name`
|
||||
|
||||
The `consul_http_addr` parameter should be set to the full HTTP(S) address of the Consul server.
|
||||
|
||||
#### `api` {#consul_api}
|
||||
### `api`
|
||||
|
||||
Two APIs for service registration are supported: `catalog` and `agent`. `catalog`, the default, will register a service using
|
||||
the `/v1/catalog` endpoints, enabling mTLS if `client_cert` and `client_key` are provided. The `agent` API uses the
|
||||
`v1/agent` endpoints instead, where an optional `token` may be provided.
|
||||
|
||||
#### `service_name` {#consul_service_name}
|
||||
### `service_name`
|
||||
|
||||
`service_name` should be set to the service name under which Garage's
|
||||
RPC ports are announced.
|
||||
|
||||
#### `client_cert`, `client_key` {#consul_client_cert}
|
||||
### `client_cert`, `client_key`
|
||||
|
||||
TLS client certificate and client key to use when communicating with Consul over TLS. Both are mandatory when doing so.
|
||||
Only available when `api = "catalog"`.
|
||||
|
||||
#### `ca_cert` {#consul_ca_cert}
|
||||
### `ca_cert`
|
||||
|
||||
TLS CA certificate to use when communicating with Consul over TLS.
|
||||
|
||||
#### `tls_skip_verify` {#consul_tls_skip_verify}
|
||||
### `tls_skip_verify`
|
||||
|
||||
Skip server hostname verification in TLS handshake.
|
||||
`ca_cert` is ignored when this is set.
|
||||
|
||||
#### `token` {#consul_token}
|
||||
### `token`
|
||||
|
||||
Uses the provided token for communication with Consul. Only available when `api = "agent"`.
|
||||
The policy assigned to this token should at least have these rules:
|
||||
|
|
@ -508,49 +437,49 @@ node_prefix "" {
|
|||
}
|
||||
```
|
||||
|
||||
#### `tags` and `meta` {#consul_tags}
|
||||
### `tags` and `meta`
|
||||
|
||||
Additional list of tags and map of service meta to add during service registration.
|
||||
|
||||
### The `[kubernetes_discovery]` section
|
||||
## The `[kubernetes_discovery]` section
|
||||
|
||||
Garage supports discovering other nodes of the cluster using kubernetes custom
|
||||
resources. For this to work, a `[kubernetes_discovery]` section must be present
|
||||
with at least the `namespace` and `service_name` parameters.
|
||||
|
||||
#### `namespace` {#kube_namespace}
|
||||
### `namespace`
|
||||
|
||||
`namespace` sets the namespace in which the custom resources are
|
||||
configured.
|
||||
|
||||
#### `service_name` {#kube_service_name}
|
||||
### `service_name`
|
||||
|
||||
`service_name` is added as a label to the advertised resources to
|
||||
filter them, to allow for multiple deployments in a single namespace.
|
||||
|
||||
#### `skip_crd` {#kube_skip_crd}
|
||||
### `skip_crd`
|
||||
|
||||
`skip_crd` can be set to true to disable the automatic creation and
|
||||
patching of the `garagenodes.deuxfleurs.fr` CRD. You will need to create the CRD
|
||||
manually.
|
||||
|
||||
|
||||
### The `[s3_api]` section
|
||||
## The `[s3_api]` section
|
||||
|
||||
#### `api_bind_addr` {#s3_api_bind_addr}
|
||||
### `api_bind_addr`
|
||||
|
||||
The IP and port on which to bind for accepting S3 API calls.
|
||||
This endpoint does not suport TLS: a reverse proxy should be used to provide it.
|
||||
|
||||
Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode.
|
||||
|
||||
#### `s3_region` {#s3_region}
|
||||
### `s3_region`
|
||||
|
||||
Garage will accept S3 API calls that are targetted to the S3 region defined here.
|
||||
API calls targetted to other regions will fail with a AuthorizationHeaderMalformed error
|
||||
message that redirects the client to the correct region.
|
||||
|
||||
#### `root_domain` {#s3_root_domain}
|
||||
### `root_domain` {#root_domain}
|
||||
|
||||
The optional suffix to access bucket using vhost-style in addition to path-style request.
|
||||
Note path-style requests are always enabled, whether or not vhost-style is configured.
|
||||
|
|
@ -562,12 +491,12 @@ using the hostname `my-bucket.s3.garage.eu`.
|
|||
|
||||
|
||||
|
||||
### The `[s3_web]` section
|
||||
## The `[s3_web]` section
|
||||
|
||||
Garage allows to publish content of buckets as websites. This section configures the
|
||||
behaviour of this module.
|
||||
|
||||
#### `bind_addr` {#web_bind_addr}
|
||||
### `bind_addr`
|
||||
|
||||
The IP and port on which to bind for accepting HTTP requests to buckets configured
|
||||
for website access.
|
||||
|
|
@ -575,7 +504,7 @@ This endpoint does not suport TLS: a reverse proxy should be used to provide it.
|
|||
|
||||
Alternatively, since `v0.8.5`, a path can be used to create a unix socket with 0222 mode.
|
||||
|
||||
#### `root_domain` {#web_root_domain}
|
||||
### `root_domain`
|
||||
|
||||
The optional suffix appended to bucket names for the corresponding HTTP Host.
|
||||
|
||||
|
|
@ -584,11 +513,11 @@ will be accessible either with hostname `deuxfleurs.fr.web.garage.eu`
|
|||
or with hostname `deuxfleurs.fr`.
|
||||
|
||||
|
||||
### The `[admin]` section
|
||||
## The `[admin]` section
|
||||
|
||||
Garage has a few administration capabilities, in particular to allow remote monitoring. These features are detailed below.
|
||||
|
||||
#### `api_bind_addr` {#admin_api_bind_addr}
|
||||
### `api_bind_addr`
|
||||
|
||||
If specified, Garage will bind an HTTP server to this port and address, on
|
||||
which it will listen to requests for administration features.
|
||||
|
|
@ -597,7 +526,7 @@ See [administration API reference](@/documentation/reference-manual/admin-api.md
|
|||
Alternatively, since `v0.8.5`, a path can be used to create a unix socket. Note that for security reasons,
|
||||
the socket will have 0220 mode. Make sure to set user and group permissions accordingly.
|
||||
|
||||
#### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env) {#admin_metrics_token}
|
||||
### `metrics_token`, `metrics_token_file` or `GARAGE_METRICS_TOKEN`, `GARAGE_METRICS_TOKEN_FILE` (env)
|
||||
|
||||
The token for accessing the Metrics endpoint. If this token is not set, the
|
||||
Metrics endpoint can be accessed without access control.
|
||||
|
|
@ -607,9 +536,9 @@ You can use any random string for this value. We recommend generating a random t
|
|||
`metrics_token` was introduced in Garage `v0.7.2`.
|
||||
`metrics_token_file` and the `GARAGE_METRICS_TOKEN` environment variable are supported since Garage `v0.8.2`.
|
||||
|
||||
`GARAGE_METRICS_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
|
||||
`GARAGE_METRICS_TOKEN_FILE` is supported since `v0.9.0`.
|
||||
|
||||
#### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env) {#admin_token}
|
||||
### `admin_token`, `admin_token_file` or `GARAGE_ADMIN_TOKEN`, `GARAGE_ADMIN_TOKEN_FILE` (env)
|
||||
|
||||
The token for accessing all of the other administration endpoints. If this
|
||||
token is not set, access to these endpoints is disabled entirely.
|
||||
|
|
@ -619,9 +548,9 @@ You can use any random string for this value. We recommend generating a random t
|
|||
`admin_token` was introduced in Garage `v0.7.2`.
|
||||
`admin_token_file` and the `GARAGE_ADMIN_TOKEN` environment variable are supported since Garage `v0.8.2`.
|
||||
|
||||
`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.8.5` / `v0.9.1`.
|
||||
`GARAGE_ADMIN_TOKEN_FILE` is supported since `v0.9.0`.
|
||||
|
||||
#### `trace_sink` {#admin_trace_sink}
|
||||
### `trace_sink`
|
||||
|
||||
Optionally, the address of an OpenTelemetry collector. If specified,
|
||||
Garage will send traces in the OpenTelemetry format to this endpoint. These
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ This is particularly usefull when nodes are far from one another and talk to one
|
|||
|
||||
Garage supports a variety of replication modes, with 1 copy, 2 copies or 3 copies of your data,
|
||||
and with various levels of consistency, in order to adapt to a variety of usage scenarios.
|
||||
Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication_mode)
|
||||
Read our reference page on [supported replication modes](@/documentation/reference-manual/configuration.md#replication-mode)
|
||||
to select the replication mode best suited to your use case (hint: in most cases, `replication_mode = "3"` is what you want).
|
||||
|
||||
### Web server for static websites
|
||||
|
|
|
|||
|
|
@ -1 +0,0 @@
|
|||
use nix
|
||||
17
script/jepsen.garage/.gitignore
vendored
17
script/jepsen.garage/.gitignore
vendored
|
|
@ -1,17 +0,0 @@
|
|||
/target
|
||||
/classes
|
||||
/checkouts
|
||||
profiles.clj
|
||||
pom.xml
|
||||
pom.xml.asc
|
||||
*.jar
|
||||
*.class
|
||||
/.lein-*
|
||||
/.nrepl-port
|
||||
/.prepl-port
|
||||
.hgignore
|
||||
.hg/
|
||||
.direnv
|
||||
/store
|
||||
/store.*
|
||||
.vagrant
|
||||
|
|
@ -1,166 +0,0 @@
|
|||
# jepsen.garage
|
||||
|
||||
Jepsen checking of Garage consistency properties.
|
||||
|
||||
## Usage
|
||||
|
||||
Requirements:
|
||||
|
||||
- vagrant
|
||||
- VirtualBox, configured so that nodes can take an IP in a private network `192.168.56.0/24` (it's the default)
|
||||
- a user that can create VirtualBox VMs
|
||||
- leiningen
|
||||
- gnuplot
|
||||
|
||||
Set up VMs before running tests:
|
||||
|
||||
```
|
||||
vagrant up
|
||||
```
|
||||
|
||||
Run tests: see commands below.
|
||||
|
||||
|
||||
## Results
|
||||
|
||||
### Register linear, without timestamp patch
|
||||
|
||||
Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 20 --workload reg1 --ops-per-key 100`
|
||||
|
||||
Results without timestamp patch:
|
||||
|
||||
- Fails with a simple clock-scramble nemesis (`--scenario c`).
|
||||
Explanation: without the timestamp patch, nodes will create objects using their
|
||||
local clock only as a timestamp, so the ordering will be all over the place if
|
||||
clocks are scrambled.
|
||||
|
||||
Results with timestamp patch (`--patch tsfix2`):
|
||||
|
||||
- No failure with clock-scramble nemesis
|
||||
|
||||
- Fails with clock-scramble nemesis + partition nemesis (`--scenario cp`).
|
||||
|
||||
**This test is expected to fail.**
|
||||
Indeed, S3 objects are not meant to behave like linearizable registers.
|
||||
TODO explain using a counter-example
|
||||
|
||||
|
||||
### Read-after-write CRDT register model
|
||||
|
||||
Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload reg2 --ops-per-key 100`
|
||||
|
||||
Results without timestamp patch:
|
||||
|
||||
- Fails with a simple clock-scramble nemesis (`--scenario c`).
|
||||
Explanation: old values are not overwritten correctly when their timestamps are in the future.
|
||||
|
||||
Results with timestamp patch (`--patch tsfix2`):
|
||||
|
||||
- No failures with clock-scramble nemesis + partition nemesis (`--scenario cp`).
|
||||
This proves that `tsfix2` (PR#543) does improve consistency.
|
||||
|
||||
- **Fails with layout reconfiguration nemesis** (`--scenario r`).
|
||||
Example of a failed run: `garage reg2/20231024T120806.899+0200`.
|
||||
This is the failure mode we are looking for and trying to fix for NLnet task 3.
|
||||
|
||||
Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`):
|
||||
|
||||
- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs),
|
||||
`--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs).
|
||||
|
||||
- Same with `--patch task3c` (commit `0041b013`, the final version).
|
||||
|
||||
|
||||
### Set, basic test (write some items, then read)
|
||||
|
||||
Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 200 --concurrency 200 --workload set1 --ops-per-key 100`
|
||||
|
||||
Results without NLnet task3 code (`--patch tsfix2`):
|
||||
|
||||
- For now, no failures with clock-scramble nemesis + partition nemesis -> TODO long test run
|
||||
|
||||
- Does not seem to fail with only the layout reconfiguation nemesis (<10 runs), although theoretically it could
|
||||
|
||||
- **Fails with the partition + layout reconfiguration nemesis** (`--scenario pr`).
|
||||
Example of a failed run: `garage set1/20231024T172214.488+0200` (1 failure in 4 runs).
|
||||
This is the failure mode we are looking for and trying to fix for NLnet task 3.
|
||||
|
||||
Results with NLnet task 3 code (commit 707442f5de, `--patch task3a`):
|
||||
|
||||
- The tests are buggy and often result in an "unknown" validity status, which
|
||||
is caused by some requests not returning results during network partitions or
|
||||
other nemesis-induced broken cluster states. However, when the tests were
|
||||
able to finish, there were no failures with scenarios `r`, `pr`, `cpr`,
|
||||
`dpr`.
|
||||
|
||||
|
||||
### Set, continuous test (interspersed reads and writes)
|
||||
|
||||
Command: `lein run test --nodes-file nodes.vagrant --time-limit 60 --rate 100 --concurrency 100 --workload set2 --ops-per-key 100`
|
||||
|
||||
Results without NLnet task3 code (`--patch tsfix2`):
|
||||
|
||||
- No failures with clock-scramble nemesis + db nemesis + partition nemesis (`--scenario cdp`) (0 failures in 10 runs).
|
||||
|
||||
- **Fails with just layout reconfiguration nemesis** (`--scenario r`).
|
||||
Example of a failed run: `garage set2/20231025T141940.198+0200` (10 failures in 10 runs).
|
||||
This is the failure mode we are looking for and trying to fix for NLnet task 3.
|
||||
|
||||
Results with NLnet task3 code (commit 707442f5de, `--patch task3a`):
|
||||
|
||||
- No failures with `--scenario r` (0 of 10 runs), `--scenario pr` (0 of 10 runs),
|
||||
`--scenario cpr` (0 of 10 runs) and `--scenario dpr` (0 of 10 runs).
|
||||
|
||||
- Same with `--patch task3c` (commit `0041b013`, the final version).
|
||||
|
||||
|
||||
## NLnet task 3 final results
|
||||
|
||||
- With code from task3 (`--patch task3c`): [reg2 and set2](results/Results-2023-12-13-task3c.png), [set1](results/Results-2023-12-14-task3-set1.png).
|
||||
- Without (`--patch tsfix2`): [reg2 and set2](results/Results-2023-12-13-tsfix2.png), set1 TBD.
|
||||
|
||||
## Investigating (and fixing) errors
|
||||
|
||||
### Segfaults
|
||||
|
||||
They are due to the download being interrupted in the middle (^C during first launch on clean VMs), the `garage` binary is truncated.
|
||||
Add `:force?` to the `cached-wget!` call in `daemon.clj` to re-download the binary,
|
||||
or restar the VMs to clear temporary files.
|
||||
|
||||
### In `jepsen.garage`: prefix wierdness
|
||||
|
||||
In `store/garage set1/20231019T163358.615+0200`:
|
||||
|
||||
```
|
||||
INFO [2023-10-19 16:35:20,977] clojure-agent-send-off-pool-207 - jepsen.garage.set list results for prefix set20/ : (set13/0 set13/1 set13/10 set13/11 set13/12 set13/13 set13/14 set13/15 set13/16 set13/17 set13/18 set13/19 set13/2 set13/20 set13/21 set13/22 set13/23 set13/24 set13/25 set13/26 set13/27 set13/28 set13/29 set13/3 set13/30 set13/31 set13/32 set13/33 set13/34 set13/35 set13/36 set13/37 set13/38 set13/39 set13/4 set13/40 set13/41 set13/42 set13/43 set13/44 set13/45 set13/46 set13/47 set13/48 set13/49 set13/5 set13/50 set13/51 set13/52 set13/53 set13/54 set13/55 set13/56 set13/57 set13/58 set13/59 set13/6 set13/60 set13/61 set13/62 set13/63 set13/64 set13/65 set13/66 set13/67 set13/68 set13/69 set13/7 set13/70 set13/71 set13/72 set13/73 set13/74 set13/75 set13/76 set13/77 set13/78 set13/79 set13/8 set13/80 set13/81 set13/82 set13/83 set13/84 set13/85 set13/86 set13/87 set13/88 set13/89 set13/9 set13/90 set13/91 set13/92 set13/93 set13/94 set13/95 set13/96 set13/97 set13/98 set13/99) (node: http://192.168.56.25:3900 )
|
||||
```
|
||||
|
||||
After inspecting, the actual S3 call made was with prefix "set13/", so at least this is not an error in Garage itself but in the jepsen code.
|
||||
|
||||
Finally found out that this was due to closures not correctly capturing their context in the list function in s3api.clj (wtf clojure?)
|
||||
Not sure exactly where it came from but it seems to have been fixed by making list-inner a separate function and not a sub-function,
|
||||
and passing all values that were previously in the context (creds and prefix) as additional arguments.
|
||||
|
||||
### `reg2` test inconsistency, even with timestamp fix
|
||||
|
||||
The reg2 test is our custom checker for CRDT read-after-write on individual object keys, acting as registers which can be updated.
|
||||
The test fails without the timestamp fix, which is expected as the clock scrambler will prevent nodes from having a correct ordering of objects.
|
||||
|
||||
With the timestamp fix (`--patch tsfix1`), the happenned-before relationship should at least be respected, meaning that when a PutObject call starts
|
||||
after another PutObject call has ended, the second call should overwrite the value of the first call, and that value should not be
|
||||
readable by future GetObject calls.
|
||||
However, we observed inconsistencies even with the timestamp fix.
|
||||
|
||||
The inconsistencies seemed to always happenned after writing a nil value, which translates to a DeleteObject call
|
||||
instead of a PutObject. By removing the possibility of writing nil values, therefore only doing
|
||||
PutObject calls, the issue disappears. There is therefore an issue to fix in DeleteObject.
|
||||
|
||||
The issue in DeleteObject seems to have been fixed by commit `c82d91c6bccf307186332b6c5c6fc0b128b1b2b1`, which can be used using `--patch tsfix2`.
|
||||
|
||||
|
||||
## License
|
||||
|
||||
Copyright © 2023 Alex Auvolat
|
||||
|
||||
This program and the accompanying materials are made available under the
|
||||
terms of the GNU Affero General Public License v3.0.
|
||||
40
script/jepsen.garage/Vagrantfile
vendored
40
script/jepsen.garage/Vagrantfile
vendored
|
|
@ -1,40 +0,0 @@
|
|||
# -*- mode: ruby -*-
|
||||
# vi: set ft=ruby :
|
||||
#
|
||||
|
||||
def vm(config, hostname, ip)
|
||||
config.vm.hostname = hostname
|
||||
config.vm.network "private_network", ip: ip
|
||||
end
|
||||
|
||||
Vagrant.configure("2") do |config|
|
||||
config.vm.box = "generic/debian10"
|
||||
|
||||
config.vm.provider "virtualbox" do |vb|
|
||||
vb.gui = false
|
||||
vb.memory = "512"
|
||||
vb.customize ["modifyvm", :id, "--vram=12"]
|
||||
end
|
||||
|
||||
config.vm.provision "shell", inline: <<-SHELL
|
||||
echo "root:root" | chpasswd
|
||||
mkdir -p /root/.ssh
|
||||
echo "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJpaBZdYxHqMxhv2RExAOa7nkKhPBOHupMP3mYaZ73w9 lx@lindy" >> /root/.ssh/authorized_keys
|
||||
SHELL
|
||||
|
||||
config.vm.define "n1" do |config| vm(config, "n1", "192.168.56.21") end
|
||||
config.vm.define "n2" do |config| vm(config, "n2", "192.168.56.22") end
|
||||
config.vm.define "n3" do |config| vm(config, "n3", "192.168.56.23") end
|
||||
config.vm.define "n4" do |config| vm(config, "n4", "192.168.56.24") end
|
||||
config.vm.define "n5" do |config| vm(config, "n5", "192.168.56.25") end
|
||||
config.vm.define "n6" do |config| vm(config, "n6", "192.168.56.26") end
|
||||
config.vm.define "n7" do |config| vm(config, "n7", "192.168.56.27") end
|
||||
|
||||
config.vm.define "n8" do |config| vm(config, "n8", "192.168.56.28") end
|
||||
config.vm.define "n9" do |config| vm(config, "n9", "192.168.56.29") end
|
||||
config.vm.define "n10" do |config| vm(config, "n10", "192.168.56.30") end
|
||||
config.vm.define "n11" do |config| vm(config, "n11", "192.168.56.31") end
|
||||
config.vm.define "n12" do |config| vm(config, "n12", "192.168.56.32") end
|
||||
config.vm.define "n13" do |config| vm(config, "n13", "192.168.56.33") end
|
||||
config.vm.define "n14" do |config| vm(config, "n14", "192.168.56.34") end
|
||||
end
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -x
|
||||
|
||||
#for ppatch in task3c task3a tsfix2; do
|
||||
for ppatch in tsfix2; do
|
||||
#for psc in c cp cdp r pr cpr dpr; do
|
||||
for psc in cdp r pr cpr dpr; do
|
||||
#for ptsk in reg2 set1 set2; do
|
||||
for ptsk in set1; do
|
||||
for irun in $(seq 10); do
|
||||
lein run test --nodes-file nodes.vagrant \
|
||||
--time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \
|
||||
--workload $ptsk --patch $ppatch --scenario $psc
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -x
|
||||
|
||||
#for ppatch in task3c tsfix2; do
|
||||
for ppatch in tsfix2; do
|
||||
for psc in cdp r pr cpr dpr; do
|
||||
for ptsk in set1; do
|
||||
for irun in $(seq 10); do
|
||||
lein run test --nodes-file nodes2.vagrant \
|
||||
--time-limit 60 --rate 100 --concurrency 100 --ops-per-key 100 \
|
||||
--workload $ptsk --patch $ppatch --scenario $psc
|
||||
done
|
||||
done
|
||||
done
|
||||
done
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
docker stop jaeger
|
||||
docker rm jaeger
|
||||
|
||||
# UI is on localhost:16686
|
||||
# otel-grpc collector is on localhost:4317
|
||||
# otel-http collector is on localhost:4318
|
||||
|
||||
docker run -d --name jaeger \
|
||||
-e COLLECTOR_OTLP_ENABLED=true \
|
||||
-p 4317:4317 \
|
||||
-p 4318:4318 \
|
||||
-p 16686:16686 \
|
||||
jaegertracing/all-in-one:1.50
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
192.168.56.21
|
||||
192.168.56.22
|
||||
192.168.56.23
|
||||
192.168.56.24
|
||||
192.168.56.25
|
||||
192.168.56.26
|
||||
192.168.56.27
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
192.168.56.28
|
||||
192.168.56.29
|
||||
192.168.56.30
|
||||
192.168.56.31
|
||||
192.168.56.32
|
||||
192.168.56.33
|
||||
192.168.56.34
|
||||
|
|
@ -1,10 +0,0 @@
|
|||
(defproject jepsen.garage "0.1.0-SNAPSHOT"
|
||||
:description "Jepsen testing for Garage"
|
||||
:url "https://git.deuxfleurs.fr/Deuxfleurs/garage"
|
||||
:license {:name "AGPLv3"
|
||||
:url "https://www.gnu.org/licenses/agpl-3.0.en.html"}
|
||||
:main jepsen.garage
|
||||
:dependencies [[org.clojure/clojure "1.11.1"]
|
||||
[jepsen "0.3.3-SNAPSHOT"]
|
||||
[amazonica "0.3.163"]]
|
||||
:repl-options {:init-ns jepsen.garage})
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.4 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1.1 MiB |
Binary file not shown.
|
Before Width: | Height: | Size: 1 MiB |
|
|
@ -1,18 +0,0 @@
|
|||
{ pkgs ? import <nixpkgs> {
|
||||
overlays = [
|
||||
(self: super: {
|
||||
jdk = super.jdk11;
|
||||
jre = super.jre11;
|
||||
})
|
||||
];
|
||||
} }:
|
||||
pkgs.mkShell {
|
||||
nativeBuildInputs = with pkgs; [
|
||||
leiningen
|
||||
jdk
|
||||
jna
|
||||
vagrant
|
||||
gnuplot
|
||||
graphviz
|
||||
];
|
||||
}
|
||||
|
|
@ -1,105 +0,0 @@
|
|||
(ns jepsen.garage
|
||||
(:require
|
||||
[clojure.string :as str]
|
||||
[jepsen
|
||||
[checker :as checker]
|
||||
[cli :as cli]
|
||||
[generator :as gen]
|
||||
[nemesis :as nemesis]
|
||||
[tests :as tests]]
|
||||
[jepsen.os.debian :as debian]
|
||||
[jepsen.garage
|
||||
[daemon :as grg]
|
||||
[nemesis :as grgNemesis]
|
||||
[reg :as reg]
|
||||
[set :as set]]))
|
||||
|
||||
(def workloads
|
||||
"A map of workload names to functions that construct workloads, given opts."
|
||||
{"reg1" reg/workload1
|
||||
"reg2" reg/workload2
|
||||
"set1" set/workload1
|
||||
"set2" set/workload2})
|
||||
|
||||
(def scenari
|
||||
"A map of scenari to the associated nemesis"
|
||||
{"c" grgNemesis/scenario-c
|
||||
"cp" grgNemesis/scenario-cp
|
||||
"r" grgNemesis/scenario-r
|
||||
"pr" grgNemesis/scenario-pr
|
||||
"cpr" grgNemesis/scenario-cpr
|
||||
"cdp" grgNemesis/scenario-cdp
|
||||
"dpr" grgNemesis/scenario-dpr})
|
||||
|
||||
(def patches
|
||||
"A map of patch names to Garage builds"
|
||||
{"default" "v0.9.0"
|
||||
"tsfix1" "d146cdd5b66ca1d3ed65ce93ca42c6db22defc09"
|
||||
"tsfix2" "c82d91c6bccf307186332b6c5c6fc0b128b1b2b1"
|
||||
"task3a" "707442f5de416fdbed4681a33b739f0a787b7834"
|
||||
"task3b" "431b28e0cfdc9cac6c649193cf602108a8b02997"
|
||||
"task3c" "0041b013a473e3ae72f50209d8f79db75a72848b"})
|
||||
|
||||
(def cli-opts
|
||||
"Additional command line options."
|
||||
[["-p" "--patch NAME" "Garage patch to use"
|
||||
:default "default"
|
||||
:validate [patches (cli/one-of patches)]]
|
||||
["-s" "--scenario NAME" "Nemesis scenario to run"
|
||||
:default "cp"
|
||||
:validate [scenari (cli/one-of scenari)]]
|
||||
["-r" "--rate HZ" "Approximate number of requests per second, per thread."
|
||||
:default 10
|
||||
:parse-fn read-string
|
||||
:validate [#(and (number? %) (pos? %)) "Must be a positive number"]]
|
||||
[nil "--ops-per-key NUM" "Maximum number of operations on any given key."
|
||||
:default 100
|
||||
:parse-fn parse-long
|
||||
:validate [pos? "Must be a positive integer."]]
|
||||
["-w" "--workload NAME" "Workload of test to run"
|
||||
:default "reg1"
|
||||
:validate [workloads (cli/one-of workloads)]]])
|
||||
|
||||
(defn garage-test
|
||||
"Given an options map from the command line runner (e.g. :nodes, :ssh,
|
||||
:concurrency, ...), constructs a test map."
|
||||
[opts]
|
||||
(let [garage-version (get patches (:patch opts))
|
||||
db (grg/db garage-version)
|
||||
workload ((get workloads (:workload opts)) opts)
|
||||
scenario ((get scenari (:scenario opts)) (assoc opts :db db))]
|
||||
(merge tests/noop-test
|
||||
opts
|
||||
{:pure-generators true
|
||||
:name (str "garage-" (name (:patch opts)) " " (name (:workload opts)) " " (name (:scenario opts)))
|
||||
:os debian/os
|
||||
:db db
|
||||
:client (:client workload)
|
||||
:generator (gen/phases
|
||||
(->>
|
||||
(:generator workload)
|
||||
(gen/stagger (/ (:rate opts)))
|
||||
(gen/nemesis (:generator scenario))
|
||||
(gen/time-limit (:time-limit opts)))
|
||||
(gen/log "Healing cluster")
|
||||
(gen/nemesis (:final-generator scenario))
|
||||
(gen/log "Waiting for recovery")
|
||||
(gen/sleep 10)
|
||||
(gen/log "Running final generator")
|
||||
(gen/clients (:final-generator workload))
|
||||
(gen/log "Generators all done"))
|
||||
:nemesis (:nemesis scenario)
|
||||
:checker (checker/compose
|
||||
{:perf (checker/perf (:perf scenario))
|
||||
:workload (:checker workload)})
|
||||
})))
|
||||
|
||||
|
||||
(defn -main
|
||||
"Handles command line arguments. Can either run a test, or a web server for
|
||||
browsing results."
|
||||
[& args]
|
||||
(cli/run! (merge (cli/single-test-cmd {:test-fn garage-test
|
||||
:opt-spec cli-opts})
|
||||
(cli/serve-cmd))
|
||||
args))
|
||||
|
|
@ -1,152 +0,0 @@
|
|||
(ns jepsen.garage.daemon
|
||||
(:require [clojure.tools.logging :refer :all]
|
||||
[jepsen [control :as c]
|
||||
[core :as jepsen]
|
||||
[db :as db]]
|
||||
[jepsen.control.util :as cu]))
|
||||
|
||||
; CONSTANTS -- HOW GARAGE IS SET UP
|
||||
|
||||
(def base-dir "/opt/garage")
|
||||
(def data-dir (str base-dir "/data"))
|
||||
(def meta-dir (str base-dir "/meta"))
|
||||
(def binary (str base-dir "/garage"))
|
||||
(def logfile (str base-dir "/garage.log"))
|
||||
(def pidfile (str base-dir "/garage.pid"))
|
||||
|
||||
(def admin-token "icanhazadmin")
|
||||
(def access-key-id "GK8bfb6a51286071c6c9cd8bc3")
|
||||
(def secret-access-key "b0be95f71c1c6f16858a9edf395078b75c12ecb6b1c03385c4ae92076e4994a3")
|
||||
(def bucket-name "jepsen")
|
||||
|
||||
; THE GARAGE DB
|
||||
|
||||
(defn install!
|
||||
"Download and install Garage"
|
||||
[node version]
|
||||
(c/su
|
||||
(c/trace
|
||||
(info node "installing garage" version)
|
||||
(c/exec :mkdir :-p base-dir)
|
||||
(let [url (str "https://garagehq.deuxfleurs.fr/_releases/" version "/x86_64-unknown-linux-musl/garage")
|
||||
cache (cu/cached-wget! url)]
|
||||
(c/exec :cp cache binary))
|
||||
(c/exec :chmod :+x binary))))
|
||||
|
||||
(defn configure!
|
||||
"Configure Garage"
|
||||
[node]
|
||||
(c/su
|
||||
(c/trace
|
||||
(cu/write-file!
|
||||
(str "rpc_secret = \"0fffabe52542c2b89a56b2efb7dfd477e9dafb285c9025cbdf1de7ca21a6b372\"\n"
|
||||
"rpc_bind_addr = \"0.0.0.0:3901\"\n"
|
||||
"rpc_public_addr = \"" node ":3901\"\n"
|
||||
"db_engine = \"lmdb\"\n"
|
||||
"replication_mode = \"2\"\n"
|
||||
"data_dir = \"" data-dir "\"\n"
|
||||
"metadata_dir = \"" meta-dir "\"\n"
|
||||
"[s3_api]\n"
|
||||
"s3_region = \"us-east-1\"\n"
|
||||
"api_bind_addr = \"0.0.0.0:3900\"\n"
|
||||
"[k2v_api]\n"
|
||||
"api_bind_addr = \"0.0.0.0:3902\"\n"
|
||||
"[admin]\n"
|
||||
"api_bind_addr = \"0.0.0.0:3903\"\n"
|
||||
"admin_token = \"" admin-token "\"\n"
|
||||
"trace_sink = \"http://192.168.56.1:4317\"\n")
|
||||
"/etc/garage.toml"))))
|
||||
|
||||
(defn connect-node!
|
||||
"Connect a Garage node to the rest of the cluster"
|
||||
[test node]
|
||||
(c/trace
|
||||
(let [node-id (c/exec binary :node :id :-q)]
|
||||
(info node "node id:" node-id)
|
||||
(c/on-many (:nodes test)
|
||||
(c/exec binary :node :connect node-id)))))
|
||||
|
||||
(defn configure-node!
|
||||
"Configure a Garage node to be part of a cluster layout"
|
||||
[test node]
|
||||
(c/trace
|
||||
(let [node-id (c/exec binary :node :id :-q)]
|
||||
(c/on (jepsen/primary test)
|
||||
(c/exec binary :layout :assign (subs node-id 0 16) :-c :1G :-z :dc1 :-t node)))))
|
||||
|
||||
(defn finalize-config!
|
||||
"Apply the layout and create a key/bucket pair in the cluster"
|
||||
[node]
|
||||
(c/trace
|
||||
(c/exec binary :layout :apply :--version 1)
|
||||
(info node "garage status:" (c/exec binary :status))
|
||||
(c/exec binary :key :import access-key-id secret-access-key :--yes)
|
||||
(c/exec binary :bucket :create bucket-name)
|
||||
(c/exec binary :bucket :allow :--read :--write bucket-name :--key access-key-id)
|
||||
(info node "key info: " (c/exec binary :key :info access-key-id))))
|
||||
|
||||
(defn db
|
||||
"Garage DB for a particular version"
|
||||
[version]
|
||||
(reify db/DB
|
||||
(setup! [_ test node]
|
||||
(install! node version)
|
||||
(configure! node)
|
||||
(cu/start-daemon!
|
||||
{:logfile logfile
|
||||
:pidfile pidfile
|
||||
:chdir base-dir
|
||||
:env {:RUST_LOG "garage=debug,garage_api=trace"}}
|
||||
binary
|
||||
:server)
|
||||
(c/exec :sleep 3)
|
||||
|
||||
(jepsen/synchronize test)
|
||||
(connect-node! test node)
|
||||
|
||||
(jepsen/synchronize test)
|
||||
(configure-node! test node)
|
||||
|
||||
(jepsen/synchronize test)
|
||||
(when (= node (jepsen/primary test))
|
||||
(finalize-config! node)))
|
||||
|
||||
(teardown! [_ test node]
|
||||
(info node "tearing down garage" version)
|
||||
(c/su
|
||||
(cu/stop-daemon! binary pidfile)
|
||||
(c/exec :rm :-rf logfile)
|
||||
(c/exec :rm :-rf data-dir)
|
||||
(c/exec :rm :-rf meta-dir)))
|
||||
|
||||
db/Pause
|
||||
(pause! [_ test node]
|
||||
(cu/grepkill! :stop binary))
|
||||
(resume! [_ test node]
|
||||
(cu/grepkill! :cont binary))
|
||||
|
||||
db/Kill
|
||||
(kill! [_ test node]
|
||||
(cu/stop-daemon! binary pidfile))
|
||||
(start! [_ test node]
|
||||
(cu/start-daemon!
|
||||
{:logfile logfile
|
||||
:pidfile pidfile
|
||||
:chdir base-dir
|
||||
:env {:RUST_LOG "garage=debug,garage_api=trace"}}
|
||||
binary
|
||||
:server))
|
||||
|
||||
db/LogFiles
|
||||
(log-files [_ test node]
|
||||
[logfile])))
|
||||
|
||||
(defn creds
|
||||
"Obtain Garage credentials for node"
|
||||
[node]
|
||||
{:access-key access-key-id
|
||||
:secret-key secret-access-key
|
||||
:endpoint (str "http://" node ":3900")
|
||||
:bucket bucket-name
|
||||
:client-config {:path-style-access-enabled true}})
|
||||
|
||||
|
|
@ -1,142 +0,0 @@
|
|||
(ns jepsen.garage.nemesis
|
||||
(:require [clojure.tools.logging :refer :all]
|
||||
[jepsen [control :as c]
|
||||
[core :as jepsen]
|
||||
[generator :as gen]
|
||||
[nemesis :as nemesis]]
|
||||
[jepsen.nemesis.combined :as combined]
|
||||
[jepsen.garage.daemon :as grg]
|
||||
[jepsen.control.util :as cu]))
|
||||
|
||||
; ---- reconfiguration nemesis ----
|
||||
|
||||
(defn configure-present!
|
||||
"Configure node to be active in new cluster layout"
|
||||
[test nodes]
|
||||
(info "configure-present!" nodes)
|
||||
(let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q))
|
||||
node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)]
|
||||
(c/on
|
||||
(jepsen/primary test)
|
||||
(apply c/exec (concat [grg/binary :layout :assign :-c :1G] node-id-strs)))))
|
||||
|
||||
(defn configure-absent!
|
||||
"Configure nodes to be active in new cluster layout"
|
||||
[test nodes]
|
||||
(info "configure-absent!" nodes)
|
||||
(let [node-ids (c/on-many nodes (c/exec grg/binary :node :id :-q))
|
||||
node-id-strs (map (fn [[_ v]] (subs v 0 16)) node-ids)]
|
||||
(c/on
|
||||
(jepsen/primary test)
|
||||
(apply c/exec (concat [grg/binary :layout :assign :-g] node-id-strs)))))
|
||||
|
||||
(defn finalize-config!
|
||||
"Apply the proposed cluster layout"
|
||||
[test]
|
||||
(let [layout-show (c/on (jepsen/primary test) (c/exec grg/binary :layout :show))
|
||||
[_ layout-next-version] (re-find #"apply --version (\d+)\n" layout-show)]
|
||||
(if layout-next-version
|
||||
(do
|
||||
(info "layout show: " layout-show "; next-version: " layout-next-version)
|
||||
(c/on (jepsen/primary test)
|
||||
(c/exec grg/binary :layout :apply :--version layout-next-version)))
|
||||
(info "no layout changes to apply"))))
|
||||
|
||||
(defn reconfigure-subset
|
||||
"Reconfigure cluster with only a subset of nodes"
|
||||
[cnt]
|
||||
(reify nemesis/Nemesis
|
||||
(setup! [this test] this)
|
||||
|
||||
(invoke! [this test op] op
|
||||
(case (:f op)
|
||||
:start
|
||||
(let [[keep-nodes remove-nodes]
|
||||
(->> (:nodes test)
|
||||
shuffle
|
||||
(split-at cnt))]
|
||||
(info "layout split: keep " keep-nodes ", remove " remove-nodes)
|
||||
(configure-present! test keep-nodes)
|
||||
(configure-absent! test remove-nodes)
|
||||
(finalize-config! test)
|
||||
(assoc op :value keep-nodes))
|
||||
:stop
|
||||
(do
|
||||
(info "layout un-split: all nodes=" (:nodes test))
|
||||
(configure-present! test (:nodes test))
|
||||
(finalize-config! test)
|
||||
(assoc op :value (:nodes test)))))
|
||||
|
||||
(teardown! [this test] this)))
|
||||
|
||||
; ---- nemesis scenari ----
|
||||
|
||||
(defn nemesis-op
|
||||
"A generator for a single nemesis operation"
|
||||
[op]
|
||||
(fn [_ _] {:type :info, :f op}))
|
||||
|
||||
(defn reconfiguration-package
|
||||
"Cluster reconfiguration nemesis package"
|
||||
[opts]
|
||||
{:generator (->>
|
||||
(gen/mix [(nemesis-op :reconfigure-start)
|
||||
(nemesis-op :reconfigure-stop)])
|
||||
(gen/stagger (:interval opts 5)))
|
||||
:final-generator {:type :info, :f :reconfigure-stop}
|
||||
:nemesis (nemesis/compose
|
||||
{{:reconfigure-start :start
|
||||
:reconfigure-stop :stop} (reconfigure-subset 3)})
|
||||
:perf #{{:name "reconfigure"
|
||||
:start #{:reconfigure-start}
|
||||
:stop #{:reconfigur-stop}
|
||||
:color "#A197E9"}}})
|
||||
|
||||
(defn scenario-c
|
||||
"Clock modifying scenario"
|
||||
[opts]
|
||||
(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}}))
|
||||
|
||||
(defn scenario-cp
|
||||
"Clock modifying + partition scenario"
|
||||
[opts]
|
||||
(combined/compose-packages
|
||||
[(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
|
||||
(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})]))
|
||||
|
||||
(defn scenario-r
|
||||
"Cluster reconfiguration scenario"
|
||||
[opts]
|
||||
(reconfiguration-package {:interval 1}))
|
||||
|
||||
(defn scenario-pr
|
||||
"Partition + cluster reconfiguration scenario"
|
||||
[opts]
|
||||
(combined/compose-packages
|
||||
[(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
|
||||
(reconfiguration-package {:interval 1})]))
|
||||
|
||||
(defn scenario-cpr
|
||||
"Clock scramble + partition + cluster reconfiguration scenario"
|
||||
[opts]
|
||||
(combined/compose-packages
|
||||
[(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
|
||||
(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
|
||||
(reconfiguration-package {:interval 1})]))
|
||||
|
||||
(defn scenario-cdp
|
||||
"Clock modifying + db + partition scenario"
|
||||
[opts]
|
||||
(combined/compose-packages
|
||||
[(combined/clock-package {:db (:db opts), :interval 1, :faults #{:clock}})
|
||||
(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}})
|
||||
(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})]))
|
||||
|
||||
(defn scenario-dpr
|
||||
"Db + partition + cluster reconfiguration scenario"
|
||||
[opts]
|
||||
(combined/compose-packages
|
||||
[(combined/db-package {:db (:db opts), :interval 1, :faults #{:db :pause :kill}})
|
||||
(combined/partition-package {:db (:db opts), :interval 1, :faults #{:partition}})
|
||||
(reconfiguration-package {:interval 1})]))
|
||||
|
||||
|
|
@ -1,143 +0,0 @@
|
|||
(ns jepsen.garage.reg
|
||||
(:require [clojure.tools.logging :refer :all]
|
||||
[clojure.string :as str]
|
||||
[clojure.set :as set]
|
||||
[jepsen [checker :as checker]
|
||||
[cli :as cli]
|
||||
[client :as client]
|
||||
[control :as c]
|
||||
[db :as db]
|
||||
[generator :as gen]
|
||||
[independent :as independent]
|
||||
[nemesis :as nemesis]
|
||||
[util :as util]
|
||||
[tests :as tests]]
|
||||
[jepsen.checker.timeline :as timeline]
|
||||
[jepsen.control.util :as cu]
|
||||
[jepsen.os.debian :as debian]
|
||||
[jepsen.garage.daemon :as grg]
|
||||
[jepsen.garage.s3api :as s3]
|
||||
[knossos.model :as model]
|
||||
[slingshot.slingshot :refer [try+]]))
|
||||
|
||||
(defn op-get [_ _] {:type :invoke, :f :read, :value nil})
|
||||
(defn op-put [_ _] {:type :invoke, :f :write, :value (str (rand-int 99))})
|
||||
(defn op-del [_ _] {:type :invoke, :f :write, :value nil})
|
||||
|
||||
(defrecord RegClient [creds]
|
||||
client/Client
|
||||
(open! [this test node]
|
||||
(assoc this :creds (grg/creds node)))
|
||||
(setup! [this test])
|
||||
(invoke! [this test op]
|
||||
(try+
|
||||
(let [[k v] (:value op)]
|
||||
(case (:f op)
|
||||
:read
|
||||
(util/timeout
|
||||
10000
|
||||
(assoc op :type :fail, :error ::timeout)
|
||||
(let [value (s3/get (:creds this) k)]
|
||||
(assoc op :type :ok, :value (independent/tuple k value))))
|
||||
:write
|
||||
(util/timeout
|
||||
10000
|
||||
(assoc op :type :info, :error ::timeout)
|
||||
(do
|
||||
(s3/put (:creds this) k v)
|
||||
(assoc op :type :ok)))))
|
||||
(catch (re-find #"Unavailable" (.getMessage %)) ex
|
||||
(assoc op :type :info, :error ::unavailable))
|
||||
(catch (re-find #"Broken pipe" (.getMessage %)) ex
|
||||
(assoc op :type :info, :error ::broken-pipe))
|
||||
(catch (re-find #"Connection refused" (.getMessage %)) ex
|
||||
(assoc op :type :info, :error ::connection-refused))))
|
||||
(teardown! [this test])
|
||||
(close! [this test]))
|
||||
|
||||
(defn reg-read-after-write
|
||||
"Read-after-Write checker for register operations"
|
||||
[]
|
||||
(reify checker/Checker
|
||||
(check [this test history opts]
|
||||
(let [init {:put-values {-1 nil}
|
||||
:put-done #{-1}
|
||||
:put-in-progress {}
|
||||
:read-can-contain {}
|
||||
:bad-reads #{}}
|
||||
final (reduce
|
||||
(fn [state op]
|
||||
(let [current-values (set/union
|
||||
(set (map (fn [idx] (get (:put-values state) idx)) (:put-done state)))
|
||||
(set (map (fn [[_ [idx _]]] (get (:put-values state) idx)) (:put-in-progress state))))
|
||||
read-can-contain (reduce
|
||||
(fn [rcc [idx v]] (assoc rcc idx (set/union current-values v)))
|
||||
{} (:read-can-contain state))]
|
||||
(info "--------")
|
||||
(info "state: " state)
|
||||
(info "current-values: " current-values)
|
||||
(info "read-can-contain: " read-can-contain)
|
||||
(info "op: " op)
|
||||
(case [(:type op) (:f op)]
|
||||
([:invoke :write])
|
||||
(assoc state
|
||||
:read-can-contain read-can-contain
|
||||
:put-values (assoc (:put-values state) (:index op) (:value op))
|
||||
:put-in-progress (assoc (:put-in-progress state) (:process op) [(:index op) (:put-done state)]))
|
||||
([:ok :write])
|
||||
(let [[index overwrites] (get (:put-in-progress state) (:process op))]
|
||||
(assoc state
|
||||
:read-can-contain read-can-contain
|
||||
:put-in-progress (dissoc (:put-in-progress state) (:process op))
|
||||
:put-done
|
||||
(conj
|
||||
(set/difference (:put-done state) overwrites)
|
||||
index)))
|
||||
([:invoke :read])
|
||||
(assoc state
|
||||
:read-can-contain (assoc read-can-contain (:process op) current-values))
|
||||
([:ok :read])
|
||||
(let [this-read-can-contain (get read-can-contain (:process op))
|
||||
bad-reads (if (contains? this-read-can-contain (:value op))
|
||||
(:bad-reads state)
|
||||
(conj (:bad-reads state) [(:process op) (:index op) (:value op) this-read-can-contain]))]
|
||||
(info "this-read-can-contain: " this-read-can-contain)
|
||||
(assoc state
|
||||
:read-can-contain (dissoc read-can-contain (:process op))
|
||||
:bad-reads bad-reads))
|
||||
state)))
|
||||
init history)
|
||||
valid? (empty? (:bad-reads final))]
|
||||
(assoc final :valid? valid?)))))
|
||||
|
||||
(defn workload-common
|
||||
"Common parts of workload"
|
||||
[opts]
|
||||
{:client (RegClient. nil)
|
||||
:generator (independent/concurrent-generator
|
||||
10
|
||||
(range)
|
||||
(fn [k]
|
||||
(->>
|
||||
(gen/mix [op-get op-put op-del])
|
||||
(gen/limit (:ops-per-key opts)))))})
|
||||
|
||||
(defn workload1
|
||||
"Tests linearizable reads and writes"
|
||||
[opts]
|
||||
(assoc (workload-common opts)
|
||||
:checker (independent/checker
|
||||
(checker/compose
|
||||
{:linear (checker/linearizable
|
||||
{:model (model/register)
|
||||
:algorithm :linear})
|
||||
:timeline (timeline/html)}))))
|
||||
|
||||
(defn workload2
|
||||
"Tests CRDT reads and writes"
|
||||
[opts]
|
||||
(assoc (workload-common opts)
|
||||
:checker (independent/checker
|
||||
(checker/compose
|
||||
{:reg-read-after-write (reg-read-after-write)
|
||||
:timeline (timeline/html)}))))
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
(ns jepsen.garage.s3api
|
||||
(:require [clojure.tools.logging :refer :all]
|
||||
[jepsen [control :as c]]
|
||||
[amazonica.aws.s3 :as s3]
|
||||
[slingshot.slingshot :refer [try+]]))
|
||||
|
||||
; GARAGE S3 HELPER FUNCTIONS
|
||||
|
||||
(defn get
|
||||
"Helper for GetObject"
|
||||
[creds k]
|
||||
(try+
|
||||
(-> (s3/get-object creds (:bucket creds) k)
|
||||
:input-stream
|
||||
slurp)
|
||||
(catch (re-find #"Key not found" (.getMessage %)) ex
|
||||
nil)))
|
||||
|
||||
(defn put
|
||||
"Helper for PutObject or DeleteObject (is a delete if value is nil)"
|
||||
[creds k v]
|
||||
(if (= v nil)
|
||||
(s3/delete-object creds
|
||||
:bucket-name (:bucket creds)
|
||||
:key k)
|
||||
(let [some-bytes (.getBytes v "UTF-8")
|
||||
bytes-stream (java.io.ByteArrayInputStream. some-bytes)]
|
||||
(s3/put-object creds
|
||||
:bucket-name (:bucket creds)
|
||||
:key k
|
||||
:input-stream bytes-stream
|
||||
:metadata {:content-length (count some-bytes)}))))
|
||||
|
||||
(defn list-inner [creds prefix ct accum]
|
||||
(let [list-result (s3/list-objects-v2 creds
|
||||
{:bucket-name (:bucket creds)
|
||||
:prefix prefix
|
||||
:continuation-token ct})
|
||||
new-object-summaries (:object-summaries list-result)
|
||||
new-objects (map (fn [d] (:key d)) new-object-summaries)
|
||||
objects (concat new-objects accum)]
|
||||
(if (:truncated? list-result)
|
||||
(list-inner creds prefix (:next-continuation-token list-result) objects)
|
||||
objects)))
|
||||
(defn list
|
||||
"Helper for ListObjects -- just lists everything in the bucket"
|
||||
[creds prefix]
|
||||
(list-inner creds prefix nil []))
|
||||
|
|
@ -1,135 +0,0 @@
|
|||
(ns jepsen.garage.set
|
||||
(:require [clojure.tools.logging :refer :all]
|
||||
[clojure.string :as str]
|
||||
[clojure.set :as set]
|
||||
[jepsen [checker :as checker]
|
||||
[cli :as cli]
|
||||
[client :as client]
|
||||
[control :as c]
|
||||
[checker :as checker]
|
||||
[db :as db]
|
||||
[generator :as gen]
|
||||
[independent :as independent]
|
||||
[nemesis :as nemesis]
|
||||
[util :as util]
|
||||
[tests :as tests]]
|
||||
[jepsen.checker.timeline :as timeline]
|
||||
[jepsen.control.util :as cu]
|
||||
[jepsen.os.debian :as debian]
|
||||
[jepsen.garage.daemon :as grg]
|
||||
[jepsen.garage.s3api :as s3]
|
||||
[knossos.model :as model]
|
||||
[slingshot.slingshot :refer [try+]]))
|
||||
|
||||
(defn op-add-rand100 [_ _] {:type :invoke, :f :add, :value (rand-int 100)})
|
||||
(defn op-read [_ _] {:type :invoke, :f :read, :value nil})
|
||||
|
||||
(defrecord SetClient [creds]
|
||||
client/Client
|
||||
(open! [this test node]
|
||||
(assoc this :creds (grg/creds node)))
|
||||
(setup! [this test])
|
||||
(invoke! [this test op]
|
||||
(try+
|
||||
(let [[k v] (:value op)
|
||||
prefix (str "set" k "/")]
|
||||
(case (:f op)
|
||||
:add
|
||||
(util/timeout
|
||||
10000
|
||||
(assoc op :type :info, :error ::timeout)
|
||||
(do
|
||||
(s3/put (:creds this) (str prefix v) "present")
|
||||
(assoc op :type :ok)))
|
||||
:read
|
||||
(util/timeout
|
||||
10000
|
||||
(assoc op :type :fail, :error ::timeout)
|
||||
(do
|
||||
(let [items (s3/list (:creds this) prefix)]
|
||||
(let [items-stripped (map (fn [o]
|
||||
(assert (str/starts-with? o prefix))
|
||||
(str/replace-first o prefix "")) items)
|
||||
items-set (set (map parse-long items-stripped))]
|
||||
(assoc op :type :ok, :value (independent/tuple k items-set))))))))
|
||||
(catch (re-find #"Unavailable" (.getMessage %)) ex
|
||||
(assoc op :type :info, :error ::unavailable))
|
||||
(catch (re-find #"Broken pipe" (.getMessage %)) ex
|
||||
(assoc op :type :info, :error ::broken-pipe))
|
||||
(catch (re-find #"Connection refused" (.getMessage %)) ex
|
||||
(assoc op :type :info, :error ::connection-refused))))
|
||||
(teardown! [this test])
|
||||
(close! [this test]))
|
||||
|
||||
(defn set-read-after-write
|
||||
"Read-after-Write checker for set operations"
|
||||
[]
|
||||
(reify checker/Checker
|
||||
(check [this test history opts]
|
||||
(let [init {:add-started #{}
|
||||
:add-done #{}
|
||||
:read-must-contain {}
|
||||
:missed #{}
|
||||
:unexpected #{}}
|
||||
final (reduce
|
||||
(fn [state op]
|
||||
(case [(:type op) (:f op)]
|
||||
([:invoke :add])
|
||||
(assoc state :add-started (conj (:add-started state) (:value op)))
|
||||
([:ok :add])
|
||||
(assoc state :add-done (conj (:add-done state) (:value op)))
|
||||
([:invoke :read])
|
||||
(assoc-in state [:read-must-contain (:process op)] (:add-done state))
|
||||
([:ok :read])
|
||||
(let [read-must-contain (get (:read-must-contain state) (:process op))
|
||||
new-missed (set/difference read-must-contain (:value op))
|
||||
new-unexpected (set/difference (:value op) (:add-started state))]
|
||||
(assoc state
|
||||
:read-must-contain (dissoc (:read-must-contain state) (:process op))
|
||||
:missed (set/union (:missed state) new-missed),
|
||||
:unexpected (set/union (:unexpected state) new-unexpected)))
|
||||
state))
|
||||
init history)
|
||||
valid? (and (empty? (:missed final)) (empty? (:unexpected final)))]
|
||||
(assoc final :valid? valid?)))))
|
||||
|
||||
(defn workload1
|
||||
"Tests insertions and deletions"
|
||||
[opts]
|
||||
{:client (SetClient. nil)
|
||||
:checker (independent/checker
|
||||
(checker/compose
|
||||
{:set (checker/set)
|
||||
:timeline (timeline/html)}))
|
||||
:generator (independent/concurrent-generator
|
||||
10
|
||||
(range 100)
|
||||
(fn [k]
|
||||
(->> (range)
|
||||
(map (fn [x] {:type :invoke, :f :add, :value x}))
|
||||
(gen/limit (:ops-per-key opts)))))
|
||||
:final-generator (independent/concurrent-generator
|
||||
10
|
||||
(range 100)
|
||||
(fn [k]
|
||||
(gen/phases
|
||||
(gen/once op-read)
|
||||
(gen/sleep 5))))})
|
||||
|
||||
(defn workload2
|
||||
"Tests insertions and deletions"
|
||||
[opts]
|
||||
{:client (SetClient. nil)
|
||||
:checker (independent/checker
|
||||
(checker/compose
|
||||
{:set-read-after-write (set-read-after-write)
|
||||
; :set-full (checker/set-full {:linearizable? false})
|
||||
:timeline (timeline/html)}))
|
||||
:generator (independent/concurrent-generator
|
||||
10
|
||||
(range)
|
||||
(fn [k]
|
||||
(->> (gen/mix [op-add-rand100 op-read])
|
||||
(gen/limit (:ops-per-key opts)))))})
|
||||
|
||||
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
(ns jepsen.garage-test
|
||||
(:require [clojure.test :refer :all]
|
||||
[jepsen.garage :refer :all]))
|
||||
|
||||
(deftest a-test
|
||||
(testing "FIXME, I fail."
|
||||
(is (= 0 1))))
|
||||
|
|
@ -182,7 +182,7 @@ impl AdminApiServer {
|
|||
),
|
||||
};
|
||||
let status_str = format!(
|
||||
"{}\nConsult the full health check API endpoint at /v1/health for more details\n",
|
||||
"{}\nConsult the full health check API endpoint at /v0/health for more details\n",
|
||||
status_str
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -344,7 +344,7 @@ impl ApiHandler for S3ApiServer {
|
|||
bucket_id,
|
||||
key,
|
||||
upload_id,
|
||||
part_number_marker: part_number_marker.map(|p| p.min(10000)),
|
||||
part_number_marker: part_number_marker.map(|p| p.clamp(1, 10000)),
|
||||
max_parts: max_parts.unwrap_or(1000).clamp(1, 1000),
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -67,7 +67,6 @@ chrono = "0.4"
|
|||
http = "0.2"
|
||||
hmac = "0.12"
|
||||
hyper = { version = "0.14", features = ["client", "http1", "runtime"] }
|
||||
mktemp = "0.5"
|
||||
sha2 = "0.10"
|
||||
|
||||
static_init = "1.0"
|
||||
|
|
|
|||
|
|
@ -7,7 +7,6 @@ extern crate tracing;
|
|||
mod admin;
|
||||
mod cli;
|
||||
mod repair;
|
||||
mod secrets;
|
||||
mod server;
|
||||
#[cfg(feature = "telemetry-otlp")]
|
||||
mod tracing_setup;
|
||||
|
|
@ -29,6 +28,7 @@ use structopt::StructOpt;
|
|||
use netapp::util::parse_and_resolve_peer_addr;
|
||||
use netapp::NetworkKey;
|
||||
|
||||
use garage_util::config::{read_secret_file, Config};
|
||||
use garage_util::error::*;
|
||||
|
||||
use garage_rpc::system::*;
|
||||
|
|
@ -38,7 +38,6 @@ use garage_model::helper::error::Error as HelperError;
|
|||
|
||||
use admin::*;
|
||||
use cli::*;
|
||||
use secrets::Secrets;
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
#[structopt(
|
||||
|
|
@ -67,6 +66,39 @@ struct Opt {
|
|||
cmd: Command,
|
||||
}
|
||||
|
||||
#[derive(StructOpt, Debug)]
|
||||
pub struct Secrets {
|
||||
/// RPC secret network key, used to replace rpc_secret in config.toml when running the
|
||||
/// daemon or doing admin operations
|
||||
#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
|
||||
pub rpc_secret: Option<String>,
|
||||
|
||||
/// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret
|
||||
/// when running the daemon or doing admin operations
|
||||
#[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")]
|
||||
pub rpc_secret_file: Option<String>,
|
||||
|
||||
/// Admin API authentication token, replaces admin.admin_token in config.toml when
|
||||
/// running the Garage daemon
|
||||
#[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
|
||||
pub admin_token: Option<String>,
|
||||
|
||||
/// Admin API authentication token file path, replaces admin.admin_token in config.toml
|
||||
/// and admin-token when running the Garage daemon
|
||||
#[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")]
|
||||
pub admin_token_file: Option<String>,
|
||||
|
||||
/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
|
||||
/// running the Garage daemon
|
||||
#[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
|
||||
pub metrics_token: Option<String>,
|
||||
|
||||
/// Metrics API authentication token file path, replaces admin.metrics_token in config.toml
|
||||
/// and metrics-token when running the Garage daemon
|
||||
#[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")]
|
||||
pub metrics_token_file: Option<String>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
// Initialize version and features info
|
||||
|
|
@ -244,3 +276,25 @@ async fn cli_command(opt: Opt) -> Result<(), Error> {
|
|||
Ok(x) => Ok(x),
|
||||
}
|
||||
}
|
||||
|
||||
fn fill_secrets(mut config: Config, secrets: Secrets) -> Result<Config, Error> {
|
||||
if secrets.rpc_secret.is_some() {
|
||||
config.rpc_secret = secrets.rpc_secret;
|
||||
} else if secrets.rpc_secret_file.is_some() {
|
||||
config.rpc_secret = Some(read_secret_file(&secrets.rpc_secret_file.unwrap())?);
|
||||
}
|
||||
|
||||
if secrets.admin_token.is_some() {
|
||||
config.admin.admin_token = secrets.admin_token;
|
||||
} else if secrets.admin_token_file.is_some() {
|
||||
config.admin.admin_token = Some(read_secret_file(&secrets.admin_token_file.unwrap())?);
|
||||
}
|
||||
|
||||
if secrets.metrics_token.is_some() {
|
||||
config.admin.metrics_token = secrets.metrics_token;
|
||||
} else if secrets.metrics_token_file.is_some() {
|
||||
config.admin.metrics_token = Some(read_secret_file(&secrets.metrics_token_file.unwrap())?);
|
||||
}
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ use garage_util::error::*;
|
|||
use garage_model::garage::Garage;
|
||||
|
||||
use crate::cli::structs::*;
|
||||
use crate::secrets::{fill_secrets, Secrets};
|
||||
use crate::{fill_secrets, Secrets};
|
||||
|
||||
pub async fn offline_repair(
|
||||
config_file: PathBuf,
|
||||
|
|
|
|||
|
|
@ -1,318 +0,0 @@
|
|||
use structopt::StructOpt;
|
||||
|
||||
use garage_util::config::Config;
|
||||
use garage_util::error::Error;
|
||||
|
||||
/// Structure for secret values or paths that are passed as CLI arguments or environment
|
||||
/// variables, instead of in the config file.
|
||||
#[derive(StructOpt, Debug, Default, Clone)]
|
||||
pub struct Secrets {
|
||||
/// Skip permission check on files containing secrets
|
||||
#[cfg(unix)]
|
||||
#[structopt(
|
||||
long = "allow-world-readable-secrets",
|
||||
env = "GARAGE_ALLOW_WORLD_READABLE_SECRETS"
|
||||
)]
|
||||
pub allow_world_readable_secrets: Option<bool>,
|
||||
|
||||
/// RPC secret network key, used to replace rpc_secret in config.toml when running the
|
||||
/// daemon or doing admin operations
|
||||
#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
|
||||
pub rpc_secret: Option<String>,
|
||||
|
||||
/// RPC secret network key, used to replace rpc_secret in config.toml and rpc-secret
|
||||
/// when running the daemon or doing admin operations
|
||||
#[structopt(long = "rpc-secret-file", env = "GARAGE_RPC_SECRET_FILE")]
|
||||
pub rpc_secret_file: Option<String>,
|
||||
|
||||
/// Admin API authentication token, replaces admin.admin_token in config.toml when
|
||||
/// running the Garage daemon
|
||||
#[structopt(long = "admin-token", env = "GARAGE_ADMIN_TOKEN")]
|
||||
pub admin_token: Option<String>,
|
||||
|
||||
/// Admin API authentication token file path, replaces admin.admin_token in config.toml
|
||||
/// and admin-token when running the Garage daemon
|
||||
#[structopt(long = "admin-token-file", env = "GARAGE_ADMIN_TOKEN_FILE")]
|
||||
pub admin_token_file: Option<String>,
|
||||
|
||||
/// Metrics API authentication token, replaces admin.metrics_token in config.toml when
|
||||
/// running the Garage daemon
|
||||
#[structopt(long = "metrics-token", env = "GARAGE_METRICS_TOKEN")]
|
||||
pub metrics_token: Option<String>,
|
||||
|
||||
/// Metrics API authentication token file path, replaces admin.metrics_token in config.toml
|
||||
/// and metrics-token when running the Garage daemon
|
||||
#[structopt(long = "metrics-token-file", env = "GARAGE_METRICS_TOKEN_FILE")]
|
||||
pub metrics_token_file: Option<String>,
|
||||
}
|
||||
|
||||
/// Single function to fill all secrets in the Config struct from their correct source (value
|
||||
/// from config or CLI param or env variable or read from a file specified in config or CLI
|
||||
/// param or env variable)
|
||||
pub fn fill_secrets(mut config: Config, secrets: Secrets) -> Result<Config, Error> {
|
||||
let allow_world_readable = secrets
|
||||
.allow_world_readable_secrets
|
||||
.unwrap_or(config.allow_world_readable_secrets);
|
||||
|
||||
fill_secret(
|
||||
&mut config.rpc_secret,
|
||||
&config.rpc_secret_file,
|
||||
&secrets.rpc_secret,
|
||||
&secrets.rpc_secret_file,
|
||||
"rpc_secret",
|
||||
allow_world_readable,
|
||||
)?;
|
||||
|
||||
fill_secret(
|
||||
&mut config.admin.admin_token,
|
||||
&config.admin.admin_token_file,
|
||||
&secrets.admin_token,
|
||||
&secrets.admin_token_file,
|
||||
"admin.admin_token",
|
||||
allow_world_readable,
|
||||
)?;
|
||||
fill_secret(
|
||||
&mut config.admin.metrics_token,
|
||||
&config.admin.metrics_token_file,
|
||||
&secrets.metrics_token,
|
||||
&secrets.metrics_token_file,
|
||||
"admin.metrics_token",
|
||||
allow_world_readable,
|
||||
)?;
|
||||
|
||||
Ok(config)
|
||||
}
|
||||
|
||||
fn fill_secret(
|
||||
config_secret: &mut Option<String>,
|
||||
config_secret_file: &Option<String>,
|
||||
cli_secret: &Option<String>,
|
||||
cli_secret_file: &Option<String>,
|
||||
name: &'static str,
|
||||
allow_world_readable: bool,
|
||||
) -> Result<(), Error> {
|
||||
let cli_value = match (&cli_secret, &cli_secret_file) {
|
||||
(Some(_), Some(_)) => {
|
||||
return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
|
||||
}
|
||||
(Some(secret), None) => Some(secret.to_string()),
|
||||
(None, Some(file)) => Some(read_secret_file(file, allow_world_readable)?),
|
||||
(None, None) => None,
|
||||
};
|
||||
|
||||
if let Some(val) = cli_value {
|
||||
if config_secret.is_some() || config_secret_file.is_some() {
|
||||
debug!("Overriding secret `{}` using value specified using CLI argument or environnement variable.", name);
|
||||
}
|
||||
|
||||
*config_secret = Some(val);
|
||||
} else if let Some(file_path) = &config_secret_file {
|
||||
if config_secret.is_some() {
|
||||
return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
|
||||
}
|
||||
|
||||
*config_secret = Some(read_secret_file(file_path, allow_world_readable)?);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_secret_file(file_path: &String, allow_world_readable: bool) -> Result<String, Error> {
|
||||
if !allow_world_readable {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let metadata = std::fs::metadata(file_path)?;
|
||||
if metadata.mode() & 0o077 != 0 {
|
||||
return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let secret_buf = std::fs::read_to_string(file_path)?;
|
||||
|
||||
// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
|
||||
// also editors sometimes add a trailing newline
|
||||
Ok(String::from(secret_buf.trim_end()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
|
||||
use garage_util::config::read_config;
|
||||
use garage_util::error::Error;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_rpc_secret_file_works() -> Result<(), Error> {
|
||||
let path_secret = mktemp::Temp::new_file()?;
|
||||
let mut file_secret = File::create(path_secret.as_path())?;
|
||||
writeln!(file_secret, "foo")?;
|
||||
drop(file_secret);
|
||||
|
||||
let path_config = mktemp::Temp::new_file()?;
|
||||
let mut file_config = File::create(path_config.as_path())?;
|
||||
let path_secret_path = path_secret.as_path();
|
||||
writeln!(
|
||||
file_config,
|
||||
r#"
|
||||
metadata_dir = "/tmp/garage/meta"
|
||||
data_dir = "/tmp/garage/data"
|
||||
replication_mode = "3"
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_secret_file = "{}"
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "[::]:3900"
|
||||
"#,
|
||||
path_secret_path.display()
|
||||
)?;
|
||||
drop(file_config);
|
||||
|
||||
// Second configuration file, same as previous one
|
||||
// except it allows world-readable secrets.
|
||||
let path_config_allow_world_readable = mktemp::Temp::new_file()?;
|
||||
let mut file_config_allow_world_readable =
|
||||
File::create(path_config_allow_world_readable.as_path())?;
|
||||
writeln!(
|
||||
file_config_allow_world_readable,
|
||||
r#"
|
||||
metadata_dir = "/tmp/garage/meta"
|
||||
data_dir = "/tmp/garage/data"
|
||||
replication_mode = "3"
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_secret_file = "{}"
|
||||
allow_world_readable_secrets = true
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "[::]:3900"
|
||||
"#,
|
||||
path_secret_path.display()
|
||||
)?;
|
||||
drop(file_config_allow_world_readable);
|
||||
|
||||
let config = read_config(path_config.to_path_buf())?;
|
||||
let config = fill_secrets(config, Secrets::default())?;
|
||||
assert_eq!("foo", config.rpc_secret.unwrap());
|
||||
|
||||
// ---- Check non world-readable secrets config ----
|
||||
#[cfg(unix)]
|
||||
{
|
||||
let secrets_allow_world_readable = Secrets {
|
||||
allow_world_readable_secrets: Some(true),
|
||||
..Default::default()
|
||||
};
|
||||
let secrets_no_allow_world_readable = Secrets {
|
||||
allow_world_readable_secrets: Some(false),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let metadata = std::fs::metadata(&path_secret_path)?;
|
||||
let mut perm = metadata.permissions();
|
||||
perm.set_mode(0o660);
|
||||
std::fs::set_permissions(&path_secret_path, perm)?;
|
||||
|
||||
// Config file that just specifies the path
|
||||
let config = read_config(path_config.to_path_buf())?;
|
||||
assert!(fill_secrets(config, Secrets::default()).is_err());
|
||||
|
||||
let config = read_config(path_config.to_path_buf())?;
|
||||
assert!(fill_secrets(config, secrets_allow_world_readable.clone()).is_ok());
|
||||
|
||||
let config = read_config(path_config.to_path_buf())?;
|
||||
assert!(fill_secrets(config, secrets_no_allow_world_readable.clone()).is_err());
|
||||
|
||||
// Config file that also specifies to allow world_readable_secrets
|
||||
let config = read_config(path_config_allow_world_readable.to_path_buf())?;
|
||||
assert!(fill_secrets(config, Secrets::default()).is_ok());
|
||||
|
||||
let config = read_config(path_config_allow_world_readable.to_path_buf())?;
|
||||
assert!(fill_secrets(config, secrets_allow_world_readable).is_ok());
|
||||
|
||||
let config = read_config(path_config_allow_world_readable.to_path_buf())?;
|
||||
assert!(fill_secrets(config, secrets_no_allow_world_readable).is_err());
|
||||
}
|
||||
|
||||
// ---- Check alternative secrets specified on CLI ----
|
||||
|
||||
let path_secret2 = mktemp::Temp::new_file()?;
|
||||
let mut file_secret2 = File::create(path_secret2.as_path())?;
|
||||
writeln!(file_secret2, "bar")?;
|
||||
drop(file_secret2);
|
||||
|
||||
let config = read_config(path_config.to_path_buf())?;
|
||||
let config = fill_secrets(
|
||||
config,
|
||||
Secrets {
|
||||
rpc_secret: Some("baz".into()),
|
||||
..Default::default()
|
||||
},
|
||||
)?;
|
||||
assert_eq!(config.rpc_secret.as_deref(), Some("baz"));
|
||||
|
||||
let config = read_config(path_config.to_path_buf())?;
|
||||
let config = fill_secrets(
|
||||
config,
|
||||
Secrets {
|
||||
rpc_secret_file: Some(path_secret2.display().to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)?;
|
||||
assert_eq!(config.rpc_secret.as_deref(), Some("bar"));
|
||||
|
||||
let config = read_config(path_config.to_path_buf())?;
|
||||
assert!(fill_secrets(
|
||||
config,
|
||||
Secrets {
|
||||
rpc_secret: Some("baz".into()),
|
||||
rpc_secret_file: Some(path_secret2.display().to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
)
|
||||
.is_err());
|
||||
|
||||
drop(path_secret);
|
||||
drop(path_secret2);
|
||||
drop(path_config);
|
||||
drop(path_config_allow_world_readable);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> {
|
||||
let path_config = mktemp::Temp::new_file()?;
|
||||
let mut file_config = File::create(path_config.as_path())?;
|
||||
writeln!(
|
||||
file_config,
|
||||
r#"
|
||||
metadata_dir = "/tmp/garage/meta"
|
||||
data_dir = "/tmp/garage/data"
|
||||
replication_mode = "3"
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_secret= "dummy"
|
||||
rpc_secret_file = "dummy"
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "[::]:3900"
|
||||
"#
|
||||
)?;
|
||||
let config = read_config(path_config.to_path_buf())?;
|
||||
assert_eq!(
|
||||
"only one of `rpc_secret` and `rpc_secret_file` can be set",
|
||||
fill_secrets(config, Secrets::default())
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
);
|
||||
drop(path_config);
|
||||
drop(file_config);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -15,9 +15,9 @@ use garage_web::WebServer;
|
|||
use garage_api::k2v::api_server::K2VApiServer;
|
||||
|
||||
use crate::admin::*;
|
||||
use crate::secrets::{fill_secrets, Secrets};
|
||||
#[cfg(feature = "telemetry-otlp")]
|
||||
use crate::tracing_setup::*;
|
||||
use crate::{fill_secrets, Secrets};
|
||||
|
||||
async fn wait_from(mut chan: watch::Receiver<bool>) {
|
||||
while !*chan.borrow() {
|
||||
|
|
@ -34,14 +34,7 @@ pub async fn run_server(config_file: PathBuf, secrets: Secrets) -> Result<(), Er
|
|||
// ---- Initialize Garage internals ----
|
||||
|
||||
#[cfg(feature = "metrics")]
|
||||
let metrics_exporter = opentelemetry_prometheus::exporter()
|
||||
.with_default_summary_quantiles(vec![0.25, 0.5, 0.75, 0.9, 0.95, 0.99])
|
||||
.with_default_histogram_boundaries(vec![
|
||||
0.001, 0.0015, 0.002, 0.003, 0.005, 0.007, 0.01, 0.015, 0.02, 0.03, 0.05, 0.07, 0.1,
|
||||
0.15, 0.2, 0.3, 0.5, 0.7, 1., 1.5, 2., 3., 5., 7., 10., 15., 20., 30., 40., 50., 60.,
|
||||
70., 100.,
|
||||
])
|
||||
.init();
|
||||
let metrics_exporter = opentelemetry_prometheus::exporter().init();
|
||||
|
||||
info!("Initializing Garage main data store...");
|
||||
let garage = Garage::new(config.clone())?;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
//! Contains type and functions related to Garage configuration file
|
||||
use std::convert::TryFrom;
|
||||
use std::io::Read;
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
|
||||
|
|
@ -44,15 +45,11 @@ pub struct Config {
|
|||
)]
|
||||
pub compression_level: Option<i32>,
|
||||
|
||||
/// Skip the permission check of secret files. Useful when
|
||||
/// POSIX ACLs (or more complex chmods) are used.
|
||||
#[serde(default)]
|
||||
pub allow_world_readable_secrets: bool,
|
||||
|
||||
/// RPC secret key: 32 bytes hex encoded
|
||||
pub rpc_secret: Option<String>,
|
||||
/// Optional file where RPC secret key is read from
|
||||
pub rpc_secret_file: Option<String>,
|
||||
|
||||
/// Address to bind for RPC
|
||||
pub rpc_bind_addr: SocketAddr,
|
||||
/// Public IP address of this node
|
||||
|
|
@ -224,13 +221,6 @@ pub struct KubernetesDiscoveryConfig {
|
|||
pub skip_crd: bool,
|
||||
}
|
||||
|
||||
/// Read and parse configuration
|
||||
pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
|
||||
let config = std::fs::read_to_string(config_file)?;
|
||||
|
||||
Ok(toml::from_str(&config)?)
|
||||
}
|
||||
|
||||
fn default_db_engine() -> String {
|
||||
"lmdb".into()
|
||||
}
|
||||
|
|
@ -245,6 +235,71 @@ fn default_block_size() -> usize {
|
|||
1048576
|
||||
}
|
||||
|
||||
/// Read and parse configuration
|
||||
pub fn read_config(config_file: PathBuf) -> Result<Config, Error> {
|
||||
let mut file = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.open(config_file.as_path())?;
|
||||
|
||||
let mut config = String::new();
|
||||
file.read_to_string(&mut config)?;
|
||||
|
||||
let mut parsed_config: Config = toml::from_str(&config)?;
|
||||
|
||||
secret_from_file(
|
||||
&mut parsed_config.rpc_secret,
|
||||
&parsed_config.rpc_secret_file,
|
||||
"rpc_secret",
|
||||
)?;
|
||||
secret_from_file(
|
||||
&mut parsed_config.admin.metrics_token,
|
||||
&parsed_config.admin.metrics_token_file,
|
||||
"admin.metrics_token",
|
||||
)?;
|
||||
secret_from_file(
|
||||
&mut parsed_config.admin.admin_token,
|
||||
&parsed_config.admin.admin_token_file,
|
||||
"admin.admin_token",
|
||||
)?;
|
||||
|
||||
Ok(parsed_config)
|
||||
}
|
||||
|
||||
pub fn read_secret_file(file_path: &String) -> Result<String, Error> {
|
||||
#[cfg(unix)]
|
||||
if std::env::var("GARAGE_ALLOW_WORLD_READABLE_SECRETS").as_deref() != Ok("true") {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let metadata = std::fs::metadata(file_path)?;
|
||||
if metadata.mode() & 0o077 != 0 {
|
||||
return Err(format!("File {} is world-readable! (mode: 0{:o}, expected 0600)\nRefusing to start until this is fixed, or environment variable GARAGE_ALLOW_WORLD_READABLE_SECRETS is set to true.", file_path, metadata.mode()).into());
|
||||
}
|
||||
}
|
||||
let mut file = std::fs::OpenOptions::new().read(true).open(file_path)?;
|
||||
let mut secret_buf = String::new();
|
||||
file.read_to_string(&mut secret_buf)?;
|
||||
|
||||
// trim_end: allows for use case such as `echo "$(openssl rand -hex 32)" > somefile`.
|
||||
// also editors sometimes add a trailing newline
|
||||
Ok(String::from(secret_buf.trim_end()))
|
||||
}
|
||||
|
||||
fn secret_from_file(
|
||||
secret: &mut Option<String>,
|
||||
secret_file: &Option<String>,
|
||||
name: &'static str,
|
||||
) -> Result<(), Error> {
|
||||
match (&secret, &secret_file) {
|
||||
(_, None) => {
|
||||
// no-op
|
||||
}
|
||||
(Some(_), Some(_)) => {
|
||||
return Err(format!("only one of `{}` and `{}_file` can be set", name, name).into());
|
||||
}
|
||||
(None, Some(file_path)) => *secret = Some(read_secret_file(file_path)?),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn default_compression() -> Option<i32> {
|
||||
Some(1)
|
||||
}
|
||||
|
|
@ -373,4 +428,83 @@ mod tests {
|
|||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rpc_secret_file_works() -> Result<(), Error> {
|
||||
let path_secret = mktemp::Temp::new_file()?;
|
||||
let mut file_secret = File::create(path_secret.as_path())?;
|
||||
writeln!(file_secret, "foo")?;
|
||||
drop(file_secret);
|
||||
|
||||
let path_config = mktemp::Temp::new_file()?;
|
||||
let mut file_config = File::create(path_config.as_path())?;
|
||||
let path_secret_path = path_secret.as_path();
|
||||
writeln!(
|
||||
file_config,
|
||||
r#"
|
||||
metadata_dir = "/tmp/garage/meta"
|
||||
data_dir = "/tmp/garage/data"
|
||||
replication_mode = "3"
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_secret_file = "{}"
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "[::]:3900"
|
||||
"#,
|
||||
path_secret_path.display()
|
||||
)?;
|
||||
let config = super::read_config(path_config.to_path_buf())?;
|
||||
assert_eq!("foo", config.rpc_secret.unwrap());
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let metadata = std::fs::metadata(&path_secret_path)?;
|
||||
let mut perm = metadata.permissions();
|
||||
perm.set_mode(0o660);
|
||||
std::fs::set_permissions(&path_secret_path, perm)?;
|
||||
|
||||
std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "false");
|
||||
assert!(super::read_config(path_config.to_path_buf()).is_err());
|
||||
|
||||
std::env::set_var("GARAGE_ALLOW_WORLD_READABLE_SECRETS", "true");
|
||||
assert!(super::read_config(path_config.to_path_buf()).is_ok());
|
||||
}
|
||||
|
||||
drop(path_config);
|
||||
drop(path_secret);
|
||||
drop(file_config);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_rcp_secret_and_rpc_secret_file_cannot_be_set_both() -> Result<(), Error> {
|
||||
let path_config = mktemp::Temp::new_file()?;
|
||||
let mut file_config = File::create(path_config.as_path())?;
|
||||
writeln!(
|
||||
file_config,
|
||||
r#"
|
||||
metadata_dir = "/tmp/garage/meta"
|
||||
data_dir = "/tmp/garage/data"
|
||||
replication_mode = "3"
|
||||
rpc_bind_addr = "[::]:3901"
|
||||
rpc_secret= "dummy"
|
||||
rpc_secret_file = "dummy"
|
||||
|
||||
[s3_api]
|
||||
s3_region = "garage"
|
||||
api_bind_addr = "[::]:3900"
|
||||
"#
|
||||
)?;
|
||||
assert_eq!(
|
||||
"only one of `rpc_secret` and `rpc_secret_file` can be set",
|
||||
super::read_config(path_config.to_path_buf())
|
||||
.unwrap_err()
|
||||
.to_string()
|
||||
);
|
||||
drop(path_config);
|
||||
drop(file_config);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue