forked from Deuxfleurs/garage
Update documentation: real_world.md
This commit is contained in:
parent
550ce7db2a
commit
d9c52e9a9c
5 changed files with 70 additions and 109 deletions
|
@ -16,10 +16,7 @@ To run a real-world deployment, make sure you the following conditions are met:
|
||||||
- You have at least three machines with sufficient storage space available.
|
- You have at least three machines with sufficient storage space available.
|
||||||
|
|
||||||
- Each machine has a public IP address which is reachable by other machines.
|
- Each machine has a public IP address which is reachable by other machines.
|
||||||
Running behind a NAT is possible, but having several Garage nodes behind a single NAT
|
Running behind a NAT is likely to be possible but hasn't been tested for the latest version (TODO).
|
||||||
is slightly more involved as each will have to have a different RPC port number
|
|
||||||
(the local port number of a node must be the same as the port number exposed publicly
|
|
||||||
by the NAT).
|
|
||||||
|
|
||||||
- Ideally, each machine should have a SSD available in addition to the HDD you are dedicating
|
- Ideally, each machine should have a SSD available in addition to the HDD you are dedicating
|
||||||
to Garage. This will allow for faster access to metadata and has the potential
|
to Garage. This will allow for faster access to metadata and has the potential
|
||||||
|
@ -45,44 +42,23 @@ For our example, we will suppose the following infrastructure with IPv6 connecti
|
||||||
## Get a Docker image
|
## Get a Docker image
|
||||||
|
|
||||||
Our docker image is currently named `lxpz/garage_amd64` and is stored on the [Docker Hub](https://hub.docker.com/r/lxpz/garage_amd64/tags?page=1&ordering=last_updated).
|
Our docker image is currently named `lxpz/garage_amd64` and is stored on the [Docker Hub](https://hub.docker.com/r/lxpz/garage_amd64/tags?page=1&ordering=last_updated).
|
||||||
We encourage you to use a fixed tag (eg. `v0.3.0`) and not the `latest` tag.
|
We encourage you to use a fixed tag (eg. `v0.4.0`) and not the `latest` tag.
|
||||||
For this example, we will use the latest published version at the time of the writing which is `v0.3.0` but it's up to you
|
For this example, we will use the latest published version at the time of the writing which is `v0.4.0` but it's up to you
|
||||||
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/lxpz/garage_amd64/tags?page=1&ordering=last_updated).
|
to check [the most recent versions on the Docker Hub](https://hub.docker.com/r/lxpz/garage_amd64/tags?page=1&ordering=last_updated).
|
||||||
|
|
||||||
For example:
|
For example:
|
||||||
|
|
||||||
```
|
```
|
||||||
sudo docker pull lxpz/garage_amd64:v0.3.0
|
sudo docker pull lxpz/garage_amd64:v0.4.0
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
## Generating TLS certificates
|
|
||||||
|
|
||||||
You first need to generate TLS certificates to encrypt traffic between Garage nodes
|
|
||||||
(reffered to as RPC traffic).
|
|
||||||
|
|
||||||
To generate your TLS certificates, run on your machine:
|
|
||||||
|
|
||||||
```
|
|
||||||
wget https://git.deuxfleurs.fr/Deuxfleurs/garage/raw/branch/main/genkeys.sh
|
|
||||||
chmod +x genkeys.sh
|
|
||||||
./genkeys.sh
|
|
||||||
```
|
|
||||||
|
|
||||||
It will creates a folder named `pki/` containing the keys that you will used for the cluster.
|
|
||||||
These files will have to be copied to all of your cluster nodes, as explained below.
|
|
||||||
|
|
||||||
|
|
||||||
## Deploying and configuring Garage
|
## Deploying and configuring Garage
|
||||||
|
|
||||||
On each machine, we will have a similar setup,
|
On each machine, we will have a similar setup,
|
||||||
especially you must consider the following folders/files:
|
especially you must consider the following folders/files:
|
||||||
|
|
||||||
- `/etc/garage/garage.toml`: Garage daemon's configuration (see below)
|
- `/etc/garage.toml`: Garage daemon's configuration (see below)
|
||||||
|
|
||||||
- `/etc/garage/pki/`: Folder containing Garage certificates,
|
|
||||||
must be generated on your computer and copied on the servers.
|
|
||||||
Only the files `garage-ca.crt`, `garage.crt` and `garage.key` are necessary.
|
|
||||||
|
|
||||||
- `/var/lib/garage/meta/`: Folder containing Garage's metadata,
|
- `/var/lib/garage/meta/`: Folder containing Garage's metadata,
|
||||||
put this folder on a SSD if possible
|
put this folder on a SSD if possible
|
||||||
|
@ -91,7 +67,7 @@ especially you must consider the following folders/files:
|
||||||
this folder will be your main data storage and must be on a large storage (e.g. large HDD)
|
this folder will be your main data storage and must be on a large storage (e.g. large HDD)
|
||||||
|
|
||||||
|
|
||||||
A valid `/etc/garage/garage.toml` for our cluster would be:
|
A valid `/etc/garage/garage.toml` for our cluster would look as follows:
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
metadata_dir = "/var/lib/garage/meta"
|
metadata_dir = "/var/lib/garage/meta"
|
||||||
|
@ -100,19 +76,13 @@ data_dir = "/var/lib/garage/data"
|
||||||
replication_mode = "3"
|
replication_mode = "3"
|
||||||
|
|
||||||
rpc_bind_addr = "[::]:3901"
|
rpc_bind_addr = "[::]:3901"
|
||||||
|
rpc_public_addr = "<this node's public IP>:3901"
|
||||||
|
rpc_secret = "<RPC secret>"
|
||||||
|
|
||||||
bootstrap_peers = [
|
bootstrap_peers = [
|
||||||
"[fc00:1::1]:3901",
|
# We will fill this in later
|
||||||
"[fc00:1::2]:3901",
|
|
||||||
"[fc00:B::1]:3901",
|
|
||||||
"[fc00:F::1]:3901",
|
|
||||||
]
|
]
|
||||||
|
|
||||||
[rpc_tls]
|
|
||||||
ca_cert = "/etc/garage/pki/garage-ca.crt"
|
|
||||||
node_cert = "/etc/garage/pki/garage.crt"
|
|
||||||
node_key = "/etc/garage/pki/garage.key"
|
|
||||||
|
|
||||||
[s3_api]
|
[s3_api]
|
||||||
s3_region = "garage"
|
s3_region = "garage"
|
||||||
api_bind_addr = "[::]:3900"
|
api_bind_addr = "[::]:3900"
|
||||||
|
@ -123,7 +93,37 @@ root_domain = ".web.garage"
|
||||||
index = "index.html"
|
index = "index.html"
|
||||||
```
|
```
|
||||||
|
|
||||||
Please make sure to change `bootstrap_peers` to **your** IP addresses!
|
Check the following for your configuration files:
|
||||||
|
|
||||||
|
- Make sure `rpc_public_addr` contains the public IP address of the node you are configuring.
|
||||||
|
This parameter is optional but recommended: if your nodes have trouble communicating with
|
||||||
|
one another, consider adding it.
|
||||||
|
|
||||||
|
- Make sure `rpc_secret` is the same value on all nodes. It should be a 32-bytes hex-encoded secret key.
|
||||||
|
You can generate such a key with `openssl rand -hex 32`.
|
||||||
|
|
||||||
|
You will now have to run `garage node-id` on all nodes to generate node keys.
|
||||||
|
This will print keys as follows:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
Mercury$ garage node-id
|
||||||
|
563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901
|
||||||
|
|
||||||
|
Venus$ garage node-id
|
||||||
|
86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332[fc00:1::2]:3901
|
||||||
|
|
||||||
|
etc.
|
||||||
|
```
|
||||||
|
|
||||||
|
You can then add these nodes to the `bootstrap_peers` list of at least one of your nodes:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
bootstrap_peers = [
|
||||||
|
"563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d@[fc00:1::1]:3901",
|
||||||
|
"86f0f26ae4afbd59aaf9cfb059eefac844951efd5b8caeec0d53f4ed6c85f332[fc00:1::2]:3901",
|
||||||
|
...
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
Check the [configuration file reference documentation](../reference_manual/configuration.md)
|
Check the [configuration file reference documentation](../reference_manual/configuration.md)
|
||||||
to learn more about all available configuration options.
|
to learn more about all available configuration options.
|
||||||
|
@ -139,11 +139,10 @@ docker run \
|
||||||
--name garaged \
|
--name garaged \
|
||||||
--restart always \
|
--restart always \
|
||||||
--network host \
|
--network host \
|
||||||
-v /etc/garage/pki:/etc/garage/pki \
|
-v /etc/garage.toml:/etc/garage.toml \
|
||||||
-v /etc/garage/garage.toml:/garage/garage.toml \
|
|
||||||
-v /var/lib/garage/meta:/var/lib/garage/meta \
|
-v /var/lib/garage/meta:/var/lib/garage/meta \
|
||||||
-v /var/lib/garage/data:/var/lib/garage/data \
|
-v /var/lib/garage/data:/var/lib/garage/data \
|
||||||
lxpz/garage_amd64:v0.3.0
|
lxpz/garage_amd64:v0.4.0
|
||||||
```
|
```
|
||||||
|
|
||||||
It should be restarted automatically at each reboot.
|
It should be restarted automatically at each reboot.
|
||||||
|
@ -159,65 +158,27 @@ start again the command with a new version of Garage.
|
||||||
## Controling the daemon
|
## Controling the daemon
|
||||||
|
|
||||||
The `garage` binary has two purposes:
|
The `garage` binary has two purposes:
|
||||||
- it acts as a daemon when launched with `garage server ...`
|
- it acts as a daemon when launched with `garage server`
|
||||||
- it acts as a control tool for the daemon when launched with any other command
|
- it acts as a control tool for the daemon when launched with any other command
|
||||||
|
|
||||||
In this section, we will see how to use the `garage` binary as a control tool for the daemon we just started.
|
Ensure an appropriate `garage` binary (the same version as your Docker image) is available in your path.
|
||||||
You first need to get a shell having access to this binary. For instance, enter the Docker container with:
|
If your configuration file is at `/etc/garage.toml`, the `garage` binary should work with no further change.
|
||||||
|
|
||||||
```bash
|
You can test your `garage` CLI utility by running a simple command such as:
|
||||||
sudo docker exec -ti garaged bash
|
|
||||||
```
|
|
||||||
|
|
||||||
You will now have a shell where the Garage binary is available as `/garage/garage`
|
|
||||||
|
|
||||||
*You can also install the binary on your machine to remotely control the cluster.*
|
|
||||||
|
|
||||||
## Talk to the daemon and create an alias
|
|
||||||
|
|
||||||
`garage` requires 4 options to talk with the daemon:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
--ca-cert <ca-cert>
|
garage status
|
||||||
--client-cert <client-cert>
|
|
||||||
--client-key <client-key>
|
|
||||||
-h, --rpc-host <rpc-host>
|
|
||||||
```
|
|
||||||
|
|
||||||
The 3 first ones are certificates and keys needed by TLS, the last one is simply the address of Garage's RPC endpoint.
|
|
||||||
|
|
||||||
If you are invoking `garage` from a server node directly, you do not need to set `--rpc-host`
|
|
||||||
as the default value `127.0.0.1:3901` will allow it to contact Garage correctly.
|
|
||||||
|
|
||||||
To avoid typing the 3 first options each time we want to run a command,
|
|
||||||
you can use the following alias:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
alias garagectl='/garage/garage \
|
|
||||||
--ca-cert /etc/garage/pki/garage-ca.crt \
|
|
||||||
--client-cert /etc/garage/pki/garage.crt \
|
|
||||||
--client-key /etc/garage/pki/garage.key'
|
|
||||||
```
|
|
||||||
|
|
||||||
You can now use all of the commands presented in the [quick start guide](../quick_start/index.md),
|
|
||||||
simply replace occurences of `garage` by `garagectl`.
|
|
||||||
|
|
||||||
#### Test the alias
|
|
||||||
|
|
||||||
You can test your alias by running a simple command such as:
|
|
||||||
|
|
||||||
```
|
|
||||||
garagectl status
|
|
||||||
```
|
```
|
||||||
|
|
||||||
You should get something like that as result:
|
You should get something like that as result:
|
||||||
|
|
||||||
```
|
```
|
||||||
Healthy nodes:
|
==== HEALTHY NODES ====
|
||||||
8781c50c410a41b3… Mercury [fc00:1::1]:3901 UNCONFIGURED/REMOVED
|
ID Hostname Address Tag Zone Capacity
|
||||||
2a638ed6c775b69a… Venus [fc00:1::2]:3901 UNCONFIGURED/REMOVED
|
563e1ac825ee3323… Mercury [fc00:1::1]:3901 NO ROLE ASSIGNED
|
||||||
68143d720f20c89d… Earth [fc00:B::1]:3901 UNCONFIGURED/REMOVED
|
86f0f26ae4afbd59… Venus [fc00:1::2]:3901 NO ROLE ASSIGNED
|
||||||
212f7572f0c89da9… Mars [fc00:F::1]:3901 UNCONFIGURED/REMOVED
|
68143d720f20c89d… Earth [fc00:B::1]:3901 NO ROLE ASSIGNED
|
||||||
|
212f7572f0c89da9… Mars [fc00:F::1]:3901 NO ROLE ASSIGNED
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
@ -230,26 +191,26 @@ For our example, we will suppose we have the following infrastructure (Capacity,
|
||||||
|
|
||||||
| Location | Name | Disk Space | `Capacity` | `Identifier` | `Zone` |
|
| Location | Name | Disk Space | `Capacity` | `Identifier` | `Zone` |
|
||||||
|----------|---------|------------|------------|--------------|--------------|
|
|----------|---------|------------|------------|--------------|--------------|
|
||||||
| Paris | Mercury | 1 To | `2` | `8781c5` | `par1` |
|
| Paris | Mercury | 1 To | `2` | `563e` | `par1` |
|
||||||
| Paris | Venus | 2 To | `4` | `2a638e` | `par1` |
|
| Paris | Venus | 2 To | `4` | `86f0` | `par1` |
|
||||||
| London | Earth | 2 To | `4` | `68143d` | `lon1` |
|
| London | Earth | 2 To | `4` | `6814` | `lon1` |
|
||||||
| Brussels | Mars | 1.5 To | `3` | `212f75` | `bru1` |
|
| Brussels | Mars | 1.5 To | `3` | `212f` | `bru1` |
|
||||||
|
|
||||||
#### Node identifiers
|
#### Node identifiers
|
||||||
|
|
||||||
After its first launch, Garage generates a random and unique identifier for each nodes, such as:
|
After its first launch, Garage generates a random and unique identifier for each nodes, such as:
|
||||||
|
|
||||||
```
|
```
|
||||||
8781c50c410a41b363167e9d49cc468b6b9e4449b6577b64f15a249a149bdcbc
|
563e1ac825ee3323aa441e72c26d1030d6d4414aeb3dd25287c531e7fc2bc95d
|
||||||
```
|
```
|
||||||
|
|
||||||
Often a shorter form can be used, containing only the beginning of the identifier, like `8781c5`,
|
Often a shorter form can be used, containing only the beginning of the identifier, like `563e`,
|
||||||
which identifies the server "Mercury" located in "Paris" according to our previous table.
|
which identifies the server "Mercury" located in "Paris" according to our previous table.
|
||||||
|
|
||||||
The most simple way to match an identifier to a node is to run:
|
The most simple way to match an identifier to a node is to run:
|
||||||
|
|
||||||
```
|
```
|
||||||
garagectl status
|
garage status
|
||||||
```
|
```
|
||||||
|
|
||||||
It will display the IP address associated with each node;
|
It will display the IP address associated with each node;
|
||||||
|
@ -287,16 +248,16 @@ have 66% chance of being stored by Venus and 33% chance of being stored by Mercu
|
||||||
Given the information above, we will configure our cluster as follow:
|
Given the information above, we will configure our cluster as follow:
|
||||||
|
|
||||||
```
|
```
|
||||||
garagectl node configure -z par1 -c 2 -t mercury 8781c5
|
garage node configure -z par1 -c 2 -t mercury 563e
|
||||||
garagectl node configure -z par1 -c 4 -t venus 2a638e
|
garage node configure -z par1 -c 4 -t venus 86f0
|
||||||
garagectl node configure -z lon1 -c 4 -t earth 68143d
|
garage node configure -z lon1 -c 4 -t earth 6814
|
||||||
garagectl node configure -z bru1 -c 3 -t mars 212f75
|
garage node configure -z bru1 -c 3 -t mars 212f
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
## Using your Garage cluster
|
## Using your Garage cluster
|
||||||
|
|
||||||
Creating buckets and managing keys is done using the `garagectl` CLI,
|
Creating buckets and managing keys is done using the `garage` CLI,
|
||||||
and is covered in the [quick start guide](../quick_start/index.md).
|
and is covered in the [quick start guide](../quick_start/index.md).
|
||||||
Remember also that the CLI is self-documented thanks to the `--help` flag and
|
Remember also that the CLI is self-documented thanks to the `--help` flag and
|
||||||
the `help` subcommand (e.g. `garage help`, `garage key --help`).
|
the `help` subcommand (e.g. `garage help`, `garage key --help`).
|
||||||
|
|
|
@ -105,7 +105,7 @@ This should show something like this:
|
||||||
```
|
```
|
||||||
==== HEALTHY NODES ====
|
==== HEALTHY NODES ====
|
||||||
ID Hostname Address Tag Zone Capacity
|
ID Hostname Address Tag Zone Capacity
|
||||||
563e1ac825ee3323… linuxbox 127.0.0.1:3901 UNCONFIGURED/REMOVED
|
563e1ac825ee3323… linuxbox 127.0.0.1:3901 NO ROLE ASSIGNED
|
||||||
```
|
```
|
||||||
|
|
||||||
## Configuring your Garage node
|
## Configuring your Garage node
|
||||||
|
|
|
@ -76,7 +76,7 @@ for count in $(seq 1 3); do
|
||||||
done
|
done
|
||||||
|
|
||||||
RETRY=120
|
RETRY=120
|
||||||
until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
|
until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
|
||||||
(( RETRY-- ))
|
(( RETRY-- ))
|
||||||
if (( RETRY <= 0 )); then
|
if (( RETRY <= 0 )); then
|
||||||
echo -en "${MAIN_LABEL} Garage did not start"
|
echo -en "${MAIN_LABEL} Garage did not start"
|
||||||
|
|
|
@ -11,7 +11,7 @@ PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
||||||
|
|
||||||
sleep 5
|
sleep 5
|
||||||
RETRY=120
|
RETRY=120
|
||||||
until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
|
until garage -c /tmp/config.1.toml status 2>&1|grep -q HEALTHY ; do
|
||||||
(( RETRY-- ))
|
(( RETRY-- ))
|
||||||
if (( RETRY <= 0 )); then
|
if (( RETRY <= 0 )); then
|
||||||
echo "garage did not start in time, failing."
|
echo "garage did not start in time, failing."
|
||||||
|
@ -22,7 +22,7 @@ until garage -c /tmp/config.1.toml status 2>&1|grep -q Healthy ; do
|
||||||
done
|
done
|
||||||
|
|
||||||
garage -c /tmp/config.1.toml status \
|
garage -c /tmp/config.1.toml status \
|
||||||
| grep UNCONFIGURED \
|
| grep 'NO ROLE' \
|
||||||
| grep -Po '^[0-9a-f]+' \
|
| grep -Po '^[0-9a-f]+' \
|
||||||
| while read id; do
|
| while read id; do
|
||||||
garage -c /tmp/config.1.toml node configure -z dc1 -c 1 $id
|
garage -c /tmp/config.1.toml node configure -z dc1 -c 1 $id
|
||||||
|
|
|
@ -371,7 +371,7 @@ pub async fn cmd_status(rpc_cli: &Endpoint<SystemRpc, ()>, rpc_host: NodeID) ->
|
||||||
));
|
));
|
||||||
} else {
|
} else {
|
||||||
healthy_nodes.push(format!(
|
healthy_nodes.push(format!(
|
||||||
"{id:?}\t{h}\t{addr}\tUNCONFIGURED/REMOVED",
|
"{id:?}\t{h}\t{addr}\tNO ROLE ASSIGNED",
|
||||||
id = adv.id,
|
id = adv.id,
|
||||||
h = adv.status.hostname,
|
h = adv.status.hostname,
|
||||||
addr = adv.addr,
|
addr = adv.addr,
|
||||||
|
|
Loading…
Reference in a new issue