Garage v0.9 #473

Merged
lx merged 175 commits from next into main 2023-10-10 13:28:29 +00:00
48 changed files with 2918 additions and 2906 deletions
Showing only changes of commit 2e90e1c124 - Show all commits

1560
Cargo.lock generated

File diff suppressed because it is too large Load diff

3532
Cargo.nix

File diff suppressed because it is too large Load diff

View file

@ -18,14 +18,14 @@ default-members = ["src/garage"]
[workspace.dependencies] [workspace.dependencies]
format_table = { version = "0.1.1", path = "src/format-table" } format_table = { version = "0.1.1", path = "src/format-table" }
garage_api = { version = "0.8.2", path = "src/api" } garage_api = { version = "0.8.3", path = "src/api" }
garage_block = { version = "0.8.2", path = "src/block" } garage_block = { version = "0.8.3", path = "src/block" }
garage_db = { version = "0.8.2", path = "src/db", default-features = false } garage_db = { version = "0.8.3", path = "src/db", default-features = false }
garage_model = { version = "0.8.2", path = "src/model", default-features = false } garage_model = { version = "0.8.3", path = "src/model", default-features = false }
garage_rpc = { version = "0.8.2", path = "src/rpc" } garage_rpc = { version = "0.8.3", path = "src/rpc" }
garage_table = { version = "0.8.2", path = "src/table" } garage_table = { version = "0.8.3", path = "src/table" }
garage_util = { version = "0.8.2", path = "src/util" } garage_util = { version = "0.8.3", path = "src/util" }
garage_web = { version = "0.8.2", path = "src/web" } garage_web = { version = "0.8.3", path = "src/web" }
k2v-client = { version = "0.0.4", path = "src/k2v-client" } k2v-client = { version = "0.0.4", path = "src/k2v-client" }
[profile.dev] [profile.dev]

View file

@ -632,7 +632,7 @@ paths:
operationId: "UpdateBucket" operationId: "UpdateBucket"
summary: "Update a bucket" summary: "Update a bucket"
description: | description: |
All fields (`websiteAccess` and `quotas`) are optionnal. All fields (`websiteAccess` and `quotas`) are optional.
If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed. If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified. In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.

View file

@ -11,6 +11,7 @@ In this section, we cover the following web applications:
| [Peertube](#peertube) | ✅ | Supported with the website endpoint, proxifying private videos unsupported | | [Peertube](#peertube) | ✅ | Supported with the website endpoint, proxifying private videos unsupported |
| [Mastodon](#mastodon) | ✅ | Natively supported | | [Mastodon](#mastodon) | ✅ | Natively supported |
| [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` | | [Matrix](#matrix) | ✅ | Tested with `synapse-s3-storage-provider` |
| [ejabberd](#ejabberd) | ✅ | `mod_s3_upload` |
| [Pixelfed](#pixelfed) | ❓ | Not yet tested | | [Pixelfed](#pixelfed) | ❓ | Not yet tested |
| [Pleroma](#pleroma) | ❓ | Not yet tested | | [Pleroma](#pleroma) | ❓ | Not yet tested |
| [Lemmy](#lemmy) | ✅ | Supported with pict-rs | | [Lemmy](#lemmy) | ✅ | Supported with pict-rs |
@ -474,6 +475,52 @@ And add a new line. For example, to run it every 10 minutes:
*External link:* [matrix-media-repo Documentation > S3](https://docs.t2bot.io/matrix-media-repo/configuration/s3-datastore.html) *External link:* [matrix-media-repo Documentation > S3](https://docs.t2bot.io/matrix-media-repo/configuration/s3-datastore.html)
## ejabberd
ejabberd is an XMPP server implementation which, with the `mod_s3_upload`
module in the [ejabberd-contrib](https://github.com/processone/ejabberd-contrib)
repository, can be integrated to store chat media files in Garage.
For uploads, this module leverages presigned URLs - this allows XMPP clients to
directly send media to Garage. Receiving clients then retrieve this media
through the [static website](@/documentation/cookbook/exposing-websites.md)
functionality.
As the data itself is publicly accessible to someone with knowledge of the
object URL - users are recommended to use
[E2EE](@/documentation/cookbook/encryption.md) to protect this data-at-rest
from unauthorized access.
Install the module with:
```bash
ejabberdctl module_install mod_s3_upload
```
Create the required key and bucket with:
```bash
garage key new --name ejabberd
garage bucket create objects.xmpp-server.fr
garage bucket allow objects.xmpp-server.fr --read --write --key ejabberd
garage bucket website --allow objects.xmpp-server.fr
```
The module can then be configured with:
```
mod_s3_upload:
#bucket_url: https://objects.xmpp-server.fr.my-garage-instance.mydomain.tld
bucket_url: https://my-garage-instance.mydomain.tld/objects.xmpp-server.fr
access_key_id: GK...
access_key_secret: ...
region: garage
download_url: https://objects.xmpp-server.fr
```
Other configuration options can be found in the
[configuration YAML file](https://github.com/processone/ejabberd-contrib/blob/master/mod_s3_upload/conf/mod_s3_upload.yml).
## Pixelfed ## Pixelfed
[Pixelfed Technical Documentation > Configuration](https://docs.pixelfed.org/technical-documentation/env.html#filesystem) [Pixelfed Technical Documentation > Configuration](https://docs.pixelfed.org/technical-documentation/env.html#filesystem)
@ -539,7 +586,7 @@ secret_key = 'abcdef0123456789...'
``` ```
PICTRS__STORE__TYPE=object_storage PICTRS__STORE__TYPE=object_storage
PICTRS__STORE__ENDPOINT=http:/my-garage-instance.mydomain.tld:3900 PICTRS__STORE__ENDPOINT=http://my-garage-instance.mydomain.tld:3900
PICTRS__STORE__BUCKET_NAME=pictrs-data PICTRS__STORE__BUCKET_NAME=pictrs-data
PICTRS__STORE__REGION=garage PICTRS__STORE__REGION=garage
PICTRS__STORE__ACCESS_KEY=GK... PICTRS__STORE__ACCESS_KEY=GK...

View file

@ -105,6 +105,7 @@ restic restore 79766175 --target /var/lib/postgresql
Restic has way more features than the ones presented here. Restic has way more features than the ones presented here.
You can discover all of them by accessing its documentation from the link below. You can discover all of them by accessing its documentation from the link below.
Files on Android devices can also be backed up with [restic-android](https://github.com/lhns/restic-android).
*External links:* [Restic Documentation > Amazon S3](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#amazon-s3) *External links:* [Restic Documentation > Amazon S3](https://restic.readthedocs.io/en/stable/030_preparing_a_new_repo.html#amazon-s3)

View file

@ -7,10 +7,23 @@ Garage is also available in binary packages on:
## Alpine Linux ## Alpine Linux
If you use Alpine Linux, you can simply install the
[garage](https://pkgs.alpinelinux.org/packages?name=garage) package from the
Alpine Linux repositories (available since v3.17):
```bash ```bash
apk install garage apk add garage
``` ```
The default configuration file is installed to `/etc/garage.toml`. You can run
Garage using: `rc-service garage start`. If you don't specify `rpc_secret`, it
will be automatically replaced with a random string on the first start.
Please note that this package is built without Consul discovery, Kubernetes
discovery, OpenTelemetry exporter, and K2V features (K2V will be enabled once
it's stable).
## Arch Linux ## Arch Linux
Garage is available in the [AUR](https://aur.archlinux.org/packages/garage). Garage is available in the [AUR](https://aur.archlinux.org/packages/garage).

View file

@ -104,5 +104,13 @@ Implementations are very specific to the various applications. Examples:
in Matrix are probably encrypted using symmetric encryption, with a key that is in Matrix are probably encrypted using symmetric encryption, with a key that is
distributed in the end-to-end encrypted message that contains the link to the object. distributed in the end-to-end encrypted message that contains the link to the object.
- XMPP: clients normally support either OMEMO / OpenPGP for the E2EE of user
messages. Media files are encrypted per
[XEP-0454](https://xmpp.org/extensions/xep-0454.html).
- Aerogramme: use the user's password as a key to decrypt data in the user's bucket - Aerogramme: use the user's password as a key to decrypt data in the user's bucket
- Cyberduck: comes with support for
[Cryptomator](https://docs.cyberduck.io/cryptomator/) which allows users to
create client-side vaults to encrypt files in before they are uploaded to a
cloud storage endpoint.

View file

@ -378,6 +378,47 @@ admin.garage.tld {
But at the same time, the `reverse_proxy` is very flexible. But at the same time, the `reverse_proxy` is very flexible.
For a production deployment, you should [read its documentation](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy) as it supports features like DNS discovery of upstreams, load balancing with checks, streaming parameters, etc. For a production deployment, you should [read its documentation](https://caddyserver.com/docs/caddyfile/directives/reverse_proxy) as it supports features like DNS discovery of upstreams, load balancing with checks, streaming parameters, etc.
### Caching
Caddy can compiled with a
[cache plugin](https://github.com/caddyserver/cache-handler) which can be used
to provide a hot-cache at the webserver-level for static websites hosted by
Garage.
This can be configured as follows:
```caddy
# Caddy global configuration section
{
# Bare minimum configuration to enable cache.
order cache before rewrite
cache
#cache
# allowed_http_verbs GET
# default_cache_control public
# ttl 8h
#}
}
# Site specific section
https:// {
cache
#cache {
# timeout {
# backend 30s
# }
#}
reverse_proxy ...
}
```
Caching is a complicated subject, and the reader is encouraged to study the
available options provided by the plugin.
### On-demand TLS ### On-demand TLS
Caddy supports a technique called Caddy supports a technique called
@ -428,3 +469,6 @@ https:// {
reverse_proxy localhost:3902 192.168.1.2:3902 example.tld:3902 reverse_proxy localhost:3902 192.168.1.2:3902 example.tld:3902
} }
``` ```
More information on how this endpoint is implemented in Garage is available
in the [Admin API Reference](@/documentation/reference-manual/admin-api.md) page.

View file

@ -4,7 +4,7 @@ weight = 30
+++ +++
To ensure the best durability of your data and to fix any inconsistencies that may To ensure the best durability of your data and to fix any inconsistencies that may
pop up in a distributed system, Garage provides a serires of repair operations. pop up in a distributed system, Garage provides a series of repair operations.
This guide will explain the meaning of each of them and when they should be applied. This guide will explain the meaning of each of them and when they should be applied.
@ -26,8 +26,11 @@ their content is correct, by verifying their hash. Any block found to be corrupt
(e.g. by bitrot or by an accidental manipulation of the datastore) will be (e.g. by bitrot or by an accidental manipulation of the datastore) will be
restored from another node that holds a valid copy. restored from another node that holds a valid copy.
A scrub is run automatically by Garage every 30 days. It can also be launched Scrubs are automatically scheduled by Garage to run every 25-35 days (the
manually using `garage repair scrub start`. actual time is randomized to spread load across nodes). The next scheduled run
can be viewed with `garage worker get`.
A scrub can also be launched manually using `garage repair scrub start`.
To view the status of an ongoing scrub, first find the task ID of the scrub worker To view the status of an ongoing scrub, first find the task ID of the scrub worker
using `garage worker list`. Then, run `garage worker info <scrub_task_id>` to using `garage worker list`. Then, run `garage worker info <scrub_task_id>` to
@ -79,7 +82,7 @@ To help make the difference between cases 1 and cases 2 and 3, you may use the
`garage block info` command to see which objects hold a reference to each block. `garage block info` command to see which objects hold a reference to each block.
In the second case (transient errors), Garage will try to fetch the block again In the second case (transient errors), Garage will try to fetch the block again
after a certain time, so the error should disappear natuarlly. You can also after a certain time, so the error should disappear naturally. You can also
request Garage to try to fetch the block immediately using `garage block retry-now` request Garage to try to fetch the block immediately using `garage block retry-now`
if you have fixed the transient issue. if you have fixed the transient issue.

View file

@ -35,6 +35,9 @@ Place this binary somewhere in your `$PATH` so that you can invoke the `garage`
command directly (for instance you can copy the binary in `/usr/local/bin` command directly (for instance you can copy the binary in `/usr/local/bin`
or in `~/.local/bin`). or in `~/.local/bin`).
You may also check whether your distribution already includes a
[binary package for Garage](@/documentation/cookbook/binary-packages.md).
If a binary of the last version is not available for your architecture, If a binary of the last version is not available for your architecture,
or if you want a build customized for your system, or if you want a build customized for your system,
you can [build Garage from source](@/documentation/cookbook/from-source.md). you can [build Garage from source](@/documentation/cookbook/from-source.md).

View file

@ -39,11 +39,95 @@ Authorization: Bearer <token>
## Administration API endpoints ## Administration API endpoints
### Metrics-related endpoints ### Metrics `GET /metrics`
#### Metrics `GET /metrics`
Returns internal Garage metrics in Prometheus format. Returns internal Garage metrics in Prometheus format.
The metrics are directly documented when returned by the API.
**Example:**
```
$ curl -i http://localhost:3903/metrics
HTTP/1.1 200 OK
content-type: text/plain; version=0.0.4
content-length: 12145
date: Tue, 08 Aug 2023 07:25:05 GMT
# HELP api_admin_error_counter Number of API calls to the various Admin API endpoints that resulted in errors
# TYPE api_admin_error_counter counter
api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="400"} 1
api_admin_error_counter{api_endpoint="CheckWebsiteEnabled",status_code="404"} 3
# HELP api_admin_request_counter Number of API calls to the various Admin API endpoints
# TYPE api_admin_request_counter counter
api_admin_request_counter{api_endpoint="CheckWebsiteEnabled"} 7
api_admin_request_counter{api_endpoint="Health"} 3
# HELP api_admin_request_duration Duration of API calls to the various Admin API endpoints
...
```
### Health `GET /health`
Returns `200 OK` if enough nodes are up to have a quorum (ie. serve requests),
otherwise returns `503 Service Unavailable`.
**Example:**
```
$ curl -i http://localhost:3903/health
HTTP/1.1 200 OK
content-type: text/plain
content-length: 102
date: Tue, 08 Aug 2023 07:22:38 GMT
Garage is fully operational
Consult the full health check API endpoint at /v0/health for more details
```
### On-demand TLS `GET /check`
To prevent abuses for on-demand TLS, Caddy developpers have specified an endpoint that can be queried by the reverse proxy
to know if a given domain is allowed to get a certificate. Garage implements this endpoints to tell if a given domain is handled by Garage or is garbage.
Garage responds with the following logic:
- If the domain matches the pattern `<bucket-name>.<s3_api.root_domain>`, returns 200 OK
- If the domain matches the pattern `<bucket-name>.<s3_web.root_domain>` and website is configured for `<bucket>`, returns 200 OK
- If the domain matches the pattern `<bucket-name>` and website is configured for `<bucket>`, returns 200 OK
- Otherwise, returns 404 Not Found, 400 Bad Request or 5xx requests.
*Note 1: because in the path-style URL mode, there is only one domain that is not known by Garage, hence it is not supported by this API endpoint.
You must manually declare the domain in your reverse-proxy. Idem for K2V.*
*Note 2: buckets in a user's namespace are not supported yet by this endpoint. This is a limitation of this endpoint currently.*
**Example:** Suppose a Garage instance configured with `s3_api.root_domain = .s3.garage.localhost` and `s3_web.root_domain = .web.garage.localhost`.
With a private `media` bucket (name in the global namespace, website is disabled), the endpoint will feature the following behavior:
```
$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=media.s3.garage.localhost
200
$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=media
400
$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=media.web.garage.localhost
400
```
With a public `example.com` bucket (name in the global namespace, website is activated), the endpoint will feature the following behavior:
```
$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=example.com.s3.garage.localhost
200
$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=example.com
200
$ curl -so /dev/null -w "%{http_code}" http://localhost:3903/check?domain=example.com.web.garage.localhost
200
```
**References:**
- [Using On-Demand TLS](https://caddyserver.com/docs/automatic-https#using-on-demand-tls)
- [Add option for a backend check to approve use of on-demand TLS](https://github.com/caddyserver/caddy/pull/1939)
- [Serving tens of thousands of domains over HTTPS with Caddy](https://caddy.community/t/serving-tens-of-thousands-of-domains-over-https-with-caddy/11179)
### Cluster operations ### Cluster operations

View file

@ -454,7 +454,7 @@ message that redirects the client to the correct region.
### `root_domain` {#root_domain} ### `root_domain` {#root_domain}
The optionnal suffix to access bucket using vhost-style in addition to path-style request. The optional suffix to access bucket using vhost-style in addition to path-style request.
Note path-style requests are always enabled, whether or not vhost-style is configured. Note path-style requests are always enabled, whether or not vhost-style is configured.
Configuring vhost-style S3 required a wildcard DNS entry, and possibly a wildcard TLS certificate, Configuring vhost-style S3 required a wildcard DNS entry, and possibly a wildcard TLS certificate,
but might be required by softwares not supporting path-style requests. but might be required by softwares not supporting path-style requests.
@ -477,7 +477,7 @@ This endpoint does not suport TLS: a reverse proxy should be used to provide it.
### `root_domain` ### `root_domain`
The optionnal suffix appended to bucket names for the corresponding HTTP Host. The optional suffix appended to bucket names for the corresponding HTTP Host.
For instance, if `root_domain` is `web.garage.eu`, a bucket called `deuxfleurs.fr` For instance, if `root_domain` is `web.garage.eu`, a bucket called `deuxfleurs.fr`
will be accessible either with hostname `deuxfleurs.fr.web.garage.eu` will be accessible either with hostname `deuxfleurs.fr.web.garage.eu`

View file

@ -3,7 +3,7 @@ title = "K2V"
weight = 100 weight = 100
+++ +++
Starting with version 0.7.2, Garage introduces an optionnal feature, K2V, Starting with version 0.7.2, Garage introduces an optional feature, K2V,
which is an alternative storage API designed to help efficiently store which is an alternative storage API designed to help efficiently store
many small values in buckets (in opposition to S3 which is more designed many small values in buckets (in opposition to S3 which is more designed
to store large blobs). to store large blobs).

View file

@ -509,7 +509,7 @@ Request body format:
} }
``` ```
All fields (`name`, `allow` and `deny`) are optionnal. All fields (`name`, `allow` and `deny`) are optional.
If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed. If they are present, the corresponding modifications are applied to the key, otherwise nothing is changed.
The possible flags in `allow` and `deny` are: `createBucket`. The possible flags in `allow` and `deny` are: `createBucket`.
@ -668,7 +668,7 @@ Request body format:
} }
``` ```
All fields (`websiteAccess` and `quotas`) are optionnal. All fields (`websiteAccess` and `quotas`) are optional.
If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed. If they are present, the corresponding modifications are applied to the bucket, otherwise nothing is changed.
In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified. In `websiteAccess`: if `enabled` is `true`, `indexDocument` must be specified.

View file

@ -2,7 +2,9 @@
"nodes": { "nodes": {
"cargo2nix": { "cargo2nix": {
"inputs": { "inputs": {
"flake-compat": "flake-compat", "flake-compat": [
"flake-compat"
],
"flake-utils": "flake-utils", "flake-utils": "flake-utils",
"nixpkgs": [ "nixpkgs": [
"nixpkgs" "nixpkgs"
@ -25,17 +27,16 @@
} }
}, },
"flake-compat": { "flake-compat": {
"flake": false,
"locked": { "locked": {
"lastModified": 1650374568, "lastModified": 1688025799,
"narHash": "sha256-Z+s0J8/r907g149rllvwhb4pKi8Wam5ij0st8PwAh+E=", "narHash": "sha256-ktpB4dRtnksm9F5WawoIkEneh1nrEvuxb5lJFt1iOyw=",
"owner": "edolstra", "owner": "nix-community",
"repo": "flake-compat", "repo": "flake-compat",
"rev": "b4a34015c698c7793d592d66adbab377907a2be8", "rev": "8bf105319d44f6b9f0d764efa4fdef9f1cc9ba1c",
"type": "github" "type": "github"
}, },
"original": { "original": {
"owner": "edolstra", "owner": "nix-community",
"repo": "flake-compat", "repo": "flake-compat",
"type": "github" "type": "github"
} }
@ -111,10 +112,7 @@
"root": { "root": {
"inputs": { "inputs": {
"cargo2nix": "cargo2nix", "cargo2nix": "cargo2nix",
"flake-compat": [ "flake-compat": "flake-compat",
"cargo2nix",
"flake-compat"
],
"flake-utils": [ "flake-utils": [
"cargo2nix", "cargo2nix",
"flake-utils" "flake-utils"

View file

@ -6,6 +6,8 @@
inputs.nixpkgs.url = inputs.nixpkgs.url =
"github:NixOS/nixpkgs/94517a501434a627c5d9e72ac6e7f26174b978d3"; "github:NixOS/nixpkgs/94517a501434a627c5d9e72ac6e7f26174b978d3";
inputs.flake-compat.url = "github:nix-community/flake-compat";
inputs.cargo2nix = { inputs.cargo2nix = {
# As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection # As of 2022-10-18: two small patches over unstable branch, one for clippy and one to fix feature detection
url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36"; url = "github:Alexis211/cargo2nix/a7a61179b66054904ef6a195d8da736eaaa06c36";
@ -20,16 +22,17 @@
"github:oxalica/rust-overlay/74f1a64dd28faeeb85ef081f32cad2989850322c"; "github:oxalica/rust-overlay/74f1a64dd28faeeb85ef081f32cad2989850322c";
inputs.nixpkgs.follows = "nixpkgs"; inputs.nixpkgs.follows = "nixpkgs";
inputs.flake-compat.follows = "flake-compat";
}; };
inputs.flake-utils.follows = "cargo2nix/flake-utils"; inputs.flake-utils.follows = "cargo2nix/flake-utils";
inputs.flake-compat.follows = "cargo2nix/flake-compat";
outputs = { self, nixpkgs, cargo2nix, flake-utils, ... }: outputs = { self, nixpkgs, cargo2nix, flake-utils, ... }:
let let
git_version = self.lastModifiedDate; git_version = self.lastModifiedDate;
compile = import ./nix/compile.nix; compile = import ./nix/compile.nix;
in flake-utils.lib.eachDefaultSystem (system: in
flake-utils.lib.eachDefaultSystem (system:
let pkgs = nixpkgs.legacyPackages.${system}; let pkgs = nixpkgs.legacyPackages.${system};
in { in {
packages = { packages = {

View file

@ -1,14 +1,17 @@
let let
lock = builtins.fromJSON (builtins.readFile ../flake.lock); lock = builtins.fromJSON (builtins.readFile ../flake.lock);
flakeCompatRev = lock.nodes.flake-compat.locked.rev;
flakeCompat = fetchTarball { inherit (lock.nodes.flake-compat.locked) owner repo rev narHash;
url =
"https://github.com/edolstra/flake-compat/archive/${flakeCompatRev}.tar.gz"; flake-compat = fetchTarball {
sha256 = lock.nodes.flake-compat.locked.narHash; url = "https://github.com/${owner}/${repo}/archive/${rev}.tar.gz";
sha256 = narHash;
}; };
flake = ((import flakeCompat) { src = ../.; }).defaultNix;
in rec { flake = (import flake-compat { system = builtins.currentSystem; src = ../.; });
pkgsSrc = flake.inputs.nixpkgs; in
cargo2nix = flake.inputs.cargo2nix; rec {
pkgsSrc = flake.defaultNix.inputs.nixpkgs;
cargo2nix = flake.defaultNix.inputs.cargo2nix;
cargo2nixOverlay = cargo2nix.overlays.default; cargo2nixOverlay = cargo2nix.overlays.default;
} }

View file

@ -21,4 +21,4 @@ version: 0.4.1
# incremented each time you make changes to the application. Versions are not expected to # incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using. # follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes. # It is recommended to use it with quotes.
appVersion: "v0.8.2" appVersion: "v0.8.3"

View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_api" name = "garage_api"
version = "0.8.2" version = "0.8.3"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -28,7 +28,7 @@ crypto-common = "0.1"
err-derive = "0.3" err-derive = "0.3"
hex = "0.4" hex = "0.4"
hmac = "0.12" hmac = "0.12"
idna = "0.3" idna = "0.4"
tracing = "0.1" tracing = "0.1"
md-5 = "0.10" md-5 = "0.10"
nom = "7.1" nom = "7.1"
@ -47,7 +47,7 @@ http-range = "0.1"
hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] } hyper = { version = "0.14", features = ["server", "http1", "runtime", "tcp", "stream"] }
multer = "2.0" multer = "2.0"
percent-encoding = "2.1.0" percent-encoding = "2.1.0"
roxmltree = "0.14" roxmltree = "0.18"
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_bytes = "0.11" serde_bytes = "0.11"
serde_json = "1.0" serde_json = "1.0"

View file

@ -26,6 +26,7 @@ use crate::admin::cluster::*;
use crate::admin::error::*; use crate::admin::error::*;
use crate::admin::key::*; use crate::admin::key::*;
use crate::admin::router::{Authorization, Endpoint}; use crate::admin::router::{Authorization, Endpoint};
use crate::helpers::host_to_bucket;
pub struct AdminApiServer { pub struct AdminApiServer {
garage: Arc<Garage>, garage: Arc<Garage>,
@ -78,10 +79,7 @@ impl AdminApiServer {
.body(Body::empty())?) .body(Body::empty())?)
} }
async fn handle_check_website_enabled( async fn handle_check_domain(&self, req: Request<Body>) -> Result<Response<Body>, Error> {
&self,
req: Request<Body>,
) -> Result<Response<Body>, Error> {
let query_params: HashMap<String, String> = req let query_params: HashMap<String, String> = req
.uri() .uri()
.query() .query()
@ -102,12 +100,56 @@ impl AdminApiServer {
.get("domain") .get("domain")
.ok_or_internal_error("Could not parse domain query string")?; .ok_or_internal_error("Could not parse domain query string")?;
let bucket_id = self if self.check_domain(domain).await? {
Ok(Response::builder()
.status(StatusCode::OK)
.body(Body::from(format!(
"Domain '{domain}' is managed by Garage"
)))?)
} else {
Err(Error::bad_request(format!(
"Domain '{domain}' is not managed by Garage"
)))
}
}
async fn check_domain(&self, domain: &str) -> Result<bool, Error> {
// Resolve bucket from domain name, inferring if the website must be activated for the
// domain to be valid.
let (bucket_name, must_check_website) = if let Some(bname) = self
.garage
.config
.s3_api
.root_domain
.as_ref()
.and_then(|rd| host_to_bucket(domain, rd))
{
(bname.to_string(), false)
} else if let Some(bname) = self
.garage
.config
.s3_web
.as_ref()
.and_then(|sw| host_to_bucket(domain, sw.root_domain.as_str()))
{
(bname.to_string(), true)
} else {
(domain.to_string(), true)
};
let bucket_id = match self
.garage .garage
.bucket_helper() .bucket_helper()
.resolve_global_bucket_name(domain) .resolve_global_bucket_name(&bucket_name)
.await? .await?
.ok_or(HelperError::NoSuchBucket(domain.to_string()))?; {
Some(bucket_id) => bucket_id,
None => return Ok(false),
};
if !must_check_website {
return Ok(true);
}
let bucket = self let bucket = self
.garage .garage
@ -119,16 +161,8 @@ impl AdminApiServer {
let bucket_website_config = bucket_state.website_config.get(); let bucket_website_config = bucket_state.website_config.get();
match bucket_website_config { match bucket_website_config {
Some(_v) => { Some(_v) => Ok(true),
Ok(Response::builder() None => Ok(false),
.status(StatusCode::OK)
.body(Body::from(format!(
"Bucket '{domain}' is authorized for website hosting"
)))?)
}
None => Err(Error::bad_request(format!(
"Bucket '{domain}' is not authorized for website hosting"
))),
} }
} }
@ -229,7 +263,7 @@ impl ApiHandler for AdminApiServer {
match endpoint { match endpoint {
Endpoint::Options => self.handle_options(&req), Endpoint::Options => self.handle_options(&req),
Endpoint::CheckWebsiteEnabled => self.handle_check_website_enabled(req).await, Endpoint::CheckDomain => self.handle_check_domain(req).await,
Endpoint::Health => self.handle_health(), Endpoint::Health => self.handle_health(),
Endpoint::Metrics => self.handle_metrics(), Endpoint::Metrics => self.handle_metrics(),
Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await, Endpoint::GetClusterStatus => handle_get_cluster_status(&self.garage).await,

View file

@ -17,7 +17,7 @@ router_match! {@func
#[derive(Debug, Clone, PartialEq, Eq)] #[derive(Debug, Clone, PartialEq, Eq)]
pub enum Endpoint { pub enum Endpoint {
Options, Options,
CheckWebsiteEnabled, CheckDomain,
Health, Health,
Metrics, Metrics,
GetClusterStatus, GetClusterStatus,
@ -93,7 +93,7 @@ impl Endpoint {
let res = router_match!(@gen_path_parser (req.method(), path, query) [ let res = router_match!(@gen_path_parser (req.method(), path, query) [
OPTIONS _ => Options, OPTIONS _ => Options,
GET "/check" => CheckWebsiteEnabled, GET "/check" => CheckDomain,
GET "/health" => Health, GET "/health" => Health,
GET "/metrics" => Metrics, GET "/metrics" => Metrics,
GET "/v1/status" => GetClusterStatus, GET "/v1/status" => GetClusterStatus,
@ -139,7 +139,7 @@ impl Endpoint {
pub fn authorization_type(&self) -> Authorization { pub fn authorization_type(&self) -> Authorization {
match self { match self {
Self::Health => Authorization::None, Self::Health => Authorization::None,
Self::CheckWebsiteEnabled => Authorization::None, Self::CheckDomain => Authorization::None,
Self::Metrics => Authorization::MetricsToken, Self::Metrics => Authorization::MetricsToken,
_ => Authorization::AdminToken, _ => Authorization::AdminToken,
} }

View file

@ -30,7 +30,7 @@ pub async fn handle_post_object(
.get(header::CONTENT_TYPE) .get(header::CONTENT_TYPE)
.and_then(|ct| ct.to_str().ok()) .and_then(|ct| ct.to_str().ok())
.and_then(|ct| multer::parse_boundary(ct).ok()) .and_then(|ct| multer::parse_boundary(ct).ok())
.ok_or_bad_request("Counld not get multipart boundary")?; .ok_or_bad_request("Could not get multipart boundary")?;
// 16k seems plenty for a header. 5G is the max size of a single part, so it seems reasonable // 16k seems plenty for a header. 5G is the max size of a single part, so it seems reasonable
// for a PostObject // for a PostObject
@ -64,15 +64,13 @@ pub async fn handle_post_object(
"tag" => (/* tag need to be reencoded, but we don't support them yet anyway */), "tag" => (/* tag need to be reencoded, but we don't support them yet anyway */),
"acl" => { "acl" => {
if params.insert("x-amz-acl", content).is_some() { if params.insert("x-amz-acl", content).is_some() {
return Err(Error::bad_request( return Err(Error::bad_request("Field 'acl' provided more than once"));
"Field 'acl' provided more than one time",
));
} }
} }
_ => { _ => {
if params.insert(&name, content).is_some() { if params.insert(&name, content).is_some() {
return Err(Error::bad_request(format!( return Err(Error::bad_request(format!(
"Field '{}' provided more than one time", "Field '{}' provided more than once",
name name
))); )));
} }
@ -149,7 +147,7 @@ pub async fn handle_post_object(
.ok_or_bad_request("Invalid expiration date")? .ok_or_bad_request("Invalid expiration date")?
.into(); .into();
if Utc::now() - expiration > Duration::zero() { if Utc::now() - expiration > Duration::zero() {
return Err(Error::bad_request("Expiration date is in the paste")); return Err(Error::bad_request("Expiration date is in the past"));
} }
let mut conditions = decoded_policy.into_conditions()?; let mut conditions = decoded_policy.into_conditions()?;
@ -330,7 +328,7 @@ impl Policy {
if map.len() != 1 { if map.len() != 1 {
return Err(Error::bad_request("Invalid policy item")); return Err(Error::bad_request("Invalid policy item"));
} }
let (mut k, v) = map.into_iter().next().expect("size was verified"); let (mut k, v) = map.into_iter().next().expect("Size could not be verified");
k.make_ascii_lowercase(); k.make_ascii_lowercase();
params.entry(k).or_default().push(Operation::Equal(v)); params.entry(k).or_default().push(Operation::Equal(v));
} }

View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_block" name = "garage_block"
version = "0.8.2" version = "0.8.3"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -28,7 +28,7 @@ hex = "0.4"
tracing = "0.1" tracing = "0.1"
rand = "0.8" rand = "0.8"
async-compression = { version = "0.3", features = ["tokio", "zstd"] } async-compression = { version = "0.4", features = ["tokio", "zstd"] }
zstd = { version = "0.12", default-features = false } zstd = { version = "0.12", default-features = false }
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }

View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_db" name = "garage_db"
version = "0.8.2" version = "0.8.3"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -22,19 +22,19 @@ hexdump = "0.1"
tracing = "0.1" tracing = "0.1"
heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true } heed = { version = "0.11", default-features = false, features = ["lmdb"], optional = true }
rusqlite = { version = "0.28", optional = true } rusqlite = { version = "0.29", optional = true }
sled = { version = "0.34", optional = true } sled = { version = "0.34", optional = true }
# cli deps # cli deps
clap = { version = "4.1", optional = true, features = ["derive", "env"] } clap = { version = "4.1", optional = true, features = ["derive", "env"] }
pretty_env_logger = { version = "0.4", optional = true } pretty_env_logger = { version = "0.5", optional = true }
[dev-dependencies] [dev-dependencies]
mktemp = "0.5" mktemp = "0.5"
[features] [features]
default = [ "sled", "lmdb", "sqlite" ] default = [ "sled", "lmdb", "sqlite" ]
bundled-libs = [ "rusqlite/bundled" ] bundled-libs = [ "rusqlite?/bundled" ]
cli = ["clap", "pretty_env_logger"] cli = ["clap", "pretty_env_logger"]
lmdb = [ "heed" ] lmdb = [ "heed" ]
sqlite = [ "rusqlite" ] sqlite = [ "rusqlite" ]

View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage" name = "garage"
version = "0.8.2" version = "0.8.3"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -33,7 +33,7 @@ garage_web.workspace = true
backtrace = "0.3" backtrace = "0.3"
bytes = "1.0" bytes = "1.0"
bytesize = "1.1" bytesize = "1.2"
timeago = { version = "0.4", default-features = false } timeago = { version = "0.4", default-features = false }
parse_duration = "2.1" parse_duration = "2.1"
hex = "0.4" hex = "0.4"
@ -61,7 +61,8 @@ opentelemetry-otlp = { version = "0.10", optional = true }
prometheus = { version = "0.13", optional = true } prometheus = { version = "0.13", optional = true }
[dev-dependencies] [dev-dependencies]
aws-sdk-s3 = "0.19" aws-config = "0.55.2"
aws-sdk-s3 = "0.28"
chrono = "0.4" chrono = "0.4"
http = "0.2" http = "0.2"
hmac = "0.12" hmac = "0.12"

View file

@ -227,7 +227,7 @@ pub struct WebsiteOpt {
#[structopt(short = "i", long = "index-document", default_value = "index.html")] #[structopt(short = "i", long = "index-document", default_value = "index.html")]
pub index_document: String, pub index_document: String,
/// Error document: the optionnal document returned when an error occurs /// Error document: the optional document returned when an error occurs
#[structopt(short = "e", long = "error-document")] #[structopt(short = "e", long = "error-document")]
pub error_document: Option<String>, pub error_document: Option<String>,
} }

View file

@ -1,7 +1,6 @@
use crate::common; use crate::common;
use crate::common::ext::CommandExt; use crate::common::ext::CommandExt;
use aws_sdk_s3::model::BucketLocationConstraint; use aws_sdk_s3::operation::delete_bucket::DeleteBucketOutput;
use aws_sdk_s3::output::DeleteBucketOutput;
#[tokio::test] #[tokio::test]
async fn test_bucket_all() { async fn test_bucket_all() {
@ -63,10 +62,7 @@ async fn test_bucket_all() {
.await .await
.unwrap(); .unwrap();
match r.location_constraint.unwrap() { assert_eq!(r.location_constraint.unwrap().as_str(), "garage-integ-test");
BucketLocationConstraint::Unknown(v) if v.as_str() == "garage-integ-test" => (),
_ => unreachable!("wrong region"),
}
} }
{ {
// (Stub) check GetVersioning // (Stub) check GetVersioning

View file

@ -1,15 +1,16 @@
use aws_sdk_s3::{Client, Config, Credentials, Endpoint}; use aws_sdk_s3::config::Credentials;
use aws_sdk_s3::{Client, Config};
use super::garage::{Instance, Key}; use super::garage::Key;
use crate::common::garage::DEFAULT_PORT;
pub fn build_client(instance: &Instance, key: &Key) -> Client { pub fn build_client(key: &Key) -> Client {
let credentials = Credentials::new(&key.id, &key.secret, None, None, "garage-integ-test"); let credentials = Credentials::new(&key.id, &key.secret, None, None, "garage-integ-test");
let endpoint = Endpoint::immutable(instance.s3_uri());
let config = Config::builder() let config = Config::builder()
.endpoint_url(format!("http://127.0.0.1:{}", DEFAULT_PORT))
.region(super::REGION) .region(super::REGION)
.credentials_provider(credentials) .credentials_provider(credentials)
.endpoint_resolver(endpoint)
.build(); .build();
Client::from_conf(config) Client::from_conf(config)

View file

@ -1,5 +1,7 @@
use aws_sdk_s3::{Client, Region}; use aws_sdk_s3::config::Region;
use aws_sdk_s3::Client;
use ext::*; use ext::*;
#[cfg(feature = "k2v")]
use k2v_client::K2vClient; use k2v_client::K2vClient;
#[macro_use] #[macro_use]
@ -20,6 +22,7 @@ pub struct Context {
pub key: garage::Key, pub key: garage::Key,
pub client: Client, pub client: Client,
pub custom_request: CustomRequester, pub custom_request: CustomRequester,
#[cfg(feature = "k2v")]
pub k2v: K2VContext, pub k2v: K2VContext,
} }
@ -32,8 +35,9 @@ impl Context {
fn new() -> Self { fn new() -> Self {
let garage = garage::instance(); let garage = garage::instance();
let key = garage.key(None); let key = garage.key(None);
let client = client::build_client(garage, &key); let client = client::build_client(&key);
let custom_request = CustomRequester::new_s3(garage, &key); let custom_request = CustomRequester::new_s3(garage, &key);
#[cfg(feature = "k2v")]
let k2v_request = CustomRequester::new_k2v(garage, &key); let k2v_request = CustomRequester::new_k2v(garage, &key);
Context { Context {
@ -41,6 +45,7 @@ impl Context {
client, client,
key, key,
custom_request, custom_request,
#[cfg(feature = "k2v")]
k2v: K2VContext { k2v: K2VContext {
request: k2v_request, request: k2v_request,
}, },
@ -71,6 +76,7 @@ impl Context {
} }
/// Build a K2vClient for a given bucket /// Build a K2vClient for a given bucket
#[cfg(feature = "k2v")]
pub fn k2v_client(&self, bucket: &str) -> K2vClient { pub fn k2v_client(&self, bucket: &str) -> K2vClient {
let config = k2v_client::K2vClientConfig { let config = k2v_client::K2vClientConfig {
region: REGION.to_string(), region: REGION.to_string(),

View file

@ -6,7 +6,7 @@ use assert_json_diff::assert_json_eq;
use base64::prelude::*; use base64::prelude::*;
use serde_json::json; use serde_json::json;
use super::json_body; use crate::json_body;
use hyper::{Method, StatusCode}; use hyper::{Method, StatusCode};
#[tokio::test] #[tokio::test]

View file

@ -6,7 +6,7 @@ use assert_json_diff::assert_json_eq;
use base64::prelude::*; use base64::prelude::*;
use serde_json::json; use serde_json::json;
use super::json_body; use crate::json_body;
use hyper::{Method, StatusCode}; use hyper::{Method, StatusCode};
#[tokio::test] #[tokio::test]

View file

@ -3,16 +3,3 @@ pub mod errorcodes;
pub mod item; pub mod item;
pub mod poll; pub mod poll;
pub mod simple; pub mod simple;
use hyper::{Body, Response};
pub async fn json_body(res: Response<Body>) -> serde_json::Value {
let res_body: serde_json::Value = serde_json::from_slice(
&hyper::body::to_bytes(res.into_body())
.await
.unwrap()
.to_vec()[..],
)
.unwrap();
res_body
}

View file

@ -5,8 +5,8 @@ use std::time::Duration;
use assert_json_diff::assert_json_eq; use assert_json_diff::assert_json_eq;
use serde_json::json; use serde_json::json;
use super::json_body;
use crate::common; use crate::common;
use crate::json_body;
#[tokio::test] #[tokio::test]
async fn test_poll_item() { async fn test_poll_item() {

View file

@ -10,3 +10,16 @@ mod s3;
mod k2v; mod k2v;
#[cfg(feature = "k2v")] #[cfg(feature = "k2v")]
mod k2v_client; mod k2v_client;
use hyper::{Body, Response};
pub async fn json_body(res: Response<Body>) -> serde_json::Value {
let res_body: serde_json::Value = serde_json::from_slice(
&hyper::body::to_bytes(res.into_body())
.await
.unwrap()
.to_vec()[..],
)
.unwrap();
res_body
}

View file

@ -1,6 +1,6 @@
use crate::common; use crate::common;
use aws_sdk_s3::model::{CompletedMultipartUpload, CompletedPart}; use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ByteStream; use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
const SZ_5MB: usize = 5 * 1024 * 1024; const SZ_5MB: usize = 5 * 1024 * 1024;
const SZ_10MB: usize = 10 * 1024 * 1024; const SZ_10MB: usize = 10 * 1024 * 1024;

View file

@ -1,6 +1,6 @@
use crate::common; use crate::common;
use aws_sdk_s3::model::{Delete, ObjectIdentifier}; use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::ByteStream; use aws_sdk_s3::types::{Delete, ObjectIdentifier};
const STD_KEY: &str = "hello world"; const STD_KEY: &str = "hello world";
const CTRL_KEY: &str = "\x00\x01\x02\x00"; const CTRL_KEY: &str = "\x00\x01\x02\x00";

View file

@ -2,7 +2,7 @@ use crate::common;
#[tokio::test] #[tokio::test]
async fn test_simple() { async fn test_simple() {
use aws_sdk_s3::types::ByteStream; use aws_sdk_s3::primitives::ByteStream;
let ctx = common::context(); let ctx = common::context();
let bucket = ctx.create_bucket("test-simple"); let bucket = ctx.create_bucket("test-simple");

View file

@ -1,11 +1,11 @@
use crate::common; use crate::common;
use crate::common::ext::*; use crate::common::ext::*;
use crate::k2v::json_body; use crate::json_body;
use assert_json_diff::assert_json_eq; use assert_json_diff::assert_json_eq;
use aws_sdk_s3::{ use aws_sdk_s3::{
model::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration}, primitives::ByteStream,
types::ByteStream, types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration},
}; };
use http::{Request, StatusCode}; use http::{Request, StatusCode};
use hyper::{ use hyper::{
@ -72,7 +72,7 @@ async fn test_website() {
res_body, res_body,
json!({ json!({
"code": "InvalidRequest", "code": "InvalidRequest",
"message": "Bad request: Bucket 'my-website' is not authorized for website hosting", "message": "Bad request: Domain 'my-website' is not managed by Garage",
"region": "garage-integ-test", "region": "garage-integ-test",
"path": "/check", "path": "/check",
}) })
@ -91,24 +91,29 @@ async fn test_website() {
BODY.as_ref() BODY.as_ref()
); );
let admin_req = || { for bname in [
Request::builder() BCKT_NAME.to_string(),
.method("GET") format!("{BCKT_NAME}.web.garage"),
.uri(format!( format!("{BCKT_NAME}.s3.garage"),
"http://127.0.0.1:{0}/check?domain={1}", ] {
ctx.garage.admin_port, let admin_req = || {
BCKT_NAME.to_string() Request::builder()
)) .method("GET")
.body(Body::empty()) .uri(format!(
.unwrap() "http://127.0.0.1:{0}/check?domain={1}",
}; ctx.garage.admin_port, bname
))
.body(Body::empty())
.unwrap()
};
let mut admin_resp = client.request(admin_req()).await.unwrap(); let mut admin_resp = client.request(admin_req()).await.unwrap();
assert_eq!(admin_resp.status(), StatusCode::OK); assert_eq!(admin_resp.status(), StatusCode::OK);
assert_eq!( assert_eq!(
to_bytes(admin_resp.body_mut()).await.unwrap().as_ref(), to_bytes(admin_resp.body_mut()).await.unwrap().as_ref(),
format!("Bucket '{BCKT_NAME}' is authorized for website hosting").as_bytes() format!("Domain '{bname}' is managed by Garage").as_bytes()
); );
}
ctx.garage ctx.garage
.command() .command()
@ -142,7 +147,7 @@ async fn test_website() {
res_body, res_body,
json!({ json!({
"code": "InvalidRequest", "code": "InvalidRequest",
"message": "Bad request: Bucket 'my-website' is not authorized for website hosting", "message": "Bad request: Domain 'my-website' is not managed by Garage",
"region": "garage-integ-test", "region": "garage-integ-test",
"path": "/check", "path": "/check",
}) })
@ -397,7 +402,7 @@ async fn test_website_s3_api() {
} }
#[tokio::test] #[tokio::test]
async fn test_website_check_website_enabled() { async fn test_website_check_domain() {
let ctx = common::context(); let ctx = common::context();
let client = Client::new(); let client = Client::new();
@ -435,13 +440,13 @@ async fn test_website_check_website_enabled() {
}; };
let admin_resp = client.request(admin_req()).await.unwrap(); let admin_resp = client.request(admin_req()).await.unwrap();
assert_eq!(admin_resp.status(), StatusCode::NOT_FOUND); assert_eq!(admin_resp.status(), StatusCode::BAD_REQUEST);
let res_body = json_body(admin_resp).await; let res_body = json_body(admin_resp).await;
assert_json_eq!( assert_json_eq!(
res_body, res_body,
json!({ json!({
"code": "NoSuchBucket", "code": "InvalidRequest",
"message": "Bucket not found: ", "message": "Bad request: Domain '' is not managed by Garage",
"region": "garage-integ-test", "region": "garage-integ-test",
"path": "/check", "path": "/check",
}) })
@ -459,13 +464,13 @@ async fn test_website_check_website_enabled() {
}; };
let admin_resp = client.request(admin_req()).await.unwrap(); let admin_resp = client.request(admin_req()).await.unwrap();
assert_eq!(admin_resp.status(), StatusCode::NOT_FOUND); assert_eq!(admin_resp.status(), StatusCode::BAD_REQUEST);
let res_body = json_body(admin_resp).await; let res_body = json_body(admin_resp).await;
assert_json_eq!( assert_json_eq!(
res_body, res_body,
json!({ json!({
"code": "NoSuchBucket", "code": "InvalidRequest",
"message": "Bucket not found: foobar", "message": "Bad request: Domain 'foobar' is not managed by Garage",
"region": "garage-integ-test", "region": "garage-integ-test",
"path": "/check", "path": "/check",
}) })
@ -483,13 +488,13 @@ async fn test_website_check_website_enabled() {
}; };
let admin_resp = client.request(admin_req()).await.unwrap(); let admin_resp = client.request(admin_req()).await.unwrap();
assert_eq!(admin_resp.status(), StatusCode::NOT_FOUND); assert_eq!(admin_resp.status(), StatusCode::BAD_REQUEST);
let res_body = json_body(admin_resp).await; let res_body = json_body(admin_resp).await;
assert_json_eq!( assert_json_eq!(
res_body, res_body,
json!({ json!({
"code": "NoSuchBucket", "code": "InvalidRequest",
"message": "Bucket not found: ☹", "message": "Bad request: Domain '☹' is not managed by Garage",
"region": "garage-integ-test", "region": "garage-integ-test",
"path": "/check", "path": "/check",
}) })

View file

@ -311,23 +311,19 @@ impl BatchOutputKind {
.collect::<Vec<_>>() .collect::<Vec<_>>()
} }
fn display_poll_range_output( fn display_poll_range_output(&self, poll_range: PollRangeResult) -> ! {
&self,
seen_marker: String,
values: BTreeMap<String, CausalValue>,
) -> ! {
if self.json { if self.json {
let json = serde_json::json!({ let json = serde_json::json!({
"values": self.values_json(values), "values": self.values_json(poll_range.items),
"seen_marker": seen_marker, "seen_marker": poll_range.seen_marker,
}); });
let stdout = std::io::stdout(); let stdout = std::io::stdout();
serde_json::to_writer_pretty(stdout, &json).unwrap(); serde_json::to_writer_pretty(stdout, &json).unwrap();
exit(0) exit(0)
} else { } else {
println!("seen marker: {}", seen_marker); println!("seen marker: {}", poll_range.seen_marker);
self.display_human_output(values) self.display_human_output(poll_range.items)
} }
} }
@ -501,8 +497,8 @@ async fn main() -> Result<(), Error> {
) )
.await?; .await?;
match res { match res {
Some((items, seen_marker)) => { Some(poll_range_output) => {
output_kind.display_poll_range_output(seen_marker, items); output_kind.display_poll_range_output(poll_range_output);
} }
None => { None => {
if output_kind.json { if output_kind.json {

View file

@ -182,7 +182,7 @@ impl K2vClient {
filter: Option<PollRangeFilter<'_>>, filter: Option<PollRangeFilter<'_>>,
seen_marker: Option<&str>, seen_marker: Option<&str>,
timeout: Option<Duration>, timeout: Option<Duration>,
) -> Result<Option<(BTreeMap<String, CausalValue>, String)>, Error> { ) -> Result<Option<PollRangeResult>, Error> {
let timeout = timeout.unwrap_or(DEFAULT_POLL_TIMEOUT); let timeout = timeout.unwrap_or(DEFAULT_POLL_TIMEOUT);
let request = PollRangeRequest { let request = PollRangeRequest {
@ -217,7 +217,10 @@ impl K2vClient {
}) })
.collect::<BTreeMap<_, _>>(); .collect::<BTreeMap<_, _>>();
Ok(Some((items, resp.seen_marker))) Ok(Some(PollRangeResult {
items,
seen_marker: resp.seen_marker,
}))
} }
/// Perform an InsertItem request, inserting a value for a single pk+sk. /// Perform an InsertItem request, inserting a value for a single pk+sk.
@ -570,6 +573,7 @@ pub struct Filter<'a> {
pub reverse: bool, pub reverse: bool,
} }
/// Filter for a poll range operations.
#[derive(Debug, Default, Clone, Serialize)] #[derive(Debug, Default, Clone, Serialize)]
pub struct PollRangeFilter<'a> { pub struct PollRangeFilter<'a> {
pub start: Option<&'a str>, pub start: Option<&'a str>,
@ -577,6 +581,15 @@ pub struct PollRangeFilter<'a> {
pub prefix: Option<&'a str>, pub prefix: Option<&'a str>,
} }
/// Response to a poll_range query
#[derive(Debug, Default, Clone, Serialize)]
pub struct PollRangeResult {
/// List of items that have changed since last PollRange call.
pub items: BTreeMap<String, CausalValue>,
/// opaque string representing items already seen for future PollRange calls.
pub seen_marker: String,
}
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
#[serde(rename_all = "camelCase")] #[serde(rename_all = "camelCase")]
struct PollRangeRequest<'a> { struct PollRangeRequest<'a> {

View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_model" name = "garage_model"
version = "0.8.2" version = "0.8.3"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"

View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_rpc" name = "garage_rpc"
version = "0.8.2" version = "0.8.3"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -14,12 +14,13 @@ path = "lib.rs"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
garage_db.workspace = true
garage_util.workspace = true garage_util.workspace = true
arc-swap = "1.0" arc-swap = "1.0"
bytes = "1.0" bytes = "1.0"
bytesize = "1.1" bytesize = "1.1"
gethostname = "0.2" gethostname = "0.4"
hex = "0.4" hex = "0.4"
tracing = "0.1" tracing = "0.1"
rand = "0.8" rand = "0.8"

View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_table" name = "garage_table"
version = "0.8.2" version = "0.8.3"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"

View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_util" name = "garage_util"
version = "0.8.2" version = "0.8.3"
authors = ["Alex Auvolat <alex@adnab.me>"] authors = ["Alex Auvolat <alex@adnab.me>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"
@ -31,7 +31,7 @@ rand = "0.8"
sha2 = "0.10" sha2 = "0.10"
chrono = "0.4" chrono = "0.4"
rmp-serde = "1.1" rmp-serde = "1.1.2"
serde = { version = "1.0", default-features = false, features = ["derive", "rc"] } serde = { version = "1.0", default-features = false, features = ["derive", "rc"] }
serde_json = "1.0" serde_json = "1.0"
toml = "0.6" toml = "0.6"

View file

@ -27,7 +27,7 @@ pub trait Migrate: Serialize + for<'de> Deserialize<'de> + 'static {
Self::Previous::decode(bytes).map(Self::migrate) Self::Previous::decode(bytes).map(Self::migrate)
} }
/// Encode this type with optionnal version marker /// Encode this type with optional version marker
fn encode(&self) -> Result<Vec<u8>, rmp_serde::encode::Error> { fn encode(&self) -> Result<Vec<u8>, rmp_serde::encode::Error> {
let mut wr = Vec::with_capacity(128); let mut wr = Vec::with_capacity(128);
wr.extend_from_slice(Self::VERSION_MARKER); wr.extend_from_slice(Self::VERSION_MARKER);

View file

@ -1,6 +1,6 @@
[package] [package]
name = "garage_web" name = "garage_web"
version = "0.8.2" version = "0.8.3"
authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"] authors = ["Alex Auvolat <alex@adnab.me>", "Quentin Dufour <quentin@dufour.io>"]
edition = "2018" edition = "2018"
license = "AGPL-3.0" license = "AGPL-3.0"

View file

@ -1,4 +1,4 @@
use std::{borrow::Cow, convert::Infallible, net::SocketAddr, sync::Arc}; use std::{convert::Infallible, net::SocketAddr, sync::Arc};
use futures::future::Future; use futures::future::Future;
@ -6,7 +6,7 @@ use hyper::{
header::{HeaderValue, HOST}, header::{HeaderValue, HOST},
server::conn::AddrStream, server::conn::AddrStream,
service::{make_service_fn, service_fn}, service::{make_service_fn, service_fn},
Body, Method, Request, Response, Server, Body, Method, Request, Response, Server, StatusCode,
}; };
use opentelemetry::{ use opentelemetry::{
@ -28,6 +28,7 @@ use garage_api::s3::get::{handle_get, handle_head};
use garage_model::garage::Garage; use garage_model::garage::Garage;
use garage_table::*; use garage_table::*;
use garage_util::data::Uuid;
use garage_util::error::Error as GarageError; use garage_util::error::Error as GarageError;
use garage_util::forwarded_headers; use garage_util::forwarded_headers;
use garage_util::metrics::{gen_trace_id, RecordDuration}; use garage_util::metrics::{gen_trace_id, RecordDuration};
@ -168,6 +169,17 @@ impl WebServer {
} }
} }
async fn check_key_exists(self: &Arc<Self>, bucket_id: Uuid, key: &str) -> Result<bool, Error> {
let exists = self
.garage
.object_table
.get(&bucket_id, &key.to_string())
.await?
.map(|object| object.versions().iter().any(|v| v.is_data()))
.unwrap_or(false);
Ok(exists)
}
async fn serve_file(self: &Arc<Self>, req: &Request<Body>) -> Result<Response<Body>, Error> { async fn serve_file(self: &Arc<Self>, req: &Request<Body>) -> Result<Response<Body>, Error> {
// Get http authority string (eg. [::1]:3902 or garage.tld:80) // Get http authority string (eg. [::1]:3902 or garage.tld:80)
let authority = req let authority = req
@ -207,11 +219,11 @@ impl WebServer {
// Get path // Get path
let path = req.uri().path().to_string(); let path = req.uri().path().to_string();
let index = &website_config.index_document; let index = &website_config.index_document;
let key = path_to_key(&path, index)?; let (key, may_redirect) = path_to_keys(&path, index)?;
debug!( debug!(
"Selected bucket: \"{}\" {:?}, selected key: \"{}\"", "Selected bucket: \"{}\" {:?}, target key: \"{}\", may redirect to: {:?}",
bucket_name, bucket_id, key bucket_name, bucket_id, key, may_redirect
); );
let ret_doc = match *req.method() { let ret_doc = match *req.method() {
@ -219,10 +231,23 @@ impl WebServer {
Method::HEAD => handle_head(self.garage.clone(), req, bucket_id, &key, None).await, Method::HEAD => handle_head(self.garage.clone(), req, bucket_id, &key, None).await,
Method::GET => handle_get(self.garage.clone(), req, bucket_id, &key, None).await, Method::GET => handle_get(self.garage.clone(), req, bucket_id, &key, None).await,
_ => Err(ApiError::bad_request("HTTP method not supported")), _ => Err(ApiError::bad_request("HTTP method not supported")),
} };
.map_err(Error::from);
match ret_doc { // Try implicit redirect on error
let ret_doc_with_redir = match (&ret_doc, may_redirect) {
(Err(ApiError::NoSuchKey), ImplicitRedirect::To { key, url })
if self.check_key_exists(bucket_id, key.as_str()).await? =>
{
Ok(Response::builder()
.status(StatusCode::FOUND)
.header("Location", url)
.body(Body::empty())
.unwrap())
}
_ => ret_doc,
};
match ret_doc_with_redir.map_err(Error::from) {
Err(error) => { Err(error) => {
// For a HEAD or OPTIONS method, and for non-4xx errors, // For a HEAD or OPTIONS method, and for non-4xx errors,
// we don't return the error document as content, // we don't return the error document as content,
@ -308,30 +333,45 @@ fn error_to_res(e: Error) -> Response<Body> {
http_error http_error
} }
#[derive(Debug, PartialEq)]
enum ImplicitRedirect {
No,
To { key: String, url: String },
}
/// Path to key /// Path to key
/// ///
/// Convert the provided path to the internal key /// Convert the provided path to the internal key
/// When a path ends with "/", we append the index name to match traditional web server behavior /// When a path ends with "/", we append the index name to match traditional web server behavior
/// which is also AWS S3 behavior. /// which is also AWS S3 behavior.
fn path_to_key<'a>(path: &'a str, index: &str) -> Result<Cow<'a, str>, Error> { ///
/// Check: https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html
fn path_to_keys<'a>(path: &'a str, index: &str) -> Result<(String, ImplicitRedirect), Error> {
let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?; let path_utf8 = percent_encoding::percent_decode_str(path).decode_utf8()?;
if !path_utf8.starts_with('/') { let base_key = match path_utf8.strip_prefix("/") {
return Err(Error::BadRequest("Path must start with a / (slash)".into())); Some(bk) => bk,
} None => return Err(Error::BadRequest("Path must start with a / (slash)".into())),
};
let is_bucket_root = base_key.len() == 0;
let is_trailing_slash = path_utf8.ends_with("/");
match path_utf8.chars().last() { match (is_bucket_root, is_trailing_slash) {
None => unreachable!(), // It is not possible to store something at the root of the bucket (ie. empty key),
Some('/') => { // the only option is to fetch the index
let mut key = String::with_capacity(path_utf8.len() + index.len()); (true, _) => Ok((index.to_string(), ImplicitRedirect::No)),
key.push_str(&path_utf8[1..]);
key.push_str(index); // "If you create a folder structure in your bucket, you must have an index document at each level. In each folder, the index document must have the same name, for example, index.html. When a user specifies a URL that resembles a folder lookup, the presence or absence of a trailing slash determines the behavior of the website. For example, the following URL, with a trailing slash, returns the photos/index.html index document."
Ok(key.into()) (false, true) => Ok((format!("{base_key}{index}"), ImplicitRedirect::No)),
}
Some(_) => match path_utf8 { // "However, if you exclude the trailing slash from the preceding URL, Amazon S3 first looks for an object photos in the bucket. If the photos object is not found, it searches for an index document, photos/index.html. If that document is found, Amazon S3 returns a 302 Found message and points to the photos/ key. For subsequent requests to photos/, Amazon S3 returns photos/index.html. If the index document is not found, Amazon S3 returns an error."
Cow::Borrowed(pu8) => Ok((&pu8[1..]).into()), (false, false) => Ok((
Cow::Owned(pu8) => Ok(pu8[1..].to_string().into()), base_key.to_string(),
}, ImplicitRedirect::To {
key: format!("{base_key}/{index}"),
url: format!("{path}/"),
},
)),
} }
} }
@ -340,13 +380,37 @@ mod tests {
use super::*; use super::*;
#[test] #[test]
fn path_to_key_test() -> Result<(), Error> { fn path_to_keys_test() -> Result<(), Error> {
assert_eq!(path_to_key("/file%20.jpg", "index.html")?, "file .jpg"); assert_eq!(
assert_eq!(path_to_key("/%20t/", "index.html")?, " t/index.html"); path_to_keys("/file%20.jpg", "index.html")?,
assert_eq!(path_to_key("/", "index.html")?, "index.html"); (
assert_eq!(path_to_key("/hello", "index.html")?, "hello"); "file .jpg".to_string(),
assert!(path_to_key("", "index.html").is_err()); ImplicitRedirect::To {
assert!(path_to_key("i/am/relative", "index.html").is_err()); key: "file .jpg/index.html".to_string(),
url: "/file%20.jpg/".to_string()
}
)
);
assert_eq!(
path_to_keys("/%20t/", "index.html")?,
(" t/index.html".to_string(), ImplicitRedirect::No)
);
assert_eq!(
path_to_keys("/", "index.html")?,
("index.html".to_string(), ImplicitRedirect::No)
);
assert_eq!(
path_to_keys("/hello", "index.html")?,
(
"hello".to_string(),
ImplicitRedirect::To {
key: "hello/index.html".to_string(),
url: "/hello/".to_string()
}
)
);
assert!(path_to_keys("", "index.html").is_err());
assert!(path_to_keys("i/am/relative", "index.html").is_err());
Ok(()) Ok(())
} }
} }