.drone.yml

@@ -80,6 +80,38 @@ steps:
       - nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
       - nix-shell --arg release false --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
+  - name: update cache
+    image: nixpkgs/nix:nixos-21.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+      NIX_PRIV_KEY:
+        from_secret: nix_priv_key
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-build --no-out-link shell.nix --arg release false -A inputDerivation))
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target x86_64-unknown-linux-musl --argstr compileMode test))
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target x86_64-unknown-linux-musl --arg release false))
+    when:
+      event:
+      - cron
+
 trigger:
   event:
   - custom
@@ -180,6 +212,27 @@ steps:
     commands:
       - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
+  - name: update cache
+    image: nixpkgs/nix:nixos-21.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+      NIX_PRIV_KEY:
+        from_secret: nix_priv_key
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target $TARGET --arg release true))
+
   - name: push static binary
     image: nixpkgs/nix:nixos-21.05
     volumes:
@@ -223,94 +276,115 @@ trigger:
 node:
   nix: 1
 
----
+# ---
-kind: pipeline
+# kind: pipeline
-type: docker
+# type: docker
-name: release-linux-i686
+# name: release-linux-i686
-
+# 
-volumes:
+# volumes:
-- name: nix_store
+# - name: nix_store
-  host:
+#   host:
-    path: /var/lib/drone/nix
+#     path: /var/lib/drone/nix
-- name: nix_config
+# - name: nix_config
-  temp: {}
+#   temp: {}
-
+# 
-environment:
+# environment:
-  TARGET: i686-unknown-linux-musl
+#   TARGET: i686-unknown-linux-musl
-
+# 
-steps:
+# steps:
-  - name: setup nix
+#   - name: setup nix
-    image: nixpkgs/nix:nixos-21.05
+#     image: nixpkgs/nix:nixos-21.05
-    volumes:
+#     volumes:
-    - name: nix_store
+#     - name: nix_store
-      path: /nix
+#       path: /nix
-    - name: nix_config
+#     - name: nix_config
-      path: /etc/nix
+#       path: /etc/nix
-    commands:
+#     commands:
-      - cp nix/nix.conf /etc/nix/nix.conf
+#       - cp nix/nix.conf /etc/nix/nix.conf
-      - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
+#       - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
-
+# 
-  - name: build
+#   - name: build
-    image: nixpkgs/nix:nixos-21.05
+#     image: nixpkgs/nix:nixos-21.05
-    volumes:
+#     volumes:
-    - name: nix_store
+#     - name: nix_store
-      path: /nix
+#       path: /nix
-    - name: nix_config
+#     - name: nix_config
-      path: /etc/nix
+#       path: /etc/nix
-    commands:
+#     commands:
-      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+#       - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
-
+# 
-  - name: integration
+#   - name: integration
-    image: nixpkgs/nix:nixos-21.05
+#     image: nixpkgs/nix:nixos-21.05
-    volumes:
+#     volumes:
-    - name: nix_store
+#     - name: nix_store
-      path: /nix
+#       path: /nix
-    - name: nix_config
+#     - name: nix_config
-      path: /etc/nix
+#       path: /etc/nix
-    commands:
+#     commands:
-      - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+#       - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-
+# 
-  - name: push static binary
+#   - name: update cache
-    image: nixpkgs/nix:nixos-21.05
+#     image: nixpkgs/nix:nixos-21.05
-    volumes:
+#     environment:
-    - name: nix_store
+#       AWS_ACCESS_KEY_ID:
-      path: /nix
+#         from_secret: cache_aws_access_key_id
-    - name: nix_config
+#       AWS_SECRET_ACCESS_KEY:
-      path: /etc/nix
+#         from_secret: cache_aws_secret_access_key
-    environment:
+#       NIX_PRIV_KEY:
-      AWS_ACCESS_KEY_ID:
+#         from_secret: nix_priv_key
-        from_secret: garagehq_aws_access_key_id
+#     volumes:
-      AWS_SECRET_ACCESS_KEY:
+#     - name: nix_store
-        from_secret: garagehq_aws_secret_access_key
+#       path: /nix
-    commands:
+#     - name: nix_config
-      - nix-shell --arg rust false --arg integration false --run "to_s3"
+#       path: /etc/nix
-
+#     commands:
-  - name: docker build and publish
+#       - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
-    image: nixpkgs/nix:nixos-21.05
+#       - |
-    volumes:
+#         nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-    - name: nix_store
+#           $(nix-store -qR --include-outputs \
-      path: /nix
+#             $(nix-instantiate --argstr target $TARGET --arg release true))
-    - name: nix_config
+# 
-      path: /etc/nix
+#   - name: push static binary
-    environment:
+#     image: nixpkgs/nix:nixos-21.05
-      DOCKER_AUTH:
+#     volumes:
-        from_secret: docker_auth
+#     - name: nix_store
-      DOCKER_PLATFORM: "linux/386"
+#       path: /nix
-      CONTAINER_NAME: "dxflrs/386_garage"
+#     - name: nix_config
-      HOME: "/kaniko"
+#       path: /etc/nix
-    commands:
+#     environment:
-      - mkdir -p /kaniko/.docker
+#       AWS_ACCESS_KEY_ID:
-      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+#         from_secret: garagehq_aws_access_key_id
-      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+#       AWS_SECRET_ACCESS_KEY:
-      - nix-shell --arg rust false --arg integration false --run "to_docker"
+#         from_secret: garagehq_aws_secret_access_key
-
+#     commands:
-trigger:
+#       - nix-shell --arg rust false --arg integration false --run "to_s3"
-  event:
+# 
-  - promote
+#   - name: docker build and publish
-  - cron
+#     image: nixpkgs/nix:nixos-21.05
-
+#     volumes:
-node:
+#     - name: nix_store
-  nix: 1
+#       path: /nix
+#     - name: nix_config
+#       path: /etc/nix
+#     environment:
+#       DOCKER_AUTH:
+#         from_secret: docker_auth
+#       DOCKER_PLATFORM: "linux/386"
+#       CONTAINER_NAME: "dxflrs/386_garage"
+#       HOME: "/kaniko"
+#     commands:
+#       - mkdir -p /kaniko/.docker
+#       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+#       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+#       - nix-shell --arg rust false --arg integration false --run "to_docker"
+# 
+# trigger:
+#   event:
+#   - promote
+#   - cron
+# 
+# node:
+#   nix: 1
 
 ---
 kind: pipeline
@@ -349,6 +423,27 @@ steps:
     commands:
       - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
 
+  - name: update cache
+    image: nixpkgs/nix:nixos-21.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: cache_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: cache_aws_secret_access_key
+      NIX_PRIV_KEY:
+        from_secret: nix_priv_key
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
+      - |
+        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+          $(nix-store -qR --include-outputs \
+            $(nix-instantiate --argstr target $TARGET --arg release true))
+
   - name: push static binary
     image: nixpkgs/nix:nixos-21.05
     volumes:
@@ -391,84 +486,105 @@ trigger:
 node:
   nix: 1
 
----
+# ---
-kind: pipeline
+# kind: pipeline
-type: docker
+# type: docker
-name: release-linux-armv6l
+# name: release-linux-armv6l
-
+# 
-volumes:
+# volumes:
-- name: nix_store
+# - name: nix_store
-  host:
+#   host:
-    path: /var/lib/drone/nix
+#     path: /var/lib/drone/nix
-- name: nix_config
+# - name: nix_config
-  temp: {}
+#   temp: {}
-
+# 
-environment:
+# environment:
-  TARGET: armv6l-unknown-linux-musleabihf
+#   TARGET: armv6l-unknown-linux-musleabihf
-
+# 
-steps:
+# steps:
-  - name: setup nix
+#   - name: setup nix
-    image: nixpkgs/nix:nixos-21.05
+#     image: nixpkgs/nix:nixos-21.05
-    volumes:
+#     volumes:
-    - name: nix_store
+#     - name: nix_store
-      path: /nix
+#       path: /nix
-    - name: nix_config
+#     - name: nix_config
-      path: /etc/nix
+#       path: /etc/nix
-    commands:
+#     commands:
-      - cp nix/nix.conf /etc/nix/nix.conf
+#       - cp nix/nix.conf /etc/nix/nix.conf
-      - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
+#       - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
-
+# 
-  - name: build
+#   - name: build
-    image: nixpkgs/nix:nixos-21.05
+#     image: nixpkgs/nix:nixos-21.05
-    volumes:
+#     volumes:
-    - name: nix_store
+#     - name: nix_store
-      path: /nix
+#       path: /nix
-    - name: nix_config
+#     - name: nix_config
-      path: /etc/nix
+#       path: /etc/nix
-    commands:
+#     commands:
-      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+#       - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
-
+# 
-  - name: push static binary
+#   - name: update cache
-    image: nixpkgs/nix:nixos-21.05
+#     image: nixpkgs/nix:nixos-21.05
-    volumes:
+#     environment:
-    - name: nix_store
+#       AWS_ACCESS_KEY_ID:
-      path: /nix
+#         from_secret: cache_aws_access_key_id
-    - name: nix_config
+#       AWS_SECRET_ACCESS_KEY:
-      path: /etc/nix
+#         from_secret: cache_aws_secret_access_key
-    environment:
+#       NIX_PRIV_KEY:
-      AWS_ACCESS_KEY_ID:
+#         from_secret: nix_priv_key
-        from_secret: garagehq_aws_access_key_id
+#     volumes:
-      AWS_SECRET_ACCESS_KEY:
+#     - name: nix_store
-        from_secret: garagehq_aws_secret_access_key
+#       path: /nix
-    commands:
+#     - name: nix_config
-      - nix-shell --arg integration false --arg rust false --run "to_s3"
+#       path: /etc/nix
-
+#     commands:
-  - name: docker build and publish
+#       - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
-    image: nixpkgs/nix:nixos-21.05
+#       - |
-    volumes:
+#         nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-    - name: nix_store
+#           $(nix-store -qR --include-outputs \
-      path: /nix
+#             $(nix-instantiate --argstr target $TARGET --arg release true))
-    - name: nix_config
+# 
-      path: /etc/nix
+#   - name: push static binary
-    environment:
+#     image: nixpkgs/nix:nixos-21.05
-      DOCKER_AUTH:
+#     volumes:
-        from_secret: docker_auth
+#     - name: nix_store
-      DOCKER_PLATFORM: "linux/arm"
+#       path: /nix
-      CONTAINER_NAME: "dxflrs/arm_garage"
+#     - name: nix_config
-      HOME: "/kaniko"
+#       path: /etc/nix
-    commands:
+#     environment:
-      - mkdir -p /kaniko/.docker
+#       AWS_ACCESS_KEY_ID:
-      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+#         from_secret: garagehq_aws_access_key_id
-      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+#       AWS_SECRET_ACCESS_KEY:
-      - nix-shell --arg rust false --arg integration false --run "to_docker"
+#         from_secret: garagehq_aws_secret_access_key
-
+#     commands:
-trigger:
+#       - nix-shell --arg integration false --arg rust false --run "to_s3"
-  event:
+# 
-  - promote
+#   - name: docker build and publish
-  - cron
+#     image: nixpkgs/nix:nixos-21.05
-
+#     volumes:
-node:
+#     - name: nix_store
-  nix: 1
+#       path: /nix
+#     - name: nix_config
+#       path: /etc/nix
+#     environment:
+#       DOCKER_AUTH:
+#         from_secret: docker_auth
+#       DOCKER_PLATFORM: "linux/arm"
+#       CONTAINER_NAME: "dxflrs/arm_garage"
+#       HOME: "/kaniko"
+#     commands:
+#       - mkdir -p /kaniko/.docker
+#       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+#       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+#       - nix-shell --arg rust false --arg integration false --run "to_docker"
+# 
+# trigger:
+#   event:
+#   - promote
+#   - cron
+# 
+# node:
+#   nix: 1
 
 ---
 kind: pipeline
@@ -497,9 +613,9 @@ steps:
 
 depends_on:
   - release-linux-x86_64
-  - release-linux-i686
+  #- release-linux-i686
   - release-linux-aarch64
-  - release-linux-armv6l
+  #- release-linux-armv6l
 
 trigger:
   event:

Dockerfile

@@ -4,4 +4,4 @@ ENV RUST_BACKTRACE=1
 ENV RUST_LOG=garage=info
 
 COPY result/bin/garage /
-CMD [ "/garage", "server"]
+CMD [ "/garage", "server", "-c", "config.toml"]

default.nix

@@ -11,17 +11,26 @@ with import ./nix/common.nix;
 let
   crossSystem = { config = target; };
 in let
+
   pkgs = import pkgsSrc {
     inherit system crossSystem;
-    overlays = [ cargo2nixOverlay ];
+    overlays = [ cargo2nixOverlay rustOverlay ];
   };
+  rustDist = pkgs.buildPackages.rust-bin.stable.latest.default;
 
   /*
    The following complexity should be abstracted by makePackageSet' (note the final quote).
    However its code uses deprecated features of rust-overlay that can lead to bug.
    Instead, we build our own rustChannel object with the recommended API of rust-overlay.
    */
-  rustChannel = pkgs.rustPlatform.rust;
+  rustChannel = rustDist // {
+    cargo = rustDist;
+    rustc = rustDist.override {
+      targets = [
+        (pkgs.rustBuilder.rustLib.realHostTriple pkgs.stdenv.targetPlatform)
+      ];
+    };
+  };
 
   overrides = pkgs.buildPackages.rustBuilder.overrides.all ++ [
     (pkgs.rustBuilder.rustLib.makeOverride {

doc/book/src/SUMMARY.md

@@ -5,19 +5,14 @@
 - [Quick start](./quick_start/index.md)
 
 - [Cookbook](./cookbook/index.md)
-  - [Building from source](./cookbook/from_source.md)
+  - [Deploying Garage](./cookbook/real_world.md)
-  - [Integration with systemd](./cookbook/systemd.md)
+  - [Configuring S3 clients](./cookbook/clients.md)
-  - [Configuring a reverse proxy](./cookbook/reverse_proxy.md)
+  - [Hosting a website](./cookbook/website.md)
-  - [Production Deployment](./cookbook/real_world.md)
   - [Recovering from failures](./cookbook/recovering.md)
-
+  - [Building from source](./cookbook/from_source.md)
-- [Integrations](./connect/index.md)
+  - [Starting with Systemd](./cookbook/systemd.md)
-  - [Apps (Nextcloud, Peertube...)](./connect/apps.md)
+  - [Integrate as a media backend]()
-  - [Websites (Hugo, Jekyll, Publii...)](./connect/websites.md)
+  - [Operate a cluster]()
-  - [Repositories (Docker, Nix...)](./connect/repositories.md)
-  - [CLI tools (rclone, awscli, mc...)](./connect/cli.md)
-  - [Your code (PHP, JS, Go...)](./connect/code.md)
-
 
 - [Reference Manual](./reference_manual/index.md)
   - [Garage configuration file](./reference_manual/configuration.md)

doc/book/src/connect/apps.md

@@ -1,388 +0,0 @@
-# Apps (Nextcloud, Peertube...)
-
-In this section, we cover the following software: [Nextcloud](#nextcloud), [Peertube](#peertube), [Mastodon](#mastodon), [Matrix](#matrix)
-
-## Nextcloud
-
-Nextcloud is a popular file synchronisation and backup service.
-By default, Nextcloud stores its data on the local filesystem.
-If you want to expand your storage to aggregate multiple servers, Garage is the way to go.
-
-A S3 backend can be configured in two ways on Nextcloud, either as Primary Storage or as an External Storage.
-Primary storage will store all your data on S3, in an opaque manner, and will provide the best performances.
-External storage enable you to select which data will be stored on S3, your file hierarchy will be preserved in S3, but it might be slower.
-
-In the following, we cover both methods but before reading our guide, we suppose you have done some preliminary steps.
-First, we expect you have an already installed and configured Nextcloud instance.
-Second, we suppose you have created a key and a bucket.
-
-As a reminder, you can create a key for your nextcloud instance as follow:
-
-```bash
-garage key new --name nextcloud-key
-```
-
-Keep the Key ID and the Secret key in a pad, they will be needed later.  
-Then you can create a bucket and give read/write rights to your key on this bucket with:
-
-```bash
-garage bucket create nextcloud
-garage bucket allow nextcloud --read --write --key nextcloud-key
-```
-
-
-### Primary Storage
-
-Now edit your Nextcloud configuration file to enable object storage.
-On my installation, the config. file is located at the following path: `/var/www/nextcloud/config/config.php`.  
-We will add a new root key to the `$CONFIG` dictionnary named `objectstore`:
-
-```php
-<?php
-$CONFIG = array(
-/* your existing configuration */
-'objectstore' => [
-    'class' => '\\OC\\Files\\ObjectStore\\S3',
-    'arguments' => [
-        'bucket' => 'nextcloud',   // Your bucket name, must be created before
-        'autocreate' => false,     // Garage does not support autocreate
-        'key'    => 'xxxxxxxxx',   // The Key ID generated previously
-        'secret' => 'xxxxxxxxx',   // The Secret key generated previously
-        'hostname' => '127.0.0.1', // Can also be a domain name, eg. garage.example.com
-        'port' => 3900,            // Put your reverse proxy port or your S3 API port
-        'use_ssl' => false,        // Set it to true if you have a TLS enabled reverse proxy
-        'region' => 'garage',      // Garage has only one region named "garage"
-        'use_path_style' => true   // Garage supports only path style, must be set to true
-    ],
-],
-```
-
-That's all, your Nextcloud will store all your data to S3.
-To test your new configuration, just reload your Nextcloud webpage and start sending data.
-
-*External link:* [Nextcloud Documentation > Primary Storage](https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/primary_storage.html)
-
-### External Storage
-
-**From the GUI.** Activate the "External storage support" app from the "Applications" page (click on your account icon on the top right corner of your screen to display the menu). Go to your parameters page (also located below your account icon). Click on external storage (or the corresponding translation in your language).
-
-[![Screenshot of the External Storage form](./cli-nextcloud-gui.png)](./cli-nextcloud-gui.png)
-*Click on the picture to zoom*
-
-Add a new external storage. Put what you want in "folder name" (eg. "shared"). Select "Amazon S3". Keep "Access Key" for the Authentication field.
-In Configuration, put your bucket name (eg. nextcloud), the host (eg. 127.0.0.1), the port (eg. 3900 or 443), the region (garage). Tick the SSL box if you have put an HTTPS proxy in front of garage. You must tick the "Path access" box and you must leave the "Legacy authentication (v2)" box empty. Put your Key ID (eg. GK...) and your Secret Key in the last two input boxes. Finally click on the tick symbol on the right of your screen.
-
-Now go to your "Files" app and a new "linked folder" has appeared with the name you chose earlier (eg. "shared").
-
-*External link:* [Nextcloud Documentation > External Storage Configuration GUI](https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/external_storage_configuration_gui.html)
-
-**From the CLI.** First install the external storage application:
-
-```bash
-php occ app:install files_external
-```
-
-Then add a new mount point with:
-
-```bash
- php occ files_external:create \
-  -c bucket=nextcloud \
-  -c hostname=127.0.0.1 \
-  -c port=3900 \
-  -c region=garage \
-  -c use_ssl=false \
-  -c use_path_style=true \
-  -c legacy_auth=false \
-  -c key=GKxxxx \
-  -c secret=xxxx \
-  shared amazons3 amazons3::accesskey
-```
-
-Adapt the `hostname`, `port`, `use_ssl`, `key`, and `secret` entries to your configuration.
-Do not change the `use_path_style` and `legacy_auth` entries, other configurations are not supported.
-
-*External link:* [Nextcloud Documentation > occ command > files external](https://docs.nextcloud.com/server/latest/admin_manual/configuration_server/occ_command.html#files-external-label)
-
-
-## Peertube
-
-Peertube proposes a clever integration of S3 by directly exposing its endpoint instead of proxifying requests through the application.
-In other words, Peertube is only responsible of the "control plane" and offload the "data plane" to Garage.
-In return, this system is a bit harder to configure, especially with Garage that supports less feature than other older S3 backends.
-We show that it is still possible to configure Garage with Peertube, allowing you to spread the load and the bandiwdth usage on the Garage cluster.
-
-### Enable path-style access by patching Peertube
-
-First, you will need to apply a small patch on Peertube ([#4510](https://github.com/Chocobozzz/PeerTube/pull/4510)):
-
-```diff
-From e3b4c641bdf67e07d406a1d49d6aa6b1fbce2ab4 Mon Sep 17 00:00:00 2001
-From: Martin Honermeyer <maze@strahlungsfrei.de>
-Date: Sun, 31 Oct 2021 12:34:04 +0100
-Subject: [PATCH] Allow setting path-style access for object storage
-
----
- config/default.yaml                                           | 4 ++++
- config/production.yaml.example                                | 4 ++++
- server/initializers/config.ts                                 | 1 +
- server/lib/object-storage/shared/client.ts                    | 3 ++-
- .../production/config/custom-environment-variables.yaml       | 2 ++
- 5 files changed, 13 insertions(+), 1 deletion(-)
-
-diff --git a/config/default.yaml b/config/default.yaml
-index cf9d69a6211..4efd56fb804 100644
---- a/config/default.yaml
-+++ b/config/default.yaml
-@@ -123,6 +123,10 @@ object_storage:
-     # You can also use AWS_SECRET_ACCESS_KEY env variable
-     secret_access_key: ''
- 
-+  # Reference buckets via path rather than subdomain
-+  # (i.e. "my-endpoint.com/bucket" instead of "bucket.my-endpoint.com")
-+  force_path_style: false
-+
-   # Maximum amount to upload in one request to object storage
-   max_upload_part: 2GB
- 
-diff --git a/config/production.yaml.example b/config/production.yaml.example
-index 70993bf57a3..9ca2de5f4c9 100644
---- a/config/production.yaml.example
-+++ b/config/production.yaml.example
-@@ -121,6 +121,10 @@ object_storage:
-     # You can also use AWS_SECRET_ACCESS_KEY env variable
-     secret_access_key: ''
- 
-+  # Reference buckets via path rather than subdomain
-+  # (i.e. "my-endpoint.com/bucket" instead of "bucket.my-endpoint.com")
-+  force_path_style: false
-+
-   # Maximum amount to upload in one request to object storage
-   max_upload_part: 2GB
- 
-diff --git a/server/initializers/config.ts b/server/initializers/config.ts
-index 8375bf4304c..d726c59a4b6 100644
---- a/server/initializers/config.ts
-+++ b/server/initializers/config.ts
-@@ -91,6 +91,7 @@ const CONFIG = {
-       ACCESS_KEY_ID: config.get<string>('object_storage.credentials.access_key_id'),
-       SECRET_ACCESS_KEY: config.get<string>('object_storage.credentials.secret_access_key')
-     },
-+    FORCE_PATH_STYLE: config.get<boolean>('object_storage.force_path_style'),
-     VIDEOS: {
-       BUCKET_NAME: config.get<string>('object_storage.videos.bucket_name'),
-       PREFIX: config.get<string>('object_storage.videos.prefix'),
-diff --git a/server/lib/object-storage/shared/client.ts b/server/lib/object-storage/shared/client.ts
-index c9a61459336..eadad02f93f 100644
---- a/server/lib/object-storage/shared/client.ts
-+++ b/server/lib/object-storage/shared/client.ts
-@@ -26,7 +26,8 @@ function getClient () {
-         accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
-         secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
-       }
--      : undefined
-+      : undefined,
-+    forcePathStyle: CONFIG.OBJECT_STORAGE.FORCE_PATH_STYLE
-   })
- 
-   logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
-diff --git a/support/docker/production/config/custom-environment-variables.yaml b/support/docker/production/config/custom-environment-variables.yaml
-index c7cd28e6521..a960bab0bc9 100644
---- a/support/docker/production/config/custom-environment-variables.yaml
-+++ b/support/docker/production/config/custom-environment-variables.yaml
-@@ -54,6 +54,8 @@ object_storage:
- 
-   region: "PEERTUBE_OBJECT_STORAGE_REGION"
- 
-+  force_path_style: "PEERTUBE_OBJECT_STORAGE_FORCE_PATH_STYLE"
-+
-   max_upload_part:
-     __name: "PEERTUBE_OBJECT_STORAGE_MAX_UPLOAD_PART"
-     __format: "json"
-```
-
-You can then recompile it with:
-
-```
-npm run build
-```
-
-And it can be started with:
-
-```
-NODE_ENV=production NODE_CONFIG_DIR=/srv/peertube/config node dist/server.js
-```
-
-
-### Create resources in Garage
-
-Create a key for Peertube:
-
-```bash
-garage key new --name peertube-key
-```
-
-Keep the Key ID and the Secret key in a pad, they will be needed later.  
-
-We need two buckets, one for normal videos (named peertube-video) and one for webtorrent videos (named peertube-playlist).
-```bash
-garage bucket create peertube-video
-garage bucket create peertube-playlist
-```
-
-Now we allow our key to read and write on these buckets:
-
-```
-garage bucket allow peertube-playlist --read --write --key peertube-key
-garage bucket allow peertube-video --read --write --key peertube-key
-```
-
-Finally, we need to expose these buckets publicly to serve their content to users:
-
-```bash
-garage bucket website --allow peertube-playlist
-garage bucket website --allow peertube-video
-```
-
-These buckets are now accessible on the web port (by default 3902) with the following URL: `http://<bucket><root_domain>:<web_port>` where the root domain is defined in your configuration file (by default `.web.garage`). So we have currently the following URLs: 
-  * http://peertube-playlist.web.garage:3902
-  * http://peertube-video.web.garage:3902
-
-Make sure you (will) have a corresponding DNS entry for them.
-
-### Configure a Reverse Proxy to serve CORS
-
-Now we will configure a reverse proxy in front of Garage.
-This is required as we have no other way to serve CORS headers yet.
-For our example, we will use nginx:
-
-```nginx
-server {
-  # In production you should use TLS instead of plain HTTP
-  listen [::]:80;
-
-  server_name peertube-video.web.garage peertube-playlist.web.garage;
-
-  location / {
-    add_header Access-Control-Allow-Origin *;
-    add_header Access-Control-Max-Age 3600;
-    add_header Access-Control-Expose-Headers Content-Length;
-    add_header Access-Control-Allow-Headers Range;
-
-    # We do not forward OPTIONS request to Garage
-    # as it does not know how to interpret them.
-    # Instead, we simply answers 200.
-    if ($request_method !~ ^(GET|HEAD)$ ) {
-      return 200;
-    }
-
-    # If your do not have a Garage instance on the reverse proxy, change the URL here.
-    proxy_pass http://127.0.0.1:3902;
-    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-    proxy_set_header Host $host;
-  }
-}
-```
-
-Now make sure that your 2 dns entries are pointing to your reverse proxy.
-
-### Configure Peertube
-
-You must edit the file named `config/production.yaml`, we are only modifying the root key named `object_storage`:
-
-```yaml
-object_storage:
-  enabled: true
-
-  # Put localhost only if you have a garage instance running on that node
-  endpoint: 'http://localhost:3900' # or "garage.example.com" if you have TLS on port 443
-
-  # This entry has been added by our patch, must be set to true
-  force_path_style: true
-
-  # Garage supports only one region for now, named garage
-  region: 'garage'
-
-  credentials:
-    access_key_id: 'GKxxxx'
-    secret_access_key: 'xxxx'
-
-  max_upload_part: 2GB
-
-  streaming_playlists:
-    bucket_name: 'peertube-playlist'
-
-    # Keep it empty for our example
-    prefix: ''
-
-    # You must fill this field to make Peertube use our reverse proxy/website logic
-    base_url: 'http://peertube-playlist.web.garage' # Example: 'https://mirror.example.com'
-
-  # Same settings but for webtorrent videos
-  videos:
-    bucket_name: 'peertube-video'
-    prefix: ''
-    # You must fill this field to make Peertube use our reverse proxy/website logic
-    base_url: 'http://peertube-video.web.garage'
-```
-
-### That's all
-
-Everything must be configured now, simply restart Peertube and try to upload a video.
-You must see in your browser console that data are fetched directly from our bucket (through the reverse proxy).
-
-### Miscellaneous
-
-*Known bug:* The playback does not start and some 400 Bad Request Errors appear in your browser console and on Garage.
-If the description of the error contains HTTP Invalid Range: InvalidRange, the error is due to a buggy ffmpeg version.
-You must avoid the 4.4.0 and use either a newer or older version.
-
-*Associated issues:* [#137](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/137), [#138](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/138), [#140](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/140). These issues are non blocking.
-
-*External link:* [Peertube Documentation > Remote Storage](https://docs.joinpeertube.org/admin-remote-storage)
-
-## Mastodon
-
-https://docs.joinmastodon.org/admin/config/#cdn
-
-## Matrix
-
-### synapse-s3-storage-provider
-
-https://github.com/matrix-org/synapse-s3-storage-provider
-
-### matrix-media-repo
-
-https://github.com/turt2live/matrix-media-repo
-
-## Pixelfed
-
-https://docs.pixelfed.org/technical-documentation/env.html#filesystem
-
-## Pleroma
-
-https://docs-develop.pleroma.social/backend/configuration/cheatsheet/#pleromauploaderss3
-
-## Lemmy
-
-via pict-rs
-https://git.asonix.dog/asonix/pict-rs/commit/f9f4fc63d670f357c93f24147c2ee3e1278e2d97
-
-## Funkwhale
-
-https://docs.funkwhale.audio/admin/configuration.html#s3-storage
-
-## Misskey
-
-https://github.com/misskey-dev/misskey/commit/9d944243a3a59e8880a360cbfe30fd5a3ec8d52d
-
-## Prismo
-
-https://gitlab.com/prismosuite/prismo/-/blob/dev/.env.production.sample#L26-33
-
-## Owncloud Infinite Scale (ocis)
-
-## Unsupported
-
-  - Mobilizon: No S3 integration
-  - WriteFreely: No S3 integration
-  - Plume: No S3 integration

doc/book/src/connect/cli.md

@@ -1,127 +0,0 @@
-# CLI tools
-
-CLI tools allow you to query the S3 API without too many abstractions.
-These tools are particularly suitable for debug, backups, website deployments or any scripted task that need to handle data.
-
-## Minio client (recommended)
-
-Use the following command to set an "alias", i.e. define a new S3 server to be
-used by the Minio client:
-
-```bash
-mc alias set \
-  garage \
-  <endpoint> \
-  <access key> \
-  <secret key> \
-  --api S3v4
-```
-
-Remember that `mc` is sometimes called `mcli` (such as on Arch Linux), to avoid conflicts
-with Midnight Commander.
-
-Some commands:
-
-```bash
-# list buckets
-mc ls garage/
-
-# list objets in a bucket
-mc ls garage/my_files
-
-# copy from your filesystem to garage
-mc cp /proc/cpuinfo garage/my_files/cpuinfo.txt
-
-# copy from garage to your filesystem
-mc cp garage/my_files/cpuinfo.txt /tmp/cpuinfo.txt
-
-# mirror a folder from your filesystem to garage
-mc mirror --overwrite ./book garage/garagehq.deuxfleurs.fr
-```
-
-
-## AWS CLI
-
-Create a file named `~/.aws/credentials` and put:
-
-```toml
-[default]
-aws_access_key_id=xxxx
-aws_secret_access_key=xxxx
-```
-
-Then a file named `~/.aws/config` and put:
-
-```toml
-[default]
-region=garage
-```
-
-Now, supposing Garage is listening on `http://127.0.0.1:3900`, you can list your buckets with:
-
-```bash
-aws --endpoint-url http://127.0.0.1:3900 s3 ls
-```
-
-Passing the `--endpoint-url` parameter to each command is annoying but AWS developers do not provide a corresponding configuration entry.
-As a workaround, you can redefine the aws command by editing the file `~/.bashrc`:
-
-```
-function aws { command aws --endpoint-url http://127.0.0.1:3900 $@ ; }
-```
-
-*Do not forget to run `source ~/.bashrc` or to start a new terminal before running the next commands.*
-
-Now you can simply run:
-
-```bash
-# list buckets
-aws s3 ls
-
-# list objects of a bucket
-aws s3 ls s3://my_files
-
-# copy from your filesystem to garage
-aws s3 cp /proc/cpuinfo s3://my_files/cpuinfo.txt
-
-# copy from garage to your filesystem
-aws s3 cp s3/my_files/cpuinfo.txt /tmp/cpuinfo.txt
-```
-
-## `rclone`
-
-`rclone` can be configured using the interactive assistant invoked using `rclone configure`.
-
-You can also configure `rclone` by writing directly its configuration file.
-Here is a template `rclone.ini` configuration file:
-
-```ini
-[garage]
-type = s3
-provider = Other
-env_auth = false
-access_key_id = <access key>
-secret_access_key = <secret key>
-region = <region>
-endpoint = <endpoint>
-force_path_style = true
-acl = private
-bucket_acl = private
-```
-
-## Cyberduck
-
-TODO
-
-## `s3cmd`
-
-Here is a template for the `s3cmd.cfg` file to talk with Garage:
-
-```ini
-[default]
-access_key = <access key>
-secret_key = <secret key>
-host_base = <endpoint without http(s)://>
-host_bucket = <same as host_base>
-use_https = False | True
-```

doc/book/src/connect/code.md

@@ -1 +0,0 @@
-# Your code (PHP, JS, Go...)

doc/book/src/connect/repositories.md

@@ -1 +0,0 @@
-# Repositories (Docker, Nix...)

doc/book/src/connect/websites.md

@@ -1 +0,0 @@
-# Websites (Hugo, Jekyll, Publii...)

doc/book/src/cookbook/clients.md

@@ -1,4 +1,4 @@
-# Connect it to
+# Configuring S3 clients to interact with Garage
 
 To configure an S3 client to interact with Garage, you will need the following
 parameters:
@@ -28,11 +28,78 @@ provided that you follow the following guidelines:
   Garage should normally redirect your client to the correct region,
   but in case your client does not support this you might have to configure it manually.
 
-We will now provide example configurations for the most common clients per category:
+We will now provide example configurations for the most common S3 clients.
 
-  - [Apps](./apps.md)
+## AWS CLI
-  - [Websites](./websites.md)
-  - [Repositories](./repositories.md)
-  - [CLI tools](./cli.md)
-  - [Your code](./code.md)
 
+Export the following environment variables:
+
+```bash
+export AWS_ACCESS_KEY_ID=<access key>
+export AWS_SECRET_ACCESS_KEY=<secret key>
+export AWS_DEFAULT_REGION=<region>
+```
+
+Now invoke `aws` as follows:
+
+```bash
+aws --endpoint-url <endpoint> s3 <command...>
+```
+
+For instance: `aws --endpoint-url http://127.0.0.1:3901 s3 ls s3://my-bucket/`.
+
+## Minio client
+
+Use the following command to set an "alias", i.e. define a new S3 server to be
+used by the Minio client:
+
+```bash
+mc alias set \
+  garage \
+  <endpoint> \
+  <access key> \
+  <secret key> \
+  --api S3v4
+```
+
+Remember that `mc` is sometimes called `mcli` (such as on Arch Linux), to avoid conflicts
+with the Midnight Commander.
+
+
+## `rclone`
+
+`rclone` can be configured using the interactive assistant invoked using `rclone configure`.
+
+You can also configure `rclone` by writing directly its configuration file.
+Here is a template `rclone.ini` configuration file:
+
+```ini
+[garage]
+type = s3
+provider = Other
+env_auth = false
+access_key_id = <access key>
+secret_access_key = <secret key>
+region = <region>
+endpoint = <endpoint>
+force_path_style = true
+acl = private
+bucket_acl = private
+```
+
+## Cyberduck
+
+TODO
+
+## `s3cmd`
+
+Here is a template for the `s3cmd.cfg` file to talk with Garage:
+
+```ini
+[default]
+access_key = <access key>
+secret_key = <secret key>
+host_base = <endpoint without http(s)://>
+host_bucket = <same as host_base>
+use_https = False | True
+```

doc/book/src/cookbook/reverse_proxy.md

@@ -1 +0,0 @@
-# Configuring a reverse proxy

doc/book/src/development/release_process.md

@@ -92,21 +92,10 @@ caching our development dependencies.
 *Currently there is no automatic garbage collection of the cache: we should monitor its growth.
 Hopefully, we can erase it totally without breaking any build, the next build will only be slower.*
 
-In practise, we concluded that we do not want to cache all the compilation dependencies.
-Instead, we want to cache the toolchain we use to build Garage each time we change it.
-So we removed from Drone any automatic update of the cache and instead handle them manually with:
-
-```
-source ~/.awsrc
-nix-shell --run 'refresh_toolchain'
-```
-
-Internally, it will run `nix-build` on  `nix/toolchain.nix` and send the output plus its depedencies to the cache.
-
 To erase the cache:
 
 ```
-mc rm --recursive --force 'garage/nix/'
+mc rm --recursive --force 'garage/nix/*'
 ```
 
 ### Publishing Garage

nix/common.nix

@@ -12,10 +12,16 @@ rec {
     url = "https://github.com/superboum/cargo2nix";
     rev = "1364752cd784764db2ef5b1e1248727cebfae2ce";
   };
+  rustOverlaySrc = fetchGit {
+    # As of 2021-10-05
+    url = "https://github.com/oxalica/rust-overlay";
+    rev = "9c2fc6a62ccbc6f420d71ecac6bf0b84dbbee64f";
+  };
 
   /*
    * Shared objects
    */
+  rustOverlay = import rustOverlaySrc;
   cargo2nix = import cargo2nixSrc;
   cargo2nixOverlay = import "${cargo2nixSrc}/overlay";
 }

nix/nix.conf

@@ -2,3 +2,6 @@ substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
 trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
 max-jobs = auto
 cores = 4
+
+# required for containers
+sandbox = false

nix/toolchain.nix

@@ -1,29 +0,0 @@
-{
-  system ? builtins.currentSystem,
-}:
-
-with import ./common.nix;
-
-let
-  platforms = [
-    "x86_64-unknown-linux-musl"
-    "i686-unknown-linux-musl"
-    "aarch64-unknown-linux-musl"
-    "armv6l-unknown-linux-musleabihf"
-  ];
-  pkgsList = builtins.map (target: import pkgsSrc { 
-    inherit system; 
-    crossSystem = { config = target; };
-  }) platforms;
-  pkgsHost = import pkgsSrc {};
-  lib = pkgsHost.lib;
-  kaniko = (import ./kaniko.nix) pkgsHost;
-in 
-  lib.flatten (builtins.map (pkgs: [
-     pkgs.rustPlatform.rust.rustc
-     pkgs.rustPlatform.rust.cargo
-     pkgs.buildPackages.stdenv.cc
-  ]) pkgsList) ++ [
-    kaniko
-  ]
-

shell.nix

@@ -1,5 +1,6 @@
 {
   system ? builtins.currentSystem,
+  crossSystem ? null,
   rust ? true,
   integration ? true,
   release ? true,
@@ -9,9 +10,10 @@ with import ./nix/common.nix;
 
 let
   pkgs = import pkgsSrc {
-    inherit system;
+    inherit system crossSystem;
-    overlays = [ cargo2nixOverlay ];
+    overlays = [ cargo2nixOverlay rustOverlay ];
   };
+  rustDist = pkgs.buildPackages.rust-bin.stable.latest.default;
   kaniko = (import ./nix/kaniko.nix) pkgs;
 
 in
@@ -55,39 +57,13 @@ function refresh_index {
       result \
       s3://garagehq.deuxfleurs.fr/_releases.html
 }
-
-function refresh_toolchain {
-  nix copy \
-    --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-    $(nix-store -qR \
-      $(nix-build --quiet --no-build-output --no-out-link nix/toolchain.nix))
-}
   '';
 
   nativeBuildInputs = 
-   (if rust then [
+   (if rust then [ rustDist (pkgs.callPackage cargo2nix {}).package ] else [])
-     pkgs.rustPlatform.rust.rustc
-     pkgs.rustPlatform.rust.cargo
-     pkgs.clippy
-     pkgs.rustfmt
-     /*(pkgs.callPackage cargo2nix {}).package*/
-    ] else [])
    ++
-   (if integration then [ 
+   (if integration then [ pkgs.s3cmd pkgs.awscli2 pkgs.minio-client pkgs.rclone pkgs.socat pkgs.psmisc pkgs.which ] else [])
-     pkgs.s3cmd
-     pkgs.awscli2
-     pkgs.minio-client
-     pkgs.rclone
-     pkgs.socat
-     pkgs.psmisc
-     pkgs.which
-     pkgs.openssl
-     pkgs.curl
-    ] else [])
    ++
-   (if release then [ 
+   (if release then [ pkgs.awscli2 kaniko ] else [])
-     pkgs.awscli2
-     kaniko 
-    ] else [])
    ;
 }

src/garage/main.rs

@@ -30,20 +30,15 @@ use cli::*;
 struct Opt {
 	/// Host to connect to for admin operations, in the format:
 	/// <public-key>@<ip>:<port>
-	#[structopt(short = "h", long = "rpc-host", env = "GARAGE_RPC_HOST")]
+	#[structopt(short = "h", long = "rpc-host")]
 	pub rpc_host: Option<String>,
 
 	/// RPC secret network key for admin operations
-	#[structopt(short = "s", long = "rpc-secret", env = "GARAGE_RPC_SECRET")]
+	#[structopt(short = "s", long = "rpc-secret")]
 	pub rpc_secret: Option<String>,
 
 	/// Configuration file (garage.toml)
-	#[structopt(
+	#[structopt(short = "c", long = "config", default_value = "/etc/garage.toml")]
-		short = "c",
-		long = "config",
-		env = "GARAGE_CONFIG_FILE",
-		default_value = "/etc/garage.toml"
-	)]
 	pub config_file: PathBuf,
 
 	#[structopt(subcommand)]