.drone.yml

@@ -0,0 +1,300 @@
+---
+kind: pipeline
+name: default
+
+node:
+  nix-daemon: 1
+
+steps:
+  - name: check formatting
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-shell --attr rust --run "cargo fmt -- --check"
+
+  - name: build
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+
+  - name: unit + func tests
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage
+      GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration
+    commands:
+      - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-build --no-build-output --attr test.amd64
+      - ./result/bin/garage_db-*
+      - ./result/bin/garage_api-*
+      - ./result/bin/garage_model-*
+      - ./result/bin/garage_rpc-*
+      - ./result/bin/garage_table-*
+      - ./result/bin/garage_util-*
+      - ./result/bin/garage_web-*
+      - ./result/bin/garage-*
+      - ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
+      - rm result
+      - rm -rv tmp-garage-integration
+
+  - name: integration tests
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+
+trigger:
+  event:
+  - custom
+  - push
+  - pull_request
+  - tag
+  - cron
+
+---
+kind: pipeline
+type: docker
+name: release-linux-amd64
+
+node:
+  nix-daemon: 1
+
+steps:
+  - name: build
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
+
+  - name: integration tests
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+
+  - name: upgrade tests
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
+
+  - name: push static binary
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+      TARGET: "x86_64-unknown-linux-musl"
+    commands:
+      - nix-shell --attr release --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/amd64"
+      CONTAINER_NAME: "dxflrs/amd64_garage"
+      HOME: "/kaniko"
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr release --run "to_docker"
+
+
+trigger:
+  event:
+  - promote
+  - cron
+
+---
+kind: pipeline
+type: docker
+name: release-linux-i386
+
+node:
+  nix-daemon: 1
+
+steps:
+  - name: build
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
+
+  - name: integration tests
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+
+  - name: upgrade tests
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
+
+  - name: push static binary
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+      TARGET: "i686-unknown-linux-musl"
+    commands:
+      - nix-shell --attr release --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/386"
+      CONTAINER_NAME: "dxflrs/386_garage"
+      HOME: "/kaniko"
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr release --run "to_docker"
+
+trigger:
+  event:
+  - promote
+  - cron
+
+---
+kind: pipeline
+type: docker
+name: release-linux-arm64
+
+node:
+  nix-daemon: 1
+
+steps:
+  - name: build
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
+
+  - name: push static binary
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+      TARGET: "aarch64-unknown-linux-musl"
+    commands:
+      - nix-shell --attr release --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/arm64"
+      CONTAINER_NAME: "dxflrs/arm64_garage"
+      HOME: "/kaniko"
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr release --run "to_docker"
+
+trigger:
+  event:
+  - promote
+  - cron
+
+---
+kind: pipeline
+type: docker
+name: release-linux-arm
+
+node:
+  nix-daemon: 1
+
+steps:
+  - name: build
+    image: nixpkgs/nix:nixos-22.05
+    commands:
+      - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
+
+  - name: push static binary
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+      TARGET: "armv6l-unknown-linux-musleabihf"
+    commands:
+      - nix-shell --attr release --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/arm"
+      CONTAINER_NAME: "dxflrs/arm_garage"
+      HOME: "/kaniko"
+    commands:
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr release --run "to_docker"
+
+trigger:
+  event:
+  - promote
+  - cron
+
+---
+kind: pipeline
+type: docker
+name: refresh-release-page
+
+node:
+  nix-daemon: 1
+
+steps:
+  - name: multiarch-docker
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      HOME: "/root"
+    commands:
+      - mkdir -p /root/.docker
+      - echo $DOCKER_AUTH > /root/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --attr release --run "multiarch_docker"
+  - name: refresh-index
+    image: nixpkgs/nix:nixos-22.05
+    environment:
+      AWS_ACCESS_KEY_ID:
+        from_secret: garagehq_aws_access_key_id
+      AWS_SECRET_ACCESS_KEY:
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
+      - nix-shell --attr release --run "refresh_index"
+
+depends_on:
+  - release-linux-amd64
+  - release-linux-i386
+  - release-linux-arm64
+  - release-linux-arm
+
+trigger:
+  event:
+  - promote
+  - cron
+
+---
+kind: signature
+hmac: 0c4b57eb4b27b7c6a6ff21ab87f0767fe3eb90f5d95d5cbcdccf794e9d2a5d86
+
+...

.woodpecker/debug.yaml

@@ -10,7 +10,7 @@ steps:
   - name: check formatting
     image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --attr devShell --run "cargo fmt -- --check"
+      - nix-shell --attr rust --run "cargo fmt -- --check"
 
   - name: build
     image: nixpkgs/nix:nixos-22.05
@@ -41,4 +41,4 @@ steps:
     image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)

.woodpecker/publish.yaml

@@ -16,14 +16,17 @@ steps:
         target: AWS_SECRET_ACCESS_KEY
     commands:
       - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
-      - nix-shell --attr ci --run "refresh_index"
+      - nix-shell --attr release --run "refresh_index"
 
-  - name: multiarch-docker
-    image: nixpkgs/nix:nixos-22.05
-    secrets:
-      - docker_auth
-    commands:
-      - mkdir -p /root/.docker
-      - echo $DOCKER_AUTH > /root/.docker/config.json
-      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-      - nix-shell --attr ci --run "multiarch_docker"
+#  - name: multiarch-docker
+#    image: nixpkgs/nix:nixos-22.05
+#    environment:
+#      HOME: "/root"
+#    secrets:
+#      - docker_auth
+#    commands:
+#      - mkdir -p /root/.docker
+#      - echo $DOCKER_AUTH > /root/.docker/config.json
+#      - sha512sum /root/.docker/config.json
+#      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
+#      - nix-shell --attr release --run "multiarch_docker"

.woodpecker/release.yaml

@@ -19,17 +19,12 @@ steps:
     image: nixpkgs/nix:nixos-22.05
     commands:
       - nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-
-  - name: check is static binary
-    image: nixpkgs/nix:nixos-22.05
-    commands:
-      - nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-      - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
+      - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
 
   - name: integration tests
     image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
+      - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
     when:
       - matrix:
           ARCH: amd64
@@ -39,7 +34,7 @@ steps:
   - name: upgrade tests
     image: nixpkgs/nix:nixos-22.05
     commands:
-      - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
+      - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
     when:
       - matrix:
           ARCH: amd64
@@ -54,17 +49,19 @@ steps:
       - source: garagehq_aws_secret_access_key
         target: AWS_SECRET_ACCESS_KEY
     commands:
-      - nix-shell --attr ci --run "to_s3"
+      - nix-shell --attr release --run "to_s3_woodpecker"
 
-  - name: docker build and publish
-    image: nixpkgs/nix:nixos-22.05
-    environment:
-      DOCKER_PLATFORM: "linux/${ARCH}"
-      CONTAINER_NAME: "dxflrs/${ARCH}_garage"
-    secrets:
-      - docker_auth
-    commands:
-      - mkdir -p /root/.docker
-      - echo $DOCKER_AUTH > /root/.docker/config.json
-      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
-      - nix-shell --attr ci --run "to_docker"
+#  - name: docker build and publish
+#    image: nixpkgs/nix:nixos-22.05
+#    environment:
+#      DOCKER_PLATFORM: "linux/${ARCH}"
+#      CONTAINER_NAME: "dxflrs/${ARCH}_garage"
+#      HOME: "/kaniko"
+#    secrets:
+#      - docker_auth
+#    commands:
+#      - mkdir -p /kaniko/.docker
+#      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+#      - sha512sum /kaniko/.docker/config.json
+#      - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA}
+#      - nix-shell --attr release --run "to_docker"

doc/book/design/goals.md

@@ -48,5 +48,7 @@ locations. They use Garage themselves for the following tasks:
 
 - As a backup target using `rclone` and `restic`
 
+- In the Drone continuous integration platform to store task logs
+
 The Deuxfleurs Garage cluster is a multi-site cluster currently composed of
 9 nodes in 3 physical locations.

doc/book/development/devenv.md

@@ -80,7 +80,7 @@ nix-build \
   --git_version $(git rev-parse HEAD)
 ```
 
-*The result is located in `result/bin`. You can pass arguments to cross compile: check `.woodpecker/release.yml` for examples.*
+*The result is located in `result/bin`. You can pass arguments to cross compile: check `.drone.yml` for examples.*
 
 If you modify a `Cargo.toml` or regenerate any `Cargo.lock`, you must run `cargo2nix`:
 

doc/book/development/miscellaneous-notes.md

@@ -81,9 +81,12 @@ Our cache will be checked.
  - http://www.lpenz.org/articles/nixchannel/index.html
 
 
-## Woodpecker
+## Drone
 
-Woodpecker can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner.
+Do not try to set a build as trusted from the interface or the CLI tool,
+your request would be ignored. Instead, directly edit the database (table `repos`, column `repo_trusted`).
+
+Drone can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner.
 
 ## Building Docker containers
 
@@ -96,4 +99,3 @@ We were:
   - Unable to use the kaniko container provided by Google as we can't run arbitrary logic: we need to put our secret in .docker/config.json.
 
 Finally we chose to build kaniko through nix and use it in a `nix-shell`.
-We then switched to using kaniko from nixpkgs when it was packaged.

doc/book/development/release-process.md

@@ -42,7 +42,7 @@ and the docker containers on Docker Hub.
 
 ## Automation
 
-We automated our release process with Nix and Woodpecker to make it more reliable.
+We automated our release process with Nix and Drone to make it more reliable.
 Here we describe how we have done in case you want to debug or improve it.
 
 ### Caching build steps
@@ -62,31 +62,52 @@ Sending to the cache is done through `nix copy`, for example:
 nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' result
 ```
 
-*The signing key possessed by the Garage maintainers is required to update the Nix cache.*
+*Note that you need the signing key. In our case, it is stored as a secret in Drone.*
 
-The previous command will only send the built package and not its dependencies.
-In the case of our CI pipeline, we want to cache all intermediate build steps
-as well. This can be done using this quite involved command (here as an example
-for the `pkgs.amd64.relase` package):
+The previous command will only send the built packet and not its dependencies.
+To send its dependency, a tool named `nix-copy-closure` has been created but it is not compatible with the S3 protocol.
+
+Instead, you can use the following commands to list all the runtime dependencies:
 
 ```bash
-nix copy -j8 \
-    --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/nix-signing-key.sec' \
-    $(nix path-info pkgs.amd64.release --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/')
+nix copy \
+  --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+  $(nix-store -qR result/)
 ```
 
-This command will simultaneously build all of the required Nix paths (using at
-most 8 parallel Nix builder jobs) and send the resulting objects to the cache.
+*We could also write this expression with xargs but this tool is not available in our container.*
 
-This can be run for all the Garage packages we build using the following command:
+But in certain cases, we want to cache compile time dependencies also.
+For example, the Nix project does not provide binaries for cross compiling to i686 and thus we need to compile gcc on our own.
+We do not want to compile gcc each time, so even if it is a compile time dependency, we want to cache it.
+
+This time, the command is a bit more involved:
+
+```bash
+nix copy --to \
+  's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+   $(nix-store -qR --include-outputs \
+     $(nix-instantiate))
+```
+
+This is the command we use in our CI as we expect the final binary to change, so we mainly focus on
+caching our development dependencies.
+
+*Currently there is no automatic garbage collection of the cache: we should monitor its growth.
+Hopefully, we can erase it totally without breaking any build, the next build will only be slower.*
+
+In practise, we concluded that we do not want to cache all the compilation dependencies.
+Instead, we want to cache the toolchain we use to build Garage each time we change it.
+So we removed from Drone any automatic update of the cache and instead handle them manually with:
 
 ```
 source ~/.awsrc
-nix-shell --attr cache --run 'refresh_cache'
+nix-shell --run 'refresh_toolchain'
 ```
 
-We don't automate this step at each CI build, as *there is currently no automatic garbage collection of the cache.*
-This means we should also monitor the cache's size; if it ever becomes too big we can erase it with:
+Internally, it will run `nix-build` on  `nix/toolchain.nix` and send the output plus its depedencies to the cache.
+
+To erase the cache:
 
 ```
 mc rm --recursive --force 'garage/nix/'
@@ -136,9 +157,9 @@ nix-shell --run refresh_index
 
 If you want to compile for different architectures, you will need to repeat all these commands for each architecture.
 
-**In practice, and except for debugging, you will never directly run these commands. Release is handled by Woodpecker.**
+**In practise, and except for debugging, you will never directly run these commands. Release is handled by drone**
 
-### Drone (obsolete)
+### Drone
 
 Our instance is available at [https://drone.deuxfleurs.fr](https://drone.deuxfleurs.fr).  
 You need an account on [https://git.deuxfleurs.fr](https://git.deuxfleurs.fr) to use it.

flake.nix

@@ -33,57 +33,27 @@
       compile = import ./nix/compile.nix;
     in
     flake-utils.lib.eachDefaultSystem (system:
-      let
-        pkgs = nixpkgs.legacyPackages.${system};
-      in
-      {
-        packages =
-          let
-            packageFor = target: (compile {
-              inherit system git_version target;
+      let pkgs = nixpkgs.legacyPackages.${system};
+      in {
+        packages = {
+          default = (compile {
+            inherit system git_version;
             pkgsSrc = nixpkgs;
             cargo2nixOverlay = cargo2nix.overlays.default;
             release = true;
           }).workspace.garage { compileMode = "build"; };
-          in
-          {
-            # default = native release build
-            default = packageFor null;
-            # other = cross-compiled, statically-linked builds
-            amd64 = packageFor "x86_64-unknown-linux-musl";
-            i386 = packageFor "i686-unknown-linux-musl";
-            arm64 = packageFor "aarch64-unknown-linux-musl";
-            arm = packageFor "armv6l-unknown-linux-musl";
         };
-
-        # ---- developpment shell, for making native builds only ----
-        devShells =
-          let
-            shellWithPackages = (packages: (compile {
+        devShell = (compile {
           inherit system git_version;
           pkgsSrc = nixpkgs;
           cargo2nixOverlay = cargo2nix.overlays.default;
-            }).workspaceShell { inherit packages; });
-          in
-          {
-            default = shellWithPackages
-              (with pkgs; [
-                rustfmt
-                clang
-                mold
-              ]);
-
-            # import the full shell using `nix develop .#full`
-            full = shellWithPackages (with pkgs; [
-              rustfmt
-              clang
-              mold
-              # ---- extra packages for dev tasks ----
+          release = false;
+        }).workspaceShell { packages = with pkgs; [
           cargo-audit
           cargo-outdated
-              cargo-machete
-              nixpkgs-fmt
-            ]);
-          };
+          rustfmt
+          clang
+          mold
+        ]; };
       });
 }

nix/common.nix

@@ -14,5 +14,4 @@ rec {
   pkgsSrc = flake.defaultNix.inputs.nixpkgs;
   cargo2nix = flake.defaultNix.inputs.cargo2nix;
   cargo2nixOverlay = cargo2nix.overlays.default;
-  devShells = builtins.getAttr builtins.currentSystem flake.defaultNix.devShells;
 }

nix/kaniko.nix

@@ -0,0 +1,24 @@
+pkgs:
+pkgs.buildGoModule rec {
+  pname = "kaniko";
+  version = "1.9.2";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "GoogleContainerTools";
+    repo = "kaniko";
+    rev = "v${version}";
+    sha256 = "dXQ0/o1qISv+sjNVIpfF85bkbM9sGOGwqVbWZpMWfMY=";
+  };
+
+  vendorSha256 = null;
+
+  checkPhase = "true";
+
+  meta = with pkgs.lib; {
+    description =
+      "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster.";
+    homepage = "https://github.com/GoogleContainerTools/kaniko";
+    license = licenses.asl20;
+    platforms = platforms.linux;
+  };
+}

nix/manifest-tool.nix

@@ -0,0 +1,24 @@
+pkgs:
+pkgs.buildGoModule rec {
+  pname = "manifest-tool";
+  version = "2.0.5";
+
+  src = pkgs.fetchFromGitHub {
+    owner = "estesp";
+    repo = "manifest-tool";
+    rev = "v${version}";
+    sha256 = "hjCGKnE0yrlnF/VIzOwcDzmQX3Wft+21KCny/opqdLg=";
+  } + "/v2";
+
+  vendorSha256 = null;
+
+  checkPhase = "true";
+
+  meta = with pkgs.lib; {
+    description =
+      "Command line tool to create and query container image manifest list/indexes";
+    homepage = "https://github.com/estesp/manifest-tool";
+    license = licenses.asl20;
+    platforms = platforms.linux;
+  };
+}

nix/toolchain.nix

@@ -0,0 +1,11 @@
+{ system ? builtins.currentSystem, }:
+
+with import ./common.nix;
+
+let
+  pkgsHost = import pkgsSrc { };
+  kaniko = (import ./kaniko.nix) pkgsHost;
+  winscp = (import ./winscp.nix) pkgsHost;
+  manifestTool = (import ./manifest-tool.nix) pkgsHost;
+in [ kaniko winscp manifestTool ]
+

shell.nix

@@ -5,35 +5,97 @@ with import ./nix/common.nix;
 let
   pkgs = import pkgsSrc {
     inherit system;
+    overlays = [ cargo2nixOverlay ];
   };
+  kaniko = (import ./nix/kaniko.nix) pkgs;
+  manifest-tool = (import ./nix/manifest-tool.nix) pkgs;
   winscp = (import ./nix/winscp.nix) pkgs;
-in
-{
-  # --- Dev shell inherited from flake.nix ---
-  devShell = devShells.default;
 
-  # --- Continuous integration shell ---
-  # The shell used for all CI jobs (along with devShell)
-  ci = pkgs.mkShell {
+in {
+  # --- Rust Shell ---
+  # Use it to compile Garage
+  rust = pkgs.mkShell {
     nativeBuildInputs = with pkgs; [
-      winscp
-
-      kaniko
-      manifest-tool
-      awscli2
+      #rustPlatform.rust.rustc
+      rustPlatform.rust.cargo
+      clang
+      mold
+      #clippy
+      rustfmt
+      #perl
+      #protobuf
+      #pkg-config
+      #openssl
       file
-      s3cmd
-      minio-client
-      rclone
-      socat
-      psmisc
-      which
-      openssl
-      curl
-      jq
+      #cargo2nix.packages.x86_64-linux.cargo2nix
     ];
+  };
+
+  # --- Integration shell ---
+  # Use it to test Garage with common S3 clients
+  integration = pkgs.mkShell {
+    nativeBuildInputs = [
+      winscp
+      pkgs.s3cmd
+      pkgs.awscli2
+      pkgs.minio-client
+      pkgs.rclone
+      pkgs.socat
+      pkgs.psmisc
+      pkgs.which
+      pkgs.openssl
+      pkgs.curl
+      pkgs.jq
+    ];
+  };
+
+  # --- Release shell ---
+  # A shell built to make releasing easier
+  release = pkgs.mkShell {
     shellHook = ''
+      function refresh_toolchain {
+        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+        nix copy \
+          --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+          $(nix-store -qR \
+            $(nix-build --no-build-output --no-out-link nix/toolchain.nix))
+        rm /tmp/nix-signing-key.sec
+      }
+
+      function refresh_cache {
+        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+        for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
+          echo "Updating cache for ''${attr}"
+          derivation=$(nix-instantiate --attr ''${attr})
+          nix copy -j8 \
+            --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+            $(nix-store -qR ''${derivation%\!bin})
+        done
+        rm /tmp/nix-signing-key.sec
+      }
+
+      function refresh_flake_cache {
+        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
+        for attr in packages.x86_64-linux.default devShell.x86_64-linux; do
+          echo "Updating cache for ''${attr}"
+          derivation=$(nix path-info --derivation ".#''${attr}")
+          nix copy -j8 \
+            --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
+            $(nix-store -qR ''${derivation})
+        done
+        rm /tmp/nix-signing-key.sec
+      }
+
       function to_s3 {
+        aws \
+            --endpoint-url https://garage.deuxfleurs.fr \
+            --region garage \
+          s3 cp \
+            ./result-bin/bin/garage \
+            s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage
+      }
+
+      function to_s3_woodpecker {
         aws \
             --endpoint-url https://garage.deuxfleurs.fr \
             --region garage \
@@ -45,8 +107,8 @@ in
       function to_docker {
         executor  \
           --force \
-          --customPlatform="$(echo "''${DOCKER_PLATFORM}" | sed 's/i386/386/')" \
-          --destination "$(echo "''${CONTAINER_NAME}" | sed 's/i386/386/'):''${CONTAINER_TAG}" \
+          --customPlatform="''${DOCKER_PLATFORM}" \
+          --destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \
           --context dir://`pwd` \
           --verbosity=debug
       }
@@ -105,25 +167,7 @@ in
             s3://garagehq.deuxfleurs.fr/
       }
     '';
-
-  };
-
-  # --- Cache shell ---
-  # A shell for refreshing caches
-  cache = pkgs.mkShell {
-    shellHook = ''
-      function refresh_cache {
-        pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
-        for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do
-          echo "Updating cache for ''${attr}"
-          nix copy -j8 \
-            --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
-            $(nix path-info ''${attr} --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/')
-
-        done
-        rm /tmp/nix-signing-key.sec
-      }
-    '';
+    nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ];
   };
 }