Extract toolchain build from the CI

.drone.yml

@@ -80,38 +80,6 @@ steps:
       - nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
       - nix-shell --arg release false --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
-  - name: update cache
-    image: nixpkgs/nix:nixos-21.05
-    environment:
-      AWS_ACCESS_KEY_ID:
-        from_secret: cache_aws_access_key_id
-      AWS_SECRET_ACCESS_KEY:
-        from_secret: cache_aws_secret_access_key
-      NIX_PRIV_KEY:
-        from_secret: nix_priv_key
-    volumes:
-    - name: nix_store
-      path: /nix
-    - name: nix_config
-      path: /etc/nix
-    commands:
-      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
-      - |
-        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-          $(nix-store -qR --include-outputs \
-            $(nix-build --no-out-link shell.nix --arg release false -A inputDerivation))
-      - |
-        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-          $(nix-store -qR --include-outputs \
-            $(nix-instantiate --argstr target x86_64-unknown-linux-musl --argstr compileMode test))
-      - |
-        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-          $(nix-store -qR --include-outputs \
-            $(nix-instantiate --argstr target x86_64-unknown-linux-musl --arg release false))
-    when:
-      event:
-      - cron
-
 trigger:
   event:
   - custom
@@ -212,26 +180,95 @@ steps:
     commands:
       - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
-  - name: update cache
+  - name: push static binary
     image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
     environment:
       AWS_ACCESS_KEY_ID:
-        from_secret: cache_aws_access_key_id
+        from_secret: garagehq_aws_access_key_id
       AWS_SECRET_ACCESS_KEY:
-        from_secret: cache_aws_secret_access_key
-      NIX_PRIV_KEY:
-        from_secret: nix_priv_key
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - nix-shell --arg rust false --arg integration false --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-21.05
     volumes:
     - name: nix_store
       path: /nix
     - name: nix_config
       path: /etc/nix
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/amd64"
+      CONTAINER_NAME: "dxflrs/amd64_garage"
+      HOME: "/kaniko"
     commands:
-      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
-      - |
-        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-          $(nix-store -qR --include-outputs \
-            $(nix-instantiate --argstr target $TARGET --arg release true))
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --arg rust false --arg integration false --run "to_docker"
+
+
+trigger:
+  event:
+  - promote
+  - cron
+
+node:
+  nix: 1
+
+---
+kind: pipeline
+type: docker
+name: release-linux-i686
+
+volumes:
+- name: nix_store
+  host:
+    path: /var/lib/drone/nix
+- name: nix_config
+  temp: {}
+
+environment:
+  TARGET: i686-unknown-linux-musl
+
+steps:
+  - name: setup nix
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - cp nix/nix.conf /etc/nix/nix.conf
+      - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
+
+  - name: build
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
+
+  - name: integration
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
 
   - name: push static binary
     image: nixpkgs/nix:nixos-21.05
@@ -258,8 +295,8 @@ steps:
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth
-      DOCKER_PLATFORM: "linux/amd64"
-      CONTAINER_NAME: "dxflrs/amd64_garage"
+      DOCKER_PLATFORM: "linux/386"
+      CONTAINER_NAME: "dxflrs/386_garage"
       HOME: "/kaniko"
     commands:
       - mkdir -p /kaniko/.docker
@@ -267,7 +304,6 @@ steps:
       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
       - nix-shell --arg rust false --arg integration false --run "to_docker"
 
-
 trigger:
   event:
   - promote
@@ -276,116 +312,6 @@ trigger:
 node:
   nix: 1
 
-# ---
-# kind: pipeline
-# type: docker
-# name: release-linux-i686
-# 
-# volumes:
-# - name: nix_store
-#   host:
-#     path: /var/lib/drone/nix
-# - name: nix_config
-#   temp: {}
-# 
-# environment:
-#   TARGET: i686-unknown-linux-musl
-# 
-# steps:
-#   - name: setup nix
-#     image: nixpkgs/nix:nixos-21.05
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     commands:
-#       - cp nix/nix.conf /etc/nix/nix.conf
-#       - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
-# 
-#   - name: build
-#     image: nixpkgs/nix:nixos-21.05
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     commands:
-#       - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
-# 
-#   - name: integration
-#     image: nixpkgs/nix:nixos-21.05
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     commands:
-#       - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
-# 
-#   - name: update cache
-#     image: nixpkgs/nix:nixos-21.05
-#     environment:
-#       AWS_ACCESS_KEY_ID:
-#         from_secret: cache_aws_access_key_id
-#       AWS_SECRET_ACCESS_KEY:
-#         from_secret: cache_aws_secret_access_key
-#       NIX_PRIV_KEY:
-#         from_secret: nix_priv_key
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     commands:
-#       - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
-#       - |
-#         nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-#           $(nix-store -qR --include-outputs \
-#             $(nix-instantiate --argstr target $TARGET --arg release true))
-# 
-#   - name: push static binary
-#     image: nixpkgs/nix:nixos-21.05
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     environment:
-#       AWS_ACCESS_KEY_ID:
-#         from_secret: garagehq_aws_access_key_id
-#       AWS_SECRET_ACCESS_KEY:
-#         from_secret: garagehq_aws_secret_access_key
-#     commands:
-#       - nix-shell --arg rust false --arg integration false --run "to_s3"
-# 
-#   - name: docker build and publish
-#     image: nixpkgs/nix:nixos-21.05
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     environment:
-#       DOCKER_AUTH:
-#         from_secret: docker_auth
-#       DOCKER_PLATFORM: "linux/386"
-#       CONTAINER_NAME: "dxflrs/386_garage"
-#       HOME: "/kaniko"
-#     commands:
-#       - mkdir -p /kaniko/.docker
-#       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
-#       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-#       - nix-shell --arg rust false --arg integration false --run "to_docker"
-# 
-# trigger:
-#   event:
-#   - promote
-#   - cron
-# 
-# node:
-#   nix: 1
-
 ---
 kind: pipeline
 type: docker
@@ -423,26 +349,84 @@ steps:
     commands:
       - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
 
-  - name: update cache
+  - name: push static binary
     image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
     environment:
       AWS_ACCESS_KEY_ID:
-        from_secret: cache_aws_access_key_id
+        from_secret: garagehq_aws_access_key_id
       AWS_SECRET_ACCESS_KEY:
-        from_secret: cache_aws_secret_access_key
-      NIX_PRIV_KEY:
-        from_secret: nix_priv_key
+        from_secret: garagehq_aws_secret_access_key
+    commands:
+      - nix-shell --arg rust false --arg integration false --run "to_s3"
+
+  - name: docker build and publish
+    image: nixpkgs/nix:nixos-21.05
     volumes:
     - name: nix_store
       path: /nix
     - name: nix_config
       path: /etc/nix
+    environment:
+      DOCKER_AUTH:
+        from_secret: docker_auth
+      DOCKER_PLATFORM: "linux/arm64"
+      CONTAINER_NAME: "dxflrs/arm64_garage"
+      HOME: "/kaniko"
     commands:
-      - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
-      - |
-        nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-          $(nix-store -qR --include-outputs \
-            $(nix-instantiate --argstr target $TARGET --arg release true))
+      - mkdir -p /kaniko/.docker
+      - echo $DOCKER_AUTH > /kaniko/.docker/config.json
+      - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
+      - nix-shell --arg rust false --arg integration false --run "to_docker"
+
+trigger:
+  event:
+  - promote
+  - cron
+
+node:
+  nix: 1
+
+---
+kind: pipeline
+type: docker
+name: release-linux-armv6l
+
+volumes:
+- name: nix_store
+  host:
+    path: /var/lib/drone/nix
+- name: nix_config
+  temp: {}
+
+environment:
+  TARGET: armv6l-unknown-linux-musleabihf
+
+steps:
+  - name: setup nix
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - cp nix/nix.conf /etc/nix/nix.conf
+      - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
+
+  - name: build
+    image: nixpkgs/nix:nixos-21.05
+    volumes:
+    - name: nix_store
+      path: /nix
+    - name: nix_config
+      path: /etc/nix
+    commands:
+      - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
 
   - name: push static binary
     image: nixpkgs/nix:nixos-21.05
@@ -457,7 +441,7 @@ steps:
       AWS_SECRET_ACCESS_KEY:
         from_secret: garagehq_aws_secret_access_key
     commands:
-      - nix-shell --arg rust false --arg integration false --run "to_s3"
+      - nix-shell --arg integration false --arg rust false --run "to_s3"
 
   - name: docker build and publish
     image: nixpkgs/nix:nixos-21.05
@@ -469,8 +453,8 @@ steps:
     environment:
       DOCKER_AUTH:
         from_secret: docker_auth
-      DOCKER_PLATFORM: "linux/arm64"
-      CONTAINER_NAME: "dxflrs/arm64_garage"
+      DOCKER_PLATFORM: "linux/arm"
+      CONTAINER_NAME: "dxflrs/arm_garage"
       HOME: "/kaniko"
     commands:
       - mkdir -p /kaniko/.docker
@@ -486,106 +470,6 @@ trigger:
 node:
   nix: 1
 
-# ---
-# kind: pipeline
-# type: docker
-# name: release-linux-armv6l
-# 
-# volumes:
-# - name: nix_store
-#   host:
-#     path: /var/lib/drone/nix
-# - name: nix_config
-#   temp: {}
-# 
-# environment:
-#   TARGET: armv6l-unknown-linux-musleabihf
-# 
-# steps:
-#   - name: setup nix
-#     image: nixpkgs/nix:nixos-21.05
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     commands:
-#       - cp nix/nix.conf /etc/nix/nix.conf
-#       - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
-# 
-#   - name: build
-#     image: nixpkgs/nix:nixos-21.05
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     commands:
-#       - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
-# 
-#   - name: update cache
-#     image: nixpkgs/nix:nixos-21.05
-#     environment:
-#       AWS_ACCESS_KEY_ID:
-#         from_secret: cache_aws_access_key_id
-#       AWS_SECRET_ACCESS_KEY:
-#         from_secret: cache_aws_secret_access_key
-#       NIX_PRIV_KEY:
-#         from_secret: nix_priv_key
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     commands:
-#       - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
-#       - |
-#         nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
-#           $(nix-store -qR --include-outputs \
-#             $(nix-instantiate --argstr target $TARGET --arg release true))
-# 
-#   - name: push static binary
-#     image: nixpkgs/nix:nixos-21.05
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     environment:
-#       AWS_ACCESS_KEY_ID:
-#         from_secret: garagehq_aws_access_key_id
-#       AWS_SECRET_ACCESS_KEY:
-#         from_secret: garagehq_aws_secret_access_key
-#     commands:
-#       - nix-shell --arg integration false --arg rust false --run "to_s3"
-# 
-#   - name: docker build and publish
-#     image: nixpkgs/nix:nixos-21.05
-#     volumes:
-#     - name: nix_store
-#       path: /nix
-#     - name: nix_config
-#       path: /etc/nix
-#     environment:
-#       DOCKER_AUTH:
-#         from_secret: docker_auth
-#       DOCKER_PLATFORM: "linux/arm"
-#       CONTAINER_NAME: "dxflrs/arm_garage"
-#       HOME: "/kaniko"
-#     commands:
-#       - mkdir -p /kaniko/.docker
-#       - echo $DOCKER_AUTH > /kaniko/.docker/config.json
-#       - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
-#       - nix-shell --arg rust false --arg integration false --run "to_docker"
-# 
-# trigger:
-#   event:
-#   - promote
-#   - cron
-# 
-# node:
-#   nix: 1
-
 ---
 kind: pipeline
 type: docker
@@ -613,9 +497,9 @@ steps:
 
 depends_on:
   - release-linux-x86_64
-  #- release-linux-i686
+  - release-linux-i686
   - release-linux-aarch64
-  #- release-linux-armv6l
+  - release-linux-armv6l
 
 trigger:
   event:

Dockerfile

@@ -4,4 +4,4 @@ ENV RUST_BACKTRACE=1
 ENV RUST_LOG=garage=info
 
 COPY result/bin/garage /
-CMD [ "/garage", "server", "-c", "config.toml"]
+CMD [ "/garage", "server"]

doc/book/src/development/release_process.md

@@ -92,10 +92,21 @@ caching our development dependencies.
 *Currently there is no automatic garbage collection of the cache: we should monitor its growth.
 Hopefully, we can erase it totally without breaking any build, the next build will only be slower.*
 
+In practise, we concluded that we do not want to cache all the compilation dependencies.
+Instead, we want to cache the toolchain we use to build Garage each time we change it.
+So we removed from Drone any automatic update of the cache and instead handle them manually with:
+
+```
+source ~/.awsrc
+nix-shell --run 'refresh_toolchain'
+```
+
+Internally, it will run `nix-build` on  `nix/toolchain.nix` and send the output plus its depedencies to the cache.
+
 To erase the cache:
 
 ```
-mc rm --recursive --force 'garage/nix/*'
+mc rm --recursive --force 'garage/nix/'
 ```
 
 ### Publishing Garage

nix/nix.conf

@@ -2,6 +2,3 @@ substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
 trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
 max-jobs = auto
 cores = 4
-
-# required for containers
-sandbox = false

nix/toolchain.nix

@@ -0,0 +1,29 @@
+{
+  system ? builtins.currentSystem,
+}:
+
+with import ./common.nix;
+
+let
+  platforms = [
+    "x86_64-unknown-linux-musl"
+    "i686-unknown-linux-musl"
+    "aarch64-unknown-linux-musl"
+    "armv6l-unknown-linux-musleabihf"
+  ];
+  pkgsList = builtins.map (target: import pkgsSrc { 
+    inherit system; 
+    crossSystem = { config = target; };
+  }) platforms;
+  pkgsHost = import pkgsSrc {};
+  lib = pkgsHost.lib;
+  kaniko = (import ./kaniko.nix) pkgsHost;
+in 
+  lib.flatten (builtins.map (pkgs: [
+     pkgs.rustPlatform.rust.rustc
+     pkgs.rustPlatform.rust.cargo
+     pkgs.buildPackages.stdenv.cc
+  ]) pkgsList) ++ [
+    kaniko
+  ]
+

shell.nix

@@ -55,6 +55,13 @@ function refresh_index {
       result \
       s3://garagehq.deuxfleurs.fr/_releases.html
 }
+
+function refresh_toolchain {
+  nix copy \
+    --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
+    $(nix-store -qR \
+      $(nix-build --quiet --no-build-output --no-out-link nix/toolchain.nix))
+}
   '';
 
   nativeBuildInputs = 
@@ -66,8 +73,21 @@ function refresh_index {
      /*(pkgs.callPackage cargo2nix {}).package*/
     ] else [])
    ++
-   (if integration then [ pkgs.s3cmd pkgs.awscli2 pkgs.minio-client pkgs.rclone pkgs.socat pkgs.psmisc pkgs.which ] else [])
+   (if integration then [ 
+     pkgs.s3cmd
+     pkgs.awscli2
+     pkgs.minio-client
+     pkgs.rclone
+     pkgs.socat
+     pkgs.psmisc
+     pkgs.which
+     pkgs.openssl
+     pkgs.curl
+    ] else [])
    ++
-   (if release then [ pkgs.awscli2 kaniko ] else [])
+   (if release then [ 
+     pkgs.awscli2
+     kaniko 
+    ] else [])
    ;
 }