From 93f8d59e4c71e6ff2f945dc2c632536f4530b13c Mon Sep 17 00:00:00 2001 From: Quentin Dufour Date: Thu, 28 Oct 2021 10:04:14 +0200 Subject: [PATCH] Extract toolchain build from the CI --- .drone.yml | 452 ++++++++------------ Dockerfile | 2 +- doc/book/src/development/release_process.md | 13 +- nix/nix.conf | 3 - nix/toolchain.nix | 29 ++ shell.nix | 24 +- 6 files changed, 232 insertions(+), 291 deletions(-) create mode 100644 nix/toolchain.nix diff --git a/.drone.yml b/.drone.yml index 70e7fee0..96a3ae13 100644 --- a/.drone.yml +++ b/.drone.yml @@ -80,38 +80,6 @@ steps: - nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT - nix-shell --arg release false --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - - name: update cache - image: nixpkgs/nix:nixos-21.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: cache_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: cache_aws_secret_access_key - NIX_PRIV_KEY: - from_secret: nix_priv_key - volumes: - - name: nix_store - path: /nix - - name: nix_config - path: /etc/nix - commands: - - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec) - - | - nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR --include-outputs \ - $(nix-build --no-out-link shell.nix --arg release false -A inputDerivation)) - - | - nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR --include-outputs \ - $(nix-instantiate --argstr target x86_64-unknown-linux-musl --argstr compileMode test)) - - | - nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR --include-outputs \ - $(nix-instantiate --argstr target x86_64-unknown-linux-musl --arg release false)) - when: - event: - - cron - trigger: event: - custom @@ -212,27 +180,6 @@ steps: commands: - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - - name: update cache - image: nixpkgs/nix:nixos-21.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: cache_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: cache_aws_secret_access_key - NIX_PRIV_KEY: - from_secret: nix_priv_key - volumes: - - name: nix_store - path: /nix - - name: nix_config - path: /etc/nix - commands: - - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec) - - | - nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR --include-outputs \ - $(nix-instantiate --argstr target $TARGET --arg release true)) - - name: push static binary image: nixpkgs/nix:nixos-21.05 volumes: @@ -276,115 +223,94 @@ trigger: node: nix: 1 -# --- -# kind: pipeline -# type: docker -# name: release-linux-i686 -# -# volumes: -# - name: nix_store -# host: -# path: /var/lib/drone/nix -# - name: nix_config -# temp: {} -# -# environment: -# TARGET: i686-unknown-linux-musl -# -# steps: -# - name: setup nix -# image: nixpkgs/nix:nixos-21.05 -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# commands: -# - cp nix/nix.conf /etc/nix/nix.conf -# - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation -# -# - name: build -# image: nixpkgs/nix:nixos-21.05 -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# commands: -# - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT -# -# - name: integration -# image: nixpkgs/nix:nixos-21.05 -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# commands: -# - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) -# -# - name: update cache -# image: nixpkgs/nix:nixos-21.05 -# environment: -# AWS_ACCESS_KEY_ID: -# from_secret: cache_aws_access_key_id -# AWS_SECRET_ACCESS_KEY: -# from_secret: cache_aws_secret_access_key -# NIX_PRIV_KEY: -# from_secret: nix_priv_key -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# commands: -# - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec) -# - | -# nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ -# $(nix-store -qR --include-outputs \ -# $(nix-instantiate --argstr target $TARGET --arg release true)) -# -# - name: push static binary -# image: nixpkgs/nix:nixos-21.05 -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# environment: -# AWS_ACCESS_KEY_ID: -# from_secret: garagehq_aws_access_key_id -# AWS_SECRET_ACCESS_KEY: -# from_secret: garagehq_aws_secret_access_key -# commands: -# - nix-shell --arg rust false --arg integration false --run "to_s3" -# -# - name: docker build and publish -# image: nixpkgs/nix:nixos-21.05 -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# environment: -# DOCKER_AUTH: -# from_secret: docker_auth -# DOCKER_PLATFORM: "linux/386" -# CONTAINER_NAME: "dxflrs/386_garage" -# HOME: "/kaniko" -# commands: -# - mkdir -p /kaniko/.docker -# - echo $DOCKER_AUTH > /kaniko/.docker/config.json -# - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} -# - nix-shell --arg rust false --arg integration false --run "to_docker" -# -# trigger: -# event: -# - promote -# - cron -# -# node: -# nix: 1 +--- +kind: pipeline +type: docker +name: release-linux-i686 + +volumes: +- name: nix_store + host: + path: /var/lib/drone/nix +- name: nix_config + temp: {} + +environment: + TARGET: i686-unknown-linux-musl + +steps: + - name: setup nix + image: nixpkgs/nix:nixos-21.05 + volumes: + - name: nix_store + path: /nix + - name: nix_config + path: /etc/nix + commands: + - cp nix/nix.conf /etc/nix/nix.conf + - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation + + - name: build + image: nixpkgs/nix:nixos-21.05 + volumes: + - name: nix_store + path: /nix + - name: nix_config + path: /etc/nix + commands: + - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT + + - name: integration + image: nixpkgs/nix:nixos-21.05 + volumes: + - name: nix_store + path: /nix + - name: nix_config + path: /etc/nix + commands: + - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + + - name: push static binary + image: nixpkgs/nix:nixos-21.05 + volumes: + - name: nix_store + path: /nix + - name: nix_config + path: /etc/nix + environment: + AWS_ACCESS_KEY_ID: + from_secret: garagehq_aws_access_key_id + AWS_SECRET_ACCESS_KEY: + from_secret: garagehq_aws_secret_access_key + commands: + - nix-shell --arg rust false --arg integration false --run "to_s3" + + - name: docker build and publish + image: nixpkgs/nix:nixos-21.05 + volumes: + - name: nix_store + path: /nix + - name: nix_config + path: /etc/nix + environment: + DOCKER_AUTH: + from_secret: docker_auth + DOCKER_PLATFORM: "linux/386" + CONTAINER_NAME: "dxflrs/386_garage" + HOME: "/kaniko" + commands: + - mkdir -p /kaniko/.docker + - echo $DOCKER_AUTH > /kaniko/.docker/config.json + - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} + - nix-shell --arg rust false --arg integration false --run "to_docker" + +trigger: + event: + - promote + - cron + +node: + nix: 1 --- kind: pipeline @@ -423,27 +349,6 @@ steps: commands: - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT - - name: update cache - image: nixpkgs/nix:nixos-21.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: cache_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: cache_aws_secret_access_key - NIX_PRIV_KEY: - from_secret: nix_priv_key - volumes: - - name: nix_store - path: /nix - - name: nix_config - path: /etc/nix - commands: - - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec) - - | - nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR --include-outputs \ - $(nix-instantiate --argstr target $TARGET --arg release true)) - - name: push static binary image: nixpkgs/nix:nixos-21.05 volumes: @@ -486,105 +391,84 @@ trigger: node: nix: 1 -# --- -# kind: pipeline -# type: docker -# name: release-linux-armv6l -# -# volumes: -# - name: nix_store -# host: -# path: /var/lib/drone/nix -# - name: nix_config -# temp: {} -# -# environment: -# TARGET: armv6l-unknown-linux-musleabihf -# -# steps: -# - name: setup nix -# image: nixpkgs/nix:nixos-21.05 -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# commands: -# - cp nix/nix.conf /etc/nix/nix.conf -# - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation -# -# - name: build -# image: nixpkgs/nix:nixos-21.05 -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# commands: -# - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT -# -# - name: update cache -# image: nixpkgs/nix:nixos-21.05 -# environment: -# AWS_ACCESS_KEY_ID: -# from_secret: cache_aws_access_key_id -# AWS_SECRET_ACCESS_KEY: -# from_secret: cache_aws_secret_access_key -# NIX_PRIV_KEY: -# from_secret: nix_priv_key -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# commands: -# - (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec) -# - | -# nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ -# $(nix-store -qR --include-outputs \ -# $(nix-instantiate --argstr target $TARGET --arg release true)) -# -# - name: push static binary -# image: nixpkgs/nix:nixos-21.05 -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# environment: -# AWS_ACCESS_KEY_ID: -# from_secret: garagehq_aws_access_key_id -# AWS_SECRET_ACCESS_KEY: -# from_secret: garagehq_aws_secret_access_key -# commands: -# - nix-shell --arg integration false --arg rust false --run "to_s3" -# -# - name: docker build and publish -# image: nixpkgs/nix:nixos-21.05 -# volumes: -# - name: nix_store -# path: /nix -# - name: nix_config -# path: /etc/nix -# environment: -# DOCKER_AUTH: -# from_secret: docker_auth -# DOCKER_PLATFORM: "linux/arm" -# CONTAINER_NAME: "dxflrs/arm_garage" -# HOME: "/kaniko" -# commands: -# - mkdir -p /kaniko/.docker -# - echo $DOCKER_AUTH > /kaniko/.docker/config.json -# - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} -# - nix-shell --arg rust false --arg integration false --run "to_docker" -# -# trigger: -# event: -# - promote -# - cron -# -# node: -# nix: 1 +--- +kind: pipeline +type: docker +name: release-linux-armv6l + +volumes: +- name: nix_store + host: + path: /var/lib/drone/nix +- name: nix_config + temp: {} + +environment: + TARGET: armv6l-unknown-linux-musleabihf + +steps: + - name: setup nix + image: nixpkgs/nix:nixos-21.05 + volumes: + - name: nix_store + path: /nix + - name: nix_config + path: /etc/nix + commands: + - cp nix/nix.conf /etc/nix/nix.conf + - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation + + - name: build + image: nixpkgs/nix:nixos-21.05 + volumes: + - name: nix_store + path: /nix + - name: nix_config + path: /etc/nix + commands: + - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT + + - name: push static binary + image: nixpkgs/nix:nixos-21.05 + volumes: + - name: nix_store + path: /nix + - name: nix_config + path: /etc/nix + environment: + AWS_ACCESS_KEY_ID: + from_secret: garagehq_aws_access_key_id + AWS_SECRET_ACCESS_KEY: + from_secret: garagehq_aws_secret_access_key + commands: + - nix-shell --arg integration false --arg rust false --run "to_s3" + + - name: docker build and publish + image: nixpkgs/nix:nixos-21.05 + volumes: + - name: nix_store + path: /nix + - name: nix_config + path: /etc/nix + environment: + DOCKER_AUTH: + from_secret: docker_auth + DOCKER_PLATFORM: "linux/arm" + CONTAINER_NAME: "dxflrs/arm_garage" + HOME: "/kaniko" + commands: + - mkdir -p /kaniko/.docker + - echo $DOCKER_AUTH > /kaniko/.docker/config.json + - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} + - nix-shell --arg rust false --arg integration false --run "to_docker" + +trigger: + event: + - promote + - cron + +node: + nix: 1 --- kind: pipeline @@ -613,9 +497,9 @@ steps: depends_on: - release-linux-x86_64 - #- release-linux-i686 + - release-linux-i686 - release-linux-aarch64 - #- release-linux-armv6l + - release-linux-armv6l trigger: event: diff --git a/Dockerfile b/Dockerfile index 05d2d81d..2e301ee9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,4 +4,4 @@ ENV RUST_BACKTRACE=1 ENV RUST_LOG=garage=info COPY result/bin/garage / -CMD [ "/garage", "server", "-c", "config.toml"] +CMD [ "/garage", "server"] diff --git a/doc/book/src/development/release_process.md b/doc/book/src/development/release_process.md index 8591fd9f..e6f9e608 100644 --- a/doc/book/src/development/release_process.md +++ b/doc/book/src/development/release_process.md @@ -92,10 +92,21 @@ caching our development dependencies. *Currently there is no automatic garbage collection of the cache: we should monitor its growth. Hopefully, we can erase it totally without breaking any build, the next build will only be slower.* +In practise, we concluded that we do not want to cache all the compilation dependencies. +Instead, we want to cache the toolchain we use to build Garage each time we change it. +So we removed from Drone any automatic update of the cache and instead handle them manually with: + +``` +source ~/.awsrc +nix-shell --run 'refresh_toolchain' +``` + +Internally, it will run `nix-build` on `nix/toolchain.nix` and send the output plus its depedencies to the cache. + To erase the cache: ``` -mc rm --recursive --force 'garage/nix/*' +mc rm --recursive --force 'garage/nix/' ``` ### Publishing Garage diff --git a/nix/nix.conf b/nix/nix.conf index 8764eb3e..871efb10 100644 --- a/nix/nix.conf +++ b/nix/nix.conf @@ -2,6 +2,3 @@ substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs= max-jobs = auto cores = 4 - -# required for containers -sandbox = false diff --git a/nix/toolchain.nix b/nix/toolchain.nix new file mode 100644 index 00000000..e8baa63a --- /dev/null +++ b/nix/toolchain.nix @@ -0,0 +1,29 @@ +{ + system ? builtins.currentSystem, +}: + +with import ./common.nix; + +let + platforms = [ + "x86_64-unknown-linux-musl" + "i686-unknown-linux-musl" + "aarch64-unknown-linux-musl" + "armv6l-unknown-linux-musleabihf" + ]; + pkgsList = builtins.map (target: import pkgsSrc { + inherit system; + crossSystem = { config = target; }; + }) platforms; + pkgsHost = import pkgsSrc {}; + lib = pkgsHost.lib; + kaniko = (import ./kaniko.nix) pkgsHost; +in + lib.flatten (builtins.map (pkgs: [ + pkgs.rustPlatform.rust.rustc + pkgs.rustPlatform.rust.cargo + pkgs.buildPackages.stdenv.cc + ]) pkgsList) ++ [ + kaniko + ] + diff --git a/shell.nix b/shell.nix index a91a9e20..a4062f79 100644 --- a/shell.nix +++ b/shell.nix @@ -55,6 +55,13 @@ function refresh_index { result \ s3://garagehq.deuxfleurs.fr/_releases.html } + +function refresh_toolchain { + nix copy \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ + $(nix-store -qR \ + $(nix-build --quiet --no-build-output --no-out-link nix/toolchain.nix)) +} ''; nativeBuildInputs = @@ -66,8 +73,21 @@ function refresh_index { /*(pkgs.callPackage cargo2nix {}).package*/ ] else []) ++ - (if integration then [ pkgs.s3cmd pkgs.awscli2 pkgs.minio-client pkgs.rclone pkgs.socat pkgs.psmisc pkgs.which ] else []) + (if integration then [ + pkgs.s3cmd + pkgs.awscli2 + pkgs.minio-client + pkgs.rclone + pkgs.socat + pkgs.psmisc + pkgs.which + pkgs.openssl + pkgs.curl + ] else []) ++ - (if release then [ pkgs.awscli2 kaniko ] else []) + (if release then [ + pkgs.awscli2 + kaniko + ] else []) ; }