From 9900368380513d8b898d6ac4116e09525341c11b Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Wed, 7 Feb 2024 19:23:32 +0100 Subject: [PATCH 1/7] [nix-improvements] modernize Nix infrastructure --- .drone.yml | 42 ++++++++-------- flake.nix | 74 +++++++++++++++++++-------- nix/common.nix | 1 + shell.nix | 133 ++++++++++++++++++++++--------------------------- 4 files changed, 134 insertions(+), 116 deletions(-) diff --git a/.drone.yml b/.drone.yml index d79a9277..16a19917 100644 --- a/.drone.yml +++ b/.drone.yml @@ -9,7 +9,7 @@ steps: - name: check formatting image: nixpkgs/nix:nixos-22.05 commands: - - nix-shell --attr rust --run "cargo fmt -- --check" + - nix-shell --attr devShell --run "cargo fmt -- --check" - name: build image: nixpkgs/nix:nixos-22.05 @@ -40,7 +40,7 @@ steps: image: nixpkgs/nix:nixos-22.05 commands: - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) trigger: event: @@ -63,17 +63,17 @@ steps: image: nixpkgs/nix:nixos-22.05 commands: - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" + - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage" - name: integration tests image: nixpkgs/nix:nixos-22.05 commands: - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - name: upgrade tests image: nixpkgs/nix:nixos-22.05 commands: - - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) + - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) - name: push static binary image: nixpkgs/nix:nixos-22.05 @@ -84,7 +84,7 @@ steps: from_secret: garagehq_aws_secret_access_key TARGET: "x86_64-unknown-linux-musl" commands: - - nix-shell --attr release --run "to_s3" + - nix-shell --attr ci --run "to_s3" - name: docker build and publish image: nixpkgs/nix:nixos-22.05 @@ -98,7 +98,7 @@ steps: - mkdir -p /kaniko/.docker - echo $DOCKER_AUTH > /kaniko/.docker/config.json - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" + - nix-shell --attr ci --run "to_docker" trigger: @@ -119,17 +119,17 @@ steps: image: nixpkgs/nix:nixos-22.05 commands: - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" + - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage" - name: integration tests image: nixpkgs/nix:nixos-22.05 commands: - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - name: upgrade tests image: nixpkgs/nix:nixos-22.05 commands: - - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false) + - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false) - name: push static binary image: nixpkgs/nix:nixos-22.05 @@ -140,7 +140,7 @@ steps: from_secret: garagehq_aws_secret_access_key TARGET: "i686-unknown-linux-musl" commands: - - nix-shell --attr release --run "to_s3" + - nix-shell --attr ci --run "to_s3" - name: docker build and publish image: nixpkgs/nix:nixos-22.05 @@ -154,7 +154,7 @@ steps: - mkdir -p /kaniko/.docker - echo $DOCKER_AUTH > /kaniko/.docker/config.json - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" + - nix-shell --attr ci --run "to_docker" trigger: event: @@ -174,7 +174,7 @@ steps: image: nixpkgs/nix:nixos-22.05 commands: - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" + - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage" - name: push static binary image: nixpkgs/nix:nixos-22.05 @@ -185,7 +185,7 @@ steps: from_secret: garagehq_aws_secret_access_key TARGET: "aarch64-unknown-linux-musl" commands: - - nix-shell --attr release --run "to_s3" + - nix-shell --attr ci --run "to_s3" - name: docker build and publish image: nixpkgs/nix:nixos-22.05 @@ -199,7 +199,7 @@ steps: - mkdir -p /kaniko/.docker - echo $DOCKER_AUTH > /kaniko/.docker/config.json - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" + - nix-shell --attr ci --run "to_docker" trigger: event: @@ -219,7 +219,7 @@ steps: image: nixpkgs/nix:nixos-22.05 commands: - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" + - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage" - name: push static binary image: nixpkgs/nix:nixos-22.05 @@ -230,7 +230,7 @@ steps: from_secret: garagehq_aws_secret_access_key TARGET: "armv6l-unknown-linux-musleabihf" commands: - - nix-shell --attr release --run "to_s3" + - nix-shell --attr ci --run "to_s3" - name: docker build and publish image: nixpkgs/nix:nixos-22.05 @@ -244,7 +244,7 @@ steps: - mkdir -p /kaniko/.docker - echo $DOCKER_AUTH > /kaniko/.docker/config.json - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "to_docker" + - nix-shell --attr ci --run "to_docker" trigger: event: @@ -270,7 +270,7 @@ steps: - mkdir -p /root/.docker - echo $DOCKER_AUTH > /root/.docker/config.json - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr release --run "multiarch_docker" + - nix-shell --attr ci --run "multiarch_docker" - name: refresh-index image: nixpkgs/nix:nixos-22.05 environment: @@ -280,7 +280,7 @@ steps: from_secret: garagehq_aws_secret_access_key commands: - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf - - nix-shell --attr release --run "refresh_index" + - nix-shell --attr ci --run "refresh_index" depends_on: - release-linux-amd64 @@ -295,6 +295,6 @@ trigger: --- kind: signature -hmac: 0c4b57eb4b27b7c6a6ff21ab87f0767fe3eb90f5d95d5cbcdccf794e9d2a5d86 +hmac: 9ff9f2ad9387b12ee49a327991a5373f01e40d28e0aeef3a13d3b741d8eb3c53 ... diff --git a/flake.nix b/flake.nix index 236b2310..06c74220 100644 --- a/flake.nix +++ b/flake.nix @@ -33,27 +33,57 @@ compile = import ./nix/compile.nix; in flake-utils.lib.eachDefaultSystem (system: - let pkgs = nixpkgs.legacyPackages.${system}; - in { - packages = { - default = (compile { - inherit system git_version; - pkgsSrc = nixpkgs; - cargo2nixOverlay = cargo2nix.overlays.default; - release = true; - }).workspace.garage { compileMode = "build"; }; - }; - devShell = (compile { - inherit system git_version; - pkgsSrc = nixpkgs; - cargo2nixOverlay = cargo2nix.overlays.default; - release = false; - }).workspaceShell { packages = with pkgs; [ - cargo-audit - cargo-outdated - rustfmt - clang - mold - ]; }; + let + pkgs = nixpkgs.legacyPackages.${system}; + in + { + packages = + let + packageFor = target: (compile { + inherit system git_version target; + pkgsSrc = nixpkgs; + cargo2nixOverlay = cargo2nix.overlays.default; + release = true; + }).workspace.garage { compileMode = "build"; }; + in + { + # default = native release build + default = packageFor null; + # other = cross-compiled, statically-linked builds + amd64 = packageFor "x86_64-unknown-linux-musl"; + i386 = packageFor "i686-unknown-linux-musl"; + arm64 = packageFor "aarch64-unknown-linux-musl"; + arm = packageFor "armv6l-unknown-linux-musl"; + }; + + # ---- developpment shell, for making native builds only ---- + devShells = + let + shellWithPackages = (packages: (compile { + inherit system git_version; + pkgsSrc = nixpkgs; + cargo2nixOverlay = cargo2nix.overlays.default; + }).workspaceShell { inherit packages; }); + in + { + default = shellWithPackages + (with pkgs; [ + rustfmt + clang + mold + ]); + + # import the full shell using `nix develop .#full` + full = shellWithPackages (with pkgs; [ + rustfmt + clang + mold + # ---- extra packages for dev tasks ---- + cargo-audit + cargo-outdated + cargo-machete + nixpkgs-fmt + ]); + }; }); } diff --git a/nix/common.nix b/nix/common.nix index 323c3882..1ad809bb 100644 --- a/nix/common.nix +++ b/nix/common.nix @@ -14,4 +14,5 @@ rec { pkgsSrc = flake.defaultNix.inputs.nixpkgs; cargo2nix = flake.defaultNix.inputs.cargo2nix; cargo2nixOverlay = cargo2nix.overlays.default; + devShells = builtins.getAttr builtins.currentSystem flake.defaultNix.devShells; } diff --git a/shell.nix b/shell.nix index 6d6cc68f..bd150743 100644 --- a/shell.nix +++ b/shell.nix @@ -5,87 +5,36 @@ with import ./nix/common.nix; let pkgs = import pkgsSrc { inherit system; - overlays = [ cargo2nixOverlay ]; }; kaniko = (import ./nix/kaniko.nix) pkgs; manifest-tool = (import ./nix/manifest-tool.nix) pkgs; winscp = (import ./nix/winscp.nix) pkgs; +in +{ + # --- Dev shell inherited from flake.nix --- + devShell = devShells.default; -in { - # --- Rust Shell --- - # Use it to compile Garage - rust = pkgs.mkShell { + # --- Continuous integration shell --- + # The shell used for all CI jobs (along with devShell) + ci = pkgs.mkShell { nativeBuildInputs = with pkgs; [ - #rustPlatform.rust.rustc - rustPlatform.rust.cargo - clang - mold - #clippy - rustfmt - #perl - #protobuf - #pkg-config - #openssl - file - #cargo2nix.packages.x86_64-linux.cargo2nix - ]; - }; - - # --- Integration shell --- - # Use it to test Garage with common S3 clients - integration = pkgs.mkShell { - nativeBuildInputs = [ + kaniko + manifest-tool winscp - pkgs.s3cmd - pkgs.awscli2 - pkgs.minio-client - pkgs.rclone - pkgs.socat - pkgs.psmisc - pkgs.which - pkgs.openssl - pkgs.curl - pkgs.jq + + awscli2 + file + s3cmd + minio-client + rclone + socat + psmisc + which + openssl + curl + jq ]; - }; - - # --- Release shell --- - # A shell built to make releasing easier - release = pkgs.mkShell { shellHook = '' - function refresh_toolchain { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - nix copy \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR \ - $(nix-build --no-build-output --no-out-link nix/toolchain.nix)) - rm /tmp/nix-signing-key.sec - } - - function refresh_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do - echo "Updating cache for ''${attr}" - derivation=$(nix-instantiate --attr ''${attr}) - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation%\!bin}) - done - rm /tmp/nix-signing-key.sec - } - - function refresh_flake_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in packages.x86_64-linux.default devShell.x86_64-linux; do - echo "Updating cache for ''${attr}" - derivation=$(nix path-info --derivation ".#''${attr}") - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR ''${derivation}) - done - rm /tmp/nix-signing-key.sec - } - function to_s3 { aws \ --endpoint-url https://garage.deuxfleurs.fr \ @@ -167,7 +116,45 @@ in { s3://garagehq.deuxfleurs.fr/ } ''; - nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ]; + + }; + + # --- Cache shell --- + # A shell for refreshing caches + cache = pkgs.mkShell { + shellHook = '' + function refresh_toolchain { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix-store -qR \ + $(nix-build -j8 --no-build-output --no-out-link nix/toolchain.nix)) + rm /tmp/nix-signing-key.sec + } + + function refresh_cache { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do + echo "Updating cache for ''${attr}" + nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + $(nix path-info ''${attr} --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/') + + done + rm /tmp/nix-signing-key.sec + } + + function refresh_flake_cache { + pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec + for attr in packages.x86_64-linux.default devShells.x86_64-linux.default; do + echo "Updating cache for ''${attr}" + nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ + ".#''${attr}" + done + rm /tmp/nix-signing-key.sec + } + ''; }; } From d35d4599de13d2bad95523547bb368f1c754a014 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 9 Feb 2024 11:15:52 +0100 Subject: [PATCH 2/7] [nix-improvements] use kaniko and manifest-tools from nixpkgs, simplify --- nix/kaniko.nix | 24 ------------------------ nix/manifest-tool.nix | 24 ------------------------ nix/toolchain.nix | 11 ----------- shell.nix | 26 ++------------------------ 4 files changed, 2 insertions(+), 83 deletions(-) delete mode 100644 nix/kaniko.nix delete mode 100644 nix/manifest-tool.nix delete mode 100644 nix/toolchain.nix diff --git a/nix/kaniko.nix b/nix/kaniko.nix deleted file mode 100644 index 8380fa2f..00000000 --- a/nix/kaniko.nix +++ /dev/null @@ -1,24 +0,0 @@ -pkgs: -pkgs.buildGoModule rec { - pname = "kaniko"; - version = "1.9.2"; - - src = pkgs.fetchFromGitHub { - owner = "GoogleContainerTools"; - repo = "kaniko"; - rev = "v${version}"; - sha256 = "dXQ0/o1qISv+sjNVIpfF85bkbM9sGOGwqVbWZpMWfMY="; - }; - - vendorSha256 = null; - - checkPhase = "true"; - - meta = with pkgs.lib; { - description = - "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster."; - homepage = "https://github.com/GoogleContainerTools/kaniko"; - license = licenses.asl20; - platforms = platforms.linux; - }; -} diff --git a/nix/manifest-tool.nix b/nix/manifest-tool.nix deleted file mode 100644 index 1090a6ef..00000000 --- a/nix/manifest-tool.nix +++ /dev/null @@ -1,24 +0,0 @@ -pkgs: -pkgs.buildGoModule rec { - pname = "manifest-tool"; - version = "2.0.5"; - - src = pkgs.fetchFromGitHub { - owner = "estesp"; - repo = "manifest-tool"; - rev = "v${version}"; - sha256 = "hjCGKnE0yrlnF/VIzOwcDzmQX3Wft+21KCny/opqdLg="; - } + "/v2"; - - vendorSha256 = null; - - checkPhase = "true"; - - meta = with pkgs.lib; { - description = - "Command line tool to create and query container image manifest list/indexes"; - homepage = "https://github.com/estesp/manifest-tool"; - license = licenses.asl20; - platforms = platforms.linux; - }; -} diff --git a/nix/toolchain.nix b/nix/toolchain.nix deleted file mode 100644 index c4fef9ab..00000000 --- a/nix/toolchain.nix +++ /dev/null @@ -1,11 +0,0 @@ -{ system ? builtins.currentSystem, }: - -with import ./common.nix; - -let - pkgsHost = import pkgsSrc { }; - kaniko = (import ./kaniko.nix) pkgsHost; - winscp = (import ./winscp.nix) pkgsHost; - manifestTool = (import ./manifest-tool.nix) pkgsHost; -in [ kaniko winscp manifestTool ] - diff --git a/shell.nix b/shell.nix index bd150743..193ed941 100644 --- a/shell.nix +++ b/shell.nix @@ -6,8 +6,6 @@ let pkgs = import pkgsSrc { inherit system; }; - kaniko = (import ./nix/kaniko.nix) pkgs; - manifest-tool = (import ./nix/manifest-tool.nix) pkgs; winscp = (import ./nix/winscp.nix) pkgs; in { @@ -18,10 +16,10 @@ in # The shell used for all CI jobs (along with devShell) ci = pkgs.mkShell { nativeBuildInputs = with pkgs; [ - kaniko - manifest-tool winscp + kaniko + manifest-tool awscli2 file s3cmd @@ -123,15 +121,6 @@ in # A shell for refreshing caches cache = pkgs.mkShell { shellHook = '' - function refresh_toolchain { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - $(nix-store -qR \ - $(nix-build -j8 --no-build-output --no-out-link nix/toolchain.nix)) - rm /tmp/nix-signing-key.sec - } - function refresh_cache { pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do @@ -143,17 +132,6 @@ in done rm /tmp/nix-signing-key.sec } - - function refresh_flake_cache { - pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec - for attr in packages.x86_64-linux.default devShells.x86_64-linux.default; do - echo "Updating cache for ''${attr}" - nix copy -j8 \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/tmp/nix-signing-key.sec' \ - ".#''${attr}" - done - rm /tmp/nix-signing-key.sec - } ''; }; } From 1c85e5e428d8b449bb0e0933f329bcba3b583697 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 9 Feb 2024 11:19:32 +0100 Subject: [PATCH 3/7] [nix-improvements] adapt woodpecker pipelines --- .woodpecker/debug.yaml | 4 ++-- .woodpecker/publish.yaml | 4 ++-- .woodpecker/release.yaml | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.woodpecker/debug.yaml b/.woodpecker/debug.yaml index 9486f672..59ae8c06 100644 --- a/.woodpecker/debug.yaml +++ b/.woodpecker/debug.yaml @@ -10,7 +10,7 @@ steps: - name: check formatting image: nixpkgs/nix:nixos-22.05 commands: - - nix-shell --attr rust --run "cargo fmt -- --check" + - nix-shell --attr devShell --run "cargo fmt -- --check" - name: build image: nixpkgs/nix:nixos-22.05 @@ -41,4 +41,4 @@ steps: image: nixpkgs/nix:nixos-22.05 commands: - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) diff --git a/.woodpecker/publish.yaml b/.woodpecker/publish.yaml index b45d379c..17b33d7f 100644 --- a/.woodpecker/publish.yaml +++ b/.woodpecker/publish.yaml @@ -16,7 +16,7 @@ steps: target: AWS_SECRET_ACCESS_KEY commands: - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf - - nix-shell --attr release --run "refresh_index" + - nix-shell --attr ci --run "refresh_index" # - name: multiarch-docker # image: nixpkgs/nix:nixos-22.05 @@ -29,4 +29,4 @@ steps: # - echo $DOCKER_AUTH > /root/.docker/config.json # - sha512sum /root/.docker/config.json # - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} -# - nix-shell --attr release --run "multiarch_docker" +# - nix-shell --attr ci --run "multiarch_docker" diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml index 8580441c..1d54e3cf 100644 --- a/.woodpecker/release.yaml +++ b/.woodpecker/release.yaml @@ -19,12 +19,12 @@ steps: image: nixpkgs/nix:nixos-22.05 commands: - nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} - - nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" + - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage" - name: integration tests image: nixpkgs/nix:nixos-22.05 commands: - - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) when: - matrix: ARCH: amd64 @@ -34,7 +34,7 @@ steps: - name: upgrade tests image: nixpkgs/nix:nixos-22.05 commands: - - nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) + - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) when: - matrix: ARCH: amd64 @@ -49,7 +49,7 @@ steps: - source: garagehq_aws_secret_access_key target: AWS_SECRET_ACCESS_KEY commands: - - nix-shell --attr release --run "to_s3_woodpecker" + - nix-shell --attr ci --run "to_s3_woodpecker" # - name: docker build and publish # image: nixpkgs/nix:nixos-22.05 @@ -64,4 +64,4 @@ steps: # - echo $DOCKER_AUTH > /kaniko/.docker/config.json # - sha512sum /kaniko/.docker/config.json # - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} -# - nix-shell --attr release --run "to_docker" +# - nix-shell --attr ci --run "to_docker" From 555ed75548d2f42f9d86cc2b8d6844e6b7d35740 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 9 Feb 2024 11:36:51 +0100 Subject: [PATCH 4/7] [nix-improvements] ci: check static as separate step --- .woodpecker/release.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml index 1d54e3cf..e5db48b8 100644 --- a/.woodpecker/release.yaml +++ b/.woodpecker/release.yaml @@ -16,6 +16,11 @@ matrix: steps: - name: build + image: nixpkgs/nix:nixos-22.05 + commands: + - nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + + - name: check is static binary image: nixpkgs/nix:nixos-22.05 commands: - nix-build --no-build-output --attr pkgs.${ARCH}.release --argstr git_version ${CI_COMMIT_TAG:-$CI_COMMIT_SHA} From 1be75fbf4e750d03e1f091016ef44814da674126 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 9 Feb 2024 11:40:35 +0100 Subject: [PATCH 5/7] [nix-improvements] fix kaniko and manifest-tool --- .woodpecker/publish.yaml | 21 +++++++++------------ .woodpecker/release.yaml | 26 ++++++++++++-------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/.woodpecker/publish.yaml b/.woodpecker/publish.yaml index 17b33d7f..739af2c4 100644 --- a/.woodpecker/publish.yaml +++ b/.woodpecker/publish.yaml @@ -18,15 +18,12 @@ steps: - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf - nix-shell --attr ci --run "refresh_index" -# - name: multiarch-docker -# image: nixpkgs/nix:nixos-22.05 -# environment: -# HOME: "/root" -# secrets: -# - docker_auth -# commands: -# - mkdir -p /root/.docker -# - echo $DOCKER_AUTH > /root/.docker/config.json -# - sha512sum /root/.docker/config.json -# - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} -# - nix-shell --attr ci --run "multiarch_docker" + - name: multiarch-docker + image: nixpkgs/nix:nixos-22.05 + secrets: + - docker_auth + commands: + - mkdir -p /root/.docker + - echo $DOCKER_AUTH > /root/.docker/config.json + - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + - nix-shell --attr ci --run "multiarch_docker" diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml index e5db48b8..8f19be82 100644 --- a/.woodpecker/release.yaml +++ b/.woodpecker/release.yaml @@ -56,17 +56,15 @@ steps: commands: - nix-shell --attr ci --run "to_s3_woodpecker" -# - name: docker build and publish -# image: nixpkgs/nix:nixos-22.05 -# environment: -# DOCKER_PLATFORM: "linux/${ARCH}" -# CONTAINER_NAME: "dxflrs/${ARCH}_garage" -# HOME: "/kaniko" -# secrets: -# - docker_auth -# commands: -# - mkdir -p /kaniko/.docker -# - echo $DOCKER_AUTH > /kaniko/.docker/config.json -# - sha512sum /kaniko/.docker/config.json -# - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} -# - nix-shell --attr ci --run "to_docker" + - name: docker build and publish + image: nixpkgs/nix:nixos-22.05 + environment: + DOCKER_PLATFORM: "linux/${ARCH}" + CONTAINER_NAME: "dxflrs/${ARCH}_garage" + secrets: + - docker_auth + commands: + - mkdir -p /root/.docker + - echo $DOCKER_AUTH > /root/.docker/config.json + - export CONTAINER_TAG=${CI_COMMIT_TAG:-$CI_COMMIT_SHA} + - nix-shell --attr ci --run "to_docker" From 561fad0b44f78372357c1c76e3708a062d703b33 Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 9 Feb 2024 12:19:16 +0100 Subject: [PATCH 6/7] [nix-improvements] get rid of Drone --- .drone.yml | 300 -------------------- .woodpecker/release.yaml | 2 +- doc/book/design/goals.md | 2 - doc/book/development/devenv.md | 2 +- doc/book/development/miscellaneous-notes.md | 8 +- doc/book/development/release-process.md | 55 ++-- shell.nix | 9 - 7 files changed, 22 insertions(+), 356 deletions(-) delete mode 100644 .drone.yml diff --git a/.drone.yml b/.drone.yml deleted file mode 100644 index 16a19917..00000000 --- a/.drone.yml +++ /dev/null @@ -1,300 +0,0 @@ ---- -kind: pipeline -name: default - -node: - nix-daemon: 1 - -steps: - - name: check formatting - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr devShell --run "cargo fmt -- --check" - - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - - name: unit + func tests - image: nixpkgs/nix:nixos-22.05 - environment: - GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage - GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration - commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-build --no-build-output --attr test.amd64 - - ./result/bin/garage_db-* - - ./result/bin/garage_api-* - - ./result/bin/garage_model-* - - ./result/bin/garage_rpc-* - - ./result/bin/garage_table-* - - ./result/bin/garage_util-* - - ./result/bin/garage_web-* - - ./result/bin/garage-* - - ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false) - - rm result - - rm -rv tmp-garage-integration - - - name: integration tests - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - -trigger: - event: - - custom - - push - - pull_request - - tag - - cron - ---- -kind: pipeline -type: docker -name: release-linux-amd64 - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: integration tests - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - - - name: upgrade tests - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "x86_64-unknown-linux-musl" - commands: - - nix-shell --attr ci --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/amd64" - CONTAINER_NAME: "dxflrs/amd64_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run "to_docker" - - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: release-linux-i386 - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: integration tests - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - - - name: upgrade tests - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false) - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "i686-unknown-linux-musl" - commands: - - nix-shell --attr ci --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/386" - CONTAINER_NAME: "dxflrs/386_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run "to_docker" - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: release-linux-arm64 - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "aarch64-unknown-linux-musl" - commands: - - nix-shell --attr ci --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/arm64" - CONTAINER_NAME: "dxflrs/arm64_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run "to_docker" - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: release-linux-arm - -node: - nix-daemon: 1 - -steps: - - name: build - image: nixpkgs/nix:nixos-22.05 - commands: - - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage" - - - name: push static binary - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - TARGET: "armv6l-unknown-linux-musleabihf" - commands: - - nix-shell --attr ci --run "to_s3" - - - name: docker build and publish - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - DOCKER_PLATFORM: "linux/arm" - CONTAINER_NAME: "dxflrs/arm_garage" - HOME: "/kaniko" - commands: - - mkdir -p /kaniko/.docker - - echo $DOCKER_AUTH > /kaniko/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run "to_docker" - -trigger: - event: - - promote - - cron - ---- -kind: pipeline -type: docker -name: refresh-release-page - -node: - nix-daemon: 1 - -steps: - - name: multiarch-docker - image: nixpkgs/nix:nixos-22.05 - environment: - DOCKER_AUTH: - from_secret: docker_auth - HOME: "/root" - commands: - - mkdir -p /root/.docker - - echo $DOCKER_AUTH > /root/.docker/config.json - - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --attr ci --run "multiarch_docker" - - name: refresh-index - image: nixpkgs/nix:nixos-22.05 - environment: - AWS_ACCESS_KEY_ID: - from_secret: garagehq_aws_access_key_id - AWS_SECRET_ACCESS_KEY: - from_secret: garagehq_aws_secret_access_key - commands: - - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf - - nix-shell --attr ci --run "refresh_index" - -depends_on: - - release-linux-amd64 - - release-linux-i386 - - release-linux-arm64 - - release-linux-arm - -trigger: - event: - - promote - - cron - ---- -kind: signature -hmac: 9ff9f2ad9387b12ee49a327991a5373f01e40d28e0aeef3a13d3b741d8eb3c53 - -... diff --git a/.woodpecker/release.yaml b/.woodpecker/release.yaml index 8f19be82..c57d7272 100644 --- a/.woodpecker/release.yaml +++ b/.woodpecker/release.yaml @@ -54,7 +54,7 @@ steps: - source: garagehq_aws_secret_access_key target: AWS_SECRET_ACCESS_KEY commands: - - nix-shell --attr ci --run "to_s3_woodpecker" + - nix-shell --attr ci --run "to_s3" - name: docker build and publish image: nixpkgs/nix:nixos-22.05 diff --git a/doc/book/design/goals.md b/doc/book/design/goals.md index 78ac7978..4efb6349 100644 --- a/doc/book/design/goals.md +++ b/doc/book/design/goals.md @@ -48,7 +48,5 @@ locations. They use Garage themselves for the following tasks: - As a backup target using `rclone` and `restic` -- In the Drone continuous integration platform to store task logs - The Deuxfleurs Garage cluster is a multi-site cluster currently composed of 9 nodes in 3 physical locations. diff --git a/doc/book/development/devenv.md b/doc/book/development/devenv.md index dd3bdec0..88f8ba06 100644 --- a/doc/book/development/devenv.md +++ b/doc/book/development/devenv.md @@ -80,7 +80,7 @@ nix-build \ --git_version $(git rev-parse HEAD) ``` -*The result is located in `result/bin`. You can pass arguments to cross compile: check `.drone.yml` for examples.* +*The result is located in `result/bin`. You can pass arguments to cross compile: check `.woodpecker/release.yml` for examples.* If you modify a `Cargo.toml` or regenerate any `Cargo.lock`, you must run `cargo2nix`: diff --git a/doc/book/development/miscellaneous-notes.md b/doc/book/development/miscellaneous-notes.md index f0083ae5..a421943f 100644 --- a/doc/book/development/miscellaneous-notes.md +++ b/doc/book/development/miscellaneous-notes.md @@ -81,12 +81,9 @@ Our cache will be checked. - http://www.lpenz.org/articles/nixchannel/index.html -## Drone +## Woodpecker -Do not try to set a build as trusted from the interface or the CLI tool, -your request would be ignored. Instead, directly edit the database (table `repos`, column `repo_trusted`). - -Drone can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner. +Woodpecker can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner. ## Building Docker containers @@ -99,3 +96,4 @@ We were: - Unable to use the kaniko container provided by Google as we can't run arbitrary logic: we need to put our secret in .docker/config.json. Finally we chose to build kaniko through nix and use it in a `nix-shell`. +We then switched to using kaniko from nixpkgs when it was packaged. diff --git a/doc/book/development/release-process.md b/doc/book/development/release-process.md index 3fed4add..0c6701c0 100644 --- a/doc/book/development/release-process.md +++ b/doc/book/development/release-process.md @@ -42,7 +42,7 @@ and the docker containers on Docker Hub. ## Automation -We automated our release process with Nix and Drone to make it more reliable. +We automated our release process with Nix and Woodpecker to make it more reliable. Here we describe how we have done in case you want to debug or improve it. ### Caching build steps @@ -62,52 +62,31 @@ Sending to the cache is done through `nix copy`, for example: nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' result ``` -*Note that you need the signing key. In our case, it is stored as a secret in Drone.* +*The signing key possessed by the Garage maintainers is required to update the Nix cache.* -The previous command will only send the built packet and not its dependencies. -To send its dependency, a tool named `nix-copy-closure` has been created but it is not compatible with the S3 protocol. - -Instead, you can use the following commands to list all the runtime dependencies: +The previous command will only send the built package and not its dependencies. +In the case of our CI pipeline, we want to cache all intermediate build steps +as well. This can be done using this quite involved command (here as an example +for the `pkgs.amd64.relase` package): ```bash -nix copy \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR result/) +nix copy -j8 \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/nix-signing-key.sec' \ + $(nix path-info pkgs.amd64.release --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/') ``` -*We could also write this expression with xargs but this tool is not available in our container.* +This command will simultaneously build all of the required Nix paths (using at +most 8 parallel Nix builder jobs) and send the resulting objects to the cache. -But in certain cases, we want to cache compile time dependencies also. -For example, the Nix project does not provide binaries for cross compiling to i686 and thus we need to compile gcc on our own. -We do not want to compile gcc each time, so even if it is a compile time dependency, we want to cache it. - -This time, the command is a bit more involved: - -```bash -nix copy --to \ - 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR --include-outputs \ - $(nix-instantiate)) -``` - -This is the command we use in our CI as we expect the final binary to change, so we mainly focus on -caching our development dependencies. - -*Currently there is no automatic garbage collection of the cache: we should monitor its growth. -Hopefully, we can erase it totally without breaking any build, the next build will only be slower.* - -In practise, we concluded that we do not want to cache all the compilation dependencies. -Instead, we want to cache the toolchain we use to build Garage each time we change it. -So we removed from Drone any automatic update of the cache and instead handle them manually with: +This can be run for all the Garage packages we build using the following command: ``` source ~/.awsrc -nix-shell --run 'refresh_toolchain' +nix-shell --attr cache --run 'refresh_cache' ``` -Internally, it will run `nix-build` on `nix/toolchain.nix` and send the output plus its depedencies to the cache. - -To erase the cache: +We don't automate this step at each CI build, as *there is currently no automatic garbage collection of the cache.* +This means we should also monitor the cache's size; if it ever becomes too big we can erase it with: ``` mc rm --recursive --force 'garage/nix/' @@ -157,9 +136,9 @@ nix-shell --run refresh_index If you want to compile for different architectures, you will need to repeat all these commands for each architecture. -**In practise, and except for debugging, you will never directly run these commands. Release is handled by drone** +**In practice, and except for debugging, you will never directly run these commands. Release is handled by Woodpecker.** -### Drone +### Drone (obsolete) Our instance is available at [https://drone.deuxfleurs.fr](https://drone.deuxfleurs.fr). You need an account on [https://git.deuxfleurs.fr](https://git.deuxfleurs.fr) to use it. diff --git a/shell.nix b/shell.nix index 193ed941..12daa502 100644 --- a/shell.nix +++ b/shell.nix @@ -34,15 +34,6 @@ in ]; shellHook = '' function to_s3 { - aws \ - --endpoint-url https://garage.deuxfleurs.fr \ - --region garage \ - s3 cp \ - ./result-bin/bin/garage \ - s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage - } - - function to_s3_woodpecker { aws \ --endpoint-url https://garage.deuxfleurs.fr \ --region garage \ From f7ae966ed34d3ca8fb20afd2f20874dd42df7f7e Mon Sep 17 00:00:00 2001 From: Alex Auvolat Date: Fri, 9 Feb 2024 12:47:03 +0100 Subject: [PATCH 7/7] [nix-improvements] special case for Docker's "386" architecture --- shell.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shell.nix b/shell.nix index 12daa502..0fa02b66 100644 --- a/shell.nix +++ b/shell.nix @@ -45,8 +45,8 @@ in function to_docker { executor \ --force \ - --customPlatform="''${DOCKER_PLATFORM}" \ - --destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \ + --customPlatform="$(echo "''${DOCKER_PLATFORM}" | sed 's/i386/386/')" \ + --destination "$(echo "''${CONTAINER_NAME}" | sed 's/i386/386/'):''${CONTAINER_TAG}" \ --context dir://`pwd` \ --verbosity=debug }