diff --git a/.drone.yml b/.drone.yml index 903be5b0..34a6eda7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -25,7 +25,7 @@ steps: path: /etc/nix commands: - cp nix/nix.conf /etc/nix/nix.conf - - nix-build --no-build-output --no-out-link shell.nix --arg release false -A inputDerivation + - nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation - name: code quality image: nixpkgs/nix:nixos-21.05 @@ -35,8 +35,8 @@ steps: - name: nix_config path: /etc/nix commands: - - nix-shell --arg release false --run "cargo fmt -- --check" - - nix-shell --arg release false --run "cargo clippy -- --deny warnings" + - nix-shell --attr rust --run "cargo fmt -- --check" + - nix-shell --attr rust --run "cargo clippy -- --deny warnings" - name: build image: nixpkgs/nix:nixos-21.05 @@ -46,7 +46,8 @@ steps: - name: nix_config path: /etc/nix commands: - - nix-build --no-build-output --option log-lines 100 --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT + - nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT + - nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage" - name: unit + func tests image: nixpkgs/nix:nixos-21.05 @@ -82,7 +83,7 @@ steps: path: /etc/nix commands: - nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT - - nix-shell --arg release false --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) trigger: event: @@ -120,7 +121,7 @@ steps: path: /etc/nix commands: - cp nix/nix.conf /etc/nix/nix.conf - - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation + - nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation - name: build image: nixpkgs/nix:nixos-21.05 @@ -131,6 +132,7 @@ steps: path: /etc/nix commands: - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT + - nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage" - name: integration image: nixpkgs/nix:nixos-21.05 @@ -140,7 +142,7 @@ steps: - name: nix_config path: /etc/nix commands: - - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - name: push static binary image: nixpkgs/nix:nixos-21.05 @@ -155,7 +157,7 @@ steps: AWS_SECRET_ACCESS_KEY: from_secret: garagehq_aws_secret_access_key commands: - - nix-shell --arg rust false --arg integration false --run "to_s3" + - nix-shell --attr release --run "to_s3" - name: docker build and publish image: nixpkgs/nix:nixos-21.05 @@ -174,7 +176,7 @@ steps: - mkdir -p /kaniko/.docker - echo $DOCKER_AUTH > /kaniko/.docker/config.json - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --arg rust false --arg integration false --run "to_docker" + - nix-shell --attr release --run "to_docker" trigger: @@ -210,7 +212,7 @@ steps: path: /etc/nix commands: - cp nix/nix.conf /etc/nix/nix.conf - - nix-build --no-build-output --no-out-link shell.nix -A inputDerivation + - nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation - name: build image: nixpkgs/nix:nixos-21.05 @@ -221,6 +223,7 @@ steps: path: /etc/nix commands: - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT + - nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage" - name: integration image: nixpkgs/nix:nixos-21.05 @@ -230,7 +233,7 @@ steps: - name: nix_config path: /etc/nix commands: - - nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) + - nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - name: push static binary image: nixpkgs/nix:nixos-21.05 @@ -245,7 +248,7 @@ steps: AWS_SECRET_ACCESS_KEY: from_secret: garagehq_aws_secret_access_key commands: - - nix-shell --arg rust false --arg integration false --run "to_s3" + - nix-shell --attr release --run "to_s3" - name: docker build and publish image: nixpkgs/nix:nixos-21.05 @@ -264,7 +267,7 @@ steps: - mkdir -p /kaniko/.docker - echo $DOCKER_AUTH > /kaniko/.docker/config.json - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --arg rust false --arg integration false --run "to_docker" + - nix-shell --attr release --run "to_docker" trigger: event: @@ -299,7 +302,7 @@ steps: path: /etc/nix commands: - cp nix/nix.conf /etc/nix/nix.conf - - nix-build --no-build-output --no-out-link ./shell.nix --arg rust false --arg integration false -A inputDerivation + - nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation - name: build image: nixpkgs/nix:nixos-21.05 @@ -310,6 +313,7 @@ steps: path: /etc/nix commands: - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT + - nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage" - name: push static binary image: nixpkgs/nix:nixos-21.05 @@ -324,7 +328,7 @@ steps: AWS_SECRET_ACCESS_KEY: from_secret: garagehq_aws_secret_access_key commands: - - nix-shell --arg rust false --arg integration false --run "to_s3" + - nix-shell --attr release --run "to_s3" - name: docker build and publish image: nixpkgs/nix:nixos-21.05 @@ -343,7 +347,7 @@ steps: - mkdir -p /kaniko/.docker - echo $DOCKER_AUTH > /kaniko/.docker/config.json - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --arg rust false --arg integration false --run "to_docker" + - nix-shell --attr release --run "to_docker" trigger: event: @@ -378,7 +382,7 @@ steps: path: /etc/nix commands: - cp nix/nix.conf /etc/nix/nix.conf - - nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation + - nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation - name: build image: nixpkgs/nix:nixos-21.05 @@ -389,6 +393,7 @@ steps: path: /etc/nix commands: - nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT + - nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage" - name: push static binary image: nixpkgs/nix:nixos-21.05 @@ -403,7 +408,7 @@ steps: AWS_SECRET_ACCESS_KEY: from_secret: garagehq_aws_secret_access_key commands: - - nix-shell --arg integration false --arg rust false --run "to_s3" + - nix-shell --attr release --run "to_s3" - name: docker build and publish image: nixpkgs/nix:nixos-21.05 @@ -422,7 +427,7 @@ steps: - mkdir -p /kaniko/.docker - echo $DOCKER_AUTH > /kaniko/.docker/config.json - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - - nix-shell --arg rust false --arg integration false --run "to_docker" + - nix-shell --attr release --run "to_docker" trigger: event: @@ -455,7 +460,7 @@ steps: from_secret: garagehq_aws_secret_access_key commands: - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf - - nix-shell --arg integration false --arg rust false --run "refresh_index" + - nix-shell --attr release --run "refresh_index" depends_on: - release-linux-x86_64 @@ -473,6 +478,6 @@ node: --- kind: signature -hmac: 3fc19d6f9a3555519c8405e3281b2e74289bb802f644740d5481d53df3a01fa4 +hmac: 60fad5d78c12616be848aae35703f250300abab5f2eda08eb48fe3afd6cc58c8 ... diff --git a/default.nix b/default.nix index de996ac1..296c7592 100644 --- a/default.nix +++ b/default.nix @@ -56,45 +56,47 @@ in let */ overrides = pkgs.rustBuilder.overrides.all ++ [ /* - [1] We need to alter Nix hardening to be able to statically compile: PIE, + [1] We need to alter Nix hardening to make static binaries: PIE, Position Independent Executables seems to be supported only on amd64. Having - this flags set either make our executables crash or compile as dynamic on many platforms. - In the following section codegenOpts, we reactive it for the supported targets - (only amd64 curently) through the `-static-pie` flag. PIE is a feature used - by ASLR, which helps mitigate security issues. - Learn more about Nix Hardening: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh - - [2] We want to inject the git version while keeping the build deterministic. - As we do not want to consider the .git folder as part of the input source, - we ask the user (the CI often) to pass the value to Nix. + this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms. + Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets + (only amd64 curently) through the `-static-pie` flag. + PIE is a feature used by ASLR, which helps mitigate security issues. + Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh */ + (pkgs.rustBuilder.rustLib.makeOverride { + name = "garage"; + overrideAttrs = drv: { hardeningDisable = [ "pie" ]; }; + }) + (pkgs.rustBuilder.rustLib.makeOverride { name = "garage_rpc"; + + /* + [2] We want to inject the git version while keeping the build deterministic. + As we do not want to consider the .git folder as part of the input source, + we ask the user (the CI often) to pass the value to Nix. + */ overrideAttrs = drv: - /* [1] */ { hardeningDisable = [ "pie" ]; } - // - /* [2] */ (if git_version != null then { + (if git_version != null then { preConfigure = '' ${drv.preConfigure or ""} export GIT_VERSION="${git_version}" ''; } else {}); + + /* + [3] We ship some parts of the code disabled by default by putting them behind a flag. + It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). + But we want to ship these additional features when we release Garage. + In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds. + Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate. + */ + overrideArgs = old: { + features = if release then [ "kubernetes-discovery" ] else []; + }; }) - /* - We ship some parts of the code disabled by default by putting them behind a flag. - It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.). - But we want to ship these additional features when we release Garage. - In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds. - Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate. - */ - (pkgs.rustBuilder.rustLib.makeOverride { - name = "garage_rpc"; - overrideArgs = old: - { - features = if release then [ "kubernetes-discovery" ] else []; - }; - }) ]; packageFun = import ./Cargo.nix; diff --git a/nix/nix.conf b/nix/nix.conf index 871efb10..5a9de951 100644 --- a/nix/nix.conf +++ b/nix/nix.conf @@ -2,3 +2,4 @@ substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs= max-jobs = auto cores = 4 +log-lines = 200 diff --git a/script/not-dynamic.sh b/script/not-dynamic.sh new file mode 100755 index 00000000..b9a13070 --- /dev/null +++ b/script/not-dynamic.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -e + +if [ "$#" -ne 1 ]; then + echo "[fail] usage: $0 binary" + exit 2 +fi + +if file $1 | grep 'dynamically linked' 2>&1; then + echo "[fail] $1 is dynamic" + exit 1 +fi +echo "[ok] $1 is probably static" diff --git a/shell.nix b/shell.nix index 13ea4a0e..eaedb6b8 100644 --- a/shell.nix +++ b/shell.nix @@ -1,8 +1,5 @@ { system ? builtins.currentSystem, - rust ? true, - integration ? true, - release ? true, }: with import ./nix/common.nix; @@ -16,9 +13,59 @@ let winscp = (import ./nix/winscp.nix) pkgs; in + { -pkgs.mkShell { - shellHook = '' + /* --- Rust Shell --- + * Use it to compile Garage + */ + rust = pkgs.mkShell { + shellHook = '' +function refresh_toolchain { + nix copy \ + --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ + $(nix-store -qR \ + $(nix-build --quiet --no-build-output --no-out-link nix/toolchain.nix)) +} + ''; + + nativeBuildInputs = [ + pkgs.rustPlatform.rust.rustc + pkgs.rustPlatform.rust.cargo + pkgs.clippy + pkgs.rustfmt + pkgs.perl + pkgs.protobuf + pkgs.pkg-config + pkgs.openssl + pkgs.file + cargo2nix.packages.x86_64-linux.cargo2nix + ]; + }; + + /* --- Integration shell --- + * Use it to test Garage with common S3 clients + */ + integration = pkgs.mkShell { + nativeBuildInputs = [ + winscp + pkgs.s3cmd + pkgs.awscli2 + pkgs.minio-client + pkgs.rclone + pkgs.socat + pkgs.psmisc + pkgs.which + pkgs.openssl + pkgs.curl + pkgs.jq + ]; + }; + + /* --- Release shell --- + * A shell built to make releasing easier + */ + release = pkgs.mkShell { + shellHook = '' function to_s3 { aws \ --endpoint-url https://garage.deuxfleurs.fr \ @@ -62,45 +109,12 @@ function refresh_index { result/share/_releases.html \ s3://garagehq.deuxfleurs.fr/ } + ''; + nativeBuildInputs = [ + pkgs.awscli2 + kaniko + ]; + }; + } -function refresh_toolchain { - nix copy \ - --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \ - $(nix-store -qR \ - $(nix-build --quiet --no-build-output --no-out-link nix/toolchain.nix)) -} - ''; - nativeBuildInputs = - (if rust then [ - pkgs.rustPlatform.rust.rustc - pkgs.rustPlatform.rust.cargo - pkgs.clippy - pkgs.rustfmt - pkgs.perl - pkgs.protobuf - pkgs.pkg-config - pkgs.openssl - cargo2nix.packages.x86_64-linux.cargo2nix - ] else []) - ++ - (if integration then [ - winscp - pkgs.s3cmd - pkgs.awscli2 - pkgs.minio-client - pkgs.rclone - pkgs.socat - pkgs.psmisc - pkgs.which - pkgs.openssl - pkgs.curl - pkgs.jq - ] else []) - ++ - (if release then [ - pkgs.awscli2 - kaniko - ] else []) - ; -}