forked from Deuxfleurs/garage
Compare commits
3 commits
main
...
bug/check_
Author | SHA1 | Date | |
---|---|---|---|
fe053957e5 | |||
e680efb361 | |||
4d269787b2 |
5 changed files with 129 additions and 93 deletions
47
.drone.yml
47
.drone.yml
|
@ -25,7 +25,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-build --no-build-output --no-out-link shell.nix --arg release false -A inputDerivation
|
||||
- nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation
|
||||
|
||||
- name: code quality
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -35,8 +35,8 @@ steps:
|
|||
- name: nix_config
|
||||
path: /etc/nix
|
||||
commands:
|
||||
- nix-shell --arg release false --run "cargo fmt -- --check"
|
||||
- nix-shell --arg release false --run "cargo clippy -- --deny warnings"
|
||||
- nix-shell --attr rust --run "cargo fmt -- --check"
|
||||
- nix-shell --attr rust --run "cargo clippy -- --deny warnings"
|
||||
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -46,7 +46,8 @@ steps:
|
|||
- name: nix_config
|
||||
path: /etc/nix
|
||||
commands:
|
||||
- nix-build --no-build-output --option log-lines 100 --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
|
||||
- nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: unit + func tests
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -82,7 +83,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --arg release false --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
trigger:
|
||||
event:
|
||||
|
@ -120,7 +121,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
|
||||
- nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation
|
||||
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -131,6 +132,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: integration
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -140,7 +142,7 @@ steps:
|
|||
- name: nix_config
|
||||
path: /etc/nix
|
||||
commands:
|
||||
- nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -155,7 +157,7 @@ steps:
|
|||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- nix-shell --arg rust false --arg integration false --run "to_s3"
|
||||
- nix-shell --attr release --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -174,7 +176,7 @@ steps:
|
|||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --arg rust false --arg integration false --run "to_docker"
|
||||
- nix-shell --attr release --run "to_docker"
|
||||
|
||||
|
||||
trigger:
|
||||
|
@ -210,7 +212,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
|
||||
- nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation
|
||||
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -221,6 +223,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: integration
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -230,7 +233,7 @@ steps:
|
|||
- name: nix_config
|
||||
path: /etc/nix
|
||||
commands:
|
||||
- nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -245,7 +248,7 @@ steps:
|
|||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- nix-shell --arg rust false --arg integration false --run "to_s3"
|
||||
- nix-shell --attr release --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -264,7 +267,7 @@ steps:
|
|||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --arg rust false --arg integration false --run "to_docker"
|
||||
- nix-shell --attr release --run "to_docker"
|
||||
|
||||
trigger:
|
||||
event:
|
||||
|
@ -299,7 +302,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-build --no-build-output --no-out-link ./shell.nix --arg rust false --arg integration false -A inputDerivation
|
||||
- nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation
|
||||
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -310,6 +313,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -324,7 +328,7 @@ steps:
|
|||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- nix-shell --arg rust false --arg integration false --run "to_s3"
|
||||
- nix-shell --attr release --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -343,7 +347,7 @@ steps:
|
|||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --arg rust false --arg integration false --run "to_docker"
|
||||
- nix-shell --attr release --run "to_docker"
|
||||
|
||||
trigger:
|
||||
event:
|
||||
|
@ -378,7 +382,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
|
||||
- nix-build --no-build-output --no-out-link shell.nix -A rust.inputDerivation -A integration.inputDerivation -A release.inputDerivation
|
||||
|
||||
- name: build
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -389,6 +393,7 @@ steps:
|
|||
path: /etc/nix
|
||||
commands:
|
||||
- nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
|
||||
- nix-shell --attr rust --run "./script/not-dynamic.sh result/bin/garage"
|
||||
|
||||
- name: push static binary
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -403,7 +408,7 @@ steps:
|
|||
AWS_SECRET_ACCESS_KEY:
|
||||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- nix-shell --arg integration false --arg rust false --run "to_s3"
|
||||
- nix-shell --attr release --run "to_s3"
|
||||
|
||||
- name: docker build and publish
|
||||
image: nixpkgs/nix:nixos-21.05
|
||||
|
@ -422,7 +427,7 @@ steps:
|
|||
- mkdir -p /kaniko/.docker
|
||||
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||
- nix-shell --arg rust false --arg integration false --run "to_docker"
|
||||
- nix-shell --attr release --run "to_docker"
|
||||
|
||||
trigger:
|
||||
event:
|
||||
|
@ -455,7 +460,7 @@ steps:
|
|||
from_secret: garagehq_aws_secret_access_key
|
||||
commands:
|
||||
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
|
||||
- nix-shell --arg integration false --arg rust false --run "refresh_index"
|
||||
- nix-shell --attr release --run "refresh_index"
|
||||
|
||||
depends_on:
|
||||
- release-linux-x86_64
|
||||
|
@ -473,6 +478,6 @@ node:
|
|||
|
||||
---
|
||||
kind: signature
|
||||
hmac: 3fc19d6f9a3555519c8405e3281b2e74289bb802f644740d5481d53df3a01fa4
|
||||
hmac: 60fad5d78c12616be848aae35703f250300abab5f2eda08eb48fe3afd6cc58c8
|
||||
|
||||
...
|
||||
|
|
56
default.nix
56
default.nix
|
@ -56,45 +56,47 @@ in let
|
|||
*/
|
||||
overrides = pkgs.rustBuilder.overrides.all ++ [
|
||||
/*
|
||||
[1] We need to alter Nix hardening to be able to statically compile: PIE,
|
||||
[1] We need to alter Nix hardening to make static binaries: PIE,
|
||||
Position Independent Executables seems to be supported only on amd64. Having
|
||||
this flags set either make our executables crash or compile as dynamic on many platforms.
|
||||
In the following section codegenOpts, we reactive it for the supported targets
|
||||
(only amd64 curently) through the `-static-pie` flag. PIE is a feature used
|
||||
by ASLR, which helps mitigate security issues.
|
||||
Learn more about Nix Hardening: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh
|
||||
|
||||
[2] We want to inject the git version while keeping the build deterministic.
|
||||
As we do not want to consider the .git folder as part of the input source,
|
||||
we ask the user (the CI often) to pass the value to Nix.
|
||||
this flag set either 1. make our executables crash or 2. compile as dynamic on some platforms.
|
||||
Here, we deactivate it. Later (find `codegenOpts`), we reactivate it for supported targets
|
||||
(only amd64 curently) through the `-static-pie` flag.
|
||||
PIE is a feature used by ASLR, which helps mitigate security issues.
|
||||
Learn more about Nix Hardening at: https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/cc-wrapper/add-hardening.sh
|
||||
*/
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage";
|
||||
overrideAttrs = drv: { hardeningDisable = [ "pie" ]; };
|
||||
})
|
||||
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_rpc";
|
||||
|
||||
/*
|
||||
[2] We want to inject the git version while keeping the build deterministic.
|
||||
As we do not want to consider the .git folder as part of the input source,
|
||||
we ask the user (the CI often) to pass the value to Nix.
|
||||
*/
|
||||
overrideAttrs = drv:
|
||||
/* [1] */ { hardeningDisable = [ "pie" ]; }
|
||||
//
|
||||
/* [2] */ (if git_version != null then {
|
||||
(if git_version != null then {
|
||||
preConfigure = ''
|
||||
${drv.preConfigure or ""}
|
||||
export GIT_VERSION="${git_version}"
|
||||
'';
|
||||
} else {});
|
||||
|
||||
/*
|
||||
[3] We ship some parts of the code disabled by default by putting them behind a flag.
|
||||
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
|
||||
But we want to ship these additional features when we release Garage.
|
||||
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
|
||||
Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate.
|
||||
*/
|
||||
overrideArgs = old: {
|
||||
features = if release then [ "kubernetes-discovery" ] else [];
|
||||
};
|
||||
})
|
||||
|
||||
/*
|
||||
We ship some parts of the code disabled by default by putting them behind a flag.
|
||||
It speeds up the compilation (when the feature is not required) and released crates have less dependency by default (less attack surface, disk space, etc.).
|
||||
But we want to ship these additional features when we release Garage.
|
||||
In the end, we chose to exclude all features from debug builds while putting (all of) them in the release builds.
|
||||
Currently, the only feature of Garage is kubernetes-discovery from the garage_rpc crate.
|
||||
*/
|
||||
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||
name = "garage_rpc";
|
||||
overrideArgs = old:
|
||||
{
|
||||
features = if release then [ "kubernetes-discovery" ] else [];
|
||||
};
|
||||
})
|
||||
];
|
||||
|
||||
packageFun = import ./Cargo.nix;
|
||||
|
|
|
@ -2,3 +2,4 @@ substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
|
|||
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
|
||||
max-jobs = auto
|
||||
cores = 4
|
||||
log-lines = 200
|
||||
|
|
14
script/not-dynamic.sh
Executable file
14
script/not-dynamic.sh
Executable file
|
@ -0,0 +1,14 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "[fail] usage: $0 binary"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if file $1 | grep 'dynamically linked' 2>&1; then
|
||||
echo "[fail] $1 is dynamic"
|
||||
exit 1
|
||||
fi
|
||||
echo "[ok] $1 is probably static"
|
104
shell.nix
104
shell.nix
|
@ -1,8 +1,5 @@
|
|||
{
|
||||
system ? builtins.currentSystem,
|
||||
rust ? true,
|
||||
integration ? true,
|
||||
release ? true,
|
||||
}:
|
||||
|
||||
with import ./nix/common.nix;
|
||||
|
@ -16,9 +13,59 @@ let
|
|||
winscp = (import ./nix/winscp.nix) pkgs;
|
||||
|
||||
in
|
||||
{
|
||||
|
||||
pkgs.mkShell {
|
||||
shellHook = ''
|
||||
/* --- Rust Shell ---
|
||||
* Use it to compile Garage
|
||||
*/
|
||||
rust = pkgs.mkShell {
|
||||
shellHook = ''
|
||||
function refresh_toolchain {
|
||||
nix copy \
|
||||
--to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||
$(nix-store -qR \
|
||||
$(nix-build --quiet --no-build-output --no-out-link nix/toolchain.nix))
|
||||
}
|
||||
'';
|
||||
|
||||
nativeBuildInputs = [
|
||||
pkgs.rustPlatform.rust.rustc
|
||||
pkgs.rustPlatform.rust.cargo
|
||||
pkgs.clippy
|
||||
pkgs.rustfmt
|
||||
pkgs.perl
|
||||
pkgs.protobuf
|
||||
pkgs.pkg-config
|
||||
pkgs.openssl
|
||||
pkgs.file
|
||||
cargo2nix.packages.x86_64-linux.cargo2nix
|
||||
];
|
||||
};
|
||||
|
||||
/* --- Integration shell ---
|
||||
* Use it to test Garage with common S3 clients
|
||||
*/
|
||||
integration = pkgs.mkShell {
|
||||
nativeBuildInputs = [
|
||||
winscp
|
||||
pkgs.s3cmd
|
||||
pkgs.awscli2
|
||||
pkgs.minio-client
|
||||
pkgs.rclone
|
||||
pkgs.socat
|
||||
pkgs.psmisc
|
||||
pkgs.which
|
||||
pkgs.openssl
|
||||
pkgs.curl
|
||||
pkgs.jq
|
||||
];
|
||||
};
|
||||
|
||||
/* --- Release shell ---
|
||||
* A shell built to make releasing easier
|
||||
*/
|
||||
release = pkgs.mkShell {
|
||||
shellHook = ''
|
||||
function to_s3 {
|
||||
aws \
|
||||
--endpoint-url https://garage.deuxfleurs.fr \
|
||||
|
@ -62,45 +109,12 @@ function refresh_index {
|
|||
result/share/_releases.html \
|
||||
s3://garagehq.deuxfleurs.fr/
|
||||
}
|
||||
'';
|
||||
nativeBuildInputs = [
|
||||
pkgs.awscli2
|
||||
kaniko
|
||||
];
|
||||
};
|
||||
}
|
||||
|
||||
function refresh_toolchain {
|
||||
nix copy \
|
||||
--to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||
$(nix-store -qR \
|
||||
$(nix-build --quiet --no-build-output --no-out-link nix/toolchain.nix))
|
||||
}
|
||||
'';
|
||||
|
||||
nativeBuildInputs =
|
||||
(if rust then [
|
||||
pkgs.rustPlatform.rust.rustc
|
||||
pkgs.rustPlatform.rust.cargo
|
||||
pkgs.clippy
|
||||
pkgs.rustfmt
|
||||
pkgs.perl
|
||||
pkgs.protobuf
|
||||
pkgs.pkg-config
|
||||
pkgs.openssl
|
||||
cargo2nix.packages.x86_64-linux.cargo2nix
|
||||
] else [])
|
||||
++
|
||||
(if integration then [
|
||||
winscp
|
||||
pkgs.s3cmd
|
||||
pkgs.awscli2
|
||||
pkgs.minio-client
|
||||
pkgs.rclone
|
||||
pkgs.socat
|
||||
pkgs.psmisc
|
||||
pkgs.which
|
||||
pkgs.openssl
|
||||
pkgs.curl
|
||||
pkgs.jq
|
||||
] else [])
|
||||
++
|
||||
(if release then [
|
||||
pkgs.awscli2
|
||||
kaniko
|
||||
] else [])
|
||||
;
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue