[nix-improvements] modernize Nix infrastructure

This commit is contained in:
Alex 2024-02-07 19:23:32 +01:00
parent e4a43bfd59
commit 9900368380
Signed by untrusted user: lx
GPG key ID: 0E496D15096376BE
4 changed files with 134 additions and 116 deletions

View file

@ -9,7 +9,7 @@ steps:
- name: check formatting - name: check formatting
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-shell --attr rust --run "cargo fmt -- --check" - nix-shell --attr devShell --run "cargo fmt -- --check"
- name: build - name: build
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
@ -40,7 +40,7 @@ steps:
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
trigger: trigger:
event: event:
@ -63,17 +63,17 @@ steps:
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: integration tests - name: integration tests
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
- name: upgrade tests - name: upgrade tests
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false) - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
- name: push static binary - name: push static binary
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
@ -84,7 +84,7 @@ steps:
from_secret: garagehq_aws_secret_access_key from_secret: garagehq_aws_secret_access_key
TARGET: "x86_64-unknown-linux-musl" TARGET: "x86_64-unknown-linux-musl"
commands: commands:
- nix-shell --attr release --run "to_s3" - nix-shell --attr ci --run "to_s3"
- name: docker build and publish - name: docker build and publish
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
@ -98,7 +98,7 @@ steps:
- mkdir -p /kaniko/.docker - mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json - echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "to_docker" - nix-shell --attr ci --run "to_docker"
trigger: trigger:
@ -119,17 +119,17 @@ steps:
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: integration tests - name: integration tests
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false) - nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
- name: upgrade tests - name: upgrade tests
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false) - nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
- name: push static binary - name: push static binary
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
@ -140,7 +140,7 @@ steps:
from_secret: garagehq_aws_secret_access_key from_secret: garagehq_aws_secret_access_key
TARGET: "i686-unknown-linux-musl" TARGET: "i686-unknown-linux-musl"
commands: commands:
- nix-shell --attr release --run "to_s3" - nix-shell --attr ci --run "to_s3"
- name: docker build and publish - name: docker build and publish
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
@ -154,7 +154,7 @@ steps:
- mkdir -p /kaniko/.docker - mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json - echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "to_docker" - nix-shell --attr ci --run "to_docker"
trigger: trigger:
event: event:
@ -174,7 +174,7 @@ steps:
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: push static binary - name: push static binary
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
@ -185,7 +185,7 @@ steps:
from_secret: garagehq_aws_secret_access_key from_secret: garagehq_aws_secret_access_key
TARGET: "aarch64-unknown-linux-musl" TARGET: "aarch64-unknown-linux-musl"
commands: commands:
- nix-shell --attr release --run "to_s3" - nix-shell --attr ci --run "to_s3"
- name: docker build and publish - name: docker build and publish
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
@ -199,7 +199,7 @@ steps:
- mkdir -p /kaniko/.docker - mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json - echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "to_docker" - nix-shell --attr ci --run "to_docker"
trigger: trigger:
event: event:
@ -219,7 +219,7 @@ steps:
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
commands: commands:
- nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT} - nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage" - nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: push static binary - name: push static binary
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
@ -230,7 +230,7 @@ steps:
from_secret: garagehq_aws_secret_access_key from_secret: garagehq_aws_secret_access_key
TARGET: "armv6l-unknown-linux-musleabihf" TARGET: "armv6l-unknown-linux-musleabihf"
commands: commands:
- nix-shell --attr release --run "to_s3" - nix-shell --attr ci --run "to_s3"
- name: docker build and publish - name: docker build and publish
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
@ -244,7 +244,7 @@ steps:
- mkdir -p /kaniko/.docker - mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json - echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "to_docker" - nix-shell --attr ci --run "to_docker"
trigger: trigger:
event: event:
@ -270,7 +270,7 @@ steps:
- mkdir -p /root/.docker - mkdir -p /root/.docker
- echo $DOCKER_AUTH > /root/.docker/config.json - echo $DOCKER_AUTH > /root/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT} - export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "multiarch_docker" - nix-shell --attr ci --run "multiarch_docker"
- name: refresh-index - name: refresh-index
image: nixpkgs/nix:nixos-22.05 image: nixpkgs/nix:nixos-22.05
environment: environment:
@ -280,7 +280,7 @@ steps:
from_secret: garagehq_aws_secret_access_key from_secret: garagehq_aws_secret_access_key
commands: commands:
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf - mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
- nix-shell --attr release --run "refresh_index" - nix-shell --attr ci --run "refresh_index"
depends_on: depends_on:
- release-linux-amd64 - release-linux-amd64
@ -295,6 +295,6 @@ trigger:
--- ---
kind: signature kind: signature
hmac: 0c4b57eb4b27b7c6a6ff21ab87f0767fe3eb90f5d95d5cbcdccf794e9d2a5d86 hmac: 9ff9f2ad9387b12ee49a327991a5373f01e40d28e0aeef3a13d3b741d8eb3c53
... ...

View file

@ -33,27 +33,57 @@
compile = import ./nix/compile.nix; compile = import ./nix/compile.nix;
in in
flake-utils.lib.eachDefaultSystem (system: flake-utils.lib.eachDefaultSystem (system:
let pkgs = nixpkgs.legacyPackages.${system}; let
in { pkgs = nixpkgs.legacyPackages.${system};
packages = { in
default = (compile { {
inherit system git_version; packages =
pkgsSrc = nixpkgs; let
cargo2nixOverlay = cargo2nix.overlays.default; packageFor = target: (compile {
release = true; inherit system git_version target;
}).workspace.garage { compileMode = "build"; }; pkgsSrc = nixpkgs;
}; cargo2nixOverlay = cargo2nix.overlays.default;
devShell = (compile { release = true;
inherit system git_version; }).workspace.garage { compileMode = "build"; };
pkgsSrc = nixpkgs; in
cargo2nixOverlay = cargo2nix.overlays.default; {
release = false; # default = native release build
}).workspaceShell { packages = with pkgs; [ default = packageFor null;
cargo-audit # other = cross-compiled, statically-linked builds
cargo-outdated amd64 = packageFor "x86_64-unknown-linux-musl";
rustfmt i386 = packageFor "i686-unknown-linux-musl";
clang arm64 = packageFor "aarch64-unknown-linux-musl";
mold arm = packageFor "armv6l-unknown-linux-musl";
]; }; };
# ---- developpment shell, for making native builds only ----
devShells =
let
shellWithPackages = (packages: (compile {
inherit system git_version;
pkgsSrc = nixpkgs;
cargo2nixOverlay = cargo2nix.overlays.default;
}).workspaceShell { inherit packages; });
in
{
default = shellWithPackages
(with pkgs; [
rustfmt
clang
mold
]);
# import the full shell using `nix develop .#full`
full = shellWithPackages (with pkgs; [
rustfmt
clang
mold
# ---- extra packages for dev tasks ----
cargo-audit
cargo-outdated
cargo-machete
nixpkgs-fmt
]);
};
}); });
} }

View file

@ -14,4 +14,5 @@ rec {
pkgsSrc = flake.defaultNix.inputs.nixpkgs; pkgsSrc = flake.defaultNix.inputs.nixpkgs;
cargo2nix = flake.defaultNix.inputs.cargo2nix; cargo2nix = flake.defaultNix.inputs.cargo2nix;
cargo2nixOverlay = cargo2nix.overlays.default; cargo2nixOverlay = cargo2nix.overlays.default;
devShells = builtins.getAttr builtins.currentSystem flake.defaultNix.devShells;
} }

133
shell.nix
View file

@ -5,87 +5,36 @@ with import ./nix/common.nix;
let let
pkgs = import pkgsSrc { pkgs = import pkgsSrc {
inherit system; inherit system;
overlays = [ cargo2nixOverlay ];
}; };
kaniko = (import ./nix/kaniko.nix) pkgs; kaniko = (import ./nix/kaniko.nix) pkgs;
manifest-tool = (import ./nix/manifest-tool.nix) pkgs; manifest-tool = (import ./nix/manifest-tool.nix) pkgs;
winscp = (import ./nix/winscp.nix) pkgs; winscp = (import ./nix/winscp.nix) pkgs;
in
{
# --- Dev shell inherited from flake.nix ---
devShell = devShells.default;
in { # --- Continuous integration shell ---
# --- Rust Shell --- # The shell used for all CI jobs (along with devShell)
# Use it to compile Garage ci = pkgs.mkShell {
rust = pkgs.mkShell {
nativeBuildInputs = with pkgs; [ nativeBuildInputs = with pkgs; [
#rustPlatform.rust.rustc kaniko
rustPlatform.rust.cargo manifest-tool
clang
mold
#clippy
rustfmt
#perl
#protobuf
#pkg-config
#openssl
file
#cargo2nix.packages.x86_64-linux.cargo2nix
];
};
# --- Integration shell ---
# Use it to test Garage with common S3 clients
integration = pkgs.mkShell {
nativeBuildInputs = [
winscp winscp
pkgs.s3cmd
pkgs.awscli2 awscli2
pkgs.minio-client file
pkgs.rclone s3cmd
pkgs.socat minio-client
pkgs.psmisc rclone
pkgs.which socat
pkgs.openssl psmisc
pkgs.curl which
pkgs.jq openssl
curl
jq
]; ];
};
# --- Release shell ---
# A shell built to make releasing easier
release = pkgs.mkShell {
shellHook = '' shellHook = ''
function refresh_toolchain {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
nix copy \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR \
$(nix-build --no-build-output --no-out-link nix/toolchain.nix))
rm /tmp/nix-signing-key.sec
}
function refresh_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
echo "Updating cache for ''${attr}"
derivation=$(nix-instantiate --attr ''${attr})
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR ''${derivation%\!bin})
done
rm /tmp/nix-signing-key.sec
}
function refresh_flake_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in packages.x86_64-linux.default devShell.x86_64-linux; do
echo "Updating cache for ''${attr}"
derivation=$(nix path-info --derivation ".#''${attr}")
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR ''${derivation})
done
rm /tmp/nix-signing-key.sec
}
function to_s3 { function to_s3 {
aws \ aws \
--endpoint-url https://garage.deuxfleurs.fr \ --endpoint-url https://garage.deuxfleurs.fr \
@ -167,7 +116,45 @@ in {
s3://garagehq.deuxfleurs.fr/ s3://garagehq.deuxfleurs.fr/
} }
''; '';
nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ];
};
# --- Cache shell ---
# A shell for refreshing caches
cache = pkgs.mkShell {
shellHook = ''
function refresh_toolchain {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR \
$(nix-build -j8 --no-build-output --no-out-link nix/toolchain.nix))
rm /tmp/nix-signing-key.sec
}
function refresh_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do
echo "Updating cache for ''${attr}"
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix path-info ''${attr} --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/')
done
rm /tmp/nix-signing-key.sec
}
function refresh_flake_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in packages.x86_64-linux.default devShells.x86_64-linux.default; do
echo "Updating cache for ''${attr}"
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
".#''${attr}"
done
rm /tmp/nix-signing-key.sec
}
'';
}; };
} }