Deuxfleurs/garage
44
56
Fork 73
Code Issues 100 Pull requests 14 Releases 30 Wiki Activity

Fix & simplify CI using Woodpecker #706

Merged
lx merged 7 commits from nix-improvements into main 2024-02-09 12:11:24 +00:00
Conversation 0 Commits 7 Files changed 14 +134 -116
4 changed files with 134 additions and 116 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 9900368380 - Show all commits

42
.drone.yml
View file

@ -9,7 +9,7 @@ steps:
- name: check formatting
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr rust --run "cargo fmt -- --check"
- nix-shell --attr devShell --run "cargo fmt -- --check"
- name: build
image: nixpkgs/nix:nixos-22.05
@ -40,7 +40,7 @@ steps:
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
trigger:
event:
@ -63,17 +63,17 @@ steps:
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: integration tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
- name: upgrade tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
- nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
- name: push static binary
image: nixpkgs/nix:nixos-22.05
@ -84,7 +84,7 @@ steps:
from_secret: garagehq_aws_secret_access_key
TARGET: "x86_64-unknown-linux-musl"
commands:
- nix-shell --attr release --run "to_s3"
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-22.05
@ -98,7 +98,7 @@ steps:
- mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "to_docker"
- nix-shell --attr ci --run "to_docker"
trigger:
@ -119,17 +119,17 @@ steps:
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: integration tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr integration --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
- name: upgrade tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr integration --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
- nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
- name: push static binary
image: nixpkgs/nix:nixos-22.05
@ -140,7 +140,7 @@ steps:
from_secret: garagehq_aws_secret_access_key
TARGET: "i686-unknown-linux-musl"
commands:
- nix-shell --attr release --run "to_s3"
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-22.05
@ -154,7 +154,7 @@ steps:
- mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "to_docker"
- nix-shell --attr ci --run "to_docker"
trigger:
event:
@ -174,7 +174,7 @@ steps:
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: push static binary
image: nixpkgs/nix:nixos-22.05
@ -185,7 +185,7 @@ steps:
from_secret: garagehq_aws_secret_access_key
TARGET: "aarch64-unknown-linux-musl"
commands:
- nix-shell --attr release --run "to_s3"
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-22.05
@ -199,7 +199,7 @@ steps:
- mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "to_docker"
- nix-shell --attr ci --run "to_docker"
trigger:
event:
@ -219,7 +219,7 @@ steps:
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr rust --run "./script/not-dynamic.sh result-bin/bin/garage"
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: push static binary
image: nixpkgs/nix:nixos-22.05
@ -230,7 +230,7 @@ steps:
from_secret: garagehq_aws_secret_access_key
TARGET: "armv6l-unknown-linux-musleabihf"
commands:
- nix-shell --attr release --run "to_s3"
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-22.05
@ -244,7 +244,7 @@ steps:
- mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "to_docker"
- nix-shell --attr ci --run "to_docker"
trigger:
event:
@ -270,7 +270,7 @@ steps:
- mkdir -p /root/.docker
- echo $DOCKER_AUTH > /root/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr release --run "multiarch_docker"
- nix-shell --attr ci --run "multiarch_docker"
- name: refresh-index
image: nixpkgs/nix:nixos-22.05
environment:
@ -280,7 +280,7 @@ steps:
from_secret: garagehq_aws_secret_access_key
commands:
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
- nix-shell --attr release --run "refresh_index"
- nix-shell --attr ci --run "refresh_index"
depends_on:
- release-linux-amd64
@ -295,6 +295,6 @@ trigger:
---
kind: signature
hmac: 0c4b57eb4b27b7c6a6ff21ab87f0767fe3eb90f5d95d5cbcdccf794e9d2a5d86
hmac: 9ff9f2ad9387b12ee49a327991a5373f01e40d28e0aeef3a13d3b741d8eb3c53
...

52
flake.nix
View file

@ -33,27 +33,57 @@
compile = import ./nix/compile.nix;
in
flake-utils.lib.eachDefaultSystem (system:
let pkgs = nixpkgs.legacyPackages.${system};
in {
packages = {
default = (compile {
inherit system git_version;
let
pkgs = nixpkgs.legacyPackages.${system};
in
{
packages =
let
packageFor = target: (compile {
inherit system git_version target;
pkgsSrc = nixpkgs;
cargo2nixOverlay = cargo2nix.overlays.default;
release = true;
}).workspace.garage { compileMode = "build"; };
in
{
# default = native release build
default = packageFor null;
# other = cross-compiled, statically-linked builds
amd64 = packageFor "x86_64-unknown-linux-musl";
i386 = packageFor "i686-unknown-linux-musl";
arm64 = packageFor "aarch64-unknown-linux-musl";
arm = packageFor "armv6l-unknown-linux-musl";
};
devShell = (compile {
# ---- developpment shell, for making native builds only ----
devShells =
let
shellWithPackages = (packages: (compile {
inherit system git_version;
pkgsSrc = nixpkgs;
cargo2nixOverlay = cargo2nix.overlays.default;
release = false;
}).workspaceShell { packages = with pkgs; [
cargo-audit
cargo-outdated
}).workspaceShell { inherit packages; });
in
{
default = shellWithPackages
(with pkgs; [
rustfmt
clang
mold
]; };
]);
# import the full shell using `nix develop .#full`
full = shellWithPackages (with pkgs; [
rustfmt
clang
mold
# ---- extra packages for dev tasks ----
cargo-audit
cargo-outdated
cargo-machete
nixpkgs-fmt
]);
};
});
}

1
nix/common.nix
View file

@ -14,4 +14,5 @@ rec {
pkgsSrc = flake.defaultNix.inputs.nixpkgs;
cargo2nix = flake.defaultNix.inputs.cargo2nix;
cargo2nixOverlay = cargo2nix.overlays.default;
devShells = builtins.getAttr builtins.currentSystem flake.defaultNix.devShells;
}

133
shell.nix
View file

@ -5,87 +5,36 @@ with import ./nix/common.nix;
let
pkgs = import pkgsSrc {
inherit system;
overlays = [ cargo2nixOverlay ];
};
kaniko = (import ./nix/kaniko.nix) pkgs;
manifest-tool = (import ./nix/manifest-tool.nix) pkgs;
winscp = (import ./nix/winscp.nix) pkgs;
in
{
# --- Dev shell inherited from flake.nix ---
devShell = devShells.default;
in {
# --- Rust Shell ---
# Use it to compile Garage
rust = pkgs.mkShell {
# --- Continuous integration shell ---
# The shell used for all CI jobs (along with devShell)
ci = pkgs.mkShell {
nativeBuildInputs = with pkgs; [
#rustPlatform.rust.rustc
rustPlatform.rust.cargo
clang
mold
#clippy
rustfmt
#perl
#protobuf
#pkg-config
#openssl
file
#cargo2nix.packages.x86_64-linux.cargo2nix
];
};
# --- Integration shell ---
# Use it to test Garage with common S3 clients
integration = pkgs.mkShell {
nativeBuildInputs = [
kaniko
manifest-tool
winscp
pkgs.s3cmd
pkgs.awscli2
pkgs.minio-client
pkgs.rclone
pkgs.socat
pkgs.psmisc
pkgs.which
pkgs.openssl
pkgs.curl
pkgs.jq
awscli2
file
s3cmd
minio-client
rclone
socat
psmisc
which
openssl
curl
jq
];
};
# --- Release shell ---
# A shell built to make releasing easier
release = pkgs.mkShell {
shellHook = ''
function refresh_toolchain {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
nix copy \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR \
$(nix-build --no-build-output --no-out-link nix/toolchain.nix))
rm /tmp/nix-signing-key.sec
}
function refresh_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.{debug,release}; do
echo "Updating cache for ''${attr}"
derivation=$(nix-instantiate --attr ''${attr})
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR ''${derivation%\!bin})
done
rm /tmp/nix-signing-key.sec
}
function refresh_flake_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in packages.x86_64-linux.default devShell.x86_64-linux; do
echo "Updating cache for ''${attr}"
derivation=$(nix path-info --derivation ".#''${attr}")
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR ''${derivation})
done
rm /tmp/nix-signing-key.sec
}
function to_s3 {
aws \
--endpoint-url https://garage.deuxfleurs.fr \
@ -167,7 +116,45 @@ in {
s3://garagehq.deuxfleurs.fr/
}
'';
nativeBuildInputs = [ pkgs.awscli2 kaniko manifest-tool ];
};
# --- Cache shell ---
# A shell for refreshing caches
cache = pkgs.mkShell {
shellHook = ''
function refresh_toolchain {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix-store -qR \
$(nix-build -j8 --no-build-output --no-out-link nix/toolchain.nix))
rm /tmp/nix-signing-key.sec
}
function refresh_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in clippy.amd64 test.amd64 pkgs.{amd64,i386,arm,arm64}.release; do
echo "Updating cache for ''${attr}"
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
$(nix path-info ''${attr} --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/')
done
rm /tmp/nix-signing-key.sec
}
function refresh_flake_cache {
pass show deuxfleurs/nix_priv_key > /tmp/nix-signing-key.sec
for attr in packages.x86_64-linux.default devShells.x86_64-linux.default; do
echo "Updating cache for ''${attr}"
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/tmp/nix-signing-key.sec' \
".#''${attr}"
done
rm /tmp/nix-signing-key.sec
}
'';
};
}