Deuxfleurs/garage
48
56
Fork 73
Code Issues 100 Pull requests 13 Releases 30 Wiki Activity

Fix & simplify CI using Woodpecker #706

Merged
lx merged 7 commits from nix-improvements into main 2024-02-09 12:11:24 +00:00
Conversation 0 Commits 7 Files changed 14 +22 -356
7 changed files with 22 additions and 356 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files
Showing only changes of commit 561fad0b44 - Show all commits

300
.drone.yml
View file

@ -1,300 +0,0 @@
---
kind: pipeline
name: default
node:
nix-daemon: 1
steps:
- name: check formatting
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr devShell --run "cargo fmt -- --check"
- name: build
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- name: unit + func tests
image: nixpkgs/nix:nixos-22.05
environment:
GARAGE_TEST_INTEGRATION_EXE: result-bin/bin/garage
GARAGE_TEST_INTEGRATION_PATH: tmp-garage-integration
commands:
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-build --no-build-output --attr test.amd64
- ./result/bin/garage_db-*
- ./result/bin/garage_api-*
- ./result/bin/garage_model-*
- ./result/bin/garage_rpc-*
- ./result/bin/garage_table-*
- ./result/bin/garage_util-*
- ./result/bin/garage_web-*
- ./result/bin/garage-*
- ./result/bin/integration-* || (cat tmp-garage-integration/stderr.log; false)
- rm result
- rm -rv tmp-garage-integration
- name: integration tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr clippy.amd64 --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
trigger:
event:
- custom
- push
- pull_request
- tag
- cron
---
kind: pipeline
type: docker
name: release-linux-amd64
node:
nix-daemon: 1
steps:
- name: build
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.amd64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: integration tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
- name: upgrade tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 x86_64-unknown-linux-musl" || (cat /tmp/garage.log; false)
- name: push static binary
image: nixpkgs/nix:nixos-22.05
environment:
AWS_ACCESS_KEY_ID:
from_secret: garagehq_aws_access_key_id
AWS_SECRET_ACCESS_KEY:
from_secret: garagehq_aws_secret_access_key
TARGET: "x86_64-unknown-linux-musl"
commands:
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-22.05
environment:
DOCKER_AUTH:
from_secret: docker_auth
DOCKER_PLATFORM: "linux/amd64"
CONTAINER_NAME: "dxflrs/amd64_garage"
HOME: "/kaniko"
commands:
- mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run "to_docker"
trigger:
event:
- promote
- cron
---
kind: pipeline
type: docker
name: release-linux-i386
node:
nix-daemon: 1
steps:
- name: build
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.i386.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: integration tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
- name: upgrade tests
image: nixpkgs/nix:nixos-22.05
commands:
- nix-shell --attr ci --run "./script/test-upgrade.sh v0.8.4 i686-unknown-linux-musl" || (cat /tmp/garage.log; false)
- name: push static binary
image: nixpkgs/nix:nixos-22.05
environment:
AWS_ACCESS_KEY_ID:
from_secret: garagehq_aws_access_key_id
AWS_SECRET_ACCESS_KEY:
from_secret: garagehq_aws_secret_access_key
TARGET: "i686-unknown-linux-musl"
commands:
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-22.05
environment:
DOCKER_AUTH:
from_secret: docker_auth
DOCKER_PLATFORM: "linux/386"
CONTAINER_NAME: "dxflrs/386_garage"
HOME: "/kaniko"
commands:
- mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run "to_docker"
trigger:
event:
- promote
- cron
---
kind: pipeline
type: docker
name: release-linux-arm64
node:
nix-daemon: 1
steps:
- name: build
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.arm64.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: push static binary
image: nixpkgs/nix:nixos-22.05
environment:
AWS_ACCESS_KEY_ID:
from_secret: garagehq_aws_access_key_id
AWS_SECRET_ACCESS_KEY:
from_secret: garagehq_aws_secret_access_key
TARGET: "aarch64-unknown-linux-musl"
commands:
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-22.05
environment:
DOCKER_AUTH:
from_secret: docker_auth
DOCKER_PLATFORM: "linux/arm64"
CONTAINER_NAME: "dxflrs/arm64_garage"
HOME: "/kaniko"
commands:
- mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run "to_docker"
trigger:
event:
- promote
- cron
---
kind: pipeline
type: docker
name: release-linux-arm
node:
nix-daemon: 1
steps:
- name: build
image: nixpkgs/nix:nixos-22.05
commands:
- nix-build --no-build-output --attr pkgs.arm.release --argstr git_version ${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run "./script/not-dynamic.sh result-bin/bin/garage"
- name: push static binary
image: nixpkgs/nix:nixos-22.05
environment:
AWS_ACCESS_KEY_ID:
from_secret: garagehq_aws_access_key_id
AWS_SECRET_ACCESS_KEY:
from_secret: garagehq_aws_secret_access_key
TARGET: "armv6l-unknown-linux-musleabihf"
commands:
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-22.05
environment:
DOCKER_AUTH:
from_secret: docker_auth
DOCKER_PLATFORM: "linux/arm"
CONTAINER_NAME: "dxflrs/arm_garage"
HOME: "/kaniko"
commands:
- mkdir -p /kaniko/.docker
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run "to_docker"
trigger:
event:
- promote
- cron
---
kind: pipeline
type: docker
name: refresh-release-page
node:
nix-daemon: 1
steps:
- name: multiarch-docker
image: nixpkgs/nix:nixos-22.05
environment:
DOCKER_AUTH:
from_secret: docker_auth
HOME: "/root"
commands:
- mkdir -p /root/.docker
- echo $DOCKER_AUTH > /root/.docker/config.json
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
- nix-shell --attr ci --run "multiarch_docker"
- name: refresh-index
image: nixpkgs/nix:nixos-22.05
environment:
AWS_ACCESS_KEY_ID:
from_secret: garagehq_aws_access_key_id
AWS_SECRET_ACCESS_KEY:
from_secret: garagehq_aws_secret_access_key
commands:
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
- nix-shell --attr ci --run "refresh_index"
depends_on:
- release-linux-amd64
- release-linux-i386
- release-linux-arm64
- release-linux-arm
trigger:
event:
- promote
- cron
---
kind: signature
hmac: 9ff9f2ad9387b12ee49a327991a5373f01e40d28e0aeef3a13d3b741d8eb3c53
...

2
.woodpecker/release.yaml
View file

@ -54,7 +54,7 @@ steps:
- source: garagehq_aws_secret_access_key
target: AWS_SECRET_ACCESS_KEY
commands:
- nix-shell --attr ci --run "to_s3_woodpecker"
- nix-shell --attr ci --run "to_s3"
- name: docker build and publish
image: nixpkgs/nix:nixos-22.05

2
doc/book/design/goals.md
View file

@ -48,7 +48,5 @@ locations. They use Garage themselves for the following tasks:
- As a backup target using `rclone` and `restic`
- In the Drone continuous integration platform to store task logs
The Deuxfleurs Garage cluster is a multi-site cluster currently composed of
9 nodes in 3 physical locations.

2
doc/book/development/devenv.md
View file

@ -80,7 +80,7 @@ nix-build \
--git_version $(git rev-parse HEAD)
```
*The result is located in `result/bin`. You can pass arguments to cross compile: check `.drone.yml` for examples.*
*The result is located in `result/bin`. You can pass arguments to cross compile: check `.woodpecker/release.yml` for examples.*
If you modify a `Cargo.toml` or regenerate any `Cargo.lock`, you must run `cargo2nix`:

8
doc/book/development/miscellaneous-notes.md
View file

@ -81,12 +81,9 @@ Our cache will be checked.
- http://www.lpenz.org/articles/nixchannel/index.html
## Drone
## Woodpecker
Do not try to set a build as trusted from the interface or the CLI tool,
your request would be ignored. Instead, directly edit the database (table `repos`, column `repo_trusted`).
Drone can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner.
Woodpecker can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner.
## Building Docker containers
@ -99,3 +96,4 @@ We were:
- Unable to use the kaniko container provided by Google as we can't run arbitrary logic: we need to put our secret in .docker/config.json.
Finally we chose to build kaniko through nix and use it in a `nix-shell`.
We then switched to using kaniko from nixpkgs when it was packaged.

55
doc/book/development/release-process.md
View file

@ -42,7 +42,7 @@ and the docker containers on Docker Hub.
## Automation
We automated our release process with Nix and Drone to make it more reliable.
We automated our release process with Nix and Woodpecker to make it more reliable.
Here we describe how we have done in case you want to debug or improve it.
### Caching build steps
@ -62,52 +62,31 @@ Sending to the cache is done through `nix copy`, for example:
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' result
```
*Note that you need the signing key. In our case, it is stored as a secret in Drone.*
*The signing key possessed by the Garage maintainers is required to update the Nix cache.*
The previous command will only send the built packet and not its dependencies.
To send its dependency, a tool named `nix-copy-closure` has been created but it is not compatible with the S3 protocol.
Instead, you can use the following commands to list all the runtime dependencies:
The previous command will only send the built package and not its dependencies.
In the case of our CI pipeline, we want to cache all intermediate build steps
as well. This can be done using this quite involved command (here as an example
for the `pkgs.amd64.relase` package):
```bash
nix copy \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
$(nix-store -qR result/)
nix copy -j8 \
--to 's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/nix-signing-key.sec' \
$(nix path-info pkgs.amd64.release --file default.nix --derivation --recursive | sed 's/\.drv$/.drv^*/')
```
*We could also write this expression with xargs but this tool is not available in our container.*
This command will simultaneously build all of the required Nix paths (using at
most 8 parallel Nix builder jobs) and send the resulting objects to the cache.
But in certain cases, we want to cache compile time dependencies also.
For example, the Nix project does not provide binaries for cross compiling to i686 and thus we need to compile gcc on our own.
We do not want to compile gcc each time, so even if it is a compile time dependency, we want to cache it.
This time, the command is a bit more involved:
```bash
nix copy --to \
's3://nix?endpoint=garage.deuxfleurs.fr&region=garage&secret-key=/etc/nix/signing-key.sec' \
$(nix-store -qR --include-outputs \
$(nix-instantiate))
```
This is the command we use in our CI as we expect the final binary to change, so we mainly focus on
caching our development dependencies.
*Currently there is no automatic garbage collection of the cache: we should monitor its growth.
Hopefully, we can erase it totally without breaking any build, the next build will only be slower.*
In practise, we concluded that we do not want to cache all the compilation dependencies.
Instead, we want to cache the toolchain we use to build Garage each time we change it.
So we removed from Drone any automatic update of the cache and instead handle them manually with:
This can be run for all the Garage packages we build using the following command:
```
source ~/.awsrc
nix-shell --run 'refresh_toolchain'
nix-shell --attr cache --run 'refresh_cache'
```
Internally, it will run `nix-build` on `nix/toolchain.nix` and send the output plus its depedencies to the cache.
To erase the cache:
We don't automate this step at each CI build, as *there is currently no automatic garbage collection of the cache.*
This means we should also monitor the cache's size; if it ever becomes too big we can erase it with:
```
mc rm --recursive --force 'garage/nix/'
@ -157,9 +136,9 @@ nix-shell --run refresh_index
If you want to compile for different architectures, you will need to repeat all these commands for each architecture.
**In practise, and except for debugging, you will never directly run these commands. Release is handled by drone**
**In practice, and except for debugging, you will never directly run these commands. Release is handled by Woodpecker.**
### Drone
### Drone (obsolete)
Our instance is available at [https://drone.deuxfleurs.fr](https://drone.deuxfleurs.fr).
You need an account on [https://git.deuxfleurs.fr](https://git.deuxfleurs.fr) to use it.

9
shell.nix
View file

@ -34,15 +34,6 @@ in
];
shellHook = ''
function to_s3 {
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \
s3 cp \
./result-bin/bin/garage \
s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage
}
function to_s3_woodpecker {
aws \
--endpoint-url https://garage.deuxfleurs.fr \
--region garage \