Build Garage with Nix
This commit is contained in:
parent
1acf7e4c66
commit
dc017a0cab
GPG key ID:
A98E9B769E4FF428
23 changed files with 4007 additions and 136 deletions
636
.drone.yml
636
.drone.yml
|
|
@ -6,103 +6,122 @@ workspace:
|
||||||
base: /drone/garage
|
base: /drone/garage
|
||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
- name: cargo_home
|
- name: nix_store
|
||||||
|
host:
|
||||||
|
path: /var/lib/drone/nix
|
||||||
|
- name: nix_config
|
||||||
temp: {}
|
temp: {}
|
||||||
|
|
||||||
|
environment:
|
||||||
|
HOME: /drone/garage
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: restore-cache
|
- name: setup nix
|
||||||
image: meltwater/drone-cache:dev
|
image: nixpkgs/nix:nixos-21.05
|
||||||
volumes:
|
volumes:
|
||||||
- name: cargo_home
|
- name: nix_store
|
||||||
path: /drone/cargo
|
path: /nix
|
||||||
environment:
|
- name: nix_config
|
||||||
AWS_ACCESS_KEY_ID:
|
path: /etc/nix
|
||||||
from_secret: cache_aws_access_key_id
|
commands:
|
||||||
AWS_SECRET_ACCESS_KEY:
|
- cp nix/nix.conf /etc/nix/nix.conf
|
||||||
from_secret: cache_aws_secret_access_key
|
- nix-build --no-build-output --no-out-link shell.nix --arg release false -A inputDerivation
|
||||||
pull: true
|
|
||||||
settings:
|
|
||||||
restore: true
|
|
||||||
archive_format: "gzip"
|
|
||||||
bucket: drone-cache
|
|
||||||
cache_key: '{{ .Repo.Name }}_{{ checksum "Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
|
|
||||||
region: garage
|
|
||||||
mount:
|
|
||||||
- '/drone/cargo'
|
|
||||||
- 'target'
|
|
||||||
path_style: true
|
|
||||||
endpoint: https://garage.deuxfleurs.fr
|
|
||||||
when:
|
|
||||||
branch:
|
|
||||||
- nonexistent_skip_this_step
|
|
||||||
|
|
||||||
- name: code quality
|
- name: code quality
|
||||||
image: superboum/garage_builder_amd64:4
|
image: nixpkgs/nix:nixos-21.05
|
||||||
volumes:
|
volumes:
|
||||||
- name: cargo_home
|
- name: nix_store
|
||||||
path: /drone/cargo
|
path: /nix
|
||||||
environment:
|
- name: nix_config
|
||||||
CARGO_HOME: /drone/cargo
|
path: /etc/nix
|
||||||
commands:
|
commands:
|
||||||
- cargo fmt -- --check
|
- nix-shell --arg release false --run "cargo fmt -- --check"
|
||||||
- cargo clippy -- --deny warnings
|
- nix-shell --arg release false --run "cargo clippy -- --allow clippy::needless_borrow --allow clippy::needless-return --deny warnings"
|
||||||
|
|
||||||
- name: build
|
- name: build
|
||||||
image: superboum/garage_builder_amd64:4
|
image: nixpkgs/nix:nixos-21.05
|
||||||
volumes:
|
volumes:
|
||||||
- name: cargo_home
|
- name: nix_store
|
||||||
path: /drone/cargo
|
path: /nix
|
||||||
environment:
|
- name: nix_config
|
||||||
CARGO_HOME: /drone/cargo
|
path: /etc/nix
|
||||||
commands:
|
commands:
|
||||||
- pwd
|
- nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
|
||||||
- cargo build
|
|
||||||
|
|
||||||
- name: cargo-test
|
- name: unit tests
|
||||||
image: superboum/garage_builder_amd64:4
|
image: nixpkgs/nix:nixos-21.05
|
||||||
volumes:
|
volumes:
|
||||||
- name: cargo_home
|
- name: nix_store
|
||||||
path: /drone/cargo
|
path: /nix
|
||||||
environment:
|
- name: nix_config
|
||||||
CARGO_HOME: /drone/cargo
|
path: /etc/nix
|
||||||
commands:
|
commands:
|
||||||
- cargo test
|
- |
|
||||||
|
nix-build \
|
||||||
|
--no-build-output \
|
||||||
|
--argstr target x86_64-unknown-linux-musl \
|
||||||
|
--argstr compileMode test
|
||||||
|
- ./result*/bin/garage_api*
|
||||||
|
- ./result*/bin/garage_model*
|
||||||
|
- ./result*/bin/garage_rpc*
|
||||||
|
- ./result*/bin/garage_table*
|
||||||
|
- ./result*/bin/garage_util*
|
||||||
|
- ./result*/bin/garage_web*
|
||||||
|
- ./result*/bin/garage*
|
||||||
|
|
||||||
- name: rebuild-cache
|
- name: smoke-test
|
||||||
image: meltwater/drone-cache:dev
|
image: nixpkgs/nix:nixos-21.05
|
||||||
volumes:
|
volumes:
|
||||||
- name: cargo_home
|
- name: nix_store
|
||||||
path: /drone/cargo
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- nix-build --no-build-output --argstr target x86_64-unknown-linux-musl --arg release false --argstr git_version $DRONE_COMMIT
|
||||||
|
- nix-shell --arg release false --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||||
|
|
||||||
|
- name: update cache
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
environment:
|
environment:
|
||||||
AWS_ACCESS_KEY_ID:
|
AWS_ACCESS_KEY_ID:
|
||||||
from_secret: cache_aws_access_key_id
|
from_secret: cache_aws_access_key_id
|
||||||
AWS_SECRET_ACCESS_KEY:
|
AWS_SECRET_ACCESS_KEY:
|
||||||
from_secret: cache_aws_secret_access_key
|
from_secret: cache_aws_secret_access_key
|
||||||
pull: true
|
NIX_PRIV_KEY:
|
||||||
settings:
|
from_secret: nix_priv_key
|
||||||
rebuild: true
|
|
||||||
archive_format: "gzip"
|
|
||||||
bucket: drone-cache
|
|
||||||
cache_key: '{{ .Repo.Name }}_{{ checksum "Cargo.lock" }}_{{ arch }}_{{ os }}_gzip'
|
|
||||||
region: garage
|
|
||||||
mount:
|
|
||||||
- '/drone/cargo'
|
|
||||||
- 'target'
|
|
||||||
path_style: true
|
|
||||||
endpoint: https://garage.deuxfleurs.fr
|
|
||||||
when:
|
|
||||||
branch:
|
|
||||||
- nonexistent_skip_this_step
|
|
||||||
|
|
||||||
- name: smoke-test
|
|
||||||
image: superboum/garage_builder_amd64:4
|
|
||||||
volumes:
|
volumes:
|
||||||
- name: cargo_home
|
- name: nix_store
|
||||||
path: /drone/cargo
|
path: /nix
|
||||||
environment:
|
- name: nix_config
|
||||||
CARGO_HOME: /drone/cargo
|
path: /etc/nix
|
||||||
commands:
|
commands:
|
||||||
- ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
- (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
|
||||||
|
- |
|
||||||
|
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||||
|
$(nix-store -qR --include-outputs \
|
||||||
|
$(nix-build --no-out-link shell.nix --arg release false -A inputDerivation))
|
||||||
|
- |
|
||||||
|
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||||
|
$(nix-store -qR --include-outputs \
|
||||||
|
$(nix-instantiate --argstr target x86_64-unknown-linux-musl --argstr compileMode test))
|
||||||
|
- |
|
||||||
|
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||||
|
$(nix-store -qR --include-outputs \
|
||||||
|
$(nix-instantiate --argstr target x86_64-unknown-linux-musl --arg release false))
|
||||||
|
when:
|
||||||
|
event:
|
||||||
|
- cron
|
||||||
|
|
||||||
|
trigger:
|
||||||
|
event:
|
||||||
|
- custom
|
||||||
|
- push
|
||||||
|
- pull_request
|
||||||
|
- tag
|
||||||
|
- cron
|
||||||
|
|
||||||
|
node:
|
||||||
|
nix: 1
|
||||||
|
|
||||||
---
|
---
|
||||||
kind: pipeline
|
kind: pipeline
|
||||||
|
|
@ -137,8 +156,477 @@ steps:
|
||||||
repo:
|
repo:
|
||||||
- Deuxfleurs/garage
|
- Deuxfleurs/garage
|
||||||
|
|
||||||
|
trigger:
|
||||||
|
event:
|
||||||
|
- custom
|
||||||
|
- push
|
||||||
|
- pull_request
|
||||||
|
|
||||||
|
node:
|
||||||
|
nix: 1
|
||||||
|
|
||||||
|
---
|
||||||
|
kind: pipeline
|
||||||
|
type: docker
|
||||||
|
name: release-linux-x86_64
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
host:
|
||||||
|
path: /var/lib/drone/nix
|
||||||
|
- name: nix_config
|
||||||
|
temp: {}
|
||||||
|
|
||||||
|
environment:
|
||||||
|
TARGET: x86_64-unknown-linux-musl
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: setup nix
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- cp nix/nix.conf /etc/nix/nix.conf
|
||||||
|
- nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
|
||||||
|
|
||||||
|
- name: build
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
|
||||||
|
|
||||||
|
- name: integration
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||||
|
|
||||||
|
- name: update cache
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: cache_aws_access_key_id
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: cache_aws_secret_access_key
|
||||||
|
NIX_PRIV_KEY:
|
||||||
|
from_secret: nix_priv_key
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
|
||||||
|
- |
|
||||||
|
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||||
|
$(nix-store -qR --include-outputs \
|
||||||
|
$(nix-instantiate --argstr target $TARGET --arg release true))
|
||||||
|
|
||||||
|
- name: push static binary
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: garagehq_aws_access_key_id
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: garagehq_aws_secret_access_key
|
||||||
|
commands:
|
||||||
|
- nix-shell --arg rust false --arg integration false --run "to_s3"
|
||||||
|
|
||||||
|
- name: docker build and publish
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
environment:
|
||||||
|
DOCKER_AUTH:
|
||||||
|
from_secret: docker_auth
|
||||||
|
DOCKER_PLATFORM: "linux/amd64"
|
||||||
|
CONTAINER_NAME: "dxflrs/amd64_garage"
|
||||||
|
HOME: "/kaniko"
|
||||||
|
commands:
|
||||||
|
- mkdir -p /kaniko/.docker
|
||||||
|
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||||
|
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||||
|
- nix-shell --arg rust false --arg integration false --run "to_docker"
|
||||||
|
|
||||||
|
|
||||||
|
trigger:
|
||||||
|
event:
|
||||||
|
- promote
|
||||||
|
- cron
|
||||||
|
|
||||||
|
node:
|
||||||
|
nix: 1
|
||||||
|
|
||||||
|
---
|
||||||
|
kind: pipeline
|
||||||
|
type: docker
|
||||||
|
name: release-linux-i686
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
host:
|
||||||
|
path: /var/lib/drone/nix
|
||||||
|
- name: nix_config
|
||||||
|
temp: {}
|
||||||
|
|
||||||
|
environment:
|
||||||
|
TARGET: i686-unknown-linux-musl
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: setup nix
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- cp nix/nix.conf /etc/nix/nix.conf
|
||||||
|
- nix-build --no-build-output --no-out-link shell.nix -A inputDerivation
|
||||||
|
|
||||||
|
- name: build
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
|
||||||
|
|
||||||
|
- name: integration
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- nix-shell --run ./script/test-smoke.sh || (cat /tmp/garage.log; false)
|
||||||
|
|
||||||
|
- name: update cache
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: cache_aws_access_key_id
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: cache_aws_secret_access_key
|
||||||
|
NIX_PRIV_KEY:
|
||||||
|
from_secret: nix_priv_key
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
|
||||||
|
- |
|
||||||
|
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||||
|
$(nix-store -qR --include-outputs \
|
||||||
|
$(nix-instantiate --argstr target $TARGET --arg release true))
|
||||||
|
|
||||||
|
- name: push static binary
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: garagehq_aws_access_key_id
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: garagehq_aws_secret_access_key
|
||||||
|
commands:
|
||||||
|
- nix-shell --arg rust false --arg integration false --run "to_s3"
|
||||||
|
|
||||||
|
- name: docker build and publish
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
environment:
|
||||||
|
DOCKER_AUTH:
|
||||||
|
from_secret: docker_auth
|
||||||
|
DOCKER_PLATFORM: "linux/386"
|
||||||
|
CONTAINER_NAME: "dxflrs/386_garage"
|
||||||
|
HOME: "/kaniko"
|
||||||
|
commands:
|
||||||
|
- mkdir -p /kaniko/.docker
|
||||||
|
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||||
|
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||||
|
- nix-shell --arg rust false --arg integration false --run "to_docker"
|
||||||
|
|
||||||
|
trigger:
|
||||||
|
event:
|
||||||
|
- promote
|
||||||
|
- cron
|
||||||
|
|
||||||
|
node:
|
||||||
|
nix: 1
|
||||||
|
|
||||||
|
---
|
||||||
|
kind: pipeline
|
||||||
|
type: docker
|
||||||
|
name: release-linux-aarch64
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
host:
|
||||||
|
path: /var/lib/drone/nix
|
||||||
|
- name: nix_config
|
||||||
|
temp: {}
|
||||||
|
|
||||||
|
environment:
|
||||||
|
TARGET: aarch64-unknown-linux-musl
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: setup nix
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- cp nix/nix.conf /etc/nix/nix.conf
|
||||||
|
- nix-build --no-build-output --no-out-link ./shell.nix --arg rust false --arg integration false -A inputDerivation
|
||||||
|
|
||||||
|
- name: build
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
|
||||||
|
|
||||||
|
- name: update cache
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: cache_aws_access_key_id
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: cache_aws_secret_access_key
|
||||||
|
NIX_PRIV_KEY:
|
||||||
|
from_secret: nix_priv_key
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
|
||||||
|
- |
|
||||||
|
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||||
|
$(nix-store -qR --include-outputs \
|
||||||
|
$(nix-instantiate --argstr target $TARGET --arg release true))
|
||||||
|
|
||||||
|
- name: push static binary
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: garagehq_aws_access_key_id
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: garagehq_aws_secret_access_key
|
||||||
|
commands:
|
||||||
|
- nix-shell --arg rust false --arg integration false --run "to_s3"
|
||||||
|
|
||||||
|
- name: docker build and publish
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
environment:
|
||||||
|
DOCKER_AUTH:
|
||||||
|
from_secret: docker_auth
|
||||||
|
DOCKER_PLATFORM: "linux/arm64"
|
||||||
|
CONTAINER_NAME: "dxflrs/arm64_garage"
|
||||||
|
HOME: "/kaniko"
|
||||||
|
commands:
|
||||||
|
- mkdir -p /kaniko/.docker
|
||||||
|
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||||
|
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||||
|
- nix-shell --arg rust false --arg integration false --run "to_docker"
|
||||||
|
|
||||||
|
trigger:
|
||||||
|
event:
|
||||||
|
- promote
|
||||||
|
- cron
|
||||||
|
|
||||||
|
node:
|
||||||
|
nix: 1
|
||||||
|
|
||||||
|
---
|
||||||
|
kind: pipeline
|
||||||
|
type: docker
|
||||||
|
name: release-linux-armv6l
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
host:
|
||||||
|
path: /var/lib/drone/nix
|
||||||
|
- name: nix_config
|
||||||
|
temp: {}
|
||||||
|
|
||||||
|
environment:
|
||||||
|
TARGET: armv6l-unknown-linux-musleabihf
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: setup nix
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- cp nix/nix.conf /etc/nix/nix.conf
|
||||||
|
- nix-build --no-build-output --no-out-link --arg rust false --arg integration false -A inputDerivation
|
||||||
|
|
||||||
|
- name: build
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- nix-build --no-build-output --argstr target $TARGET --arg release true --argstr git_version $DRONE_COMMIT
|
||||||
|
|
||||||
|
- name: update cache
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: cache_aws_access_key_id
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: cache_aws_secret_access_key
|
||||||
|
NIX_PRIV_KEY:
|
||||||
|
from_secret: nix_priv_key
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
commands:
|
||||||
|
- (umask 377 && echo $NIX_PRIV_KEY > /etc/nix/signing-key.sec)
|
||||||
|
- |
|
||||||
|
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||||
|
$(nix-store -qR --include-outputs \
|
||||||
|
$(nix-instantiate --argstr target $TARGET --arg release true))
|
||||||
|
|
||||||
|
- name: push static binary
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: garagehq_aws_access_key_id
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: garagehq_aws_secret_access_key
|
||||||
|
commands:
|
||||||
|
- nix-shell --arg integration false --arg rust false --run "to_s3"
|
||||||
|
|
||||||
|
- name: docker build and publish
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
- name: nix_config
|
||||||
|
path: /etc/nix
|
||||||
|
environment:
|
||||||
|
DOCKER_AUTH:
|
||||||
|
from_secret: docker_auth
|
||||||
|
DOCKER_PLATFORM: "linux/arm"
|
||||||
|
CONTAINER_NAME: "dxflrs/arm_garage"
|
||||||
|
HOME: "/kaniko"
|
||||||
|
commands:
|
||||||
|
- mkdir -p /kaniko/.docker
|
||||||
|
- echo $DOCKER_AUTH > /kaniko/.docker/config.json
|
||||||
|
- export CONTAINER_TAG=${DRONE_TAG:-$DRONE_COMMIT}
|
||||||
|
- nix-shell --arg rust false --arg integration false --run "to_docker"
|
||||||
|
|
||||||
|
trigger:
|
||||||
|
event:
|
||||||
|
- promote
|
||||||
|
- cron
|
||||||
|
|
||||||
|
node:
|
||||||
|
nix: 1
|
||||||
|
|
||||||
|
---
|
||||||
|
kind: pipeline
|
||||||
|
type: docker
|
||||||
|
name: refresh-release-page
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
host:
|
||||||
|
path: /var/lib/drone/nix
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: refresh-index
|
||||||
|
image: nixpkgs/nix:nixos-21.05
|
||||||
|
volumes:
|
||||||
|
- name: nix_store
|
||||||
|
path: /nix
|
||||||
|
environment:
|
||||||
|
AWS_ACCESS_KEY_ID:
|
||||||
|
from_secret: garagehq_aws_access_key_id
|
||||||
|
AWS_SECRET_ACCESS_KEY:
|
||||||
|
from_secret: garagehq_aws_secret_access_key
|
||||||
|
commands:
|
||||||
|
- mkdir -p /etc/nix && cp nix/nix.conf /etc/nix/nix.conf
|
||||||
|
- nix-shell --arg integration false --arg rust false --run "refresh_index"
|
||||||
|
|
||||||
|
depends_on:
|
||||||
|
- release-linux-x86_64
|
||||||
|
- release-linux-i686
|
||||||
|
- release-linux-aarch64
|
||||||
|
- release-linux-armv6l
|
||||||
|
|
||||||
|
trigger:
|
||||||
|
event:
|
||||||
|
- promote
|
||||||
|
- cron
|
||||||
|
|
||||||
|
node:
|
||||||
|
nix: 1
|
||||||
|
|
||||||
---
|
---
|
||||||
kind: signature
|
kind: signature
|
||||||
hmac: f0f2e947c8aa8bc5b83d25b4da22f3eb711b3fe1cc80ead4f93428dbd3d44164
|
hmac: 1c33490cc2902564c4250a409c156683d0d549b8c9d5aee4e46d1bde4e0ccf2c
|
||||||
|
|
||||||
...
|
...
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,7 @@
|
||||||
FROM archlinux:latest
|
FROM scratch
|
||||||
|
|
||||||
RUN mkdir -p /garage/meta
|
|
||||||
RUN mkdir -p /garage/data
|
|
||||||
ENV RUST_BACKTRACE=1
|
ENV RUST_BACKTRACE=1
|
||||||
ENV RUST_LOG=garage=info
|
ENV RUST_LOG=garage=info
|
||||||
|
|
||||||
COPY target/release/garage.stripped /garage/garage
|
COPY result/bin/garage /
|
||||||
|
CMD [ "/garage", "server", "-c", "config.toml"]
|
||||||
CMD /garage/garage server -c /garage/config.toml
|
|
||||||
|
|
|
||||||
7
Makefile
7
Makefile
|
|
@ -1,4 +1,4 @@
|
||||||
.PHONY: doc
|
.PHONY: doc all release shell
|
||||||
|
|
||||||
all:
|
all:
|
||||||
clear; cargo build
|
clear; cargo build
|
||||||
|
|
@ -7,4 +7,7 @@ doc:
|
||||||
cd doc/book; mdbook build
|
cd doc/book; mdbook build
|
||||||
|
|
||||||
release:
|
release:
|
||||||
RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features
|
nix-build --arg release true
|
||||||
|
|
||||||
|
shell:
|
||||||
|
nix-shell
|
||||||
|
|
|
||||||
75
default.nix
Normal file
75
default.nix
Normal file
|
|
@ -0,0 +1,75 @@
|
||||||
|
{
|
||||||
|
system ? builtins.currentSystem,
|
||||||
|
release ? false,
|
||||||
|
target ? "x86_64-unknown-linux-musl",
|
||||||
|
compileMode ? null,
|
||||||
|
git_version ? null,
|
||||||
|
}:
|
||||||
|
|
||||||
|
with import ./nix/common.nix;
|
||||||
|
|
||||||
|
let
|
||||||
|
crossSystem = { config = target; };
|
||||||
|
in let
|
||||||
|
|
||||||
|
pkgs = import pkgsSrc {
|
||||||
|
inherit system crossSystem;
|
||||||
|
overlays = [ cargo2nixOverlay rustOverlay ];
|
||||||
|
};
|
||||||
|
rustDist = pkgs.buildPackages.rust-bin.stable.latest.default;
|
||||||
|
|
||||||
|
/*
|
||||||
|
The following complexity should be abstracted by makePackageSet' (note the final quote).
|
||||||
|
However its code uses deprecated features of rust-overlay that can lead to bug.
|
||||||
|
Instead, we build our own rustChannel object with the recommended API of rust-overlay.
|
||||||
|
*/
|
||||||
|
rustChannel = rustDist // {
|
||||||
|
cargo = rustDist;
|
||||||
|
rustc = rustDist.override {
|
||||||
|
targets = [
|
||||||
|
(pkgs.rustBuilder.rustLib.realHostTriple pkgs.stdenv.targetPlatform)
|
||||||
|
];
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
overrides = pkgs.buildPackages.rustBuilder.overrides.all ++ [
|
||||||
|
(pkgs.rustBuilder.rustLib.makeOverride {
|
||||||
|
name = "garage";
|
||||||
|
overrideAttrs = drv: if git_version != null then {
|
||||||
|
preConfigure = ''
|
||||||
|
${drv.preConfigure or ""}
|
||||||
|
export GIT_VERSION="${git_version}"
|
||||||
|
'';
|
||||||
|
} else {};
|
||||||
|
})
|
||||||
|
];
|
||||||
|
|
||||||
|
packageFun = import ./Cargo.nix;
|
||||||
|
|
||||||
|
rustPkgs = pkgs.rustBuilder.makePackageSet {
|
||||||
|
inherit packageFun rustChannel release;
|
||||||
|
packageOverrides = overrides;
|
||||||
|
|
||||||
|
buildRustPackages = pkgs.buildPackages.rustBuilder.makePackageSet {
|
||||||
|
inherit rustChannel packageFun;
|
||||||
|
packageOverrides = overrides;
|
||||||
|
};
|
||||||
|
|
||||||
|
localPatterns = [
|
||||||
|
/*
|
||||||
|
The way the default rules are written make think we match recursively, on full path, but the rules are misleading.
|
||||||
|
In fact, the regex is only called on root elements of the crate (and not recursively).
|
||||||
|
This behavior does not work well with our nested modules.
|
||||||
|
We tried to build a "deny list" but negative lookup ahead are not supported on Nix.
|
||||||
|
As a workaround, we have to register all our submodules in this allow list...
|
||||||
|
*/
|
||||||
|
''^(src|tests)'' # fixed default
|
||||||
|
''.*\.(rs|toml)$'' # fixed default
|
||||||
|
''^(crdt|replication)'' # our crate submodules
|
||||||
|
];
|
||||||
|
};
|
||||||
|
|
||||||
|
in
|
||||||
|
if compileMode == "test"
|
||||||
|
then builtins.mapAttrs (name: value: rustPkgs.workspace.${name} { inherit compileMode; }) rustPkgs.workspace
|
||||||
|
else rustPkgs.workspace.garage { inherit compileMode; }
|
||||||
|
|
@ -25,7 +25,8 @@
|
||||||
|
|
||||||
- [Development](./development/index.md)
|
- [Development](./development/index.md)
|
||||||
- [Setup your environment](./development/devenv.md)
|
- [Setup your environment](./development/devenv.md)
|
||||||
- [Your first contribution]()
|
- [Development scripts](./development/scripts.md)
|
||||||
|
- [Release process](./development/release_process.md)
|
||||||
|
|
||||||
- [Working Documents](./working_documents/index.md)
|
- [Working Documents](./working_documents/index.md)
|
||||||
- [Load Balancing Data](./working_documents/load_balancing.md)
|
- [Load Balancing Data](./working_documents/load_balancing.md)
|
||||||
|
|
|
||||||
|
|
@ -1,17 +1,145 @@
|
||||||
# Setup your development environment
|
# Setup your development environment
|
||||||
|
|
||||||
We propose the following quickstart to setup a full dev. environment as quickly as possible:
|
Depending on your tastes, you can bootstrap your development environment in a traditional Rust way or through Nix.
|
||||||
|
|
||||||
1. Setup a rust/cargo environment. eg. `dnf install rust cargo`
|
## The Nix way
|
||||||
2. Install awscli v2 by following the guide [here](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2.html).
|
|
||||||
3. Run `cargo build` to build the project
|
|
||||||
4. Run `./script/dev-cluster.sh` to launch a test cluster (feel free to read the script)
|
|
||||||
5. Run `./script/dev-configure.sh` to configure your test cluster with default values (same datacenter, 100 tokens)
|
|
||||||
6. Run `./script/dev-bucket.sh` to create a bucket named `eprouvette` and an API key that will be stored in `/tmp/garage.s3`
|
|
||||||
7. Run `source ./script/dev-env-aws.sh` to configure your CLI environment
|
|
||||||
8. You can use `garage` to manage the cluster. Try `garage --help`.
|
|
||||||
9. You can use the `awsgrg` alias to add, remove, and delete files. Try `awsgrg help`, `awsgrg cp /proc/cpuinfo s3://eprouvette/cpuinfo.txt`, or `awsgrg ls s3://eprouvette`. `awsgrg` is a wrapper on the `aws s3` command pre-configured with the previously generated API key (the one in `/tmp/garage.s3`) and localhost as the endpoint.
|
|
||||||
|
|
||||||
Now you should be ready to start hacking on garage!
|
Nix is a generic package manager we use to precisely define our development environment.
|
||||||
|
Instructions on how to install it are given on their [Download page](https://nixos.org/download.html).
|
||||||
|
|
||||||
|
Check that your installation is working by running the following commands:
|
||||||
|
|
||||||
|
```
|
||||||
|
nix-shell --version
|
||||||
|
nix-build --version
|
||||||
|
nix-env --version
|
||||||
|
```
|
||||||
|
|
||||||
|
Now, you can clone our git repository (run `nix-env -iA git` if you do not have git yet):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
|
||||||
|
cd garage
|
||||||
|
```
|
||||||
|
|
||||||
|
*Optionnaly, you can use our nix.conf file to speed up compilations:*
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo mkdir -p /etc/nix
|
||||||
|
sudo cp nix/nix.conf /etc/nix/nix.conf
|
||||||
|
sudo killall nix-daemon
|
||||||
|
```
|
||||||
|
|
||||||
|
Now you can enter our nix-shell, all the required packages will be downloaded but they will not pollute your environment outside of the shell:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nix-shell
|
||||||
|
```
|
||||||
|
|
||||||
|
You can use the traditionnal Rust development workflow:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build # compile the project
|
||||||
|
cargo run # execute the project
|
||||||
|
cargo test # run the tests
|
||||||
|
cargo fmt # format the project, run it before any commit!
|
||||||
|
cargo clippy # run the linter, run it before any commit!
|
||||||
|
```
|
||||||
|
|
||||||
|
You can build the project with Nix by running:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nix-build
|
||||||
|
```
|
||||||
|
|
||||||
|
You can parallelize the build (if you use our nix.conf file, it is already automatically done).
|
||||||
|
To use all your cores when building a derivation use `-j`, and to build multiple derivations at once use `--max-jobs`.
|
||||||
|
The special value `auto` will be replaced by the number of cores of your computer.
|
||||||
|
An example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nix-build -j $(nproc) --max-jobs auto
|
||||||
|
```
|
||||||
|
|
||||||
|
Our build has multiple parameters you might want to set:
|
||||||
|
- `release` build with release optimisations instead of debug
|
||||||
|
- `target allows` for cross compilation
|
||||||
|
- `compileMode` can be set to test or bench to build a unit test runner
|
||||||
|
- `git_version` to inject the hash to display when running `garage stats`
|
||||||
|
|
||||||
|
An example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nix-build \
|
||||||
|
--arg release true \
|
||||||
|
--argstr target x86_64-unknown-linux-musl \
|
||||||
|
--argstr compileMode build \
|
||||||
|
--git_version $(git rev-parse HEAD)
|
||||||
|
```
|
||||||
|
|
||||||
|
*The result is located in `result/bin`. You can pass arguments to cross compile: check `.drone.yml` for examples.*
|
||||||
|
|
||||||
|
If you modify a `Cargo.toml` or regenerate any `Cargo.lock`, you must run `cargo2nix`:
|
||||||
|
|
||||||
|
```
|
||||||
|
cargo2nix -f
|
||||||
|
```
|
||||||
|
|
||||||
|
Many tools like rclone, `mc` (minio-client), or `aws` (awscliv2) will be available in your environment and will be useful to test Garage.
|
||||||
|
|
||||||
|
**This is the recommended method.**
|
||||||
|
|
||||||
|
## The Rust way
|
||||||
|
|
||||||
|
You need a Rust distribution installed on your computer.
|
||||||
|
The most simple way is to install it from [rustup](https://rustup.rs).
|
||||||
|
Please avoid using your package manager to install Rust as some tools might be outdated or missing.
|
||||||
|
|
||||||
|
Now, check your Rust distribution works by running the following commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
rustc --version
|
||||||
|
cargo --version
|
||||||
|
rustfmt --version
|
||||||
|
clippy-driver --version
|
||||||
|
```
|
||||||
|
|
||||||
|
Now, you need to clone our git repository ([how to install git](https://git-scm.com/downloads)):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
|
||||||
|
cd garage
|
||||||
|
```
|
||||||
|
|
||||||
|
You can now use the following commands:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build # compile the project
|
||||||
|
cargo run # execute the project
|
||||||
|
cargo test # run the tests
|
||||||
|
cargo fmt # format the project, run it before any commit!
|
||||||
|
cargo clippy # run the linter, run it before any commit!
|
||||||
|
```
|
||||||
|
|
||||||
|
This is specific to our project, but you will need one last tool, `cargo2nix`.
|
||||||
|
To install it, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo install --git https://github.com/superboum/cargo2nix --branch main cargo2nix
|
||||||
|
```
|
||||||
|
|
||||||
|
You must use it every time you modify a `Cargo.toml` or regenerate a `Cargo.lock` file as follow:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cargo build # Rebuild Cargo.lock if needed
|
||||||
|
cargo2nix -f
|
||||||
|
```
|
||||||
|
|
||||||
|
It will output a `Cargo.nix` file which is a specific `Cargo.lock` file dedicated to Nix that is required by our CI
|
||||||
|
which means you must include it in your commits.
|
||||||
|
|
||||||
|
Later, to use our scripts and integration tests, you might need additional tools.
|
||||||
|
These tools are listed at the end of the `shell.nix` package in the `nativeBuildInputs` part.
|
||||||
|
It is up to you to find a way to install the ones you need on your computer.
|
||||||
|
|
||||||
|
**A global drawback of this method is that it is up to you to adapt your environment to the one defined in the Nix files.**
|
||||||
|
|
|
||||||
134
doc/book/src/development/miscellaneous_notes.md
Normal file
134
doc/book/src/development/miscellaneous_notes.md
Normal file
|
|
@ -0,0 +1,134 @@
|
||||||
|
# Miscellaneous Notes
|
||||||
|
|
||||||
|
## Quirks about cargo2nix/rust in Nix
|
||||||
|
|
||||||
|
If you use submodules in your crate (like `crdt` and `replication` in `garage_table`), you must list them in `default.nix`
|
||||||
|
|
||||||
|
The Windows target does not work. it might be solvable through [overrides](https://github.com/cargo2nix/cargo2nix/blob/master/overlay/overrides.nix). Indeed, we pass `x86_64-pc-windows-gnu` but mingw need `x86_64-w64-mingw32`
|
||||||
|
|
||||||
|
We have a simple [PR on cargo2nix](https://github.com/cargo2nix/cargo2nix/pull/201) that fixes critical bugs but the project does not seem very active currently. We must use [my patched version of cargo2nix](https://github.com/superboum/cargo2nix) to enable i686 and armv6l compilation. We might need to contribute to cargo2nix in the future.
|
||||||
|
|
||||||
|
|
||||||
|
## Nix
|
||||||
|
|
||||||
|
Nix has no armv7 + musl toolchains but armv7l is backward compatible with armv6l.
|
||||||
|
|
||||||
|
Signing keys are generated with:
|
||||||
|
|
||||||
|
```
|
||||||
|
nix-store --generate-binary-cache-key nix.web.deuxfleurs.fr cache-priv-key.pem cache-pub-key.pem
|
||||||
|
```
|
||||||
|
|
||||||
|
We copy the secret key in our nix folder:
|
||||||
|
|
||||||
|
```
|
||||||
|
cp cache-priv-key.pem /etc/nix/signing-key.sec
|
||||||
|
```
|
||||||
|
|
||||||
|
Manually sign
|
||||||
|
|
||||||
|
We can sign the whole store with:
|
||||||
|
|
||||||
|
```
|
||||||
|
nix sign-paths --all -k /etc/nix/signing-key.sec
|
||||||
|
```
|
||||||
|
|
||||||
|
Or simply the current package and its dependencies with:
|
||||||
|
|
||||||
|
```
|
||||||
|
nix sign-paths --recursive -k /etc/nix/signing-key.sec
|
||||||
|
```
|
||||||
|
|
||||||
|
Setting a key in `nix.conf` will do the signature at build time automatically without additional commands, edit the `nix.conf` of your builder:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
secret-key-files = /etc/nix/signing-key.sec
|
||||||
|
max-jobs = auto
|
||||||
|
cores = 8
|
||||||
|
```
|
||||||
|
|
||||||
|
Now you are ready to build your packages:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cat > $HOME/.awsrc <<EOF
|
||||||
|
export AWS_ACCESS_KEY_ID="xxx"
|
||||||
|
export AWS_SECRET_ACCESS_KEY="xxx"
|
||||||
|
EOF
|
||||||
|
|
||||||
|
# source each time you want to send on the cache
|
||||||
|
source ~/.awsrc
|
||||||
|
|
||||||
|
# copy garage build dependencies (and not only the output)
|
||||||
|
nix-build
|
||||||
|
nix-store -qR --include-outputs $(nix-instantiate default.nix)
|
||||||
|
| xargs nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage'
|
||||||
|
|
||||||
|
# copy shell dependencies
|
||||||
|
nix-build shell.nix -A inputDerivation
|
||||||
|
nix copy $(nix-store -qR result/) --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage'
|
||||||
|
```
|
||||||
|
|
||||||
|
More example of nix-copy
|
||||||
|
|
||||||
|
```
|
||||||
|
# nix-build produces a result/ symlink
|
||||||
|
nix copy result/ --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage'
|
||||||
|
|
||||||
|
# alternative ways to use nix copy
|
||||||
|
nix copy nixpkgs.garage --to ...
|
||||||
|
nix copy /nix/store/3rbb9qsc2w6xl5xccz5ncfhy33nzv3dp-crate-garage-0.3.0 --to ...
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
Clear the cache:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mc rm --recursive --force garage/nix/
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
A desirable `nix.conf` for a consumer:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
|
||||||
|
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
|
||||||
|
```
|
||||||
|
|
||||||
|
And now, whenever you run a command like:
|
||||||
|
|
||||||
|
```
|
||||||
|
nix-shell
|
||||||
|
nix-build
|
||||||
|
```
|
||||||
|
|
||||||
|
Our cache will be checked.
|
||||||
|
|
||||||
|
### Some references about Nix
|
||||||
|
|
||||||
|
|
||||||
|
- https://doc.rust-lang.org/nightly/rustc/platform-support.html
|
||||||
|
- https://nix.dev/tutorials/cross-compilation
|
||||||
|
- https://nixos.org/manual/nix/unstable/package-management/s3-substituter.html
|
||||||
|
- https://fzakaria.com/2020/09/28/nix-copy-closure-your-nix-shell.html
|
||||||
|
- http://www.lpenz.org/articles/nixchannel/index.html
|
||||||
|
|
||||||
|
|
||||||
|
## Drone
|
||||||
|
|
||||||
|
Do not try to set a build as trusted from the interface or the CLI tool,
|
||||||
|
your request would be ignored. Instead, directly edit the database (table `repos`, column `repo_trusted`).
|
||||||
|
|
||||||
|
Drone can do parallelism both at the step and the pipeline level. At the step level, parallelism is restricted to the same runner.
|
||||||
|
|
||||||
|
## Building Docker containers
|
||||||
|
|
||||||
|
We were:
|
||||||
|
- Unable to use the official Docker plugin because
|
||||||
|
- it requires to mount docker socket in the container but it is not recommended
|
||||||
|
- you cant set the platform when building
|
||||||
|
- Unable to use buildah because it needs `CLONE_USERNS` capability
|
||||||
|
- Unable to use the kaniko plugin for Drone as we can't set the target platform
|
||||||
|
- Unable to use the kaniko container provided by Google as we can't run arbitrary logic: we need to put our secret in .docker/config.json.
|
||||||
|
|
||||||
|
Finally we chose to build kaniko through nix and use it in a `nix-shell`.
|
||||||
184
doc/book/src/development/release_process.md
Normal file
184
doc/book/src/development/release_process.md
Normal file
|
|
@ -0,0 +1,184 @@
|
||||||
|
# Release process
|
||||||
|
|
||||||
|
Before releasing a new version of Garage, our code pass through a succession of checks and transformations.
|
||||||
|
We define them as our release process.
|
||||||
|
|
||||||
|
## Trigger and classify a release
|
||||||
|
|
||||||
|
While we run some tests on every commits, we do not make a release for all of them.
|
||||||
|
|
||||||
|
A release can be triggered manually by "promoting" a successful build.
|
||||||
|
Otherwise, every weeks, a release build is triggered on the `main` branch.
|
||||||
|
|
||||||
|
If the build is from a tag following the regex: `v[0-9]+\.[0-9]+\.[0-9]+`, it will be listed as stable.
|
||||||
|
If it is a tag but with a different format, it will be listed as Extra.
|
||||||
|
Otherwise, if it is a commit, it will be listed as development.
|
||||||
|
This logic is defined in `nix/build_index.nix`.
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
For each commit, we first pass the code to a formatter (rustfmt) and a linter (clippy).
|
||||||
|
Then we try to build it in debug mode and run both unit tests and our integration tests.
|
||||||
|
|
||||||
|
Additionnaly, when releasing, our integration tests are run on the release build for amd64 and i686.
|
||||||
|
|
||||||
|
## Generated Artifacts
|
||||||
|
|
||||||
|
We generate the following binary artifacts for now:
|
||||||
|
- **architecture**: amd64, i686, aarch64, armv6
|
||||||
|
- **os**: linux
|
||||||
|
- **format**: static binary, docker container
|
||||||
|
|
||||||
|
Additionnaly we also build two web pages:
|
||||||
|
- the documentation (this website)
|
||||||
|
- [the release page](https://garagehq.deuxfleurs.fr/releases.html)
|
||||||
|
|
||||||
|
We publish the static binaries on our own garage cluster (you can access them through the releases page)
|
||||||
|
and the docker containers on Docker Hub.
|
||||||
|
|
||||||
|
## Automation
|
||||||
|
|
||||||
|
We automated our release process with Nix and Drone to make it more reliable.
|
||||||
|
Here we describe how we have done in case you want to debug or improve it.
|
||||||
|
|
||||||
|
### Caching build steps
|
||||||
|
|
||||||
|
To speed up the CI, we use the caching feature provided by Nix.
|
||||||
|
|
||||||
|
You can benefit from it by using our provided `nix.conf` as recommended or by simply adding the following lines to your file:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
|
||||||
|
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
|
||||||
|
```
|
||||||
|
|
||||||
|
Sending to the cache is done through `nix copy`, for example:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nix copy --to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' result
|
||||||
|
```
|
||||||
|
|
||||||
|
*Note that you need the signing key. In our case, it is stored as a secret in Drone.*
|
||||||
|
|
||||||
|
The previous command will only send the built packet and not its dependencies.
|
||||||
|
To send its dependency, a tool named `nix-copy-closure` has been created but it is not compatible with the S3 protocol.
|
||||||
|
|
||||||
|
Instead, you can use the following commands to list all the runtime dependencies:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nix copy \
|
||||||
|
--to 's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||||
|
$(nix-store -qR result/)
|
||||||
|
```
|
||||||
|
|
||||||
|
*We could also write this expression with xargs but this tool is not available in our container.*
|
||||||
|
|
||||||
|
But in certain cases, we want to cache compile time dependencies also.
|
||||||
|
For example, the Nix project does not provide binaries for cross compiling to i686 and thus we need to compile gcc on our own.
|
||||||
|
We do not want to compile gcc each time, so even if it is a compile time dependency, we want to cache it.
|
||||||
|
|
||||||
|
This time, the command is a bit more involved:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nix copy --to \
|
||||||
|
's3://nix?endpoint=garage.deuxfleurs.fr®ion=garage&secret-key=/etc/nix/signing-key.sec' \
|
||||||
|
$(nix-store -qR --include-outputs \
|
||||||
|
$(nix-instantiate))
|
||||||
|
```
|
||||||
|
|
||||||
|
This is the command we use in our CI as we expect the final binary to change, so we mainly focus on
|
||||||
|
caching our development dependencies.
|
||||||
|
|
||||||
|
*Currently there is no automatic garbage collection of the cache: we should monitor its growth.
|
||||||
|
Hopefully, we can erase it totally without breaking any build, the next build will only be slower.*
|
||||||
|
|
||||||
|
To erase the cache:
|
||||||
|
|
||||||
|
```
|
||||||
|
mc rm --recursive --force 'garage/nix/*'
|
||||||
|
```
|
||||||
|
|
||||||
|
### Publishing Garage
|
||||||
|
|
||||||
|
We defined our publishing logic in Nix, mostly as shell hooks.
|
||||||
|
You can inspect them in `shell.nix` to see exactly how.
|
||||||
|
Here, we will give a quick explanation on how to use them to manually publish a release.
|
||||||
|
|
||||||
|
Supposing you just have built garage as follow:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nix-build --arg release true
|
||||||
|
```
|
||||||
|
|
||||||
|
To publish a static binary in `result/bin` on garagehq, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export AWS_ACCESS_KEY_ID=xxx
|
||||||
|
export AWS_SECRET_ACCESS_KEY=xxx
|
||||||
|
export DRONE_TAG=handcrafted-1.0.0 # or DRONE_COMMIT
|
||||||
|
export TARGET=x86_64-unknown-linux-musl
|
||||||
|
|
||||||
|
nix-shell --run to_s3
|
||||||
|
```
|
||||||
|
|
||||||
|
To create and publish a docker container, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export DOCKER_AUTH='{ "auths": { "https://index.docker.io/v1/": { "auth": "xxxx" }}}'
|
||||||
|
export DOCKER_PLATFORM='linux/amd64' # check GOARCH and GOOS from golang.org
|
||||||
|
export CONTAINER_NAME='me/amd64_garage'
|
||||||
|
export CONTAINER_TAG='handcrafted-1.0.0'
|
||||||
|
|
||||||
|
nix-shell --run to_docker
|
||||||
|
```
|
||||||
|
|
||||||
|
To rebuild the release page, run:
|
||||||
|
```bash
|
||||||
|
export AWS_ACCESS_KEY_ID=xxx
|
||||||
|
export AWS_SECRET_ACCESS_KEY=xxx
|
||||||
|
|
||||||
|
nix-shell --run refresh_index
|
||||||
|
```
|
||||||
|
|
||||||
|
If you want to compile for different architectures, you will need to repeat all these commands for each architecture.
|
||||||
|
|
||||||
|
**In practise, and except for debugging, you will never directly run these commands. Release is handled by drone**
|
||||||
|
|
||||||
|
### Drone
|
||||||
|
|
||||||
|
Our instance is available at [https://drone.deuxfleurs.fr](https://drone.deuxfleurs.fr).
|
||||||
|
You need an account on [https://git.deuxfleurs.fr](https://git.deuxfleurs.fr) to use it.
|
||||||
|
|
||||||
|
**Drone CLI** - Drone has a CLI tool to interact with.
|
||||||
|
It can be downloaded from its Github [release page](https://github.com/drone/drone-cli/releases).
|
||||||
|
|
||||||
|
To communicate with our instance, you must setup some environment variables.
|
||||||
|
You can get them from your [Account Settings](https://drone.deuxfleurs.fr/account).
|
||||||
|
|
||||||
|
To make drone easier to use, you could create a `~/.dronerc` that you could source each time you want to use it.
|
||||||
|
|
||||||
|
```
|
||||||
|
export DRONE_SERVER=https://drone.deuxfleurs.fr
|
||||||
|
export DRONE_TOKEN=xxx
|
||||||
|
drone info
|
||||||
|
```
|
||||||
|
|
||||||
|
The CLI tool is very self-discoverable, just append `--help` to each subcommands.
|
||||||
|
Start with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
drone --help
|
||||||
|
```
|
||||||
|
|
||||||
|
**.drone.yml** - The builds steps are defined in `.drone.yml`.
|
||||||
|
You can not edit this file without resigning it.
|
||||||
|
|
||||||
|
To sign it, you must be a maintainer and then run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
drone sign --save Deuxfleurs/garage
|
||||||
|
```
|
||||||
|
|
||||||
|
Looking at the file, you will see that most of the commands are `nix-shell` and `nix-build` commands with various parameters.
|
||||||
|
|
||||||
|
|
||||||
113
doc/book/src/development/scripts.md
Normal file
113
doc/book/src/development/scripts.md
Normal file
|
|
@ -0,0 +1,113 @@
|
||||||
|
# Development scripts
|
||||||
|
|
||||||
|
We maintain a `script/` folder that contains some useful script to ease testing on Garage.
|
||||||
|
|
||||||
|
A fully integrated script, `test-smoke.sh`, runs some basic tests on various tools such as minio client, awscli and rclone.
|
||||||
|
To run it, enter a `nix-shell` (or install all required tools) and simply run:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
nix-build # or cargo build
|
||||||
|
./script/test-smoke.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
If something fails, you can find useful logs in `/tmp/garage.log`.
|
||||||
|
You can inspect the generated configuration and local data created by inspecting your `/tmp` directory:
|
||||||
|
the script creates files and folder prefixed with the name "garage".
|
||||||
|
|
||||||
|
## Bootstrapping a test cluster
|
||||||
|
|
||||||
|
Under the hood `test-smoke.sh` uses multiple helpers scripts you can also run in case you want to manually test Garage.
|
||||||
|
In this section, we introduce 3 scripts to quickly bootstrap a full test cluster with 3 instances.
|
||||||
|
|
||||||
|
### 1. Start each daemon
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./script/dev-cluster.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
This script spawns 3 Garage instances with 3 configuration files.
|
||||||
|
You can inspect the detailed configuration, including ports, by inspecting `/tmp/config.1` (change 1 by the instance number you want).
|
||||||
|
|
||||||
|
This script also spawns a simple HTTPS reverse proxy through `socat` for the S3 endpoint that listens on port `4443`.
|
||||||
|
Some libraries might require a TLS endpoint to work, refer to our issue [#64](https://git.deuxfleurs.fr/Deuxfleurs/garage/issues/64) for more detailed information on this subject.
|
||||||
|
|
||||||
|
This script covers the [Launching the garage server](/quick_start/index.html#launching-the-garage-server) section of our Quick start page.
|
||||||
|
|
||||||
|
### 2. Make them join the cluster
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./script/dev-configure.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
This script will configure each instance by assigning them a zone (`dc1`) and a weight (`1`).
|
||||||
|
|
||||||
|
This script covers the [Configuring your Garage node](/quick_start/index.html#configuring-your-garage-node) section of our Quick start page.
|
||||||
|
|
||||||
|
### 3. Create a key and a bucket
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./script/dev-bucket.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
This script will create a bucket named `eprouvette` with a key having read and write rights on this bucket.
|
||||||
|
The key is stored in a filed named `/tmp/garage.s3` and can be used by the following tools to pre-configure them.
|
||||||
|
|
||||||
|
This script covers the [Creating buckets and keys](/quick_start/index.html#creating-buckets-and-keys) section of our Quick start page.
|
||||||
|
|
||||||
|
## Handlers for generic tools
|
||||||
|
|
||||||
|
We provide wrappers for some CLI tools that configure themselves for your development cluster.
|
||||||
|
They are meant to save you some configuration time as to use them, you are only required to source the right file.
|
||||||
|
|
||||||
|
### awscli
|
||||||
|
|
||||||
|
```bash
|
||||||
|
source ./script/dev-env-aws.sh
|
||||||
|
|
||||||
|
# some examples
|
||||||
|
aws s3 ls s3://eprouvette
|
||||||
|
aws s3 cp /proc/cpuinfo s3://eprouvette/cpuinfo.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### minio-client
|
||||||
|
|
||||||
|
|
||||||
|
```bash
|
||||||
|
source ./script/dev-env-mc.sh
|
||||||
|
|
||||||
|
# some examples
|
||||||
|
mc ls garage/
|
||||||
|
mc cp /proc/cpuinfo garage/eprouvette/cpuinfo.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### rclone
|
||||||
|
|
||||||
|
```bash
|
||||||
|
source ./script/dev-env-rclone.sh
|
||||||
|
|
||||||
|
# some examples
|
||||||
|
rclone lsd garage:
|
||||||
|
rclone copy /proc/cpuinfo garage:eprouvette/cpuinfo.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### s3cmd
|
||||||
|
|
||||||
|
```bash
|
||||||
|
source ./script/dev-env-s3cmd.sh
|
||||||
|
|
||||||
|
# some examples
|
||||||
|
s3cmd ls
|
||||||
|
s3cmd put /proc/cpuinfo s3://eprouvette/cpuinfo.txt
|
||||||
|
```
|
||||||
|
|
||||||
|
### duck
|
||||||
|
|
||||||
|
*Warning! Duck is not yet provided by nix-shell.*
|
||||||
|
|
||||||
|
```bash
|
||||||
|
source ./script/dev-env-duck.sh
|
||||||
|
|
||||||
|
# some examples
|
||||||
|
duck --list garage:/
|
||||||
|
duck --upload garage:/eprouvette/ /proc/cpuinfo
|
||||||
|
```
|
||||||
|
|
@ -1,20 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
|
|
||||||
BIN=target/release/garage
|
|
||||||
DOCKER=lxpz/garage_amd64
|
|
||||||
|
|
||||||
TAG=$1
|
|
||||||
if [ -z "$1" ]; then
|
|
||||||
echo "Usage: $0 <tag>"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
RUSTFLAGS="-C link-arg=-fuse-ld=lld -C target-cpu=x86-64 -C target-feature=+sse2" cargo build --release --no-default-features
|
|
||||||
cp $BIN $BIN.stripped
|
|
||||||
strip $BIN.stripped
|
|
||||||
|
|
||||||
docker pull archlinux:latest
|
|
||||||
docker build -t $DOCKER:$TAG .
|
|
||||||
docker push $DOCKER:$TAG
|
|
||||||
docker tag $DOCKER:$TAG $DOCKER:latest
|
|
||||||
docker push $DOCKER:latest
|
|
||||||
146
nix/build_index.nix
Normal file
146
nix/build_index.nix
Normal file
|
|
@ -0,0 +1,146 @@
|
||||||
|
{
|
||||||
|
path ? "/../aws-list.txt",
|
||||||
|
}:
|
||||||
|
|
||||||
|
with import ./common.nix;
|
||||||
|
let
|
||||||
|
pkgs = import pkgsSrc {};
|
||||||
|
lib = pkgs.lib;
|
||||||
|
|
||||||
|
/* Converts a key list and a value list to a set
|
||||||
|
|
||||||
|
Example:
|
||||||
|
listToSet [ "name" "version" ] [ "latex" "3.14" ]
|
||||||
|
=> { name = "latex"; version = "3.14"; }
|
||||||
|
*/
|
||||||
|
listToSet = keys: values:
|
||||||
|
builtins.listToAttrs
|
||||||
|
(lib.zipListsWith
|
||||||
|
(a: b: { name = a; value = b; })
|
||||||
|
keys
|
||||||
|
values);
|
||||||
|
|
||||||
|
/* Says if datetime a is more recent than datetime b
|
||||||
|
|
||||||
|
Example:
|
||||||
|
cmpDate { date = "2021-09-10"; time = "22:12:15"; } { date = "2021-02-03"; time = "23:54:12"; }
|
||||||
|
=> true
|
||||||
|
*/
|
||||||
|
cmpDate = a: b:
|
||||||
|
let da = (builtins.head a.builds).date;
|
||||||
|
db = (builtins.head b.builds).date;
|
||||||
|
in
|
||||||
|
if da == db then (builtins.head a.builds).time > (builtins.head b.builds).time
|
||||||
|
else da > db;
|
||||||
|
|
||||||
|
/* Pretty platforms */
|
||||||
|
prettyPlatform = name:
|
||||||
|
if name == "aarch64-unknown-linux-musl" then "linux/arm64"
|
||||||
|
else if name == "armv6l-unknown-linux-musleabihf" then "linux/arm"
|
||||||
|
else if name == "x86_64-unknown-linux-musl" then "linux/amd64"
|
||||||
|
else if name == "i686-unknown-linux-musl" then "linux/386"
|
||||||
|
else name;
|
||||||
|
|
||||||
|
/* Parsing */
|
||||||
|
list = builtins.readFile (./. + path);
|
||||||
|
entries = lib.splitString "\n" list;
|
||||||
|
|
||||||
|
elems = builtins.filter
|
||||||
|
(e: (builtins.length e) == 4)
|
||||||
|
(map
|
||||||
|
(x: builtins.filter (e: e != "") (lib.splitString " " x))
|
||||||
|
entries);
|
||||||
|
|
||||||
|
keys = ["date" "time" "size" "path"];
|
||||||
|
parsed = map (entry: listToSet keys entry) elems;
|
||||||
|
|
||||||
|
subkeys = ["root" "version" "platform" "binary" ];
|
||||||
|
builds = map (entry: entry // listToSet subkeys (lib.splitString "/" entry.path)) parsed;
|
||||||
|
|
||||||
|
/* Aggregation */
|
||||||
|
builds_per_version = lib.foldl (acc: v: acc // { ${v.version} = if builtins.hasAttr v.version acc then acc.${v.version} ++ [ v ] else [ v ]; }) {} builds;
|
||||||
|
|
||||||
|
versions = builtins.attrNames builds_per_version;
|
||||||
|
versions_release = builtins.filter (x: builtins.match "v[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?" x != null) versions;
|
||||||
|
versions_commit = builtins.filter (x: builtins.match "[0-9a-f]{40}" x != null) versions;
|
||||||
|
versions_extra = lib.subtractLists (versions_release ++ versions_commit) versions;
|
||||||
|
|
||||||
|
sorted_builds = [
|
||||||
|
{
|
||||||
|
name = "Release";
|
||||||
|
hide = false;
|
||||||
|
type = "tag";
|
||||||
|
description = "Release builds are the official builds, they are tailored for productions and are the most tested.";
|
||||||
|
builds = builtins.sort (a: b: a.version > b.version) (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_release);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "Extra";
|
||||||
|
hide = true;
|
||||||
|
type = "tag";
|
||||||
|
description = "Extra builds are built on demand to test a specific feature or a specific need.";
|
||||||
|
builds = builtins.sort cmpDate (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_extra);
|
||||||
|
}
|
||||||
|
{
|
||||||
|
name = "Development";
|
||||||
|
hide = true;
|
||||||
|
type = "commit";
|
||||||
|
description = "Development builds are built periodically. Use them if you want to test a specific feature that is not yet released.";
|
||||||
|
builds = builtins.sort cmpDate (map (x: { version = x; builds = builtins.getAttr x builds_per_version; }) versions_commit);
|
||||||
|
}
|
||||||
|
];
|
||||||
|
|
||||||
|
in
|
||||||
|
pkgs.writeText "index.html" ''
|
||||||
|
<!doctype html>
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<title>Garage releases</title>
|
||||||
|
<style>
|
||||||
|
html, body { margin:0; padding: 0 }
|
||||||
|
body { font-family: 'Helvetica', Sans; }
|
||||||
|
section { margin: 1rem; }
|
||||||
|
ul { padding:0; margin: 0.2rem }
|
||||||
|
li {
|
||||||
|
border-radius: 0.2rem;
|
||||||
|
display: inline;
|
||||||
|
border: 2px #0b5d83 solid;
|
||||||
|
padding: 0.5rem;
|
||||||
|
line-height: 3rem;
|
||||||
|
color: #0b5d83;
|
||||||
|
}
|
||||||
|
li:hover { background-color: #0b5d83; color: #fff; }
|
||||||
|
li a, li a:hover { color: inherit; text-decoration: none }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
${ builtins.toString (lib.forEach sorted_builds (r: ''
|
||||||
|
<section>
|
||||||
|
<h2>${r.name} builds</h2>
|
||||||
|
|
||||||
|
<p>${r.description}</p>
|
||||||
|
|
||||||
|
${if r.hide then "<details><summary>Show ${r.name} builds</summary>" else ""}
|
||||||
|
${ builtins.toString (lib.forEach r.builds (x: ''
|
||||||
|
<h3> ${x.version} (${(builtins.head x.builds).date}) </h3>
|
||||||
|
<p>See this build on</p>
|
||||||
|
<p> Binaries:
|
||||||
|
<ul>
|
||||||
|
${ builtins.toString (lib.forEach x.builds (b: ''
|
||||||
|
<li><a href="/${b.path}">${prettyPlatform b.platform}</a></li>
|
||||||
|
''))}
|
||||||
|
</ul></p>
|
||||||
|
<p> Sources:
|
||||||
|
<ul>
|
||||||
|
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/src/${r.type}/${x.version}">gitea</a></li>
|
||||||
|
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.zip">.zip</a></li>
|
||||||
|
<li><a href="https://git.deuxfleurs.fr/Deuxfleurs/garage/archive/${x.version}.tar.gz">.tar.gz</a></li>
|
||||||
|
</ul></p>
|
||||||
|
'')) }
|
||||||
|
${ if builtins.length r.builds == 0 then "<em>There is no build for this category</em>" else "" }
|
||||||
|
${if r.hide then "</details>" else ""}
|
||||||
|
</section>
|
||||||
|
''))}
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
|
''
|
||||||
27
nix/common.nix
Normal file
27
nix/common.nix
Normal file
|
|
@ -0,0 +1,27 @@
|
||||||
|
rec {
|
||||||
|
/*
|
||||||
|
* Fixed dependencies
|
||||||
|
*/
|
||||||
|
pkgsSrc = fetchTarball {
|
||||||
|
# As of 2021-10-04
|
||||||
|
url ="https://github.com/NixOS/nixpkgs/archive/b27d18a412b071f5d7991d1648cfe78ee7afe68a.tar.gz";
|
||||||
|
sha256 = "1xy9zpypqfxs5gcq5dcla4bfkhxmh5nzn9dyqkr03lqycm9wg5cr";
|
||||||
|
};
|
||||||
|
cargo2nixSrc = fetchGit {
|
||||||
|
# As of 2021-10-06
|
||||||
|
url = "https://github.com/superboum/cargo2nix";
|
||||||
|
rev = "1364752cd784764db2ef5b1e1248727cebfae2ce";
|
||||||
|
};
|
||||||
|
rustOverlaySrc = fetchGit {
|
||||||
|
# As of 2021-10-05
|
||||||
|
url = "https://github.com/oxalica/rust-overlay";
|
||||||
|
rev = "9c2fc6a62ccbc6f420d71ecac6bf0b84dbbee64f";
|
||||||
|
};
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Shared objects
|
||||||
|
*/
|
||||||
|
rustOverlay = import rustOverlaySrc;
|
||||||
|
cargo2nix = import cargo2nixSrc;
|
||||||
|
cargo2nixOverlay = import "${cargo2nixSrc}/overlay";
|
||||||
|
}
|
||||||
23
nix/kaniko.nix
Normal file
23
nix/kaniko.nix
Normal file
|
|
@ -0,0 +1,23 @@
|
||||||
|
pkgs:
|
||||||
|
pkgs.buildGoModule rec {
|
||||||
|
pname = "kaniko";
|
||||||
|
version = "1.6.0";
|
||||||
|
|
||||||
|
src = pkgs.fetchFromGitHub {
|
||||||
|
owner = "GoogleContainerTools";
|
||||||
|
repo = "kaniko";
|
||||||
|
rev = "v${version}";
|
||||||
|
sha256 = "1fnclr556avxay6pvgw5ya3xbxfnf2gv4njq2hr4fd6fcjyslq5h";
|
||||||
|
};
|
||||||
|
|
||||||
|
vendorSha256 = null;
|
||||||
|
|
||||||
|
checkPhase = "true";
|
||||||
|
|
||||||
|
meta = with pkgs.lib; {
|
||||||
|
description = "kaniko is a tool to build container images from a Dockerfile, inside a container or Kubernetes cluster.";
|
||||||
|
homepage = "https://github.com/GoogleContainerTools/kaniko";
|
||||||
|
license = licenses.asl20;
|
||||||
|
platforms = platforms.linux;
|
||||||
|
};
|
||||||
|
}
|
||||||
7
nix/nix.conf
Normal file
7
nix/nix.conf
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
substituters = https://cache.nixos.org https://nix.web.deuxfleurs.fr
|
||||||
|
trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= nix.web.deuxfleurs.fr:eTGL6kvaQn6cDR/F9lDYUIP9nCVR/kkshYfLDJf1yKs=
|
||||||
|
max-jobs = auto
|
||||||
|
cores = 4
|
||||||
|
|
||||||
|
# required for containers
|
||||||
|
sandbox = false
|
||||||
|
|
@ -1,7 +0,0 @@
|
||||||
FROM rust:buster
|
|
||||||
RUN apt-get update && \
|
|
||||||
apt-get install --yes libsodium-dev awscli python-pip wget rclone openssl socat && \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
RUN wget https://dl.min.io/client/mc/release/linux-amd64/mc -O /usr/local/bin/mc && chmod +x /usr/local/bin/mc
|
|
||||||
RUN rustup component add rustfmt clippy
|
|
||||||
RUN pip install s3cmd
|
|
||||||
|
|
@ -1,8 +0,0 @@
|
||||||
DOCKER=lxpz/garage_builder_amd64
|
|
||||||
|
|
||||||
docker:
|
|
||||||
docker build -t $(DOCKER):$(TAG) .
|
|
||||||
docker push $(DOCKER):$(TAG)
|
|
||||||
docker tag $(DOCKER):$(TAG) $(DOCKER):latest
|
|
||||||
docker push $(DOCKER):latest
|
|
||||||
|
|
||||||
|
|
@ -6,7 +6,8 @@ SCRIPT_FOLDER="`dirname \"$0\"`"
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
||||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
|
||||||
|
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
||||||
|
|
||||||
garage bucket create eprouvette
|
garage bucket create eprouvette
|
||||||
KEY_INFO=`garage key new --name opérateur`
|
KEY_INFO=`garage key new --name opérateur`
|
||||||
|
|
|
||||||
|
|
@ -6,7 +6,8 @@ SCRIPT_FOLDER="`dirname \"$0\"`"
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
||||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
|
||||||
|
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
||||||
FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m")
|
FANCYCOLORS=("41m" "42m" "44m" "45m" "100m" "104m")
|
||||||
|
|
||||||
export RUST_BACKTRACE=1
|
export RUST_BACKTRACE=1
|
||||||
|
|
@ -63,7 +64,13 @@ fi
|
||||||
(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
|
(garage server -c /tmp/config.$count.toml 2>&1|while read r; do echo -en "$LABEL $r\n"; done) &
|
||||||
done
|
done
|
||||||
|
|
||||||
|
RETRY=120
|
||||||
until garage status 2>&1|grep -q Healthy ; do
|
until garage status 2>&1|grep -q Healthy ; do
|
||||||
|
(( RETRY-- ))
|
||||||
|
if (( RETRY <= 0 )); then
|
||||||
|
echo -en "${MAIN_LABEL} Garage did not start"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo -en "${MAIN_LABEL} cluster starting...\n"
|
echo -en "${MAIN_LABEL} cluster starting...\n"
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
|
|
||||||
|
|
@ -6,10 +6,17 @@ SCRIPT_FOLDER="`dirname \"$0\"`"
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
||||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
|
||||||
|
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
||||||
|
|
||||||
sleep 5
|
sleep 5
|
||||||
|
RETRY=120
|
||||||
until garage status 2>&1|grep -q Healthy ; do
|
until garage status 2>&1|grep -q Healthy ; do
|
||||||
|
(( RETRY-- ))
|
||||||
|
if (( RETRY <= 0 )); then
|
||||||
|
echo "garage did not start in time, failing."
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
echo "cluster starting..."
|
echo "cluster starting..."
|
||||||
sleep 1
|
sleep 1
|
||||||
done
|
done
|
||||||
|
|
|
||||||
|
|
@ -8,13 +8,13 @@ SCRIPT_FOLDER="`dirname \"$0\"`"
|
||||||
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
REPO_FOLDER="${SCRIPT_FOLDER}/../"
|
||||||
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
GARAGE_DEBUG="${REPO_FOLDER}/target/debug/"
|
||||||
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
GARAGE_RELEASE="${REPO_FOLDER}/target/release/"
|
||||||
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:$PATH"
|
NIX_RELEASE="${REPO_FOLDER}/result/bin/"
|
||||||
|
PATH="${GARAGE_DEBUG}:${GARAGE_RELEASE}:${NIX_RELEASE}:$PATH"
|
||||||
|
|
||||||
# @FIXME Duck is not ready for testing, we have a bug
|
# @FIXME Duck is not ready for testing, we have a bug
|
||||||
SKIP_DUCK=1
|
SKIP_DUCK=1
|
||||||
|
|
||||||
echo "⏳ Setup"
|
echo "⏳ Setup"
|
||||||
cargo build
|
|
||||||
${SCRIPT_FOLDER}/dev-clean.sh
|
${SCRIPT_FOLDER}/dev-clean.sh
|
||||||
${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
|
${SCRIPT_FOLDER}/dev-cluster.sh > /tmp/garage.log 2>&1 &
|
||||||
${SCRIPT_FOLDER}/dev-configure.sh
|
${SCRIPT_FOLDER}/dev-configure.sh
|
||||||
|
|
|
||||||
69
shell.nix
Normal file
69
shell.nix
Normal file
|
|
@ -0,0 +1,69 @@
|
||||||
|
{
|
||||||
|
system ? builtins.currentSystem,
|
||||||
|
crossSystem ? null,
|
||||||
|
rust ? true,
|
||||||
|
integration ? true,
|
||||||
|
release ? true,
|
||||||
|
}:
|
||||||
|
|
||||||
|
with import ./nix/common.nix;
|
||||||
|
|
||||||
|
let
|
||||||
|
pkgs = import pkgsSrc {
|
||||||
|
inherit system crossSystem;
|
||||||
|
overlays = [ cargo2nixOverlay rustOverlay ];
|
||||||
|
};
|
||||||
|
rustDist = pkgs.buildPackages.rust-bin.stable.latest.default;
|
||||||
|
kaniko = (import ./nix/kaniko.nix) pkgs;
|
||||||
|
|
||||||
|
in
|
||||||
|
|
||||||
|
pkgs.mkShell {
|
||||||
|
shellHook = ''
|
||||||
|
function to_s3 {
|
||||||
|
aws \
|
||||||
|
--endpoint-url https://garage.deuxfleurs.fr \
|
||||||
|
--region garage \
|
||||||
|
s3 cp \
|
||||||
|
./result/bin/garage \
|
||||||
|
s3://garagehq.deuxfleurs.fr/_releases/''${DRONE_TAG:-$DRONE_COMMIT}/''${TARGET}/garage
|
||||||
|
}
|
||||||
|
|
||||||
|
function to_docker {
|
||||||
|
executor \
|
||||||
|
--force \
|
||||||
|
--customPlatform="''${DOCKER_PLATFORM}" \
|
||||||
|
--destination "''${CONTAINER_NAME}:''${CONTAINER_TAG}" \
|
||||||
|
--context dir://`pwd` \
|
||||||
|
--verbosity=debug
|
||||||
|
}
|
||||||
|
|
||||||
|
function refresh_index {
|
||||||
|
aws \
|
||||||
|
--endpoint-url https://garage.deuxfleurs.fr \
|
||||||
|
--region garage \
|
||||||
|
s3 ls \
|
||||||
|
--recursive \
|
||||||
|
s3://garagehq.deuxfleurs.fr/_releases/ \
|
||||||
|
> aws-list.txt
|
||||||
|
|
||||||
|
nix-build nix/build_index.nix
|
||||||
|
|
||||||
|
aws \
|
||||||
|
--endpoint-url https://garage.deuxfleurs.fr \
|
||||||
|
--region garage \
|
||||||
|
s3 cp \
|
||||||
|
--content-type "text/html" \
|
||||||
|
result \
|
||||||
|
s3://garagehq.deuxfleurs.fr/_releases.html
|
||||||
|
}
|
||||||
|
'';
|
||||||
|
|
||||||
|
nativeBuildInputs =
|
||||||
|
(if rust then [ rustDist (pkgs.callPackage cargo2nix {}).package ] else [])
|
||||||
|
++
|
||||||
|
(if integration then [ pkgs.s3cmd pkgs.awscli2 pkgs.minio-client pkgs.rclone pkgs.socat pkgs.psmisc pkgs.which ] else [])
|
||||||
|
++
|
||||||
|
(if release then [ pkgs.awscli2 kaniko ] else [])
|
||||||
|
;
|
||||||
|
}
|
||||||
|
|
@ -423,11 +423,11 @@ impl AdminRpcHandler {
|
||||||
writeln!(
|
writeln!(
|
||||||
&mut ret,
|
&mut ret,
|
||||||
"\nGarage version: {}",
|
"\nGarage version: {}",
|
||||||
git_version::git_version!(
|
option_env!("GIT_VERSION").unwrap_or(git_version::git_version!(
|
||||||
prefix = "git:",
|
prefix = "git:",
|
||||||
cargo_prefix = "cargo:",
|
cargo_prefix = "cargo:",
|
||||||
fallback = "unknown"
|
fallback = "unknown"
|
||||||
)
|
))
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue