Merge pull request 'Add helm chart' (#331) from chemicstry/garage:helm_chart into main

Reviewed-on: Deuxfleurs/garage#331
Reviewed-by: maximilien <me@mricher.fr>
This commit is contained in:
Alex 2022-10-02 16:40:54 +02:00
commit e21b672c96
13 changed files with 709 additions and 0 deletions

View file

@ -0,0 +1,87 @@
+++
title = "Deploying on Kubernetes"
weight = 32
+++
Garage can also be deployed on a kubernetes cluster via helm chart.
## Deploying
Firstly clone the repository:
```bash
git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
cd garage/scripts/helm
```
Deploy with default options:
```bash
helm install --create-namespace --namespace garage garage ./garage
```
Or deploy with custom values:
```bash
helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
```
After deploying, cluster layout must be configured manually as described in [Creating a cluster layout](@/documentation/quick-start/_index.md#creating-a-cluster-layout). Use the following command to access garage CLI:
```bash
kubectl exec --stdin --tty -n garage garage-0 -- ./garage status
```
## Overriding default values
All possible configuration values can be found with:
```bash
helm show values ./garage
```
This is an example `values.overrride.yaml` for deploying in a microk8s cluster with a https s3 api ingress route:
```yaml
garage:
# Use only 2 replicas per object
replicationMode: "2"
# Start 4 instances (StatefulSets) of garage
replicaCount: 4
# Override default storage class and size
persistence:
meta:
storageClass: "openebs-hostpath"
size: 100Mi
data:
storageClass: "openebs-hostpath"
size: 1Gi
ingress:
s3:
api:
enabled: true
className: "public"
annotations:
cert-manager.io/cluster-issuer: "letsencrypt-prod"
nginx.ingress.kubernetes.io/proxy-body-size: 500m
hosts:
- host: s3-api.my-domain.com
paths:
- path: /
pathType: Prefix
tls:
- secretName: garage-ingress-cert
hosts:
- s3-api.my-domain.com
```
## Removing
```bash
helm delete --namespace garage garage
```
Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired.

3
script/helm/README.md Normal file
View file

@ -0,0 +1,3 @@
# Garage helm3 chart
Documentation is located [here](/doc/book/cookbook/kubernetes.md).

View file

@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

View file

@ -0,0 +1,24 @@
apiVersion: v2
name: garage
description: S3-compatible object store for small self-hosted geo-distributed deployments
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.1.0
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "v0.7.2.1"

View file

@ -0,0 +1,88 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "garage.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "garage.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create the name of the rpc secret
*/}}
{{- define "garage.rpcSecretName" -}}
{{- printf "%s-rpc-secret" (include "garage.fullname" .) -}}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "garage.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "garage.labels" -}}
helm.sh/chart: {{ include "garage.chart" . }}
{{ include "garage.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "garage.selectorLabels" -}}
app.kubernetes.io/name: {{ include "garage.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "garage.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "garage.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}
{{/*
Returns given number of random Hex characters.
In practice, it generates up to 100 randAlphaNum strings
that are filtered from non-hex characters and augmented
to the resulting string that is finally trimmed down.
*/}}
{{- define "jupyterhub.randHex" -}}
{{- $result := "" }}
{{- range $i := until 100 }}
{{- if lt (len $result) . }}
{{- $rand_list := randAlphaNum . | splitList "" -}}
{{- $reduced_list := without $rand_list "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" }}
{{- $rand_string := join "" $reduced_list }}
{{- $result = print $result $rand_string -}}
{{- end }}
{{- end }}
{{- $result | trunc . }}
{{- end }}

View file

@ -0,0 +1,28 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: manage-crds-{{ .Release.Namespace }}-{{ .Release.Name }}
labels:
{{- include "garage.labels" . | nindent 4 }}
rules:
- apiGroups: ["apiextensions.k8s.io"]
resources: ["customresourcedefinitions"]
verbs: ["get", "list", "watch", "create", "patch"]
- apiGroups: ["deuxfleurs.fr"]
resources: ["garagenodes"]
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: allow-crds-for-{{ .Release.Namespace }}-{{ .Release.Name }}
labels:
{{- include "garage.labels" . | nindent 4 }}
subjects:
- kind: ServiceAccount
name: {{ include "garage.serviceAccountName" . }}
namespace: {{ .Release.Namespace }}
roleRef:
kind: ClusterRole
name: manage-crds-{{ .Release.Namespace }}-{{ .Release.Name }}
apiGroup: rbac.authorization.k8s.io

View file

@ -0,0 +1,30 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "garage.fullname" . }}-config
data:
garage.toml: |-
metadata_dir = "{{ .Values.garage.metadataDir }}"
data_dir = "{{ .Values.garage.dataDir }}"
replication_mode = "{{ .Values.garage.replicationMode }}"
rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"
# rpc_secret will be populated by the init container from a k8s secret object
rpc_secret = "__RPC_SECRET_REPLACE__"
bootstrap_peers = {{ .Values.garage.bootstrapPeers }}
kubernetes_namespace = "{{ .Release.Namespace }}"
kubernetes_service_name = "{{ include "garage.fullname" . }}"
kubernetes_skip_crd = {{ .Values.garage.kubernetesSkipCrd }}
[s3_api]
s3_region = "{{ .Values.garage.s3.api.region }}"
api_bind_addr = "[::]:3900"
root_domain = "{{ .Values.garage.s3.api.rootDomain }}"
[s3_web]
bind_addr = "[::]:3902"
root_domain = "{{ .Values.garage.s3.web.rootDomain }}"
index = "{{ .Values.garage.s3.web.index }}"

View file

@ -0,0 +1,123 @@
{{- if .Values.ingress.s3.api.enabled -}}
{{- $fullName := include "garage.fullname" . -}}
{{- $svcPort := .Values.service.s3.api.port -}}
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.s3.api.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.s3.api.annotations "kubernetes.io/ingress.class" .Values.ingress.s3.api.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}-s3-api
labels:
{{- include "garage.labels" . | nindent 4 }}
{{- with .Values.ingress.s3.api.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.s3.api.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.s3.api.className }}
{{- end }}
{{- if .Values.ingress.s3.api.tls }}
tls:
{{- range .Values.ingress.s3.api.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.s3.api.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
---
{{- if .Values.ingress.s3.web.enabled -}}
{{- $fullName := include "garage.fullname" . -}}
{{- $svcPort := .Values.service.s3.web.port -}}
{{- if and .Values.ingress.s3.web.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
{{- if not (hasKey .Values.ingress.s3.web.annotations "kubernetes.io/ingress.class") }}
{{- $_ := set .Values.ingress.s3.web.annotations "kubernetes.io/ingress.class" .Values.ingress.s3.web.className}}
{{- end }}
{{- end }}
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
apiVersion: networking.k8s.io/v1beta1
{{- else -}}
apiVersion: extensions/v1beta1
{{- end }}
kind: Ingress
metadata:
name: {{ $fullName }}-s3-web
labels:
{{- include "garage.labels" . | nindent 4 }}
{{- with .Values.ingress.s3.web.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
spec:
{{- if and .Values.ingress.s3.web.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
ingressClassName: {{ .Values.ingress.s3.web.className }}
{{- end }}
{{- if .Values.ingress.s3.web.tls }}
tls:
{{- range .Values.ingress.s3.web.tls }}
- hosts:
{{- range .hosts }}
- {{ . | quote }}
{{- end }}
secretName: {{ .secretName }}
{{- end }}
{{- end }}
rules:
{{- range .Values.ingress.s3.web.hosts }}
- host: {{ .host | quote }}
http:
paths:
{{- range .paths }}
- path: {{ .path }}
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
pathType: {{ .pathType }}
{{- end }}
backend:
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
service:
name: {{ $fullName }}
port:
number: {{ $svcPort }}
{{- else }}
serviceName: {{ $fullName }}
servicePort: {{ $svcPort }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,14 @@
apiVersion: v1
kind: Secret
metadata:
name: {{ include "garage.rpcSecretName" . }}
labels:
{{- include "garage.labels" . | nindent 4 }}
type: Opaque
data:
{{/* retrieve the secret data using lookup function and when not exists, return an empty dictionary / map as result */}}
{{- $prevSecret := (lookup "v1" "Secret" .Release.Namespace (include "garage.rpcSecretName" .)) | default dict }}
{{- $prevSecretData := $prevSecret.data | default dict }}
{{- $prevRpcSecret := $prevSecretData.rpcSecret | default "" | b64dec }}
{{/* Priority is: 1. from values, 2. previous value, 3. generate random */}}
rpcSecret: {{ .Values.garage.rpcSecret | default $prevRpcSecret | default (include "jupyterhub.randHex" 64) | b64enc | quote }}

View file

@ -0,0 +1,19 @@
apiVersion: v1
kind: Service
metadata:
name: {{ include "garage.fullname" . }}
labels:
{{- include "garage.labels" . | nindent 4 }}
spec:
type: {{ .Values.service.type }}
ports:
- port: {{ .Values.service.s3.api.port }}
targetPort: 3900
protocol: TCP
name: s3-api
- port: {{ .Values.service.s3.web.port }}
targetPort: 3902
protocol: TCP
name: s3-web
selector:
{{- include "garage.selectorLabels" . | nindent 4 }}

View file

@ -0,0 +1,12 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "garage.serviceAccountName" . }}
labels:
{{- include "garage.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

View file

@ -0,0 +1,116 @@
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: {{ include "garage.fullname" . }}
labels:
{{- include "garage.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
{{- include "garage.selectorLabels" . | nindent 6 }}
serviceName: {{ include "garage.fullname" . }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "garage.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "garage.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
initContainers:
# Copies garage.toml from configmap to temporary etc volume and replaces RPC secret placeholder
- name: {{ .Chart.Name }}-init
image: busybox:1.28
command: ["sh", "-c", "sed \"s/__RPC_SECRET_REPLACE__/$RPC_SECRET/\" /mnt/garage.toml > /mnt/etc/garage.toml"]
env:
- name: RPC_SECRET
valueFrom:
secretKeyRef:
name: {{ include "garage.rpcSecretName" . }}
key: rpcSecret
volumeMounts:
- name: configmap
mountPath: /mnt/garage.toml
subPath: garage.toml
- name: etc
mountPath: /mnt/etc
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
ports:
- containerPort: 3900
name: s3-api
- containerPort: 3902
name: web-api
volumeMounts:
- name: meta
mountPath: /mnt/meta
- name: data
mountPath: /mnt/data
- name: etc
mountPath: /etc/garage.toml
subPath: garage.toml
# TODO
# livenessProbe:
# httpGet:
# path: /
# port: 3900
# readinessProbe:
# httpGet:
# path: /
# port: 3900
resources:
{{- toYaml .Values.resources | nindent 12 }}
volumes:
- name: configmap
configMap:
name: {{ include "garage.fullname" . }}-config
- name: etc
emptyDir: {}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- if .Values.persistence.enabled }}
volumeClaimTemplates:
- metadata:
name: meta
spec:
accessModes: [ "ReadWriteOnce" ]
{{- if hasKey .Values.persistence.meta "storageClass" }}
storageClassName: {{ .Values.persistence.meta.storageClass | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.persistence.meta.size | quote }}
- metadata:
name: data
spec:
accessModes: [ "ReadWriteOnce" ]
{{- if hasKey .Values.persistence.data "storageClass" }}
storageClassName: {{ .Values.persistence.data.storageClass | quote }}
{{- end }}
resources:
requests:
storage: {{ .Values.persistence.data.size | quote }}
{{- end }}

View file

@ -0,0 +1,142 @@
# Default values for garage.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
# Garage configuration. These values go to garage.toml
garage:
metadataDir: "/mnt/meta"
dataDir: "/mnt/data"
# Default to 3 replicas, see the replication_mode section at
# https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/
replicationMode: "3"
rpcBindAddr: "[::]:3901"
# If not given, a random secret will be generated and stored in a Secret object
rpcSecret: ""
# This is not required if you use the integrated kubernetes discovery
bootstrapPeers: []
kubernetesSkipCrd: false
s3:
api:
region: "garage"
rootDomain: ".s3.garage.tld"
web:
rootDomain: ".web.garage.tld"
index: "index.html"
# Data persistence
persistence:
enabled: true
meta:
# storageClass: "fast-storage-class"
size: 100Mi
data:
# storageClass: "slow-storage-class"
size: 100Mi
# Number of StatefulSet replicas/garage nodes to start
replicaCount: 3
image:
repository: dxflrs/amd64_garage
# please prefer using the chart version and not this tag
tag: ""
pullPolicy: IfNotPresent
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
podAnnotations: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext:
# The default security context is heavily restricted
# feel free to tune it to your requirements
capabilities:
drop:
- ALL
readOnlyRootFilesystem: true
runAsNonRoot: true
runAsUser: 1000
service:
# You can rely on any service to expose your cluster
# - ClusterIP (+ Ingress)
# - NodePort (+ Ingress)
# - LoadBalancer
type: ClusterIP
s3:
api:
port: 3900
web:
port: 3902
# NOTE: the admin API is excluded for now as it is not consistent across nodes
ingress:
s3:
api:
enabled: true
# Rely either on the className or the annotation below but not both
# replace "nginx" by an Ingress controller
# you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
className: "nginx"
annotations:
# kubernetes.io/ingress.class: "nginx"
# kubernetes.io/tls-acme: "true"
hosts:
- host: "s3.garage.tld" # garage S3 API endpoint
paths:
- path: /
pathType: Prefix
- host: "*.s3.garage.tld" # garage S3 API endpoint, DNS style bucket access
paths:
- path: /
pathType: Prefix
tls: []
# - secretName: my-garage-cluster-tls
# hosts:
# - kubernetes.docker.internal
web:
enabled: true
className: "nginx"
annotations: {}
# kubernetes.io/ingress.class: nginx
# kubernetes.io/tls-acme: "true"
hosts:
- host: "*.web.garage.tld" # wildcard website access with bucket name prefix
paths:
- path: /
pathType: Prefix
- host: "mywebpage.example.com" # specific bucket access with FQDN bucket
paths:
- path: /
pathType: Prefix
tls: []
# - secretName: my-garage-cluster-tls
# hosts:
# - kubernetes.docker.internal
resources: {}
# The following are indicative for a small-size deployement, for anything serious double them.
# limits:
# cpu: 100m
# memory: 1024Mi
# requests:
# cpu: 100m
# memory: 512Mi
nodeSelector: {}
tolerations: []
affinity: {}