From a93dcce84196bb8ffc8cef091d1343597b15b9a6 Mon Sep 17 00:00:00 2001
From: chemicstry <chemicstry@gmail.com>
Date: Mon, 20 Jun 2022 15:52:43 +0300
Subject: [PATCH 01/10] Add helm chart

---
 script/helm/README.md                         |  63 +++++++++
 script/helm/garage/.helmignore                |  23 ++++
 script/helm/garage/Chart.yaml                 |  24 ++++
 script/helm/garage/templates/_helpers.tpl     |  62 +++++++++
 script/helm/garage/templates/configmap.yaml   |  29 ++++
 script/helm/garage/templates/ingress.yaml     | 123 +++++++++++++++++
 script/helm/garage/templates/service.yaml     |  19 +++
 .../helm/garage/templates/serviceaccount.yaml |  12 ++
 script/helm/garage/templates/statefulset.yaml |  97 ++++++++++++++
 script/helm/garage/values.yaml                | 124 ++++++++++++++++++
 10 files changed, 576 insertions(+)
 create mode 100644 script/helm/README.md
 create mode 100644 script/helm/garage/.helmignore
 create mode 100644 script/helm/garage/Chart.yaml
 create mode 100644 script/helm/garage/templates/_helpers.tpl
 create mode 100644 script/helm/garage/templates/configmap.yaml
 create mode 100644 script/helm/garage/templates/ingress.yaml
 create mode 100644 script/helm/garage/templates/service.yaml
 create mode 100644 script/helm/garage/templates/serviceaccount.yaml
 create mode 100644 script/helm/garage/templates/statefulset.yaml
 create mode 100644 script/helm/garage/values.yaml

diff --git a/script/helm/README.md b/script/helm/README.md
new file mode 100644
index 00000000..715cbab1
--- /dev/null
+++ b/script/helm/README.md
@@ -0,0 +1,63 @@
+# Garage helm3 chart
+
+This chart deploys garage on a kubernetes cluster.
+
+## Deploying
+
+With default options:
+
+```bash
+helm install --create-namespace --namespace garage garage ./garage
+```
+
+With custom values:
+
+```bash
+helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
+```
+
+## Overriding default values
+
+All possible configuration values can be found in [values.yaml](garage/values.yaml).
+
+This is an example `values.overrride.yaml` for deploying in a microk8s cluster with a https s3 api ingress route:
+
+```yaml
+# Start 4 instances (StatefulSets) of garage
+replicaCount: 4
+
+# Override default storage class and size
+persistence:
+  meta:
+    storageClass: "openebs-hostpath"
+    size: 100Mi
+  data:
+    storageClass: "openebs-hostpath"
+    size: 1Gi
+
+ingress:
+  s3:
+    api:
+      enabled: true
+      className: "public"
+      annotations:
+        cert-manager.io/cluster-issuer: "letsencrypt-prod"
+        nginx.ingress.kubernetes.io/proxy-body-size: 500m
+      hosts:
+        - host: s3-api.my-domain.com
+          paths:
+            - path: /
+              pathType: Prefix
+      tls:
+        - secretName: garage-ingress-cert
+          hosts:
+            - s3-api.my-domain.com
+```
+
+## Removing
+
+```bash
+helm delete --namespace garage garage
+```
+
+Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired.
diff --git a/script/helm/garage/.helmignore b/script/helm/garage/.helmignore
new file mode 100644
index 00000000..0e8a0eb3
--- /dev/null
+++ b/script/helm/garage/.helmignore
@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml
new file mode 100644
index 00000000..9455488a
--- /dev/null
+++ b/script/helm/garage/Chart.yaml
@@ -0,0 +1,24 @@
+apiVersion: v2
+name: garage
+description: S3-compatible object store for small self-hosted geo-distributed deployments
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: "v0.7.2"
diff --git a/script/helm/garage/templates/_helpers.tpl b/script/helm/garage/templates/_helpers.tpl
new file mode 100644
index 00000000..1a651f47
--- /dev/null
+++ b/script/helm/garage/templates/_helpers.tpl
@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "garage.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "garage.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "garage.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "garage.labels" -}}
+helm.sh/chart: {{ include "garage.chart" . }}
+{{ include "garage.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "garage.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "garage.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "garage.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "garage.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
diff --git a/script/helm/garage/templates/configmap.yaml b/script/helm/garage/templates/configmap.yaml
new file mode 100644
index 00000000..587746f6
--- /dev/null
+++ b/script/helm/garage/templates/configmap.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "garage.fullname" . }}-config
+data:
+  garage.toml: |-
+    metadata_dir = "{{ .Values.garage.metadataDir }}"
+    data_dir = "{{ .Values.garage.dataDir }}"
+
+    replication_mode = "{{ .Values.garage.replicationMode }}"
+
+    rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"
+    rpc_secret = "{{ .Values.garage.rpcSecret }}"
+
+    bootstrap_peers = {{ .Values.garage.bootstrapPeers }}
+
+    kubernetes_namespace = "{{ .Release.Namespace }}"
+    kubernetes_service_name = "{{ include "garage.fullname" . }}"
+    kubernetes_skip_crd = {{ .Values.garage.kubernetesSkipCrd }}
+
+    [s3_api]
+    s3_region = "{{ .Values.garage.s3.api.region }}"
+    api_bind_addr = "[::]:3900"
+    root_domain = "{{ .Values.garage.s3.api.rootDomain }}"
+
+    [s3_web]
+    bind_addr = "[::]:3902"
+    root_domain = "{{ .Values.garage.s3.web.rootDomain }}"
+    index = "{{ .Values.garage.s3.web.index }}"
\ No newline at end of file
diff --git a/script/helm/garage/templates/ingress.yaml b/script/helm/garage/templates/ingress.yaml
new file mode 100644
index 00000000..c4ee5a3f
--- /dev/null
+++ b/script/helm/garage/templates/ingress.yaml
@@ -0,0 +1,123 @@
+{{- if .Values.ingress.s3.api.enabled -}}
+{{- $fullName := include "garage.fullname" . -}}
+{{- $svcPort := .Values.service.s3.api.port -}}
+{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
+  {{- if not (hasKey .Values.ingress.s3.api.annotations "kubernetes.io/ingress.class") }}
+  {{- $_ := set .Values.ingress.s3.api.annotations "kubernetes.io/ingress.class" .Values.ingress.s3.api.className}}
+  {{- end }}
+{{- end }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}-s3-api
+  labels:
+    {{- include "garage.labels" . | nindent 4 }}
+  {{- with .Values.ingress.s3.api.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if and .Values.ingress.s3.api.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.s3.api.className }}
+  {{- end }}
+  {{- if .Values.ingress.s3.api.tls }}
+  tls:
+    {{- range .Values.ingress.s3.api.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.s3.api.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+            pathType: {{ .pathType }}
+            {{- end }}
+            backend:
+              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+              service:
+                name: {{ $fullName }}
+                port:
+                  number: {{ $svcPort }}
+              {{- else }}
+              serviceName: {{ $fullName }}
+              servicePort: {{ $svcPort }}
+              {{- end }}
+          {{- end }}
+    {{- end }}
+{{- end }}
+---
+{{- if .Values.ingress.s3.web.enabled -}}
+{{- $fullName := include "garage.fullname" . -}}
+{{- $svcPort := .Values.service.s3.web.port -}}
+{{- if and .Values.ingress.s3.web.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
+  {{- if not (hasKey .Values.ingress.s3.web.annotations "kubernetes.io/ingress.class") }}
+  {{- $_ := set .Values.ingress.s3.web.annotations "kubernetes.io/ingress.class" .Values.ingress.s3.web.className}}
+  {{- end }}
+{{- end }}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ $fullName }}-s3-web
+  labels:
+    {{- include "garage.labels" . | nindent 4 }}
+  {{- with .Values.ingress.s3.web.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if and .Values.ingress.s3.web.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.s3.web.className }}
+  {{- end }}
+  {{- if .Values.ingress.s3.web.tls }}
+  tls:
+    {{- range .Values.ingress.s3.web.tls }}
+    - hosts:
+        {{- range .hosts }}
+        - {{ . | quote }}
+        {{- end }}
+      secretName: {{ .secretName }}
+    {{- end }}
+  {{- end }}
+  rules:
+    {{- range .Values.ingress.s3.web.hosts }}
+    - host: {{ .host | quote }}
+      http:
+        paths:
+          {{- range .paths }}
+          - path: {{ .path }}
+            {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+            pathType: {{ .pathType }}
+            {{- end }}
+            backend:
+              {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+              service:
+                name: {{ $fullName }}
+                port:
+                  number: {{ $svcPort }}
+              {{- else }}
+              serviceName: {{ $fullName }}
+              servicePort: {{ $svcPort }}
+              {{- end }}
+          {{- end }}
+    {{- end }}
+{{- end }}
diff --git a/script/helm/garage/templates/service.yaml b/script/helm/garage/templates/service.yaml
new file mode 100644
index 00000000..2bfff99d
--- /dev/null
+++ b/script/helm/garage/templates/service.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "garage.fullname" . }}
+  labels:
+    {{- include "garage.labels" . | nindent 4 }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.s3.api.port }}
+      targetPort: 3900
+      protocol: TCP
+      name: s3-api
+    - port: {{ .Values.service.s3.web.port }}
+      targetPort: 3902
+      protocol: TCP
+      name: s3-web
+  selector:
+    {{- include "garage.selectorLabels" . | nindent 4 }}
diff --git a/script/helm/garage/templates/serviceaccount.yaml b/script/helm/garage/templates/serviceaccount.yaml
new file mode 100644
index 00000000..a0a89a33
--- /dev/null
+++ b/script/helm/garage/templates/serviceaccount.yaml
@@ -0,0 +1,12 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "garage.serviceAccountName" . }}
+  labels:
+    {{- include "garage.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
diff --git a/script/helm/garage/templates/statefulset.yaml b/script/helm/garage/templates/statefulset.yaml
new file mode 100644
index 00000000..82fe89a9
--- /dev/null
+++ b/script/helm/garage/templates/statefulset.yaml
@@ -0,0 +1,97 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "garage.fullname" . }}
+  labels:
+    {{- include "garage.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "garage.selectorLabels" . | nindent 6 }}
+  serviceName: {{ include "garage.fullname" . }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "garage.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "garage.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - containerPort: 3900
+              name: s3-api
+            - containerPort: 3902
+              name: web-api
+          volumeMounts:
+            - name: meta
+              mountPath: /mnt/meta
+            - name: data
+              mountPath: /mnt/data
+            - name: etc
+              mountPath: /etc/garage.toml
+              subPath: garage.toml
+          # TODO
+          # livenessProbe:
+          #   httpGet:
+          #     path: /
+          #     port: 3900
+          # readinessProbe:
+          #   httpGet:
+          #     path: /
+          #     port: 3900
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      volumes:
+        - name: etc
+          configMap:
+            name: {{ include "garage.fullname" . }}-config
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+  {{- if .Values.persistence.enabled }}
+  volumeClaimTemplates:
+  - metadata:
+      name: meta
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+    {{- if hasKey .Values.persistence.meta "storageClass" }}
+      storageClassName: {{ .Values.persistence.meta.storageClass | quote }}
+    {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.persistence.meta.size | quote }}
+  - metadata:
+      name: data
+    spec:
+      accessModes: [ "ReadWriteOnce" ]
+    {{- if hasKey .Values.persistence.data "storageClass" }}
+      storageClassName: {{ .Values.persistence.data.storageClass | quote }}
+    {{- end }}
+      resources:
+        requests:
+          storage: {{ .Values.persistence.data.size | quote }}
+  {{- end }}
diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml
new file mode 100644
index 00000000..dd1c99f0
--- /dev/null
+++ b/script/helm/garage/values.yaml
@@ -0,0 +1,124 @@
+# Default values for garage.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+# Garage configuration. These values go to garage.toml
+garage:
+  metadataDir: "/mnt/meta"
+  dataDir: "/mnt/data"
+  replicationMode: "3"
+  rpcBindAddr: "[::]:3901"
+  rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
+  bootstrapPeers: []
+  # kubernetes_namespace: "default"
+  # kubernetes_service_name: "garage-daemon"
+  kubernetesSkipCrd: false
+  s3:
+    api:
+      region: "garage"
+      rootDomain: ".s3.garage.tld"
+    web:
+      rootDomain: ".web.garage.tld"
+      index: "index.html"
+
+# Data persistence
+persistence:
+  enabled: true
+  meta:
+    # storageClass: ""
+    size: 100Mi
+  data:
+    # storageClass: ""
+    size: 100Mi
+
+# Number of StatefulSet replicas to start
+replicaCount: 3
+
+image:
+  repository: dxflrs/amd64_garage
+  pullPolicy: IfNotPresent
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+service:
+  type: ClusterIP
+  s3:
+    api:
+      port: 3900
+    web:
+      port: 3902
+
+ingress:
+  s3:
+    api:
+      enabled: false
+      className: ""
+      annotations: {}
+        # kubernetes.io/ingress.class: nginx
+        # kubernetes.io/tls-acme: "true"
+      hosts:
+        - host: chart-example.local
+          paths:
+            - path: /
+              pathType: ImplementationSpecific
+      tls: []
+      #  - secretName: chart-example-tls
+      #    hosts:
+      #      - chart-example.local
+    web:
+      enabled: false
+      className: ""
+      annotations: {}
+        # kubernetes.io/ingress.class: nginx
+        # kubernetes.io/tls-acme: "true"
+      hosts:
+        - host: chart-example.local
+          paths:
+            - path: /
+              pathType: ImplementationSpecific
+      tls: []
+      #  - secretName: chart-example-tls
+      #    hosts:
+      #      - chart-example.local
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}

From 131cc2532b13acfb90d38e04c5dac5fa9cd3cb0e Mon Sep 17 00:00:00 2001
From: chemicstry <chemicstry@gmail.com>
Date: Mon, 20 Jun 2022 16:02:23 +0300
Subject: [PATCH 02/10] Cleanup values.yaml

---
 script/helm/garage/values.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml
index dd1c99f0..06cf9d16 100644
--- a/script/helm/garage/values.yaml
+++ b/script/helm/garage/values.yaml
@@ -10,8 +10,6 @@ garage:
   rpcBindAddr: "[::]:3901"
   rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
   bootstrapPeers: []
-  # kubernetes_namespace: "default"
-  # kubernetes_service_name: "garage-daemon"
   kubernetesSkipCrd: false
   s3:
     api:

From fa52558ca12573763bea392f9b44f71d3a6bb457 Mon Sep 17 00:00:00 2001
From: chemicstry <chemicstry@gmail.com>
Date: Mon, 20 Jun 2022 16:02:53 +0300
Subject: [PATCH 03/10] Add configuration instructions to README

---
 script/helm/README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/script/helm/README.md b/script/helm/README.md
index 715cbab1..925d51ce 100644
--- a/script/helm/README.md
+++ b/script/helm/README.md
@@ -16,6 +16,12 @@ With custom values:
 helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
 ```
 
+After deploying, cluster layout must be configured manually as per garage's documentation. Use the following command to access garage CLI:
+
+```bash
+kubectl exec --stdin --tty -n garage garage-0 -- ./garage status
+```
+
 ## Overriding default values
 
 All possible configuration values can be found in [values.yaml](garage/values.yaml).

From d0f08c254e3cc996dbf3d565df9ff2c89e15c639 Mon Sep 17 00:00:00 2001
From: chemicstry <chemicstry@gmail.com>
Date: Mon, 20 Jun 2022 16:08:41 +0300
Subject: [PATCH 04/10] Add secret to overrides

---
 script/helm/README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/script/helm/README.md b/script/helm/README.md
index 925d51ce..b7a3a35b 100644
--- a/script/helm/README.md
+++ b/script/helm/README.md
@@ -29,6 +29,10 @@ All possible configuration values can be found in [values.yaml](garage/values.ya
 This is an example `values.overrride.yaml` for deploying in a microk8s cluster with a https s3 api ingress route:
 
 ```yaml
+garage:
+  # Make sure to generate a new secret for your deployment
+  rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
+
 # Start 4 instances (StatefulSets) of garage
 replicaCount: 4
 

From 37a73d7d3782ec8a5cd8b0e71a00722f90321ced Mon Sep 17 00:00:00 2001
From: chemicstry <chemicstry@gmail.com>
Date: Mon, 20 Jun 2022 17:11:09 +0300
Subject: [PATCH 05/10] Move documentation to book

---
 doc/book/cookbook/kubernetes.md | 87 +++++++++++++++++++++++++++++++++
 script/helm/README.md           | 72 +--------------------------
 2 files changed, 88 insertions(+), 71 deletions(-)
 create mode 100644 doc/book/cookbook/kubernetes.md

diff --git a/doc/book/cookbook/kubernetes.md b/doc/book/cookbook/kubernetes.md
new file mode 100644
index 00000000..8fd12fdf
--- /dev/null
+++ b/doc/book/cookbook/kubernetes.md
@@ -0,0 +1,87 @@
++++
+title = "Deploying on Kubernetes"
+weight = 32
++++
+
+Garage can also be deployed on a kubernetes cluster via helm chart.
+
+## Deploying
+
+Firstly clone the repository:
+
+```bash
+git clone https://git.deuxfleurs.fr/Deuxfleurs/garage
+cd garage/scripts/helm
+```
+
+Deploy with default options:
+
+```bash
+helm install --create-namespace --namespace garage garage ./garage
+```
+
+Or deploy with custom values:
+
+```bash
+helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
+```
+
+After deploying, cluster layout must be configured manually as described in [Creating a cluster layout](@/documentation/quick-start/_index.md#creating-a-cluster-layout). Use the following command to access garage CLI:
+
+```bash
+kubectl exec --stdin --tty -n garage garage-0 -- ./garage status
+```
+
+## Overriding default values
+
+All possible configuration values can be found with:
+
+```bash
+helm show values ./garage
+```
+
+This is an example `values.overrride.yaml` for deploying in a microk8s cluster with a https s3 api ingress route:
+
+```yaml
+garage:
+  # Make sure to generate a new secret for your deployment
+  rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
+
+# Start 4 instances (StatefulSets) of garage
+replicaCount: 4
+
+# Override default storage class and size
+persistence:
+  meta:
+    storageClass: "openebs-hostpath"
+    size: 100Mi
+  data:
+    storageClass: "openebs-hostpath"
+    size: 1Gi
+
+ingress:
+  s3:
+    api:
+      enabled: true
+      className: "public"
+      annotations:
+        cert-manager.io/cluster-issuer: "letsencrypt-prod"
+        nginx.ingress.kubernetes.io/proxy-body-size: 500m
+      hosts:
+        - host: s3-api.my-domain.com
+          paths:
+            - path: /
+              pathType: Prefix
+      tls:
+        - secretName: garage-ingress-cert
+          hosts:
+            - s3-api.my-domain.com
+```
+
+## Removing
+
+```bash
+helm delete --namespace garage garage
+```
+
+Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired.
diff --git a/script/helm/README.md b/script/helm/README.md
index b7a3a35b..5f919a23 100644
--- a/script/helm/README.md
+++ b/script/helm/README.md
@@ -1,73 +1,3 @@
 # Garage helm3 chart
 
-This chart deploys garage on a kubernetes cluster.
-
-## Deploying
-
-With default options:
-
-```bash
-helm install --create-namespace --namespace garage garage ./garage
-```
-
-With custom values:
-
-```bash
-helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml
-```
-
-After deploying, cluster layout must be configured manually as per garage's documentation. Use the following command to access garage CLI:
-
-```bash
-kubectl exec --stdin --tty -n garage garage-0 -- ./garage status
-```
-
-## Overriding default values
-
-All possible configuration values can be found in [values.yaml](garage/values.yaml).
-
-This is an example `values.overrride.yaml` for deploying in a microk8s cluster with a https s3 api ingress route:
-
-```yaml
-garage:
-  # Make sure to generate a new secret for your deployment
-  rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
-
-# Start 4 instances (StatefulSets) of garage
-replicaCount: 4
-
-# Override default storage class and size
-persistence:
-  meta:
-    storageClass: "openebs-hostpath"
-    size: 100Mi
-  data:
-    storageClass: "openebs-hostpath"
-    size: 1Gi
-
-ingress:
-  s3:
-    api:
-      enabled: true
-      className: "public"
-      annotations:
-        cert-manager.io/cluster-issuer: "letsencrypt-prod"
-        nginx.ingress.kubernetes.io/proxy-body-size: 500m
-      hosts:
-        - host: s3-api.my-domain.com
-          paths:
-            - path: /
-              pathType: Prefix
-      tls:
-        - secretName: garage-ingress-cert
-          hosts:
-            - s3-api.my-domain.com
-```
-
-## Removing
-
-```bash
-helm delete --namespace garage garage
-```
-
-Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired.
+Documentation is located [here](/doc/book/cookbook/kubernetes.md).

From b71fa2ddf45e21f40067fc021b3a81d738556eca Mon Sep 17 00:00:00 2001
From: chemicstry <chemicstry@gmail.com>
Date: Mon, 20 Jun 2022 18:49:38 +0300
Subject: [PATCH 06/10] Generate random RPC secret if not provided

---
 script/helm/garage/templates/_helpers.tpl     | 26 +++++++++++++++++++
 script/helm/garage/templates/configmap.yaml   |  3 ++-
 script/helm/garage/templates/secret.yaml      | 14 ++++++++++
 script/helm/garage/templates/statefulset.yaml | 21 ++++++++++++++-
 script/helm/garage/values.yaml                |  3 ++-
 5 files changed, 64 insertions(+), 3 deletions(-)
 create mode 100644 script/helm/garage/templates/secret.yaml

diff --git a/script/helm/garage/templates/_helpers.tpl b/script/helm/garage/templates/_helpers.tpl
index 1a651f47..037a5f1c 100644
--- a/script/helm/garage/templates/_helpers.tpl
+++ b/script/helm/garage/templates/_helpers.tpl
@@ -23,6 +23,13 @@ If release name contains chart name it will be used as a full name.
 {{- end }}
 {{- end }}
 
+{{/*
+Create the name of the rpc secret
+*/}}
+{{- define "garage.rpcSecretName" -}}
+{{- printf "%s-rpc-secret" (include "garage.fullname" .) -}}
+{{- end }}
+
 {{/*
 Create chart name and version as used by the chart label.
 */}}
@@ -60,3 +67,22 @@ Create the name of the service account to use
 {{- default "default" .Values.serviceAccount.name }}
 {{- end }}
 {{- end }}
+
+{{/*
+    Returns given number of random Hex characters.
+    In practice, it generates up to 100 randAlphaNum strings
+    that are filtered from non-hex characters and augmented
+    to the resulting string that is finally trimmed down.
+*/}}
+{{- define "jupyterhub.randHex" -}}
+    {{- $result := "" }}
+    {{- range $i := until 100 }}
+        {{- if lt (len $result) . }}
+            {{- $rand_list := randAlphaNum . | splitList "" -}}
+            {{- $reduced_list := without $rand_list "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" }}
+            {{- $rand_string := join "" $reduced_list }}
+            {{- $result = print $result $rand_string -}}
+        {{- end }}
+    {{- end }}
+    {{- $result | trunc . }}
+{{- end }}
diff --git a/script/helm/garage/templates/configmap.yaml b/script/helm/garage/templates/configmap.yaml
index 587746f6..e33a4dbd 100644
--- a/script/helm/garage/templates/configmap.yaml
+++ b/script/helm/garage/templates/configmap.yaml
@@ -10,7 +10,8 @@ data:
     replication_mode = "{{ .Values.garage.replicationMode }}"
 
     rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"
-    rpc_secret = "{{ .Values.garage.rpcSecret }}"
+    # rpc_secret will be populated by the init container from a k8s secret object
+    rpc_secret = "__RPC_SECRET_REPLACE__"
 
     bootstrap_peers = {{ .Values.garage.bootstrapPeers }}
 
diff --git a/script/helm/garage/templates/secret.yaml b/script/helm/garage/templates/secret.yaml
new file mode 100644
index 00000000..54749424
--- /dev/null
+++ b/script/helm/garage/templates/secret.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "garage.rpcSecretName" . }}
+  labels:
+    {{- include "garage.labels" . | nindent 4 }}
+type: Opaque
+data:
+  {{/* retrieve the secret data using lookup function and when not exists, return an empty dictionary / map as result */}}
+  {{- $prevSecret := (lookup "v1" "Secret" .Release.Namespace (include "garage.rpcSecretName" .)) | default dict }}
+  {{- $prevSecretData := $prevSecret.data | default dict }}
+  {{- $prevRpcSecret := $prevSecretData.rpcSecret | default "" | b64dec }}
+  {{/* Priority is: 1. from values, 2. previous value, 3. generate random */}}
+  rpcSecret: {{ .Values.garage.rpcSecret | default $prevRpcSecret | default (include "jupyterhub.randHex" 64) | b64enc | quote }}
diff --git a/script/helm/garage/templates/statefulset.yaml b/script/helm/garage/templates/statefulset.yaml
index 82fe89a9..bda40117 100644
--- a/script/helm/garage/templates/statefulset.yaml
+++ b/script/helm/garage/templates/statefulset.yaml
@@ -26,6 +26,23 @@ spec:
       serviceAccountName: {{ include "garage.serviceAccountName" . }}
       securityContext:
         {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      initContainers:
+        # Copies garage.toml from configmap to temporary etc volume and replaces RPC secret placeholder
+        - name: {{ .Chart.Name }}-init
+          image: busybox:1.28
+          command: ["sh", "-c", "sed \"s/__RPC_SECRET_REPLACE__/$RPC_SECRET/\" /mnt/garage.toml > /mnt/etc/garage.toml"]
+          env:
+            - name: RPC_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "garage.rpcSecretName" . }}
+                  key: rpcSecret
+          volumeMounts:
+            - name: configmap
+              mountPath: /mnt/garage.toml
+              subPath: garage.toml
+            - name: etc
+              mountPath: /mnt/etc
       containers:
         - name: {{ .Chart.Name }}
           securityContext:
@@ -57,9 +74,11 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       volumes:
-        - name: etc
+        - name: configmap
           configMap:
             name: {{ include "garage.fullname" . }}-config
+        - name: etc
+          emptyDir: {}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml
index 06cf9d16..d011f63e 100644
--- a/script/helm/garage/values.yaml
+++ b/script/helm/garage/values.yaml
@@ -8,7 +8,8 @@ garage:
   dataDir: "/mnt/data"
   replicationMode: "3"
   rpcBindAddr: "[::]:3901"
-  rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
+  # If not given, a random secret will be generated
+  rpcSecret: ""
   bootstrapPeers: []
   kubernetesSkipCrd: false
   s3:

From 744c3b4d9487045ab04a221572722afa0ca34b09 Mon Sep 17 00:00:00 2001
From: chemicstry <chemicstry@gmail.com>
Date: Mon, 20 Jun 2022 18:52:32 +0300
Subject: [PATCH 07/10] Update docs

---
 doc/book/cookbook/kubernetes.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/book/cookbook/kubernetes.md b/doc/book/cookbook/kubernetes.md
index 8fd12fdf..0bf89c96 100644
--- a/doc/book/cookbook/kubernetes.md
+++ b/doc/book/cookbook/kubernetes.md
@@ -44,8 +44,8 @@ This is an example `values.overrride.yaml` for deploying in a microk8s cluster w
 
 ```yaml
 garage:
-  # Make sure to generate a new secret for your deployment
-  rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec"
+  # Use only 2 replicas per object
+  replicationMode: "3"
 
 # Start 4 instances (StatefulSets) of garage
 replicaCount: 4

From d2c937a931b6549ffd2f2afdd7a871be8d8eefbf Mon Sep 17 00:00:00 2001
From: chemicstry <chemicstry@gmail.com>
Date: Tue, 21 Jun 2022 16:16:42 +0300
Subject: [PATCH 08/10] Fix typo

---
 doc/book/cookbook/kubernetes.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/book/cookbook/kubernetes.md b/doc/book/cookbook/kubernetes.md
index 0bf89c96..9eafe3e1 100644
--- a/doc/book/cookbook/kubernetes.md
+++ b/doc/book/cookbook/kubernetes.md
@@ -45,7 +45,7 @@ This is an example `values.overrride.yaml` for deploying in a microk8s cluster w
 ```yaml
 garage:
   # Use only 2 replicas per object
-  replicationMode: "3"
+  replicationMode: "2"
 
 # Start 4 instances (StatefulSets) of garage
 replicaCount: 4

From 6dba7dadf44781abfb878f06fba86e731b267c87 Mon Sep 17 00:00:00 2001
From: Maximilien R <maximilien@deuxfleurs.fr>
Date: Wed, 22 Jun 2022 10:04:59 +0200
Subject: [PATCH 09/10] Add missing ClusterRole and bindings for CRDs

---
 script/helm/garage/templates/clusterrole.yaml | 28 +++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 script/helm/garage/templates/clusterrole.yaml

diff --git a/script/helm/garage/templates/clusterrole.yaml b/script/helm/garage/templates/clusterrole.yaml
new file mode 100644
index 00000000..fa3e6405
--- /dev/null
+++ b/script/helm/garage/templates/clusterrole.yaml
@@ -0,0 +1,28 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: manage-crds-{{ .Release.Namespace }}-{{ .Release.Name }}
+  labels:
+    {{- include "garage.labels" . | nindent 4 }}
+rules:
+- apiGroups: ["apiextensions.k8s.io"]
+  resources: ["customresourcedefinitions"]
+  verbs: ["get", "list", "watch", "create", "patch"]
+- apiGroups: ["deuxfleurs.fr"]
+  resources: ["garagenodes"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: allow-crds-for-{{ .Release.Namespace }}-{{ .Release.Name }}
+  labels:
+    {{- include "garage.labels" . | nindent 4 }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "garage.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: manage-crds-{{ .Release.Namespace }}-{{ .Release.Name }}
+  apiGroup: rbac.authorization.k8s.io
\ No newline at end of file

From db0c8b3980c5cb056c9402332dd09a1bfb276997 Mon Sep 17 00:00:00 2001
From: Maximilien R <maximilien@deuxfleurs.fr>
Date: Thu, 11 Aug 2022 01:35:41 +0200
Subject: [PATCH 10/10] Updates values.yml with some opinionated and untested
 defaults

---
 script/helm/garage/Chart.yaml  |  2 +-
 script/helm/garage/values.yaml | 87 +++++++++++++++++++++-------------
 2 files changed, 54 insertions(+), 35 deletions(-)

diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml
index 9455488a..56598ea4 100644
--- a/script/helm/garage/Chart.yaml
+++ b/script/helm/garage/Chart.yaml
@@ -21,4 +21,4 @@ version: 0.1.0
 # incremented each time you make changes to the application. Versions are not expected to
 # follow Semantic Versioning. They should reflect the version the application is using.
 # It is recommended to use it with quotes.
-appVersion: "v0.7.2"
+appVersion: "v0.7.2.1"
diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml
index d011f63e..08d0c09b 100644
--- a/script/helm/garage/values.yaml
+++ b/script/helm/garage/values.yaml
@@ -6,10 +6,13 @@
 garage:
   metadataDir: "/mnt/meta"
   dataDir: "/mnt/data"
+  # Default to 3 replicas, see the replication_mode section at
+  # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/
   replicationMode: "3"
   rpcBindAddr: "[::]:3901"
-  # If not given, a random secret will be generated
+  # If not given, a random secret will be generated and stored in a Secret object
   rpcSecret: ""
+  # This is not required if you use the integrated kubernetes discovery
   bootstrapPeers: []
   kubernetesSkipCrd: false
   s3:
@@ -24,17 +27,19 @@ garage:
 persistence:
   enabled: true
   meta:
-    # storageClass: ""
+    # storageClass: "fast-storage-class"
     size: 100Mi
   data:
-    # storageClass: ""
+    # storageClass: "slow-storage-class"
     size: 100Mi
 
-# Number of StatefulSet replicas to start
+# Number of StatefulSet replicas/garage nodes to start
 replicaCount: 3
 
 image:
   repository: dxflrs/amd64_garage
+  # please prefer using the chart version and not this tag
+  tag: ""
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -55,66 +60,80 @@ podAnnotations: {}
 podSecurityContext: {}
   # fsGroup: 2000
 
-securityContext: {}
-  # capabilities:
-  #   drop:
-  #   - ALL
-  # readOnlyRootFilesystem: true
-  # runAsNonRoot: true
-  # runAsUser: 1000
+securityContext:
+  # The default security context is heavily restricted
+  # feel free to tune it to your requirements
+  capabilities:
+    drop:
+    - ALL
+  readOnlyRootFilesystem: true
+  runAsNonRoot: true
+  runAsUser: 1000
 
 service:
+  # You can rely on any service to expose your cluster
+  # - ClusterIP (+ Ingress)
+  # - NodePort (+ Ingress)
+  # - LoadBalancer
   type: ClusterIP
   s3:
     api:
       port: 3900
     web:
       port: 3902
-
+    # NOTE: the admin API is excluded for now as it is not consistent across nodes
 ingress:
   s3:
     api:
-      enabled: false
-      className: ""
-      annotations: {}
-        # kubernetes.io/ingress.class: nginx
+      enabled: true
+      # Rely either on the className or the annotation below but not both
+      # replace "nginx" by an Ingress controller
+      # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers
+      className: "nginx"
+      annotations:
+        # kubernetes.io/ingress.class: "nginx"
         # kubernetes.io/tls-acme: "true"
       hosts:
-        - host: chart-example.local
+        - host: "s3.garage.tld" # garage S3 API endpoint
           paths:
             - path: /
-              pathType: ImplementationSpecific
+              pathType: Prefix
+        - host: "*.s3.garage.tld" # garage S3 API endpoint, DNS style bucket access
+          paths:
+            - path: /
+              pathType: Prefix
       tls: []
-      #  - secretName: chart-example-tls
+      #  - secretName: my-garage-cluster-tls
       #    hosts:
-      #      - chart-example.local
+      #      - kubernetes.docker.internal
     web:
-      enabled: false
-      className: ""
+      enabled: true
+      className: "nginx"
       annotations: {}
         # kubernetes.io/ingress.class: nginx
         # kubernetes.io/tls-acme: "true"
       hosts:
-        - host: chart-example.local
-          paths:
-            - path: /
-              pathType: ImplementationSpecific
+       - host: "*.web.garage.tld" # wildcard website access with bucket name prefix
+         paths:
+           - path: /
+             pathType: Prefix
+       - host: "mywebpage.example.com" # specific bucket access with FQDN bucket
+         paths:
+           - path: /
+             pathType: Prefix
       tls: []
-      #  - secretName: chart-example-tls
+      #  - secretName: my-garage-cluster-tls
       #    hosts:
-      #      - chart-example.local
+      #      - kubernetes.docker.internal
 
 resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # The following are indicative for a small-size deployement, for anything serious double them.
   # limits:
   #   cpu: 100m
-  #   memory: 128Mi
+  #   memory: 1024Mi
   # requests:
   #   cpu: 100m
-  #   memory: 128Mi
+  #   memory: 512Mi
 
 nodeSelector: {}