Make db engine configurable through helm values

DaemonSet is a k8s resource that schedules one instance per node,
which is useful for some garage deployment use cases, including
managing garage nodes using k8s node labels

doc/book/cookbook/kubernetes.md

@@ -47,7 +47,11 @@ garage:
   # Use only 2 replicas per object
   replicationMode: "2"
 
+  # Use recommended lmdb db engine
+  dbEngine: "lmdb"
+
 # Start 4 instances (StatefulSets) of garage
+deployment:
   replicaCount: 4
 
 # Override default storage class and size

script/helm/garage/templates/configmap.yaml

@@ -7,6 +7,7 @@ data:
     metadata_dir = "{{ .Values.garage.metadataDir }}"
     data_dir = "{{ .Values.garage.dataDir }}"
 
+    db_engine = "{{ .Values.garage.dbEngine }}"
     replication_mode = "{{ .Values.garage.replicationMode }}"
 
     rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}"

script/helm/garage/templates/workload.yaml

@@ -1,15 +1,17 @@
 apiVersion: apps/v1
-kind: StatefulSet
+kind: {{ .Values.deployment.kind }}
 metadata:
   name: {{ include "garage.fullname" . }}
   labels:
     {{- include "garage.labels" . | nindent 4 }}
 spec:
-  replicas: {{ .Values.replicaCount }}
   selector:
     matchLabels:
       {{- include "garage.selectorLabels" . | nindent 6 }}
+  {{- if eq .Values.deployment.kind "StatefulSet" }}
+  replicas: {{ .Values.deployment.replicaCount }}
   serviceName: {{ include "garage.fullname" . }}
+  {{- end }}
   template:
     metadata:
       {{- with .Values.podAnnotations }}
@@ -79,6 +81,23 @@ spec:
             name: {{ include "garage.fullname" . }}-config
         - name: etc
           emptyDir: {}
+        {{- if .Values.persistence.enabled }}
+        {{- if eq .Values.deployment.kind "DaemonSet" }}
+        - name: meta
+          hostPath:
+            path: {{ .Values.persistence.meta.hostPath }}
+            type: DirectoryOrCreate
+        - name: data
+          hostPath:
+            path: {{ .Values.persistence.data.hostPath }}
+            type: DirectoryOrCreate
+        {{- end }}
+        {{- else }}
+        - name: meta
+          emptyDir: {}
+        - name: data
+          emptyDir: {}
+        {{- end }}
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -91,7 +110,7 @@ spec:
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-  {{- if .Values.persistence.enabled }}
+  {{- if and .Values.persistence.enabled (eq .Values.deployment.kind "StatefulSet") }}
   volumeClaimTemplates:
   - metadata:
       name: meta

script/helm/garage/values.yaml

@@ -29,10 +29,18 @@ persistence:
   meta:
     # storageClass: "fast-storage-class"
     size: 100Mi
+    # used only for daemon sets
+    hostPath: /var/lib/garage/meta
   data:
     # storageClass: "slow-storage-class"
     size: 100Mi
+    # used only for daemon sets
+    hostPath: /var/lib/garage/data
 
+# Deployment configuration
+deployment:
+  # Switchable to DaemonSet
+  kind: StatefulSet
   # Number of StatefulSet replicas/garage nodes to start
   replicaCount: 3