From e94d6f78d7584b64115905d4d6f7959160dd1936 Mon Sep 17 00:00:00 2001 From: kaiyou Date: Sat, 29 Oct 2022 21:07:02 +0200 Subject: Enable daemonset deployment using the helm chart DaemonSet is a k8s resource that schedules one instance per node, which is useful for some garage deployment use cases, including managing garage nodes using k8s node labels --- script/helm/garage/templates/statefulset.yaml | 116 ---------------------- script/helm/garage/templates/workload.yaml | 133 ++++++++++++++++++++++++++ script/helm/garage/values.yaml | 12 ++- 3 files changed, 143 insertions(+), 118 deletions(-) delete mode 100644 script/helm/garage/templates/statefulset.yaml create mode 100644 script/helm/garage/templates/workload.yaml diff --git a/script/helm/garage/templates/statefulset.yaml b/script/helm/garage/templates/statefulset.yaml deleted file mode 100644 index bda40117..00000000 --- a/script/helm/garage/templates/statefulset.yaml +++ /dev/null @@ -1,116 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "garage.fullname" . }} - labels: - {{- include "garage.labels" . | nindent 4 }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - {{- include "garage.selectorLabels" . | nindent 6 }} - serviceName: {{ include "garage.fullname" . }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "garage.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "garage.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - initContainers: - # Copies garage.toml from configmap to temporary etc volume and replaces RPC secret placeholder - - name: {{ .Chart.Name }}-init - image: busybox:1.28 - command: ["sh", "-c", "sed \"s/__RPC_SECRET_REPLACE__/$RPC_SECRET/\" /mnt/garage.toml > /mnt/etc/garage.toml"] - env: - - name: RPC_SECRET - valueFrom: - secretKeyRef: - name: {{ include "garage.rpcSecretName" . }} - key: rpcSecret - volumeMounts: - - name: configmap - mountPath: /mnt/garage.toml - subPath: garage.toml - - name: etc - mountPath: /mnt/etc - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - containerPort: 3900 - name: s3-api - - containerPort: 3902 - name: web-api - volumeMounts: - - name: meta - mountPath: /mnt/meta - - name: data - mountPath: /mnt/data - - name: etc - mountPath: /etc/garage.toml - subPath: garage.toml - # TODO - # livenessProbe: - # httpGet: - # path: / - # port: 3900 - # readinessProbe: - # httpGet: - # path: / - # port: 3900 - resources: - {{- toYaml .Values.resources | nindent 12 }} - volumes: - - name: configmap - configMap: - name: {{ include "garage.fullname" . }}-config - - name: etc - emptyDir: {} - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- if .Values.persistence.enabled }} - volumeClaimTemplates: - - metadata: - name: meta - spec: - accessModes: [ "ReadWriteOnce" ] - {{- if hasKey .Values.persistence.meta "storageClass" }} - storageClassName: {{ .Values.persistence.meta.storageClass | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.persistence.meta.size | quote }} - - metadata: - name: data - spec: - accessModes: [ "ReadWriteOnce" ] - {{- if hasKey .Values.persistence.data "storageClass" }} - storageClassName: {{ .Values.persistence.data.storageClass | quote }} - {{- end }} - resources: - requests: - storage: {{ .Values.persistence.data.size | quote }} - {{- end }} diff --git a/script/helm/garage/templates/workload.yaml b/script/helm/garage/templates/workload.yaml new file mode 100644 index 00000000..f41eff6f --- /dev/null +++ b/script/helm/garage/templates/workload.yaml @@ -0,0 +1,133 @@ +apiVersion: apps/v1 +kind: {{ .Values.deployment.kind }} +metadata: + name: {{ include "garage.fullname" . }} + labels: + {{- include "garage.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + {{- include "garage.selectorLabels" . | nindent 6 }} + {{- if eq .Values.deployment.kind "StatefulSet" }} + replicas: {{ .Values.deployment.replicaCount }} + serviceName: {{ include "garage.fullname" . }} + {{- end }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "garage.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "garage.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + # Copies garage.toml from configmap to temporary etc volume and replaces RPC secret placeholder + - name: {{ .Chart.Name }}-init + image: busybox:1.28 + command: ["sh", "-c", "sed \"s/__RPC_SECRET_REPLACE__/$RPC_SECRET/\" /mnt/garage.toml > /mnt/etc/garage.toml"] + env: + - name: RPC_SECRET + valueFrom: + secretKeyRef: + name: {{ include "garage.rpcSecretName" . }} + key: rpcSecret + volumeMounts: + - name: configmap + mountPath: /mnt/garage.toml + subPath: garage.toml + - name: etc + mountPath: /mnt/etc + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 3900 + name: s3-api + - containerPort: 3902 + name: web-api + volumeMounts: + - name: meta + mountPath: /mnt/meta + - name: data + mountPath: /mnt/data + - name: etc + mountPath: /etc/garage.toml + subPath: garage.toml + # TODO + # livenessProbe: + # httpGet: + # path: / + # port: 3900 + # readinessProbe: + # httpGet: + # path: / + # port: 3900 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumes: + - name: configmap + configMap: + name: {{ include "garage.fullname" . }}-config + - name: etc + emptyDir: {} + {{- if eq .Values.deployment.kind "DaemonSet" }} + {{- if .Values.persistence.enabled }} + - name: meta + hostPath: + path: {{ .Values.persistence.meta.hostPath }} + - name: data + hostPath: + path: {{ .Values.persistence.data.hostPath }} + {{- else }} + {{- end }} + - name: meta + emptyDir: {} + - name: data + emptyDir: {} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if and .Values.persistence.enabled (eq .Values.deployment.kind "StatefulSet") }} + volumeClaimTemplates: + - metadata: + name: meta + spec: + accessModes: [ "ReadWriteOnce" ] + {{- if hasKey .Values.persistence.meta "storageClass" }} + storageClassName: {{ .Values.persistence.meta.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.meta.size | quote }} + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + {{- if hasKey .Values.persistence.data "storageClass" }} + storageClassName: {{ .Values.persistence.data.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.data.size | quote }} + {{- end }} diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml index 608ee53c..52f1910a 100644 --- a/script/helm/garage/values.yaml +++ b/script/helm/garage/values.yaml @@ -29,12 +29,20 @@ persistence: meta: # storageClass: "fast-storage-class" size: 100Mi + # used only for daemon sets + hostPath: /var/lib/garage/meta data: # storageClass: "slow-storage-class" size: 100Mi + # used only for daemon sets + hostPath: /var/lib/garage/data -# Number of StatefulSet replicas/garage nodes to start -replicaCount: 3 +# Deployment configuration +deployment: + # Switchable to DaemonSet + kind: StatefulSet + # Number of StatefulSet replicas/garage nodes to start + replicaCount: 3 image: repository: dxflrs/amd64_garage -- cgit v1.2.3