From a93dcce84196bb8ffc8cef091d1343597b15b9a6 Mon Sep 17 00:00:00 2001 From: chemicstry Date: Mon, 20 Jun 2022 15:52:43 +0300 Subject: Add helm chart --- script/helm/README.md | 63 ++++++++++++ script/helm/garage/.helmignore | 23 +++++ script/helm/garage/Chart.yaml | 24 +++++ script/helm/garage/templates/_helpers.tpl | 62 ++++++++++++ script/helm/garage/templates/configmap.yaml | 29 ++++++ script/helm/garage/templates/ingress.yaml | 123 ++++++++++++++++++++++ script/helm/garage/templates/service.yaml | 19 ++++ script/helm/garage/templates/serviceaccount.yaml | 12 +++ script/helm/garage/templates/statefulset.yaml | 97 ++++++++++++++++++ script/helm/garage/values.yaml | 124 +++++++++++++++++++++++ 10 files changed, 576 insertions(+) create mode 100644 script/helm/README.md create mode 100644 script/helm/garage/.helmignore create mode 100644 script/helm/garage/Chart.yaml create mode 100644 script/helm/garage/templates/_helpers.tpl create mode 100644 script/helm/garage/templates/configmap.yaml create mode 100644 script/helm/garage/templates/ingress.yaml create mode 100644 script/helm/garage/templates/service.yaml create mode 100644 script/helm/garage/templates/serviceaccount.yaml create mode 100644 script/helm/garage/templates/statefulset.yaml create mode 100644 script/helm/garage/values.yaml diff --git a/script/helm/README.md b/script/helm/README.md new file mode 100644 index 00000000..715cbab1 --- /dev/null +++ b/script/helm/README.md @@ -0,0 +1,63 @@ +# Garage helm3 chart + +This chart deploys garage on a kubernetes cluster. + +## Deploying + +With default options: + +```bash +helm install --create-namespace --namespace garage garage ./garage +``` + +With custom values: + +```bash +helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml +``` + +## Overriding default values + +All possible configuration values can be found in [values.yaml](garage/values.yaml). + +This is an example `values.overrride.yaml` for deploying in a microk8s cluster with a https s3 api ingress route: + +```yaml +# Start 4 instances (StatefulSets) of garage +replicaCount: 4 + +# Override default storage class and size +persistence: + meta: + storageClass: "openebs-hostpath" + size: 100Mi + data: + storageClass: "openebs-hostpath" + size: 1Gi + +ingress: + s3: + api: + enabled: true + className: "public" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/proxy-body-size: 500m + hosts: + - host: s3-api.my-domain.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: garage-ingress-cert + hosts: + - s3-api.my-domain.com +``` + +## Removing + +```bash +helm delete --namespace garage garage +``` + +Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired. diff --git a/script/helm/garage/.helmignore b/script/helm/garage/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/script/helm/garage/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml new file mode 100644 index 00000000..9455488a --- /dev/null +++ b/script/helm/garage/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: garage +description: S3-compatible object store for small self-hosted geo-distributed deployments + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "v0.7.2" diff --git a/script/helm/garage/templates/_helpers.tpl b/script/helm/garage/templates/_helpers.tpl new file mode 100644 index 00000000..1a651f47 --- /dev/null +++ b/script/helm/garage/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "garage.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "garage.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "garage.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "garage.labels" -}} +helm.sh/chart: {{ include "garage.chart" . }} +{{ include "garage.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "garage.selectorLabels" -}} +app.kubernetes.io/name: {{ include "garage.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "garage.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "garage.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/script/helm/garage/templates/configmap.yaml b/script/helm/garage/templates/configmap.yaml new file mode 100644 index 00000000..587746f6 --- /dev/null +++ b/script/helm/garage/templates/configmap.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "garage.fullname" . }}-config +data: + garage.toml: |- + metadata_dir = "{{ .Values.garage.metadataDir }}" + data_dir = "{{ .Values.garage.dataDir }}" + + replication_mode = "{{ .Values.garage.replicationMode }}" + + rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}" + rpc_secret = "{{ .Values.garage.rpcSecret }}" + + bootstrap_peers = {{ .Values.garage.bootstrapPeers }} + + kubernetes_namespace = "{{ .Release.Namespace }}" + kubernetes_service_name = "{{ include "garage.fullname" . }}" + kubernetes_skip_crd = {{ .Values.garage.kubernetesSkipCrd }} + + [s3_api] + s3_region = "{{ .Values.garage.s3.api.region }}" + api_bind_addr = "[::]:3900" + root_domain = "{{ .Values.garage.s3.api.rootDomain }}" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = "{{ .Values.garage.s3.web.rootDomain }}" + index = "{{ .Values.garage.s3.web.index }}" \ No newline at end of file diff --git a/script/helm/garage/templates/ingress.yaml b/script/helm/garage/templates/ingress.yaml new file mode 100644 index 00000000..c4ee5a3f --- /dev/null +++ b/script/helm/garage/templates/ingress.yaml @@ -0,0 +1,123 @@ +{{- if .Values.ingress.s3.api.enabled -}} +{{- $fullName := include "garage.fullname" . -}} +{{- $svcPort := .Values.service.s3.api.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.s3.api.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.s3.api.annotations "kubernetes.io/ingress.class" .Values.ingress.s3.api.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }}-s3-api + labels: + {{- include "garage.labels" . | nindent 4 }} + {{- with .Values.ingress.s3.api.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.s3.api.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.s3.api.className }} + {{- end }} + {{- if .Values.ingress.s3.api.tls }} + tls: + {{- range .Values.ingress.s3.api.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.s3.api.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} +--- +{{- if .Values.ingress.s3.web.enabled -}} +{{- $fullName := include "garage.fullname" . -}} +{{- $svcPort := .Values.service.s3.web.port -}} +{{- if and .Values.ingress.s3.web.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.s3.web.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.s3.web.annotations "kubernetes.io/ingress.class" .Values.ingress.s3.web.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }}-s3-web + labels: + {{- include "garage.labels" . | nindent 4 }} + {{- with .Values.ingress.s3.web.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.s3.web.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.s3.web.className }} + {{- end }} + {{- if .Values.ingress.s3.web.tls }} + tls: + {{- range .Values.ingress.s3.web.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.s3.web.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/script/helm/garage/templates/service.yaml b/script/helm/garage/templates/service.yaml new file mode 100644 index 00000000..2bfff99d --- /dev/null +++ b/script/helm/garage/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "garage.fullname" . }} + labels: + {{- include "garage.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.s3.api.port }} + targetPort: 3900 + protocol: TCP + name: s3-api + - port: {{ .Values.service.s3.web.port }} + targetPort: 3902 + protocol: TCP + name: s3-web + selector: + {{- include "garage.selectorLabels" . | nindent 4 }} diff --git a/script/helm/garage/templates/serviceaccount.yaml b/script/helm/garage/templates/serviceaccount.yaml new file mode 100644 index 00000000..a0a89a33 --- /dev/null +++ b/script/helm/garage/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "garage.serviceAccountName" . }} + labels: + {{- include "garage.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/script/helm/garage/templates/statefulset.yaml b/script/helm/garage/templates/statefulset.yaml new file mode 100644 index 00000000..82fe89a9 --- /dev/null +++ b/script/helm/garage/templates/statefulset.yaml @@ -0,0 +1,97 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "garage.fullname" . }} + labels: + {{- include "garage.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "garage.selectorLabels" . | nindent 6 }} + serviceName: {{ include "garage.fullname" . }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "garage.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "garage.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - containerPort: 3900 + name: s3-api + - containerPort: 3902 + name: web-api + volumeMounts: + - name: meta + mountPath: /mnt/meta + - name: data + mountPath: /mnt/data + - name: etc + mountPath: /etc/garage.toml + subPath: garage.toml + # TODO + # livenessProbe: + # httpGet: + # path: / + # port: 3900 + # readinessProbe: + # httpGet: + # path: / + # port: 3900 + resources: + {{- toYaml .Values.resources | nindent 12 }} + volumes: + - name: etc + configMap: + name: {{ include "garage.fullname" . }}-config + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + - metadata: + name: meta + spec: + accessModes: [ "ReadWriteOnce" ] + {{- if hasKey .Values.persistence.meta "storageClass" }} + storageClassName: {{ .Values.persistence.meta.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.meta.size | quote }} + - metadata: + name: data + spec: + accessModes: [ "ReadWriteOnce" ] + {{- if hasKey .Values.persistence.data "storageClass" }} + storageClassName: {{ .Values.persistence.data.storageClass | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.persistence.data.size | quote }} + {{- end }} diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml new file mode 100644 index 00000000..dd1c99f0 --- /dev/null +++ b/script/helm/garage/values.yaml @@ -0,0 +1,124 @@ +# Default values for garage. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Garage configuration. These values go to garage.toml +garage: + metadataDir: "/mnt/meta" + dataDir: "/mnt/data" + replicationMode: "3" + rpcBindAddr: "[::]:3901" + rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" + bootstrapPeers: [] + # kubernetes_namespace: "default" + # kubernetes_service_name: "garage-daemon" + kubernetesSkipCrd: false + s3: + api: + region: "garage" + rootDomain: ".s3.garage.tld" + web: + rootDomain: ".web.garage.tld" + index: "index.html" + +# Data persistence +persistence: + enabled: true + meta: + # storageClass: "" + size: 100Mi + data: + # storageClass: "" + size: 100Mi + +# Number of StatefulSet replicas to start +replicaCount: 3 + +image: + repository: dxflrs/amd64_garage + pullPolicy: IfNotPresent + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + s3: + api: + port: 3900 + web: + port: 3902 + +ingress: + s3: + api: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + web: + enabled: false + className: "" + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} -- cgit v1.2.3 From 131cc2532b13acfb90d38e04c5dac5fa9cd3cb0e Mon Sep 17 00:00:00 2001 From: chemicstry Date: Mon, 20 Jun 2022 16:02:23 +0300 Subject: Cleanup values.yaml --- script/helm/garage/values.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml index dd1c99f0..06cf9d16 100644 --- a/script/helm/garage/values.yaml +++ b/script/helm/garage/values.yaml @@ -10,8 +10,6 @@ garage: rpcBindAddr: "[::]:3901" rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" bootstrapPeers: [] - # kubernetes_namespace: "default" - # kubernetes_service_name: "garage-daemon" kubernetesSkipCrd: false s3: api: -- cgit v1.2.3 From fa52558ca12573763bea392f9b44f71d3a6bb457 Mon Sep 17 00:00:00 2001 From: chemicstry Date: Mon, 20 Jun 2022 16:02:53 +0300 Subject: Add configuration instructions to README --- script/helm/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/script/helm/README.md b/script/helm/README.md index 715cbab1..925d51ce 100644 --- a/script/helm/README.md +++ b/script/helm/README.md @@ -16,6 +16,12 @@ With custom values: helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml ``` +After deploying, cluster layout must be configured manually as per garage's documentation. Use the following command to access garage CLI: + +```bash +kubectl exec --stdin --tty -n garage garage-0 -- ./garage status +``` + ## Overriding default values All possible configuration values can be found in [values.yaml](garage/values.yaml). -- cgit v1.2.3 From d0f08c254e3cc996dbf3d565df9ff2c89e15c639 Mon Sep 17 00:00:00 2001 From: chemicstry Date: Mon, 20 Jun 2022 16:08:41 +0300 Subject: Add secret to overrides --- script/helm/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/script/helm/README.md b/script/helm/README.md index 925d51ce..b7a3a35b 100644 --- a/script/helm/README.md +++ b/script/helm/README.md @@ -29,6 +29,10 @@ All possible configuration values can be found in [values.yaml](garage/values.ya This is an example `values.overrride.yaml` for deploying in a microk8s cluster with a https s3 api ingress route: ```yaml +garage: + # Make sure to generate a new secret for your deployment + rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" + # Start 4 instances (StatefulSets) of garage replicaCount: 4 -- cgit v1.2.3 From 37a73d7d3782ec8a5cd8b0e71a00722f90321ced Mon Sep 17 00:00:00 2001 From: chemicstry Date: Mon, 20 Jun 2022 17:11:09 +0300 Subject: Move documentation to book --- doc/book/cookbook/kubernetes.md | 87 +++++++++++++++++++++++++++++++++++++++++ script/helm/README.md | 72 +--------------------------------- 2 files changed, 88 insertions(+), 71 deletions(-) create mode 100644 doc/book/cookbook/kubernetes.md diff --git a/doc/book/cookbook/kubernetes.md b/doc/book/cookbook/kubernetes.md new file mode 100644 index 00000000..8fd12fdf --- /dev/null +++ b/doc/book/cookbook/kubernetes.md @@ -0,0 +1,87 @@ ++++ +title = "Deploying on Kubernetes" +weight = 32 ++++ + +Garage can also be deployed on a kubernetes cluster via helm chart. + +## Deploying + +Firstly clone the repository: + +```bash +git clone https://git.deuxfleurs.fr/Deuxfleurs/garage +cd garage/scripts/helm +``` + +Deploy with default options: + +```bash +helm install --create-namespace --namespace garage garage ./garage +``` + +Or deploy with custom values: + +```bash +helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml +``` + +After deploying, cluster layout must be configured manually as described in [Creating a cluster layout](@/documentation/quick-start/_index.md#creating-a-cluster-layout). Use the following command to access garage CLI: + +```bash +kubectl exec --stdin --tty -n garage garage-0 -- ./garage status +``` + +## Overriding default values + +All possible configuration values can be found with: + +```bash +helm show values ./garage +``` + +This is an example `values.overrride.yaml` for deploying in a microk8s cluster with a https s3 api ingress route: + +```yaml +garage: + # Make sure to generate a new secret for your deployment + rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" + +# Start 4 instances (StatefulSets) of garage +replicaCount: 4 + +# Override default storage class and size +persistence: + meta: + storageClass: "openebs-hostpath" + size: 100Mi + data: + storageClass: "openebs-hostpath" + size: 1Gi + +ingress: + s3: + api: + enabled: true + className: "public" + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/proxy-body-size: 500m + hosts: + - host: s3-api.my-domain.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: garage-ingress-cert + hosts: + - s3-api.my-domain.com +``` + +## Removing + +```bash +helm delete --namespace garage garage +``` + +Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired. diff --git a/script/helm/README.md b/script/helm/README.md index b7a3a35b..5f919a23 100644 --- a/script/helm/README.md +++ b/script/helm/README.md @@ -1,73 +1,3 @@ # Garage helm3 chart -This chart deploys garage on a kubernetes cluster. - -## Deploying - -With default options: - -```bash -helm install --create-namespace --namespace garage garage ./garage -``` - -With custom values: - -```bash -helm install --create-namespace --namespace garage garage ./garage -f values.override.yaml -``` - -After deploying, cluster layout must be configured manually as per garage's documentation. Use the following command to access garage CLI: - -```bash -kubectl exec --stdin --tty -n garage garage-0 -- ./garage status -``` - -## Overriding default values - -All possible configuration values can be found in [values.yaml](garage/values.yaml). - -This is an example `values.overrride.yaml` for deploying in a microk8s cluster with a https s3 api ingress route: - -```yaml -garage: - # Make sure to generate a new secret for your deployment - rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" - -# Start 4 instances (StatefulSets) of garage -replicaCount: 4 - -# Override default storage class and size -persistence: - meta: - storageClass: "openebs-hostpath" - size: 100Mi - data: - storageClass: "openebs-hostpath" - size: 1Gi - -ingress: - s3: - api: - enabled: true - className: "public" - annotations: - cert-manager.io/cluster-issuer: "letsencrypt-prod" - nginx.ingress.kubernetes.io/proxy-body-size: 500m - hosts: - - host: s3-api.my-domain.com - paths: - - path: / - pathType: Prefix - tls: - - secretName: garage-ingress-cert - hosts: - - s3-api.my-domain.com -``` - -## Removing - -```bash -helm delete --namespace garage garage -``` - -Note that this will leave behind custom CRD `garagenodes.deuxfleurs.fr`, which must be removed manually if desired. +Documentation is located [here](/doc/book/cookbook/kubernetes.md). -- cgit v1.2.3 From b71fa2ddf45e21f40067fc021b3a81d738556eca Mon Sep 17 00:00:00 2001 From: chemicstry Date: Mon, 20 Jun 2022 18:49:38 +0300 Subject: Generate random RPC secret if not provided --- script/helm/garage/templates/_helpers.tpl | 26 ++++++++++++++++++++++++++ script/helm/garage/templates/configmap.yaml | 3 ++- script/helm/garage/templates/secret.yaml | 14 ++++++++++++++ script/helm/garage/templates/statefulset.yaml | 21 ++++++++++++++++++++- script/helm/garage/values.yaml | 3 ++- 5 files changed, 64 insertions(+), 3 deletions(-) create mode 100644 script/helm/garage/templates/secret.yaml diff --git a/script/helm/garage/templates/_helpers.tpl b/script/helm/garage/templates/_helpers.tpl index 1a651f47..037a5f1c 100644 --- a/script/helm/garage/templates/_helpers.tpl +++ b/script/helm/garage/templates/_helpers.tpl @@ -23,6 +23,13 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} +{{/* +Create the name of the rpc secret +*/}} +{{- define "garage.rpcSecretName" -}} +{{- printf "%s-rpc-secret" (include "garage.fullname" .) -}} +{{- end }} + {{/* Create chart name and version as used by the chart label. */}} @@ -60,3 +67,22 @@ Create the name of the service account to use {{- default "default" .Values.serviceAccount.name }} {{- end }} {{- end }} + +{{/* + Returns given number of random Hex characters. + In practice, it generates up to 100 randAlphaNum strings + that are filtered from non-hex characters and augmented + to the resulting string that is finally trimmed down. +*/}} +{{- define "jupyterhub.randHex" -}} + {{- $result := "" }} + {{- range $i := until 100 }} + {{- if lt (len $result) . }} + {{- $rand_list := randAlphaNum . | splitList "" -}} + {{- $reduced_list := without $rand_list "g" "h" "i" "j" "k" "l" "m" "n" "o" "p" "q" "r" "s" "t" "u" "v" "w" "x" "y" "z" "A" "B" "C" "D" "E" "F" "G" "H" "I" "J" "K" "L" "M" "N" "O" "P" "Q" "R" "S" "T" "U" "V" "W" "X" "Y" "Z" }} + {{- $rand_string := join "" $reduced_list }} + {{- $result = print $result $rand_string -}} + {{- end }} + {{- end }} + {{- $result | trunc . }} +{{- end }} diff --git a/script/helm/garage/templates/configmap.yaml b/script/helm/garage/templates/configmap.yaml index 587746f6..e33a4dbd 100644 --- a/script/helm/garage/templates/configmap.yaml +++ b/script/helm/garage/templates/configmap.yaml @@ -10,7 +10,8 @@ data: replication_mode = "{{ .Values.garage.replicationMode }}" rpc_bind_addr = "{{ .Values.garage.rpcBindAddr }}" - rpc_secret = "{{ .Values.garage.rpcSecret }}" + # rpc_secret will be populated by the init container from a k8s secret object + rpc_secret = "__RPC_SECRET_REPLACE__" bootstrap_peers = {{ .Values.garage.bootstrapPeers }} diff --git a/script/helm/garage/templates/secret.yaml b/script/helm/garage/templates/secret.yaml new file mode 100644 index 00000000..54749424 --- /dev/null +++ b/script/helm/garage/templates/secret.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "garage.rpcSecretName" . }} + labels: + {{- include "garage.labels" . | nindent 4 }} +type: Opaque +data: + {{/* retrieve the secret data using lookup function and when not exists, return an empty dictionary / map as result */}} + {{- $prevSecret := (lookup "v1" "Secret" .Release.Namespace (include "garage.rpcSecretName" .)) | default dict }} + {{- $prevSecretData := $prevSecret.data | default dict }} + {{- $prevRpcSecret := $prevSecretData.rpcSecret | default "" | b64dec }} + {{/* Priority is: 1. from values, 2. previous value, 3. generate random */}} + rpcSecret: {{ .Values.garage.rpcSecret | default $prevRpcSecret | default (include "jupyterhub.randHex" 64) | b64enc | quote }} diff --git a/script/helm/garage/templates/statefulset.yaml b/script/helm/garage/templates/statefulset.yaml index 82fe89a9..bda40117 100644 --- a/script/helm/garage/templates/statefulset.yaml +++ b/script/helm/garage/templates/statefulset.yaml @@ -26,6 +26,23 @@ spec: serviceAccountName: {{ include "garage.serviceAccountName" . }} securityContext: {{- toYaml .Values.podSecurityContext | nindent 8 }} + initContainers: + # Copies garage.toml from configmap to temporary etc volume and replaces RPC secret placeholder + - name: {{ .Chart.Name }}-init + image: busybox:1.28 + command: ["sh", "-c", "sed \"s/__RPC_SECRET_REPLACE__/$RPC_SECRET/\" /mnt/garage.toml > /mnt/etc/garage.toml"] + env: + - name: RPC_SECRET + valueFrom: + secretKeyRef: + name: {{ include "garage.rpcSecretName" . }} + key: rpcSecret + volumeMounts: + - name: configmap + mountPath: /mnt/garage.toml + subPath: garage.toml + - name: etc + mountPath: /mnt/etc containers: - name: {{ .Chart.Name }} securityContext: @@ -57,9 +74,11 @@ spec: resources: {{- toYaml .Values.resources | nindent 12 }} volumes: - - name: etc + - name: configmap configMap: name: {{ include "garage.fullname" . }}-config + - name: etc + emptyDir: {} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml index 06cf9d16..d011f63e 100644 --- a/script/helm/garage/values.yaml +++ b/script/helm/garage/values.yaml @@ -8,7 +8,8 @@ garage: dataDir: "/mnt/data" replicationMode: "3" rpcBindAddr: "[::]:3901" - rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" + # If not given, a random secret will be generated + rpcSecret: "" bootstrapPeers: [] kubernetesSkipCrd: false s3: -- cgit v1.2.3 From 744c3b4d9487045ab04a221572722afa0ca34b09 Mon Sep 17 00:00:00 2001 From: chemicstry Date: Mon, 20 Jun 2022 18:52:32 +0300 Subject: Update docs --- doc/book/cookbook/kubernetes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/book/cookbook/kubernetes.md b/doc/book/cookbook/kubernetes.md index 8fd12fdf..0bf89c96 100644 --- a/doc/book/cookbook/kubernetes.md +++ b/doc/book/cookbook/kubernetes.md @@ -44,8 +44,8 @@ This is an example `values.overrride.yaml` for deploying in a microk8s cluster w ```yaml garage: - # Make sure to generate a new secret for your deployment - rpcSecret: "1799bccfd7411eddcf9ebd316bc1f5287ad12a68094e1c6ac6abde7e6feae1ec" + # Use only 2 replicas per object + replicationMode: "3" # Start 4 instances (StatefulSets) of garage replicaCount: 4 -- cgit v1.2.3 From d2c937a931b6549ffd2f2afdd7a871be8d8eefbf Mon Sep 17 00:00:00 2001 From: chemicstry Date: Tue, 21 Jun 2022 16:16:42 +0300 Subject: Fix typo --- doc/book/cookbook/kubernetes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/book/cookbook/kubernetes.md b/doc/book/cookbook/kubernetes.md index 0bf89c96..9eafe3e1 100644 --- a/doc/book/cookbook/kubernetes.md +++ b/doc/book/cookbook/kubernetes.md @@ -45,7 +45,7 @@ This is an example `values.overrride.yaml` for deploying in a microk8s cluster w ```yaml garage: # Use only 2 replicas per object - replicationMode: "3" + replicationMode: "2" # Start 4 instances (StatefulSets) of garage replicaCount: 4 -- cgit v1.2.3 From 6dba7dadf44781abfb878f06fba86e731b267c87 Mon Sep 17 00:00:00 2001 From: Maximilien R Date: Wed, 22 Jun 2022 10:04:59 +0200 Subject: Add missing ClusterRole and bindings for CRDs --- script/helm/garage/templates/clusterrole.yaml | 28 +++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 script/helm/garage/templates/clusterrole.yaml diff --git a/script/helm/garage/templates/clusterrole.yaml b/script/helm/garage/templates/clusterrole.yaml new file mode 100644 index 00000000..fa3e6405 --- /dev/null +++ b/script/helm/garage/templates/clusterrole.yaml @@ -0,0 +1,28 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: manage-crds-{{ .Release.Namespace }}-{{ .Release.Name }} + labels: + {{- include "garage.labels" . | nindent 4 }} +rules: +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get", "list", "watch", "create", "patch"] +- apiGroups: ["deuxfleurs.fr"] + resources: ["garagenodes"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: allow-crds-for-{{ .Release.Namespace }}-{{ .Release.Name }} + labels: + {{- include "garage.labels" . | nindent 4 }} +subjects: +- kind: ServiceAccount + name: {{ include "garage.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: manage-crds-{{ .Release.Namespace }}-{{ .Release.Name }} + apiGroup: rbac.authorization.k8s.io \ No newline at end of file -- cgit v1.2.3 From db0c8b3980c5cb056c9402332dd09a1bfb276997 Mon Sep 17 00:00:00 2001 From: Maximilien R Date: Thu, 11 Aug 2022 01:35:41 +0200 Subject: Updates values.yml with some opinionated and untested defaults --- script/helm/garage/Chart.yaml | 2 +- script/helm/garage/values.yaml | 87 +++++++++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 35 deletions(-) diff --git a/script/helm/garage/Chart.yaml b/script/helm/garage/Chart.yaml index 9455488a..56598ea4 100644 --- a/script/helm/garage/Chart.yaml +++ b/script/helm/garage/Chart.yaml @@ -21,4 +21,4 @@ version: 0.1.0 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "v0.7.2" +appVersion: "v0.7.2.1" diff --git a/script/helm/garage/values.yaml b/script/helm/garage/values.yaml index d011f63e..08d0c09b 100644 --- a/script/helm/garage/values.yaml +++ b/script/helm/garage/values.yaml @@ -6,10 +6,13 @@ garage: metadataDir: "/mnt/meta" dataDir: "/mnt/data" + # Default to 3 replicas, see the replication_mode section at + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/configuration/ replicationMode: "3" rpcBindAddr: "[::]:3901" - # If not given, a random secret will be generated + # If not given, a random secret will be generated and stored in a Secret object rpcSecret: "" + # This is not required if you use the integrated kubernetes discovery bootstrapPeers: [] kubernetesSkipCrd: false s3: @@ -24,17 +27,19 @@ garage: persistence: enabled: true meta: - # storageClass: "" + # storageClass: "fast-storage-class" size: 100Mi data: - # storageClass: "" + # storageClass: "slow-storage-class" size: 100Mi -# Number of StatefulSet replicas to start +# Number of StatefulSet replicas/garage nodes to start replicaCount: 3 image: repository: dxflrs/amd64_garage + # please prefer using the chart version and not this tag + tag: "" pullPolicy: IfNotPresent imagePullSecrets: [] @@ -55,66 +60,80 @@ podAnnotations: {} podSecurityContext: {} # fsGroup: 2000 -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 +securityContext: + # The default security context is heavily restricted + # feel free to tune it to your requirements + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 1000 service: + # You can rely on any service to expose your cluster + # - ClusterIP (+ Ingress) + # - NodePort (+ Ingress) + # - LoadBalancer type: ClusterIP s3: api: port: 3900 web: port: 3902 - + # NOTE: the admin API is excluded for now as it is not consistent across nodes ingress: s3: api: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx + enabled: true + # Rely either on the className or the annotation below but not both + # replace "nginx" by an Ingress controller + # you can find examples here https://kubernetes.io/docs/concepts/services-networking/ingress-controllers + className: "nginx" + annotations: + # kubernetes.io/ingress.class: "nginx" # kubernetes.io/tls-acme: "true" hosts: - - host: chart-example.local + - host: "s3.garage.tld" # garage S3 API endpoint paths: - path: / - pathType: ImplementationSpecific + pathType: Prefix + - host: "*.s3.garage.tld" # garage S3 API endpoint, DNS style bucket access + paths: + - path: / + pathType: Prefix tls: [] - # - secretName: chart-example-tls + # - secretName: my-garage-cluster-tls # hosts: - # - chart-example.local + # - kubernetes.docker.internal web: - enabled: false - className: "" + enabled: true + className: "nginx" annotations: {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific + - host: "*.web.garage.tld" # wildcard website access with bucket name prefix + paths: + - path: / + pathType: Prefix + - host: "mywebpage.example.com" # specific bucket access with FQDN bucket + paths: + - path: / + pathType: Prefix tls: [] - # - secretName: chart-example-tls + # - secretName: my-garage-cluster-tls # hosts: - # - chart-example.local + # - kubernetes.docker.internal resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # The following are indicative for a small-size deployement, for anything serious double them. # limits: # cpu: 100m - # memory: 128Mi + # memory: 1024Mi # requests: # cpu: 100m - # memory: 128Mi + # memory: 512Mi nodeSelector: {} -- cgit v1.2.3