chore: add emqx enterprise helm chart

.github/workflows/release.yaml

@@ -58,6 +58,7 @@ jobs:
                -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ github.ref_name }}\" }" \
                ${{ secrets.EMQX_IO_RELEASE_API }}
       - uses: emqx/push-helm-action@v1
+        if: github.event_name == 'release' && startsWith(github.ref_name, 'v')
         with:
           charts_dir: "${{ github.workspace }}/deploy/charts/emqx"
           version: ${{ github.ref_name }}
@@ -65,6 +66,15 @@ jobs:
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws_region: "us-west-2"
           aws_bucket_name: "repos-emqx-io"
+      - uses: emqx/push-helm-action@v1
+        if: github.event_name == 'release' && startsWith(github.ref_name, 'e')
+        with:
+          charts_dir: "${{ github.workspace }}/deploy/charts/emqx-ee"
+          version: ${{ github.ref_name }}
+          aws_access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws_region: "us-west-2"
+          aws_bucket_name: "repos-emqx-io"
       - name: update homebrew packages
         if: github.event_name == 'release'
         run: |

.github/workflows/run_fvt_tests.yaml

@@ -116,6 +116,7 @@ jobs:
         - dns
         profile:
         - emqx
+        - emqx-enterprise
         os:
         - ["debian11", "debian:11-slim"]
         otp:
@@ -156,6 +157,12 @@ jobs:
       working-directory: source
       if: matrix.discovery == 'k8s'
       run: |
+        if [ ${{ matrix.profile }} = "emqx-enterprise" ]; then
+          chart_name="emqx-ee"
+        else
+          chart_name="emqx"
+        fi
+
         helm install emqx \
             --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \
             --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \
@@ -169,12 +176,18 @@ jobs:
             --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \
             --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
             --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
-            deploy/charts/emqx \
+            deploy/charts/${chart_name} \
             --debug
     - name: run emqx on chart
       working-directory: source
       if: matrix.discovery == 'dns'
       run: |
+        if [ ${{ matrix.profile }} = "emqx-enterprise" ]; then
+          chart_name="emqx-ee"
+        else
+          chart_name="emqx"
+        fi
+
         helm install emqx \
             --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \
             --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \
@@ -182,18 +195,17 @@ jobs:
             --set image.repository=$TARGET \
             --set image.pullPolicy=Never \
             --set emqxAclConfig="" \
-            --set image.pullPolicy=Never \
             --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \
             --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \
             --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
             --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
-            deploy/charts/emqx \
+            deploy/charts/${chart_name} \
             --debug
     - name: waiting emqx started
       timeout-minutes: 10
       run: |
-        while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \
-          != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
+        while [ "$(kubectl get StatefulSet -l app.kubernetes.io/instance=emqx -o jsonpath='{.items[0].status.replicas}')" \
+          != "$(kubectl get StatefulSet -l app.kubernetes.io/instance=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
           echo "==============================";
           kubectl get pods;
           echo "==============================";

deploy/charts/emqx-ee/Chart.yaml

@@ -0,0 +1,21 @@
+apiVersion: v2
+name: emqx-ee
+icon: https://github.com/emqx.png
+description: A Helm chart for EMQX
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 5.0.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: 5.0.0

deploy/charts/emqx-ee/README.md

@@ -0,0 +1,121 @@
+# Introduction
+
+This chart bootstraps an emqx deployment on a Kubernetes cluster using the Helm package manager.
+
+# Prerequisites
+
++ Kubernetes 1.6+
++ Helm
+
+# Installing the Chart
+
+To install the chart with the release name `my-emqx`:
+
++ From github
+  ```
+  $ git clone https://github.com/emqx/emqx.git
+  $ cd emqx/deploy/charts/emqx
+  $ helm install my-emqx .
+  ```
+
++ From chart repos
+  ```
+  helm repo add emqx https://repos.emqx.io/charts
+  helm install my-emqx emqx/emqx
+  ```
+  > If you want to install an unstable version, you need to add `--devel` when you execute the `helm install` command.
+
+# Uninstalling the Chart
+
+To uninstall/delete the `my-emqx` deployment:
+
+```
+$ helm del  my-emqx
+```
+
+# Configuration
+
+The following table lists the configurable parameters of the emqx chart and their default values.
+
+| Parameter                            | Description                                                                                                                                                  | Default Value                                           |
+|--------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------|
+| `replicaCount`                       | It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split.                  | 3                                                       |
+| `image.repository`                   | EMQX Image name                                                                                                                                              | emqx/emqx                                               |
+| `image.pullPolicy`                   | The image pull policy                                                                                                                                        | IfNotPresent                                            |
+| `image.pullSecrets `                 | The image pull secrets                                                                                                                                       | `[]` (does not add image pull secrets to deployed pods) |
+| `envFromSecret`                      | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment                                          | nil                                                     |
+| `recreatePods`                       | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration.                                            | false                                                   |
+| `podAnnotations `                    | Annotations for pod                                                                                                                                          | `{}`                                                    |
+| `podManagementPolicy`                | To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock                                                                | `Parallel`                                              |
+| `persistence.enabled`                | Enable EMQX persistence using PVC                                                                                                                            | false                                                   |
+| `persistence.storageClass`           | Storage class of backing PVC                                                                                                                                 | `nil` (uses alpha storage class annotation)             |
+| `persistence.existingClaim`          | EMQX data Persistent Volume existing claim name, evaluated as a template                                                                                     | ""                                                      |
+| `persistence.accessMode`             | PVC Access Mode for EMQX volume                                                                                                                              | ReadWriteOnce                                           |
+| `persistence.size`                   | PVC Storage Request for EMQX volume                                                                                                                          | 20Mi                                                    |
+| `initContainers`                     | Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts.                                                     | `{}`                                                    |
+| `resources`                          | CPU/Memory resource requests/limits                                                                                                                          | {}                                                      |
+| `nodeSelector`                       | Node labels for pod assignment                                                                                                                               | `{}`                                                    |
+| `tolerations`                        | Toleration labels for pod assignment                                                                                                                         | `[]`                                                    |
+| `affinity`                           | Map of node/pod affinities                                                                                                                                   | `{}`                                                    |
+| `service.type`                       | Kubernetes Service type.                                                                                                                                     | ClusterIP                                               |
+| `service.mqtt`                       | Port for MQTT.                                                                                                                                               | 1883                                                    |
+| `service.mqttssl`                    | Port for MQTT(SSL).                                                                                                                                          | 8883                                                    |
+| `service.mgmt`                       | Port for mgmt API.                                                                                                                                           | 8081                                                    |
+| `service.ws`                         | Port for WebSocket/HTTP.                                                                                                                                     | 8083                                                    |
+| `service.wss`                        | Port for WSS/HTTPS.                                                                                                                                          | 8084                                                    |
+| `service.dashboard`                  | Port for dashboard.                                                                                                                                          | 18083                                                   |
+| `service.nodePorts.mqtt`             | Kubernetes node port for MQTT.                                                                                                                               | nil                                                     |
+| `service.nodePorts.mqttssl`          | Kubernetes node port for MQTT(SSL).                                                                                                                          | nil                                                     |
+| `service.nodePorts.mgmt`             | Kubernetes node port for mgmt API.                                                                                                                           | nil                                                     |
+| `service.nodePorts.ws`               | Kubernetes node port for WebSocket/HTTP.                                                                                                                     | nil                                                     |
+| `service.nodePorts.wss`              | Kubernetes node port for WSS/HTTPS.                                                                                                                          | nil                                                     |
+| `service.nodePorts.dashboard`        | Kubernetes node port for dashboard.                                                                                                                          | nil                                                     |
+| `service.loadBalancerIP`             | loadBalancerIP for Service                                                                                                                                   | nil                                                     |
+| `service.loadBalancerSourceRanges`   | Address(es) that are allowed when service is LoadBalancer                                                                                                    | []                                                      |
+| `service.externalIPs`                | ExternalIPs for the service                                                                                                                                  | []                                                      |
+| `service.annotations`                | Service annotations                                                                                                                                          | {}(evaluated as a template)                             |
+| `ingress.dashboard.enabled`          | Enable ingress for EMQX Dashboard                                                                                                                            | false                                                   |
+| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Dashboard                                                                                                                     |                                                         |
+| `ingress.dashboard.path`             | Ingress path for EMQX Dashboard                                                                                                                              | /                                                       |
+| `ingress.dashboard.pathType`         | Ingress pathType for EMQX Dashboard                                                                                                                          | `ImplementationSpecific`                                |
+| `ingress.dashboard.hosts`            | Ingress hosts for EMQX Mgmt API                                                                                                                              | dashboard.emqx.local                                    |
+| `ingress.dashboard.tls`              | Ingress tls for EMQX Mgmt API                                                                                                                                | []                                                      |
+| `ingress.dashboard.annotations`      | Ingress annotations for EMQX Mgmt API                                                                                                                        | {}                                                      |
+| `ingress.mgmt.enabled`               | Enable ingress for EMQX Mgmt API                                                                                                                             | false                                                   |
+| `ingress.dashboard.ingressClassName` | Set the ingress class for EMQX Mgmt API                                                                                                                      |                                                         |
+| `ingress.mgmt.path`                  | Ingress path for EMQX Mgmt API                                                                                                                               | /                                                       |
+| `ingress.mgmt.hosts`                 | Ingress hosts for EMQX Mgmt API                                                                                                                              | api.emqx.local                                          |
+| `ingress.mgmt.tls`                   | Ingress tls for EMQX Mgmt API                                                                                                                                | []                                                      |
+| `ingress.mgmt.annotations`           | Ingress annotations for EMQX Mgmt API                                                                                                                        | {}                                                      |
+| `metrics.enable`                     | If set to true, [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) needs to be installed, and emqx_prometheus needs to enable | false                                                   |
+| `metrics.type`                       | Now we only supported "prometheus"                                                                                                                           | "prometheus"                                            |
+| `ssl.enabled`                        | Enable SSL support                                                                                                                                           | false                                                   |
+| `ssl.useExisting`                    | Use existing certificate or let cert-manager generate one                                                                                                    | false                                                   |
+| `ssl.existingName`                   | Name of existing certificate                                                                                                                                 | emqx-tls                                                |
+| `ssl.dnsnames`                       | DNS name(s) for certificate to be generated                                                                                                                  | {}                                                      |
+| `ssl.issuer.name`                    | Issuer name for certificate generation                                                                                                                       | letsencrypt-dns                                         |
+| `ssl.issuer.kind`                    | Issuer kind for certificate generation                                                                                                                       | ClusterIssuer                                           |
+
+## EMQX specific settings
+
+The following table lists the configurable [EMQX](https://www.emqx.io/)-specific parameters of the chart and their
+default values.
+Parameter | Description | Default Value
+--- | --- | ---
+`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items
+expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix
+can be omitted) or using the configuration
+files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil`
+`emqxLicenseSecretName` | Name of the secret that holds the license information | `nil`
+
+## SSL settings
+`cert-manager` generates secrets with certificate data using the keys `tls.crt` and `tls.key`. The helm chart always mounts those keys as files to `/tmp/ssl/`
+which needs to explicitly configured by either changing the emqx config file or by passing the following environment variables:
+
+```
+  EMQX_LISTENERS__SSL__DEFAULT__SSL_OPTIONS__CERTFILE: /tmp/ssl/tls.crt
+  EMQX_LISTENERS__SSL__DEFAULT__SSL_OPTIONS__KEYFILE: /tmp/ssl/tls.key
+```
+
+If you chose to use an existing certificate, make sure, you update the filenames accordingly.
+

deploy/charts/emqx-ee/templates/StatefulSet.yaml

@@ -0,0 +1,168 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: {{ include "emqx.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  serviceName: {{ include "emqx.fullname" . }}-headless
+  podManagementPolicy: {{ .Values.podManagementPolicy }}
+  {{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+  volumeClaimTemplates:
+    - metadata:
+        name: emqx-data
+        namespace: {{ .Release.Namespace }}
+        labels:
+          app.kubernetes.io/name: {{ include "emqx.name" . }}
+          app.kubernetes.io/instance: {{ .Release.Name }}
+          app.kubernetes.io/managed-by: {{ .Release.Service }}
+      spec:
+        {{- if .Values.persistence.storageClassName }}
+        storageClassName: {{ .Values.persistence.storageClassName | quote }}
+        {{- end }}
+        accessModes:
+          - {{ .Values.persistence.accessMode | quote }}
+        resources:
+         requests:
+           storage: {{ .Values.persistence.size | quote }}
+  {{- end }}
+  updateStrategy:
+    type: RollingUpdate
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "emqx.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ include "emqx.name" . }}
+        version: {{ .Chart.AppVersion }}
+        app.kubernetes.io/name: {{ include "emqx.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+      annotations:
+      {{- with .Values.podAnnotations }}
+          {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.recreatePods }}
+        checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }}
+      {{- end }}
+    spec:
+      volumes:
+      {{- if .Values.ssl.enabled }}
+      - name: ssl-cert
+        secret:
+          secretName: {{ include "emqx.fullname" . }}-tls
+      {{- end }}
+      {{- if not .Values.persistence.enabled }}
+      - name: emqx-data
+        emptyDir: {}
+      {{- else if .Values.persistence.existingClaim }}
+      - name: emqx-data
+        persistentVolumeClaim:
+        {{- with .Values.persistence.existingClaim }}
+          claimName: {{ tpl . $ }}
+        {{- end }}
+      {{- end }}
+      {{- if .Values.emqxLicenseSecretName  }}
+      - name: emqx-license
+        secret:
+          secretName: {{ .Values.emqxLicenseSecretName }}
+      {{- end }}
+      {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}}
+      serviceAccountName:  {{ include "emqx.fullname" . }}
+      {{- end }}
+      {{- if .Values.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      {{- if .Values.initContainers }}
+      initContainers:
+{{ toYaml .Values.initContainers | indent 8 }}
+      {{- end }}
+      {{- if .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- range .Values.image.pullSecrets }}
+        - name: {{ . }}
+      {{- end }}
+      {{- end }}
+      containers:
+        - name: emqx
+          image: "{{ .Values.image.repository }}:{{ .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          {{- if .Values.containerSecurityContext.enabled }}
+          securityContext: {{- omit .Values.containerSecurityContext "enabled" | toYaml | nindent 12 }}
+          {{- end }}
+          ports:
+          - name: mqtt
+            containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT | default 1883 }}
+          - name: mqttssl
+            containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__SSL__DEFAULT | default 8883 }}
+          - name: ws
+            containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WS__DEFAULT | default 8083 }}
+          - name: wss
+            containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT | default 8084 }}
+          - name: dashboard
+            containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP | default 18083 }}
+          {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }}
+          - name: internalmqtt
+            containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT }}
+          {{- end }}
+          {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS) }}
+          - name: dashboardtls
+            containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS }}
+          {{- end }}
+          - name: ekka
+            containerPort: 4370
+          envFrom:
+            - configMapRef:
+                name: {{ include "emqx.fullname" . }}-env
+         {{- if .Values.envFromSecret }}
+            - secretRef:
+                name: {{ .Values.envFromSecret }}
+         {{- end }}
+          resources:
+{{ toYaml .Values.resources | indent 12 }}
+          volumeMounts:
+          - name: emqx-data
+            mountPath: "/opt/emqx/data"
+          {{- if .Values.ssl.enabled }}
+          - name: ssl-cert
+            mountPath: /tmp/ssl
+            readOnly: true
+          {{- end}}
+          {{ if .Values.emqxLicenseSecretName }}
+          - name: emqx-license
+            mountPath: "/opt/emqx/etc/emqx.lic"
+            subPath: "emqx.lic"
+            readOnly: true
+          {{- end }}
+          readinessProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP | default 18083 }}
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            failureThreshold: 30
+          livenessProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP | default 18083 }}
+            initialDelaySeconds: 60
+            periodSeconds: 30
+            failureThreshold: 10
+    {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}

deploy/charts/emqx-ee/templates/_helpers.tpl

@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "emqx.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "emqx.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "emqx.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}

deploy/charts/emqx-ee/templates/certificate.yaml

@@ -0,0 +1,16 @@
+{{- if and (.Values.ssl.enable) (not .Values.ssl.useExisting) -}}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: {{ include "emqx.fullname" . }}-tls
+spec:
+  secretName: {{ include "emqx.fullname" . }}-tls
+  issuerRef:
+    name: {{ default "letsencrypt-staging" .Values.ssl.issuer.name }}
+    kind: {{ default "ClusterIssuer" .Values.ssl.issuer.kind }}
+  dnsNames:
+    {{- range .Values.ssl.dnsnames }}
+    - {{ . }}
+    {{- end }}
+{{- end -}}

deploy/charts/emqx-ee/templates/configmap.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.emqxConfig }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "emqx.fullname" . }}-env
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  {{- range $index, $value := .Values.emqxConfig }}
+  {{- if $value }}
+  {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
+  {{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}"
+  {{- end }}
+  {{- end }}
+{{- end }}

deploy/charts/emqx-ee/templates/ingress.yaml

@@ -0,0 +1,50 @@
+{{- if .Values.ingress.dashboard.enabled -}}
+{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1
+{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
+apiVersion: networking.k8s.io/v1beta1
+{{- else -}}
+apiVersion: extensions/v1beta1
+{{- end }}
+kind: Ingress
+metadata:
+  name: {{ printf "%s-%s" (include "emqx.fullname" .) "dashboard" }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- if .Values.ingress.dashboard.annotations }}
+  annotations:
+    {{- toYaml .Values.ingress.dashboard.annotations | nindent 4 }}
+  {{- end }}
+spec:
+{{- if and .Values.ingress.dashboard.ingressClassName (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
+  ingressClassName: {{ .Values.ingress.dashboard.ingressClassName }}
+{{- end }}
+  rules:
+  {{- range $host := .Values.ingress.dashboard.hosts }}
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: {{ $.Values.ingress.dashboard.path | default "/" }}
+        {{- if (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
+        pathType: {{ $.Values.ingress.dashboard.pathType | default "ImplementationSpecific" }}
+        {{- end }}
+        backend:
+          {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+          service:
+            name: {{ include "emqx.fullname" $ }}
+            port:
+              number: {{ $.Values.service.dashboard }}
+          {{- else }}
+          serviceName: {{ include "emqx.fullname" $ }}
+          servicePort: {{ $.Values.service.dashboard }}
+          {{- end }}
+  {{- end -}}
+  {{- if .Values.ingress.dashboard.tls }}
+  tls:
+    {{- toYaml .Values.ingress.dashboard.tls | nindent 4 }}
+  {{- end }}
+---
+{{- end }}

deploy/charts/emqx-ee/templates/rbac.yaml

@@ -0,0 +1,44 @@
+{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ include "emqx.fullname" . }}
+---
+kind: Role
+{{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- else }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+{{- end }}
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ include "emqx.fullname" . }}
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - endpoints
+  verbs:
+  - get
+  - watch
+  - list
+---
+kind: RoleBinding
+{{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- else }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+{{- end }}
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ include "emqx.fullname" . }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "emqx.fullname" . }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: {{ include "emqx.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

deploy/charts/emqx-ee/templates/secret.yaml

@@ -0,0 +1,19 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "emqx.fullname" . }}-basic-auth
+  namespace: {{ .Release.Namespace }}
+type: kubernetes.io/basic-auth
+stringData:
+  {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__DEFAULT_USERNAME) }}
+  username: admin
+  {{- else }}
+  username: {{ .Values.emqxConfig.EMQX_DASHBOARD__DEFAULT_USERNAME }}
+  {{- end }}
+  {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__DEFAULT_PASSWORD) }}
+  password: public
+  {{- else }}
+  password: {{ .Values.emqxConfig.EMQX_DASHBOARD__DEFAULT_PASSWORD}}
+  {{- end }}
+{{- end }}

deploy/charts/emqx-ee/templates/service-monitor.yaml

@@ -0,0 +1,39 @@
+{{- if and (.Values.metrics.enabled) (eq .Values.metrics.type "prometheus") }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "emqx.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- if .Values.service.annotations }}
+  annotations:
+    {{ toYaml .Values.service.annotations | indent 4 }}
+  {{- end }}
+spec:
+  endpoints:
+  - interval: 10s
+    port: dashboard
+    scheme: http
+    path: /api/v5/prometheus/stats
+    params:
+      type:
+        - prometheus
+    basicAuth:
+      password:
+        name: {{ include "emqx.fullname" . }}-basic-auth
+        key: password
+      username:
+        name: {{ include "emqx.fullname" . }}-basic-auth
+        key: username
+  jobLabel: {{ .Release.Name }}-scraping
+  namespaceSelector:
+    matchNames:
+      -  {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "emqx.name" . }}
+{{- end }}

deploy/charts/emqx-ee/templates/service.yaml

@@ -0,0 +1,149 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "emqx.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- if .Values.service.annotations }}
+  annotations:
+{{ toYaml .Values.service.annotations | indent 4 }}
+  {{- end }}
+spec:
+  type: {{ .Values.service.type }}
+  {{- if eq .Values.service.type "LoadBalancer" }}
+  {{- if .Values.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.service.loadBalancerIP }}
+  {{- end }}
+  {{- if .Values.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges: {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }}
+  {{- end }}
+  {{- if .Values.service.externalIPs }}
+  externalIPs: {{- toYaml .Values.service.externalIPs | nindent 4 }}
+  {{- end }}
+  {{- end }}
+  ports:
+  - name: mqtt
+    port: {{ .Values.service.mqtt | default 1883 }}
+    protocol: TCP
+    targetPort: mqtt
+    {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.mqtt)) }}
+    nodePort: {{ .Values.service.nodePorts.mqtt }}
+    {{- else if eq .Values.service.type "ClusterIP" }}
+    nodePort: null
+    {{- end }}
+    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }}
+  - name: internalmqtt
+    port: {{ .Values.service.internalmqtt | default 11883 }}
+    protocol: TCP
+    targetPort: internalmqtt
+    {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.internalmqtt)) }}
+    nodePort: {{ .Values.service.nodePorts.internalmqtt }}
+    {{- else if eq .Values.service.type "ClusterIP" }}
+    nodePort: null
+    {{- end }}
+    {{ end }}
+  - name: mqttssl
+    port: {{ .Values.service.mqttssl | default 8883 }}
+    protocol: TCP
+    targetPort: mqttssl
+    {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.mqttssl)) }}
+    nodePort: {{ .Values.service.nodePorts.mqttssl }}
+    {{- else if eq .Values.service.type "ClusterIP" }}
+    nodePort: null
+    {{- end }}
+  - name: ws
+    port: {{ .Values.service.ws | default 8083 }}
+    protocol: TCP
+    targetPort: ws
+    {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.ws)) }}
+    nodePort: {{ .Values.service.nodePorts.ws }}
+    {{- else if eq .Values.service.type "ClusterIP" }}
+    nodePort: null
+    {{- end }}
+  - name: wss
+    port: {{ .Values.service.wss | default 8084 }}
+    protocol: TCP
+    targetPort: wss
+    {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.wss)) }}
+    nodePort: {{ .Values.service.nodePorts.wss }}
+    {{- else if eq .Values.service.type "ClusterIP" }}
+    nodePort: null
+    {{- end }}
+  - name: dashboard
+    port: {{ .Values.service.dashboard | default 18083 }}
+    protocol: TCP
+    targetPort: dashboard
+    {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.dashboard)) }}
+    nodePort: {{ .Values.service.nodePorts.dashboard }}
+    {{- else if eq .Values.service.type "ClusterIP" }}
+    nodePort: null
+    {{- end }}
+  {{- if not (empty .Values.service.dashboardtls) }}
+  - name: dashboardtls
+    port: {{ .Values.service.dashboardtls }}
+    protocol: TCP
+    targetPort: dashboardtls
+    {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.dashboardtls)) }}
+    nodePort: {{ .Values.service.nodePorts.dashboardtls }}
+    {{- else if eq .Values.service.type "ClusterIP" }}
+    nodePort: null
+    {{- end }}
+  {{- end }}
+  selector:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "emqx.fullname" . }}-headless
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  type: ClusterIP
+  sessionAffinity: None
+  clusterIP: None
+  publishNotReadyAddresses: true
+  ports:
+  - name: mqtt
+    port: {{ .Values.service.mqtt | default 1883 }}
+    protocol: TCP
+    targetPort: mqtt
+    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }}
+  - name: internalmqtt
+    port: {{ .Values.service.internalmqtt | default 11883 }}
+    protocol: TCP
+    targetPort: internalmqtt
+    {{ end }}
+  - name: mqttssl
+    port: {{ .Values.service.mqttssl | default 8883 }}
+    protocol: TCP
+    targetPort: mqttssl
+  - name: ws
+    port: {{ .Values.service.ws | default 8083 }}
+    protocol: TCP
+    targetPort: ws
+  - name: wss
+    port: {{ .Values.service.wss | default 8084 }}
+    protocol: TCP
+    targetPort: wss
+  - name: dashboard
+    port: {{ .Values.service.dashboard | default 18083 }}
+    protocol: TCP
+    targetPort: dashboard
+  - name: ekka
+    port: 4370
+    protocol: TCP
+    targetPort: ekka
+  selector:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}

deploy/charts/emqx-ee/values.yaml

@@ -0,0 +1,214 @@
+## Default values for emqx.
+## This is a YAML-formatted file.
+## Declare variables to be passed into your templates.
+
+## It is recommended to have odd number of nodes in a cluster, otherwise the emqx cluster cannot be automatically healed in case of net-split.
+replicaCount: 3
+image:
+  repository: emqx/emqx-enterprise
+  pullPolicy: IfNotPresent
+  ## Optionally specify an array of imagePullSecrets.
+  ## Secrets must be manually created in the namespace.
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ##
+  # pullSecrets:
+  # - myRegistryKeySecretName
+
+
+# The name of a secret in the same kubernetes namespace which contains values to
+# be added to the environment (must be manually created)
+# This can be useful for passwords and logins, etc.
+
+# envFromSecret: "emqx-secrets"
+
+## Forces the recreation of pods during helm upgrades. This can be useful to update configuration values even if the container image did not change.
+recreatePods: false
+
+podAnnotations: {}
+
+# Pod deployment policy
+# value: OrderedReady | Parallel
+# To redeploy a chart with existing PVC(s), the value must be set to Parallel to avoid deadlock
+podManagementPolicy: Parallel
+
+persistence:
+  enabled: false
+  size: 20Mi
+  storageClassName: ""
+  accessMode: ReadWriteOnce
+  ## Existing PersistentVolumeClaims
+  ## The value is evaluated as a template
+  ## So, for example, the name can depend on .Release or .Chart
+  # existingClaim: ""
+
+resources: {}
+  # limits:
+  #   cpu: 500m
+  #   memory: 512Mi
+  # requests:
+  #   cpu: 500m
+  #   memory: 512Mi
+
+# Containers that run before the creation of EMQX containers. They can contain utilities or setup scripts.
+initContainers: {}
+  # - name: sysctl
+  #   image: busybox
+  #   securityContext:
+  #     runAsUser: 0
+  #     runAsGroup: 0
+  #     capabilities:
+  #       add:
+  #       - SYS_ADMIN
+  #       drop:
+  #       - ALL
+  #   command:
+  #     - /bin/sh
+  #     - -c
+  #     - |
+  #       mount -o remount rw /proc/sys
+  #       sysctl -w net.core.somaxconn=65535
+  #       sysctl -w net.ipv4.ip_local_port_range="1024 65535"
+  #       sysctl -w kernel.core_uses_pid=0
+  #       sysctl -w net.ipv4.tcp_tw_reuse=1
+  #       sysctl -w fs.nr_open=1000000000
+  #       sysctl -w fs.file-max=1000000000
+  #       sysctl -w net.ipv4.ip_local_port_range='1025 65534'
+  #       sysctl -w net.ipv4.udp_mem='74583000 499445000 749166000'
+  #       sysctl -w net.ipv4.tcp_max_sync_backlog=163840
+  #       sysctl -w net.core.netdev_max_backlog=163840
+  #       sysctl -w net.core.optmem_max=16777216
+  #       sysctl -w net.ipv4.tcp_rmem='1024 4096 16777216'
+  #       sysctl -w net.ipv4.tcp_wmem='1024 4096 16777216'
+  #       sysctl -w net.ipv4.tcp_max_tw_buckets=1048576
+  #       sysctl -w net.ipv4.tcp_fin_timeout=15
+  #       sysctl -w net.core.rmem_default=262144000
+  #       sysctl -w net.core.wmem_default=262144000
+  #       sysctl -w net.core.rmem_max=262144000
+  #       sysctl -w net.core.wmem_max=262144000
+  #       sysctl -w net.ipv4.tcp_mem='378150000  504200000  756300000'
+  #       sysctl -w net.netfilter.nf_conntrack_max=1000000
+  #       sysctl -w net.netfilter.nf_conntrack_tcp_timeout_time_wait=30
+
+## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
+emqxConfig:
+  EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns"
+  EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
+  EMQX_CLUSTER__DNS__RECORD_TYPE: "srv"
+  # EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s"
+  # EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
+  # EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless"
+  # EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}"
+  ## The address type is used to extract host from k8s service.
+  ## Value: ip | dns | hostname
+  ## Note：Hostname is only supported after v4.0-rc.2
+  EMQX_CLUSTER__K8S__ADDRESS_TYPE: "hostname"
+  EMQX_CLUSTER__K8S__SUFFIX: "svc.cluster.local"
+  ## if EMQX_CLUSTER__K8S__ADDRESS_TYPE eq dns
+  # EMQX_CLUSTER__K8S__SUFFIX: "pod.cluster.local"
+  EMQX_DASHBOARD__DEFAULT_USERNAME: "admin"
+  EMQX_DASHBOARD__DEFAULT_PASSWORD: "public"
+
+## EMQX Enterprise Edition requires manual creation of a Secret containing the licensed content. Write the name of Secret to the value of "emqxLicenseSecretName"
+## Example:
+##   kubectl create secret generic emqx-license-secret-name --from-file=/path/to/emqx.lic
+emqxLicenseSecretName:
+
+service:
+  ## Service type
+  ##
+  type: ClusterIP
+  ## Port for MQTT
+  ##
+  mqtt: 1883
+  ## Port for MQTT(SSL)
+  ##
+  mqttssl: 8883
+  ## Port for mgmt API
+  ##
+  mgmt: 8081
+  ## Port for WebSocket/HTTP
+  ##
+  ws: 8083
+  ## Port for WSS/HTTPS
+  ##
+  wss: 8084
+  ## Port for dashboard
+  ##
+  dashboard: 18083
+  ## Port for dashboard HTTPS
+  ##
+  # dashboardtls: 18084
+  ## Specify the nodePort(s) value for the LoadBalancer and NodePort service types.
+  ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
+  ##
+  nodePorts:
+    mqtt:
+    mqttssl:
+    mgmt:
+    ws:
+    wss:
+    dashboard:
+    dashboardtls:
+  ## Set the LoadBalancer service type to internal only.
+  ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
+  ##
+  # loadBalancerIP:
+  ## Load Balancer sources
+  ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service
+  ## Example:
+  ## loadBalancerSourceRanges:
+  ## - 10.10.10.0/24
+  ##
+  loadBalancerSourceRanges: []
+  ## Set the ExternalIPs
+  ##
+  externalIPs: []
+  ## Provide any additional annotations which may be required. Evaluated as a template
+  ##
+  annotations: {}
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+
+ingress:
+  ## ingress for EMQX Dashboard
+  dashboard:
+    enabled: false
+    # ingressClassName: nginx
+    annotations: {}
+      # kubernetes.io/ingress.class: nginx
+      # kubernetes.io/tls-acme: "true"
+    path: /
+    pathType: ImplementationSpecific
+    hosts:
+    - dashboard.emqx.local
+    tls: []
+
+podSecurityContext:
+  enabled: true
+  fsGroup: 1000
+  fsGroupChangePolicy: Always
+  runAsUser: 1000
+  supplementalGroups:
+    - 1000
+
+containerSecurityContext:
+  enabled: true
+  runAsNonRoot: true
+  runAsUser: 1000
+
+metrics:
+  enabled: false
+  type: prometheus
+
+ssl:
+  enabled: false
+  useExisting: false
+  existingName: emqx-tls
+  dnsnames: {}
+  issuer:
+    name: letsencrypt-dns
+    kind: ClusterIssuer