Merge pull request #8246 from Rory-Z/feat/helm-support-dns-cluster

feat(helm): support dns cluster

.github/workflows/run_fvt_tests.yaml

@@ -126,16 +126,19 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
+        discovery:
+        - k8s
+        - dns
         profile:
         - emqx
         os:
-          - ["debian11", "debian:11-slim"]
+        - ["debian11", "debian:11-slim"]
         otp:
-          - 24.2.1-1
+        - 24.2.1-1
         elixir:
-          - 1.13.4
+        - 1.13.4
         arch:
-          - amd64
+        - amd64
       # - emqx-enterprise # TODO test enterprise
 
     steps:
@@ -171,14 +174,15 @@ jobs:
         echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
         echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
     - run: minikube start
-    - name: run emqx on chart
-      timeout-minutes: 5
+    - name: setup helm chart
       working-directory: source
       run: |
         minikube image load $TARGET:$EMQX_TAG
-
         sed -i -r "s/^appVersion: .*$/appVersion: \"$EMQX_TAG\"/g" deploy/charts/emqx/Chart.yaml
-
+    - name: run emqx on chart
+      working-directory: source
+      if: matrix.discovery == 'k8s'
+      run: |
         helm install emqx \
             --set image.repository=$TARGET \
             --set image.pullPolicy=Never \
@@ -190,7 +194,27 @@ jobs:
             --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
             deploy/charts/emqx \
             --debug
-
+    - name: run emqx on chart
+      working-directory: source
+      if: matrix.discovery == 'dns'
+      run: |
+        helm install emqx \
+            --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \
+            --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \
+            --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \
+            --set image.repository=$TARGET \
+            --set image.pullPolicy=Never \
+            --set emqxAclConfig="" \
+            --set image.pullPolicy=Never \
+            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \
+            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \
+            --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
+            --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
+            deploy/charts/emqx \
+            --debug
+    - name: waiting emqx started
+      timeout-minutes: 10
+      run: |
         while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \
           != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
           echo "==============================";
@@ -199,6 +223,16 @@ jobs:
           echo "waiting emqx started";
           sleep 10;
         done
+    - name: Check cluster
+      timeout-minutes: 10
+      run: |
+        kubectl port-forward svc/emqx 18083:18083 &
+        while
+          [ "$(curl --silent --basic -u admin:public -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ];
+        do
+          echo "waiting emqx cluster scale"
+          sleep 1
+        done
     - name: get emqx-0 pods log
       if: failure()
       run: |

deploy/charts/emqx/templates/StatefulSet.yaml

@@ -68,7 +68,9 @@ spec:
         secret:
           secretName: {{ .Values.emqxLicenseSecretName }}
       {{- end }}
+      {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}}
       serviceAccountName:  {{ include "emqx.fullname" . }}
+      {{- end }}
       {{- if .Values.podSecurityContext.enabled }}
       securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
@@ -117,15 +119,6 @@ spec:
             - secretRef:
                 name: {{ .Values.envFromSecret }}
          {{- end }}
-          env:
-          - name: EMQX_NAME
-            value: {{ .Release.Name }}
-          - name: EMQX_CLUSTER__DISCOVERY_STRATEGY
-            value: k8s
-          - name: EMQX_CLUSTER__K8S__SERVICE_NAME
-            value: {{ include "emqx.fullname" . }}-headless
-          - name: EMQX_CLUSTER__K8S__NAMESPACE
-            value: {{ .Release.Namespace }}
           resources:
 {{ toYaml .Values.resources | indent 12 }}
           volumeMounts:

deploy/charts/emqx/templates/configmap.yaml

@@ -1,3 +1,4 @@
+{{- if .Values.emqxConfig }}
 apiVersion: v1
 kind: ConfigMap
 metadata:
@@ -9,6 +10,10 @@ metadata:
     app.kubernetes.io/instance: {{ .Release.Name }}
     app.kubernetes.io/managed-by: {{ .Release.Service }}
 data:
-  {{- range $index, $value := .Values.emqxConfig}}
-    {{$index}}: "{{ $value }}" 
-  {{- end}}
+  {{- range $index, $value := .Values.emqxConfig }}
+  {{- if $value }}
+  {{- $key := (regexReplaceAllLiteral "\\." (regexReplaceAllLiteral "EMQX[_\\.]" (upper (trimAll " " $index)) "") "__") }}
+  {{ print "EMQX_" $key }}: "{{ tpl (printf "%v" $value) $ }}"
+  {{- end }}
+  {{- end }}
+{{- end }}

deploy/charts/emqx/templates/rbac.yaml

@@ -1,3 +1,4 @@
+{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
@@ -40,3 +41,4 @@ roleRef:
   kind: Role
   name: {{ include "emqx.fullname" . }}
   apiGroup: rbac.authorization.k8s.io
+{{- end }}

deploy/charts/emqx/values.yaml

@@ -91,7 +91,13 @@ initContainers: {}
 
 ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
 emqxConfig:
+  EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s"
+  # EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns"
+  # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
+  # EMQX_CLUSTER__DNS__RECORD_TYPE: "srv"
   EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
+  EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless"
+  EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}"
   ## The address type is used to extract host from k8s service.
   ## Value: ip | dns | hostname
   ## Note：Hostname is only supported after v4.0-rc.2