Merge pull request #9833 from zmstone/0123-chore-sync-release-50-to-master

0123 chore sync release 50 to master

Makefile

@@ -6,8 +6,8 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.1.5
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1-beta.13
+export EMQX_DASHBOARD_VERSION ?= v1.1.6
+export EMQX_EE_DASHBOARD_VERSION ?= e1.0.1
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)

apps/emqx/include/emqx_release.hrl

@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.14").
+-define(EMQX_RELEASE_CE, "5.0.15").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.0.0-rc.1").

apps/emqx_bridge/src/emqx_bridge.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "EMQX bridges"},
-    {vsn, "0.1.9"},
+    {vsn, "0.1.10"},
     {registered, []},
     {mod, {emqx_bridge_app, []}},
     {applications, [

apps/emqx_connector/i18n/emqx_connector_redis.conf

@@ -54,7 +54,7 @@ The Redis default port 6379 is used if `[:Port]` is not specified.
           zh: """
 将要连接的 IPv4 或 IPv6 地址，或者主机名。<br/>
 主机名具有以下形式：`Host[:Port]`。<br/>
-如果未指定 `[:Port]`，则使用 MongoDB 默认端口 27017。
+如果未指定 `[:Port]`，则使用 Redis 默认端口 6379。
 """
         }
         label: {

apps/emqx_connector/src/emqx_connector_jwt_worker.erl

@@ -120,7 +120,7 @@ init(#{private_key := PrivateKeyPEM} = Config) ->
 
 handle_continue({make_key, PrivateKeyPEM}, State0) ->
     ?tp(connector_jwt_worker_make_key, #{state => State0}),
-    case jose_jwk:from_pem(PrivateKeyPEM) of
+    try jose_jwk:from_pem(PrivateKeyPEM) of
         JWK = #jose_jwk{} ->
             State = State0#{jwk := JWK},
             {noreply, State, {continue, create_token}};
@@ -135,6 +135,17 @@ handle_continue({make_key, PrivateKeyPEM}, State0) ->
             Error = {invalid_private_key, Error0},
             ?tp(connector_jwt_worker_startup_error, #{error => Error}),
             {stop, {shutdown, {error, Error}}, State0}
+    catch
+        Kind:Error ->
+            ?tp(
+                error,
+                connector_jwt_worker_startup_error,
+                #{
+                    kind => Kind,
+                    error => Error
+                }
+            ),
+            {stop, {shutdown, {error, Error}}, State0}
     end;
 handle_continue(create_token, State0) ->
     State = generate_and_store_jwt(State0),

apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl

@@ -364,3 +364,23 @@ t_unknown_requests(_Config) ->
     gen_server:cast(Worker, unknown_cast),
     ?assertEqual({error, bad_call}, gen_server:call(Worker, unknown_call)),
     ok.
+
+t_truncated_private_key(_Config) ->
+    Config0 = generate_config(),
+    Config = Config0#{private_key := <<"-----BEGIN PRIVATE KEY-----\nMIIEvQI...">>},
+    process_flag(trap_exit, true),
+    ?check_trace(
+        ?wait_async_action(
+            ?assertMatch({ok, _}, emqx_connector_jwt_worker:start_link(Config)),
+            #{?snk_kind := connector_jwt_worker_startup_error},
+            1_000
+        ),
+        fun(Trace) ->
+            ?assertMatch(
+                [#{error := function_clause}],
+                ?of_kind(connector_jwt_worker_startup_error, Trace)
+            ),
+            ok
+        end
+    ),
+    ok.

apps/emqx_resource/i18n/emqx_resource_schema_i18n.conf

@@ -26,7 +26,7 @@ emqx_resource_schema {
     desc {
       en: """The number of buffer workers. Only applicable for egress type bridges.
 For bridges only have ingress direction data flow, it can be set to 0 otherwise must be greater than 0."""
-      zh: """缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时，可设置为 0，否则必须大于 0）。"""
+      zh: """缓存队列 worker 数量。仅对 egress 类型的桥接有意义。当桥接仅有 ingress 方向时，可设置为 0，否则必须大于 0。"""
     }
     label {
       en: """Buffer Pool Size"""

apps/emqx_resource/src/emqx_resource.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_resource, [
     {description, "Manager for all external resources"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {mod, {emqx_resource_app, []}},
     {applications, [

changes/v5.0.15-en.md

@@ -0,0 +1,79 @@
+# v5.0.15
+
+## Enhancements
+
+- [#9569](https://github.com/emqx/emqx/pull/9569) Refactor `/authorization/sources/built_in_database/` by adding `rules/` to the path.
+
+- [#9585](https://github.com/emqx/emqx/pull/9585) `/bridges_probe` API endpoint to test params for creating a new data bridge.
+
+- [#9586](https://github.com/emqx/emqx/pull/9586) Basic auth is no longer allowed for API calls, must use API key instead.
+
+- [#9628](https://github.com/emqx/emqx/pull/9628) Expose additional resource configuration parameters: `start_after_created` and `start_timeout`.
+
+- [#9722](https://github.com/emqx/emqx/pull/9722) Add the following configuration options for Pushing metrics to Prometheus Push Gateway:
+  - `headers`: Allows custom HTTP request headers.
+  - `job_name`: allows to customize the name of the Job pushed to Push Gateway.
+
+- [#9725](https://github.com/emqx/emqx/pull/9725) Remove the config `auto_reconnect` from the emqx_authz, emqx_authn and data-bridge componets.
+  This is because we have another config with similar functions: `resource_opts.auto_restart_interval`。
+  
+  The functions of these two config are difficult to distinguish, which will lead to confusion.
+  After this change, `auto_reconnect` will not be configurable (always be true), and the underlying
+  drivers that support this config will automatically reconnect the abnormally disconnected
+  connection every `2s`.
+  
+  And the config `resource_opts.auto_restart_interval` is still available for user.
+  It is the time interval that emqx restarts the resource when the connection cannot be
+  established for some reason.
+
+- [#9736](https://github.com/emqx/emqx/pull/9736) Refactor of /bridges API to make it more consistent with other APIs:
+  - bridge enable/disable is now done via the endpoint `/bridges/{id}/enable/[true,false]`
+  - `/bridges/{id}/operation/{operation}` endpoints are now `/bridges/{id}/{operation}`
+  - metrics are moved out from the GET `/bridges/{id}` response and can now be fetched via `/bridges/{id}/metrics`
+  - the `bridges/{id}/reset_metrics` endpoint is now `/bridges/{id}/metrics/reset`
+
+- [#9774](https://github.com/emqx/emqx/pull/9774) Add a password complexity requirement when adding or modifying Dashboard users via the API.
+  Now password must contain at least 2 of alphabetic, numeric and special characters,
+  and must be 8 to 64 characters long.
+
+## Bug fixes
+
+- [#9626](https://github.com/emqx/emqx/pull/9626) Return authorization settings with default values.
+  The authorization cache is enabled by default, but due to the missing default value in `GET` response of `/authorization/settings`, it seemed to be disabled from the dashboard.
+
+- [#9680](https://github.com/emqx/emqx/pull/9680) Fix the problem that username and password authentication is mandatory in Influxdb v1 write API.
+
+- [#9726](https://github.com/emqx/emqx/pull/9726) Client fuzzy search API results were missing information which could tell if more results are available in the next pages, this is now fixed by providing `hasnext` flag in the response.
+
+- [#9735](https://github.com/emqx/emqx/pull/9735) Password information has been removed from information log messages for http, ldap, mongo, mqtt, mysql, pgsql and redis.
+
+- [#9748](https://github.com/emqx/emqx/pull/9748) Listeners not configured with `max_connections` will cause the cluster `/listeners` API to return 500 error.
+
+- [#9749](https://github.com/emqx/emqx/pull/9749) In some cases search APIs could respond with an incorrect `count` value in the metadata, that is usually much bigger than expected, this is now fixed.
+
+- [#9750](https://github.com/emqx/emqx/pull/9750) Reload overriding configs after boot.
+  Prior to this change, two configs were allow to change from dashboard, but will not take effect after reboot:
+  * Logging (such as level)
+  * Prometheus configs
+  
+
+- [#9751](https://github.com/emqx/emqx/pull/9751) Fix that obsoleted cert file will not be deleted after the listener is updated/deleted
+
+- [#9763](https://github.com/emqx/emqx/pull/9763) Fix an authentication exception when password is not provided
+
+- [#9765](https://github.com/emqx/emqx/pull/9765) Parse decimals as password from environment variable overrides correctly.
+  Prior to this change, config values for passwords are not allowed to be decimals.
+  e.g. `EMQX_FOOBAR__PASSWORD=12344` or `emqx.foobar.password=1234`
+  would result in a type check error, unless quoted as:
+  `EMQX_FOOBAR__PASSWORD='"12344"'` or `emqx.foobar.password="1234"`.
+  After this fix, the value does not have to be quoted.
+
+- [#9769](https://github.com/emqx/emqx/pull/9769) Fix Erlang shell prompt version prefix. e5.0.15 -> v5.0.15
+
+- [#9780](https://github.com/emqx/emqx/pull/9780) When creating disk queue directory for resource worker, substitute ':' with '-' in worker id.
+
+- [#9781](https://github.com/emqx/emqx/pull/9781) Trace files were left on a node when creating a zip file for download. They are now removed when the file is sent. Also, concurrent downloads will no longer interfere with each other.
+
+- [#9785](https://github.com/emqx/emqx/pull/9785) Stop authentication hook chain if `emqx_authentication` provides a definitive result.
+
+- [#9787](https://github.com/emqx/emqx/pull/9787) Fix a compatible problem for the `webhook` bridge configuration which was created before the v5.0.12.

changes/v5.0.15-zh.md

@@ -0,0 +1,76 @@
+# v5.0.15
+
+## 增强
+
+- [#9569](https://github.com/emqx/emqx/pull/9569) 重构 `/authorization/sources/built_in_database/`  接口，将 `rules/` 添加到了其路径中。
+
+- [#9585](https://github.com/emqx/emqx/pull/9585) 添加新 API 接口 `/bridges_probe` 用于测试创建桥接的参数是否可用。
+
+- [#9586](https://github.com/emqx/emqx/pull/9586) API 调用不再支持基于 `username:password` 的 `baisc` 认证, 现在 API 必须通过 API Key 才能进行调用。
+
+- [#9628](https://github.com/emqx/emqx/pull/9628) 为桥接资源增加了配置参数：`start_after_created` 和 `start_timeout`。
+
+- [#9722](https://github.com/emqx/emqx/pull/9722) 为 Prometheus 推送到 Push Gateway 新增以下配置项：
+  - `headers`：允许自定义 HTTP 请求头。
+  - `job_name`：允许自定义推送到 Push Gateway 的 Job 名称。
+
+- [#9725](https://github.com/emqx/emqx/pull/9725) 从认证、鉴权和数据桥接功能中，删除 `auto_reconnect` 配置项，因为我们还有另一个功能类似的配置项：
+  `resource_opts.auto_restart_interval`。
+  
+  这两个配置项的功能难以区分，会导致困惑。此修改之后，`auto_reconnect` 将不可配置(永远为 true)，
+  支持此配置的底层驱动将以 `2s` 为周期自动重连异常断开的连接。
+  
+  而 `resource_opts.auto_restart_interval` 配置项仍然开放给用户配置，它是资源因为某些原因
+  无法建立连接的时候，emqx 重新启动该资源的时间间隔。
+
+- [#9736](https://github.com/emqx/emqx/pull/9736) 重构部分 /bridges 的API 使得其和其他 API 能够更加一致：
+  - 桥接的启用和禁用现在是通过 `/bridges/{id}/enable/[true,false]` API 来实现的
+  - 使用 `/bridges/{id}/{operation}` 替换了旧的 `/bridges/{id}/operation/{operation}` API
+  - 指标数据从 `/bridges/{id}` 的响应消息中移除，现在可以使用新的 API  `/bridges/{id}/metrics` 进行访问
+  - 使用  `/bridges/{id}/metrics/reset` 替换了旧的 `bridges/{id}/reset_metrics` API
+
+- [#9774](https://github.com/emqx/emqx/pull/9774) 通过 API 添加、修改 Dashboard 用户时，增加对密码复杂度的要求。
+  现在密码必须包含字母、数字以及特殊字符中的至少 2 种，并且长度范围必须是 8~64 个字符。
+
+## 修复
+
+- [#9626](https://github.com/emqx/emqx/pull/9626) 为授权设置 API 返回默认值。
+  授权缓存默认为开启，但是在此修复前，因为默认值在 `/authorization/settings` 这个 API 的返回值中缺失，
+  使得在仪表盘配置页面中看起来是关闭了。
+
+- [#9680](https://github.com/emqx/emqx/pull/9680) 修复 InfluxDB v1 桥接写入 API 配置中强制需要用户名密码认证的问题。
+
+- [#9726](https://github.com/emqx/emqx/pull/9726) 在此修复前，客户端模糊搜索 API 缺少一些可以用于判断是否可以继续翻页的信息，现在通过在响应中提供 `hasnext` 标志来解决这个问题。
+
+- [#9735](https://github.com/emqx/emqx/pull/9735) 密码信息已从http、ldap、mongo、mqtt、mysql、pgsql和redis的信息日志消息中删除。
+
+- [#9748](https://github.com/emqx/emqx/pull/9748) 监听器不配置 `max_connections` 时会导致集群 `/listeners` 接口返回 500 错误。
+
+- [#9749](https://github.com/emqx/emqx/pull/9749) 在某些情况下，搜索 API 可能会在元数据中响应不正确的 `count` 值，这通常比预期的要大得多，现在已经修复了。
+
+- [#9750](https://github.com/emqx/emqx/pull/9750) 启动后重新加载一些重载配置项。
+  在此修复前，下面两个配置项允许从 Dashboard 控制台修改，但是在重启后无法生效：
+  * 日志 （例如日志级别）
+  * Prometheus 配置
+
+- [#9751](https://github.com/emqx/emqx/pull/9751) 修复在更新或者删除监听器后，过时的证书文件没有被删除的问题。
+
+- [#9763](https://github.com/emqx/emqx/pull/9763) 修复客户端没有提供密码时的一个异常
+
+- [#9765](https://github.com/emqx/emqx/pull/9765) 允许使用纯数字作为密码配置。
+  在此修复前，密码的配置必须是字符串，使用纯数字时，会报类型检查错误。
+  例如，`EMQX_FOOBAR__PASSWORD=12344` 或 `emqx.foobar.password=1234` 会出错，
+  必须用引把值括起来才行：
+  `EMQX_FOOBAR__PASSWORD='"12344"'` 或 `emqx.foobar.password="1234"`。
+  修复后可以不使用引号。在环境变量重载中使用更加方便。
+  
+
+- [#9769](https://github.com/emqx/emqx/pull/9769) 修复 Eralng 控制台版本号前缀的打印错误 e5.0.15 -> v5.0.15
+
+- [#9780](https://github.com/emqx/emqx/pull/9780) 在为资源缓存进程创建磁盘队列目录时，在ID中用 '-' 代替 ':'。
+
+- [#9781](https://github.com/emqx/emqx/pull/9781) 当下载 日志追踪 的日志时，一些中间文件将存留在处理节点上，现在这个问题得到了修复。同时，并发下载日志将不再相互干扰。
+
+- [#9785](https://github.com/emqx/emqx/pull/9785) 如果 `emqx_authentication` 提供了确定的结果，则停止认证钩子链。
+
+- [#9787](https://github.com/emqx/emqx/pull/9787) 修复对在 v5.0.12 之前创建的 `webhook` 桥接配置的兼容问题。

deploy/charts/emqx-enterprise/README.md

@@ -14,14 +14,14 @@ To install the chart with the release name `my-emqx`:
 + From github
   ```
   $ git clone https://github.com/emqx/emqx.git
-  $ cd emqx/deploy/charts/emqx
+  $ cd emqx/deploy/charts/emqx-enterprise
   $ helm install my-emqx .
   ```
 
 + From chart repos
   ```
   helm repo add emqx https://repos.emqx.io/charts
-  helm install my-emqx emqx/emqx
+  helm install my-emqx emqx/emqx-enterprise
   ```
   > If you want to install an unstable version, you need to add `--devel` when you execute the `helm install` command.
 
@@ -43,6 +43,9 @@ The following table lists the configurable parameters of the emqx chart and thei
 | `image.repository` | EMQX Image name | `emqx/emqx-enterprise` |
 | `image.pullPolicy` | The image pull policy | IfNotPresent |
 | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) |
+| `serviceAccount.create` | If `true`, create a new service account | `true` |
+| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the fullname template |  |
+| `serviceAccount.annotations` | Annotations to add to the service account |  |
 | `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil |
 | `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false |
 | `podAnnotations ` | Annotations for pod | `{}` |
@@ -102,10 +105,9 @@ The following table lists the configurable [EMQX](https://www.emqx.io/)-specific
 default values.
 Parameter | Description | Default Value
 --- | --- | ---
-`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/latest/configuration/configuration.html) items
-expressed as [environment variables](https://www.emqx.io/docs/en/v4.3/configuration/environment-variable.html) (prefix
-can be omitted) or using the configuration
-files [namespaced dotted notation](https://www.emqx.io/docs/en/latest/configuration/configuration.html) | `nil`
+`emqxConfig` | Map of [configuration](https://www.emqx.io/docs/en/v5.0/admin/cfg.html) items
+expressed as [environment variables](https://www.emqx.io/docs/en/v5.0/admin/cfg.html#environment-variables) (prefix `EMQX_` can be omitted) or using the configuration
+files [namespaced dotted notation](https://www.emqx.io/docs/en/v5.0/admin/cfg.html#syntax) | `nil`
 `emqxLicenseSecretName` | Name of the secret that holds the license information | `nil`
 
 ## SSL settings

deploy/charts/emqx-enterprise/templates/StatefulSet.yaml

@@ -52,6 +52,7 @@ spec:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }}
       {{- end }}
     spec:
+      serviceAccountName: {{ include "emqx.serviceAccountName" . }}
       volumes:
       {{- if .Values.ssl.enabled }}
       - name: ssl-cert
@@ -73,9 +74,6 @@ spec:
         secret:
           secretName: {{ .Values.emqxLicenseSecretName }}
       {{- end }}
-      {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}}
-      serviceAccountName:  {{ include "emqx.fullname" . }}
-      {{- end }}
       {{- if .Values.podSecurityContext.enabled }}
       securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
@@ -107,13 +105,13 @@ spec:
             containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }}
           - name: dashboard
             containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }}
-          {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }}
+          {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }}
           - name: internalmqtt
-            containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT }}
+            containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }}
           {{- end }}
-          {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS) }}
+          {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND) }}
           - name: dashboardtls
-            containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS }}
+            containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND }}
           {{- end }}
           - name: ekka
             containerPort: 4370

deploy/charts/emqx-enterprise/templates/_helpers.tpl

@@ -42,3 +42,14 @@ Get ssl secret name .
     {{ include "emqx.fullname" . }}-tls
 {{- end -}}
 {{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "emqx.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "emqx.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

deploy/charts/emqx-enterprise/templates/rbac.yaml

@@ -1,10 +1,23 @@
-{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}}
+{{- if .Values.serviceAccount.create }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  name: {{ include "emqx.serviceAccountName" . }}
   namespace: {{ .Release.Namespace }}
-  name: {{ include "emqx.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
+
 ---
+{{- if .Values.serviceAccount.create }}
+{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }}
 kind: Role
 {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }}
 apiVersion: rbac.authorization.k8s.io/v1
@@ -23,7 +36,12 @@ rules:
   - get
   - watch
   - list
+{{- end }}
+{{- end }}
+
 ---
+{{- if .Values.serviceAccount.create }}
+{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }}
 kind: RoleBinding
 {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }}
 apiVersion: rbac.authorization.k8s.io/v1
@@ -35,10 +53,11 @@ metadata:
   name: {{ include "emqx.fullname" . }}
 subjects:
   - kind: ServiceAccount
-    name: {{ include "emqx.fullname" . }}
+    name: {{ include "emqx.serviceAccountName" . }}
     namespace: {{ .Release.Namespace }}
 roleRef:
   kind: Role
   name: {{ include "emqx.fullname" . }}
   apiGroup: rbac.authorization.k8s.io
 {{- end }}
+{{- end }}

deploy/charts/emqx-enterprise/templates/service.yaml

@@ -38,7 +38,7 @@ spec:
     {{- else if eq .Values.service.type "ClusterIP" }}
     nodePort: null
     {{- end }}
-    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }}
+    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }}
   - name: internalmqtt
     port: {{ .Values.service.internalmqtt | default 11883 }}
     protocol: TCP

deploy/charts/emqx-enterprise/values.yaml

@@ -16,6 +16,15 @@ image:
   # pullSecrets:
   # - myRegistryKeySecretName
 
+serviceAccount:
+  # Specifies whether a service account should be created
+  # If set false, means you need create service account by yourself
+  create: true
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+  # Annotations to add to the service account
+  annotations: {}
 
 # The name of a secret in the same kubernetes namespace which contains values to
 # be added to the environment (must be manually created)

deploy/charts/emqx/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.0.14
+version: 5.0.15
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.0.14
+appVersion: 5.0.15

deploy/charts/emqx/README.md

@@ -43,6 +43,9 @@ The following table lists the configurable parameters of the emqx chart and thei
 | `image.repository` | EMQX Image name | emqx/emqx |
 | `image.pullPolicy` | The image pull policy | IfNotPresent |
 | `image.pullSecrets ` | The image pull secrets | `[]` (does not add image pull secrets to deployed pods) |
+| `serviceAccount.create` | If `true`, create a new service account | `true` |
+| `serviceAccount.name` | Service account to be used. If not set and `serviceAccount.create` is `true`, a name is generated using the fullname template |  |
+| `serviceAccount.annotations` | Annotations to add to the service account |  |
 | `envFromSecret` | The name pull a secret in the same kubernetes namespace which contains values that will be added to the environment | nil |
 | `recreatePods` | Forces the recreation of pods during upgrades, which can be useful to always apply the most recent configuration. | false |
 | `podAnnotations ` | Annotations for pod | `{}` |

deploy/charts/emqx/templates/StatefulSet.yaml

@@ -52,6 +52,7 @@ spec:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum | quote }}
       {{- end }}
     spec:
+      serviceAccountName: {{ include "emqx.serviceAccountName" . }}
       volumes:
       {{- if .Values.ssl.enabled }}
       - name: ssl-cert
@@ -73,9 +74,6 @@ spec:
         secret:
           secretName: {{ .Values.emqxLicenseSecretName }}
       {{- end }}
-      {{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}}
-      serviceAccountName:  {{ include "emqx.fullname" . }}
-      {{- end }}
       {{- if .Values.podSecurityContext.enabled }}
       securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
       {{- end }}
@@ -107,13 +105,13 @@ spec:
             containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__WSS__DEFAULT__BIND | default 8084 }}
           - name: dashboard
             containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTP__BIND | default 18083 }}
-          {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }}
+          {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }}
           - name: internalmqtt
-            containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT }}
+            containerPort: {{ .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND }}
           {{- end }}
-          {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS) }}
+          {{- if not (empty .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND) }}
           - name: dashboardtls
-            containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS }}
+            containerPort: {{ .Values.emqxConfig.EMQX_DASHBOARD__LISTENER__HTTPS__BIND }}
           {{- end }}
           - name: ekka
             containerPort: 4370

deploy/charts/emqx/templates/_helpers.tpl

@@ -42,3 +42,14 @@ Get ssl secret name .
     {{ include "emqx.fullname" . }}-tls
 {{- end -}}
 {{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "emqx.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "emqx.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

deploy/charts/emqx/templates/rbac.yaml

@@ -1,10 +1,23 @@
-{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s"}}
+{{- if .Values.serviceAccount.create }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
+  name: {{ include "emqx.serviceAccountName" . }}
   namespace: {{ .Release.Namespace }}
-  name: {{ include "emqx.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "emqx.name" . }}
+    helm.sh/chart: {{ include "emqx.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
+
 ---
+{{- if .Values.serviceAccount.create }}
+{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }}
 kind: Role
 {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }}
 apiVersion: rbac.authorization.k8s.io/v1
@@ -23,7 +36,12 @@ rules:
   - get
   - watch
   - list
+{{- end }}
+{{- end }}
+
 ---
+{{- if .Values.serviceAccount.create }}
+{{- if eq .Values.emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY "k8s" }}
 kind: RoleBinding
 {{- if semverCompare ">=1.17-0" .Capabilities.KubeVersion.GitVersion }}
 apiVersion: rbac.authorization.k8s.io/v1
@@ -35,10 +53,11 @@ metadata:
   name: {{ include "emqx.fullname" . }}
 subjects:
   - kind: ServiceAccount
-    name: {{ include "emqx.fullname" . }}
+    name: {{ include "emqx.serviceAccountName" . }}
     namespace: {{ .Release.Namespace }}
 roleRef:
   kind: Role
   name: {{ include "emqx.fullname" . }}
   apiGroup: rbac.authorization.k8s.io
 {{- end }}
+{{- end }}

deploy/charts/emqx/templates/service.yaml

@@ -121,7 +121,7 @@ spec:
     port: {{ .Values.service.mqtt | default 1883 }}
     protocol: TCP
     targetPort: mqtt
-    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__DEFAULT) }}
+    {{- if not (empty .Values.emqxConfig.EMQX_LISTENERS__TCP__INTERNAL__BIND) }}
   - name: internalmqtt
     port: {{ .Values.service.internalmqtt | default 11883 }}
     protocol: TCP

deploy/charts/emqx/values.yaml

@@ -16,6 +16,15 @@ image:
   # pullSecrets:
   # - myRegistryKeySecretName
 
+serviceAccount:
+  # Specifies whether a service account should be created
+  # If set false, means you need create service account by yourself
+  create: true
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+  # Annotations to add to the service account
+  annotations: {}
 
 # The name of a secret in the same kubernetes namespace which contains values to
 # be added to the environment (must be manually created)

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src

@@ -1,6 +1,6 @@
 {application, emqx_ee_bridge, [
     {description, "EMQX Enterprise data bridges"},
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {registered, []},
     {applications, [
         kernel,

lib-ee/emqx_ee_bridge/src/kafka/emqx_bridge_impl_kafka_producer.erl

@@ -75,7 +75,16 @@ on_start(InstId, Config) ->
             }),
             throw(failed_to_start_kafka_client)
     end,
-    WolffProducerConfig = producers_config(BridgeName, ClientId, ProducerConfig),
+    %% Check if this is a dry run
+    TestIdStart = string:find(InstId, ?TEST_ID_PREFIX),
+    IsDryRun =
+        case TestIdStart of
+            nomatch ->
+                false;
+            _ ->
+                string:equal(TestIdStart, InstId)
+        end,
+    WolffProducerConfig = producers_config(BridgeName, ClientId, ProducerConfig, IsDryRun),
     case wolff:ensure_supervised_producers(ClientId, KafkaTopic, WolffProducerConfig) of
         {ok, Producers} ->
             {ok, #{
@@ -241,7 +250,7 @@ ssl(#{enable := true} = SSL) ->
 ssl(_) ->
     [].
 
-producers_config(BridgeName, ClientId, Input) ->
+producers_config(BridgeName, ClientId, Input, IsDryRun) ->
     #{
         max_batch_bytes := MaxBatchBytes,
         compression := Compression,
@@ -271,7 +280,7 @@ producers_config(BridgeName, ClientId, Input) ->
     BridgeType = kafka,
     ResourceID = emqx_bridge_resource:resource_id(BridgeType, BridgeName),
     #{
-        name => make_producer_name(BridgeName),
+        name => make_producer_name(BridgeName, IsDryRun),
         partitioner => partitioner(PartitionStrategy),
         partition_count_refresh_interval_seconds => PCntRefreshInterval,
         replayq_dir => ReplayqDir,
@@ -302,12 +311,20 @@ make_client_id(BridgeName) ->
 
 %% Producer name must be an atom which will be used as a ETS table name for
 %% partition worker lookup.
-make_producer_name(BridgeName) when is_atom(BridgeName) ->
-    make_producer_name(atom_to_list(BridgeName));
-make_producer_name(BridgeName) ->
-    %% Woff needs atom for ets table name registration
-    %% The assumption here is bridge is not often re-created
-    binary_to_atom(iolist_to_binary(["kafka_producer_", BridgeName])).
+make_producer_name(BridgeName, IsDryRun) when is_atom(BridgeName) ->
+    make_producer_name(atom_to_list(BridgeName), IsDryRun);
+make_producer_name(BridgeName, IsDryRun) ->
+    %% Woff needs an atom for ets table name registration. The assumption here is
+    %% that bridges with new names are not often created.
+    case IsDryRun of
+        true ->
+            %% It is a dry run and we don't want to leak too many atoms
+            %% so we use the default producer name instead of creating
+            %% an unique name.
+            probing_wolff_producers;
+        false ->
+            binary_to_atom(iolist_to_binary(["kafka_producer_", BridgeName]))
+    end.
 
 with_log_at_error(Fun, Log) ->
     try