Merge branch 'master' into licence-conf-update

.github/pull_request_template.md

@@ -1,6 +1,6 @@
 Fixes <issue-or-jira-number>
 
-<!-- Make sure to target release-50 branch if this PR is intended to fix the issues for the release candidate. -->
+<!-- Make sure to target release-51 branch if this PR is intended to fix the issues for the release candidate. -->
 
 ## Summary
 copilot:summary

.github/workflows/build_and_push_docker_images.yaml

@@ -97,7 +97,6 @@ jobs:
         env:
           PROFILE: ${{ steps.get_profile.outputs.PROFILE }}
         run: |
-          PROFILE=$PROFILE make -C source deps-$PROFILE
           zip -ryq source.zip source/* source/.[^.]*
       - uses: actions/upload-artifact@v3
         with:

.github/workflows/build_packages_cron.yaml

@@ -23,7 +23,7 @@ jobs:
       matrix:
         profile:
           - ['emqx', 'master']
-          - ['emqx-enterprise', 'release-50']
+          - ['emqx-enterprise', 'release-51']
         otp:
           - 24.3.4.2-3
         arch:

.github/workflows/build_slim_packages.yaml

@@ -8,7 +8,7 @@ on:
   push:
     branches:
       - master
-      - release-50
+      - release-51
   pull_request:
     # GitHub pull_request action is by default triggered when
     # opened reopened or synchronize,

Makefile

@@ -15,7 +15,7 @@ endif
 
 # Dashbord version
 # from https://github.com/emqx/emqx-dashboard5
-export EMQX_DASHBOARD_VERSION ?= v1.2.5
+export EMQX_DASHBOARD_VERSION ?= v1.2.5-1
 export EMQX_EE_DASHBOARD_VERSION ?= e1.0.7
 
 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used
@@ -104,7 +104,7 @@ APPS=$(shell $(SCRIPTS)/find-apps.sh)
 
 .PHONY: $(APPS:%=%-ct)
 define gen-app-ct-target
-$1-ct: $(REBAR) merge-config
+$1-ct: $(REBAR) merge-config clean-test-cluster-config
 	$(eval SUITES := $(shell $(SCRIPTS)/find-suites.sh $1))
 ifneq ($(SUITES),)
 		@ENABLE_COVER_COMPILE=1 $(REBAR) ct -c -v \
@@ -127,7 +127,7 @@ endef
 $(foreach app,$(APPS),$(eval $(call gen-app-prop-target,$(app))))
 
 .PHONY: ct-suite
-ct-suite: $(REBAR) merge-config
+ct-suite: $(REBAR) merge-config clean-test-cluster-config
 ifneq ($(TESTCASE),)
 ifneq ($(GROUP),)
 	$(REBAR) ct -v --readable=$(CT_READABLE) --name $(CT_NODE_NAME) --suite $(SUITE)  --case $(TESTCASE) --group $(GROUP)
@@ -294,3 +294,7 @@ fmt: $(REBAR)
 	@$(SCRIPTS)/erlfmt -w '{apps,lib-ee}/*/{src,include,test}/**/*.{erl,hrl,app.src}'
 	@$(SCRIPTS)/erlfmt -w 'rebar.config.erl'
 	@mix format
+
+.PHONY: clean-test-cluster-config
+clean-test-cluster-config:
+	@rm -f apps/emqx_conf/data/configs/cluster.hocon || true

apps/emqx/include/emqx_release.hrl

@@ -32,10 +32,10 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Community edition
--define(EMQX_RELEASE_CE, "5.0.25").
+-define(EMQX_RELEASE_CE, "5.1.0-alpha.1").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.0.4").
+-define(EMQX_RELEASE_EE, "5.1.0-alpha.1").
 
 %% the HTTP API version
 -define(EMQX_API_VERSION, "5.0").

apps/emqx/src/bpapi/README.md

@@ -149,6 +149,7 @@ After releasing, let's say, 5.1.0, the following actions should be performed to
 
 1. Checkout 5.1.0 tag
 1. Build the code
+1. Replace api version string `"master"` in `apps/emqx/test/emqx_static_checks_data/master.bpapi` with `"5.1"`
 1. Rename `apps/emqx/test/emqx_static_checks_data/master.bpapi` to `apps/emqx/test/emqx_static_checks_data/5.1.bpapi`
 1. Add `apps/emqx/test/emqx_static_checks_data/5.1.bpapi` to the repo
 1. Delete the previous file (e.g. `5.0.bpapi`), unless there is plan to support rolling upgrade from 5.0 to 5.2

apps/emqx/src/emqx.app.src

@@ -2,8 +2,7 @@
 {application, emqx, [
     {id, "emqx"},
     {description, "EMQX Core"},
-    % strict semver, bump manually!
-    {vsn, "5.0.27"},
+    {vsn, "5.1.0"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/persistent_session/emqx_persistent_session.erl

@@ -229,7 +229,7 @@ discard_if_present(ClientID) ->
             ok
     end.
 
--spec discard(binary(), emgx_session:session()) -> emgx_session:session().
+-spec discard(binary(), emqx_session:session()) -> emqx_session:session().
 discard(ClientID, Session) ->
     discard_opt(is_store_enabled(), ClientID, Session).
 

apps/emqx/test/emqx_access_control_SUITE.erl

@@ -120,6 +120,3 @@ clientinfo(InitProps) ->
         },
         InitProps
     ).
-
-toggle_auth(Bool) when is_boolean(Bool) ->
-    emqx_config:put_zone_conf(default, [auth, enable], Bool).

apps/emqx/test/emqx_authentication_SUITE.erl

@@ -98,6 +98,8 @@ init_per_suite(Config) ->
     LogLevel = emqx_logger:get_primary_log_level(),
     ok = emqx_logger:set_log_level(debug),
     application:set_env(ekka, strict_mode, true),
+    emqx_config:erase_all(),
+    emqx_common_test_helpers:stop_apps([]),
     emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
     [{log_level, LogLevel} | Config].

apps/emqx/test/emqx_banned_SUITE.erl

@@ -145,11 +145,12 @@ t_kick(_) ->
 t_session_taken(_) ->
     erlang:process_flag(trap_exit, true),
     Topic = <<"t/banned">>,
-    ClientId2 = <<"t_session_taken">>,
+    ClientId2 = emqx_guid:to_hexstr(emqx_guid:gen()),
     MsgNum = 3,
     Connect = fun() ->
+        ClientId = emqx_guid:to_hexstr(emqx_guid:gen()),
         {ok, C} = emqtt:start_link([
-            {clientid, <<"client1">>},
+            {clientid, ClientId},
             {proto_ver, v5},
             {clean_start, false},
             {properties, #{'Session-Expiry-Interval' => 120}}
@@ -188,9 +189,9 @@ t_session_taken(_) ->
         end,
         15_000
     ),
-    Publish(),
 
     C2 = Connect(),
+    Publish(),
     ?assertEqual(MsgNum, length(receive_messages(MsgNum + 1))),
     ok = emqtt:disconnect(C2),
 

apps/emqx/test/emqx_broker_SUITE.erl

@@ -93,6 +93,7 @@ end_per_group(_Group, _Config) ->
     emqx_common_test_helpers:stop_apps([]).
 
 init_per_suite(Config) ->
+    emqx_common_test_helpers:clear_screen(),
     Config.
 
 end_per_suite(_Config) ->
@@ -442,7 +443,7 @@ t_connected_client_count_persistent(Config) when is_list(Config) ->
             emqx_cm_process_down
         ]
     ),
-    ?assertEqual(0, emqx_cm:get_connected_client_count()),
+    ?retry(_Sleep = 100, _Retries = 20, ?assertEqual(0, emqx_cm:get_connected_client_count())),
     ok;
 t_connected_client_count_persistent({'end', _Config}) ->
     snabbkaffe:stop(),

apps/emqx/test/emqx_channel_SUITE.erl

@@ -892,7 +892,7 @@ t_handle_kicked_publish_will_msg(_) ->
     ),
     receive
         {pub, Msg} -> ok
-    after 200 -> exit(will_message_not_published)
+    after 10_000 -> exit(will_message_not_published)
     end.
 
 t_handle_call_discard(_) ->

apps/emqx/test/emqx_exclusive_sub_SUITE.erl

@@ -34,6 +34,7 @@
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
+    emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
     ok = ekka:start(),
     OldConf = emqx:get_config([zones], #{}),

apps/emqx/test/emqx_olp_SUITE.erl

@@ -28,6 +28,7 @@
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
+    emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
     OldSch = erlang:system_flag(schedulers_online, 1),
     [{old_sch, OldSch} | Config].

apps/emqx/test/emqx_quic_multistreams_SUITE.erl

@@ -24,6 +24,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("quicer/include/quicer.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 suite() ->
     [{timetrap, {seconds, 30}}].
@@ -1089,6 +1090,11 @@ t_multi_streams_unsub(Config) ->
     ),
 
     emqtt:unsubscribe_via(C, SubVia, Topic),
+    ?retry(
+        _Sleep2 = 100,
+        _Attempts2 = 50,
+        false = emqx_router:has_routes(Topic)
+    ),
 
     case emqtt:publish_via(C, PubVia, Topic, #{}, <<6, 7, 8, 9>>, [{qos, PubQos}]) of
         ok when PubQos == 0 ->
@@ -1569,7 +1575,7 @@ t_multi_streams_remote_shutdown(Config) ->
 
     ok = stop_emqx(),
     %% Client should be closed
-    assert_client_die(C, 100, 50).
+    assert_client_die(C, 100, 200).
 
 t_multi_streams_remote_shutdown_with_reconnect(Config) ->
     erlang:process_flag(trap_exit, true),

apps/emqx/test/emqx_shared_sub_SUITE.erl

@@ -666,7 +666,7 @@ t_local_fallback(Config) when is_list(Config) ->
     {true, UsedSubPid1} = last_message(<<"hello1">>, [ConnPid1]),
 
     rpc:call(Node, emqx, publish, [Message2]),
-    {true, UsedSubPid2} = last_message(<<"hello2">>, [ConnPid1]),
+    {true, UsedSubPid2} = last_message(<<"hello2">>, [ConnPid1], 2_000),
 
     emqtt:stop(ConnPid1),
     stop_slave(Node),

apps/emqx_bridge/src/emqx_bridge.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge, [
     {description, "EMQX bridges"},
-    {vsn, "0.1.20"},
+    {vsn, "0.1.21"},
     {registered, [emqx_bridge_sup]},
     {mod, {emqx_bridge_app, []}},
     {applications, [

apps/emqx_bridge/src/emqx_bridge.erl

@@ -227,9 +227,13 @@ post_config_update(_, _Req, NewConf, OldConf, _AppEnv) ->
         diff_confs(NewConf, OldConf),
     %% The config update will be failed if any task in `perform_bridge_changes` failed.
     Result = perform_bridge_changes([
-        {fun emqx_bridge_resource:remove/4, Removed},
-        {fun emqx_bridge_resource:create/4, Added},
-        {fun emqx_bridge_resource:update/4, Updated}
+        #{action => fun emqx_bridge_resource:remove/4, data => Removed},
+        #{
+            action => fun emqx_bridge_resource:create/4,
+            data => Added,
+            on_exception_fn => fun emqx_bridge_resource:remove/4
+        },
+        #{action => fun emqx_bridge_resource:update/4, data => Updated}
     ]),
     ok = unload_hook(),
     ok = load_hook(NewConf),
@@ -345,7 +349,8 @@ perform_bridge_changes(Tasks) ->
 
 perform_bridge_changes([], Result) ->
     Result;
-perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) ->
+perform_bridge_changes([#{action := Action, data := MapConfs} = Task | Tasks], Result0) ->
+    OnException = maps:get(on_exception_fn, Task, fun(_Type, _Name, _Conf, _Opts) -> ok end),
     Result = maps:fold(
         fun
             ({_Type, _Name}, _Conf, {error, Reason}) ->
@@ -359,9 +364,21 @@ perform_bridge_changes([{Action, MapConfs} | Tasks], Result0) ->
                 end;
             ({Type, Name}, Conf, _) ->
                 ResOpts = emqx_resource:fetch_creation_opts(Conf),
-                case Action(Type, Name, Conf, ResOpts) of
+                try Action(Type, Name, Conf, ResOpts) of
                     {error, Reason} -> {error, Reason};
                     Return -> Return
+                catch
+                    Kind:Error:Stacktrace ->
+                        ?SLOG(error, #{
+                            msg => "bridge_config_update_exception",
+                            kind => Kind,
+                            error => Error,
+                            type => Type,
+                            name => Name,
+                            stacktrace => Stacktrace
+                        }),
+                        OnException(Type, Name, Conf, ResOpts),
+                        erlang:raise(Kind, Error, Stacktrace)
                 end
         end,
         Result0,

apps/emqx_bridge/src/emqx_bridge_resource.erl

@@ -58,14 +58,14 @@
 ).
 
 -if(?EMQX_RELEASE_EDITION == ee).
-bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt;
-bridge_to_resource_type(mqtt) -> emqx_connector_mqtt;
+bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector;
+bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector;
 bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http;
 bridge_to_resource_type(webhook) -> emqx_connector_http;
 bridge_to_resource_type(BridgeType) -> emqx_ee_bridge:resource_type(BridgeType).
 -else.
-bridge_to_resource_type(<<"mqtt">>) -> emqx_connector_mqtt;
-bridge_to_resource_type(mqtt) -> emqx_connector_mqtt;
+bridge_to_resource_type(<<"mqtt">>) -> emqx_bridge_mqtt_connector;
+bridge_to_resource_type(mqtt) -> emqx_bridge_mqtt_connector;
 bridge_to_resource_type(<<"webhook">>) -> emqx_connector_http;
 bridge_to_resource_type(webhook) -> emqx_connector_http.
 -endif.

apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl

@@ -47,7 +47,14 @@
     <<"server">> => SERVER,
     <<"username">> => <<"user1">>,
     <<"password">> => <<"">>,
-    <<"proto_ver">> => <<"v5">>
+    <<"proto_ver">> => <<"v5">>,
+    <<"egress">> => #{
+        <<"remote">> => #{
+            <<"topic">> => <<"emqx/${topic}">>,
+            <<"qos">> => <<"${qos}">>,
+            <<"retain">> => false
+        }
+    }
 }).
 -define(MQTT_BRIDGE(SERVER), ?MQTT_BRIDGE(SERVER, <<"mqtt_egress_test_bridge">>)).
 

apps/emqx_bridge_cassandra/test/emqx_bridge_cassandra_SUITE.erl

@@ -541,7 +541,9 @@ t_write_failure(Config) ->
         end),
         fun(Trace0) ->
             ct:pal("trace: ~p", [Trace0]),
-            Trace = ?of_kind(buffer_worker_flush_nack, Trace0),
+            Trace = ?of_kind(
+                [buffer_worker_flush_nack, buffer_worker_retry_inflight_failed], Trace0
+            ),
             [#{result := Result} | _] = Trace,
             case Result of
                 {async_return, {error, {resource_error, _}}} ->
@@ -606,11 +608,12 @@ t_missing_data(Config) ->
     %% to ecql driver
     ?check_trace(
         begin
-            ?wait_async_action(
-                send_message(Config, #{}),
-                #{?snk_kind := handle_async_reply, result := {error, {8704, _}}},
-                10_000
-            ),
+            {_, {ok, _}} =
+                ?wait_async_action(
+                    send_message(Config, #{}),
+                    #{?snk_kind := handle_async_reply, result := {error, {8704, _}}},
+                    30_000
+                ),
             ok
         end,
         fun(Trace0) ->

apps/emqx_bridge_kafka/rebar.config

@@ -1,9 +1,10 @@
 %% -*- mode: erlang; -*-
 {erl_opts, [debug_info]}.
 {deps, [ {wolff, {git, "https://github.com/kafka4beam/wolff.git", {tag, "1.7.5"}}}
-       , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.2"}}}
+       , {kafka_protocol, {git, "https://github.com/kafka4beam/kafka_protocol.git", {tag, "4.1.3"}}}
        , {brod_gssapi, {git, "https://github.com/kafka4beam/brod_gssapi.git", {tag, "v0.1.0"}}}
        , {brod, {git, "https://github.com/kafka4beam/brod.git", {tag, "3.16.8"}}}
+       , {snappyer, "1.2.9"}
        , {emqx_connector, {path, "../../apps/emqx_connector"}}
        , {emqx_resource, {path, "../../apps/emqx_resource"}}
        , {emqx_bridge, {path, "../../apps/emqx_bridge"}}

apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_kafka, [
     {description, "EMQX Enterprise Kafka Bridge"},
-    {vsn, "0.1.2"},
+    {vsn, "0.1.3"},
     {registered, [emqx_bridge_kafka_consumer_sup]},
     {applications, [
         kernel,

apps/emqx_bridge_kafka/src/emqx_bridge_kafka.erl

@@ -247,7 +247,8 @@ fields(producer_opts) ->
         {kafka,
             mk(ref(producer_kafka_opts), #{
                 required => true,
-                desc => ?DESC(producer_kafka_opts)
+                desc => ?DESC(producer_kafka_opts),
+                validator => fun producer_strategy_key_validator/1
             })}
     ];
 fields(producer_kafka_opts) ->
@@ -459,3 +460,11 @@ consumer_topic_mapping_validator(TopicMapping = [_ | _]) ->
         false ->
             {error, "Kafka topics must not be repeated in a bridge"}
     end.
+
+producer_strategy_key_validator(#{
+    <<"partition_strategy">> := key_dispatch,
+    <<"message">> := #{<<"key">> := ""}
+}) ->
+    {error, "Message key cannot be empty when `key_dispatch` strategy is used"};
+producer_strategy_key_validator(_) ->
+    ok.

apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl

@@ -101,6 +101,10 @@
     " the connection parameters."
 ).
 
+%% Allocatable resources
+-define(kafka_client_id, kafka_client_id).
+-define(kafka_subscriber_id, kafka_subscriber_id).
+
 %%-------------------------------------------------------------------------------------
 %% `emqx_resource' API
 %%-------------------------------------------------------------------------------------
@@ -140,6 +144,7 @@ on_start(ResourceId, Config) ->
             Auth -> [{sasl, emqx_bridge_kafka_impl:sasl(Auth)}]
         end,
     ClientOpts = add_ssl_opts(ClientOpts0, SSL),
+    ok = emqx_resource:allocate_resource(ResourceId, ?kafka_client_id, ClientID),
     case brod:start_client(BootstrapHosts, ClientID, ClientOpts) of
         ok ->
             ?tp(
@@ -163,7 +168,21 @@ on_start(ResourceId, Config) ->
     start_consumer(Config, ResourceId, ClientID).
 
 -spec on_stop(resource_id(), state()) -> ok.
-on_stop(_ResourceID, State) ->
+on_stop(ResourceId, _State = undefined) ->
+    case emqx_resource:get_allocated_resources(ResourceId) of
+        #{?kafka_client_id := ClientID, ?kafka_subscriber_id := SubscriberId} ->
+            stop_subscriber(SubscriberId),
+            stop_client(ClientID),
+            ?tp(kafka_consumer_subcriber_and_client_stopped, #{}),
+            ok;
+        #{?kafka_client_id := ClientID} ->
+            stop_client(ClientID),
+            ?tp(kafka_consumer_just_client_stopped, #{}),
+            ok;
+        _ ->
+            ok
+    end;
+on_stop(_ResourceId, State) ->
     #{
         subscriber_id := SubscriberId,
         kafka_client_id := ClientID
@@ -333,6 +352,9 @@ start_consumer(Config, ResourceId, ClientID) ->
     %% spawns one worker for each assigned topic-partition
     %% automatically, so we should not spawn duplicate workers.
     SubscriberId = make_subscriber_id(BridgeName),
+    ?tp(kafka_consumer_about_to_start_subscriber, #{}),
+    ok = emqx_resource:allocate_resource(ResourceId, ?kafka_subscriber_id, SubscriberId),
+    ?tp(kafka_consumer_subscriber_allocated, #{}),
     case emqx_bridge_kafka_consumer_sup:start_child(SubscriberId, GroupSubscriberConfig) of
         {ok, _ConsumerPid} ->
             ?tp(
@@ -359,7 +381,13 @@ start_consumer(Config, ResourceId, ClientID) ->
 stop_subscriber(SubscriberId) ->
     _ = log_when_error(
         fun() ->
-            emqx_bridge_kafka_consumer_sup:ensure_child_deleted(SubscriberId)
+            try
+                emqx_bridge_kafka_consumer_sup:ensure_child_deleted(SubscriberId)
+            catch
+                exit:{noproc, _} ->
+                    %% may happen when node is shutting down
+                    ok
+            end
         end,
         #{
             msg => "failed_to_delete_kafka_subscriber",
@@ -443,16 +471,22 @@ do_get_topic_status(ClientID, KafkaTopic, SubscriberId, NPartitions) ->
     end.
 
 are_subscriber_workers_alive(SubscriberId) ->
-    Children = supervisor:which_children(emqx_bridge_kafka_consumer_sup),
-    case lists:keyfind(SubscriberId, 1, Children) of
-        false ->
-            false;
-        {_, Pid, _, _} ->
-            Workers = brod_group_subscriber_v2:get_workers(Pid),
-            %% we can't enforce the number of partitions on a single
-            %% node, as the group might be spread across an emqx
-            %% cluster.
-            lists:all(fun is_process_alive/1, maps:values(Workers))
+    try
+        Children = supervisor:which_children(emqx_bridge_kafka_consumer_sup),
+        case lists:keyfind(SubscriberId, 1, Children) of
+            false ->
+                false;
+            {_, Pid, _, _} ->
+                Workers = brod_group_subscriber_v2:get_workers(Pid),
+                %% we can't enforce the number of partitions on a single
+                %% node, as the group might be spread across an emqx
+                %% cluster.
+                lists:all(fun is_process_alive/1, maps:values(Workers))
+        end
+    catch
+        exit:{shutdown, _} ->
+            %% may happen if node is shutting down
+            false
     end.
 
 log_when_error(Fun, Log) ->

apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_producer.erl

@@ -23,6 +23,11 @@
 
 -include_lib("emqx/include/logger.hrl").
 
+%% Allocatable resources
+-define(kafka_resource_id, kafka_resource_id).
+-define(kafka_client_id, kafka_client_id).
+-define(kafka_producers, kafka_producers).
+
 %% TODO: rename this to `kafka_producer' after alias support is added
 %% to hocon; keeping this as just `kafka' for backwards compatibility.
 -define(BRIDGE_TYPE, kafka).
@@ -46,9 +51,11 @@ on_start(InstId, Config) ->
     } = Config,
     BridgeType = ?BRIDGE_TYPE,
     ResourceId = emqx_bridge_resource:resource_id(BridgeType, BridgeName),
+    ok = emqx_resource:allocate_resource(InstId, ?kafka_resource_id, ResourceId),
     _ = maybe_install_wolff_telemetry_handlers(ResourceId),
     Hosts = emqx_bridge_kafka_impl:hosts(Hosts0),
     ClientId = emqx_bridge_kafka_impl:make_client_id(BridgeType, BridgeName),
+    ok = emqx_resource:allocate_resource(InstId, ?kafka_client_id, ClientId),
     ClientConfig = #{
         min_metadata_refresh_interval => MinMetaRefreshInterval,
         connect_timeout => ConnTimeout,
@@ -86,6 +93,7 @@ on_start(InstId, Config) ->
     WolffProducerConfig = producers_config(BridgeName, ClientId, KafkaConfig, IsDryRun),
     case wolff:ensure_supervised_producers(ClientId, KafkaTopic, WolffProducerConfig) of
         {ok, Producers} ->
+            ok = emqx_resource:allocate_resource(InstId, ?kafka_producers, Producers),
             {ok, #{
                 message_template => compile_message_template(MessageTemplate),
                 client_id => ClientId,
@@ -120,28 +128,63 @@ on_start(InstId, Config) ->
             )
     end.
 
-on_stop(_InstanceID, #{client_id := ClientID, producers := Producers, resource_id := ResourceID}) ->
-    _ = with_log_at_error(
-        fun() -> wolff:stop_and_delete_supervised_producers(Producers) end,
-        #{
-            msg => "failed_to_delete_kafka_producer",
-            client_id => ClientID
-        }
-    ),
-    _ = with_log_at_error(
-        fun() -> wolff:stop_and_delete_supervised_client(ClientID) end,
-        #{
-            msg => "failed_to_delete_kafka_client",
-            client_id => ClientID
-        }
-    ),
-    with_log_at_error(
-        fun() -> uninstall_telemetry_handlers(ResourceID) end,
+on_stop(InstanceId, _State) ->
+    case emqx_resource:get_allocated_resources(InstanceId) of
         #{
-            msg => "failed_to_uninstall_telemetry_handlers",
-            client_id => ClientID
-        }
-    ).
+            ?kafka_client_id := ClientId,
+            ?kafka_producers := Producers,
+            ?kafka_resource_id := ResourceId
+        } ->
+            _ = with_log_at_error(
+                fun() -> wolff:stop_and_delete_supervised_producers(Producers) end,
+                #{
+                    msg => "failed_to_delete_kafka_producer",
+                    client_id => ClientId
+                }
+            ),
+            _ = with_log_at_error(
+                fun() -> wolff:stop_and_delete_supervised_client(ClientId) end,
+                #{
+                    msg => "failed_to_delete_kafka_client",
+                    client_id => ClientId
+                }
+            ),
+            _ = with_log_at_error(
+                fun() -> uninstall_telemetry_handlers(ResourceId) end,
+                #{
+                    msg => "failed_to_uninstall_telemetry_handlers",
+                    resource_id => ResourceId
+                }
+            ),
+            ok;
+        #{?kafka_client_id := ClientId, ?kafka_resource_id := ResourceId} ->
+            _ = with_log_at_error(
+                fun() -> wolff:stop_and_delete_supervised_client(ClientId) end,
+                #{
+                    msg => "failed_to_delete_kafka_client",
+                    client_id => ClientId
+                }
+            ),
+            _ = with_log_at_error(
+                fun() -> uninstall_telemetry_handlers(ResourceId) end,
+                #{
+                    msg => "failed_to_uninstall_telemetry_handlers",
+                    resource_id => ResourceId
+                }
+            ),
+            ok;
+        #{?kafka_resource_id := ResourceId} ->
+            _ = with_log_at_error(
+                fun() -> uninstall_telemetry_handlers(ResourceId) end,
+                #{
+                    msg => "failed_to_uninstall_telemetry_handlers",
+                    resource_id => ResourceId
+                }
+            ),
+            ok;
+        _ ->
+            ok
+    end.
 
 on_query(
     _InstId,

apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_consumer_SUITE.erl

@@ -59,7 +59,9 @@ only_once_tests() ->
         t_cluster_group,
         t_node_joins_existing_cluster,
         t_cluster_node_down,
-        t_multiple_topic_mappings
+        t_multiple_topic_mappings,
+        t_resource_manager_crash_after_subscriber_started,
+        t_resource_manager_crash_before_subscriber_started
     ].
 
 init_per_suite(Config) ->
@@ -333,6 +335,7 @@ init_per_testcase(TestCase, Config) ->
 common_init_per_testcase(TestCase, Config0) ->
     ct:timetrap(timer:seconds(60)),
     delete_all_bridges(),
+    emqx_config:delete_override_conf_files(),
     KafkaTopic =
         <<
             (atom_to_binary(TestCase))/binary,
@@ -1117,6 +1120,24 @@ stop_async_publisher(Pid) ->
     end,
     ok.
 
+kill_resource_managers() ->
+    ct:pal("gonna kill resource managers"),
+    lists:foreach(
+        fun({_, Pid, _, _}) ->
+            ct:pal("terminating resource manager ~p", [Pid]),
+            Ref = monitor(process, Pid),
+            exit(Pid, kill),
+            receive
+                {'DOWN', Ref, process, Pid, killed} ->
+                    ok
+            after 500 ->
+                ct:fail("pid ~p didn't die!", [Pid])
+            end,
+            ok
+        end,
+        supervisor:which_children(emqx_resource_manager_sup)
+    ).
+
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
@@ -2019,3 +2040,118 @@ t_begin_offset_earliest(Config) ->
         end
     ),
     ok.
+
+t_resource_manager_crash_after_subscriber_started(Config) ->
+    ?check_trace(
+        begin
+            ?force_ordering(
+                #{?snk_kind := kafka_consumer_subscriber_allocated},
+                #{?snk_kind := will_kill_resource_manager}
+            ),
+            ?force_ordering(
+                #{?snk_kind := resource_manager_killed},
+                #{?snk_kind := kafka_consumer_subscriber_started}
+            ),
+            spawn_link(fun() ->
+                ?tp(will_kill_resource_manager, #{}),
+                kill_resource_managers(),
+                ?tp(resource_manager_killed, #{}),
+                ok
+            end),
+
+            %% even if the resource manager is dead, we can still
+            %% clear the allocated resources.
+
+            %% We avoid asserting only the `config_update_crashed'
+            %% error here because there's a race condition (just a
+            %% problem for the test assertion below) in which the
+            %% `emqx_resource_manager:create/5' call returns a failure
+            %% (not checked) and then `lookup' in that module is
+            %% delayed enough so that the manager supervisor has time
+            %% to restart the manager process and for the latter to
+            %% startup successfully.  Occurs frequently in CI...
+
+            {Res, {ok, _}} =
+                ?wait_async_action(
+                    create_bridge(Config),
+                    #{?snk_kind := kafka_consumer_subcriber_and_client_stopped},
+                    10_000
+                ),
+            case Res of
+                {error, {config_update_crashed, {killed, _}}} ->
+                    ok;
+                {ok, _} ->
+                    %% the new manager may have had time to startup
+                    %% before the resource status cache is read...
+                    ok;
+                _ ->
+                    ct:fail("unexpected result: ~p", [Res])
+            end,
+            ?assertMatch({ok, _}, delete_bridge(Config)),
+            ?retry(
+                _Sleep = 50,
+                _Attempts = 50,
+                ?assertEqual([], supervisor:which_children(emqx_bridge_kafka_consumer_sup))
+            ),
+            ok
+        end,
+        []
+    ),
+    ok.
+
+t_resource_manager_crash_before_subscriber_started(Config) ->
+    ?check_trace(
+        begin
+            ?force_ordering(
+                #{?snk_kind := kafka_consumer_client_started},
+                #{?snk_kind := will_kill_resource_manager}
+            ),
+            ?force_ordering(
+                #{?snk_kind := resource_manager_killed},
+                #{?snk_kind := kafka_consumer_about_to_start_subscriber}
+            ),
+            spawn_link(fun() ->
+                ?tp(will_kill_resource_manager, #{}),
+                kill_resource_managers(),
+                ?tp(resource_manager_killed, #{}),
+                ok
+            end),
+
+            %% even if the resource manager is dead, we can still
+            %% clear the allocated resources.
+
+            %% We avoid asserting only the `config_update_crashed'
+            %% error here because there's a race condition (just a
+            %% problem for the test assertion below) in which the
+            %% `emqx_resource_manager:create/5' call returns a failure
+            %% (not checked) and then `lookup' in that module is
+            %% delayed enough so that the manager supervisor has time
+            %% to restart the manager process and for the latter to
+            %% startup successfully.  Occurs frequently in CI...
+            {Res, {ok, _}} =
+                ?wait_async_action(
+                    create_bridge(Config),
+                    #{?snk_kind := kafka_consumer_just_client_stopped},
+                    10_000
+                ),
+            case Res of
+                {error, {config_update_crashed, {killed, _}}} ->
+                    ok;
+                {ok, _} ->
+                    %% the new manager may have had time to startup
+                    %% before the resource status cache is read...
+                    ok;
+                _ ->
+                    ct:fail("unexpected result: ~p", [Res])
+            end,
+            ?assertMatch({ok, _}, delete_bridge(Config)),
+            ?retry(
+                _Sleep = 50,
+                _Attempts = 50,
+                ?assertEqual([], supervisor:which_children(emqx_bridge_kafka_consumer_sup))
+            ),
+            ok
+        end,
+        []
+    ),
+    ok.

apps/emqx_bridge_kafka/test/emqx_bridge_kafka_impl_producer_SUITE.erl

@@ -446,6 +446,8 @@ t_failed_creation_then_fix(Config) ->
     ?assertMatch(#kafka_message{key = BinTime}, KafkaMsg),
     %% TODO: refactor those into init/end per testcase
     ok = ?PRODUCER:on_stop(ResourceId, State),
+    ?assertEqual([], supervisor:which_children(wolff_client_sup)),
+    ?assertEqual([], supervisor:which_children(wolff_producers_sup)),
     ok = emqx_bridge_resource:remove(BridgeId),
     delete_all_bridges(),
     ok.

apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl

@@ -138,6 +138,36 @@ kafka_consumer_test() ->
 
     ok.
 
+message_key_dispatch_validations_test() ->
+    Conf0 = kafka_producer_new_hocon(),
+    Conf1 =
+        Conf0 ++
+            "\n"
+            "bridges.kafka.myproducer.kafka.message.key = \"\""
+            "\n"
+            "bridges.kafka.myproducer.kafka.partition_strategy = \"key_dispatch\"",
+    Conf = parse(Conf1),
+    ?assertMatch(
+        #{
+            <<"kafka">> :=
+                #{
+                    <<"partition_strategy">> := <<"key_dispatch">>,
+                    <<"message">> := #{<<"key">> := <<>>}
+                }
+        },
+        emqx_utils_maps:deep_get([<<"bridges">>, <<"kafka">>, <<"myproducer">>], Conf)
+    ),
+    ?assertThrow(
+        {_, [
+            #{
+                path := "bridges.kafka.myproducer.kafka",
+                reason := "Message key cannot be empty when `key_dispatch` strategy is used"
+            }
+        ]},
+        check(Conf)
+    ),
+    ok.
+
 %%===========================================================================
 %% Helper functions
 %%===========================================================================

apps/emqx_bridge_mqtt/README.md

@@ -0,0 +1,32 @@
+# EMQX MQTT Broker Bridge
+
+This application connects EMQX to virtually any MQTT broker adhering to either [MQTTv3][1] or [MQTTv5][2] standard. The connection is facilitated through the _MQTT bridge_ abstraction, allowing for the flow of data in both directions: from the remote broker to EMQX (ingress) and from EMQX to the remote broker (egress).
+
+User can create a rule and easily ingest into a remote MQTT broker by leveraging [EMQX Rules][3].
+
+
+# Documentation
+
+- Refer to [Bridge Data into MQTT Broker][4] for how to use EMQX dashboard to set up ingress or egress bridge, or even both at the same time.
+
+- Refer to [EMQX Rules][3] for the EMQX rules engine introduction.
+
+
+# HTTP APIs
+
+Several APIs are provided for bridge management, refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges) for more detailed information.
+
+
+# Contributing
+
+Please see our [contributing guide](../../CONTRIBUTING.md).
+
+
+# License
+
+Apache License 2.0, see [LICENSE](../../APL.txt).
+
+[1]: https://docs.oasis-open.org/mqtt/mqtt/v3.1.1
+[2]: https://docs.oasis-open.org/mqtt/mqtt/v5.0/os/mqtt-v5.0-os.html
+[3]: https://docs.emqx.com/en/enterprise/v5.0/data-integration/rules.html
+[4]: https://www.emqx.io/docs/en/v5.0/data-integration/data-bridge-mqtt.html

apps/emqx_bridge_mqtt/rebar.config

@@ -0,0 +1,3 @@
+{deps, [
+    {emqx, {path, "../../apps/emqx"}}
+]}.

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt.app.src

@@ -0,0 +1,18 @@
+%% -*- mode: erlang -*-
+{application, emqx_bridge_mqtt, [
+    {description, "EMQX MQTT Broker Bridge"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib,
+        emqx,
+        emqx_resource,
+        emqx_bridge,
+        emqtt
+    ]},
+    {env, []},
+    {modules, []},
+    {licenses, ["Apache 2.0"]},
+    {links, []}
+]}.

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_connector.erl

@@ -0,0 +1,340 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_bridge_mqtt_connector).
+
+-include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("emqx/include/logger.hrl").
+
+-behaviour(emqx_resource).
+
+-export([on_message_received/3]).
+
+%% callbacks of behaviour emqx_resource
+-export([
+    callback_mode/0,
+    on_start/2,
+    on_stop/2,
+    on_query/3,
+    on_query_async/4,
+    on_get_status/2
+]).
+
+-export([on_async_result/2]).
+
+-define(HEALTH_CHECK_TIMEOUT, 1000).
+
+%% ===================================================================
+%% When use this bridge as a data source, ?MODULE:on_message_received will be called
+%% if the bridge received msgs from the remote broker.
+on_message_received(Msg, HookPoint, ResId) ->
+    emqx_resource_metrics:received_inc(ResId),
+    emqx:run_hook(HookPoint, [Msg]).
+
+%% ===================================================================
+callback_mode() -> async_if_possible.
+
+on_start(ResourceId, Conf) ->
+    ?SLOG(info, #{
+        msg => "starting_mqtt_connector",
+        connector => ResourceId,
+        config => emqx_utils:redact(Conf)
+    }),
+    case start_ingress(ResourceId, Conf) of
+        {ok, Result1} ->
+            case start_egress(ResourceId, Conf) of
+                {ok, Result2} ->
+                    {ok, maps:merge(Result1, Result2)};
+                {error, Reason} ->
+                    _ = stop_ingress(Result1),
+                    {error, Reason}
+            end;
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+start_ingress(ResourceId, Conf) ->
+    ClientOpts = mk_client_opts(ResourceId, "ingress", Conf),
+    case mk_ingress_config(ResourceId, Conf) of
+        Ingress = #{} ->
+            start_ingress(ResourceId, Ingress, ClientOpts);
+        undefined ->
+            {ok, #{}}
+    end.
+
+start_ingress(ResourceId, Ingress, ClientOpts) ->
+    PoolName = <<ResourceId/binary, ":ingress">>,
+    PoolSize = choose_ingress_pool_size(ResourceId, Ingress),
+    Options = [
+        {name, PoolName},
+        {pool_size, PoolSize},
+        {ingress, Ingress},
+        {client_opts, ClientOpts}
+    ],
+    case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_ingress, Options) of
+        ok ->
+            {ok, #{ingress_pool_name => PoolName}};
+        {error, {start_pool_failed, _, Reason}} ->
+            {error, Reason}
+    end.
+
+choose_ingress_pool_size(
+    ResourceId,
+    #{remote := #{topic := RemoteTopic}, pool_size := PoolSize}
+) ->
+    case emqx_topic:parse(RemoteTopic) of
+        {_Filter, #{share := _Name}} ->
+            % NOTE: this is shared subscription, many workers may subscribe
+            PoolSize;
+        {_Filter, #{}} ->
+            % NOTE: this is regular subscription, only one worker should subscribe
+            ?SLOG(warning, #{
+                msg => "mqtt_bridge_ingress_pool_size_ignored",
+                connector => ResourceId,
+                reason =>
+                    "Remote topic filter is not a shared subscription, "
+                    "ingress pool will start with a single worker",
+                config_pool_size => PoolSize,
+                pool_size => 1
+            }),
+            1
+    end.
+
+start_egress(ResourceId, Conf) ->
+    % NOTE
+    % We are ignoring the user configuration here because there's currently no reliable way
+    % to ensure proper session recovery according to the MQTT spec.
+    ClientOpts = maps:put(clean_start, true, mk_client_opts(ResourceId, "egress", Conf)),
+    case mk_egress_config(Conf) of
+        Egress = #{} ->
+            start_egress(ResourceId, Egress, ClientOpts);
+        undefined ->
+            {ok, #{}}
+    end.
+
+start_egress(ResourceId, Egress, ClientOpts) ->
+    PoolName = <<ResourceId/binary, ":egress">>,
+    PoolSize = maps:get(pool_size, Egress),
+    Options = [
+        {name, PoolName},
+        {pool_size, PoolSize},
+        {client_opts, ClientOpts}
+    ],
+    case emqx_resource_pool:start(PoolName, emqx_bridge_mqtt_egress, Options) of
+        ok ->
+            {ok, #{
+                egress_pool_name => PoolName,
+                egress_config => emqx_bridge_mqtt_egress:config(Egress)
+            }};
+        {error, {start_pool_failed, _, Reason}} ->
+            {error, Reason}
+    end.
+
+on_stop(ResourceId, State) ->
+    ?SLOG(info, #{
+        msg => "stopping_mqtt_connector",
+        connector => ResourceId
+    }),
+    ok = stop_ingress(State),
+    ok = stop_egress(State).
+
+stop_ingress(#{ingress_pool_name := PoolName}) ->
+    emqx_resource_pool:stop(PoolName);
+stop_ingress(#{}) ->
+    ok.
+
+stop_egress(#{egress_pool_name := PoolName}) ->
+    emqx_resource_pool:stop(PoolName);
+stop_egress(#{}) ->
+    ok.
+
+on_query(
+    ResourceId,
+    {send_message, Msg},
+    #{egress_pool_name := PoolName, egress_config := Config}
+) ->
+    ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => ResourceId}),
+    handle_send_result(with_egress_client(PoolName, send, [Msg, Config]));
+on_query(ResourceId, {send_message, Msg}, #{}) ->
+    ?SLOG(error, #{
+        msg => "forwarding_unavailable",
+        connector => ResourceId,
+        message => Msg,
+        reason => "Egress is not configured"
+    }).
+
+on_query_async(
+    ResourceId,
+    {send_message, Msg},
+    CallbackIn,
+    #{egress_pool_name := PoolName, egress_config := Config}
+) ->
+    ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => ResourceId}),
+    Callback = {fun on_async_result/2, [CallbackIn]},
+    Result = with_egress_client(PoolName, send_async, [Msg, Callback, Config]),
+    case Result of
+        ok ->
+            ok;
+        {ok, Pid} when is_pid(Pid) ->
+            {ok, Pid};
+        {error, Reason} ->
+            {error, classify_error(Reason)}
+    end;
+on_query_async(ResourceId, {send_message, Msg}, _Callback, #{}) ->
+    ?SLOG(error, #{
+        msg => "forwarding_unavailable",
+        connector => ResourceId,
+        message => Msg,
+        reason => "Egress is not configured"
+    }).
+
+with_egress_client(ResourceId, Fun, Args) ->
+    ecpool:pick_and_do(ResourceId, {emqx_bridge_mqtt_egress, Fun, Args}, no_handover).
+
+on_async_result(Callback, Result) ->
+    apply_callback_function(Callback, handle_send_result(Result)).
+
+apply_callback_function(F, Result) when is_function(F) ->
+    erlang:apply(F, [Result]);
+apply_callback_function({F, A}, Result) when is_function(F), is_list(A) ->
+    erlang:apply(F, A ++ [Result]);
+apply_callback_function({M, F, A}, Result) when is_atom(M), is_atom(F), is_list(A) ->
+    erlang:apply(M, F, A ++ [Result]).
+
+handle_send_result(ok) ->
+    ok;
+handle_send_result({ok, #{reason_code := ?RC_SUCCESS}}) ->
+    ok;
+handle_send_result({ok, #{reason_code := ?RC_NO_MATCHING_SUBSCRIBERS}}) ->
+    ok;
+handle_send_result({ok, Reply}) ->
+    {error, classify_reply(Reply)};
+handle_send_result({error, Reason}) ->
+    {error, classify_error(Reason)}.
+
+classify_reply(Reply = #{reason_code := _}) ->
+    {unrecoverable_error, Reply}.
+
+classify_error(disconnected = Reason) ->
+    {recoverable_error, Reason};
+classify_error(ecpool_empty) ->
+    {recoverable_error, disconnected};
+classify_error({disconnected, _RC, _} = Reason) ->
+    {recoverable_error, Reason};
+classify_error({shutdown, _} = Reason) ->
+    {recoverable_error, Reason};
+classify_error(shutdown = Reason) ->
+    {recoverable_error, Reason};
+classify_error(Reason) ->
+    {unrecoverable_error, Reason}.
+
+on_get_status(_ResourceId, State) ->
+    Pools = maps:to_list(maps:with([ingress_pool_name, egress_pool_name], State)),
+    Workers = [{Pool, Worker} || {Pool, PN} <- Pools, {_Name, Worker} <- ecpool:workers(PN)],
+    try emqx_utils:pmap(fun get_status/1, Workers, ?HEALTH_CHECK_TIMEOUT) of
+        Statuses ->
+            combine_status(Statuses)
+    catch
+        exit:timeout ->
+            connecting
+    end.
+
+get_status({Pool, Worker}) ->
+    case ecpool_worker:client(Worker) of
+        {ok, Client} when Pool == ingress_pool_name ->
+            emqx_bridge_mqtt_ingress:status(Client);
+        {ok, Client} when Pool == egress_pool_name ->
+            emqx_bridge_mqtt_egress:status(Client);
+        {error, _} ->
+            disconnected
+    end.
+
+combine_status(Statuses) ->
+    %% NOTE
+    %% Natural order of statuses: [connected, connecting, disconnected]
+    %% * `disconnected` wins over any other status
+    %% * `connecting` wins over `connected`
+    case lists:reverse(lists:usort(Statuses)) of
+        [Status | _] ->
+            Status;
+        [] ->
+            disconnected
+    end.
+
+mk_ingress_config(
+    ResourceId,
+    #{
+        ingress := Ingress = #{remote := _},
+        server := Server,
+        hookpoint := HookPoint
+    }
+) ->
+    Ingress#{
+        server => Server,
+        on_message_received => {?MODULE, on_message_received, [HookPoint, ResourceId]}
+    };
+mk_ingress_config(ResourceId, #{ingress := #{remote := _}} = Conf) ->
+    error({no_hookpoint_provided, ResourceId, Conf});
+mk_ingress_config(_ResourceId, #{}) ->
+    undefined.
+
+mk_egress_config(#{egress := Egress = #{remote := _}}) ->
+    Egress;
+mk_egress_config(#{}) ->
+    undefined.
+
+mk_client_opts(
+    ResourceId,
+    ClientScope,
+    Config = #{
+        server := Server,
+        keepalive := KeepAlive,
+        ssl := #{enable := EnableSsl} = Ssl
+    }
+) ->
+    HostPort = emqx_bridge_mqtt_connector_schema:parse_server(Server),
+    Options = maps:with(
+        [
+            proto_ver,
+            username,
+            password,
+            clean_start,
+            retry_interval,
+            max_inflight,
+            % Opening a connection in bridge mode will form a non-standard mqtt connection message.
+            % A load balancing server (such as haproxy) is often set up before the emqx broker server.
+            % When the load balancing server enables mqtt connection packet inspection,
+            % non-standard mqtt connection packets might be filtered out by LB.
+            bridge_mode
+        ],
+        Config
+    ),
+    Options#{
+        hosts => [HostPort],
+        clientid => clientid(ResourceId, ClientScope, Config),
+        connect_timeout => 30,
+        keepalive => ms_to_s(KeepAlive),
+        force_ping => true,
+        ssl => EnableSsl,
+        ssl_opts => maps:to_list(maps:remove(enable, Ssl))
+    }.
+
+ms_to_s(Ms) ->
+    erlang:ceil(Ms / 1000).
+
+clientid(Id, ClientScope, _Conf = #{clientid_prefix := Prefix}) when is_binary(Prefix) ->
+    iolist_to_binary([Prefix, ":", Id, ":", ClientScope, ":", atom_to_list(node())]);
+clientid(Id, ClientScope, _Conf) ->
+    iolist_to_binary([Id, ":", ClientScope, ":", atom_to_list(node())]).

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl

@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--module(emqx_connector_mqtt_schema).
+-module(emqx_bridge_mqtt_connector_schema).
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
@@ -68,7 +68,8 @@ fields("server_configs") ->
                 hoconsc:enum([cluster_shareload]),
                 #{
                     default => cluster_shareload,
-                    desc => ?DESC("mode")
+                    desc => ?DESC("mode"),
+                    deprecated => {since, "v5.1.0 & e5.1.0"}
                 }
             )},
         {server, emqx_schema:servers_sc(#{desc => ?DESC("server")}, ?MQTT_HOST_OPTS)},
@@ -133,16 +134,17 @@ fields("server_configs") ->
     ] ++ emqx_connector_schema_lib:ssl_fields();
 fields("ingress") ->
     [
-        {"remote",
+        {pool_size, fun ingress_pool_size/1},
+        {remote,
             mk(
                 ref(?MODULE, "ingress_remote"),
-                #{desc => ?DESC(emqx_connector_mqtt_schema, "ingress_remote")}
+                #{desc => ?DESC("ingress_remote")}
             )},
-        {"local",
+        {local,
             mk(
                 ref(?MODULE, "ingress_local"),
                 #{
-                    desc => ?DESC(emqx_connector_mqtt_schema, "ingress_local")
+                    desc => ?DESC("ingress_local")
                 }
             )}
     ];
@@ -204,19 +206,20 @@ fields("ingress_local") ->
     ];
 fields("egress") ->
     [
-        {"local",
+        {pool_size, fun egress_pool_size/1},
+        {local,
             mk(
                 ref(?MODULE, "egress_local"),
                 #{
-                    desc => ?DESC(emqx_connector_mqtt_schema, "egress_local"),
+                    desc => ?DESC("egress_local"),
                     required => false
                 }
             )},
-        {"remote",
+        {remote,
             mk(
                 ref(?MODULE, "egress_remote"),
                 #{
-                    desc => ?DESC(emqx_connector_mqtt_schema, "egress_remote"),
+                    desc => ?DESC("egress_remote"),
                     required => true
                 }
             )}
@@ -272,6 +275,16 @@ fields("egress_remote") ->
             )}
     ].
 
+ingress_pool_size(desc) ->
+    ?DESC("ingress_pool_size");
+ingress_pool_size(Prop) ->
+    emqx_connector_schema_lib:pool_size(Prop).
+
+egress_pool_size(desc) ->
+    ?DESC("egress_pool_size");
+egress_pool_size(Prop) ->
+    emqx_connector_schema_lib:pool_size(Prop).
+
 desc("server_configs") ->
     ?DESC("server_configs");
 desc("ingress") ->

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl

@@ -0,0 +1,162 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_mqtt_egress).
+
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
+
+-behaviour(ecpool_worker).
+
+%% ecpool
+-export([connect/1]).
+
+-export([
+    config/1,
+    send/3,
+    send_async/4
+]).
+
+%% management APIs
+-export([
+    status/1,
+    info/1
+]).
+
+-type name() :: term().
+-type message() :: emqx_types:message() | map().
+-type callback() :: {function(), [_Arg]} | {module(), atom(), [_Arg]}.
+-type remote_message() :: #mqtt_msg{}.
+
+-type option() ::
+    {name, name()}
+    %% see `emqtt:option()`
+    | {client_opts, map()}.
+
+-type egress() :: #{
+    local => #{
+        topic => emqx_topic:topic()
+    },
+    remote := emqx_bridge_mqtt_msg:msgvars()
+}.
+
+%% @doc Start an ingress bridge worker.
+-spec connect([option() | {ecpool_worker_id, pos_integer()}]) ->
+    {ok, pid()} | {error, _Reason}.
+connect(Options) ->
+    ?SLOG(debug, #{
+        msg => "egress_client_starting",
+        options => emqx_utils:redact(Options)
+    }),
+    Name = proplists:get_value(name, Options),
+    WorkerId = proplists:get_value(ecpool_worker_id, Options),
+    ClientOpts = proplists:get_value(client_opts, Options),
+    case emqtt:start_link(mk_client_opts(WorkerId, ClientOpts)) of
+        {ok, Pid} ->
+            connect(Pid, Name);
+        {error, Reason} = Error ->
+            ?SLOG(error, #{
+                msg => "egress_client_start_failed",
+                config => emqx_utils:redact(ClientOpts),
+                reason => Reason
+            }),
+            Error
+    end.
+
+mk_client_opts(WorkerId, ClientOpts = #{clientid := ClientId}) ->
+    ClientOpts#{clientid := mk_clientid(WorkerId, ClientId)}.
+
+mk_clientid(WorkerId, ClientId) ->
+    iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]).
+
+connect(Pid, Name) ->
+    case emqtt:connect(Pid) of
+        {ok, _Props} ->
+            {ok, Pid};
+        {error, Reason} = Error ->
+            ?SLOG(warning, #{
+                msg => "egress_client_connect_failed",
+                reason => Reason,
+                name => Name
+            }),
+            _ = catch emqtt:stop(Pid),
+            Error
+    end.
+
+%%
+
+-spec config(map()) ->
+    egress().
+config(#{remote := RC = #{}} = Conf) ->
+    Conf#{remote => emqx_bridge_mqtt_msg:parse(RC)}.
+
+-spec send(pid(), message(), egress()) ->
+    ok.
+send(Pid, MsgIn, Egress) ->
+    emqtt:publish(Pid, export_msg(MsgIn, Egress)).
+
+-spec send_async(pid(), message(), callback(), egress()) ->
+    ok | {ok, pid()}.
+send_async(Pid, MsgIn, Callback, Egress) ->
+    ok = emqtt:publish_async(Pid, export_msg(MsgIn, Egress), _Timeout = infinity, Callback),
+    {ok, Pid}.
+
+export_msg(Msg, #{remote := Remote}) ->
+    to_remote_msg(Msg, Remote).
+
+-spec to_remote_msg(message(), emqx_bridge_mqtt_msg:msgvars()) ->
+    remote_message().
+to_remote_msg(#message{flags = Flags} = Msg, Vars) ->
+    {EventMsg, _} = emqx_rule_events:eventmsg_publish(Msg),
+    to_remote_msg(EventMsg#{retain => maps:get(retain, Flags, false)}, Vars);
+to_remote_msg(Msg = #{}, Remote) ->
+    #{
+        topic := Topic,
+        payload := Payload,
+        qos := QoS,
+        retain := Retain
+    } = emqx_bridge_mqtt_msg:render(Msg, Remote),
+    PubProps = maps:get(pub_props, Msg, #{}),
+    #mqtt_msg{
+        qos = QoS,
+        retain = Retain,
+        topic = Topic,
+        props = emqx_utils:pub_props_to_packet(PubProps),
+        payload = Payload
+    }.
+
+%%
+
+-spec info(pid()) ->
+    [{atom(), term()}].
+info(Pid) ->
+    emqtt:info(Pid).
+
+-spec status(pid()) ->
+    emqx_resource:resource_status().
+status(Pid) ->
+    try
+        case proplists:get_value(socket, info(Pid)) of
+            Socket when Socket /= undefined ->
+                connected;
+            undefined ->
+                connecting
+        end
+    catch
+        exit:{noproc, _} ->
+            disconnected
+    end.

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl

@@ -0,0 +1,274 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_mqtt_ingress).
+
+-include_lib("emqx/include/logger.hrl").
+
+-behaviour(ecpool_worker).
+
+%% ecpool
+-export([connect/1]).
+
+%% management APIs
+-export([
+    status/1,
+    info/1
+]).
+
+-export([handle_publish/5]).
+-export([handle_disconnect/1]).
+
+-type name() :: term().
+
+-type option() ::
+    {name, name()}
+    | {ingress, map()}
+    %% see `emqtt:option()`
+    | {client_opts, map()}.
+
+-type ingress() :: #{
+    server := string(),
+    remote := #{
+        topic := emqx_topic:topic(),
+        qos => emqx_types:qos()
+    },
+    local := emqx_bridge_mqtt_msg:msgvars(),
+    on_message_received := {module(), atom(), [term()]}
+}.
+
+%% @doc Start an ingress bridge worker.
+-spec connect([option() | {ecpool_worker_id, pos_integer()}]) ->
+    {ok, pid()} | {error, _Reason}.
+connect(Options) ->
+    ?SLOG(debug, #{
+        msg => "ingress_client_starting",
+        options => emqx_utils:redact(Options)
+    }),
+    Name = proplists:get_value(name, Options),
+    WorkerId = proplists:get_value(ecpool_worker_id, Options),
+    Ingress = config(proplists:get_value(ingress, Options), Name),
+    ClientOpts = proplists:get_value(client_opts, Options),
+    case emqtt:start_link(mk_client_opts(Name, WorkerId, Ingress, ClientOpts)) of
+        {ok, Pid} ->
+            connect(Pid, Name, Ingress);
+        {error, Reason} = Error ->
+            ?SLOG(error, #{
+                msg => "client_start_failed",
+                config => emqx_utils:redact(ClientOpts),
+                reason => Reason
+            }),
+            Error
+    end.
+
+mk_client_opts(Name, WorkerId, Ingress, ClientOpts = #{clientid := ClientId}) ->
+    ClientOpts#{
+        clientid := mk_clientid(WorkerId, ClientId),
+        msg_handler => mk_client_event_handler(Name, Ingress)
+    }.
+
+mk_clientid(WorkerId, ClientId) ->
+    iolist_to_binary([ClientId, $: | integer_to_list(WorkerId)]).
+
+mk_client_event_handler(Name, Ingress = #{}) ->
+    IngressVars = maps:with([server], Ingress),
+    OnMessage = maps:get(on_message_received, Ingress, undefined),
+    LocalPublish =
+        case Ingress of
+            #{local := Local = #{topic := _}} ->
+                Local;
+            #{} ->
+                undefined
+        end,
+    #{
+        publish => {fun ?MODULE:handle_publish/5, [Name, OnMessage, LocalPublish, IngressVars]},
+        disconnected => {fun ?MODULE:handle_disconnect/1, []}
+    }.
+
+-spec connect(pid(), name(), ingress()) ->
+    {ok, pid()} | {error, _Reason}.
+connect(Pid, Name, Ingress) ->
+    case emqtt:connect(Pid) of
+        {ok, _Props} ->
+            case subscribe_remote_topic(Pid, Ingress) of
+                {ok, _, _RCs} ->
+                    {ok, Pid};
+                {error, Reason} = Error ->
+                    ?SLOG(error, #{
+                        msg => "ingress_client_subscribe_failed",
+                        ingress => Ingress,
+                        name => Name,
+                        reason => Reason
+                    }),
+                    _ = catch emqtt:stop(Pid),
+                    Error
+            end;
+        {error, Reason} = Error ->
+            ?SLOG(warning, #{
+                msg => "ingress_client_connect_failed",
+                reason => Reason,
+                name => Name
+            }),
+            _ = catch emqtt:stop(Pid),
+            Error
+    end.
+
+subscribe_remote_topic(Pid, #{remote := #{topic := RemoteTopic, qos := QoS}}) ->
+    emqtt:subscribe(Pid, RemoteTopic, QoS).
+
+%%
+
+-spec config(map(), name()) ->
+    ingress().
+config(#{remote := RC, local := LC} = Conf, BridgeName) ->
+    Conf#{
+        remote => parse_remote(RC, BridgeName),
+        local => emqx_bridge_mqtt_msg:parse(LC)
+    }.
+
+parse_remote(#{qos := QoSIn} = Conf, BridgeName) ->
+    QoS = downgrade_ingress_qos(QoSIn),
+    case QoS of
+        QoSIn ->
+            ok;
+        _ ->
+            ?SLOG(warning, #{
+                msg => "downgraded_unsupported_ingress_qos",
+                qos_configured => QoSIn,
+                qos_used => QoS,
+                name => BridgeName
+            })
+    end,
+    Conf#{qos => QoS}.
+
+downgrade_ingress_qos(2) ->
+    1;
+downgrade_ingress_qos(QoS) ->
+    QoS.
+
+%%
+
+-spec info(pid()) ->
+    [{atom(), term()}].
+info(Pid) ->
+    emqtt:info(Pid).
+
+-spec status(pid()) ->
+    emqx_resource:resource_status().
+status(Pid) ->
+    try
+        case proplists:get_value(socket, info(Pid)) of
+            Socket when Socket /= undefined ->
+                connected;
+            undefined ->
+                connecting
+        end
+    catch
+        exit:{noproc, _} ->
+            disconnected
+    end.
+
+%%
+
+handle_publish(#{properties := Props} = MsgIn, Name, OnMessage, LocalPublish, IngressVars) ->
+    Msg = import_msg(MsgIn, IngressVars),
+    ?SLOG(debug, #{
+        msg => "ingress_publish_local",
+        message => Msg,
+        name => Name
+    }),
+    maybe_on_message_received(Msg, OnMessage),
+    maybe_publish_local(Msg, LocalPublish, Props).
+
+handle_disconnect(_Reason) ->
+    ok.
+
+maybe_on_message_received(Msg, {Mod, Func, Args}) ->
+    erlang:apply(Mod, Func, [Msg | Args]);
+maybe_on_message_received(_Msg, undefined) ->
+    ok.
+
+maybe_publish_local(Msg, Local = #{}, Props) ->
+    emqx_broker:publish(to_broker_msg(Msg, Local, Props));
+maybe_publish_local(_Msg, undefined, _Props) ->
+    ok.
+
+%%
+
+import_msg(
+    #{
+        dup := Dup,
+        payload := Payload,
+        properties := Props,
+        qos := QoS,
+        retain := Retain,
+        topic := Topic
+    },
+    #{server := Server}
+) ->
+    #{
+        id => emqx_guid:to_hexstr(emqx_guid:gen()),
+        server => Server,
+        payload => Payload,
+        topic => Topic,
+        qos => QoS,
+        dup => Dup,
+        retain => Retain,
+        pub_props => printable_maps(Props),
+        message_received_at => erlang:system_time(millisecond)
+    }.
+
+printable_maps(undefined) ->
+    #{};
+printable_maps(Headers) ->
+    maps:fold(
+        fun
+            ('User-Property', V0, AccIn) when is_list(V0) ->
+                AccIn#{
+                    'User-Property' => maps:from_list(V0),
+                    'User-Property-Pairs' => [
+                        #{
+                            key => Key,
+                            value => Value
+                        }
+                     || {Key, Value} <- V0
+                    ]
+                };
+            (K, V0, AccIn) ->
+                AccIn#{K => V0}
+        end,
+        #{},
+        Headers
+    ).
+
+%% published from remote node over a MQTT connection
+to_broker_msg(Msg, Vars, undefined) ->
+    to_broker_msg(Msg, Vars, #{});
+to_broker_msg(#{dup := Dup} = Msg, Local, Props) ->
+    #{
+        topic := Topic,
+        payload := Payload,
+        qos := QoS,
+        retain := Retain
+    } = emqx_bridge_mqtt_msg:render(Msg, Local),
+    PubProps = maps:get(pub_props, Msg, #{}),
+    emqx_message:set_headers(
+        Props#{properties => emqx_utils:pub_props_to_packet(PubProps)},
+        emqx_message:set_flags(
+            #{dup => Dup, retain => Retain},
+            emqx_message:make(bridge, QoS, Topic, Payload)
+        )
+    ).

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_msg.erl

@@ -0,0 +1,95 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_mqtt_msg).
+
+-export([parse/1]).
+-export([render/2]).
+
+-export_type([msgvars/0]).
+
+-type template() :: emqx_plugin_libs_rule:tmpl_token().
+
+-type msgvars() :: #{
+    topic => template(),
+    qos => template() | emqx_types:qos(),
+    retain => template() | boolean(),
+    payload => template() | undefined
+}.
+
+%%
+
+-spec parse(#{
+    topic => iodata(),
+    qos => iodata() | emqx_types:qos(),
+    retain => iodata() | boolean(),
+    payload => iodata()
+}) ->
+    msgvars().
+parse(Conf) ->
+    Acc1 = parse_field(topic, Conf, Conf),
+    Acc2 = parse_field(qos, Conf, Acc1),
+    Acc3 = parse_field(payload, Conf, Acc2),
+    parse_field(retain, Conf, Acc3).
+
+parse_field(Key, Conf, Acc) ->
+    case Conf of
+        #{Key := Val} when is_binary(Val) ->
+            Acc#{Key => emqx_plugin_libs_rule:preproc_tmpl(Val)};
+        #{Key := Val} ->
+            Acc#{Key => Val};
+        #{} ->
+            Acc
+    end.
+
+render(
+    Msg,
+    #{
+        topic := TopicToken,
+        qos := QoSToken,
+        retain := RetainToken
+    } = Vars
+) ->
+    #{
+        topic => render_string(TopicToken, Msg),
+        payload => render_payload(Vars, Msg),
+        qos => render_simple_var(QoSToken, Msg),
+        retain => render_simple_var(RetainToken, Msg)
+    }.
+
+render_payload(From, MapMsg) ->
+    do_render_payload(maps:get(payload, From, undefined), MapMsg).
+
+do_render_payload(undefined, Msg) ->
+    emqx_utils_json:encode(Msg);
+do_render_payload(Tks, Msg) ->
+    render_string(Tks, Msg).
+
+%% Replace a string contains vars to another string in which the placeholders are replace by the
+%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be:
+%% "a: 1".
+render_string(Tokens, Data) when is_list(Tokens) ->
+    emqx_placeholder:proc_tmpl(Tokens, Data, #{return => full_binary});
+render_string(Val, _Data) ->
+    Val.
+
+%% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result
+%% value will be an integer 1.
+render_simple_var(Tokens, Data) when is_list(Tokens) ->
+    [Var] = emqx_placeholder:proc_tmpl(Tokens, Data, #{return => rawlist}),
+    Var;
+render_simple_var(Val, _Data) ->
+    Val.

apps/emqx_bridge/src/schema/emqx_bridge_mqtt_schema.erl

@@ -42,7 +42,7 @@ fields("config") ->
                     }
                 )}
         ] ++
-        emqx_connector_mqtt_schema:fields("config");
+        emqx_bridge_mqtt_connector_schema:fields("config");
 fields("creation_opts") ->
     Opts = emqx_resource_schema:fields("creation_opts"),
     [O || {Field, _} = O <- Opts, not is_hidden_opts(Field)];

apps/emqx_bridge/test/emqx_bridge_mqtt_SUITE.erl

@@ -22,9 +22,7 @@
 
 -include("emqx/include/emqx.hrl").
 -include_lib("eunit/include/eunit.hrl").
--include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
--include("emqx_dashboard/include/emqx_dashboard.hrl").
 
 %% output functions
 -export([inspect/3]).
@@ -132,13 +130,11 @@ suite() ->
 
 init_per_suite(Config) ->
     _ = application:load(emqx_conf),
-    %% some testcases (may from other app) already get emqx_connector started
-    _ = application:stop(emqx_resource),
-    _ = application:stop(emqx_connector),
     ok = emqx_common_test_helpers:start_apps(
         [
             emqx_rule_engine,
             emqx_bridge,
+            emqx_bridge_mqtt,
             emqx_dashboard
         ],
         fun set_special_configs/1
@@ -152,9 +148,10 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     emqx_common_test_helpers:stop_apps([
-        emqx_rule_engine,
+        emqx_dashboard,
+        emqx_bridge_mqtt,
         emqx_bridge,
-        emqx_dashboard
+        emqx_rule_engine
     ]),
     ok.
 
@@ -221,6 +218,12 @@ t_mqtt_conn_bridge_ingress(_) ->
         request(put, uri(["bridges", BridgeIDIngress]), ServerConf)
     ),
 
+    %% non-shared subscription, verify that only one client is subscribed
+    ?assertEqual(
+        1,
+        length(emqx:subscribers(<<?INGRESS_REMOTE_TOPIC, "/#">>))
+    ),
+
     %% we now test if the bridge works as expected
     RemoteTopic = <<?INGRESS_REMOTE_TOPIC, "/1">>,
     LocalTopic = <<?INGRESS_LOCAL_TOPIC, "/", RemoteTopic/binary>>,
@@ -245,6 +248,48 @@ t_mqtt_conn_bridge_ingress(_) ->
 
     ok.
 
+t_mqtt_conn_bridge_ingress_shared_subscription(_) ->
+    PoolSize = 4,
+    Ns = lists:seq(1, 10),
+    BridgeName = atom_to_binary(?FUNCTION_NAME),
+    BridgeID = create_bridge(
+        ?SERVER_CONF(<<>>)#{
+            <<"type">> => ?TYPE_MQTT,
+            <<"name">> => BridgeName,
+            <<"ingress">> => #{
+                <<"pool_size">> => PoolSize,
+                <<"remote">> => #{
+                    <<"topic">> => <<"$share/ingress/", ?INGRESS_REMOTE_TOPIC, "/#">>,
+                    <<"qos">> => 1
+                },
+                <<"local">> => #{
+                    <<"topic">> => <<?INGRESS_LOCAL_TOPIC, "/${topic}">>,
+                    <<"qos">> => <<"${qos}">>,
+                    <<"payload">> => <<"${clientid}">>,
+                    <<"retain">> => <<"${retain}">>
+                }
+            }
+        }
+    ),
+
+    RemoteTopic = <<?INGRESS_REMOTE_TOPIC, "/1">>,
+    LocalTopic = <<?INGRESS_LOCAL_TOPIC, "/", RemoteTopic/binary>>,
+    ok = emqx:subscribe(LocalTopic),
+
+    _ = emqx_utils:pmap(
+        fun emqx:publish/1,
+        [emqx_message:make(RemoteTopic, <<>>) || _ <- Ns]
+    ),
+    _ = [assert_mqtt_msg_received(LocalTopic) || _ <- Ns],
+
+    ?assertEqual(
+        PoolSize,
+        length(emqx_shared_sub:subscribers(<<"ingress">>, <<?INGRESS_REMOTE_TOPIC, "/#">>))
+    ),
+
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
+    ok.
+
 t_mqtt_egress_bridge_ignores_clean_start(_) ->
     BridgeName = atom_to_binary(?FUNCTION_NAME),
     BridgeID = create_bridge(
@@ -256,11 +301,17 @@ t_mqtt_egress_bridge_ignores_clean_start(_) ->
         }
     ),
 
-    {ok, _, #{state := #{name := WorkerName}}} =
-        emqx_resource:get_instance(emqx_bridge_resource:resource_id(BridgeID)),
+    ResourceID = emqx_bridge_resource:resource_id(BridgeID),
+    {ok, _Group, #{state := #{egress_pool_name := EgressPoolName}}} =
+        emqx_resource_manager:lookup_cached(ResourceID),
+    ClientInfo = ecpool:pick_and_do(
+        EgressPoolName,
+        {emqx_bridge_mqtt_egress, info, []},
+        no_handover
+    ),
     ?assertMatch(
         #{clean_start := true},
-        maps:from_list(emqx_connector_mqtt_worker:info(WorkerName))
+        maps:from_list(ClientInfo)
     ),
 
     %% delete the bridge

apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_pulsar, [
     {description, "EMQX Pulsar Bridge"},
-    {vsn, "0.1.3"},
+    {vsn, "0.1.4"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar.erl

@@ -18,6 +18,8 @@
 %% emqx_ee_bridge "unofficial" API
 -export([conn_bridge_examples/1]).
 
+-export([producer_strategy_key_validator/1]).
+
 %%-------------------------------------------------------------------------------------------------
 %% `hocon_schema' API
 %%-------------------------------------------------------------------------------------------------
@@ -218,6 +220,14 @@ conn_bridge_examples(_Method) ->
         }
     ].
 
+producer_strategy_key_validator(#{
+    <<"strategy">> := key_dispatch,
+    <<"message">> := #{<<"key">> := ""}
+}) ->
+    {error, "Message key cannot be empty when `key_dispatch` strategy is used"};
+producer_strategy_key_validator(_) ->
+    ok.
+
 %%-------------------------------------------------------------------------------------------------
 %% Internal fns
 %%-------------------------------------------------------------------------------------------------

apps/emqx_bridge_pulsar/src/emqx_bridge_pulsar_impl_producer.erl

@@ -60,6 +60,10 @@
     sync_timeout := emqx_schema:duration_ms()
 }.
 
+%% Allocatable resources
+-define(pulsar_client_id, pulsar_client_id).
+-define(pulsar_producers, pulsar_producers).
+
 %%-------------------------------------------------------------------------------------
 %% `emqx_resource' API
 %%-------------------------------------------------------------------------------------
@@ -81,7 +85,7 @@ on_start(InstanceId, Config) ->
     } = Config,
     Servers = format_servers(Servers0),
     ClientId = make_client_id(InstanceId, BridgeName),
-    ok = emqx_resource:allocate_resource(InstanceId, pulsar_client_id, ClientId),
+    ok = emqx_resource:allocate_resource(InstanceId, ?pulsar_client_id, ClientId),
     SSLOpts = emqx_tls_lib:to_client_opts(SSL),
     ConnectTimeout = maps:get(connect_timeout, Config, timer:seconds(5)),
     ClientOpts = #{
@@ -119,7 +123,7 @@ on_start(InstanceId, Config) ->
 -spec on_stop(resource_id(), state()) -> ok.
 on_stop(InstanceId, _State) ->
     case emqx_resource:get_allocated_resources(InstanceId) of
-        #{pulsar_client_id := ClientId, pulsar_producers := Producers} ->
+        #{?pulsar_client_id := ClientId, ?pulsar_producers := Producers} ->
             stop_producers(ClientId, Producers),
             stop_client(ClientId),
             ?tp(pulsar_bridge_stopped, #{
@@ -128,7 +132,7 @@ on_stop(InstanceId, _State) ->
                 pulsar_producers => Producers
             }),
             ok;
-        #{pulsar_client_id := ClientId} ->
+        #{?pulsar_client_id := ClientId} ->
             stop_client(ClientId),
             ?tp(pulsar_bridge_stopped, #{
                 instance_id => InstanceId,
@@ -340,7 +344,7 @@ start_producer(Config, InstanceId, ClientId, ClientOpts) ->
     ?tp(pulsar_producer_about_to_start_producers, #{producer_name => ProducerName}),
     try pulsar:ensure_supervised_producers(ClientId, PulsarTopic, ProducerOpts) of
         {ok, Producers} ->
-            ok = emqx_resource:allocate_resource(InstanceId, pulsar_producers, Producers),
+            ok = emqx_resource:allocate_resource(InstanceId, ?pulsar_producers, Producers),
             ?tp(pulsar_producer_producers_allocated, #{}),
             State = #{
                 pulsar_client_id => ClientId,

apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_impl_producer_SUITE.erl

@@ -285,6 +285,11 @@ create_bridge(Config, Overrides) ->
     PulsarConfig = emqx_utils_maps:deep_merge(PulsarConfig0, Overrides),
     emqx_bridge:create(Type, Name, PulsarConfig).
 
+delete_bridge(Config) ->
+    Type = ?BRIDGE_TYPE_BIN,
+    Name = ?config(pulsar_name, Config),
+    emqx_bridge:remove(Type, Name).
+
 create_bridge_api(Config) ->
     create_bridge_api(Config, _Overrides = #{}).
 
@@ -541,8 +546,14 @@ kill_resource_managers() ->
     lists:foreach(
         fun({_, Pid, _, _}) ->
             ct:pal("terminating resource manager ~p", [Pid]),
-            %% sys:terminate(Pid, stop),
+            Ref = monitor(process, Pid),
             exit(Pid, kill),
+            receive
+                {'DOWN', Ref, process, Pid, killed} ->
+                    ok
+            after 500 ->
+                ct:fail("pid ~p didn't die!", [Pid])
+            end,
             ok
         end,
         supervisor:which_children(emqx_resource_manager_sup)
@@ -1002,6 +1013,8 @@ t_resource_manager_crash_after_producers_started(Config) ->
                         Producers =/= undefined,
                     10_000
                 ),
+            ?assertMatch({ok, _}, delete_bridge(Config)),
+            ?assertEqual([], get_pulsar_producers()),
             ok
         end,
         []
@@ -1033,6 +1046,8 @@ t_resource_manager_crash_before_producers_started(Config) ->
                     #{?snk_kind := pulsar_bridge_stopped, pulsar_producers := undefined},
                     10_000
                 ),
+            ?assertMatch({ok, _}, delete_bridge(Config)),
+            ?assertEqual([], get_pulsar_producers()),
             ok
         end,
         []

apps/emqx_bridge_pulsar/test/emqx_bridge_pulsar_tests.erl

@@ -0,0 +1,75 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_pulsar_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+%%===========================================================================
+%% Test cases
+%%===========================================================================
+
+pulsar_producer_validations_test() ->
+    Conf0 = pulsar_producer_hocon(),
+    Conf1 =
+        Conf0 ++
+            "\n"
+            "bridges.pulsar_producer.my_producer.strategy = key_dispatch"
+            "\n"
+            "bridges.pulsar_producer.my_producer.message.key = \"\"",
+    Conf = parse(Conf1),
+    ?assertMatch(
+        #{
+            <<"strategy">> := <<"key_dispatch">>,
+            <<"message">> := #{<<"key">> := <<>>}
+        },
+        emqx_utils_maps:deep_get([<<"bridges">>, <<"pulsar_producer">>, <<"my_producer">>], Conf)
+    ),
+    ?assertThrow(
+        {_, [
+            #{
+                path := "bridges.pulsar_producer.my_producer",
+                reason := "Message key cannot be empty when `key_dispatch` strategy is used"
+            }
+        ]},
+        check(Conf)
+    ),
+
+    ok.
+
+%%===========================================================================
+%% Helper functions
+%%===========================================================================
+
+parse(Hocon) ->
+    {ok, Conf} = hocon:binary(Hocon),
+    Conf.
+
+check(Conf) when is_map(Conf) ->
+    hocon_tconf:check_plain(emqx_bridge_schema, Conf).
+
+%%===========================================================================
+%% Data section
+%%===========================================================================
+
+%% erlfmt-ignore
+pulsar_producer_hocon() ->
+"""
+bridges.pulsar_producer.my_producer {
+  enable = true
+  servers = \"localhost:6650\"
+  pulsar_topic = pulsar_topic
+  strategy = random
+  message {
+    key = \"${.clientid}\"
+    value = \"${.}\"
+  }
+  authentication = none
+  ssl {
+    enable = false
+    verify = verify_none
+    server_name_indication = \"auto\"
+  }
+}
+""".

apps/emqx_conf/test/emqx_conf_schema_tests.erl

@@ -414,17 +414,7 @@ doc_gen_test() ->
         60,
         fun() ->
             Dir = "tmp",
-            ok = filelib:ensure_dir(filename:join("tmp", foo)),
-            I18nFile = filename:join([
-                "_build",
-                "test",
-                "lib",
-                "emqx_dashboard",
-                "priv",
-                "i18n.conf"
-            ]),
-            _ = emqx_conf:dump_schema(Dir, emqx_conf_schema, I18nFile),
-            ok
+            ok = emqx_conf:dump_schema(Dir, emqx_conf_schema)
         end
     }.
 

apps/emqx_connector/src/emqx_connector.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_connector, [
     {description, "EMQX Data Integration Connectors"},
-    {vsn, "0.1.24"},
+    {vsn, "0.1.25"},
     {registered, []},
     {mod, {emqx_connector_app, []}},
     {applications, [

apps/emqx_connector/src/emqx_connector_mqtt.erl

@@ -1,319 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
--module(emqx_connector_mqtt).
-
--include("emqx_connector.hrl").
-
--include_lib("typerefl/include/types.hrl").
--include_lib("hocon/include/hoconsc.hrl").
--include_lib("emqx/include/logger.hrl").
-
--behaviour(supervisor).
--behaviour(emqx_resource).
-
-%% API and callbacks for supervisor
--export([
-    callback_mode/0,
-    start_link/0,
-    init/1,
-    create_bridge/1,
-    drop_bridge/1,
-    bridges/0
-]).
-
--export([on_message_received/3]).
-
-%% callbacks of behaviour emqx_resource
--export([
-    on_start/2,
-    on_stop/2,
-    on_query/3,
-    on_query_async/4,
-    on_get_status/2
-]).
-
--export([on_async_result/2]).
-
--behaviour(hocon_schema).
-
--import(hoconsc, [mk/2]).
-
--export([
-    roots/0,
-    fields/1
-]).
-
-%%=====================================================================
-%% Hocon schema
-roots() ->
-    fields("config").
-
-fields("config") ->
-    emqx_connector_mqtt_schema:fields("config");
-fields("get") ->
-    [
-        {num_of_bridges,
-            mk(
-                integer(),
-                #{desc => ?DESC("num_of_bridges")}
-            )}
-    ] ++ fields("post");
-fields("put") ->
-    emqx_connector_mqtt_schema:fields("server_configs");
-fields("post") ->
-    [
-        {type,
-            mk(
-                mqtt,
-                #{
-                    required => true,
-                    desc => ?DESC("type")
-                }
-            )},
-        {name,
-            mk(
-                binary(),
-                #{
-                    required => true,
-                    desc => ?DESC("name")
-                }
-            )}
-    ] ++ fields("put").
-
-%% ===================================================================
-%% supervisor APIs
-start_link() ->
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-
-init([]) ->
-    SupFlag = #{
-        strategy => one_for_one,
-        intensity => 100,
-        period => 10
-    },
-    {ok, {SupFlag, []}}.
-
-bridge_spec(Config) ->
-    {Name, NConfig} = maps:take(name, Config),
-    #{
-        id => Name,
-        start => {emqx_connector_mqtt_worker, start_link, [Name, NConfig]},
-        restart => temporary,
-        shutdown => 1000
-    }.
-
--spec bridges() -> [{_Name, _Status}].
-bridges() ->
-    [
-        {Name, emqx_connector_mqtt_worker:status(Name)}
-     || {Name, _Pid, _, _} <- supervisor:which_children(?MODULE)
-    ].
-
-create_bridge(Config) ->
-    supervisor:start_child(?MODULE, bridge_spec(Config)).
-
-drop_bridge(Name) ->
-    case supervisor:terminate_child(?MODULE, Name) of
-        ok ->
-            supervisor:delete_child(?MODULE, Name);
-        {error, not_found} ->
-            ok;
-        {error, Error} ->
-            {error, Error}
-    end.
-
-%% ===================================================================
-%% When use this bridge as a data source, ?MODULE:on_message_received will be called
-%% if the bridge received msgs from the remote broker.
-on_message_received(Msg, HookPoint, ResId) ->
-    emqx_resource_metrics:received_inc(ResId),
-    emqx:run_hook(HookPoint, [Msg]).
-
-%% ===================================================================
-callback_mode() -> async_if_possible.
-
-on_start(InstanceId, Conf) ->
-    ?SLOG(info, #{
-        msg => "starting_mqtt_connector",
-        connector => InstanceId,
-        config => emqx_utils:redact(Conf)
-    }),
-    BasicConf = basic_config(Conf),
-    BridgeConf = BasicConf#{
-        name => InstanceId,
-        clientid => clientid(InstanceId, Conf),
-        subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined), Conf, InstanceId),
-        forwards => make_forward_confs(maps:get(egress, Conf, undefined))
-    },
-    case ?MODULE:create_bridge(BridgeConf) of
-        {ok, _Pid} ->
-            ensure_mqtt_worker_started(InstanceId, BridgeConf);
-        {error, {already_started, _Pid}} ->
-            ok = ?MODULE:drop_bridge(InstanceId),
-            {ok, _} = ?MODULE:create_bridge(BridgeConf),
-            ensure_mqtt_worker_started(InstanceId, BridgeConf);
-        {error, Reason} ->
-            {error, Reason}
-    end.
-
-on_stop(_InstId, #{name := InstanceId}) ->
-    ?SLOG(info, #{
-        msg => "stopping_mqtt_connector",
-        connector => InstanceId
-    }),
-    case ?MODULE:drop_bridge(InstanceId) of
-        ok ->
-            ok;
-        {error, not_found} ->
-            ok;
-        {error, Reason} ->
-            ?SLOG(error, #{
-                msg => "stop_mqtt_connector_error",
-                connector => InstanceId,
-                reason => Reason
-            })
-    end.
-
-on_query(_InstId, {send_message, Msg}, #{name := InstanceId}) ->
-    ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
-    case emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg) of
-        ok ->
-            ok;
-        {error, Reason} ->
-            classify_error(Reason)
-    end.
-
-on_query_async(_InstId, {send_message, Msg}, CallbackIn, #{name := InstanceId}) ->
-    ?TRACE("QUERY", "async_send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
-    Callback = {fun on_async_result/2, [CallbackIn]},
-    case emqx_connector_mqtt_worker:send_to_remote_async(InstanceId, Msg, Callback) of
-        ok ->
-            ok;
-        {ok, Pid} ->
-            {ok, Pid};
-        {error, Reason} ->
-            classify_error(Reason)
-    end.
-
-on_async_result(Callback, ok) ->
-    apply_callback_function(Callback, ok);
-on_async_result(Callback, {ok, _} = Ok) ->
-    apply_callback_function(Callback, Ok);
-on_async_result(Callback, {error, Reason}) ->
-    apply_callback_function(Callback, classify_error(Reason)).
-
-apply_callback_function(F, Result) when is_function(F) ->
-    erlang:apply(F, [Result]);
-apply_callback_function({F, A}, Result) when is_function(F), is_list(A) ->
-    erlang:apply(F, A ++ [Result]);
-apply_callback_function({M, F, A}, Result) when is_atom(M), is_atom(F), is_list(A) ->
-    erlang:apply(M, F, A ++ [Result]).
-
-on_get_status(_InstId, #{name := InstanceId}) ->
-    emqx_connector_mqtt_worker:status(InstanceId).
-
-classify_error(disconnected = Reason) ->
-    {error, {recoverable_error, Reason}};
-classify_error({disconnected, _RC, _} = Reason) ->
-    {error, {recoverable_error, Reason}};
-classify_error({shutdown, _} = Reason) ->
-    {error, {recoverable_error, Reason}};
-classify_error(shutdown = Reason) ->
-    {error, {recoverable_error, Reason}};
-classify_error(Reason) ->
-    {error, {unrecoverable_error, Reason}}.
-
-ensure_mqtt_worker_started(InstanceId, BridgeConf) ->
-    case emqx_connector_mqtt_worker:connect(InstanceId) of
-        {ok, Properties} ->
-            {ok, #{name => InstanceId, config => BridgeConf, props => Properties}};
-        {error, Reason} ->
-            {error, Reason}
-    end.
-
-make_sub_confs(EmptyMap, _Conf, _) when map_size(EmptyMap) == 0 ->
-    undefined;
-make_sub_confs(undefined, _Conf, _) ->
-    undefined;
-make_sub_confs(SubRemoteConf, Conf, ResourceId) ->
-    case maps:find(hookpoint, Conf) of
-        error ->
-            error({no_hookpoint_provided, Conf});
-        {ok, HookPoint} ->
-            MFA = {?MODULE, on_message_received, [HookPoint, ResourceId]},
-            SubRemoteConf#{on_message_received => MFA}
-    end.
-
-make_forward_confs(EmptyMap) when map_size(EmptyMap) == 0 ->
-    undefined;
-make_forward_confs(undefined) ->
-    undefined;
-make_forward_confs(FrowardConf) ->
-    FrowardConf.
-
-basic_config(
-    #{
-        server := Server,
-        proto_ver := ProtoVer,
-        bridge_mode := BridgeMode,
-        clean_start := CleanStart,
-        keepalive := KeepAlive,
-        retry_interval := RetryIntv,
-        max_inflight := MaxInflight,
-        ssl := #{enable := EnableSsl} = Ssl
-    } = Conf
-) ->
-    BasicConf = #{
-        %% connection opts
-        server => Server,
-        %% 30s
-        connect_timeout => 30,
-        auto_reconnect => true,
-        proto_ver => ProtoVer,
-        %% Opening bridge_mode will form a non-standard mqtt connection message.
-        %% A load balancing server (such as haproxy) is often set up before the emqx broker server.
-        %% When the load balancing server enables mqtt connection packet inspection,
-        %% non-standard mqtt connection packets will be filtered out by LB.
-        %% So let's disable bridge_mode.
-        bridge_mode => BridgeMode,
-        keepalive => ms_to_s(KeepAlive),
-        clean_start => CleanStart,
-        retry_interval => RetryIntv,
-        max_inflight => MaxInflight,
-        ssl => EnableSsl,
-        ssl_opts => maps:to_list(maps:remove(enable, Ssl))
-    },
-    maybe_put_fields([username, password], Conf, BasicConf).
-
-maybe_put_fields(Fields, Conf, Acc0) ->
-    lists:foldl(
-        fun(Key, Acc) ->
-            case maps:find(Key, Conf) of
-                error -> Acc;
-                {ok, Val} -> Acc#{Key => Val}
-            end
-        end,
-        Acc0,
-        Fields
-    ).
-
-ms_to_s(Ms) ->
-    erlang:ceil(Ms / 1000).
-
-clientid(Id, _Conf = #{clientid_prefix := Prefix = <<_/binary>>}) ->
-    iolist_to_binary([Prefix, ":", Id, ":", atom_to_list(node())]);
-clientid(Id, _Conf) ->
-    iolist_to_binary([Id, ":", atom_to_list(node())]).

apps/emqx_connector/src/emqx_connector_sup.erl

@@ -33,7 +33,6 @@ init([]) ->
         period => 20
     },
     ChildSpecs = [
-        child_spec(emqx_connector_mqtt),
         child_spec(emqx_connector_jwt_sup)
     ],
     {ok, {SupFlags, ChildSpecs}}.

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl

@@ -1,168 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_connector_mqtt_msg).
-
--export([
-    to_binary/1,
-    from_binary/1,
-    make_pub_vars/2,
-    to_remote_msg/2,
-    to_broker_msg/3,
-    estimate_size/1
-]).
-
--export([
-    replace_vars_in_str/2,
-    replace_simple_var/2
-]).
-
--export_type([msg/0]).
-
--include_lib("emqx/include/emqx.hrl").
-
--include_lib("emqtt/include/emqtt.hrl").
-
--type msg() :: emqx_types:message().
--type exp_msg() :: emqx_types:message() | #mqtt_msg{}.
--type remote_config() :: #{
-    topic := binary(),
-    qos := original | integer(),
-    retain := original | boolean(),
-    payload := binary()
-}.
--type variables() :: #{
-    mountpoint := undefined | binary(),
-    remote := remote_config()
-}.
-
-make_pub_vars(_, undefined) ->
-    undefined;
-make_pub_vars(Mountpoint, Conf) when is_map(Conf) ->
-    Conf#{mountpoint => Mountpoint}.
-
-%% @doc Make export format:
-%% 1. Mount topic to a prefix
-%% 2. Fix QoS to 1
-%% @end
-%% Shame that we have to know the callback module here
-%% would be great if we can get rid of #mqtt_msg{} record
-%% and use #message{} in all places.
--spec to_remote_msg(msg() | map(), variables()) ->
-    exp_msg().
-to_remote_msg(#message{flags = Flags0} = Msg, Vars) ->
-    Retain0 = maps:get(retain, Flags0, false),
-    {Columns, _} = emqx_rule_events:eventmsg_publish(Msg),
-    MapMsg = maps:put(retain, Retain0, Columns),
-    to_remote_msg(MapMsg, Vars);
-to_remote_msg(MapMsg, #{
-    remote := #{
-        topic := TopicToken,
-        qos := QoSToken,
-        retain := RetainToken
-    } = Remote,
-    mountpoint := Mountpoint
-}) when is_map(MapMsg) ->
-    Topic = replace_vars_in_str(TopicToken, MapMsg),
-    Payload = process_payload(Remote, MapMsg),
-    QoS = replace_simple_var(QoSToken, MapMsg),
-    Retain = replace_simple_var(RetainToken, MapMsg),
-    PubProps = maps:get(pub_props, MapMsg, #{}),
-    #mqtt_msg{
-        qos = QoS,
-        retain = Retain,
-        topic = topic(Mountpoint, Topic),
-        props = emqx_utils:pub_props_to_packet(PubProps),
-        payload = Payload
-    };
-to_remote_msg(#message{topic = Topic} = Msg, #{mountpoint := Mountpoint}) ->
-    Msg#message{topic = topic(Mountpoint, Topic)}.
-
-%% published from remote node over a MQTT connection
-to_broker_msg(Msg, Vars, undefined) ->
-    to_broker_msg(Msg, Vars, #{});
-to_broker_msg(
-    #{dup := Dup} = MapMsg,
-    #{
-        local := #{
-            topic := TopicToken,
-            qos := QoSToken,
-            retain := RetainToken
-        } = Local,
-        mountpoint := Mountpoint
-    },
-    Props
-) ->
-    Topic = replace_vars_in_str(TopicToken, MapMsg),
-    Payload = process_payload(Local, MapMsg),
-    QoS = replace_simple_var(QoSToken, MapMsg),
-    Retain = replace_simple_var(RetainToken, MapMsg),
-    PubProps = maps:get(pub_props, MapMsg, #{}),
-    set_headers(
-        Props#{properties => emqx_utils:pub_props_to_packet(PubProps)},
-        emqx_message:set_flags(
-            #{dup => Dup, retain => Retain},
-            emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload)
-        )
-    ).
-
-process_payload(From, MapMsg) ->
-    do_process_payload(maps:get(payload, From, undefined), MapMsg).
-
-do_process_payload(undefined, Msg) ->
-    emqx_utils_json:encode(Msg);
-do_process_payload(Tks, Msg) ->
-    replace_vars_in_str(Tks, Msg).
-
-%% Replace a string contains vars to another string in which the placeholders are replace by the
-%% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be:
-%% "a: 1".
-replace_vars_in_str(Tokens, Data) when is_list(Tokens) ->
-    emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => full_binary});
-replace_vars_in_str(Val, _Data) ->
-    Val.
-
-%% Replace a simple var to its value. For example, given "${var}", if the var=1, then the result
-%% value will be an integer 1.
-replace_simple_var(Tokens, Data) when is_list(Tokens) ->
-    [Var] = emqx_plugin_libs_rule:proc_tmpl(Tokens, Data, #{return => rawlist}),
-    Var;
-replace_simple_var(Val, _Data) ->
-    Val.
-
-%% @doc Make `binary()' in order to make iodata to be persisted on disk.
--spec to_binary(msg()) -> binary().
-to_binary(Msg) -> term_to_binary(Msg).
-
-%% @doc Unmarshal binary into `msg()'.
--spec from_binary(binary()) -> msg().
-from_binary(Bin) -> binary_to_term(Bin).
-
-%% @doc Estimate the size of a message.
-%% Count only the topic length + payload size
-%% There is no topic and payload for event message. So count all `Msg` term
--spec estimate_size(msg()) -> integer().
-estimate_size(#message{topic = Topic, payload = Payload}) ->
-    size(Topic) + size(Payload);
-estimate_size(#{topic := Topic, payload := Payload}) ->
-    size(Topic) + size(Payload);
-estimate_size(Term) ->
-    erlang:external_size(Term).
-
-set_headers(Val, Msg) ->
-    emqx_message:set_headers(Val, Msg).
-topic(undefined, Topic) -> Topic;
-topic(Prefix, Topic) -> emqx_topic:prepend(Prefix, Topic).

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl

@@ -1,465 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
-%% @doc Bridge works in two layers (1) batching layer (2) transport layer
-%% The `bridge' batching layer collects local messages in batches and sends over
-%% to remote MQTT node/cluster via `connection' transport layer.
-%% In case `REMOTE' is also an EMQX node, `connection' is recommended to be
-%% the `gen_rpc' based implementation `emqx_bridge_rpc'. Otherwise `connection'
-%% has to be `emqx_connector_mqtt_mod'.
-%%
-%% ```
-%% +------+                        +--------+
-%% | EMQX |                        | REMOTE |
-%% |      |                        |        |
-%% |   (bridge) <==(connection)==> |        |
-%% |      |                        |        |
-%% |      |                        |        |
-%% +------+                        +--------+
-%% '''
-%%
-%%
-%% This module implements 2 kinds of APIs with regards to batching and
-%% messaging protocol. (1) A `gen_statem' based local batch collector;
-%% (2) APIs for incoming remote batches/messages.
-%%
-%% Batch collector state diagram
-%%
-%% [idle] --(0) --> [connecting] --(2)--> [connected]
-%%                  |        ^                 |
-%%                  |        |                 |
-%%                  '--(1)---'--------(3)------'
-%%
-%% (0): auto or manual start
-%% (1): retry timeout
-%% (2): successfully connected to remote node/cluster
-%% (3): received {disconnected, Reason} OR
-%%      failed to send to remote node/cluster.
-%%
-%% NOTE: A bridge worker may subscribe to multiple (including wildcard)
-%% local topics, and the underlying `emqx_bridge_connect' may subscribe to
-%% multiple remote topics, however, worker/connections are not designed
-%% to support automatic load-balancing, i.e. in case it can not keep up
-%% with the amount of messages coming in, administrator should split and
-%% balance topics between worker/connections manually.
-%%
-%% NOTES:
-%% * Local messages are all normalised to QoS-1 when exporting to remote
-
--module(emqx_connector_mqtt_worker).
-
--include_lib("snabbkaffe/include/snabbkaffe.hrl").
--include_lib("emqx/include/logger.hrl").
-
-%% APIs
--export([
-    start_link/2,
-    stop/1
-]).
-
-%% management APIs
--export([
-    connect/1,
-    status/1,
-    ping/1,
-    info/1,
-    send_to_remote/2,
-    send_to_remote_async/3
-]).
-
--export([handle_publish/3]).
--export([handle_disconnect/1]).
-
--export_type([
-    config/0,
-    ack_ref/0
-]).
-
--type name() :: term().
-% -type qos() :: emqx_types:qos().
--type config() :: map().
--type ack_ref() :: term().
-% -type topic() :: emqx_types:topic().
-
--include_lib("emqx/include/logger.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
-
--define(REF(Name), {via, gproc, ?NAME(Name)}).
--define(NAME(Name), {n, l, Name}).
-
-%% @doc Start a bridge worker. Supported configs:
-%% mountpoint: The topic mount point for messages sent to remote node/cluster
-%%      `undefined', `<<>>' or `""' to disable
-%% forwards: Local topics to subscribe.
-%%
-%% Find more connection specific configs in the callback modules
-%% of emqx_bridge_connect behaviour.
--spec start_link(name(), map()) ->
-    {ok, pid()} | {error, _Reason}.
-start_link(Name, BridgeOpts) ->
-    ?SLOG(debug, #{
-        msg => "client_starting",
-        name => Name,
-        options => BridgeOpts
-    }),
-    Conf = init_config(Name, BridgeOpts),
-    Options = mk_client_options(Conf, BridgeOpts),
-    case emqtt:start_link(Options) of
-        {ok, Pid} ->
-            true = gproc:reg_other(?NAME(Name), Pid, Conf),
-            {ok, Pid};
-        {error, Reason} = Error ->
-            ?SLOG(error, #{
-                msg => "client_start_failed",
-                config => emqx_utils:redact(BridgeOpts),
-                reason => Reason
-            }),
-            Error
-    end.
-
-init_config(Name, Opts) ->
-    Mountpoint = maps:get(forward_mountpoint, Opts, undefined),
-    Subscriptions = maps:get(subscriptions, Opts, undefined),
-    Forwards = maps:get(forwards, Opts, undefined),
-    #{
-        mountpoint => format_mountpoint(Mountpoint),
-        subscriptions => pre_process_subscriptions(Subscriptions, Name, Opts),
-        forwards => pre_process_forwards(Forwards)
-    }.
-
-mk_client_options(Conf, BridgeOpts) ->
-    Server = iolist_to_binary(maps:get(server, BridgeOpts)),
-    HostPort = emqx_connector_mqtt_schema:parse_server(Server),
-    Mountpoint = maps:get(receive_mountpoint, BridgeOpts, undefined),
-    Subscriptions = maps:get(subscriptions, Conf),
-    Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Subscriptions),
-    CleanStart =
-        case Subscriptions of
-            #{remote := _} ->
-                maps:get(clean_start, BridgeOpts);
-            undefined ->
-                %% NOTE
-                %% We are ignoring the user configuration here because there's currently no reliable way
-                %% to ensure proper session recovery according to the MQTT spec.
-                true
-        end,
-    Opts = maps:without(
-        [
-            address,
-            auto_reconnect,
-            conn_type,
-            mountpoint,
-            forwards,
-            receive_mountpoint,
-            subscriptions
-        ],
-        BridgeOpts
-    ),
-    Opts#{
-        msg_handler => mk_client_event_handler(Vars, #{server => Server}),
-        hosts => [HostPort],
-        clean_start => CleanStart,
-        force_ping => true,
-        proto_ver => maps:get(proto_ver, BridgeOpts, v4)
-    }.
-
-mk_client_event_handler(Vars, Opts) when Vars /= undefined ->
-    #{
-        publish => {fun ?MODULE:handle_publish/3, [Vars, Opts]},
-        disconnected => {fun ?MODULE:handle_disconnect/1, []}
-    };
-mk_client_event_handler(undefined, _Opts) ->
-    undefined.
-
-connect(Name) ->
-    #{subscriptions := Subscriptions} = get_config(Name),
-    case emqtt:connect(get_pid(Name)) of
-        {ok, Properties} ->
-            case subscribe_remote_topics(Name, Subscriptions) of
-                ok ->
-                    {ok, Properties};
-                {ok, _, _RCs} ->
-                    {ok, Properties};
-                {error, Reason} = Error ->
-                    ?SLOG(error, #{
-                        msg => "client_subscribe_failed",
-                        subscriptions => Subscriptions,
-                        reason => Reason
-                    }),
-                    Error
-            end;
-        {error, Reason} = Error ->
-            ?SLOG(warning, #{
-                msg => "client_connect_failed",
-                reason => Reason,
-                name => Name
-            }),
-            Error
-    end.
-subscribe_remote_topics(Ref, #{remote := #{topic := FromTopic, qos := QoS}}) ->
-    emqtt:subscribe(ref(Ref), FromTopic, QoS);
-subscribe_remote_topics(_Ref, undefined) ->
-    ok.
-
-stop(Ref) ->
-    emqtt:stop(ref(Ref)).
-
-info(Ref) ->
-    emqtt:info(ref(Ref)).
-
-status(Ref) ->
-    try
-        case proplists:get_value(socket, info(Ref)) of
-            Socket when Socket /= undefined ->
-                connected;
-            undefined ->
-                connecting
-        end
-    catch
-        exit:{noproc, _} ->
-            disconnected
-    end.
-
-ping(Ref) ->
-    emqtt:ping(ref(Ref)).
-
-send_to_remote(Name, MsgIn) ->
-    trycall(fun() -> do_send(Name, export_msg(Name, MsgIn)) end).
-
-do_send(Name, {true, Msg}) ->
-    case emqtt:publish(get_pid(Name), Msg) of
-        ok ->
-            ok;
-        {ok, #{reason_code := RC}} when
-            RC =:= ?RC_SUCCESS;
-            RC =:= ?RC_NO_MATCHING_SUBSCRIBERS
-        ->
-            ok;
-        {ok, #{reason_code := RC, reason_code_name := Reason}} ->
-            ?SLOG(warning, #{
-                msg => "remote_publish_failed",
-                message => Msg,
-                reason_code => RC,
-                reason_code_name => Reason
-            }),
-            {error, Reason};
-        {error, Reason} ->
-            ?SLOG(info, #{
-                msg => "client_failed",
-                reason => Reason
-            }),
-            {error, Reason}
-    end;
-do_send(_Name, false) ->
-    ok.
-
-send_to_remote_async(Name, MsgIn, Callback) ->
-    trycall(fun() -> do_send_async(Name, export_msg(Name, MsgIn), Callback) end).
-
-do_send_async(Name, {true, Msg}, Callback) ->
-    Pid = get_pid(Name),
-    ok = emqtt:publish_async(Pid, Msg, _Timeout = infinity, Callback),
-    {ok, Pid};
-do_send_async(_Name, false, _Callback) ->
-    ok.
-
-ref(Pid) when is_pid(Pid) ->
-    Pid;
-ref(Term) ->
-    ?REF(Term).
-
-trycall(Fun) ->
-    try
-        Fun()
-    catch
-        throw:noproc ->
-            {error, disconnected};
-        exit:{noproc, _} ->
-            {error, disconnected}
-    end.
-
-format_mountpoint(undefined) ->
-    undefined;
-format_mountpoint(Prefix) ->
-    binary:replace(iolist_to_binary(Prefix), <<"${node}">>, atom_to_binary(node(), utf8)).
-
-pre_process_subscriptions(undefined, _, _) ->
-    undefined;
-pre_process_subscriptions(
-    #{remote := RC, local := LC} = Conf,
-    BridgeName,
-    BridgeOpts
-) when is_map(Conf) ->
-    Conf#{
-        remote => pre_process_in_remote(RC, BridgeName, BridgeOpts),
-        local => pre_process_in_out_common(LC)
-    };
-pre_process_subscriptions(Conf, _, _) when is_map(Conf) ->
-    %% have no 'local' field in the config
-    undefined.
-
-pre_process_forwards(undefined) ->
-    undefined;
-pre_process_forwards(#{remote := RC} = Conf) when is_map(Conf) ->
-    Conf#{remote => pre_process_in_out_common(RC)};
-pre_process_forwards(Conf) when is_map(Conf) ->
-    %% have no 'remote' field in the config
-    undefined.
-
-pre_process_in_out_common(Conf0) ->
-    Conf1 = pre_process_conf(topic, Conf0),
-    Conf2 = pre_process_conf(qos, Conf1),
-    Conf3 = pre_process_conf(payload, Conf2),
-    pre_process_conf(retain, Conf3).
-
-pre_process_conf(Key, Conf) ->
-    case maps:find(Key, Conf) of
-        error ->
-            Conf;
-        {ok, Val} when is_binary(Val) ->
-            Conf#{Key => emqx_plugin_libs_rule:preproc_tmpl(Val)};
-        {ok, Val} ->
-            Conf#{Key => Val}
-    end.
-
-pre_process_in_remote(#{qos := QoSIn} = Conf, BridgeName, BridgeOpts) ->
-    QoS = downgrade_ingress_qos(QoSIn),
-    case QoS of
-        QoSIn ->
-            ok;
-        _ ->
-            ?SLOG(warning, #{
-                msg => "downgraded_unsupported_ingress_qos",
-                qos_configured => QoSIn,
-                qos_used => QoS,
-                name => BridgeName,
-                options => BridgeOpts
-            })
-    end,
-    Conf#{qos => QoS}.
-
-downgrade_ingress_qos(2) ->
-    1;
-downgrade_ingress_qos(QoS) ->
-    QoS.
-
-get_pid(Name) ->
-    case gproc:where(?NAME(Name)) of
-        Pid when is_pid(Pid) ->
-            Pid;
-        undefined ->
-            throw(noproc)
-    end.
-
-get_config(Name) ->
-    try
-        gproc:lookup_value(?NAME(Name))
-    catch
-        error:badarg ->
-            throw(noproc)
-    end.
-
-export_msg(Name, Msg) ->
-    case get_config(Name) of
-        #{forwards := Forwards = #{}, mountpoint := Mountpoint} ->
-            {true, export_msg(Mountpoint, Forwards, Msg)};
-        #{forwards := undefined} ->
-            ?SLOG(error, #{
-                msg => "forwarding_unavailable",
-                message => Msg,
-                reason => "egress is not configured"
-            }),
-            false
-    end.
-
-export_msg(Mountpoint, Forwards, Msg) ->
-    Vars = emqx_connector_mqtt_msg:make_pub_vars(Mountpoint, Forwards),
-    emqx_connector_mqtt_msg:to_remote_msg(Msg, Vars).
-
-%%
-
-handle_publish(#{properties := Props} = MsgIn, Vars, Opts) ->
-    Msg = import_msg(MsgIn, Opts),
-    ?SLOG(debug, #{
-        msg => "publish_local",
-        message => Msg,
-        vars => Vars
-    }),
-    case Vars of
-        #{on_message_received := {Mod, Func, Args}} ->
-            _ = erlang:apply(Mod, Func, [Msg | Args]);
-        _ ->
-            ok
-    end,
-    maybe_publish_local(Msg, Vars, Props).
-
-handle_disconnect(_Reason) ->
-    ok.
-
-maybe_publish_local(Msg, Vars, Props) ->
-    case emqx_utils_maps:deep_get([local, topic], Vars, undefined) of
-        %% local topic is not set, discard it
-        undefined ->
-            ok;
-        _ ->
-            emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars, Props))
-    end.
-
-import_msg(
-    #{
-        dup := Dup,
-        payload := Payload,
-        properties := Props,
-        qos := QoS,
-        retain := Retain,
-        topic := Topic
-    },
-    #{server := Server}
-) ->
-    #{
-        id => emqx_guid:to_hexstr(emqx_guid:gen()),
-        server => Server,
-        payload => Payload,
-        topic => Topic,
-        qos => QoS,
-        dup => Dup,
-        retain => Retain,
-        pub_props => printable_maps(Props),
-        message_received_at => erlang:system_time(millisecond)
-    }.
-
-printable_maps(undefined) ->
-    #{};
-printable_maps(Headers) ->
-    maps:fold(
-        fun
-            ('User-Property', V0, AccIn) when is_list(V0) ->
-                AccIn#{
-                    'User-Property' => maps:from_list(V0),
-                    'User-Property-Pairs' => [
-                        #{
-                            key => Key,
-                            value => Value
-                        }
-                     || {Key, Value} <- V0
-                    ]
-                };
-            (K, V0, AccIn) ->
-                AccIn#{K => V0}
-        end,
-        #{},
-        Headers
-    ).

apps/emqx_dashboard/src/emqx_dashboard.app.src

@@ -2,7 +2,7 @@
 {application, emqx_dashboard, [
     {description, "EMQX Web Dashboard"},
     % strict semver, bump manually!
-    {vsn, "5.0.22"},
+    {vsn, "5.0.23"},
     {modules, []},
     {registered, [emqx_dashboard_sup]},
     {applications, [kernel, stdlib, mnesia, minirest, emqx, emqx_ctl]},

apps/emqx_dashboard/src/emqx_dashboard_desc_cache.erl

@@ -27,15 +27,12 @@
 %% @doc Global ETS table to cache the description of the configuration items.
 %% The table is owned by the emqx_dashboard_sup the root supervisor of emqx_dashboard.
 %% The cache is initialized with the default language (English) and
-%% all the desc.<lang>.hocon files in the www/static directory (extracted from dashboard package).
+%% all the desc.<lang>.hocon files in the app's priv directory
 init() ->
     ok = ensure_app_loaded(emqx_dashboard),
     PrivDir = code:priv_dir(emqx_dashboard),
-    EngDesc = filename:join([PrivDir, "desc.en.hocon"]),
-    WwwStaticDir = filename:join([PrivDir, "www", "static"]),
-    OtherLangDesc0 = filelib:wildcard("desc.*.hocon", WwwStaticDir),
-    OtherLangDesc = lists:map(fun(F) -> filename:join([WwwStaticDir, F]) end, OtherLangDesc0),
-    Files = [EngDesc | OtherLangDesc],
+    Files0 = filelib:wildcard("desc.*.hocon", PrivDir),
+    Files = lists:map(fun(F) -> filename:join([PrivDir, F]) end, Files0),
     ok = emqx_utils_ets:new(?MODULE, [public, ordered_set, {read_concurrency, true}]),
     ok = lists:foreach(fun(F) -> load_desc(?MODULE, F) end, Files).
 

apps/emqx_dashboard/test/emqx_dashboard_api_test_helpers.erl

@@ -64,11 +64,6 @@ set_default_config(DefaultUsername, HAProxyEnabled, Opts) ->
         i18n_lang => en
     },
     emqx_config:put([dashboard], Config),
-    I18nFile = filename:join([
-        code:priv_dir(emqx_dashboard),
-        "i18n.conf"
-    ]),
-    application:set_env(emqx_dashboard, i18n_file, I18nFile),
     ok.
 
 request(Method, Url) ->

apps/emqx_enterprise/README.md

@@ -1,6 +1,5 @@
 # EMQX Enterprise Application
 
-This application so fart only holds EMQX config schema for enterprise edition.
+This application only holds EMQX config schema for enterprise edition so far.
 In the future this application will collect more responsibilities in managing
 enterprise edition specific features.
-

apps/emqx_enterprise/src/emqx_enterprise.app.src

@@ -1,6 +1,6 @@
 {application, emqx_enterprise, [
     {description, "EMQX Enterprise Edition"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_enterprise/src/emqx_enterprise_schema.erl

@@ -64,7 +64,7 @@ redefine_roots(Roots) ->
     override(Roots, Overrides).
 
 redefine_node(Fields) ->
-    Overrides = [{"applications", #{default => <<"emqx_license">>}}],
+    Overrides = [],
     override(Fields, Overrides).
 
 override(Fields, []) ->

apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl

@@ -13,16 +13,6 @@ doc_gen_test() ->
         60,
         fun() ->
             Dir = "tmp",
-            ok = filelib:ensure_dir(filename:join("tmp", foo)),
-            I18nFile = filename:join([
-                "_build",
-                "test",
-                "lib",
-                "emqx_dashboard",
-                "priv",
-                "i18n.conf"
-            ]),
-            _ = emqx_conf:dump_schema(Dir, emqx_enterprise_schema, I18nFile),
-            ok
+            ok = emqx_conf:dump_schema(Dir, emqx_enterprise_schema)
         end
     }.

apps/emqx_ft/src/emqx_ft.app.src

@@ -1,6 +1,6 @@
 {application, emqx_ft, [
     {description, "EMQX file transfer over MQTT"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {mod, {emqx_ft_app, []}},
     {applications, [

apps/emqx_ft/src/emqx_ft_storage_exporter.erl

@@ -38,7 +38,7 @@
 
 -export_type([export/0]).
 
--type storage() :: emxt_ft_storage_fs:storage().
+-type storage() :: emqx_ft_storage_fs:storage() | undefined.
 -type transfer() :: emqx_ft:transfer().
 -type filemeta() :: emqx_ft:filemeta().
 -type checksum() :: emqx_ft:checksum().

apps/emqx_resource/src/emqx_resource.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_resource, [
     {description, "Manager for all external resources"},
-    {vsn, "0.1.17"},
+    {vsn, "0.1.18"},
     {registered, []},
     {mod, {emqx_resource_app, []}},
     {applications, [

apps/emqx_resource/src/emqx_resource.erl

@@ -121,7 +121,8 @@
 
 -export_type([
     resource_id/0,
-    resource_data/0
+    resource_data/0,
+    resource_status/0
 ]).
 
 -optional_callbacks([
@@ -533,7 +534,7 @@ clean_allocated_resources(ResourceId, ResourceMod) ->
         true ->
             %% The resource entries in the ETS table are erased inside
             %% `call_stop' if the call is successful.
-            ok = emqx_resource:call_stop(ResourceId, ResourceMod, _ResourceState = undefined),
+            ok = call_stop(ResourceId, ResourceMod, _ResourceState = undefined),
             ok;
         false ->
             ok

apps/emqx_resource/src/emqx_resource_buffer_worker.erl

@@ -247,7 +247,7 @@ running(info, Info, _St) ->
     keep_state_and_data.
 
 blocked(enter, _, #{resume_interval := ResumeT} = St0) ->
-    ?tp(buffer_worker_enter_blocked, #{}),
+    ?tp(buffer_worker_enter_blocked, #{buffer_worker => self()}),
     %% discard the old timer, new timer will be started when entering running state again
     St = cancel_flush_timer(St0),
     {keep_state, St, {state_timeout, ResumeT, unblock}};
@@ -403,7 +403,8 @@ retry_inflight_sync(Ref, QueryOrBatch, Data0) ->
                 buffer_worker_retry_inflight_failed,
                 #{
                     ref => Ref,
-                    query_or_batch => QueryOrBatch
+                    query_or_batch => QueryOrBatch,
+                    result => Result
                 }
             ),
             {keep_state, Data1, {state_timeout, ResumeT, unblock}};
@@ -976,7 +977,7 @@ handle_async_worker_down(Data0, Pid) ->
     {AsyncWorkerMRef, AsyncWorkers} = maps:take(Pid, AsyncWorkers0),
     Data = Data0#{async_workers := AsyncWorkers},
     mark_inflight_items_as_retriable(Data, AsyncWorkerMRef),
-    {keep_state, Data}.
+    {next_state, blocked, Data}.
 
 -spec call_query(force_sync | async_if_possible, _, _, _, _, _) -> _.
 call_query(QM, Id, Index, Ref, Query, QueryOpts) ->
@@ -1256,6 +1257,13 @@ handle_async_batch_reply2([Inflight], ReplyContext, Result, Now) ->
             %% some queries are not expired, put them back to the inflight batch
             %% so it can be either acked now or retried later
             ok = update_inflight_item(InflightTID, Ref, RealNotExpired, NumExpired),
+            ?tp_ignore_side_effects_in_prod(
+                handle_async_reply_partially_expired,
+                #{
+                    inflight_count => inflight_count(InflightTID),
+                    num_inflight_messages => inflight_num_msgs(InflightTID)
+                }
+            ),
             do_handle_async_batch_reply(ReplyContext#{min_batch := RealNotExpired}, Result)
     end.
 
@@ -1556,7 +1564,7 @@ mark_inflight_items_as_retriable(Data, AsyncWorkerMRef) ->
             end
         ),
     _NumAffected = ets:select_replace(InflightTID, MatchSpec),
-    ?tp(buffer_worker_async_agent_down, #{num_affected => _NumAffected}),
+    ?tp(buffer_worker_async_agent_down, #{num_affected => _NumAffected, buffer_worker => self()}),
     ok.
 
 %% used to update a batch after dropping expired individual queries.

apps/emqx_resource/src/emqx_resource_manager.erl

@@ -192,14 +192,13 @@ remove(ResId) when is_binary(ResId) ->
 %% @doc Stops a running resource_manager and optionally clears the metrics for the resource
 -spec remove(resource_id(), boolean()) -> ok | {error, Reason :: term()}.
 remove(ResId, ClearMetrics) when is_binary(ResId) ->
-    ResourceManagerPid = gproc:whereis_name(?NAME(ResId)),
     try
         safe_call(ResId, {remove, ClearMetrics}, ?T_OPERATION)
     after
         %% Ensure the supervisor has it removed, otherwise the immediate re-add will see a stale process
         %% If the 'remove' call babove had succeeded, this is mostly a no-op but still needed to avoid race condition.
         %% Otherwise this is a 'infinity' shutdown, so it may take arbitrary long.
-        emqx_resource_manager_sup:delete_child(ResourceManagerPid)
+        emqx_resource_manager_sup:delete_child(ResId)
     end.
 
 %% @doc Stops and then starts an instance that was already running

apps/emqx_resource/src/emqx_resource_manager_sup.erl

@@ -26,12 +26,12 @@
 -export([init/1]).
 
 ensure_child(ResId, Group, ResourceType, Config, Opts) ->
-    _ = supervisor:start_child(?MODULE, [ResId, Group, ResourceType, Config, Opts]),
+    _ = supervisor:start_child(?MODULE, child_spec(ResId, Group, ResourceType, Config, Opts)),
     ok.
 
-delete_child(Pid) ->
-    _ = supervisor:terminate_child(?MODULE, Pid),
-    _ = supervisor:delete_child(?MODULE, Pid),
+delete_child(ResId) ->
+    _ = supervisor:terminate_child(?MODULE, ResId),
+    _ = supervisor:delete_child(?MODULE, ResId),
     ok.
 
 start_link() ->
@@ -44,18 +44,19 @@ init([]) ->
         public,
         {read_concurrency, true}
     ]),
-    ChildSpecs = [
-        #{
-            id => emqx_resource_manager,
-            start => {emqx_resource_manager, start_link, []},
-            restart => transient,
-            %% never force kill a resource manager.
-            %% becasue otherwise it may lead to release leak,
-            %% resource_manager's terminate callback calls resource on_stop
-            shutdown => infinity,
-            type => worker,
-            modules => [emqx_resource_manager]
-        }
-    ],
-    SupFlags = #{strategy => simple_one_for_one, intensity => 10, period => 10},
+    ChildSpecs = [],
+    SupFlags = #{strategy => one_for_one, intensity => 10, period => 10},
     {ok, {SupFlags, ChildSpecs}}.
+
+child_spec(ResId, Group, ResourceType, Config, Opts) ->
+    #{
+        id => ResId,
+        start => {emqx_resource_manager, start_link, [ResId, Group, ResourceType, Config, Opts]},
+        restart => transient,
+        %% never force kill a resource manager.
+        %% becasue otherwise it may lead to release leak,
+        %% resource_manager's terminate callback calls resource on_stop
+        shutdown => infinity,
+        type => worker,
+        modules => [emqx_resource_manager]
+    }.

apps/emqx_resource/test/emqx_resource_SUITE.erl

@@ -1832,6 +1832,18 @@ t_async_pool_worker_death(_Config) ->
                 NumReqs,
                 lists:sum([N || #{num_affected := N} <- Events])
             ),
+
+            %% The `DOWN' signal must trigger the transition to the `blocked' state,
+            %% otherwise the request won't be retried until the buffer worker is `blocked'
+            %% for other reasons.
+            ?assert(
+                ?strict_causality(
+                    #{?snk_kind := buffer_worker_async_agent_down, buffer_worker := _Pid0},
+                    #{?snk_kind := buffer_worker_enter_blocked, buffer_worker := _Pid1},
+                    _Pid0 =:= _Pid1,
+                    Trace
+                )
+            ),
             ok
         end
     ),
@@ -2248,6 +2260,15 @@ do_t_expiration_async_after_reply(IsBatch) ->
                             }
                         ],
                         ?of_kind(handle_async_reply_expired, Trace)
+                    ),
+                    ?assertMatch(
+                        [
+                            #{
+                                inflight_count := 1,
+                                num_inflight_messages := 1
+                            }
+                        ],
+                        ?of_kind(handle_async_reply_partially_expired, Trace)
                     );
                 single ->
                     ?assertMatch(

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.18"},
+    {vsn, "5.0.19"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]},

apps/emqx_rule_engine/src/emqx_rule_engine_api.erl

@@ -529,47 +529,69 @@ printable_function_name(Mod, Func) ->
     list_to_binary(lists:concat([Mod, ":", Func])).
 
 get_rule_metrics(Id) ->
-    Format = fun(
-        Node,
-        #{
-            counters :=
-                #{
-                    'matched' := Matched,
-                    'passed' := Passed,
-                    'failed' := Failed,
-                    'failed.exception' := FailedEx,
-                    'failed.no_result' := FailedNoRes,
-                    'actions.total' := OTotal,
-                    'actions.failed' := OFailed,
-                    'actions.failed.out_of_service' := OFailedOOS,
-                    'actions.failed.unknown' := OFailedUnknown,
-                    'actions.success' := OFailedSucc
-                },
-            rate :=
-                #{
-                    'matched' :=
-                        #{current := Current, max := Max, last5m := Last5M}
-                }
-        }
-    ) ->
-        #{
-            metrics => ?METRICS(
-                Matched,
-                Passed,
-                Failed,
-                FailedEx,
-                FailedNoRes,
-                OTotal,
-                OFailed,
-                OFailedOOS,
-                OFailedUnknown,
-                OFailedSucc,
-                Current,
-                Max,
-                Last5M
-            ),
-            node => Node
-        }
+    Format = fun
+        (
+            Node,
+            #{
+                counters :=
+                    #{
+                        'matched' := Matched,
+                        'passed' := Passed,
+                        'failed' := Failed,
+                        'failed.exception' := FailedEx,
+                        'failed.no_result' := FailedNoRes,
+                        'actions.total' := OTotal,
+                        'actions.failed' := OFailed,
+                        'actions.failed.out_of_service' := OFailedOOS,
+                        'actions.failed.unknown' := OFailedUnknown,
+                        'actions.success' := OFailedSucc
+                    },
+                rate :=
+                    #{
+                        'matched' :=
+                            #{current := Current, max := Max, last5m := Last5M}
+                    }
+            }
+        ) ->
+            #{
+                metrics => ?METRICS(
+                    Matched,
+                    Passed,
+                    Failed,
+                    FailedEx,
+                    FailedNoRes,
+                    OTotal,
+                    OFailed,
+                    OFailedOOS,
+                    OFailedUnknown,
+                    OFailedSucc,
+                    Current,
+                    Max,
+                    Last5M
+                ),
+                node => Node
+            };
+        (Node, _Metrics) ->
+            %% Empty metrics: can happen when a node joins another and a bridge is not yet
+            %% replicated to it, so the counters map is empty.
+            #{
+                metrics => ?METRICS(
+                    _Matched = 0,
+                    _Passed = 0,
+                    _Failed = 0,
+                    _FailedEx = 0,
+                    _FailedNoRes = 0,
+                    _OTotal = 0,
+                    _OFailed = 0,
+                    _OFailedOOS = 0,
+                    _OFailedUnknown = 0,
+                    _OFailedSucc = 0,
+                    _Current = 0,
+                    _Max = 0,
+                    _Last5M = 0
+                ),
+                node => Node
+            }
     end,
     [
         Format(Node, emqx_plugin_libs_proto_v1:get_metrics(Node, rule_metrics, Id))

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -228,7 +228,9 @@
     format_date/3,
     format_date/4,
     date_to_unix_ts/3,
-    date_to_unix_ts/4
+    date_to_unix_ts/4,
+    timezone_to_second/1,
+    timezone_to_offset_seconds/1
 ]).
 
 %% MongoDB specific date functions. These functions return a date tuple. The
@@ -1104,6 +1106,12 @@ date_to_unix_ts(TimeUnit, Offset, FormatString, InputString) ->
     OffsetDelta = erlang:convert_time_unit(OffsetSecond, second, Unit),
     date_to_unix_ts(Unit, FormatString, InputString) - OffsetDelta.
 
+timezone_to_second(TimeZone) ->
+    timezone_to_offset_seconds(TimeZone).
+
+timezone_to_offset_seconds(TimeZone) ->
+    emqx_calendar:offset_second(TimeZone).
+
 %% @doc This is for sql funcs that should be handled in the specific modules.
 %% Here the emqx_rule_funcs module acts as a proxy, forwarding
 %% the function handling to the worker module.

apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl

@@ -94,6 +94,26 @@ t_crud_rule_api(_Config) ->
     ct:pal("RMetrics : ~p", [Metrics]),
     ?assertMatch(#{id := RuleId, metrics := _, node_metrics := _}, Metrics),
 
+    %% simulating a node joining a cluster and lagging the configuration replication; in
+    %% such cases, when fetching metrics, a rule may exist in the cluster but not on the
+    %% new node.  We just check that it doesn't provoke a crash.
+    emqx_common_test_helpers:with_mock(
+        emqx_metrics_worker,
+        get_metrics,
+        fun(HandlerName, MetricId) ->
+            %% change the metric id to some unknown id.
+            meck:passthrough([HandlerName, <<"unknown-", MetricId/binary>>])
+        end,
+        fun() ->
+            {200, Metrics1} = emqx_rule_engine_api:'/rules/:id/metrics'(get, #{
+                bindings => #{id => RuleId}
+            }),
+            ct:pal("RMetrics : ~p", [Metrics1]),
+            ?assertMatch(#{id := RuleId, metrics := _, node_metrics := _}, Metrics1),
+            ok
+        end
+    ),
+
     {200, Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{
         bindings => #{id => RuleId},
         body => ?SIMPLE_RULE(RuleId)#{<<"sql">> => <<"select * from \"t/b\"">>}

apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl

@@ -1012,6 +1012,25 @@ prop_format_date_fun() ->
     Args3DTUS = [<<"second">>, <<"+04:00">>, <<"--%m--%d--%Y---%H:%M:%S">>, Formatters3],
     Second == apply_func(date_to_unix_ts, Args3DTUS).
 
+t_timezone_to_offset_seconds(_) ->
+    timezone_to_offset_seconds_helper(timezone_to_offset_seconds),
+    %% The timezone_to_second function is kept for compatibility with 4.X.
+    timezone_to_offset_seconds_helper(timezone_to_second).
+
+timezone_to_offset_seconds_helper(FunctionName) ->
+    ?assertEqual(120 * 60, apply_func(FunctionName, [<<"+02:00:00">>])),
+    ?assertEqual(-120 * 60, apply_func(FunctionName, [<<"-02:00:00">>])),
+    ?assertEqual(102, apply_func(FunctionName, [<<"+00:01:42">>])),
+    ?assertEqual(0, apply_func(FunctionName, [<<"z">>])),
+    ?assertEqual(0, apply_func(FunctionName, [<<"Z">>])),
+    ?assertEqual(42, apply_func(FunctionName, [42])),
+    ?assertEqual(0, apply_func(FunctionName, [undefined])),
+    %% Check that the following does not crash
+    apply_func(FunctionName, [<<"local">>]),
+    apply_func(FunctionName, ["local"]),
+    apply_func(FunctionName, [local]),
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Utility functions
 %%------------------------------------------------------------------------------

apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl

@@ -430,25 +430,25 @@ t_num_clients(_Config) ->
         {port, 1883},
         {clean_start, false}
     ]),
-    {{ok, _}, _} = ?wait_async_action(
+    {{ok, _}, {ok, _}} = ?wait_async_action(
         {ok, _} = emqtt:connect(Client),
         #{
             ?snk_kind := emqx_stats_setstat,
             count_stat := 'live_connections.count',
             value := 1
         },
-        2000
+        5_000
     ),
     {ok, TelemetryData0} = emqx_telemetry:get_telemetry(),
     ?assertEqual(1, get_value(num_clients, TelemetryData0)),
-    {ok, _} = ?wait_async_action(
+    {ok, {ok, _}} = ?wait_async_action(
         ok = emqtt:disconnect(Client),
         #{
             ?snk_kind := emqx_stats_setstat,
             count_stat := 'live_connections.count',
             value := 0
         },
-        2000
+        5_000
     ),
     {ok, TelemetryData1} = emqx_telemetry:get_telemetry(),
     ?assertEqual(0, get_value(num_clients, TelemetryData1)),

changes/ce/feat-10858.en.md

@@ -0,0 +1 @@
+A new utility function timezone_to_offset_seconds/1 has been added to the rule engine SQL language. This function converts a timezone string (for example, "+02:00", "Z" and "local") to the corresponding offset in seconds.

changes/ce/fix-10884.en.md

@@ -0,0 +1 @@
+Fixes an issue where trying to get rule info or metrics could result in a crash when a node is joining a cluster.

changes/ce/fix-10887.en.md

@@ -0,0 +1,3 @@
+Fixed a potential issue where requests to bridges might take a long time to be retried.
+
+This only affected low throughput scenarios, where the buffering layer could take a long time to detect connectivity and driver problems.

changes/ce/perf-10754.en.md

@@ -0,0 +1,3 @@
+The MQTT bridge has been enhanced to utilize connection pooling and leverage available parallelism, substantially improving throughput.
+
+As a consequence, single MQTT bridge now uses a pool of `clientid`s to connect to the remote broker.

changes/ee/feat-10778.en.md

@@ -1 +1 @@
-Refactored Pulsar Producer bridge to avoid leaking resources during crashes.
+Refactored Pulsar Producer bridge to avoid leaking resources during crashes at creation.

changes/ee/feat-10813.en.md

@@ -0,0 +1 @@
+Refactored Kafka Producer and Consumer bridges to avoid leaking resources during crashes at creation.

changes/ee/feat-10841.en.md

@@ -0,0 +1 @@
+Added a schema validation to ensure message key is not empty when dispatching by key in Kafka and Pulsar Producer bridges.

changes/v5.0.26.en.md

@@ -0,0 +1,118 @@
+# v5.0.26
+
+## Enhancements
+
+- [#10584](https://github.com/emqx/emqx/pull/10584) Add log level configuration to SSL communication
+
+- [#10702](https://github.com/emqx/emqx/pull/10702) Introduce a more straightforward configuration option `keepalive_multiplier` and
+  deprecate the old `keepalive_backoff` configuration.
+  After this enhancement, EMQX checks the client's keepalive timeout status
+  period by multiplying the "Client Requested Keepalive Interval" with `keepalive_multiplier`.
+
+- [#10713](https://github.com/emqx/emqx/pull/10713) We hide the request_timeout in resource_option of the webhook to keep it consistent with the http request_timeout of the webhook.
+  From now on, when configuring a webhook through API or configuration files,
+  it is no longer necessary to configure the request_timeout of the resource. Only configuring the http request_timeout is sufficient, and the request_timeout in the resource will automatically be consistent with the http request_timeout.
+
+- [#10511](https://github.com/emqx/emqx/pull/10511) Improve the security and privacy of some resource logs by masking sensitive information in the data.
+
+- [#10678](https://github.com/emqx/emqx/pull/10678) Optimized counter increment calls to avoid work if increment is zero.
+
+- [#10690](https://github.com/emqx/emqx/pull/10690) Added a retry mechanism to webhook bridge that attempts to improve throughput.
+
+  This optimization retries request failures without blocking the buffering layer, which can improve throughput in situations of high messaging rate.
+
+- [#10698](https://github.com/emqx/emqx/pull/10698) Optimize memory usage when accessing the configuration during runtime.
+
+## Bug Fixes
+
+- [#10340](https://github.com/emqx/emqx/pull/10340) Fixed the issue that could lead to crash logs being printed when stopping EMQX via systemd.
+  ```
+  2023-03-29T16:43:25.915761+08:00 [error] Generic server memsup terminating. Reason: {port_died,normal}. Last message: {'EXIT',<0.2117.0>,{port_died,normal}}. State: [{data,[{"Timeout",60000}]},{items,{"Memory Usage",[{"Allocated",929959936},{"Total",3832242176}]}},{items,{"Worst Memory User",[{"Pid",<0.2031.0>},{"Memory",4720472}]}}].
+  2023-03-29T16:43:25.924764+08:00 [error] crasher: initial call: memsup:init/1, pid: <0.2116.0>, registered_name: memsup, exit: {{port_died,normal},[{gen_server,handle_common_reply,8,[{file,"gen_server.erl"},{line,811}]},{proc_lib,init_p_do_apply,3,[{file,"proc_lib.erl"},{line,226}]}]}, ancestors: [os_mon_sup,<0.2114.0>], message_queue_len: 0, messages: [], links: [<0.2115.0>], dictionary: [], trap_exit: true, status: running, heap_size: 4185, stack_size: 29, reductions: 187637; neighbours:
+  2023-03-29T16:43:25.924979+08:00 [error] Supervisor: {local,os_mon_sup}. Context: child_terminated. Reason: {port_died,normal}. Offender: id=memsup,pid=<0.2116.0>.
+  ```
+
+- [#10563](https://github.com/emqx/emqx/pull/10563) Corrected an issue where the no_local flag was not functioning correctly.
+
+
+- [#10600](https://github.com/emqx/emqx/pull/10600) Deleted emqx_statsd application.
+
+
+- [#10653](https://github.com/emqx/emqx/pull/10653) Store gateway authentication TLS certificates and keys in the data directory.
+
+- [#10677](https://github.com/emqx/emqx/pull/10677) In Rule API, reapond with 404 HTTP error code when trying to delete a rule that does not exist.
+
+- [#10682](https://github.com/emqx/emqx/pull/10682) Fix the timestamp for the will message is incorrectly assigned at the session creation time, now this timestamp is the disconnected time of the session.
+
+- [#10701](https://github.com/emqx/emqx/pull/10701) RPM package for Amazon Linux 2 did not support TLS v1.3 as it was assembled with Erlang/OTP built with openssl 1.0.
+
+- [#10715](https://github.com/emqx/emqx/pull/10715) Postpone trimming the connection information structure until after `client.connected` hooks have been executed. These hooks once again have access to the client's peer certificate.
+
+- [#10717](https://github.com/emqx/emqx/pull/10717) Fixed an issue where the buffering layer processes could use a lot of CPU when inflight window is full.
+
+- [#10724](https://github.com/emqx/emqx/pull/10724) A summary has been added for all endpoints in the HTTP API documentation (accessible at "http://emqx_host_name:18083/api-docs").
+
+- [#10726](https://github.com/emqx/emqx/pull/10726) Validate Health Check Interval and Auto Restart Interval against the range from 1ms to 1 hour.
+
+- [#10728](https://github.com/emqx/emqx/pull/10728) Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` in the `DO` clause.
+
+  Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement:
+  ```
+  FOREACH payload.date as date, payload.array as elem
+  DO date, elem
+  FROM "t/#"
+  ```
+  Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement:
+  `[{"elem": "a","date": "undefined"}]`.
+  After the fix, the output of the SQL statement is: `[{"elem": "a","date": "2023-05-06"}]`
+
+- [#10737](https://github.com/emqx/emqx/pull/10737) Fix the issue where the HTTP API interface of Gateway cannot handle ClientIDs with
+  special characters, such as: `!@#$%^&*()_+{}:"<>?/`.
+
+- [#10742](https://github.com/emqx/emqx/pull/10742) Check the correctness of the rules before saving the authorization file source.
+  Previously, Saving wrong rules could lead to restart failure.
+
+- [#10743](https://github.com/emqx/emqx/pull/10743) Fixes an issue where trying to get a bridge info or metrics could result in a crash when a node is joining a cluster.
+
+- [#10746](https://github.com/emqx/emqx/pull/10746) Add missing support of the event `$events/delivery_dropped` into the rule engine test API `rule_test`.
+
+- [#10747](https://github.com/emqx/emqx/pull/10747) Refactor date and time functions, `format_date` and `date_to_unix_ts`, in the rule engine to fix the implementation problem.
+
+- [#10755](https://github.com/emqx/emqx/pull/10755) Fixed data bridge resource update race condition.
+
+  In the 'delete + create' process for EMQX resource updates,
+  long bridge creation times could cause dashboard request timeouts.
+  If a bridge resource update was initiated before completion of its creation,
+  it led to an erroneous deletion from the runtime, despite being present in the config file.
+
+  This fix addresses the race condition in bridge resource updates,
+  ensuring the accurate identification and addition of new resources,
+  maintaining consistency between runtime and configuration file statuses.
+
+- [#10760](https://github.com/emqx/emqx/pull/10760) Fix Internal Error 500 that occurred sometimes when bridge statistics page was updated while a node was (re)joining the cluster.
+
+- [#10761](https://github.com/emqx/emqx/pull/10761) Fixing the issue where the default value of SSL certificate for Dashboard Listener was not correctly interpolated, which caused HTTPS to be inaccessible when verify_peer and cacertfile were using the default configuration.
+
+- [#10785](https://github.com/emqx/emqx/pull/10785) Ensure `EMQX_LOG_DIR` is set by Windows boot script.
+
+  The environment variable `EMQX_LOG_DIR` was missing in v5.0.25, caused EMQX Windows package fail to boot unless set by sysadmin.
+
+- [#10801](https://github.com/emqx/emqx/pull/10801) Avoid duplicated percent decode the topic name in API `/topics/{topic}` and `/topics`.
+
+- [#10809](https://github.com/emqx/emqx/pull/10809) Address  `** ERROR ** Mnesia post_commit hook failed: error:badarg` error messages happening during node shutdown or restart.
+  Mria pull request: https://github.com/emqx/mria/pull/142
+
+- [#10817](https://github.com/emqx/emqx/pull/10817) Fix the error of not being able to configure `auto_restart_interval` as infinity
+
+- [#10818](https://github.com/emqx/emqx/pull/10818) Fixing `emqx_ctl traces` command.
+
+- [#10820](https://github.com/emqx/emqx/pull/10820) In case the cluster updated license before the new node join in. The new node will not apply the updated license.
+  After this change, the new joined node will use the cluster's license key.
+
+  Sometimes the new node must start with a outdated license.
+  e.g. use emqx-operator deployed and needed to scale up after license expired.
+  At the time the cluster's license key already updated by API/CLI, but the new node won't use it.
+
+- [#10833](https://github.com/emqx/emqx/pull/10833) Only include enabled authenticators and authorizers in telemetry report, not all of them.
+
+- [#10851](https://github.com/emqx/emqx/pull/10851) Obfuscated sensitive data in the bad API logging.

deploy/charts/emqx-enterprise/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.0.4
+version: 5.1.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.0.4
+appVersion: 5.1.0

deploy/charts/emqx/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.0.25
+version: 5.1.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.0.25
+appVersion: 5.1.0

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src

@@ -1,6 +1,6 @@
 {application, emqx_ee_bridge, [
     {description, "EMQX Enterprise data bridges"},
-    {vsn, "0.1.14"},
+    {vsn, "0.1.15"},
     {registered, []},
     {applications, [
         kernel,

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl

@@ -242,7 +242,8 @@ pulsar_structs() ->
                 hoconsc:map(name, ref(emqx_bridge_pulsar, pulsar_producer)),
                 #{
                     desc => <<"Pulsar Producer Bridge Config">>,
-                    required => false
+                    required => false,
+                    validator => fun emqx_bridge_pulsar:producer_strategy_key_validator/1
                 }
             )}
     ].

lib-ee/emqx_license/src/emqx_license_schema.erl

@@ -82,7 +82,7 @@ check_license_watermark(Conf) ->
 %% @doc The default license key.
 %% This default license has 1000 connections limit.
 %% It is issued on 2023-01-09 and valid for 5 years (1825 days)
-%% NOTE: when updating a new key, the schema doc in emqx_license_schema_i18n.conf
+%% NOTE: when updating a new key, the schema doc in emqx_license_schema.hocon
 %% should be updated accordingly
 default_license() ->
     <<

mix.exs

@@ -54,12 +54,12 @@ defmodule EMQXUmbrella.MixProject do
       {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
       {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
       {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true},
-      {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true},
+      {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-11", override: true},
       {:ekka, github: "emqx/ekka", tag: "0.15.2", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.3.10", override: true},
-      {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true},
+      {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true},
       {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
       # maybe forbid to fetch quicer
@@ -195,10 +195,10 @@ defmodule EMQXUmbrella.MixProject do
       {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
       {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.9", override: true},
       {:wolff, github: "kafka4beam/wolff", tag: "1.7.5"},
-      {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.2", override: true},
+      {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true},
       {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0"},
       {:brod, github: "kafka4beam/brod", tag: "3.16.8"},
-      {:snappyer, "1.2.8", override: true},
+      {:snappyer, "1.2.9", override: true},
       {:crc32cer, "0.1.8", override: true},
       {:supervisor3, "1.1.12", override: true},
       {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true},
@@ -310,6 +310,7 @@ defmodule EMQXUmbrella.MixProject do
             :emqx_connector,
             :emqx_exhook,
             :emqx_bridge,
+            :emqx_bridge_mqtt,
             :emqx_modules,
             :emqx_management,
             :emqx_retainer,
@@ -372,6 +373,7 @@ defmodule EMQXUmbrella.MixProject do
         emqx_gateway_exproto: :permanent,
         emqx_exhook: :permanent,
         emqx_bridge: :permanent,
+        emqx_bridge_mqtt: :permanent,
         emqx_rule_engine: :permanent,
         emqx_modules: :permanent,
         emqx_management: :permanent,

rebar.config

@@ -61,12 +61,12 @@
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.6"}}}
-    , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-9"}}}
+    , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.7.2-emqx-11"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.2"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
     , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.7"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.10"}}}
-    , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.3"}}}
+    , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}}
     , {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.7"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
     , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.8.5"}}}

rebar.config.erl

@@ -91,6 +91,7 @@ is_community_umbrella_app("apps/emqx_bridge_matrix") -> false;
 is_community_umbrella_app("apps/emqx_bridge_mongodb") -> false;
 is_community_umbrella_app("apps/emqx_bridge_mysql") -> false;
 is_community_umbrella_app("apps/emqx_bridge_pgsql") -> false;
+is_community_umbrella_app("apps/emqx_bridge_pulsar") -> false;
 is_community_umbrella_app("apps/emqx_bridge_redis") -> false;
 is_community_umbrella_app("apps/emqx_bridge_rocketmq") -> false;
 is_community_umbrella_app("apps/emqx_bridge_tdengine") -> false;
@@ -426,6 +427,7 @@ relx_apps(ReleaseType, Edition) ->
             emqx_gateway_exproto,
             emqx_exhook,
             emqx_bridge,
+            emqx_bridge_mqtt,
             emqx_rule_engine,
             emqx_modules,
             emqx_management,

rel/i18n/emqx_connector_mqtt_schema.hocon

@@ -1,4 +1,4 @@
-emqx_connector_mqtt_schema {
+emqx_bridge_mqtt_connector_schema {
 
 bridge_mode.desc:
 """If enable bridge mode.
@@ -32,6 +32,14 @@ is configured, then both the data got from the rule and the MQTT messages that m
 egress_desc.label:
 """Egress Configs"""
 
+egress_pool_size.desc:
+"""Size of the pool of MQTT clients that will publish messages to the remote broker.<br/>
+Each MQTT client will be assigned 'clientid' of the form '${clientid_prefix}:${bridge_name}:egress:${node}:${n}'
+where 'n' is the number of a client inside the pool."""
+
+egress_pool_size.label:
+"""Pool Size"""
+
 egress_local.desc:
 """The configs about receiving messages from local broker."""
 
@@ -75,6 +83,17 @@ ingress_desc.desc:
 ingress_desc.label:
 """Ingress Configs"""
 
+ingress_pool_size.desc:
+"""Size of the pool of MQTT clients that will ingest messages from the remote broker.<br/>
+This value will be respected only if 'remote.topic' is a shared subscription topic or topic-filter
+(for example `$share/name1/topic1` or `$share/name2/topic2/#`), otherwise only a single MQTT client will be used.
+Each MQTT client will be assigned 'clientid' of the form '${clientid_prefix}:${bridge_name}:ingress:${node}:${n}'
+where 'n' is the number of a client inside the pool.
+NOTE: Non-shared subscription will not work well when EMQX is clustered."""
+
+ingress_pool_size.label:
+"""Pool Size"""
+
 ingress_local.desc:
 """The configs about sending message to the local broker."""
 

rel/i18n/emqx_connector_mqtt.hocon

@@ -1,21 +0,0 @@
-emqx_connector_mqtt {
-
-name.desc:
-"""Connector name, used as a human-readable description of the connector."""
-
-name.label:
-"""Connector Name"""
-
-num_of_bridges.desc:
-"""The current number of bridges that are using this connector."""
-
-num_of_bridges.label:
-"""Num of Bridges"""
-
-type.desc:
-"""The Connector Type."""
-
-type.label:
-"""Connector Type"""
-
-}

rel/i18n/zh/emqx_authn_api.hocon

@@ -1,96 +0,0 @@
-emqx_authn_api {
-
-authentication_get.desc:
-"""列出全局认证链上的认证器。"""
-
-authentication_id_delete.desc:
-"""删除全局认证链上的指定认证器。"""
-
-authentication_id_get.desc:
-"""获取全局认证链上的指定认证器。"""
-
-authentication_id_position_put.desc:
-"""更改全局认证链上指定认证器的顺序。"""
-
-authentication_id_put.desc:
-"""更新全局认证链上的指定认证器。"""
-
-authentication_id_status_get.desc:
-"""获取全局认证链上指定认证器的状态。"""
-
-authentication_id_users_get.desc:
-"""获取全局认证链上指定认证器中的用户数据。"""
-
-authentication_id_users_post.desc:
-"""为全局认证链上的指定认证器创建用户数据。"""
-
-authentication_id_users_user_id_delete.desc:
-"""删除全局认证链上指定认证器中的指定用户数据。"""
-
-authentication_id_users_user_id_get.desc:
-"""获取全局认证链上指定认证器中的指定用户数据。"""
-
-authentication_id_users_user_id_put.desc:
-"""更新全局认证链上指定认证器中的指定用户数据。"""
-
-authentication_post.desc:
-"""为全局认证链创建认证器。"""
-
-is_superuser.desc:
-"""是否是超级用户"""
-
-like_user_id.desc:
-"""使用用户 ID （username 或 clientid）模糊查询。"""
-
-like_user_id.label:
-"""like_user_id"""
-
-listeners_listener_id_authentication_get.desc:
-"""列出监听器认证链上的认证器。"""
-
-listeners_listener_id_authentication_id_delete.desc:
-"""删除监听器认证链上的指定认证器。"""
-
-listeners_listener_id_authentication_id_get.desc:
-"""获取监听器认证链上的指定认证器。"""
-
-listeners_listener_id_authentication_id_position_put.desc:
-"""更改监听器认证链上指定认证器的顺序。"""
-
-listeners_listener_id_authentication_id_put.desc:
-"""更新监听器认证链上的指定认证器。"""
-
-listeners_listener_id_authentication_id_status_get.desc:
-"""获取监听器认证链上指定认证器的状态。"""
-
-listeners_listener_id_authentication_id_users_get.desc:
-"""列出监听器认证链上指定认证器中的用户数据。"""
-
-listeners_listener_id_authentication_id_users_post.desc:
-"""为监听器认证链上的指定认证器创建用户数据。"""
-
-listeners_listener_id_authentication_id_users_user_id_delete.desc:
-"""删除监听器认证链上指定认证器中的指定用户数据。"""
-
-listeners_listener_id_authentication_id_users_user_id_get.desc:
-"""获取监听器认证链上指定认证器中的指定用户数据。"""
-
-listeners_listener_id_authentication_id_users_user_id_put.desc:
-"""更新监听器认证链上指定认证器中的指定用户数据。"""
-
-listeners_listener_id_authentication_post.desc:
-"""在监听器认证链上创建认证器。"""
-
-param_auth_id.desc:
-"""认证器 ID。"""
-
-param_listener_id.desc:
-"""监听器 ID。"""
-
-param_position.desc:
-"""认证者在链中的位置。可能的值是 'front', 'rear', 'before:{other_authenticator}', 'after:{other_authenticator}'"""
-
-param_user_id.desc:
-"""用户 ID。"""
-
-}

rel/i18n/zh/emqx_authn_http.hocon

@@ -1,45 +0,0 @@
-emqx_authn_http {
-
-body.desc:
-"""HTTP request body。"""
-
-body.label:
-"""Request Body"""
-
-get.desc:
-"""使用 HTTP Server 作为认证服务的认证器的配置项 (使用 GET 请求)。"""
-
-headers.desc:
-"""HTTP Headers 列表"""
-
-headers.label:
-"""请求头"""
-
-headers_no_content_type.desc:
-"""HTTP Headers 列表 (无 <code>content-type</code>) 。"""
-
-headers_no_content_type.label:
-"""请求头(无 content-type)"""
-
-method.desc:
-"""HTTP 请求方法。"""
-
-method.label:
-"""请求方法"""
-
-post.desc:
-"""使用 HTTP Server 作为认证服务的认证器的配置项 (使用 POST 请求)。"""
-
-request_timeout.desc:
-"""HTTP 请求超时时长。"""
-
-request_timeout.label:
-"""请求超时时间"""
-
-url.desc:
-"""认证 HTTP 服务器地址。"""
-
-url.label:
-"""URL"""
-
-}

rel/i18n/zh/emqx_authn_jwt.hocon

@@ -1,118 +0,0 @@
-emqx_authn_jwt {
-
-acl_claim_name.desc:
-"""JWT claim name to use for getting ACL rules."""
-
-acl_claim_name.label:
-"""ACL claim name"""
-
-algorithm.desc:
-"""JWT 签名算法，支持 HMAC (配置为 <code>hmac-based</code>）和 RSA、ECDSA (配置为 <code>public-key</code>)。"""
-
-algorithm.label:
-"""JWT 签名算法"""
-
-cacertfile.desc:
-"""包含 PEM 编码的 CA 证书的文件的路径。"""
-
-cacertfile.label:
-"""CA 证书文件"""
-
-certfile.desc:
-"""包含用户证书的文件的路径。"""
-
-certfile.label:
-"""证书文件"""
-
-enable.desc:
-"""启用/禁用 SSL。"""
-
-enable.label:
-"""启用/禁用 SSL"""
-
-endpoint.desc:
-"""JWKS 端点， 它是一个以 JWKS 格式返回服务端的公钥集的只读端点。"""
-
-endpoint.label:
-"""JWKS Endpoint"""
-
-from.desc:
-"""要从中获取 JWT 的字段。"""
-
-from.label:
-"""源字段"""
-
-jwt_hmac.desc:
-"""用于认证的 JWT 使用 HMAC 算法签发时的配置。"""
-
-jwt_jwks.desc:
-"""用于认证的 JWTs 需要从 JWKS 端点获取时的配置。"""
-
-keyfile.desc:
-"""包含 PEM 编码的用户私钥的文件的路径。"""
-
-keyfile.label:
-"""私钥文件"""
-
-jwt_public_key.desc:
-"""用于认证的 JWT 使用 RSA 或 ECDSA 算法签发时的配置。"""
-
-public_key.desc:
-"""用于验证 JWT 的公钥。"""
-
-public_key.label:
-"""公钥"""
-
-refresh_interval.desc:
-"""JWKS 刷新间隔。"""
-
-refresh_interval.label:
-"""JWKS 刷新间隔"""
-
-secret.desc:
-"""使用 HMAC 算法时用于验证 JWT 的密钥"""
-
-secret.label:
-"""Secret"""
-
-secret_base64_encoded.desc:
-"""密钥是否为 Base64 编码。"""
-
-secret_base64_encoded.label:
-"""密钥是否为 Base64 编码"""
-
-server_name_indication.desc:
-"""服务器名称指示（SNI）。"""
-
-server_name_indication.label:
-"""服务器名称指示"""
-
-ssl.desc:
-"""SSL 选项。"""
-
-ssl.label:
-"""SSL 选项"""
-
-use_jwks.desc:
-"""是否使用 JWKS。"""
-
-use_jwks.label:
-"""是否使用 JWKS"""
-
-verify.desc:
-"""指定握手过程中是否校验对端证书。"""
-
-verify.label:
-"""Verify"""
-
-verify_claims.desc:
-"""需要验证的自定义声明列表，它是一个名称/值对列表。
-值可以使用以下占位符：
-- <code>${username}</code>: 将在运行时被替换为客户端连接时使用的用户名
-- <code>${clientid}</code>: 将在运行时被替换为客户端连接时使用的客户端标识符
-认证时将验证 JWT（取自 Password 字段）中 claims 的值是否与 <code>verify_claims</code> 中要求的相匹配。"""
-
-verify_claims.label:
-"""Verify Claims"""
-
-}

rel/i18n/zh/emqx_authn_mnesia.hocon

@@ -1,12 +0,0 @@
-emqx_authn_mnesia {
-
-builtin_db.desc:
-"""使用内置数据库作为认证数据源的认证器的配置项。"""
-
-user_id_type.desc:
-"""指定使用客户端ID `clientid` 还是用户名 `username` 进行认证。"""
-
-user_id_type.label:
-"""认证 ID 类型"""
-
-}

rel/i18n/zh/emqx_authn_mongodb.hocon

@@ -1,45 +0,0 @@
-emqx_authn_mongodb {
-
-collection.desc:
-"""存储认证数据的集合。"""
-
-collection.label:
-"""集合"""
-
-filter.desc:
-"""在查询中定义过滤条件的条件表达式。
-过滤器支持如下占位符：
-- <code>${username}</code>: 将在运行时被替换为客户端连接时使用的用户名
-- <code>${clientid}</code>: 将在运行时被替换为客户端连接时使用的客户端标识符"""
-
-filter.label:
-"""过滤器"""
-
-is_superuser_field.desc:
-"""文档中用于定义用户是否具有超级用户权限的字段。"""
-
-is_superuser_field.label:
-"""超级用户字段"""
-
-password_hash_field.desc:
-"""文档中用于存放密码散列的字段。"""
-
-password_hash_field.label:
-"""密码散列字段"""
-
-replica-set.desc:
-"""使用 MongoDB (Replica Set) 作为认证数据源的认证器的配置项。"""
-
-salt_field.desc:
-"""文档中用于存放盐值的字段。"""
-
-salt_field.label:
-"""盐值字段"""
-
-sharded-cluster.desc:
-"""使用 MongoDB (Sharded Cluster) 作为认证数据源的认证器的配置项。"""
-
-single.desc:
-"""使用 MongoDB (Standalone) 作为认证数据源的认证器的配置项。"""
-
-}

rel/i18n/zh/emqx_authn_mysql.hocon

@@ -1,18 +0,0 @@
-emqx_authn_mysql {
-
-mysql.desc:
-"""使用 MySQL 作为认证数据源的认证器的配置项。"""
-
-query.desc:
-"""用于查询密码散列等用于认证的数据的 SQL 语句。"""
-
-query.label:
-"""查询语句"""
-
-query_timeout.desc:
-"""SQL 查询的超时时间。"""
-
-query_timeout.label:
-"""查询超时"""
-
-}

rel/i18n/zh/emqx_authn_pgsql.hocon

@@ -1,12 +0,0 @@
-emqx_authn_pgsql {
-
-postgresql.desc:
-"""使用 PostgreSQL 作为认证数据源的认证器的配置项。"""
-
-query.desc:
-"""用于查询密码散列等用于认证的数据的 SQL 语句。"""
-
-query.label:
-"""查询语句"""
-
-}