Merge remote-tracking branch 'origin/master' into release-51

Makefile

@@ -15,8 +15,8 @@ endif
 
 # Dashbord version
 # from https://github.com/emqx/emqx-dashboard5
-export EMQX_DASHBOARD_VERSION ?= v1.2.5-1
-export EMQX_EE_DASHBOARD_VERSION ?= e1.0.8-beta.1
+export EMQX_DASHBOARD_VERSION ?= v1.2.6-beta.1
+export EMQX_EE_DASHBOARD_VERSION ?= e1.1.0-beta.2
 
 # `:=` should be used here, otherwise the `$(shell ...)` will be executed every time when the variable is used
 # In make 4.4+, for backward-compatibility the value from the original environment is used.

apps/emqx/src/emqx_cm.hrl

@@ -13,9 +13,19 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
+
 -ifndef(EMQX_CM_HRL).
 -define(EMQX_CM_HRL, true).
 
+%% Tables for channel management.
+-define(CHAN_TAB, emqx_channel).
+-define(CHAN_CONN_TAB, emqx_channel_conn).
+-define(CHAN_INFO_TAB, emqx_channel_info).
+-define(CHAN_LIVE_TAB, emqx_channel_live).
+
+%% Mria/Mnesia Tables for channel management.
+-define(CHAN_REG_TAB, emqx_channel_registry).
+
 -define(T_KICK, 5_000).
 -define(T_GET_INFO, 5_000).
 -define(T_TAKEOVER, 15_000).

apps/emqx/include/emqx_mqtt.hrl

@@ -48,6 +48,13 @@
     {?MQTT_PROTO_V5, <<"MQTT">>}
 ]).
 
+%%--------------------------------------------------------------------
+%% MQTT Topic and TopitFilter byte length
+%%--------------------------------------------------------------------
+
+%% MQTT-3.1.1 and MQTT-5.0 [MQTT-4.7.3-3]
+-define(MAX_TOPIC_LEN, 65535).
+
 %%--------------------------------------------------------------------
 %% MQTT QoS Levels
 %%--------------------------------------------------------------------
@@ -662,6 +669,11 @@ end).
     end
 ).
 
+-define(SHARE_EMPTY_FILTER, share_subscription_topic_cannot_be_empty).
+-define(SHARE_EMPTY_GROUP, share_subscription_group_name_cannot_be_empty).
+-define(SHARE_RECURSIVELY, '$share_cannot_be_used_as_real_topic_filter').
+-define(SHARE_NAME_INVALID_CHAR, share_subscription_group_name_cannot_include_wildcard).
+
 -define(FRAME_PARSE_ERROR, frame_parse_error).
 -define(FRAME_SERIALIZE_ERROR, frame_serialize_error).
 -define(THROW_FRAME_ERROR(Reason), erlang:throw({?FRAME_PARSE_ERROR, Reason})).

apps/emqx/include/emqx_release.hrl

@@ -31,11 +31,11 @@
 %% NOTE: ALso make sure to follow the instructions in end of
 %% `apps/emqx/src/bpapi/README.md'
 
-%% Community edition
+%% Opensource edition
 -define(EMQX_RELEASE_CE, "5.1.0-alpha.3").
 
 %% Enterprise edition
 -define(EMQX_RELEASE_EE, "5.1.0-alpha.3").
 
-%% the HTTP API version
+%% The HTTP API version
 -define(EMQX_API_VERSION, "5.0").

apps/emqx/include/emqx_router.hrl

@@ -0,0 +1,31 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-ifndef(EMQX_ROUTER_HRL).
+-define(EMQX_ROUTER_HRL, true).
+
+%% ETS table for message routing
+-define(ROUTE_TAB, emqx_route).
+
+%% Mnesia table for message routing
+-define(ROUTING_NODE, emqx_routing_node).
+
+%% ETS tables for PubSub
+-define(SUBOPTION, emqx_suboption).
+-define(SUBSCRIBER, emqx_subscriber).
+-define(SUBSCRIPTION, emqx_subscription).
+
+-endif.

apps/emqx/src/config/emqx_config_zones.erl

@@ -0,0 +1,35 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_config_zones).
+
+-behaviour(emqx_config_handler).
+
+%% API
+-export([add_handler/0, remove_handler/0, pre_config_update/3]).
+
+-define(ZONES, [zones]).
+
+add_handler() ->
+    ok = emqx_config_handler:add_handler(?ZONES, ?MODULE),
+    ok.
+
+remove_handler() ->
+    ok = emqx_config_handler:remove_handler(?ZONES),
+    ok.
+
+%% replace the old config with the new config
+pre_config_update(?ZONES, NewRaw, _OldRaw) ->
+    {ok, NewRaw}.

apps/emqx/src/emqx_broker.erl

@@ -19,6 +19,8 @@
 -behaviour(gen_server).
 
 -include("emqx.hrl").
+-include("emqx_router.hrl").
+
 -include("logger.hrl").
 -include("types.hrl").
 -include("emqx_mqtt.hrl").
@@ -80,11 +82,6 @@
 
 -define(BROKER, ?MODULE).
 
-%% ETS tables for PubSub
--define(SUBOPTION, emqx_suboption).
--define(SUBSCRIBER, emqx_subscriber).
--define(SUBSCRIPTION, emqx_subscription).
-
 %% Guards
 -define(IS_SUBID(Id), (is_binary(Id) orelse is_atom(Id))).
 
@@ -106,10 +103,10 @@ start_link(Pool, Id) ->
 create_tabs() ->
     TabOpts = [public, {read_concurrency, true}, {write_concurrency, true}],
 
-    %% SubOption: {Topic, SubPid} -> SubOption
+    %% SubOption: {TopicFilter, SubPid} -> SubOption
     ok = emqx_utils_ets:new(?SUBOPTION, [ordered_set | TabOpts]),
 
-    %% Subscription: SubPid -> Topic1, Topic2, Topic3, ...
+    %% Subscription: SubPid -> TopicFilter1, TopicFilter2, TopicFilter3, ...
     %% duplicate_bag: o(1) insert
     ok = emqx_utils_ets:new(?SUBSCRIPTION, [duplicate_bag | TabOpts]),
 

apps/emqx/src/emqx_channel.erl

@@ -484,13 +484,13 @@ handle_in(
             {ok, Channel}
     end;
 handle_in(
-    Packet = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
+    SubPkt = ?SUBSCRIBE_PACKET(PacketId, Properties, TopicFilters),
     Channel = #channel{clientinfo = ClientInfo}
 ) ->
-    case emqx_packet:check(Packet) of
+    case emqx_packet:check(SubPkt) of
         ok ->
             TopicFilters0 = parse_topic_filters(TopicFilters),
-            TopicFilters1 = put_subid_in_subopts(Properties, TopicFilters0),
+            TopicFilters1 = enrich_subopts_subid(Properties, TopicFilters0),
             TupleTopicFilters0 = check_sub_authzs(TopicFilters1, Channel),
             HasAuthzDeny = lists:any(
                 fun({_TopicFilter, ReasonCode}) ->
@@ -503,7 +503,10 @@ handle_in(
                 true ->
                     handle_out(disconnect, ?RC_NOT_AUTHORIZED, Channel);
                 false ->
-                    TopicFilters2 = [TopicFilter || {TopicFilter, 0} <- TupleTopicFilters0],
+                    TopicFilters2 = [
+                        TopicFilter
+                     || {TopicFilter, ?RC_SUCCESS} <- TupleTopicFilters0
+                    ],
                     TopicFilters3 = run_hooks(
                         'client.subscribe',
                         [ClientInfo, Properties],
@@ -1632,10 +1635,9 @@ check_banned(_ConnPkt, #channel{clientinfo = ClientInfo}) ->
 %%--------------------------------------------------------------------
 %% Flapping
 
-count_flapping_event(_ConnPkt, Channel = #channel{clientinfo = ClientInfo = #{zone := Zone}}) ->
-    is_integer(emqx_config:get_zone_conf(Zone, [flapping_detect, window_time])) andalso
-        emqx_flapping:detect(ClientInfo),
-    {ok, Channel}.
+count_flapping_event(_ConnPkt, #channel{clientinfo = ClientInfo}) ->
+    _ = emqx_flapping:detect(ClientInfo),
+    ok.
 
 %%--------------------------------------------------------------------
 %% Authenticate
@@ -1866,6 +1868,9 @@ check_pub_caps(
 %%--------------------------------------------------------------------
 %% Check Sub Authorization
 
+%% TODO: not only check topic filter. Qos chould be checked too.
+%% Not implemented yet:
+%% MQTT-3.1.1 [MQTT-3.8.4-6] and MQTT-5.0 [MQTT-3.8.4-7]
 check_sub_authzs(TopicFilters, Channel) ->
     check_sub_authzs(TopicFilters, Channel, []).
 
@@ -1876,7 +1881,7 @@ check_sub_authzs(
 ) ->
     case emqx_access_control:authorize(ClientInfo, subscribe, Topic) of
         allow ->
-            check_sub_authzs(More, Channel, [{TopicFilter, 0} | Acc]);
+            check_sub_authzs(More, Channel, [{TopicFilter, ?RC_SUCCESS} | Acc]);
         deny ->
             check_sub_authzs(More, Channel, [{TopicFilter, ?RC_NOT_AUTHORIZED} | Acc])
     end;
@@ -1892,9 +1897,9 @@ check_sub_caps(TopicFilter, SubOpts, #channel{clientinfo = ClientInfo}) ->
 %%--------------------------------------------------------------------
 %% Enrich SubId
 
-put_subid_in_subopts(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
+enrich_subopts_subid(#{'Subscription-Identifier' := SubId}, TopicFilters) ->
     [{Topic, SubOpts#{subid => SubId}} || {Topic, SubOpts} <- TopicFilters];
-put_subid_in_subopts(_Properties, TopicFilters) ->
+enrich_subopts_subid(_Properties, TopicFilters) ->
     TopicFilters.
 
 %%--------------------------------------------------------------------

apps/emqx/src/emqx_cm.erl

@@ -20,6 +20,7 @@
 -behaviour(gen_server).
 
 -include("emqx.hrl").
+-include("emqx_cm.hrl").
 -include("logger.hrl").
 -include("types.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -118,14 +119,6 @@
     _Stats :: emqx_types:stats()
 }.
 
--include("emqx_cm.hrl").
-
-%% Tables for channel management.
--define(CHAN_TAB, emqx_channel).
--define(CHAN_CONN_TAB, emqx_channel_conn).
--define(CHAN_INFO_TAB, emqx_channel_info).
--define(CHAN_LIVE_TAB, emqx_channel_live).
-
 -define(CHAN_STATS, [
     {?CHAN_TAB, 'channels.count', 'channels.max'},
     {?CHAN_TAB, 'sessions.count', 'sessions.max'},
@@ -669,12 +662,12 @@ lookup_client({username, Username}) ->
     MatchSpec = [
         {{'_', #{clientinfo => #{username => '$1'}}, '_'}, [{'=:=', '$1', Username}], ['$_']}
     ],
-    ets:select(emqx_channel_info, MatchSpec);
+    ets:select(?CHAN_INFO_TAB, MatchSpec);
 lookup_client({clientid, ClientId}) ->
     [
         Rec
-     || Key <- ets:lookup(emqx_channel, ClientId),
-        Rec <- ets:lookup(emqx_channel_info, Key)
+     || Key <- ets:lookup(?CHAN_TAB, ClientId),
+        Rec <- ets:lookup(?CHAN_INFO_TAB, Key)
     ].
 
 %% @private

apps/emqx/src/emqx_cm_registry.erl

@@ -20,6 +20,7 @@
 -behaviour(gen_server).
 
 -include("emqx.hrl").
+-include("emqx_cm.hrl").
 -include("logger.hrl").
 -include("types.hrl").
 
@@ -50,7 +51,6 @@
 ]).
 
 -define(REGISTRY, ?MODULE).
--define(TAB, emqx_channel_registry).
 -define(LOCK, {?MODULE, cleanup_down}).
 
 -record(channel, {chid, pid}).
@@ -78,7 +78,7 @@ register_channel(ClientId) when is_binary(ClientId) ->
     register_channel({ClientId, self()});
 register_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
     case is_enabled() of
-        true -> mria:dirty_write(?TAB, record(ClientId, ChanPid));
+        true -> mria:dirty_write(?CHAN_REG_TAB, record(ClientId, ChanPid));
         false -> ok
     end.
 
@@ -91,14 +91,14 @@ unregister_channel(ClientId) when is_binary(ClientId) ->
     unregister_channel({ClientId, self()});
 unregister_channel({ClientId, ChanPid}) when is_binary(ClientId), is_pid(ChanPid) ->
     case is_enabled() of
-        true -> mria:dirty_delete_object(?TAB, record(ClientId, ChanPid));
+        true -> mria:dirty_delete_object(?CHAN_REG_TAB, record(ClientId, ChanPid));
         false -> ok
     end.
 
 %% @doc Lookup the global channels.
 -spec lookup_channels(emqx_types:clientid()) -> list(pid()).
 lookup_channels(ClientId) ->
-    [ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(?TAB, ClientId)].
+    [ChanPid || #channel{pid = ChanPid} <- mnesia:dirty_read(?CHAN_REG_TAB, ClientId)].
 
 record(ClientId, ChanPid) ->
     #channel{chid = ClientId, pid = ChanPid}.
@@ -109,7 +109,7 @@ record(ClientId, ChanPid) ->
 
 init([]) ->
     mria_config:set_dirty_shard(?CM_SHARD, true),
-    ok = mria:create_table(?TAB, [
+    ok = mria:create_table(?CHAN_REG_TAB, [
         {type, bag},
         {rlog_shard, ?CM_SHARD},
         {storage, ram_copies},
@@ -166,7 +166,7 @@ cleanup_channels(Node) ->
 
 do_cleanup_channels(Node) ->
     Pat = [{#channel{pid = '$1', _ = '_'}, [{'==', {node, '$1'}, Node}], ['$_']}],
-    lists:foreach(fun delete_channel/1, mnesia:select(?TAB, Pat, write)).
+    lists:foreach(fun delete_channel/1, mnesia:select(?CHAN_REG_TAB, Pat, write)).
 
 delete_channel(Chan) ->
-    mnesia:delete_object(?TAB, Chan, write).
+    mnesia:delete_object(?CHAN_REG_TAB, Chan, write).

apps/emqx/src/emqx_config.erl

@@ -91,7 +91,7 @@
 -export([ensure_atom_conf_path/2]).
 
 -ifdef(TEST).
--export([erase_all/0]).
+-export([erase_all/0, backup_and_write/2]).
 -endif.
 
 -include("logger.hrl").
@@ -105,6 +105,7 @@
 -define(LISTENER_CONF_PATH(TYPE, LISTENER, PATH), [listeners, TYPE, LISTENER | PATH]).
 
 -define(CONFIG_NOT_FOUND_MAGIC, '$0tFound').
+-define(MAX_KEEP_BACKUP_CONFIGS, 10).
 
 -export_type([
     update_request/0,
@@ -597,51 +598,104 @@ save_to_config_map(Conf, RawConf) ->
 -spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}.
 save_to_override_conf(_, undefined, _) ->
     ok;
-%% TODO: Remove deprecated override conf file when 5.1
 save_to_override_conf(true, RawConf, Opts) ->
     case deprecated_conf_file(Opts) of
         undefined ->
             ok;
         FileName ->
-            ok = filelib:ensure_dir(FileName),
-            case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
-                ok ->
-                    ok;
-                {error, Reason} ->
-                    ?SLOG(error, #{
-                        msg => "failed_to_write_override_file",
-                        filename => FileName,
-                        reason => Reason
-                    }),
-                    {error, Reason}
-            end
+            backup_and_write(FileName, hocon_pp:do(RawConf, #{}))
     end;
 save_to_override_conf(false, RawConf, _Opts) ->
     case cluster_hocon_file() of
         undefined ->
             ok;
         FileName ->
-            ok = filelib:ensure_dir(FileName),
-            case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
+            backup_and_write(FileName, hocon_pp:do(RawConf, #{}))
+    end.
+
+%% @private This is the same human-readable timestamp format as
+%% hocon-cli generated app.<time>.config file name.
+now_time() ->
+    Ts = os:system_time(millisecond),
+    {{Y, M, D}, {HH, MM, SS}} = calendar:system_time_to_local_time(Ts, millisecond),
+    Res = io_lib:format(
+        "~0p.~2..0b.~2..0b.~2..0b.~2..0b.~2..0b.~3..0b",
+        [Y, M, D, HH, MM, SS, Ts rem 1000]
+    ),
+    lists:flatten(Res).
+
+%% @private Backup the current config to a file with a timestamp suffix and
+%% then save the new config to the config file.
+backup_and_write(Path, Content) ->
+    %% this may fail, but we don't care
+    %% e.g. read-only file system
+    _ = filelib:ensure_dir(Path),
+    TmpFile = Path ++ ".tmp",
+    case file:write_file(TmpFile, Content) of
+        ok ->
+            backup_and_replace(Path, TmpFile);
+        {error, Reason} ->
+            ?SLOG(error, #{
+                msg => "failed_to_save_conf_file",
+                hint =>
+                    "The updated cluster config is note saved on this node, please check the file system.",
+                filename => TmpFile,
+                reason => Reason
+            }),
+            %% e.g. read-only, it's not the end of the world
+            ok
+    end.
+
+backup_and_replace(Path, TmpPath) ->
+    Backup = Path ++ "." ++ now_time() ++ ".bak",
+    case file:rename(Path, Backup) of
+        ok ->
+            ok = file:rename(TmpPath, Path),
+            ok = prune_backup_files(Path);
+        {error, enoent} ->
+            %% not created yet
+            ok = file:rename(TmpPath, Path);
+        {error, Reason} ->
+            ?SLOG(warning, #{
+                msg => "failed_to_backup_conf_file",
+                filename => Backup,
+                reason => Reason
+            }),
+            ok
+    end.
+
+prune_backup_files(Path) ->
+    Files0 = filelib:wildcard(Path ++ ".*"),
+    Re = "\\.[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{3}\\.bak$",
+    Files = lists:filter(fun(F) -> re:run(F, Re) =/= nomatch end, Files0),
+    Sorted = lists:reverse(lists:sort(Files)),
+    {_Keeps, Deletes} = lists:split(min(?MAX_KEEP_BACKUP_CONFIGS, length(Sorted)), Sorted),
+    lists:foreach(
+        fun(F) ->
+            case file:delete(F) of
                 ok ->
                     ok;
                 {error, Reason} ->
-                    ?SLOG(error, #{
-                        msg => "failed_to_save_conf_file",
-                        filename => FileName,
+                    ?SLOG(warning, #{
+                        msg => "failed_to_delete_backup_conf_file",
+                        filename => F,
                         reason => Reason
                     }),
-                    {error, Reason}
+                    ok
             end
-    end.
+        end,
+        Deletes
+    ).
 
 add_handlers() ->
     ok = emqx_config_logger:add_handler(),
+    ok = emqx_config_zones:add_handler(),
     emqx_sys_mon:add_handler(),
     ok.
 
 remove_handlers() ->
     ok = emqx_config_logger:remove_handler(),
+    ok = emqx_config_zones:remove_handler(),
     emqx_sys_mon:remove_handler(),
     ok.
 
@@ -707,7 +761,10 @@ do_put(Type, Putter, [], DeepValue) ->
 do_put(Type, Putter, [RootName | KeyPath], DeepValue) ->
     OldValue = do_get(Type, [RootName], #{}),
     NewValue = do_deep_put(Type, Putter, KeyPath, OldValue, DeepValue),
-    persistent_term:put(?PERSIS_KEY(Type, RootName), NewValue).
+    Key = ?PERSIS_KEY(Type, RootName),
+    persistent_term:put(Key, NewValue),
+    put_config_post_change_actions(Key, NewValue),
+    ok.
 
 do_deep_get(?CONF, AtomKeyPath, Map, Default) ->
     emqx_utils_maps:deep_get(AtomKeyPath, Map, Default);
@@ -828,16 +885,14 @@ merge_with_global_defaults(GlobalDefaults, ZoneVal) ->
     NewZoneVal :: map().
 maybe_update_zone([zones | T], ZonesValue, Value) ->
     %% note, do not write to PT, return *New value* instead
-    NewZonesValue = emqx_utils_maps:deep_put(T, ZonesValue, Value),
-    ExistingZoneNames = maps:keys(?MODULE:get([zones], #{})),
-    %% Update only new zones with global defaults
     GLD = zone_global_defaults(),
-    maps:fold(
-        fun(ZoneName, ZoneValue, Acc) ->
-            Acc#{ZoneName := merge_with_global_defaults(GLD, ZoneValue)}
+    NewZonesValue0 = emqx_utils_maps:deep_put(T, ZonesValue, Value),
+    NewZonesValue1 = emqx_utils_maps:deep_merge(#{default => GLD}, NewZonesValue0),
+    maps:map(
+        fun(_ZoneName, ZoneValue) ->
+            merge_with_global_defaults(GLD, ZoneValue)
         end,
-        NewZonesValue,
-        maps:without(ExistingZoneNames, NewZonesValue)
+        NewZonesValue1
     );
 maybe_update_zone([RootName | T], RootValue, Value) when is_atom(RootName) ->
     NewRootValue = emqx_utils_maps:deep_put(T, RootValue, Value),
@@ -911,3 +966,12 @@ rawconf_to_conf(SchemaModule, RawPath, RawValue) ->
         ),
     AtomPath = to_atom_conf_path(RawPath, {raise_error, maybe_update_zone_error}),
     emqx_utils_maps:deep_get(AtomPath, RawUserDefinedValues).
+
+%% When the global zone change, the zones is updated with the new global zone.
+%% The global zone's keys is too many,
+%% so we don't choose to write a global zone change emqx_config_handler callback to hook
+put_config_post_change_actions(?PERSIS_KEY(?CONF, zones), _Zones) ->
+    emqx_flapping:update_config(),
+    ok;
+put_config_post_change_actions(_Key, _NewValue) ->
+    ok.

apps/emqx/src/emqx_connection.erl

@@ -49,7 +49,7 @@
 
 -export([
     async_set_keepalive/3,
-    async_set_keepalive/4,
+    async_set_keepalive/5,
     async_set_socket_options/2
 ]).
 
@@ -273,16 +273,30 @@ stats(#state{
 %% NOTE: This API sets TCP socket options, which has nothing to do with
 %%       the MQTT layer's keepalive (PINGREQ and PINGRESP).
 async_set_keepalive(Idle, Interval, Probes) ->
-    async_set_keepalive(self(), Idle, Interval, Probes).
+    async_set_keepalive(os:type(), self(), Idle, Interval, Probes).
 
-async_set_keepalive(Pid, Idle, Interval, Probes) ->
+async_set_keepalive({unix, linux}, Pid, Idle, Interval, Probes) ->
     Options = [
         {keepalive, true},
         {raw, 6, 4, <<Idle:32/native>>},
         {raw, 6, 5, <<Interval:32/native>>},
         {raw, 6, 6, <<Probes:32/native>>}
     ],
-    async_set_socket_options(Pid, Options).
+    async_set_socket_options(Pid, Options);
+async_set_keepalive({unix, darwin}, Pid, Idle, Interval, Probes) ->
+    Options = [
+        {keepalive, true},
+        {raw, 6, 16#10, <<Idle:32/native>>},
+        {raw, 6, 16#101, <<Interval:32/native>>},
+        {raw, 6, 16#102, <<Probes:32/native>>}
+    ],
+    async_set_socket_options(Pid, Options);
+async_set_keepalive(OS, _Pid, _Idle, _Interval, _Probes) ->
+    ?SLOG(warning, #{
+        msg => "Unsupported operation: set TCP keepalive",
+        os => OS
+    }),
+    ok.
 
 %% @doc Set custom socket options.
 %% This API is made async because the call might be originated from
@@ -353,6 +367,9 @@ init_state(
             false -> disabled
         end,
     IdleTimeout = emqx_channel:get_mqtt_conf(Zone, idle_timeout),
+
+    set_tcp_keepalive(Listener),
+
     IdleTimer = start_timer(IdleTimeout, idle_timeout),
     #state{
         transport = Transport,
@@ -948,8 +965,15 @@ handle_cast(
     }
 ) ->
     case Transport:setopts(Socket, Opts) of
-        ok -> ?tp(info, "custom_socket_options_successfully", #{opts => Opts});
-        Err -> ?tp(error, "failed_to_set_custom_socket_optionn", #{reason => Err})
+        ok ->
+            ?tp(debug, "custom_socket_options_successfully", #{opts => Opts});
+        {error, einval} ->
+            %% socket is already closed, ignore this error
+            ?tp(debug, "socket already closed", #{reason => socket_already_closed}),
+            ok;
+        Err ->
+            %% other errors
+            ?tp(error, "failed_to_set_custom_socket_option", #{reason => Err})
     end,
     State;
 handle_cast(Req, State) ->
@@ -1199,6 +1223,19 @@ inc_counter(Key, Inc) ->
     _ = emqx_pd:inc_counter(Key, Inc),
     ok.
 
+set_tcp_keepalive({quic, _Listener}) ->
+    ok;
+set_tcp_keepalive({Type, Id}) ->
+    Conf = emqx_config:get_listener_conf(Type, Id, [tcp_options, keepalive], <<"none">>),
+    case iolist_to_binary(Conf) of
+        <<"none">> ->
+            ok;
+        Value ->
+            %% the value is already validated by schema, so we do not validate it again.
+            {Idle, Interval, Probes} = emqx_schema:parse_tcp_keepalive(Value),
+            async_set_keepalive(Idle, Interval, Probes)
+    end.
+
 %%--------------------------------------------------------------------
 %% For CT tests
 %%--------------------------------------------------------------------

apps/emqx/src/emqx_flapping.erl

@@ -22,13 +22,13 @@
 -include("types.hrl").
 -include("logger.hrl").
 
--export([start_link/0, stop/0]).
+-export([start_link/0, update_config/0, stop/0]).
 
 %% API
 -export([detect/1]).
 
 -ifdef(TEST).
--export([get_policy/2]).
+-export([get_policy/1]).
 -endif.
 
 %% gen_server callbacks
@@ -59,12 +59,17 @@
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
 
+update_config() ->
+    gen_server:cast(?MODULE, update_config).
+
 stop() -> gen_server:stop(?MODULE).
 
 %% @doc Detect flapping when a MQTT client disconnected.
 -spec detect(emqx_types:clientinfo()) -> boolean().
 detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone}) ->
-    Policy = #{max_count := Threshold} = get_policy([max_count, window_time, ban_time], Zone),
+    detect(ClientId, PeerHost, get_policy(Zone)).
+
+detect(ClientId, PeerHost, #{enable := true, max_count := Threshold} = Policy) ->
     %% The initial flapping record sets the detect_cnt to 0.
     InitVal = #flapping{
         clientid = ClientId,
@@ -82,25 +87,22 @@ detect(#{clientid := ClientId, peerhost := PeerHost, zone := Zone}) ->
                 [] ->
                     false
             end
+    end;
+detect(_ClientId, _PeerHost, #{enable := false}) ->
+    false.
+
+get_policy(Zone) ->
+    Flapping = [flapping_detect],
+    case emqx_config:get_zone_conf(Zone, Flapping, undefined) of
+        undefined ->
+            %% If zone has be deleted at running time,
+            %% we don't crash the connection and disable flapping detect.
+            Policy = emqx_config:get(Flapping),
+            Policy#{enable => false};
+        Policy ->
+            Policy
     end.
 
-get_policy(Keys, Zone) when is_list(Keys) ->
-    RootKey = flapping_detect,
-    Conf = emqx_config:get_zone_conf(Zone, [RootKey]),
-    lists:foldl(
-        fun(Key, Acc) ->
-            case maps:find(Key, Conf) of
-                {ok, V} -> Acc#{Key => V};
-                error -> Acc#{Key => emqx_config:get([RootKey, Key])}
-            end
-        end,
-        #{},
-        Keys
-    );
-get_policy(Key, Zone) ->
-    #{Key := Conf} = get_policy([Key], Zone),
-    Conf.
-
 now_diff(TS) -> erlang:system_time(millisecond) - TS.
 
 %%--------------------------------------------------------------------
@@ -115,8 +117,8 @@ init([]) ->
         {read_concurrency, true},
         {write_concurrency, true}
     ]),
-    start_timers(),
-    {ok, #{}, hibernate}.
+    Timers = start_timers(),
+    {ok, Timers, hibernate}.
 
 handle_call(Req, _From, State) ->
     ?SLOG(error, #{msg => "unexpected_call", call => Req}),
@@ -169,17 +171,20 @@ handle_cast(
             )
     end,
     {noreply, State};
+handle_cast(update_config, State) ->
+    NState = update_timer(State),
+    {noreply, NState};
 handle_cast(Msg, State) ->
     ?SLOG(error, #{msg => "unexpected_cast", cast => Msg}),
     {noreply, State}.
 
 handle_info({timeout, _TRef, {garbage_collect, Zone}}, State) ->
-    Timestamp =
-        erlang:system_time(millisecond) - get_policy(window_time, Zone),
+    Policy = #{window_time := WindowTime} = get_policy(Zone),
+    Timestamp = erlang:system_time(millisecond) - WindowTime,
     MatchSpec = [{{'_', '_', '_', '$1', '_'}, [{'<', '$1', Timestamp}], [true]}],
     ets:select_delete(?FLAPPING_TAB, MatchSpec),
-    _ = start_timer(Zone),
-    {noreply, State, hibernate};
+    Timer = start_timer(Policy, Zone),
+    {noreply, State#{Zone => Timer}, hibernate};
 handle_info(Info, State) ->
     ?SLOG(error, #{msg => "unexpected_info", info => Info}),
     {noreply, State}.
@@ -190,18 +195,30 @@ terminate(_Reason, _State) ->
 code_change(_OldVsn, State, _Extra) ->
     {ok, State}.
 
-start_timer(Zone) ->
-    case get_policy(window_time, Zone) of
-        WindowTime when is_integer(WindowTime) ->
-            emqx_utils:start_timer(WindowTime, {garbage_collect, Zone});
-        disabled ->
-            ok
-    end.
+start_timer(#{enable := true, window_time := WindowTime}, Zone) ->
+    emqx_utils:start_timer(WindowTime, {garbage_collect, Zone});
+start_timer(_Policy, _Zone) ->
+    undefined.
 
 start_timers() ->
-    maps:foreach(
-        fun(Zone, _ZoneConf) ->
-            start_timer(Zone)
+    maps:map(
+        fun(ZoneName, #{flapping_detect := FlappingDetect}) ->
+            start_timer(FlappingDetect, ZoneName)
+        end,
+        emqx:get_config([zones], #{})
+    ).
+
+update_timer(Timers) ->
+    maps:map(
+        fun(ZoneName, #{flapping_detect := FlappingDetect = #{enable := Enable}}) ->
+            case maps:get(ZoneName, Timers, undefined) of
+                undefined ->
+                    start_timer(FlappingDetect, ZoneName);
+                TRef when Enable -> TRef;
+                TRef ->
+                    _ = erlang:cancel_timer(TRef),
+                    undefined
+            end
         end,
         emqx:get_config([zones], #{})
     ).

apps/emqx/src/emqx_frame.erl

@@ -300,6 +300,7 @@ parse_connect2(
     ConnPacket = #mqtt_packet_connect{
         proto_name = ProtoName,
         proto_ver = ProtoVer,
+        %% For bridge mode, non-standard implementation
         is_bridge = (BridgeTag =:= 8),
         clean_start = bool(CleanStart),
         will_flag = bool(WillFlag),
@@ -762,6 +763,7 @@ serialize_variable(
     #mqtt_packet_connect{
         proto_name = ProtoName,
         proto_ver = ProtoVer,
+        %% For bridge mode, non-standard implementation
         is_bridge = IsBridge,
         clean_start = CleanStart,
         will_flag = WillFlag,

apps/emqx/src/emqx_hocon.erl

@@ -100,7 +100,16 @@ no_stacktrace(Map) ->
 %% it's maybe too much when reporting to the user
 -spec compact_errors(any(), Stacktrace :: list()) -> {error, any()}.
 compact_errors({SchemaModule, Errors}, Stacktrace) ->
-    compact_errors(SchemaModule, Errors, Stacktrace).
+    compact_errors(SchemaModule, Errors, Stacktrace);
+compact_errors(ErrorContext0, _Stacktrace) when is_map(ErrorContext0) ->
+    case ErrorContext0 of
+        #{exception := #{schema_module := _Mod, message := _Msg} = Detail} ->
+            Error0 = maps:remove(exception, ErrorContext0),
+            Error = maps:merge(Error0, Detail),
+            {error, Error};
+        _ ->
+            {error, ErrorContext0}
+    end.
 
 compact_errors(SchemaModule, [Error0 | More], _Stacktrace) when is_map(Error0) ->
     Error1 =

apps/emqx/src/emqx_listeners.erl

@@ -57,6 +57,7 @@
 ]).
 
 -export([pre_config_update/3, post_config_update/5]).
+-export([create_listener/3, remove_listener/3, update_listener/3]).
 
 -export([format_bind/1]).
 
@@ -65,8 +66,8 @@
 -endif.
 
 -type listener_id() :: atom() | binary().
-
--define(CONF_KEY_PATH, [listeners, '?', '?']).
+-define(ROOT_KEY, listeners).
+-define(CONF_KEY_PATH, [?ROOT_KEY, '?', '?']).
 -define(TYPES_STRING, ["tcp", "ssl", "ws", "wss", "quic"]).
 -define(MARK_DEL, ?TOMBSTONE_CONFIG_CHANGE_REQ).
 
@@ -212,7 +213,10 @@ shutdown_count(_, _, _) ->
 start() ->
     %% The ?MODULE:start/0 will be called by emqx_app when emqx get started,
     %% so we install the config handler here.
+    %% callback when http api request
     ok = emqx_config_handler:add_handler(?CONF_KEY_PATH, ?MODULE),
+    %% callback when reload from config file
+    ok = emqx_config_handler:add_handler([?ROOT_KEY], ?MODULE),
     foreach_listeners(fun start_listener/3).
 
 -spec start_listener(listener_id()) -> ok | {error, term()}.
@@ -287,7 +291,8 @@ restart_listener(Type, ListenerName, OldConf, NewConf) ->
 stop() ->
     %% The ?MODULE:stop/0 will be called by emqx_app when emqx is going to shutdown,
     %% so we uninstall the config handler here.
-    _ = emqx_config_handler:remove_handler(?CONF_KEY_PATH),
+    ok = emqx_config_handler:remove_handler(?CONF_KEY_PATH),
+    ok = emqx_config_handler:remove_handler([?ROOT_KEY]),
     foreach_listeners(fun stop_listener/3).
 
 -spec stop_listener(listener_id()) -> ok | {error, term()}.
@@ -463,48 +468,34 @@ do_start_listener(quic, ListenerName, #{bind := Bind} = Opts) ->
     end.
 
 %% Update the listeners at runtime
-pre_config_update([listeners, Type, Name], {create, NewConf}, V) when
+pre_config_update([?ROOT_KEY, Type, Name], {create, NewConf}, V) when
     V =:= undefined orelse V =:= ?TOMBSTONE_VALUE
 ->
-    CertsDir = certs_dir(Type, Name),
-    {ok, convert_certs(CertsDir, NewConf)};
-pre_config_update([listeners, _Type, _Name], {create, _NewConf}, _RawConf) ->
+    {ok, convert_certs(Type, Name, NewConf)};
+pre_config_update([?ROOT_KEY, _Type, _Name], {create, _NewConf}, _RawConf) ->
     {error, already_exist};
-pre_config_update([listeners, _Type, _Name], {update, _Request}, undefined) ->
+pre_config_update([?ROOT_KEY, _Type, _Name], {update, _Request}, undefined) ->
     {error, not_found};
-pre_config_update([listeners, Type, Name], {update, Request}, RawConf) ->
-    NewConfT = emqx_utils_maps:deep_merge(RawConf, Request),
-    NewConf = ensure_override_limiter_conf(NewConfT, Request),
-    CertsDir = certs_dir(Type, Name),
-    {ok, convert_certs(CertsDir, NewConf)};
-pre_config_update([listeners, _Type, _Name], {action, _Action, Updated}, RawConf) ->
-    NewConf = emqx_utils_maps:deep_merge(RawConf, Updated),
-    {ok, NewConf};
-pre_config_update([listeners, _Type, _Name], ?MARK_DEL, _RawConf) ->
+pre_config_update([?ROOT_KEY, Type, Name], {update, Request}, RawConf) ->
+    RawConf1 = emqx_utils_maps:deep_merge(RawConf, Request),
+    RawConf2 = ensure_override_limiter_conf(RawConf1, Request),
+    {ok, convert_certs(Type, Name, RawConf2)};
+pre_config_update([?ROOT_KEY, _Type, _Name], {action, _Action, Updated}, RawConf) ->
+    {ok, emqx_utils_maps:deep_merge(RawConf, Updated)};
+pre_config_update([?ROOT_KEY, _Type, _Name], ?MARK_DEL, _RawConf) ->
     {ok, ?TOMBSTONE_VALUE};
-pre_config_update(_Path, _Request, RawConf) ->
-    {ok, RawConf}.
-
-post_config_update([listeners, Type, Name], {create, _Request}, NewConf, undefined, _AppEnvs) ->
-    start_listener(Type, Name, NewConf);
-post_config_update([listeners, Type, Name], {update, _Request}, NewConf, OldConf, _AppEnvs) ->
-    ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf),
-    case NewConf of
-        #{enabled := true} -> restart_listener(Type, Name, {OldConf, NewConf});
-        _ -> ok
-    end;
-post_config_update([listeners, Type, Name], Op, _, OldConf, _AppEnvs) when
-    Op =:= ?MARK_DEL andalso is_map(OldConf)
-->
-    ok = unregister_ocsp_stapling_refresh(Type, Name),
-    case stop_listener(Type, Name, OldConf) of
-        ok ->
-            _ = emqx_authentication:delete_chain(listener_id(Type, Name)),
-            ok;
-        Err ->
-            Err
-    end;
-post_config_update([listeners, Type, Name], {action, _Action, _}, NewConf, OldConf, _AppEnvs) ->
+pre_config_update([?ROOT_KEY], RawConf, RawConf) ->
+    {ok, RawConf};
+pre_config_update([?ROOT_KEY], NewConf, _RawConf) ->
+    {ok, convert_certs(NewConf)}.
+
+post_config_update([?ROOT_KEY, Type, Name], {create, _Request}, NewConf, undefined, _AppEnvs) ->
+    create_listener(Type, Name, NewConf);
+post_config_update([?ROOT_KEY, Type, Name], {update, _Request}, NewConf, OldConf, _AppEnvs) ->
+    update_listener(Type, Name, {OldConf, NewConf});
+post_config_update([?ROOT_KEY, Type, Name], ?MARK_DEL, _, OldConf = #{}, _AppEnvs) ->
+    remove_listener(Type, Name, OldConf);
+post_config_update([?ROOT_KEY, Type, Name], {action, _Action, _}, NewConf, OldConf, _AppEnvs) ->
     #{enabled := NewEnabled} = NewConf,
     #{enabled := OldEnabled} = OldConf,
     case {NewEnabled, OldEnabled} of
@@ -521,9 +512,72 @@ post_config_update([listeners, Type, Name], {action, _Action, _}, NewConf, OldCo
             ok = unregister_ocsp_stapling_refresh(Type, Name),
             stop_listener(Type, Name, OldConf)
     end;
+post_config_update([?ROOT_KEY], _Request, OldConf, OldConf, _AppEnvs) ->
+    ok;
+post_config_update([?ROOT_KEY], _Request, NewConf, OldConf, _AppEnvs) ->
+    #{added := Added, removed := Removed, changed := Changed} = diff_confs(NewConf, OldConf),
+    Updated = lists:map(fun({{{T, N}, Old}, {_, New}}) -> {{T, N}, {Old, New}} end, Changed),
+    perform_listener_changes([
+        {fun ?MODULE:remove_listener/3, Removed},
+        {fun ?MODULE:update_listener/3, Updated},
+        {fun ?MODULE:create_listener/3, Added}
+    ]);
 post_config_update(_Path, _Request, _NewConf, _OldConf, _AppEnvs) ->
     ok.
 
+create_listener(Type, Name, NewConf) ->
+    Res = start_listener(Type, Name, NewConf),
+    recreate_authenticators(Res, Type, Name, NewConf).
+
+recreate_authenticators(ok, Type, Name, Conf) ->
+    Chain = listener_id(Type, Name),
+    _ = emqx_authentication:delete_chain(Chain),
+    do_create_authneticators(Chain, maps:get(authentication, Conf, []));
+recreate_authenticators(Error, _Type, _Name, _NewConf) ->
+    Error.
+
+do_create_authneticators(Chain, [AuthN | T]) ->
+    case emqx_authentication:create_authenticator(Chain, AuthN) of
+        {ok, _} ->
+            do_create_authneticators(Chain, T);
+        Error ->
+            _ = emqx_authentication:delete_chain(Chain),
+            Error
+    end;
+do_create_authneticators(_Chain, []) ->
+    ok.
+
+remove_listener(Type, Name, OldConf) ->
+    ok = unregister_ocsp_stapling_refresh(Type, Name),
+    case stop_listener(Type, Name, OldConf) of
+        ok ->
+            _ = emqx_authentication:delete_chain(listener_id(Type, Name)),
+            ok;
+        Err ->
+            Err
+    end.
+
+update_listener(Type, Name, {OldConf, NewConf}) ->
+    ok = maybe_unregister_ocsp_stapling_refresh(Type, Name, NewConf),
+    Res = restart_listener(Type, Name, {OldConf, NewConf}),
+    recreate_authenticators(Res, Type, Name, NewConf).
+
+perform_listener_changes([]) ->
+    ok;
+perform_listener_changes([{Action, ConfL} | Tasks]) ->
+    case perform_listener_changes(Action, ConfL) of
+        ok -> perform_listener_changes(Tasks);
+        {error, Reason} -> {error, Reason}
+    end.
+
+perform_listener_changes(_Action, []) ->
+    ok;
+perform_listener_changes(Action, [{{Type, Name}, Diff} | MapConf]) ->
+    case Action(Type, Name, Diff) of
+        ok -> perform_listener_changes(Action, MapConf);
+        {error, Reason} -> {error, Reason}
+    end.
+
 esockd_opts(ListenerId, Type, Opts0) ->
     Opts1 = maps:with([acceptors, max_connections, proxy_protocol, proxy_protocol_timeout], Opts0),
     Limiter = limiter(Opts0),
@@ -699,6 +753,29 @@ del_limiter_bucket(Id, Conf) ->
             )
     end.
 
+diff_confs(NewConfs, OldConfs) ->
+    emqx_utils:diff_lists(
+        flatten_confs(NewConfs),
+        flatten_confs(OldConfs),
+        fun({Key, _}) -> Key end
+    ).
+
+flatten_confs(Conf0) ->
+    lists:flatmap(
+        fun({Type, Conf}) ->
+            do_flatten_confs(Type, Conf)
+        end,
+        maps:to_list(Conf0)
+    ).
+
+do_flatten_confs(Type, Conf0) ->
+    FilterFun =
+        fun
+            ({_Name, ?TOMBSTONE_TYPE}) -> false;
+            ({Name, Conf}) -> {true, {{Type, Name}, Conf}}
+        end,
+    lists:filtermap(FilterFun, maps:to_list(Conf0)).
+
 enable_authn(Opts) ->
     maps:get(enable_authn, Opts, true).
 
@@ -708,7 +785,7 @@ ssl_opts(Opts) ->
 tcp_opts(Opts) ->
     maps:to_list(
         maps:without(
-            [active_n],
+            [active_n, keepalive],
             maps:get(tcp_options, Opts, #{})
         )
     ).
@@ -760,14 +837,32 @@ parse_bind(#{<<"bind">> := Bind}) ->
 certs_dir(Type, Name) ->
     iolist_to_binary(filename:join(["listeners", Type, Name])).
 
-convert_certs(CertsDir, Conf) ->
+convert_certs(ListenerConf) ->
+    maps:fold(
+        fun(Type, Listeners0, Acc) ->
+            Listeners1 =
+                maps:fold(
+                    fun(Name, Conf, Acc1) ->
+                        Acc1#{Name => convert_certs(Type, Name, Conf)}
+                    end,
+                    #{},
+                    Listeners0
+                ),
+            Acc#{Type => Listeners1}
+        end,
+        #{},
+        ListenerConf
+    ).
+
+convert_certs(Type, Name, Conf) ->
+    CertsDir = certs_dir(Type, Name),
     case emqx_tls_lib:ensure_ssl_files(CertsDir, get_ssl_options(Conf)) of
         {ok, undefined} ->
             Conf;
         {ok, SSL} ->
             Conf#{<<"ssl_options">> => SSL};
         {error, Reason} ->
-            ?SLOG(error, Reason#{msg => "bad_ssl_config"}),
+            ?SLOG(error, Reason#{msg => "bad_ssl_config", type => Type, name => Name}),
             throw({bad_ssl_config, Reason})
     end.
 
@@ -780,13 +875,15 @@ ensure_override_limiter_conf(Conf, #{<<"limiter">> := Limiter}) ->
 ensure_override_limiter_conf(Conf, _) ->
     Conf.
 
-get_ssl_options(Conf) ->
+get_ssl_options(Conf = #{}) ->
     case maps:find(ssl_options, Conf) of
         {ok, SSL} ->
             SSL;
         error ->
             maps:get(<<"ssl_options">>, Conf, undefined)
-    end.
+    end;
+get_ssl_options(_) ->
+    undefined.
 
 %% @doc Get QUIC optional settings for low level tunings.
 %% @see quicer:quic_settings()
@@ -878,8 +975,5 @@ unregister_ocsp_stapling_refresh(Type, Name) ->
     emqx_ocsp_cache:unregister_listener(ListenerId),
     ok.
 
-%% There is currently an issue with frontend
-%% infinity is not a good value for it, so we use 5m for now
 default_max_conn() ->
-    %% TODO: <<"infinity">>
-    5_000_000.
+    <<"infinity">>.

apps/emqx/src/emqx_mqueue.erl

@@ -75,11 +75,10 @@
 
 -export_type([mqueue/0, options/0]).
 
--type topic() :: emqx_types:topic().
 -type priority() :: infinity | integer().
 -type pq() :: emqx_pqueue:q().
 -type count() :: non_neg_integer().
--type p_table() :: ?NO_PRIORITY_TABLE | #{topic() := priority()}.
+-type p_table() :: ?NO_PRIORITY_TABLE | #{emqx_types:topic() := priority()}.
 -type options() :: #{
     max_len := count(),
     priorities => p_table(),

apps/emqx/src/emqx_release.erl

@@ -21,7 +21,10 @@
     edition_vsn_prefix/0,
     edition_longstr/0,
     description/0,
-    version/0
+    version/0,
+    version_with_prefix/0,
+    vsn_compare/1,
+    vsn_compare/2
 ]).
 
 -include("emqx_release.hrl").
@@ -68,6 +71,10 @@ edition_vsn_prefix() ->
 edition_longstr() ->
     maps:get(edition(), ?EMQX_REL_NAME).
 
+%% @doc Return the release version with prefix.
+version_with_prefix() ->
+    edition_vsn_prefix() ++ version().
+
 %% @doc Return the release version.
 version() ->
     case lists:keyfind(emqx_vsn, 1, ?MODULE:module_info(compile)) of
@@ -92,3 +99,47 @@ version() ->
 
 build_vsn() ->
     maps:get(edition(), ?EMQX_REL_VSNS).
+
+%% @doc Compare the given version with the current running version,
+%% return 'newer' 'older' or 'same'.
+vsn_compare("v" ++ Vsn) ->
+    vsn_compare(?EMQX_RELEASE_CE, Vsn);
+vsn_compare("e" ++ Vsn) ->
+    vsn_compare(?EMQX_RELEASE_EE, Vsn).
+
+%% @private Compare the second argument with the first argument, return
+%% 'newer' 'older' or 'same' semver comparison result.
+vsn_compare(Vsn1, Vsn2) ->
+    ParsedVsn1 = parse_vsn(Vsn1),
+    ParsedVsn2 = parse_vsn(Vsn2),
+    case ParsedVsn1 =:= ParsedVsn2 of
+        true ->
+            same;
+        false when ParsedVsn1 < ParsedVsn2 ->
+            newer;
+        false ->
+            older
+    end.
+
+%% @private Parse the version string to a tuple.
+%% Return {{Major, Minor, Patch}, Suffix}.
+%% Where Suffix is either an empty string or a tuple like {"rc", 1}.
+%% NOTE: taking the nature ordering of the suffix:
+%% {"alpha", _} < {"beta", _} < {"rc", _} < ""
+parse_vsn(Vsn) ->
+    try
+        [V1, V2, V3 | Suffix0] = string:tokens(Vsn, ".-"),
+        Suffix =
+            case Suffix0 of
+                "" ->
+                    %% For the case like "5.1.0"
+                    "";
+                [ReleaseStage, Number] ->
+                    %% For the case like "5.1.0-rc.1"
+                    {ReleaseStage, list_to_integer(Number)}
+            end,
+        {{list_to_integer(V1), list_to_integer(V2), list_to_integer(V3)}, Suffix}
+    catch
+        _:_ ->
+            erlang:error({invalid_version_string, Vsn})
+    end.

apps/emqx/src/emqx_router.erl

@@ -22,6 +22,7 @@
 -include("logger.hrl").
 -include("types.hrl").
 -include_lib("mria/include/mria.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).
@@ -69,8 +70,6 @@
 
 -type dest() :: node() | {group(), node()}.
 
--define(ROUTE_TAB, emqx_route).
-
 %%--------------------------------------------------------------------
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------

apps/emqx/src/emqx_router_helper.erl

@@ -19,6 +19,7 @@
 -behaviour(gen_server).
 
 -include("emqx.hrl").
+-include("emqx_router.hrl").
 -include("logger.hrl").
 -include("types.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -54,8 +55,6 @@
 
 -record(routing_node, {name, const = unused}).
 
--define(ROUTE, emqx_route).
--define(ROUTING_NODE, emqx_routing_node).
 -define(LOCK, {?MODULE, cleanup_routes}).
 
 -dialyzer({nowarn_function, [cleanup_routes/1]}).
@@ -185,7 +184,7 @@ code_change(_OldVsn, State, _Extra) ->
 %%--------------------------------------------------------------------
 
 stats_fun() ->
-    case ets:info(?ROUTE, size) of
+    case ets:info(?ROUTE_TAB, size) of
         undefined ->
             ok;
         Size ->
@@ -198,6 +197,6 @@ cleanup_routes(Node) ->
         #route{_ = '_', dest = {'_', Node}}
     ],
     [
-        mnesia:delete_object(?ROUTE, Route, write)
-     || Pat <- Patterns, Route <- mnesia:match_object(?ROUTE, Pat, write)
+        mnesia:delete_object(?ROUTE_TAB, Route, write)
+     || Pat <- Patterns, Route <- mnesia:match_object(?ROUTE_TAB, Pat, write)
     ].

apps/emqx/src/emqx_schema.erl

@@ -33,6 +33,7 @@
 -define(MAX_INT_TIMEOUT_MS, 4294967295).
 %% floor(?MAX_INT_TIMEOUT_MS / 1000).
 -define(MAX_INT_TIMEOUT_S, 4294967).
+-define(DEFAULT_WINDOW_TIME, <<"1m">>).
 
 -type duration() :: integer().
 -type duration_s() :: integer().
@@ -94,7 +95,10 @@
     validate_keepalive_multiplier/1,
     non_empty_string/1,
     validations/0,
-    naive_env_interpolation/1
+    naive_env_interpolation/1,
+    validate_server_ssl_opts/1,
+    validate_tcp_keepalive/1,
+    parse_tcp_keepalive/1
 ]).
 
 -export([qos/0]).
@@ -274,7 +278,10 @@ roots(low) ->
         {"flapping_detect",
             sc(
                 ref("flapping_detect"),
-                #{importance => ?IMPORTANCE_HIDDEN}
+                #{
+                    importance => ?IMPORTANCE_MEDIUM,
+                    converter => fun flapping_detect_converter/2
+                }
             )},
         {"persistent_session_store",
             sc(
@@ -684,15 +691,14 @@ fields("flapping_detect") ->
                 boolean(),
                 #{
                     default => false,
-                    deprecated => {since, "5.0.23"},
                     desc => ?DESC(flapping_detect_enable)
                 }
             )},
         {"window_time",
             sc(
-                hoconsc:union([disabled, duration()]),
+                duration(),
                 #{
-                    default => disabled,
+                    default => ?DEFAULT_WINDOW_TIME,
                     importance => ?IMPORTANCE_HIGH,
                     desc => ?DESC(flapping_detect_window_time)
                 }
@@ -958,7 +964,7 @@ fields("mqtt_wss_listener") ->
             {"ssl_options",
                 sc(
                     ref("listener_wss_opts"),
-                    #{}
+                    #{validator => fun validate_server_ssl_opts/1}
                 )},
             {"websocket",
                 sc(
@@ -1388,6 +1394,15 @@ fields("tcp_opts") ->
                     default => true,
                     desc => ?DESC(fields_tcp_opts_reuseaddr)
                 }
+            )},
+        {"keepalive",
+            sc(
+                string(),
+                #{
+                    default => <<"none">>,
+                    desc => ?DESC(fields_tcp_opts_keepalive),
+                    validator => fun validate_tcp_keepalive/1
+                }
             )}
     ];
 fields("listener_ssl_opts") ->
@@ -2426,8 +2441,21 @@ server_ssl_opts_schema(Defaults, IsRanchListener) ->
             ]
         ].
 
+validate_server_ssl_opts(#{<<"fail_if_no_peer_cert">> := true, <<"verify">> := Verify}) ->
+    validate_verify(Verify);
+validate_server_ssl_opts(#{fail_if_no_peer_cert := true, verify := Verify}) ->
+    validate_verify(Verify);
+validate_server_ssl_opts(_SSLOpts) ->
+    ok.
+
+validate_verify(verify_peer) ->
+    ok;
+validate_verify(_) ->
+    {error, "verify must be verify_peer when fail_if_no_peer_cert is true"}.
+
 mqtt_ssl_listener_ssl_options_validator(Conf) ->
     Checks = [
+        fun validate_server_ssl_opts/1,
         fun ocsp_outer_validator/1,
         fun crl_outer_validator/1
     ],
@@ -2681,7 +2709,11 @@ do_to_timeout_duration(Str, Fn, Max, Unit) ->
                     Msg = lists:flatten(
                         io_lib:format("timeout value too large (max: ~b ~s)", [Max, Unit])
                     ),
-                    throw(Msg)
+                    throw(#{
+                        schema_module => ?MODULE,
+                        message => Msg,
+                        kind => validation_error
+                    })
             end;
         Err ->
             Err
@@ -2828,6 +2860,44 @@ validate_alarm_actions(Actions) ->
         Error -> {error, Error}
     end.
 
+validate_tcp_keepalive(Value) ->
+    case iolist_to_binary(Value) of
+        <<"none">> ->
+            ok;
+        _ ->
+            _ = parse_tcp_keepalive(Value),
+            ok
+    end.
+
+%% @doc This function is used as value validator and also run-time parser.
+parse_tcp_keepalive(Str) ->
+    try
+        [Idle, Interval, Probes] = binary:split(iolist_to_binary(Str), <<",">>, [global]),
+        %% use 10 times the Linux defaults as range limit
+        IdleInt = parse_ka_int(Idle, "Idle", 1, 7200_0),
+        IntervalInt = parse_ka_int(Interval, "Interval", 1, 75_0),
+        ProbesInt = parse_ka_int(Probes, "Probes", 1, 9_0),
+        {IdleInt, IntervalInt, ProbesInt}
+    catch
+        error:_ ->
+            throw(#{
+                reason => "Not comma separated positive integers of 'Idle,Interval,Probes' format",
+                value => Str
+            })
+    end.
+
+parse_ka_int(Bin, Name, Min, Max) ->
+    I = binary_to_integer(string:trim(Bin)),
+    case I >= Min andalso I =< Max of
+        true ->
+            I;
+        false ->
+            Msg = io_lib:format("TCP-Keepalive '~s' value must be in the rage of [~p, ~p].", [
+                Name, Min, Max
+            ]),
+            throw(#{reason => lists:flatten(Msg), value => I})
+    end.
+
 user_lookup_fun_tr(Lookup, #{make_serializable := true}) ->
     fmt_user_lookup_fun(Lookup);
 user_lookup_fun_tr(Lookup, _) ->
@@ -3482,3 +3552,9 @@ mqtt_converter(#{<<"keepalive_backoff">> := Backoff} = Mqtt, _Opts) ->
     Mqtt#{<<"keepalive_multiplier">> => Backoff * 2};
 mqtt_converter(Mqtt, _Opts) ->
     Mqtt.
+
+%% For backward compatibility with window_time is disable
+flapping_detect_converter(Conf = #{<<"window_time">> := <<"disable">>}, _Opts) ->
+    Conf#{<<"window_time">> => ?DEFAULT_WINDOW_TIME, <<"enable">> => false};
+flapping_detect_converter(Conf, _Opts) ->
+    Conf.

apps/emqx/src/emqx_session_router.erl

@@ -57,9 +57,7 @@
     code_change/3
 ]).
 
--type group() :: binary().
-
--type dest() :: node() | {group(), node()}.
+-type dest() :: node() | {emqx_types:group(), node()}.
 
 -define(ROUTE_RAM_TAB, emqx_session_route_ram).
 -define(ROUTE_DISC_TAB, emqx_session_route_disc).
@@ -114,7 +112,7 @@ start_link(Pool, Id) ->
 %% Route APIs
 %%--------------------------------------------------------------------
 
--spec do_add_route(emqx_topic:topic(), dest()) -> ok | {error, term()}.
+-spec do_add_route(emqx_types:topic(), dest()) -> ok | {error, term()}.
 do_add_route(Topic, SessionID) when is_binary(Topic) ->
     Route = #route{topic = Topic, dest = SessionID},
     case lists:member(Route, lookup_routes(Topic)) of
@@ -135,7 +133,7 @@ do_add_route(Topic, SessionID) when is_binary(Topic) ->
     end.
 
 %% @doc Match routes
--spec match_routes(emqx_topic:topic()) -> [emqx_types:route()].
+-spec match_routes(emqx_types:topic()) -> [emqx_types:route()].
 match_routes(Topic) when is_binary(Topic) ->
     case match_trie(Topic) of
         [] -> lookup_routes(Topic);
@@ -153,7 +151,7 @@ match_trie(Topic) ->
 delete_routes(SessionID, Subscriptions) ->
     cast(pick(SessionID), {delete_routes, SessionID, Subscriptions}).
 
--spec do_delete_route(emqx_topic:topic(), dest()) -> ok | {error, term()}.
+-spec do_delete_route(emqx_types:topic(), dest()) -> ok | {error, term()}.
 do_delete_route(Topic, SessionID) ->
     Route = #route{topic = Topic, dest = SessionID},
     case emqx_topic:wildcard(Topic) of
@@ -165,7 +163,7 @@ do_delete_route(Topic, SessionID) ->
     end.
 
 %% @doc Print routes to a topic
--spec print_routes(emqx_topic:topic()) -> ok.
+-spec print_routes(emqx_types:topic()) -> ok.
 print_routes(Topic) ->
     lists:foreach(
         fun(#route{topic = To, dest = SessionID}) ->

apps/emqx/src/emqx_shared_sub.erl

@@ -97,7 +97,7 @@
 -define(REDISPATCH_TO(GROUP, TOPIC), {GROUP, TOPIC}).
 -define(SUBSCRIBER_DOWN, noproc).
 
--type redispatch_to() :: ?REDISPATCH_TO(emqx_topic:group(), emqx_topic:topic()).
+-type redispatch_to() :: ?REDISPATCH_TO(emqx_types:group(), emqx_types:topic()).
 
 -record(state, {pmon}).
 
@@ -156,7 +156,7 @@ dispatch(Group, Topic, Delivery = #delivery{message = Msg}, FailedSubs) ->
             end
     end.
 
--spec strategy(emqx_topic:group()) -> strategy().
+-spec strategy(emqx_types:group()) -> strategy().
 strategy(Group) ->
     try
         emqx:get_config([

apps/emqx/src/emqx_topic.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_topic).
 
+-include("emqx_mqtt.hrl").
+
 %% APIs
 -export([
     match/2,
@@ -33,18 +35,14 @@
     parse/2
 ]).
 
--export_type([
-    group/0,
-    topic/0,
-    word/0
-]).
-
--type group() :: binary().
--type topic() :: binary().
--type word() :: '' | '+' | '#' | binary().
--type words() :: list(word()).
+-type topic() :: emqx_types:topic().
+-type word() :: emqx_types:word().
+-type words() :: emqx_types:words().
 
--define(MAX_TOPIC_LEN, 65535).
+%% Guards
+-define(MULTI_LEVEL_WILDCARD_NOT_LAST(C, REST),
+    ((C =:= '#' orelse C =:= <<"#">>) andalso REST =/= [])
+).
 
 %%--------------------------------------------------------------------
 %% APIs
@@ -97,11 +95,15 @@ validate({Type, Topic}) when Type =:= name; Type =:= filter ->
 
 -spec validate(name | filter, topic()) -> true.
 validate(_, <<>>) ->
+    %% MQTT-5.0 [MQTT-4.7.3-1]
     error(empty_topic);
 validate(_, Topic) when is_binary(Topic) andalso (size(Topic) > ?MAX_TOPIC_LEN) ->
+    %% MQTT-5.0 [MQTT-4.7.3-3]
     error(topic_too_long);
-validate(filter, Topic) when is_binary(Topic) ->
-    validate2(words(Topic));
+validate(filter, SharedFilter = <<"$share/", _Rest/binary>>) ->
+    validate_share(SharedFilter);
+validate(filter, Filter) when is_binary(Filter) ->
+    validate2(words(Filter));
 validate(name, Topic) when is_binary(Topic) ->
     Words = words(Topic),
     validate2(Words) andalso
@@ -113,7 +115,8 @@ validate2([]) ->
 % end with '#'
 validate2(['#']) ->
     true;
-validate2(['#' | Words]) when length(Words) > 0 ->
+%% MQTT-5.0 [MQTT-4.7.1-1]
+validate2([C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
     error('topic_invalid_#');
 validate2(['' | Words]) ->
     validate2(Words);
@@ -129,6 +132,32 @@ validate3(<<C/utf8, _Rest/binary>>) when C == $#; C == $+; C == 0 ->
 validate3(<<_/utf8, Rest/binary>>) ->
     validate3(Rest).
 
+validate_share(<<"$share/", Rest/binary>>) when
+    Rest =:= <<>> orelse Rest =:= <<"/">>
+->
+    %% MQTT-5.0 [MQTT-4.8.2-1]
+    error(?SHARE_EMPTY_FILTER);
+validate_share(<<"$share/", Rest/binary>>) ->
+    case binary:split(Rest, <<"/">>) of
+        %% MQTT-5.0 [MQTT-4.8.2-1]
+        [<<>>, _] ->
+            error(?SHARE_EMPTY_GROUP);
+        %% MQTT-5.0 [MQTT-4.7.3-1]
+        [_, <<>>] ->
+            error(?SHARE_EMPTY_FILTER);
+        [ShareName, Filter] ->
+            validate_share(ShareName, Filter)
+    end.
+
+validate_share(_, <<"$share/", _Rest/binary>>) ->
+    error(?SHARE_RECURSIVELY);
+validate_share(ShareName, Filter) ->
+    case binary:match(ShareName, [<<"+">>, <<"#">>]) of
+        %% MQTT-5.0 [MQTT-4.8.2-2]
+        nomatch -> validate2(words(Filter));
+        _ -> error(?SHARE_NAME_INVALID_CHAR)
+    end.
+
 %% @doc Prepend a topic prefix.
 %% Ensured to have only one / between prefix and suffix.
 prepend(undefined, W) ->
@@ -142,6 +171,7 @@ prepend(Parent0, W) ->
         _ -> <<Parent/binary, $/, (bin(W))/binary>>
     end.
 
+-spec bin(word()) -> binary().
 bin('') -> <<>>;
 bin('+') -> <<"+">>;
 bin('#') -> <<"#">>;
@@ -163,6 +193,7 @@ tokens(Topic) ->
 words(Topic) when is_binary(Topic) ->
     [word(W) || W <- tokens(Topic)].
 
+-spec word(binary()) -> word().
 word(<<>>) -> '';
 word(<<"+">>) -> '+';
 word(<<"#">>) -> '#';
@@ -185,23 +216,19 @@ feed_var(Var, Val, [Var | Words], Acc) ->
 feed_var(Var, Val, [W | Words], Acc) ->
     feed_var(Var, Val, Words, [W | Acc]).
 
--spec join(list(binary())) -> binary().
+-spec join(list(word())) -> binary().
 join([]) ->
     <<>>;
-join([W]) ->
-    bin(W);
-join(Words) ->
-    {_, Bin} = lists:foldr(
-        fun
-            (W, {true, Tail}) ->
-                {false, <<W/binary, Tail/binary>>};
-            (W, {false, Tail}) ->
-                {false, <<W/binary, "/", Tail/binary>>}
-        end,
-        {true, <<>>},
-        [bin(W) || W <- Words]
-    ),
-    Bin.
+join([Word | Words]) ->
+    do_join(bin(Word), Words).
+
+do_join(TopicAcc, []) ->
+    TopicAcc;
+%% MQTT-5.0 [MQTT-4.7.1-1]
+do_join(_TopicAcc, [C | Words]) when ?MULTI_LEVEL_WILDCARD_NOT_LAST(C, Words) ->
+    error('topic_invalid_#');
+do_join(TopicAcc, [Word | Words]) ->
+    do_join(<<TopicAcc/binary, "/", (bin(Word))/binary>>, Words).
 
 -spec parse(topic() | {topic(), map()}) -> {topic(), #{share => binary()}}.
 parse(TopicFilter) when is_binary(TopicFilter) ->

apps/emqx/src/emqx_trie.erl

@@ -114,7 +114,7 @@ create_session_trie(Type) ->
 insert(Topic) when is_binary(Topic) ->
     insert(Topic, ?TRIE).
 
--spec insert_session(emqx_topic:topic()) -> ok.
+-spec insert_session(emqx_types:topic()) -> ok.
 insert_session(Topic) when is_binary(Topic) ->
     insert(Topic, session_trie()).
 
@@ -132,7 +132,7 @@ delete(Topic) when is_binary(Topic) ->
     delete(Topic, ?TRIE).
 
 %% @doc Delete a topic filter from the trie.
--spec delete_session(emqx_topic:topic()) -> ok.
+-spec delete_session(emqx_types:topic()) -> ok.
 delete_session(Topic) when is_binary(Topic) ->
     delete(Topic, session_trie()).
 
@@ -148,7 +148,7 @@ delete(Topic, Trie) when is_binary(Topic) ->
 match(Topic) when is_binary(Topic) ->
     match(Topic, ?TRIE).
 
--spec match_session(emqx_topic:topic()) -> list(emqx_topic:topic()).
+-spec match_session(emqx_types:topic()) -> list(emqx_types:topic()).
 match_session(Topic) when is_binary(Topic) ->
     match(Topic, session_trie()).
 

apps/emqx/src/emqx_types.erl

@@ -29,10 +29,16 @@
 -export_type([
     zone/0,
     pubsub/0,
-    topic/0,
     subid/0
 ]).
 
+-export_type([
+    group/0,
+    topic/0,
+    word/0,
+    words/0
+]).
+
 -export_type([
     socktype/0,
     sockstate/0,
@@ -122,9 +128,13 @@
 
 -type zone() :: atom().
 -type pubsub() :: publish | subscribe.
--type topic() :: emqx_topic:topic().
 -type subid() :: binary() | atom().
 
+-type group() :: binary() | undefined.
+-type topic() :: binary().
+-type word() :: '' | '+' | '#' | binary().
+-type words() :: list(word()).
+
 -type socktype() :: tcp | udp | ssl | proxy | atom().
 -type sockstate() :: idle | running | blocked | closed.
 -type conninfo() :: #{
@@ -230,7 +240,6 @@
     | {share, topic(), deliver_result()}
 ].
 -type route() :: #route{}.
--type group() :: emqx_topic:group().
 -type route_entry() :: {topic(), node()} | {topic, group()}.
 -type command() :: #command{}.
 

apps/emqx/src/emqx_ws_connection.erl

@@ -18,6 +18,7 @@
 -module(emqx_ws_connection).
 
 -include("emqx.hrl").
+-include("emqx_cm.hrl").
 -include("emqx_mqtt.hrl").
 -include("logger.hrl").
 -include("types.hrl").
@@ -1034,7 +1035,7 @@ check_max_connection(Type, Listener) ->
             allow;
         Max ->
             MatchSpec = [{{'_', emqx_ws_connection}, [], [true]}],
-            Curr = ets:select_count(emqx_channel_conn, MatchSpec),
+            Curr = ets:select_count(?CHAN_CONN_TAB, MatchSpec),
             case Curr >= Max of
                 false ->
                     allow;

apps/emqx/src/emqx_zone_schema.erl

@@ -58,8 +58,7 @@ hidden() ->
     [
         "stats",
         "overload_protection",
-        "conn_congestion",
-        "flapping_detect"
+        "conn_congestion"
     ].
 
 %% zone schemas are clones from the same name from root level

apps/emqx/src/proto/emqx_cm_proto_v1.erl

@@ -33,7 +33,7 @@
 ]).
 
 -include("bpapi.hrl").
--include("src/emqx_cm.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 
 introduced_in() ->
     "5.0.0".

apps/emqx/src/proto/emqx_cm_proto_v2.erl

@@ -34,7 +34,7 @@
 ]).
 
 -include("bpapi.hrl").
--include("src/emqx_cm.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 
 introduced_in() ->
     "5.0.0".

apps/emqx/test/emqx_cm_SUITE.erl

@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
@@ -200,10 +201,10 @@ t_open_session_race_condition(_) ->
         end,
     Winner = WaitForDowns(Pids),
 
-    ?assertMatch([_], ets:lookup(emqx_channel, ClientId)),
+    ?assertMatch([_], ets:lookup(?CHAN_TAB, ClientId)),
     ?assertEqual([Winner], emqx_cm:lookup_channels(ClientId)),
-    ?assertMatch([_], ets:lookup(emqx_channel_conn, {ClientId, Winner})),
-    ?assertMatch([_], ets:lookup(emqx_channel_registry, ClientId)),
+    ?assertMatch([_], ets:lookup(?CHAN_CONN_TAB, {ClientId, Winner})),
+    ?assertMatch([_], ets:lookup(?CHAN_REG_TAB, ClientId)),
 
     exit(Winner, kill),
     receive

apps/emqx/test/emqx_config_SUITE.erl

@@ -32,7 +32,23 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     emqx_common_test_helpers:stop_apps([]).
 
-t_fill_default_values(_) ->
+init_per_testcase(TestCase, Config) ->
+    try
+        ?MODULE:TestCase({init, Config})
+    catch
+        error:function_clause ->
+            Config
+    end.
+
+end_per_testcase(TestCase, Config) ->
+    try
+        ?MODULE:TestCase({'end', Config})
+    catch
+        error:function_clause ->
+            ok
+    end.
+
+t_fill_default_values(C) when is_list(C) ->
     Conf = #{
         <<"broker">> => #{
             <<"perf">> => #{},
@@ -61,7 +77,7 @@ t_fill_default_values(_) ->
     _ = emqx_utils_json:encode(WithDefaults),
     ok.
 
-t_init_load(_Config) ->
+t_init_load(C) when is_list(C) ->
     ConfFile = "./test_emqx.conf",
     ok = file:write_file(ConfFile, <<"">>),
     ExpectRootNames = lists:sort(hocon_schema:root_names(emqx_schema)),
@@ -80,7 +96,7 @@ t_init_load(_Config) ->
     ?assertMatch({ok, #{raw_config := 128}}, emqx:update_config([mqtt, max_topic_levels], 128)),
     ok = file:delete(DeprecatedFile).
 
-t_unknown_rook_keys(_) ->
+t_unknown_root_keys(C) when is_list(C) ->
     ?check_trace(
         #{timetrap => 1000},
         begin
@@ -98,7 +114,50 @@ t_unknown_rook_keys(_) ->
     ),
     ok.
 
-t_init_load_emqx_schema(Config) ->
+t_cluster_hocon_backup({init, C}) ->
+    C;
+t_cluster_hocon_backup({'end', _C}) ->
+    File = "backup-test.hocon",
+    Files = [File | filelib:wildcard(File ++ ".*.bak")],
+    lists:foreach(fun file:delete/1, Files);
+t_cluster_hocon_backup(C) when is_list(C) ->
+    Write = fun(Path, Content) ->
+        %% avoid name clash
+        timer:sleep(1),
+        emqx_config:backup_and_write(Path, Content)
+    end,
+    File = "backup-test.hocon",
+    %% write 12 times, 10 backups should be kept
+    %% the latest one is File itself without suffix
+    %% the oldest one is expected to be deleted
+    N = 12,
+    Inputs = lists:seq(1, N),
+    Backups = lists:seq(N - 10, N - 1),
+    InputContents = [integer_to_binary(I) || I <- Inputs],
+    BackupContents = [integer_to_binary(I) || I <- Backups],
+    lists:foreach(
+        fun(Content) ->
+            Write(File, Content)
+        end,
+        InputContents
+    ),
+    LatestContent = integer_to_binary(N),
+    ?assertEqual({ok, LatestContent}, file:read_file(File)),
+    Re = "\\.[0-9]{4}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{2}\\.[0-9]{3}\\.bak$",
+    Files = filelib:wildcard(File ++ ".*.bak"),
+    ?assert(lists:all(fun(F) -> re:run(F, Re) =/= nomatch end, Files)),
+    %% keep only the latest 10
+    ?assertEqual(10, length(Files)),
+    FilesSorted = lists:zip(lists:sort(Files), BackupContents),
+    lists:foreach(
+        fun({BackupFile, ExpectedContent}) ->
+            ?assertEqual({ok, ExpectedContent}, file:read_file(BackupFile))
+        end,
+        FilesSorted
+    ),
+    ok.
+
+t_init_load_emqx_schema(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given empty config file
     ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@@ -127,7 +186,7 @@ t_init_load_emqx_schema(Config) ->
         Default
     ).
 
-t_init_zones_load_emqx_schema_no_default_for_none_existing(Config) ->
+t_init_zones_load_emqx_schema_no_default_for_none_existing(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given empty config file
     ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@@ -140,7 +199,7 @@ t_init_zones_load_emqx_schema_no_default_for_none_existing(Config) ->
         emqx_config:get([zones, no_exists])
     ).
 
-t_init_zones_load_other_schema(Config) ->
+t_init_zones_load_other_schema(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given empty config file
     ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@@ -159,7 +218,7 @@ t_init_zones_load_other_schema(Config) ->
         emqx_config:get([zones, default])
     ).
 
-t_init_zones_with_user_defined_default_zone(Config) ->
+t_init_zones_with_user_defined_default_zone(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given user defined config for default zone
     ConfFile = prepare_conf_file(
@@ -176,7 +235,7 @@ t_init_zones_with_user_defined_default_zone(Config) ->
     %% Then others are defaults
     ?assertEqual(ExpectedOthers, Others).
 
-t_init_zones_with_user_defined_other_zone(Config) ->
+t_init_zones_with_user_defined_other_zone(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given user defined config for default zone
     ConfFile = prepare_conf_file(
@@ -196,7 +255,7 @@ t_init_zones_with_user_defined_other_zone(Config) ->
     %% Then default zone still have the defaults
     ?assertEqual(zone_global_defaults(), emqx_config:get([zones, default])).
 
-t_init_zones_with_cust_root_mqtt(Config) ->
+t_init_zones_with_cust_root_mqtt(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given config file with mqtt user overrides
     ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"mqtt.retry_interval=10m">>, Config),
@@ -211,7 +270,7 @@ t_init_zones_with_cust_root_mqtt(Config) ->
         emqx_config:get([zones, default, mqtt])
     ).
 
-t_default_zone_is_updated_after_global_defaults_updated(Config) ->
+t_default_zone_is_updated_after_global_defaults_updated(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given empty emqx conf
     ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@@ -227,7 +286,7 @@ t_default_zone_is_updated_after_global_defaults_updated(Config) ->
         emqx_config:get([zones, default, mqtt])
     ).
 
-t_myzone_is_updated_after_global_defaults_updated(Config) ->
+t_myzone_is_updated_after_global_defaults_updated(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given emqx conf file with user override in myzone (none default zone)
     ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"zones.myzone.mqtt.max_inflight=32">>, Config),
@@ -251,7 +310,7 @@ t_myzone_is_updated_after_global_defaults_updated(Config) ->
         emqx_config:get([zones, default, mqtt])
     ).
 
-t_zone_no_user_defined_overrides(Config) ->
+t_zone_no_user_defined_overrides(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given emqx conf file with user specified myzone
     ConfFile = prepare_conf_file(
@@ -268,7 +327,7 @@ t_zone_no_user_defined_overrides(Config) ->
     %% Then user defined value from config is not overwritten
     ?assertMatch(600000, emqx_config:get([zones, myzone, mqtt, retry_interval])).
 
-t_zone_no_user_defined_overrides_internal_represent(Config) ->
+t_zone_no_user_defined_overrides_internal_represent(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given emqx conf file with user specified myzone
     ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"zones.myzone.mqtt.max_inflight=1">>, Config),
@@ -281,7 +340,7 @@ t_zone_no_user_defined_overrides_internal_represent(Config) ->
     ?assertMatch(2, emqx_config:get([zones, default, mqtt, max_inflight])),
     ?assertMatch(1, emqx_config:get([zones, myzone, mqtt, max_inflight])).
 
-t_update_global_defaults_no_updates_on_user_overrides(Config) ->
+t_update_global_defaults_no_updates_on_user_overrides(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given default zone config in conf file.
     ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"zones.default.mqtt.max_inflight=1">>, Config),
@@ -293,7 +352,7 @@ t_update_global_defaults_no_updates_on_user_overrides(Config) ->
     %% Then the value is not reflected in default `zone'
     ?assertMatch(1, emqx_config:get([zones, default, mqtt, max_inflight])).
 
-t_zone_update_with_new_zone(Config) ->
+t_zone_update_with_new_zone(Config) when is_list(Config) ->
     emqx_config:erase_all(),
     %% Given loaded an empty conf file
     ConfFile = prepare_conf_file(?FUNCTION_NAME, <<"">>, Config),
@@ -308,7 +367,7 @@ t_zone_update_with_new_zone(Config) ->
         emqx_config:get([zones, myzone, mqtt])
     ).
 
-t_init_zone_with_global_defaults(_Config) ->
+t_init_zone_with_global_defaults(Config) when is_list(Config) ->
     %% Given uninitialized empty config
     emqx_config:erase_all(),
     Zones = #{myzone => #{mqtt => #{max_inflight => 3}}},
@@ -344,7 +403,7 @@ zone_global_defaults() ->
         conn_congestion =>
             #{enable_alarm => true, min_alarm_sustain_duration => 60000},
         flapping_detect =>
-            #{ban_time => 300000, max_count => 15, window_time => disabled},
+            #{ban_time => 300000, max_count => 15, window_time => 60000, enable => false},
         force_gc =>
             #{bytes => 16777216, count => 16000, enable => true},
         force_shutdown =>

apps/emqx/test/emqx_flapping_SUITE.erl

@@ -26,15 +26,16 @@ all() -> emqx_common_test_helpers:all(?MODULE).
 init_per_suite(Config) ->
     emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
-    emqx_config:put_zone_conf(
-        default,
+    %% update global default config
+    {ok, _} = emqx:update_config(
         [flapping_detect],
         #{
-            max_count => 3,
+            <<"enable">> => true,
+            <<"max_count">> => 3,
             % 0.1s
-            window_time => 100,
+            <<"window_time">> => 100,
             %% 2s
-            ban_time => 2000
+            <<"ban_time">> => "2s"
         }
     ),
     Config.
@@ -102,20 +103,73 @@ t_expired_detecting(_) ->
         )
     ).
 
-t_conf_without_window_time(_) ->
-    %% enable is deprecated, so we need to make sure it won't be used.
+t_conf_update(_) ->
     Global = emqx_config:get([flapping_detect]),
-    ?assertNot(maps:is_key(enable, Global)),
-    %% zones don't have default value, so we need to make sure fallback to global conf.
-    %% this new_zone will fallback to global conf.
+    #{
+        ban_time := _BanTime,
+        enable := _Enable,
+        max_count := _MaxCount,
+        window_time := _WindowTime
+    } = Global,
+
     emqx_config:put_zone_conf(new_zone, [flapping_detect], #{}),
     ?assertEqual(Global, get_policy(new_zone)),
 
-    emqx_config:put_zone_conf(new_zone_1, [flapping_detect], #{window_time => 100}),
-    ?assertEqual(100, emqx_flapping:get_policy(window_time, new_zone_1)),
-    ?assertEqual(maps:get(ban_time, Global), emqx_flapping:get_policy(ban_time, new_zone_1)),
-    ?assertEqual(maps:get(max_count, Global), emqx_flapping:get_policy(max_count, new_zone_1)),
+    emqx_config:put_zone_conf(zone_1, [flapping_detect], #{window_time => 100}),
+    ?assertEqual(Global#{window_time := 100}, emqx_flapping:get_policy(zone_1)),
+
+    Zones = #{
+        <<"zone_1">> => #{<<"flapping_detect">> => #{<<"window_time">> => 123}},
+        <<"zone_2">> => #{<<"flapping_detect">> => #{<<"window_time">> => 456}}
+    },
+    ?assertMatch({ok, _}, emqx:update_config([zones], Zones)),
+    %% new_zone is already deleted
+    ?assertError({config_not_found, _}, get_policy(new_zone)),
+    %% update zone(zone_1) has default.
+    ?assertEqual(Global#{window_time := 123}, emqx_flapping:get_policy(zone_1)),
+    %% create zone(zone_2) has default
+    ?assertEqual(Global#{window_time := 456}, emqx_flapping:get_policy(zone_2)),
+    %% reset to default(empty) andalso get default from global
+    ?assertMatch({ok, _}, emqx:update_config([zones], #{})),
+    ?assertEqual(Global, emqx:get_config([zones, default, flapping_detect])),
+    ?assertError({config_not_found, _}, get_policy(zone_1)),
+    ?assertError({config_not_found, _}, get_policy(zone_2)),
+    ok.
+
+t_conf_update_timer(_Config) ->
+    _ = emqx_flapping:start_link(),
+    validate_timer([default]),
+    {ok, _} =
+        emqx:update_config([zones], #{
+            <<"timer_1">> => #{<<"flapping_detect">> => #{<<"enable">> => true}},
+            <<"timer_2">> => #{<<"flapping_detect">> => #{<<"enable">> => true}},
+            <<"timer_3">> => #{<<"flapping_detect">> => #{<<"enable">> => false}}
+        }),
+    validate_timer([timer_1, timer_2, timer_3, default]),
+    ok.
+
+validate_timer(Names) ->
+    Zones = emqx:get_config([zones]),
+    ?assertEqual(lists:sort(Names), lists:sort(maps:keys(Zones))),
+    Timers = sys:get_state(emqx_flapping),
+    maps:foreach(
+        fun(Name, #{flapping_detect := #{enable := Enable}}) ->
+            ?assertEqual(Enable, is_reference(maps:get(Name, Timers)), Timers)
+        end,
+        Zones
+    ),
+    ?assertEqual(maps:keys(Zones), maps:keys(Timers)),
+    ok.
+
+t_window_compatibility_check(_Conf) ->
+    Flapping = emqx:get_config([flapping_detect]),
+    ok = emqx_config:init_load(emqx_schema, <<"flapping_detect {window_time = disable}">>),
+    ?assertMatch(#{window_time := 60000, enable := false}, emqx:get_config([flapping_detect])),
+    %% reset
+    FlappingBin = iolist_to_binary(["flapping_detect {", hocon_pp:do(Flapping, #{}), "}"]),
+    ok = emqx_config:init_load(emqx_schema, FlappingBin),
+    ?assertEqual(Flapping, emqx:get_config([flapping_detect])),
     ok.
 
 get_policy(Zone) ->
-    emqx_flapping:get_policy([window_time, ban_time, max_count], Zone).
+    emqx_config:get_zone_conf(Zone, [flapping_detect]).

apps/emqx/test/emqx_listeners_update_SUITE.erl

@@ -0,0 +1,154 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_listeners_update_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_schema.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+-import(emqx_listeners, [current_conns/2, is_running/1]).
+
+-define(LISTENERS, [listeners]).
+
+all() -> emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    emqx_common_test_helpers:boot_modules(all),
+    emqx_common_test_helpers:start_apps([]),
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_common_test_helpers:stop_apps([]).
+
+init_per_testcase(_TestCase, Config) ->
+    Init = emqx:get_raw_config(?LISTENERS),
+    [{init_conf, Init} | Config].
+
+end_per_testcase(_TestCase, Config) ->
+    Conf = ?config(init_conf, Config),
+    {ok, _} = emqx:update_config(?LISTENERS, Conf),
+    ok.
+
+t_default_conf(_Config) ->
+    ?assertMatch(
+        #{
+            <<"tcp">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:1883">>}},
+            <<"ssl">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8883">>}},
+            <<"ws">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8083">>}},
+            <<"wss">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8084">>}}
+        },
+        emqx:get_raw_config(?LISTENERS)
+    ),
+    ?assertMatch(
+        #{
+            tcp := #{default := #{bind := {{0, 0, 0, 0}, 1883}}},
+            ssl := #{default := #{bind := {{0, 0, 0, 0}, 8883}}},
+            ws := #{default := #{bind := {{0, 0, 0, 0}, 8083}}},
+            wss := #{default := #{bind := {{0, 0, 0, 0}, 8084}}}
+        },
+        emqx:get_config(?LISTENERS)
+    ),
+    ok.
+
+t_update_conf(_Conf) ->
+    Raw = emqx:get_raw_config(?LISTENERS),
+    Raw1 = emqx_utils_maps:deep_put(
+        [<<"tcp">>, <<"default">>, <<"bind">>], Raw, <<"127.0.0.1:1883">>
+    ),
+    Raw2 = emqx_utils_maps:deep_put(
+        [<<"ssl">>, <<"default">>, <<"bind">>], Raw1, <<"127.0.0.1:8883">>
+    ),
+    Raw3 = emqx_utils_maps:deep_put(
+        [<<"ws">>, <<"default">>, <<"bind">>], Raw2, <<"0.0.0.0:8083">>
+    ),
+    Raw4 = emqx_utils_maps:deep_put(
+        [<<"wss">>, <<"default">>, <<"bind">>], Raw3, <<"127.0.0.1:8084">>
+    ),
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw4)),
+    ?assertMatch(
+        #{
+            <<"tcp">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:1883">>}},
+            <<"ssl">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:8883">>}},
+            <<"ws">> := #{<<"default">> := #{<<"bind">> := <<"0.0.0.0:8083">>}},
+            <<"wss">> := #{<<"default">> := #{<<"bind">> := <<"127.0.0.1:8084">>}}
+        },
+        emqx:get_raw_config(?LISTENERS)
+    ),
+    BindTcp = {{127, 0, 0, 1}, 1883},
+    BindSsl = {{127, 0, 0, 1}, 8883},
+    BindWs = {{0, 0, 0, 0}, 8083},
+    BindWss = {{127, 0, 0, 1}, 8084},
+    ?assertMatch(
+        #{
+            tcp := #{default := #{bind := BindTcp}},
+            ssl := #{default := #{bind := BindSsl}},
+            ws := #{default := #{bind := BindWs}},
+            wss := #{default := #{bind := BindWss}}
+        },
+        emqx:get_config(?LISTENERS)
+    ),
+    ?assertError(not_found, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})),
+    ?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+
+    ?assertEqual(0, current_conns(<<"tcp:default">>, BindTcp)),
+    ?assertEqual(0, current_conns(<<"ssl:default">>, BindSsl)),
+
+    ?assertEqual({0, 0, 0, 0}, proplists:get_value(ip, ranch:info('ws:default'))),
+    ?assertEqual({127, 0, 0, 1}, proplists:get_value(ip, ranch:info('wss:default'))),
+    ?assert(is_running('ws:default')),
+    ?assert(is_running('wss:default')),
+    ok.
+
+t_add_delete_conf(_Conf) ->
+    Raw = emqx:get_raw_config(?LISTENERS),
+    %% add
+    #{<<"tcp">> := #{<<"default">> := Tcp}} = Raw,
+    NewBind = <<"127.0.0.1:1987">>,
+    Raw1 = emqx_utils_maps:deep_put([<<"tcp">>, <<"new">>], Raw, Tcp#{<<"bind">> => NewBind}),
+    Raw2 = emqx_utils_maps:deep_put([<<"ssl">>, <<"default">>], Raw1, ?TOMBSTONE_VALUE),
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw2)),
+    ?assertEqual(0, current_conns(<<"tcp:new">>, {{127, 0, 0, 1}, 1987})),
+    ?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+    %% deleted
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw)),
+    ?assertError(not_found, current_conns(<<"tcp:new">>, {{127, 0, 0, 1}, 1987})),
+    ?assertEqual(0, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+    ok.
+
+t_delete_default_conf(_Conf) ->
+    Raw = emqx:get_raw_config(?LISTENERS),
+    %% delete default listeners
+    Raw1 = emqx_utils_maps:deep_put([<<"tcp">>, <<"default">>], Raw, ?TOMBSTONE_VALUE),
+    Raw2 = emqx_utils_maps:deep_put([<<"ssl">>, <<"default">>], Raw1, ?TOMBSTONE_VALUE),
+    Raw3 = emqx_utils_maps:deep_put([<<"ws">>, <<"default">>], Raw2, ?TOMBSTONE_VALUE),
+    Raw4 = emqx_utils_maps:deep_put([<<"wss">>, <<"default">>], Raw3, ?TOMBSTONE_VALUE),
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw4)),
+    ?assertError(not_found, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})),
+    ?assertError(not_found, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+    ?assertMatch({error, not_found}, is_running('ws:default')),
+    ?assertMatch({error, not_found}, is_running('wss:default')),
+
+    %% reset
+    ?assertMatch({ok, _}, emqx:update_config(?LISTENERS, Raw)),
+    ?assertEqual(0, current_conns(<<"tcp:default">>, {{0, 0, 0, 0}, 1883})),
+    ?assertEqual(0, current_conns(<<"ssl:default">>, {{0, 0, 0, 0}, 8883})),
+    ?assert(is_running('ws:default')),
+    ?assert(is_running('wss:default')),
+    ok.

apps/emqx/test/emqx_mqtt_SUITE.erl

@@ -219,13 +219,15 @@ t_async_set_keepalive('end', _Config) ->
 t_async_set_keepalive(_) ->
     case os:type() of
         {unix, darwin} ->
-            %% Mac OSX don't support the feature
-            ok;
+            do_async_set_keepalive(16#10, 16#101, 16#102);
+        {unix, linux} ->
+            do_async_set_keepalive(4, 5, 6);
         _ ->
-            do_async_set_keepalive()
+            %% don't support the feature on other OS
+            ok
     end.
 
-do_async_set_keepalive() ->
+do_async_set_keepalive(OptKeepIdle, OptKeepInterval, OptKeepCount) ->
     ClientID = <<"client-tcp-keepalive">>,
     {ok, Client} = emqtt:start_link([
         {host, "localhost"},
@@ -247,19 +249,19 @@ do_async_set_keepalive() ->
     Transport = maps:get(transport, State),
     Socket = maps:get(socket, State),
     ?assert(is_port(Socket)),
-    Opts = [{raw, 6, 4, 4}, {raw, 6, 5, 4}, {raw, 6, 6, 4}],
+    Opts = [{raw, 6, OptKeepIdle, 4}, {raw, 6, OptKeepInterval, 4}, {raw, 6, OptKeepCount, 4}],
     {ok, [
-        {raw, 6, 4, <<Idle:32/native>>},
-        {raw, 6, 5, <<Interval:32/native>>},
-        {raw, 6, 6, <<Probes:32/native>>}
+        {raw, 6, OptKeepIdle, <<Idle:32/native>>},
+        {raw, 6, OptKeepInterval, <<Interval:32/native>>},
+        {raw, 6, OptKeepCount, <<Probes:32/native>>}
     ]} = Transport:getopts(Socket, Opts),
     ct:pal("Idle=~p, Interval=~p, Probes=~p", [Idle, Interval, Probes]),
-    emqx_connection:async_set_keepalive(Pid, Idle + 1, Interval + 1, Probes + 1),
+    emqx_connection:async_set_keepalive(os:type(), Pid, Idle + 1, Interval + 1, Probes + 1),
     {ok, _} = ?block_until(#{?snk_kind := "custom_socket_options_successfully"}, 1000),
     {ok, [
-        {raw, 6, 4, <<NewIdle:32/native>>},
-        {raw, 6, 5, <<NewInterval:32/native>>},
-        {raw, 6, 6, <<NewProbes:32/native>>}
+        {raw, 6, OptKeepIdle, <<NewIdle:32/native>>},
+        {raw, 6, OptKeepInterval, <<NewInterval:32/native>>},
+        {raw, 6, OptKeepCount, <<NewProbes:32/native>>}
     ]} = Transport:getopts(Socket, Opts),
     ?assertEqual(NewIdle, Idle + 1),
     ?assertEqual(NewInterval, Interval + 1),

apps/emqx/test/emqx_quic_multistreams_SUITE.erl

@@ -23,6 +23,7 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("quicer/include/quicer.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
@@ -1465,7 +1466,7 @@ t_multi_streams_emqx_ctrl_kill(Config) ->
     ),
 
     ClientId = proplists:get_value(clientid, emqtt:info(C)),
-    [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId),
+    [{ClientId, TransPid}] = ets:lookup(?CHAN_TAB, ClientId),
     exit(TransPid, kill),
 
     %% Client should be closed
@@ -1518,7 +1519,7 @@ t_multi_streams_emqx_ctrl_exit_normal(Config) ->
     ),
 
     ClientId = proplists:get_value(clientid, emqtt:info(C)),
-    [{ClientId, TransPid}] = ets:lookup(emqx_channel, ClientId),
+    [{ClientId, TransPid}] = ets:lookup(?CHAN_TAB, ClientId),
 
     emqx_connection:stop(TransPid),
     %% Client exit normal.

apps/emqx/test/emqx_release_tests.erl

@@ -0,0 +1,56 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_release_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+vsn_compre_test_() ->
+    CurrentVersion = emqx_release:version_with_prefix(),
+    [
+        {"must be 'same' when comparing with current version", fun() ->
+            ?assertEqual(same, emqx_release:vsn_compare(CurrentVersion))
+        end},
+        {"must be 'same' when comparing same version strings", fun() ->
+            ?assertEqual(same, emqx_release:vsn_compare("1.1.1", "1.1.1"))
+        end},
+        {"1.1.1 is older than 1.1.2", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.2", "1.1.1")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.1", "1.1.2"))
+        end},
+        {"1.1.9 is older than 1.1.10", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.10", "1.1.9")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.9", "1.1.10"))
+        end},
+        {"alpha is older than beta", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.1-beta.1", "1.1.1-alpha.2")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.1-alpha.2", "1.1.1-beta.1"))
+        end},
+        {"beta is older than rc", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.1-rc.1", "1.1.1-beta.2")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.1-beta.2", "1.1.1-rc.1"))
+        end},
+        {"rc is older than official cut", fun() ->
+            ?assertEqual(older, emqx_release:vsn_compare("1.1.1", "1.1.1-rc.1")),
+            ?assertEqual(newer, emqx_release:vsn_compare("1.1.1-rc.1", "1.1.1"))
+        end},
+        {"invalid version string will crash", fun() ->
+            ?assertError({invalid_version_string, "1.1.a"}, emqx_release:vsn_compare("v1.1.a")),
+            ?assertError(
+                {invalid_version_string, "1.1.1-alpha"}, emqx_release:vsn_compare("e1.1.1-alpha")
+            )
+        end}
+    ].

apps/emqx/test/emqx_router_SUITE.erl

@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
@@ -127,5 +128,5 @@ t_unexpected(_) ->
 clear_tables() ->
     lists:foreach(
         fun mnesia:clear_table/1,
-        [emqx_route, emqx_trie, emqx_trie_node]
+        [?ROUTE_TAB, ?TRIE, emqx_trie_node]
     ).

apps/emqx/test/emqx_router_helper_SUITE.erl

@@ -20,11 +20,11 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -define(ROUTER_HELPER, emqx_router_helper).
--define(ROUTE_TAB, emqx_route).
 
 all() -> emqx_common_test_helpers:all(?MODULE).
 
@@ -82,9 +82,9 @@ t_monitor(_) ->
     emqx_router_helper:monitor(undefined).
 
 t_mnesia(_) ->
-    ?ROUTER_HELPER ! {mnesia_table_event, {delete, {emqx_routing_node, node()}, undefined}},
+    ?ROUTER_HELPER ! {mnesia_table_event, {delete, {?ROUTING_NODE, node()}, undefined}},
     ?ROUTER_HELPER ! {mnesia_table_event, testing},
-    ?ROUTER_HELPER ! {mnesia_table_event, {write, {emqx_routing_node, node()}, undefined}},
+    ?ROUTER_HELPER ! {mnesia_table_event, {write, {?ROUTING_NODE, node()}, undefined}},
     ?ROUTER_HELPER ! {membership, testing},
     ?ROUTER_HELPER ! {membership, {mnesia, down, node()}},
     ct:sleep(200).

apps/emqx/test/emqx_schema_tests.erl

@@ -106,6 +106,67 @@ bad_cipher_test() ->
     ),
     ok.
 
+fail_if_no_peer_cert_test_() ->
+    Sc = #{
+        roots => [mqtt_ssl_listener],
+        fields => #{mqtt_ssl_listener => emqx_schema:fields("mqtt_ssl_listener")}
+    },
+    Opts = #{atom_key => false, required => false},
+    OptsAtomKey = #{atom_key => true, required => false},
+    InvalidConf = #{
+        <<"bind">> => <<"0.0.0.0:9883">>,
+        <<"ssl_options">> => #{
+            <<"fail_if_no_peer_cert">> => true,
+            <<"verify">> => <<"verify_none">>
+        }
+    },
+    InvalidListener = #{<<"mqtt_ssl_listener">> => InvalidConf},
+    ValidListener = #{
+        <<"mqtt_ssl_listener">> => InvalidConf#{
+            <<"ssl_options">> =>
+                #{
+                    <<"fail_if_no_peer_cert">> => true,
+                    <<"verify">> => <<"verify_peer">>
+                }
+        }
+    },
+    ValidListener1 = #{
+        <<"mqtt_ssl_listener">> => InvalidConf#{
+            <<"ssl_options">> =>
+                #{
+                    <<"fail_if_no_peer_cert">> => false,
+                    <<"verify">> => <<"verify_none">>
+                }
+        }
+    },
+    Reason = "verify must be verify_peer when fail_if_no_peer_cert is true",
+    [
+        ?_assertThrow(
+            {_Sc, [#{kind := validation_error, reason := Reason}]},
+            hocon_tconf:check_plain(Sc, InvalidListener, Opts)
+        ),
+        ?_assertThrow(
+            {_Sc, [#{kind := validation_error, reason := Reason}]},
+            hocon_tconf:check_plain(Sc, InvalidListener, OptsAtomKey)
+        ),
+        ?_assertMatch(
+            #{mqtt_ssl_listener := #{}},
+            hocon_tconf:check_plain(Sc, ValidListener, OptsAtomKey)
+        ),
+        ?_assertMatch(
+            #{mqtt_ssl_listener := #{}},
+            hocon_tconf:check_plain(Sc, ValidListener1, OptsAtomKey)
+        ),
+        ?_assertMatch(
+            #{<<"mqtt_ssl_listener">> := #{}},
+            hocon_tconf:check_plain(Sc, ValidListener, Opts)
+        ),
+        ?_assertMatch(
+            #{<<"mqtt_ssl_listener">> := #{}},
+            hocon_tconf:check_plain(Sc, ValidListener1, Opts)
+        )
+    ].
+
 validate(Schema, Data0) ->
     Sc = #{
         roots => [ssl_opts],
@@ -825,15 +886,27 @@ timeout_types_test_() ->
             typerefl:from_string(emqx_schema:timeout_duration_s(), <<"4294967000ms">>)
         ),
         ?_assertThrow(
-            "timeout value too large (max: 4294967295 ms)",
+            #{
+                kind := validation_error,
+                message := "timeout value too large (max: 4294967295 ms)",
+                schema_module := emqx_schema
+            },
             typerefl:from_string(emqx_schema:timeout_duration(), <<"4294967296ms">>)
         ),
         ?_assertThrow(
-            "timeout value too large (max: 4294967295 ms)",
+            #{
+                kind := validation_error,
+                message := "timeout value too large (max: 4294967295 ms)",
+                schema_module := emqx_schema
+            },
             typerefl:from_string(emqx_schema:timeout_duration_ms(), <<"4294967296ms">>)
         ),
         ?_assertThrow(
-            "timeout value too large (max: 4294967 s)",
+            #{
+                kind := validation_error,
+                message := "timeout value too large (max: 4294967 s)",
+                schema_module := emqx_schema
+            },
             typerefl:from_string(emqx_schema:timeout_duration_s(), <<"4294967001ms">>)
         )
     ].

apps/emqx/test/emqx_takeover_SUITE.erl

@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
@@ -117,7 +118,7 @@ load_meck(ClientId) ->
     [ChanPid] = emqx_cm:lookup_channels(ClientId),
     ChanInfo = #{conninfo := ConnInfo} = emqx_cm:get_chan_info(ClientId),
     NChanInfo = ChanInfo#{conninfo := ConnInfo#{conn_mod := fake_conn_mod}},
-    true = ets:update_element(emqx_channel_info, {ClientId, ChanPid}, {2, NChanInfo}).
+    true = ets:update_element(?CHAN_INFO_TAB, {ClientId, ChanPid}, {2, NChanInfo}).
 
 unload_meck(_ClientId) ->
     meck:unload(fake_conn_mod).

apps/emqx/test/emqx_topic_SUITE.erl

@@ -20,6 +20,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx/include/emqx_placeholder.hrl").
 
 -import(
@@ -130,14 +131,35 @@ t_validate(_) ->
     true = validate({filter, <<"x">>}),
     true = validate({name, <<"x//y">>}),
     true = validate({filter, <<"sport/tennis/#">>}),
+    %% MQTT-5.0 [MQTT-4.7.3-1]
     ?assertError(empty_topic, validate({name, <<>>})),
+    ?assertError(empty_topic, validate({filter, <<>>})),
     ?assertError(topic_name_error, validate({name, <<"abc/#">>})),
     ?assertError(topic_too_long, validate({name, long_topic()})),
-    ?assertError('topic_invalid_#', validate({filter, <<"abc/#/1">>})),
     ?assertError(topic_invalid_char, validate({filter, <<"abc/#xzy/+">>})),
     ?assertError(topic_invalid_char, validate({filter, <<"abc/xzy/+9827">>})),
     ?assertError(topic_invalid_char, validate({filter, <<"sport/tennis#">>})),
-    ?assertError('topic_invalid_#', validate({filter, <<"sport/tennis/#/ranking">>})).
+    %% MQTT-5.0 [MQTT-4.7.1-1]
+    ?assertError('topic_invalid_#', validate({filter, <<"abc/#/1">>})),
+    ?assertError('topic_invalid_#', validate({filter, <<"sport/tennis/#/ranking">>})),
+    %% MQTT-5.0 [MQTT-4.8.2-1]
+    ?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share/">>})),
+    ?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share//">>})),
+    ?assertError(?SHARE_EMPTY_GROUP, validate({filter, <<"$share//t">>})),
+    ?assertError(?SHARE_EMPTY_GROUP, validate({filter, <<"$share//test">>})),
+    %% MQTT-5.0 [MQTT-4.7.3-1] for shared-sub
+    ?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share/g/">>})),
+    ?assertError(?SHARE_EMPTY_FILTER, validate({filter, <<"$share/g2/">>})),
+    %% MQTT-5.0 [MQTT-4.8.2-2]
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/p+q/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/m+/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/+n/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/x#y/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/x#/1">>})),
+    ?assertError(?SHARE_NAME_INVALID_CHAR, validate({filter, <<"$share/#y/1">>})),
+    %% share recursively
+    ?assertError(?SHARE_RECURSIVELY, validate({filter, <<"$share/g1/$share/t">>})),
+    true = validate({filter, <<"$share/g1/topic/$share">>}).
 
 t_sigle_level_validate(_) ->
     true = validate({filter, <<"+">>}),
@@ -177,7 +199,10 @@ t_join(_) ->
     ?assertEqual(<<"+//#">>, join(['+', '', '#'])),
     ?assertEqual(<<"x/y/z/+">>, join([<<"x">>, <<"y">>, <<"z">>, '+'])),
     ?assertEqual(<<"/ab/cd/ef/">>, join(words(<<"/ab/cd/ef/">>))),
-    ?assertEqual(<<"ab/+/#">>, join(words(<<"ab/+/#">>))).
+    ?assertEqual(<<"ab/+/#">>, join(words(<<"ab/+/#">>))),
+    %% MQTT-5.0 [MQTT-4.7.1-1]
+    ?assertError('topic_invalid_#', join(['+', <<"a">>, '#', <<"b">>, '', '+'])),
+    ?assertError('topic_invalid_#', join(['+', <<"c">>, <<"#">>, <<"d">>, '', '+'])).
 
 t_systop(_) ->
     SysTop1 = iolist_to_binary(["$SYS/brokers/", atom_to_list(node()), "/xyz"]),

apps/emqx_authz/src/emqx_authz_mnesia.erl

@@ -33,7 +33,7 @@
 -type clientid() :: {clientid, binary()}.
 -type who() :: username() | clientid() | all.
 
--type rule() :: {emqx_authz_rule:permission(), emqx_authz_rule:action(), emqx_topic:topic()}.
+-type rule() :: {emqx_authz_rule:permission(), emqx_authz_rule:action(), emqx_types:topic()}.
 -type rules() :: [rule()].
 
 -record(emqx_acl, {

apps/emqx_bridge_cassandra/src/emqx_bridge_cassandra_connector.erl

@@ -143,12 +143,12 @@ on_start(
             {error, Reason}
     end.
 
-on_stop(InstId, #{pool_name := PoolName}) ->
+on_stop(InstId, _State) ->
     ?SLOG(info, #{
         msg => "stopping_cassandra_connector",
         connector => InstId
     }),
-    emqx_resource_pool:stop(PoolName).
+    emqx_resource_pool:stop(InstId).
 
 -type request() ::
     % emqx_bridge.erl

apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_connector.erl

@@ -274,12 +274,12 @@ connect(Options) ->
 
 -spec on_stop(resource_id(), resource_state()) -> term().
 
-on_stop(InstanceID, #{pool_name := PoolName}) ->
+on_stop(InstanceID, _State) ->
     ?SLOG(info, #{
         msg => "stopping clickouse connector",
         connector => InstanceID
     }),
-    emqx_resource_pool:stop(PoolName).
+    emqx_resource_pool:stop(InstanceID).
 
 %% -------------------------------------------------------------------
 %% on_get_status emqx_resouce callback and related functions

apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo_connector.erl

@@ -111,12 +111,12 @@ on_start(
             Error
     end.
 
-on_stop(InstanceId, #{pool_name := PoolName}) ->
+on_stop(InstanceId, _State) ->
     ?SLOG(info, #{
         msg => "stopping_dynamo_connector",
         connector => InstanceId
     }),
-    emqx_resource_pool:stop(PoolName).
+    emqx_resource_pool:stop(InstanceId).
 
 on_query(InstanceId, Query, State) ->
     do_query(InstanceId, Query, State).

apps/emqx_bridge_gcp_pubsub/src/emqx_bridge_gcp_pubsub_connector.erl

@@ -6,6 +6,7 @@
 
 -behaviour(emqx_resource).
 
+-include_lib("jose/include/jose_jwk.hrl").
 -include_lib("emqx_connector/include/emqx_connector_tables.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 -include_lib("typerefl/include/types.hrl").
@@ -26,7 +27,6 @@
 ]).
 -export([reply_delegator/3]).
 
--type jwt_worker() :: binary().
 -type service_account_json() :: emqx_bridge_gcp_pubsub:service_account_json().
 -type config() :: #{
     connect_timeout := emqx_schema:duration_ms(),
@@ -38,7 +38,7 @@
 }.
 -type state() :: #{
     connect_timeout := timer:time(),
-    jwt_worker_id := jwt_worker(),
+    jwt_config := emqx_connector_jwt:jwt_config(),
     max_retries := non_neg_integer(),
     payload_template := emqx_plugin_libs_rule:tmpl_token(),
     pool_name := binary(),
@@ -97,12 +97,12 @@ on_start(
         {enable_pipelining, maps:get(enable_pipelining, Config, ?DEFAULT_PIPELINE_SIZE)}
     ],
     #{
-        jwt_worker_id := JWTWorkerId,
+        jwt_config := JWTConfig,
         project_id := ProjectId
-    } = ensure_jwt_worker(ResourceId, Config),
+    } = parse_jwt_config(ResourceId, Config),
     State = #{
         connect_timeout => ConnectTimeout,
-        jwt_worker_id => JWTWorkerId,
+        jwt_config => JWTConfig,
         max_retries => MaxRetries,
         payload_template => emqx_plugin_libs_rule:preproc_tmpl(PayloadTemplate),
         pool_name => ResourceId,
@@ -134,18 +134,21 @@ on_start(
     end.
 
 -spec on_stop(resource_id(), state()) -> ok | {error, term()}.
-on_stop(
-    ResourceId,
-    _State = #{jwt_worker_id := JWTWorkerId}
-) ->
-    ?tp(gcp_pubsub_stop, #{resource_id => ResourceId, jwt_worker_id => JWTWorkerId}),
+on_stop(ResourceId, _State) ->
+    ?tp(gcp_pubsub_stop, #{resource_id => ResourceId}),
     ?SLOG(info, #{
         msg => "stopping_gcp_pubsub_bridge",
         connector => ResourceId
     }),
-    emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
-    emqx_connector_jwt:delete_jwt(?JWT_TABLE, ResourceId),
-    ehttpc_sup:stop_pool(ResourceId).
+    ok = emqx_connector_jwt:delete_jwt(?JWT_TABLE, ResourceId),
+    case ehttpc_sup:stop_pool(ResourceId) of
+        ok ->
+            ok;
+        {error, not_found} ->
+            ok;
+        Error ->
+            Error
+    end.
 
 -spec on_query(
     resource_id(),
@@ -228,12 +231,12 @@ on_get_status(ResourceId, #{connect_timeout := Timeout} = State) ->
 %% Helper fns
 %%-------------------------------------------------------------------------------------------------
 
--spec ensure_jwt_worker(resource_id(), config()) ->
+-spec parse_jwt_config(resource_id(), config()) ->
     #{
-        jwt_worker_id := jwt_worker(),
+        jwt_config := emqx_connector_jwt:jwt_config(),
         project_id := binary()
     }.
-ensure_jwt_worker(ResourceId, #{
+parse_jwt_config(ResourceId, #{
     service_account_json := ServiceAccountJSON
 }) ->
     #{
@@ -246,8 +249,32 @@ ensure_jwt_worker(ResourceId, #{
     Aud = <<"https://pubsub.googleapis.com/">>,
     ExpirationMS = timer:hours(1),
     Alg = <<"RS256">>,
-    Config = #{
-        private_key => PrivateKeyPEM,
+    JWK =
+        try jose_jwk:from_pem(PrivateKeyPEM) of
+            JWK0 = #jose_jwk{} ->
+                %% Don't wrap the JWK with `emqx_secret:wrap' here;
+                %% this is stored in mnesia and synchronized among the
+                %% nodes, and will easily become a bad fun.
+                JWK0;
+            [] ->
+                ?tp(error, gcp_pubsub_connector_startup_error, #{error => empty_key}),
+                throw("empty private in service account json");
+            {error, Reason} ->
+                Error = {invalid_private_key, Reason},
+                ?tp(error, gcp_pubsub_connector_startup_error, #{error => Error}),
+                throw("invalid private key in service account json");
+            Error0 ->
+                Error = {invalid_private_key, Error0},
+                ?tp(error, gcp_pubsub_connector_startup_error, #{error => Error}),
+                throw("invalid private key in service account json")
+        catch
+            Kind:Reason ->
+                Error = {Kind, Reason},
+                ?tp(error, gcp_pubsub_connector_startup_error, #{error => Error}),
+                throw("invalid private key in service account json")
+        end,
+    JWTConfig = #{
+        jwk => emqx_secret:wrap(JWK),
         resource_id => ResourceId,
         expiration => ExpirationMS,
         table => ?JWT_TABLE,
@@ -257,46 +284,8 @@ ensure_jwt_worker(ResourceId, #{
         kid => KId,
         alg => Alg
     },
-
-    JWTWorkerId = <<"gcp_pubsub_jwt_worker:", ResourceId/binary>>,
-    Worker =
-        case emqx_connector_jwt_sup:ensure_worker_present(JWTWorkerId, Config) of
-            {ok, Worker0} ->
-                Worker0;
-            Error ->
-                ?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{
-                    connector => ResourceId,
-                    reason => Error
-                }),
-                _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
-                throw(failed_to_start_jwt_worker)
-        end,
-    MRef = monitor(process, Worker),
-    Ref = emqx_connector_jwt_worker:ensure_jwt(Worker),
-
-    %% to ensure that this resource and its actions will be ready to
-    %% serve when started, we must ensure that the first JWT has been
-    %% produced by the worker.
-    receive
-        {Ref, token_created} ->
-            ?tp(gcp_pubsub_bridge_jwt_created, #{resource_id => ResourceId}),
-            demonitor(MRef, [flush]),
-            ok;
-        {'DOWN', MRef, process, Worker, Reason} ->
-            ?tp(error, "gcp_pubsub_bridge_jwt_worker_failed_to_start", #{
-                connector => ResourceId,
-                reason => Reason
-            }),
-            _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
-            throw(failed_to_start_jwt_worker)
-    after 10_000 ->
-        ?tp(warning, "gcp_pubsub_bridge_jwt_timeout", #{connector => ResourceId}),
-        demonitor(MRef, [flush]),
-        _ = emqx_connector_jwt_sup:ensure_worker_deleted(JWTWorkerId),
-        throw(timeout_creating_jwt)
-    end,
     #{
-        jwt_worker_id => JWTWorkerId,
+        jwt_config => JWTConfig,
         project_id => ProjectId
     }.
 
@@ -322,14 +311,10 @@ publish_path(
 ) ->
     <<"/v1/projects/", ProjectId/binary, "/topics/", PubSubTopic/binary, ":publish">>.
 
--spec get_jwt_authorization_header(resource_id()) -> [{binary(), binary()}].
-get_jwt_authorization_header(ResourceId) ->
-    case emqx_connector_jwt:lookup_jwt(?JWT_TABLE, ResourceId) of
-        %% Since we synchronize the JWT creation during resource start
-        %% (see `on_start/2'), this will be always be populated.
-        {ok, JWT} ->
-            [{<<"Authorization">>, <<"Bearer ", JWT/binary>>}]
-    end.
+-spec get_jwt_authorization_header(emqx_connector_jwt:jwt_config()) -> [{binary(), binary()}].
+get_jwt_authorization_header(JWTConfig) ->
+    JWT = emqx_connector_jwt:ensure_jwt(JWTConfig),
+    [{<<"Authorization">>, <<"Bearer ", JWT/binary>>}].
 
 -spec do_send_requests_sync(
     state(),
@@ -342,6 +327,7 @@ get_jwt_authorization_header(ResourceId) ->
     | {error, term()}.
 do_send_requests_sync(State, Requests, ResourceId) ->
     #{
+        jwt_config := JWTConfig,
         pool_name := PoolName,
         max_retries := MaxRetries,
         request_ttl := RequestTTL
@@ -354,7 +340,7 @@ do_send_requests_sync(State, Requests, ResourceId) ->
             requests => Requests
         }
     ),
-    Headers = get_jwt_authorization_header(ResourceId),
+    Headers = get_jwt_authorization_header(JWTConfig),
     Payloads =
         lists:map(
             fun({send_message, Selected}) ->
@@ -466,6 +452,7 @@ do_send_requests_sync(State, Requests, ResourceId) ->
 ) -> {ok, pid()}.
 do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId) ->
     #{
+        jwt_config := JWTConfig,
         pool_name := PoolName,
         request_ttl := RequestTTL
     } = State,
@@ -477,7 +464,7 @@ do_send_requests_async(State, Requests, ReplyFunAndArgs, ResourceId) ->
             requests => Requests
         }
     ),
-    Headers = get_jwt_authorization_header(ResourceId),
+    Headers = get_jwt_authorization_header(JWTConfig),
     Payloads =
         lists:map(
             fun({send_message, Selected}) ->

apps/emqx_bridge_gcp_pubsub/test/emqx_bridge_gcp_pubsub_SUITE.erl

@@ -55,8 +55,9 @@ single_config_tests() ->
         t_not_of_service_account_type,
         t_json_missing_fields,
         t_invalid_private_key,
-        t_jwt_worker_start_timeout,
-        t_failed_to_start_jwt_worker,
+        t_truncated_private_key,
+        t_jose_error_tuple,
+        t_jose_other_error,
         t_stop,
         t_get_status_ok,
         t_get_status_down,
@@ -580,14 +581,7 @@ t_publish_success(Config) ->
     ServiceAccountJSON = ?config(service_account_json, Config),
     TelemetryTable = ?config(telemetry_table, Config),
     Topic = <<"t/topic">>,
-    ?check_trace(
-        create_bridge(Config),
-        fun(Res, Trace) ->
-            ?assertMatch({ok, _}, Res),
-            ?assertMatch([_], ?of_kind(gcp_pubsub_bridge_jwt_created, Trace)),
-            ok
-        end
-    ),
+    ?assertMatch({ok, _}, create_bridge(Config)),
     {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config),
     on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end),
     assert_empty_metrics(ResourceId),
@@ -686,14 +680,7 @@ t_publish_success_local_topic(Config) ->
     ok.
 
 t_create_via_http(Config) ->
-    ?check_trace(
-        create_bridge_http(Config),
-        fun(Res, Trace) ->
-            ?assertMatch({ok, _}, Res),
-            ?assertMatch([_, _], ?of_kind(gcp_pubsub_bridge_jwt_created, Trace)),
-            ok
-        end
-    ),
+    ?assertMatch({ok, _}, create_bridge_http(Config)),
     ok.
 
 t_publish_templated(Config) ->
@@ -705,16 +692,12 @@ t_publish_templated(Config) ->
         "{\"payload\": \"${payload}\","
         " \"pub_props\": ${pub_props}}"
     >>,
-    ?check_trace(
+    ?assertMatch(
+        {ok, _},
         create_bridge(
             Config,
             #{<<"payload_template">> => PayloadTemplate}
-        ),
-        fun(Res, Trace) ->
-            ?assertMatch({ok, _}, Res),
-            ?assertMatch([_], ?of_kind(gcp_pubsub_bridge_jwt_created, Trace)),
-            ok
-        end
+        )
     ),
     {ok, #{<<"id">> := RuleId}} = create_rule_and_action_http(Config),
     on_exit(fun() -> ok = emqx_rule_engine:delete_rule(RuleId) end),
@@ -908,36 +891,26 @@ t_invalid_private_key(Config) ->
                                 #{<<"private_key">> => InvalidPrivateKeyPEM}
                         }
                     ),
-                    #{?snk_kind := "gcp_pubsub_bridge_jwt_worker_failed_to_start"},
+                    #{?snk_kind := gcp_pubsub_connector_startup_error},
                     20_000
                 ),
             Res
         end,
         fun(Res, Trace) ->
             ?assertMatch({ok, _}, Res),
-            ?assertMatch(
-                [#{reason := Reason}] when
-                    Reason =:= noproc orelse
-                        Reason =:= {shutdown, {error, empty_key}},
-                ?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace)
-            ),
             ?assertMatch(
                 [#{error := empty_key}],
-                ?of_kind(connector_jwt_worker_startup_error, Trace)
+                ?of_kind(gcp_pubsub_connector_startup_error, Trace)
             ),
             ok
         end
     ),
     ok.
 
-t_jwt_worker_start_timeout(Config) ->
-    InvalidPrivateKeyPEM = <<"xxxxxx">>,
+t_truncated_private_key(Config) ->
+    InvalidPrivateKeyPEM = <<"-----BEGIN PRIVATE KEY-----\nMIIEvQI...">>,
     ?check_trace(
         begin
-            ?force_ordering(
-                #{?snk_kind := will_never_happen},
-                #{?snk_kind := connector_jwt_worker_make_key}
-            ),
             {Res, {ok, _Event}} =
                 ?wait_async_action(
                     create_bridge(
@@ -947,14 +920,71 @@ t_jwt_worker_start_timeout(Config) ->
                                 #{<<"private_key">> => InvalidPrivateKeyPEM}
                         }
                     ),
-                    #{?snk_kind := "gcp_pubsub_bridge_jwt_timeout"},
+                    #{?snk_kind := gcp_pubsub_connector_startup_error},
                     20_000
                 ),
             Res
         end,
         fun(Res, Trace) ->
             ?assertMatch({ok, _}, Res),
-            ?assertMatch([_], ?of_kind("gcp_pubsub_bridge_jwt_timeout", Trace)),
+            ?assertMatch(
+                [#{error := {error, function_clause}}],
+                ?of_kind(gcp_pubsub_connector_startup_error, Trace)
+            ),
+            ok
+        end
+    ),
+    ok.
+
+t_jose_error_tuple(Config) ->
+    ?check_trace(
+        begin
+            {Res, {ok, _Event}} =
+                ?wait_async_action(
+                    emqx_common_test_helpers:with_mock(
+                        jose_jwk,
+                        from_pem,
+                        fun(_PrivateKeyPEM) -> {error, some_error} end,
+                        fun() -> create_bridge(Config) end
+                    ),
+                    #{?snk_kind := gcp_pubsub_connector_startup_error},
+                    20_000
+                ),
+            Res
+        end,
+        fun(Res, Trace) ->
+            ?assertMatch({ok, _}, Res),
+            ?assertMatch(
+                [#{error := {invalid_private_key, some_error}}],
+                ?of_kind(gcp_pubsub_connector_startup_error, Trace)
+            ),
+            ok
+        end
+    ),
+    ok.
+
+t_jose_other_error(Config) ->
+    ?check_trace(
+        begin
+            {Res, {ok, _Event}} =
+                ?wait_async_action(
+                    emqx_common_test_helpers:with_mock(
+                        jose_jwk,
+                        from_pem,
+                        fun(_PrivateKeyPEM) -> {unknown, error} end,
+                        fun() -> create_bridge(Config) end
+                    ),
+                    #{?snk_kind := gcp_pubsub_connector_startup_error},
+                    20_000
+                ),
+            Res
+        end,
+        fun(Res, Trace) ->
+            ?assertMatch({ok, _}, Res),
+            ?assertMatch(
+                [#{error := {invalid_private_key, {unknown, error}}}],
+                ?of_kind(gcp_pubsub_connector_startup_error, Trace)
+            ),
             ok
         end
     ),
@@ -1309,26 +1339,6 @@ t_unrecoverable_error(Config) ->
     ),
     ok.
 
-t_failed_to_start_jwt_worker(Config) ->
-    ?check_trace(
-        emqx_common_test_helpers:with_mock(
-            emqx_connector_jwt_sup,
-            ensure_worker_present,
-            fun(_JWTWorkerId, _Config) -> {error, restarting} end,
-            fun() ->
-                ?assertMatch({ok, _}, create_bridge(Config))
-            end
-        ),
-        fun(Trace) ->
-            ?assertMatch(
-                [#{reason := {error, restarting}}],
-                ?of_kind("gcp_pubsub_bridge_jwt_worker_failed_to_start", Trace)
-            ),
-            ok
-        end
-    ),
-    ok.
-
 t_stop(Config) ->
     Name = ?config(gcp_pubsub_name, Config),
     {ok, _} = create_bridge(Config),

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_egress.erl

@@ -49,7 +49,7 @@
 
 -type egress() :: #{
     local => #{
-        topic => emqx_topic:topic()
+        topic => emqx_types:topic()
     },
     remote := emqx_bridge_mqtt_msg:msgvars()
 }.

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_ingress.erl

@@ -43,7 +43,7 @@
 -type ingress() :: #{
     server := string(),
     remote := #{
-        topic := emqx_topic:topic(),
+        topic := emqx_types:topic(),
         qos => emqx_types:qos()
     },
     local := emqx_bridge_mqtt_msg:msgvars(),

apps/emqx_bridge_opents/src/emqx_bridge_opents_connector.erl

@@ -89,12 +89,12 @@ on_start(
             Error
     end.
 
-on_stop(InstanceId, #{pool_name := PoolName} = _State) ->
+on_stop(InstanceId, _State) ->
     ?SLOG(info, #{
         msg => "stopping_opents_connector",
         connector => InstanceId
     }),
-    emqx_resource_pool:stop(PoolName).
+    emqx_resource_pool:stop(InstanceId).
 
 on_query(InstanceId, Request, State) ->
     on_batch_query(InstanceId, [Request], State).

apps/emqx_bridge_rabbitmq/src/emqx_bridge_rabbitmq_connector.erl

@@ -261,12 +261,15 @@ on_start(
 -spec on_stop(resource_id(), resource_state()) -> term().
 on_stop(
     ResourceID,
-    #{poolname := PoolName} = _State
+    _State
 ) ->
     ?SLOG(info, #{
         msg => "stopping RabbitMQ connector",
         connector => ResourceID
     }),
+    stop_clients_and_pool(ResourceID).
+
+stop_clients_and_pool(PoolName) ->
     Workers = [Worker || {_WorkerName, Worker} <- ecpool:workers(PoolName)],
     Clients = [
         begin

apps/emqx_bridge_tdengine/src/emqx_bridge_tdengine_connector.erl

@@ -108,7 +108,6 @@ on_start(
 
     Prepares = parse_prepare_sql(Config),
     State = Prepares#{pool_name => InstanceId, query_opts => query_opts(Config)},
-    ok = emqx_resource:allocate_resource(InstanceId, pool_name, InstanceId),
     case emqx_resource_pool:start(InstanceId, ?MODULE, Options) of
         ok ->
             {ok, State};
@@ -121,12 +120,7 @@ on_stop(InstanceId, _State) ->
         msg => "stopping_tdengine_connector",
         connector => InstanceId
     }),
-    case emqx_resource:get_allocated_resources(InstanceId) of
-        #{pool_name := PoolName} ->
-            emqx_resource_pool:stop(PoolName);
-        _ ->
-            ok
-    end.
+    emqx_resource_pool:stop(InstanceId).
 
 on_query(InstanceId, {query, SQL}, State) ->
     do_query(InstanceId, SQL, State);

apps/emqx_conf/src/emqx_cluster_rpc.erl

@@ -575,7 +575,7 @@ maybe_init_tnx_id(Node, TnxId) ->
     {atomic, _} = transaction(fun ?MODULE:commit/2, [Node, TnxId]),
     ok.
 
-%% @priv Cannot proceed until emqx app is ready.
+%% @private Cannot proceed until emqx app is ready.
 %% Otherwise the committed transaction catch up may fail.
 wait_for_emqx_ready() ->
     %% wait 10 seconds for emqx to start

apps/emqx_conf/src/emqx_conf_app.erl

@@ -42,6 +42,8 @@ start(_StartType, _StartArgs) ->
 stop(_State) ->
     ok.
 
+%% Read the cluster config from the local node.
+%% This function is named 'override' due to historical reasons.
 get_override_config_file() ->
     Node = node(),
     case emqx_app:get_init_config_load_done() of
@@ -63,7 +65,7 @@ get_override_config_file() ->
                             tnx_id => TnxId,
                             node => Node,
                             has_deprecated_file => HasDeprecateFile,
-                            release => emqx_app:get_release()
+                            release => emqx_release:version_with_prefix()
                         }
                     end,
                     case mria:ro_transaction(?CLUSTER_RPC_SHARD, Fun) of
@@ -95,7 +97,7 @@ init_conf() ->
     %% Workaround for https://github.com/emqx/mria/issues/94:
     _ = mria_rlog:wait_for_shards([?CLUSTER_RPC_SHARD], 1000),
     _ = mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT]),
-    {ok, TnxId} = copy_override_conf_from_core_node(),
+    {ok, TnxId} = sync_cluster_conf(),
     _ = emqx_app:set_init_tnx_id(TnxId),
     ok = init_load(),
     ok = emqx_app:set_init_config_load_done().
@@ -103,88 +105,137 @@ init_conf() ->
 cluster_nodes() ->
     mria:cluster_nodes(cores) -- [node()].
 
-copy_override_conf_from_core_node() ->
+%% @doc Try to sync the cluster config from other core nodes.
+sync_cluster_conf() ->
     case cluster_nodes() of
-        %% The first core nodes is self.
         [] ->
-            ?SLOG(debug, #{msg => "skip_copy_override_conf_from_core_node"}),
+            %% The first core nodes is self.
+            ?SLOG(debug, #{
+                msg => "skip_sync_cluster_conf",
+                reason => "This is a single node, or the first node in the cluster"
+            }),
             {ok, ?DEFAULT_INIT_TXN_ID};
         Nodes ->
-            {Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes),
-            {Ready, NotReady0} = lists:partition(fun(Res) -> element(1, Res) =:= ok end, Results),
-            NotReady = lists:filter(fun(Res) -> element(1, Res) =:= error end, NotReady0),
-            case (Failed =/= [] orelse NotReady =/= []) andalso Ready =/= [] of
+            sync_cluster_conf2(Nodes)
+    end.
+
+%% @private Some core nodes are running, try to sync the cluster config from them.
+sync_cluster_conf2(Nodes) ->
+    {Results, Failed} = emqx_conf_proto_v2:get_override_config_file(Nodes),
+    {Ready, NotReady0} = lists:partition(fun(Res) -> element(1, Res) =:= ok end, Results),
+    NotReady = lists:filter(fun(Res) -> element(1, Res) =:= error end, NotReady0),
+    case (Failed =/= [] orelse NotReady =/= []) of
+        true when Ready =/= [] ->
+            %% Some core nodes failed to reply.
+            Warning = #{
+                nodes => Nodes,
+                failed => Failed,
+                not_ready => NotReady,
+                msg => "ignored_nodes_when_sync_cluster_conf"
+            },
+            ?SLOG(warning, Warning);
+        true ->
+            %% There are core nodes running but no one was able to reply.
+            ?SLOG(error, #{
+                msg => "failed_to_sync_cluster_conf",
+                nodes => Nodes,
+                failed => Failed,
+                not_ready => NotReady
+            });
+        false ->
+            ok
+    end,
+    case Ready of
+        [] ->
+            case should_proceed_with_boot() of
                 true ->
-                    Warning = #{
+                    %% Act as if this node is alone, so it can
+                    %% finish the boot sequence and load the
+                    %% config for other nodes to copy it.
+                    ?SLOG(info, #{
+                        msg => "skip_sync_cluster_conf",
+                        loading_from_disk => true,
                         nodes => Nodes,
                         failed => Failed,
-                        not_ready => NotReady,
-                        msg => "ignored_bad_nodes_when_copy_init_config"
-                    },
-                    ?SLOG(warning, Warning);
+                        not_ready => NotReady
+                    }),
+                    {ok, ?DEFAULT_INIT_TXN_ID};
                 false ->
-                    ok
-            end,
-            case Ready of
-                [] ->
-                    %% Other core nodes running but no one replicated it successfully.
-                    ?SLOG(error, #{
-                        msg => "copy_override_conf_from_core_node_failed",
+                    %% retry in some time
+                    Jitter = rand:uniform(2000),
+                    Timeout = 10000 + Jitter,
+                    timer:sleep(Timeout),
+                    ?SLOG(warning, #{
+                        msg => "sync_cluster_conf_retry",
+                        timeout => Timeout,
                         nodes => Nodes,
                         failed => Failed,
                         not_ready => NotReady
                     }),
+                    sync_cluster_conf()
+            end;
+        _ ->
+            sync_cluster_conf3(Ready)
+    end.
 
-                    case should_proceed_with_boot() of
-                        true ->
-                            %% Act as if this node is alone, so it can
-                            %% finish the boot sequence and load the
-                            %% config for other nodes to copy it.
-                            ?SLOG(info, #{
-                                msg => "skip_copy_override_conf_from_core_node",
-                                loading_from_disk => true,
-                                nodes => Nodes,
-                                failed => Failed,
-                                not_ready => NotReady
-                            }),
-                            {ok, ?DEFAULT_INIT_TXN_ID};
-                        false ->
-                            %% retry in some time
-                            Jitter = rand:uniform(2000),
-                            Timeout = 10000 + Jitter,
-                            ?SLOG(info, #{
-                                msg => "copy_cluster_conf_from_core_node_retry",
-                                timeout => Timeout,
-                                nodes => Nodes,
-                                failed => Failed,
-                                not_ready => NotReady
-                            }),
-                            timer:sleep(Timeout),
-                            copy_override_conf_from_core_node()
-                    end;
-                _ ->
-                    [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
-                    #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
-                    HasDeprecatedFile = has_deprecated_file(Info),
-                    ?SLOG(debug, #{
-                        msg => "copy_cluster_conf_from_core_node_success",
-                        node => Node,
-                        has_deprecated_file => HasDeprecatedFile,
-                        local_release => emqx_app:get_release(),
-                        remote_release => maps:get(release, Info, "before_v5.0.24|e5.0.3"),
-                        data_dir => emqx:data_dir(),
-                        tnx_id => TnxId
-                    }),
-                    ok = emqx_config:save_to_override_conf(
-                        HasDeprecatedFile,
-                        RawOverrideConf,
-                        #{override_to => cluster}
-                    ),
-                    ok = sync_data_from_node(Node),
-                    {ok, TnxId}
-            end
+%% @private Filter out the nodes which are running a newer version than this node.
+sync_cluster_conf3(Ready) ->
+    NotNewer = fun({ok, #{release := RemoteRelease}}) ->
+        try
+            emqx_release:vsn_compare(RemoteRelease) =/= newer
+        catch
+            _:_ ->
+                %% If the version is not valid (without v or e prefix),
+                %% we know it's older than v5.1.0/e5.1.0
+                true
+        end
+    end,
+    case lists:filter(NotNewer, Ready) of
+        [] ->
+            %% All available core nodes are running a newer version than this node.
+            %% Start this node without syncing cluster config from them.
+            %% This is likely a restart of an older version node during cluster upgrade.
+            NodesAndVersions = lists:map(
+                fun({ok, #{node := Node, release := Release}}) ->
+                    #{node => Node, version => Release}
+                end,
+                Ready
+            ),
+            ?SLOG(warning, #{
+                msg => "all_available_nodes_running_newer_version",
+                hint =>
+                    "Booting this node without syncing cluster config from peer core nodes "
+                    "because other nodes are running a newer version",
+                peer_nodes => NodesAndVersions
+            }),
+            {ok, ?DEFAULT_INIT_TXN_ID};
+        Ready2 ->
+            sync_cluster_conf4(Ready2)
     end.
 
+%% @private Some core nodes are running and replied with their configs successfully.
+%% Try to sort the results and save the first one for local use.
+sync_cluster_conf4(Ready) ->
+    [{ok, Info} | _] = lists:sort(fun conf_sort/2, Ready),
+    #{node := Node, conf := RawOverrideConf, tnx_id := TnxId} = Info,
+    HasDeprecatedFile = has_deprecated_file(Info),
+    ?SLOG(debug, #{
+        msg => "sync_cluster_conf_success",
+        synced_from_node => Node,
+        has_deprecated_file => HasDeprecatedFile,
+        local_release => emqx_app:get_release(),
+        remote_release => maps:get(release, Info, "before_v5.0.24|e5.0.3"),
+        data_dir => emqx:data_dir(),
+        tnx_id => TnxId
+    }),
+    ok = emqx_config:save_to_override_conf(
+        HasDeprecatedFile,
+        RawOverrideConf,
+        #{override_to => cluster}
+    ),
+    ok = sync_data_from_node(Node),
+    {ok, TnxId}.
+
 should_proceed_with_boot() ->
     TablesStatus = emqx_cluster_rpc:get_tables_status(),
     LocalNode = node(),

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -143,7 +143,7 @@ fields("cluster") ->
             )},
         {"discovery_strategy",
             sc(
-                hoconsc:enum([manual, static, mcast, dns, etcd, k8s]),
+                hoconsc:enum([manual, static, dns, etcd, k8s, mcast]),
                 #{
                     default => manual,
                     desc => ?DESC(cluster_discovery_strategy),
@@ -198,7 +198,7 @@ fields("cluster") ->
         {"mcast",
             sc(
                 ?R_REF(cluster_mcast),
-                #{}
+                #{importance => ?IMPORTANCE_HIDDEN}
             )},
         {"dns",
             sc(

apps/emqx_conf/test/emqx_conf_app_SUITE.erl

@@ -98,6 +98,34 @@ t_copy_deprecated_data_dir(_Config) ->
         stop_cluster(Nodes)
     end.
 
+t_no_copy_from_newer_version_node(_Config) ->
+    net_kernel:start(['master2@127.0.0.1', longnames]),
+    ct:timetrap({seconds, 120}),
+    snabbkaffe:fix_ct_logging(),
+    Cluster = cluster([cluster_spec({core, 10}), cluster_spec({core, 11}), cluster_spec({core, 12})]),
+    OKs = [ok, ok, ok],
+    [First | Rest] = Nodes = start_cluster(Cluster),
+    try
+        File = "/configs/cluster.hocon",
+        assert_config_load_done(Nodes),
+        rpc:call(First, ?MODULE, create_data_dir, [File]),
+        {OKs, []} = rpc:multicall(Nodes, application, stop, [emqx_conf]),
+        {OKs, []} = rpc:multicall(Nodes, ?MODULE, set_data_dir_env, []),
+        {OKs, []} = rpc:multicall(Nodes, meck, new, [
+            emqx_release, [passthrough, no_history, no_link, non_strict]
+        ]),
+        %% 99.9.9 is always newer than the current version
+        {OKs, []} = rpc:multicall(Nodes, meck, expect, [
+            emqx_release, version_with_prefix, 0, "e99.9.9"
+        ]),
+        ok = rpc:call(First, application, start, [emqx_conf]),
+        {[ok, ok], []} = rpc:multicall(Rest, application, start, [emqx_conf]),
+        ok = assert_no_cluster_conf_copied(Rest, File),
+        stop_cluster(Nodes),
+        ok
+    after
+        stop_cluster(Nodes)
+    end.
 %%------------------------------------------------------------------------------
 %% Helper functions
 %%------------------------------------------------------------------------------
@@ -158,6 +186,17 @@ assert_data_copy_done([First0 | Rest], File) ->
         Rest
     ).
 
+assert_no_cluster_conf_copied([], _) ->
+    ok;
+assert_no_cluster_conf_copied([Node | Nodes], File) ->
+    NodeStr = atom_to_list(Node),
+    ?assertEqual(
+        {error, enoent},
+        file:read_file(NodeStr ++ File),
+        #{node => Node}
+    ),
+    assert_no_cluster_conf_copied(Nodes, File).
+
 assert_config_load_done(Nodes) ->
     lists:foreach(
         fun(Node) ->

apps/emqx_connector/src/emqx_connector_http.erl

@@ -219,7 +219,6 @@ on_start(
         base_path => BasePath,
         request => preprocess_request(maps:get(request, Config, undefined))
     },
-    ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
     case ehttpc_sup:start_pool(InstId, PoolOpts) of
         {ok, _} -> {ok, State};
         {error, {already_started, _}} -> {ok, State};
@@ -231,12 +230,7 @@ on_stop(InstId, _State) ->
         msg => "stopping_http_connector",
         connector => InstId
     }),
-    case emqx_resource:get_allocated_resources(InstId) of
-        #{pool_name := PoolName} ->
-            ehttpc_sup:stop_pool(PoolName);
-        _ ->
-            ok
-    end.
+    ehttpc_sup:stop_pool(InstId).
 
 on_query(InstId, {send_message, Msg}, State) ->
     case maps:get(request, State, undefined) of

apps/emqx_connector/src/emqx_connector_jwt.erl

@@ -19,15 +19,33 @@
 -include_lib("emqx_connector/include/emqx_connector_tables.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
+-include_lib("jose/include/jose_jwt.hrl").
+-include_lib("jose/include/jose_jws.hrl").
 
 %% API
 -export([
     lookup_jwt/1,
     lookup_jwt/2,
-    delete_jwt/2
+    delete_jwt/2,
+    ensure_jwt/1
 ]).
 
 -type jwt() :: binary().
+-type wrapped_jwk() :: fun(() -> jose_jwk:key()).
+-type jwk() :: jose_jwk:key().
+-type jwt_config() :: #{
+    expiration := timer:time(),
+    resource_id := resource_id(),
+    table := ets:table(),
+    jwk := wrapped_jwk() | jwk(),
+    iss := binary(),
+    sub := binary(),
+    aud := binary(),
+    kid := binary(),
+    alg := binary()
+}.
+
+-export_type([jwt_config/0, jwt/0]).
 
 -spec lookup_jwt(resource_id()) -> {ok, jwt()} | {error, not_found}.
 lookup_jwt(ResourceId) ->
@@ -57,3 +75,70 @@ delete_jwt(TId, ResourceId) ->
         error:badarg ->
             ok
     end.
+
+%% @doc Attempts to retrieve a valid JWT from the cache.  If there is
+%% none or if the cached token is expired, generates an caches a fresh
+%% one.
+-spec ensure_jwt(jwt_config()) -> jwt().
+ensure_jwt(JWTConfig) ->
+    #{resource_id := ResourceId, table := Table} = JWTConfig,
+    case lookup_jwt(Table, ResourceId) of
+        {error, not_found} ->
+            JWT = do_generate_jwt(JWTConfig),
+            store_jwt(JWTConfig, JWT),
+            JWT;
+        {ok, JWT0} ->
+            case is_about_to_expire(JWT0) of
+                true ->
+                    JWT = do_generate_jwt(JWTConfig),
+                    store_jwt(JWTConfig, JWT),
+                    JWT;
+                false ->
+                    JWT0
+            end
+    end.
+
+%%-----------------------------------------------------------------------------------------
+%% Helper fns
+%%-----------------------------------------------------------------------------------------
+
+-spec do_generate_jwt(jwt_config()) -> jwt().
+do_generate_jwt(#{
+    expiration := ExpirationMS,
+    iss := Iss,
+    sub := Sub,
+    aud := Aud,
+    kid := KId,
+    alg := Alg,
+    jwk := WrappedJWK
+}) ->
+    JWK = emqx_secret:unwrap(WrappedJWK),
+    Headers = #{
+        <<"alg">> => Alg,
+        <<"kid">> => KId
+    },
+    Now = erlang:system_time(seconds),
+    ExpirationS = erlang:convert_time_unit(ExpirationMS, millisecond, second),
+    Claims = #{
+        <<"iss">> => Iss,
+        <<"sub">> => Sub,
+        <<"aud">> => Aud,
+        <<"iat">> => Now,
+        <<"exp">> => Now + ExpirationS
+    },
+    JWT0 = jose_jwt:sign(JWK, Headers, Claims),
+    {_, JWT} = jose_jws:compact(JWT0),
+    JWT.
+
+-spec store_jwt(jwt_config(), jwt()) -> ok.
+store_jwt(#{resource_id := ResourceId, table := TId}, JWT) ->
+    true = ets:insert(TId, {{ResourceId, jwt}, JWT}),
+    ?tp(emqx_connector_jwt_token_stored, #{resource_id => ResourceId}),
+    ok.
+
+-spec is_about_to_expire(jwt()) -> boolean().
+is_about_to_expire(JWT) ->
+    #jose_jwt{fields = #{<<"exp">> := Exp}} = jose_jwt:peek(JWT),
+    Now = erlang:system_time(seconds),
+    GraceExp = Exp - timer:seconds(5),
+    Now >= GraceExp.

apps/emqx_connector/src/emqx_connector_jwt_worker.erl

@@ -189,49 +189,14 @@ terminate(_Reason, State) ->
 %% Helper fns
 %%-----------------------------------------------------------------------------------------
 
--spec do_generate_jwt(state()) -> jwt().
-do_generate_jwt(
-    #{
-        expiration := ExpirationMS,
-        iss := Iss,
-        sub := Sub,
-        aud := Aud,
-        kid := KId,
-        alg := Alg,
-        jwk := JWK
-    } = _State
-) ->
-    Headers = #{
-        <<"alg">> => Alg,
-        <<"kid">> => KId
-    },
-    Now = erlang:system_time(seconds),
-    ExpirationS = erlang:convert_time_unit(ExpirationMS, millisecond, second),
-    Claims = #{
-        <<"iss">> => Iss,
-        <<"sub">> => Sub,
-        <<"aud">> => Aud,
-        <<"iat">> => Now,
-        <<"exp">> => Now + ExpirationS
-    },
-    JWT0 = jose_jwt:sign(JWK, Headers, Claims),
-    {_, JWT} = jose_jws:compact(JWT0),
-    JWT.
-
 -spec generate_and_store_jwt(state()) -> state().
 generate_and_store_jwt(State0) ->
-    JWT = do_generate_jwt(State0),
-    store_jwt(State0, JWT),
+    JWTConfig = maps:without([jwt, refresh_timer], State0),
+    JWT = emqx_connector_jwt:ensure_jwt(JWTConfig),
     ?tp(connector_jwt_worker_refresh, #{jwt => JWT}),
     State1 = State0#{jwt := JWT},
     ensure_timer(State1).
 
--spec store_jwt(state(), jwt()) -> ok.
-store_jwt(#{resource_id := ResourceId, table := TId}, JWT) ->
-    true = ets:insert(TId, {{ResourceId, jwt}, JWT}),
-    ?tp(connector_jwt_worker_token_stored, #{resource_id => ResourceId}),
-    ok.
-
 -spec ensure_timer(state()) -> state().
 ensure_timer(
     State = #{

apps/emqx_connector/src/emqx_connector_ldap.erl

@@ -97,7 +97,6 @@ on_start(
         {pool_size, PoolSize},
         {auto_reconnect, ?AUTO_RECONNECT_INTERVAL}
     ],
-    ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
     case emqx_resource_pool:start(InstId, ?MODULE, Opts ++ SslOpts) of
         ok -> {ok, #{pool_name => InstId}};
         {error, Reason} -> {error, Reason}
@@ -108,12 +107,7 @@ on_stop(InstId, _State) ->
         msg => "stopping_ldap_connector",
         connector => InstId
     }),
-    case emqx_resource:get_allocated_resources(InstId) of
-        #{pool_name := PoolName} ->
-            emqx_resource_pool:stop(PoolName);
-        _ ->
-            ok
-    end.
+    emqx_resource_pool:stop(InstId).
 
 on_query(InstId, {search, Base, Filter, Attributes}, #{pool_name := PoolName} = State) ->
     Request = {Base, Filter, Attributes},

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -183,7 +183,6 @@ on_start(
         {worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)}
     ],
     Collection = maps:get(collection, Config, <<"mqtt">>),
-    ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
     case emqx_resource_pool:start(InstId, ?MODULE, Opts) of
         ok ->
             {ok, #{
@@ -200,12 +199,7 @@ on_stop(InstId, _State) ->
         msg => "stopping_mongodb_connector",
         connector => InstId
     }),
-    case emqx_resource:get_allocated_resources(InstId) of
-        #{pool_name := PoolName} ->
-            emqx_resource_pool:stop(PoolName);
-        _ ->
-            ok
-    end.
+    emqx_resource_pool:stop(InstId).
 
 on_query(
     InstId,

apps/emqx_connector/src/emqx_connector_mysql.erl

@@ -124,7 +124,6 @@ on_start(
             ]
         ),
     State = parse_prepare_sql(Config),
-    ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
     case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of
         ok ->
             {ok, init_prepare(State#{pool_name => InstId})};
@@ -146,12 +145,7 @@ on_stop(InstId, _State) ->
         msg => "stopping_mysql_connector",
         connector => InstId
     }),
-    case emqx_resource:get_allocated_resources(InstId) of
-        #{pool_name := PoolName} ->
-            emqx_resource_pool:stop(PoolName);
-        _ ->
-            ok
-    end.
+    emqx_resource_pool:stop(InstId).
 
 on_query(InstId, {TypeOrKey, SQLOrKey}, State) ->
     on_query(InstId, {TypeOrKey, SQLOrKey, [], default_timeout}, State);

apps/emqx_connector/src/emqx_connector_pgsql.erl

@@ -121,7 +121,6 @@ on_start(
         {pool_size, PoolSize}
     ],
     State = parse_prepare_sql(Config),
-    ok = emqx_resource:allocate_resource(InstId, pool_name, InstId),
     case emqx_resource_pool:start(InstId, ?MODULE, Options ++ SslOpts) of
         ok ->
             {ok, init_prepare(State#{pool_name => InstId, prepare_statement => #{}})};
@@ -138,12 +137,7 @@ on_stop(InstId, _State) ->
         msg => "stopping postgresql connector",
         connector => InstId
     }),
-    case emqx_resource:get_allocated_resources(InstId) of
-        #{pool_name := PoolName} ->
-            emqx_resource_pool:stop(PoolName);
-        _ ->
-            ok
-    end.
+    emqx_resource_pool:stop(InstId).
 
 on_query(InstId, {TypeOrKey, NameOrSQL}, State) ->
     on_query(InstId, {TypeOrKey, NameOrSQL, []}, State);

apps/emqx_connector/test/emqx_connector_jwt_SUITE.erl

@@ -18,7 +18,10 @@
 
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
+-include_lib("jose/include/jose_jwt.hrl").
+-include_lib("jose/include/jose_jws.hrl").
 -include("emqx_connector_tables.hrl").
+-include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -compile([export_all, nowarn_export_all]).
 
@@ -51,6 +54,33 @@ end_per_testcase(_TestCase, _Config) ->
 insert_jwt(TId, ResourceId, JWT) ->
     ets:insert(TId, {{ResourceId, jwt}, JWT}).
 
+generate_private_key_pem() ->
+    PublicExponent = 65537,
+    Size = 2048,
+    Key = public_key:generate_key({rsa, Size, PublicExponent}),
+    DERKey = public_key:der_encode('PrivateKeyInfo', Key),
+    public_key:pem_encode([{'PrivateKeyInfo', DERKey, not_encrypted}]).
+
+generate_config() ->
+    PrivateKeyPEM = generate_private_key_pem(),
+    ResourceID = emqx_guid:gen(),
+    #{
+        private_key => PrivateKeyPEM,
+        expiration => timer:hours(1),
+        resource_id => ResourceID,
+        table => ets:new(test_jwt_table, [ordered_set, public]),
+        iss => <<"issuer">>,
+        sub => <<"subject">>,
+        aud => <<"audience">>,
+        kid => <<"key id">>,
+        alg => <<"RS256">>
+    }.
+
+is_expired(JWT) ->
+    #jose_jwt{fields = #{<<"exp">> := Exp}} = jose_jwt:peek(JWT),
+    Now = erlang:system_time(seconds),
+    Now >= Exp.
+
 %%-----------------------------------------------------------------------------
 %% Test cases
 %%-----------------------------------------------------------------------------
@@ -77,3 +107,39 @@ t_delete_jwt(_Config) ->
     ?assertEqual(ok, emqx_connector_jwt:delete_jwt(TId, ResourceId)),
     ?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(TId, ResourceId)),
     ok.
+
+t_ensure_jwt(_Config) ->
+    Config0 =
+        #{
+            table := Table,
+            resource_id := ResourceId,
+            private_key := PrivateKeyPEM
+        } = generate_config(),
+    JWK = jose_jwk:from_pem(PrivateKeyPEM),
+    Config1 = maps:without([private_key], Config0),
+    Expiration = timer:seconds(10),
+    JWTConfig = Config1#{jwk => JWK, expiration := Expiration},
+    ?assertEqual({error, not_found}, emqx_connector_jwt:lookup_jwt(Table, ResourceId)),
+    ?check_trace(
+        begin
+            JWT0 = emqx_connector_jwt:ensure_jwt(JWTConfig),
+            ?assertNot(is_expired(JWT0)),
+            %% should refresh 5 s before expiration
+            ct:sleep(Expiration - 5500),
+            JWT1 = emqx_connector_jwt:ensure_jwt(JWTConfig),
+            ?assertNot(is_expired(JWT1)),
+            %% fully expired
+            ct:sleep(2 * Expiration),
+            JWT2 = emqx_connector_jwt:ensure_jwt(JWTConfig),
+            ?assertNot(is_expired(JWT2)),
+            {JWT0, JWT1, JWT2}
+        end,
+        fun({JWT0, JWT1, JWT2}, Trace) ->
+            ?assertNotEqual(JWT0, JWT1),
+            ?assertNotEqual(JWT1, JWT2),
+            ?assertNotEqual(JWT2, JWT0),
+            ?assertMatch([_, _, _], ?of_kind(emqx_connector_jwt_token_stored, Trace)),
+            ok
+        end
+    ),
+    ok.

apps/emqx_connector/test/emqx_connector_jwt_worker_SUITE.erl

@@ -176,7 +176,7 @@ t_refresh(_Config) ->
             {{ok, _Pid}, {ok, _Event}} =
                 ?wait_async_action(
                     emqx_connector_jwt_worker:start_link(Config),
-                    #{?snk_kind := connector_jwt_worker_token_stored},
+                    #{?snk_kind := emqx_connector_jwt_token_stored},
                     5_000
                 ),
             {ok, FirstJWT} = emqx_connector_jwt:lookup_jwt(Table, ResourceId),
@@ -209,7 +209,7 @@ t_refresh(_Config) ->
         fun({FirstJWT, SecondJWT, ThirdJWT}, Trace) ->
             ?assertMatch(
                 [_, _, _ | _],
-                ?of_kind(connector_jwt_worker_token_stored, Trace)
+                ?of_kind(emqx_connector_jwt_token_stored, Trace)
             ),
             ?assertNotEqual(FirstJWT, SecondJWT),
             ?assertNotEqual(SecondJWT, ThirdJWT),

apps/emqx_dashboard/include/emqx_dashboard.hrl

@@ -61,7 +61,8 @@
 -define(GAUGE_SAMPLER_LIST, [
     subscriptions,
     topics,
-    connections
+    connections,
+    live_connections
 ]).
 
 -define(SAMPLER_LIST, ?GAUGE_SAMPLER_LIST ++ ?DELTA_SAMPLER_LIST).

apps/emqx_dashboard/src/emqx_dashboard_monitor.erl

@@ -401,6 +401,7 @@ getstats(Key) ->
     end.
 
 stats(connections) -> emqx_stats:getstat('connections.count');
+stats(live_connections) -> emqx_stats:getstat('live_connections.count');
 stats(topics) -> emqx_stats:getstat('topics.count');
 stats(subscriptions) -> emqx_stats:getstat('subscriptions.count');
 stats(received) -> emqx_metrics:val('messages.received');

apps/emqx_dashboard/src/emqx_dashboard_monitor_api.erl

@@ -171,6 +171,11 @@ swagger_desc(topics) ->
         " Can only represent the approximate state"
     >>;
 swagger_desc(connections) ->
+    <<
+        "Sessions at the time of sampling."
+        " Can only represent the approximate state"
+    >>;
+swagger_desc(live_connections) ->
     <<
         "Connections at the time of sampling."
         " Can only represent the approximate state"

apps/emqx_dashboard/test/emqx_dashboard_monitor_SUITE.erl

@@ -90,6 +90,27 @@ t_monitor_current_api(_) ->
     ],
     ok.
 
+t_monitor_current_api_live_connections(_) ->
+    process_flag(trap_exit, true),
+    ClientId = <<"live_conn_tests">>,
+    ClientId1 = <<"live_conn_tests1">>,
+    {ok, C} = emqtt:start_link([{clean_start, false}, {clientid, ClientId}]),
+    {ok, _} = emqtt:connect(C),
+    ok = emqtt:disconnect(C),
+    {ok, C1} = emqtt:start_link([{clean_start, true}, {clientid, ClientId1}]),
+    {ok, _} = emqtt:connect(C1),
+    %% waiting for emqx_stats ticker
+    timer:sleep(1500),
+    _ = emqx_dashboard_monitor:current_rate(),
+    {ok, Rate} = request(["monitor_current"]),
+    ?assertEqual(1, maps:get(<<"live_connections">>, Rate)),
+    ?assertEqual(2, maps:get(<<"connections">>, Rate)),
+    %% clears
+    ok = emqtt:disconnect(C1),
+    {ok, C2} = emqtt:start_link([{clean_start, true}, {clientid, ClientId}]),
+    {ok, _} = emqtt:connect(C2),
+    ok = emqtt:disconnect(C2).
+
 t_monitor_reset(_) ->
     restart_monitor(),
     {ok, Rate} = request(["monitor_current"]),

apps/emqx_eviction_agent/test/emqx_eviction_agent_SUITE.erl

@@ -11,6 +11,7 @@
 -include_lib("common_test/include/ct.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx/include/asserts.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 
 -import(
     emqx_eviction_agent_test_helpers,
@@ -295,7 +296,7 @@ t_session_serialization(_Config) ->
     ?assertMatch(
         #{data := [#{clientid := <<"client_with_session">>}]},
         emqx_mgmt_api:cluster_query(
-            emqx_channel_info,
+            ?CHAN_INFO_TAB,
             #{},
             [],
             fun emqx_mgmt_api_clients:qs2ms/2,

apps/emqx_exhook/src/emqx_exhook_app.erl

@@ -22,8 +22,7 @@
 
 -export([
     start/2,
-    stop/1,
-    prep_stop/1
+    stop/1
 ]).
 
 %%--------------------------------------------------------------------
@@ -34,10 +33,6 @@ start(_StartType, _StartArgs) ->
     {ok, Sup} = emqx_exhook_sup:start_link(),
     {ok, Sup}.
 
-prep_stop(State) ->
-    emqx_ctl:unregister_command(exhook),
-    State.
-
 stop(_State) ->
     ok.
 

apps/emqx_exhook/src/emqx_exhook_mgr.erl

@@ -23,6 +23,9 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
+-define(SERVERS, [exhook, servers]).
+-define(EXHOOK, [exhook]).
+
 %% APIs
 -export([start_link/0]).
 
@@ -148,7 +151,7 @@ update_config(KeyPath, UpdateReq) ->
             Error
     end.
 
-pre_config_update(_, {add, #{<<"name">> := Name} = Conf}, OldConf) ->
+pre_config_update(?SERVERS, {add, #{<<"name">> := Name} = Conf}, OldConf) ->
     case lists:any(fun(#{<<"name">> := ExistedName}) -> ExistedName =:= Name end, OldConf) of
         true ->
             throw(already_exists);
@@ -156,47 +159,35 @@ pre_config_update(_, {add, #{<<"name">> := Name} = Conf}, OldConf) ->
             NConf = maybe_write_certs(Conf),
             {ok, OldConf ++ [NConf]}
     end;
-pre_config_update(_, {update, Name, Conf}, OldConf) ->
-    case replace_conf(Name, fun(_) -> Conf end, OldConf) of
-        not_found -> throw(not_found);
-        NewConf -> {ok, lists:map(fun maybe_write_certs/1, NewConf)}
-    end;
-pre_config_update(_, {delete, ToDelete}, OldConf) ->
-    case do_delete(ToDelete, OldConf) of
-        not_found -> throw(not_found);
-        NewConf -> {ok, NewConf}
-    end;
-pre_config_update(_, {move, Name, Position}, OldConf) ->
-    case do_move(Name, Position, OldConf) of
-        not_found -> throw(not_found);
-        NewConf -> {ok, NewConf}
-    end;
-pre_config_update(_, {enable, Name, Enable}, OldConf) ->
-    case
-        replace_conf(
-            Name,
-            fun(Conf) -> Conf#{<<"enable">> => Enable} end,
-            OldConf
-        )
-    of
-        not_found -> throw(not_found);
-        NewConf -> {ok, lists:map(fun maybe_write_certs/1, NewConf)}
-    end.
-
-post_config_update(_KeyPath, UpdateReq, NewConf, _OldConf, _AppEnvs) ->
-    Result = call({update_config, UpdateReq, NewConf}),
+pre_config_update(?SERVERS, {update, Name, Conf}, OldConf) ->
+    NewConf = replace_conf(Name, fun(_) -> Conf end, OldConf),
+    {ok, lists:map(fun maybe_write_certs/1, NewConf)};
+pre_config_update(?SERVERS, {delete, ToDelete}, OldConf) ->
+    {ok, do_delete(ToDelete, OldConf)};
+pre_config_update(?SERVERS, {move, Name, Position}, OldConf) ->
+    {ok, do_move(Name, Position, OldConf)};
+pre_config_update(?SERVERS, {enable, Name, Enable}, OldConf) ->
+    ReplaceFun = fun(Conf) -> Conf#{<<"enable">> => Enable} end,
+    NewConf = replace_conf(Name, ReplaceFun, OldConf),
+    {ok, lists:map(fun maybe_write_certs/1, NewConf)};
+pre_config_update(?EXHOOK, NewConf, _OldConf) when NewConf =:= #{} ->
+    {ok, NewConf#{<<"servers">> => []}};
+pre_config_update(?EXHOOK, NewConf = #{<<"servers">> := Servers}, _OldConf) ->
+    {ok, NewConf#{<<"servers">> => lists:map(fun maybe_write_certs/1, Servers)}}.
+
+post_config_update(_KeyPath, UpdateReq, NewConf, OldConf, _AppEnvs) ->
+    Result = call({update_config, UpdateReq, NewConf, OldConf}),
     {ok, Result}.
 
-%%=====================================================================
-
 %%--------------------------------------------------------------------
 %% gen_server callbacks
 %%--------------------------------------------------------------------
 
 init([]) ->
     process_flag(trap_exit, true),
-    emqx_conf:add_handler([exhook, servers], ?MODULE),
-    ServerL = emqx:get_config([exhook, servers]),
+    emqx_conf:add_handler(?EXHOOK, ?MODULE),
+    emqx_conf:add_handler(?SERVERS, ?MODULE),
+    ServerL = emqx:get_config(?SERVERS),
     Servers = load_all_servers(ServerL),
     Servers2 = reorder(ServerL, Servers),
     refresh_tick(),
@@ -221,22 +212,16 @@ handle_call(
     OrderServers = sort_name_by_order(Infos, Servers),
     {reply, OrderServers, State};
 handle_call(
-    {update_config, {move, _Name, _Position}, NewConfL},
+    {update_config, {move, _Name, _Position}, NewConfL, _},
     _From,
     #{servers := Servers} = State
 ) ->
     Servers2 = reorder(NewConfL, Servers),
     {reply, ok, State#{servers := Servers2}};
-handle_call({update_config, {delete, ToDelete}, _}, _From, State) ->
-    emqx_exhook_metrics:on_server_deleted(ToDelete),
-
-    #{servers := Servers} = State2 = do_unload_server(ToDelete, State),
-
-    Servers2 = maps:remove(ToDelete, Servers),
-
-    {reply, ok, update_servers(Servers2, State2)};
+handle_call({update_config, {delete, ToDelete}, _, _}, _From, State) ->
+    {reply, ok, remove_server(ToDelete, State)};
 handle_call(
-    {update_config, {add, RawConf}, NewConfL},
+    {update_config, {add, RawConf}, NewConfL, _},
     _From,
     #{servers := Servers} = State
 ) ->
@@ -245,14 +230,30 @@ handle_call(
     Servers2 = Servers#{Name => Server},
     Servers3 = reorder(NewConfL, Servers2),
     {reply, Result, State#{servers := Servers3}};
-handle_call({lookup, Name}, _From, State) ->
-    {reply, where_is_server(Name, State), State};
-handle_call({update_config, {update, Name, _Conf}, NewConfL}, _From, State) ->
+handle_call({update_config, {update, Name, _Conf}, NewConfL, _}, _From, State) ->
     {Result, State2} = restart_server(Name, NewConfL, State),
     {reply, Result, State2};
-handle_call({update_config, {enable, Name, _Enable}, NewConfL}, _From, State) ->
+handle_call({update_config, {enable, Name, _Enable}, NewConfL, _}, _From, State) ->
     {Result, State2} = restart_server(Name, NewConfL, State),
     {reply, Result, State2};
+handle_call({update_config, _, ConfL, ConfL}, _From, State) ->
+    {reply, ok, State};
+handle_call({update_config, _, #{servers := NewConfL}, #{servers := OldConfL}}, _From, State) ->
+    #{
+        removed := Removed,
+        added := Added,
+        changed := Updated
+    } = emqx_utils:diff_lists(NewConfL, OldConfL, fun(#{name := Name}) -> Name end),
+    State2 = remove_servers(Removed, State),
+    {UpdateRes, State3} = restart_servers(Updated, NewConfL, State2),
+    {AddRes, State4 = #{servers := Servers4}} = add_servers(Added, State3),
+    State5 = State4#{servers => reorder(NewConfL, Servers4)},
+    case UpdateRes =:= [] andalso AddRes =:= [] of
+        true -> {reply, ok, State5};
+        false -> {reply, {error, #{added => AddRes, updated => UpdateRes}}, State5}
+    end;
+handle_call({lookup, Name}, _From, State) ->
+    {reply, where_is_server(Name, State), State};
 handle_call({server_info, Name}, _From, State) ->
     case where_is_server(Name, State) of
         not_found ->
@@ -286,6 +287,22 @@ handle_call(_Request, _From, State) ->
     Reply = ok,
     {reply, Reply, State}.
 
+remove_servers(Removes, State) ->
+    lists:foldl(
+        fun(Conf, Acc) ->
+            ToDelete = maps:get(name, Conf),
+            remove_server(ToDelete, Acc)
+        end,
+        State,
+        Removes
+    ).
+
+remove_server(ToDelete, State) ->
+    emqx_exhook_metrics:on_server_deleted(ToDelete),
+    #{servers := Servers} = State2 = do_unload_server(ToDelete, State),
+    Servers2 = maps:remove(ToDelete, Servers),
+    update_servers(Servers2, State2).
+
 handle_cast(_Msg, State) ->
     {noreply, State}.
 
@@ -309,6 +326,8 @@ terminate(Reason, State = #{servers := Servers}) ->
         Servers
     ),
     ?tp(info, exhook_mgr_terminated, #{reason => Reason, servers => Servers}),
+    emqx_conf:remove_handler(?SERVERS),
+    emqx_conf:remove_handler(?EXHOOK),
     ok.
 
 code_change(_OldVsn, State, _Extra) ->
@@ -324,6 +343,22 @@ unload_exhooks() ->
      || {Name, {M, F, _A}} <- ?ENABLED_HOOKS
     ].
 
+add_servers(Added, State) ->
+    lists:foldl(
+        fun(Conf = #{name := Name}, {ResAcc, StateAcc}) ->
+            case do_load_server(options_to_server(Conf)) of
+                {ok, Server} ->
+                    #{servers := Servers} = StateAcc,
+                    Servers2 = Servers#{Name => Server},
+                    {ResAcc, update_servers(Servers2, StateAcc)};
+                {Err, StateAcc1} ->
+                    {[Err | ResAcc], StateAcc1}
+            end
+        end,
+        {[], State},
+        Added
+    ).
+
 do_load_server(#{name := Name} = Server) ->
     case emqx_exhook_server:load(Name, Server) of
         {ok, ServerState} ->
@@ -400,8 +435,7 @@ clean_reload_timer(#{timer := Timer}) ->
     _ = erlang:cancel_timer(Timer),
     ok.
 
--spec do_move(binary(), position(), list(server_options())) ->
-    not_found | list(server_options()).
+-spec do_move(binary(), position(), list(server_options())) -> list(server_options()).
 do_move(Name, Position, ConfL) ->
     move(ConfL, Name, Position, []).
 
@@ -410,7 +444,7 @@ move([#{<<"name">> := Name} = Server | T], Name, Position, HeadL) ->
 move([Server | T], Name, Position, HeadL) ->
     move(T, Name, Position, [Server | HeadL]);
 move([], _Name, _Position, _HeadL) ->
-    not_found.
+    throw(not_found).
 
 move_to(?CMD_MOVE_FRONT, Server, ServerL) ->
     [Server | ServerL];
@@ -428,8 +462,7 @@ move_to([H | T], Position, Server, HeadL) ->
 move_to([], _Position, _Server, _HeadL) ->
     not_found.
 
--spec do_delete(binary(), list(server_options())) ->
-    not_found | list(server_options()).
+-spec do_delete(binary(), list(server_options())) -> list(server_options()).
 do_delete(ToDelete, OldConf) ->
     case lists:any(fun(#{<<"name">> := ExistedName}) -> ExistedName =:= ToDelete end, OldConf) of
         true ->
@@ -438,7 +471,7 @@ do_delete(ToDelete, OldConf) ->
                 OldConf
             );
         false ->
-            not_found
+            throw(not_found)
     end.
 
 -spec reorder(list(server_options()), servers()) -> servers().
@@ -470,9 +503,7 @@ where_is_server(Name, #{servers := Servers}) ->
 
 -type replace_fun() :: fun((server_options()) -> server_options()).
 
--spec replace_conf(binary(), replace_fun(), list(server_options())) ->
-    not_found
-    | list(server_options()).
+-spec replace_conf(binary(), replace_fun(), list(server_options())) -> list(server_options()).
 replace_conf(Name, ReplaceFun, ConfL) ->
     replace_conf(ConfL, Name, ReplaceFun, []).
 
@@ -482,7 +513,20 @@ replace_conf([#{<<"name">> := Name} = H | T], Name, ReplaceFun, HeadL) ->
 replace_conf([H | T], Name, ReplaceFun, HeadL) ->
     replace_conf(T, Name, ReplaceFun, [H | HeadL]);
 replace_conf([], _, _, _) ->
-    not_found.
+    throw(not_found).
+
+restart_servers(Updated, NewConfL, State) ->
+    lists:foldl(
+        fun({_Old, Conf}, {ResAcc, StateAcc}) ->
+            Name = maps:get(name, Conf),
+            case restart_server(Name, NewConfL, StateAcc) of
+                {ok, StateAcc1} -> {ResAcc, StateAcc1};
+                {Err, StateAcc1} -> {[Err | ResAcc], StateAcc1}
+            end
+        end,
+        {[], State},
+        Updated
+    ).
 
 -spec restart_server(binary(), list(server_options()), state()) ->
     {ok, state()}

apps/emqx_exhook/test/emqx_exhook_SUITE.erl

@@ -196,9 +196,9 @@ t_error_update_conf(_) ->
     Path = [exhook, servers],
     Name = <<"error_update">>,
     ErrorCfg = #{<<"name">> => Name},
-    {error, _} = emqx_exhook_mgr:update_config(Path, {update, Name, ErrorCfg}),
-    {error, _} = emqx_exhook_mgr:update_config(Path, {move, Name, top, <<>>}),
-    {error, _} = emqx_exhook_mgr:update_config(Path, {enable, Name, true}),
+    {error, not_found} = emqx_exhook_mgr:update_config(Path, {update, Name, ErrorCfg}),
+    {error, not_found} = emqx_exhook_mgr:update_config(Path, {move, Name, top}),
+    {error, not_found} = emqx_exhook_mgr:update_config(Path, {enable, Name, true}),
 
     ErrorAnd = #{<<"name">> => Name, <<"url">> => <<"http://127.0.0.1:9001">>},
     {ok, _} = emqx_exhook_mgr:update_config(Path, {add, ErrorAnd}),
@@ -210,12 +210,37 @@ t_error_update_conf(_) ->
     },
     {ok, _} = emqx_exhook_mgr:update_config(Path, {update, Name, DisableAnd}),
 
-    {ok, _} = emqx_exhook_mgr:update_config(Path, {delete, <<"error">>}),
-    {error, not_found} = emqx_exhook_mgr:update_config(
-        Path, {delete, <<"delete_not_exists">>}
-    ),
+    {ok, _} = emqx_exhook_mgr:update_config(Path, {delete, Name}),
+    {error, not_found} = emqx_exhook_mgr:update_config(Path, {delete, Name}),
+    ok.
+
+t_update_conf(_Config) ->
+    Path = [exhook],
+    Conf = #{<<"servers">> := Servers} = emqx_config:get_raw(Path),
+    ?assert(length(Servers) > 1),
+    Servers1 = shuffle(Servers),
+    ReOrderedConf = Conf#{<<"servers">> => Servers1},
+    validate_servers(Path, ReOrderedConf, Servers1),
+    [_ | Servers2] = Servers,
+    DeletedConf = Conf#{<<"servers">> => Servers2},
+    validate_servers(Path, DeletedConf, Servers2),
+    [L1, L2 | Servers3] = Servers,
+    UpdateL2 = L2#{<<"pool_size">> => 1, <<"request_timeout">> => 1000},
+    UpdatedServers = [L1, UpdateL2 | Servers3],
+    UpdatedConf = Conf#{<<"servers">> => UpdatedServers},
+    validate_servers(Path, UpdatedConf, UpdatedServers),
+    %% reset
+    validate_servers(Path, Conf, Servers),
     ok.
 
+validate_servers(Path, ReOrderConf, Servers1) ->
+    {ok, _} = emqx_exhook_mgr:update_config(Path, ReOrderConf),
+    ?assertEqual(ReOrderConf, emqx_config:get_raw(Path)),
+    List = emqx_exhook_mgr:list(),
+    ExpectL = lists:map(fun(#{<<"name">> := Name}) -> Name end, Servers1),
+    L1 = lists:map(fun(#{name := Name}) -> Name end, List),
+    ?assertEqual(ExpectL, L1).
+
 t_error_server_info(_) ->
     not_found = emqx_exhook_mgr:server_info(<<"not_exists">>),
     ok.
@@ -483,6 +508,10 @@ data_file(Name) ->
 cert_file(Name) ->
     data_file(filename:join(["certs", Name])).
 
-%% FIXME: this creats inter-test dependency
+%% FIXME: this creates inter-test dependency
 stop_apps(Apps) ->
     emqx_common_test_helpers:stop_apps(Apps, #{erase_all_configs => false}).
+
+shuffle(List) ->
+    Sorted = lists:sort(lists:map(fun(L) -> {rand:uniform(), L} end, List)),
+    lists:map(fun({_, L}) -> L end, Sorted).

apps/emqx_gateway/src/emqx_gateway.erl

@@ -16,7 +16,7 @@
 
 -module(emqx_gateway).
 
--include("include/emqx_gateway.hrl").
+-include("emqx_gateway.hrl").
 
 %% Gateway APIs
 -export([

apps/emqx_gateway/src/emqx_gateway_api_listeners.erl

@@ -723,7 +723,8 @@ examples_listener() ->
                                 buffer => <<"10KB">>,
                                 high_watermark => <<"1MB">>,
                                 nodelay => false,
-                                reuseaddr => true
+                                reuseaddr => true,
+                                keepalive => "none"
                             }
                     }
             },

apps/emqx_gateway/src/emqx_gateway_conf.erl

@@ -74,18 +74,20 @@
 -type listener_ref() :: {ListenerType :: atom_or_bin(), ListenerName :: atom_or_bin()}.
 
 -define(IS_SSL(T), (T == <<"ssl_options">> orelse T == <<"dtls_options">>)).
+-define(IGNORE_KEYS, [<<"listeners">>, ?AUTHN_BIN]).
 
 %%--------------------------------------------------------------------
 %%  Load/Unload
 %%--------------------------------------------------------------------
+-define(GATEWAY, [gateway]).
 
 -spec load() -> ok.
 load() ->
-    emqx_conf:add_handler([gateway], ?MODULE).
+    emqx_conf:add_handler(?GATEWAY, ?MODULE).
 
 -spec unload() -> ok.
 unload() ->
-    emqx_conf:remove_handler([gateway]).
+    emqx_conf:remove_handler(?GATEWAY).
 
 %%--------------------------------------------------------------------
 %% APIs
@@ -104,7 +106,7 @@ unconvert_listeners(Ls) when is_list(Ls) ->
     lists:foldl(
         fun(Lis, Acc) ->
             {[Type, Name], Lis1} = maps_key_take([<<"type">>, <<"name">>], Lis),
-            _ = vaildate_listener_name(Name),
+            _ = validate_listener_name(Name),
             NLis1 = maps:without([<<"id">>, <<"running">>], Lis1),
             emqx_utils_maps:deep_merge(Acc, #{Type => #{Name => NLis1}})
         end,
@@ -122,7 +124,7 @@ maps_key_take([K | Ks], M, Acc) ->
         {V, M1} -> maps_key_take(Ks, M1, [V | Acc])
     end.
 
-vaildate_listener_name(Name) ->
+validate_listener_name(Name) ->
     try
         {match, _} = re:run(Name, "^[0-9a-zA-Z_-]+$"),
         ok
@@ -373,7 +375,7 @@ ret_listener_or_err(_, _, Err) ->
     emqx_config:raw_config()
 ) ->
     {ok, emqx_config:update_request()} | {error, term()}.
-pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) ->
+pre_config_update(?GATEWAY, {load_gateway, GwName, Conf}, RawConf) ->
     case maps:get(GwName, RawConf, undefined) of
         undefined ->
             NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf),
@@ -381,24 +383,20 @@ pre_config_update(_, {load_gateway, GwName, Conf}, RawConf) ->
         _ ->
             badres_gateway(already_exist, GwName)
     end;
-pre_config_update(_, {update_gateway, GwName, Conf}, RawConf) ->
+pre_config_update(?GATEWAY, {update_gateway, GwName, Conf}, RawConf) ->
     case maps:get(GwName, RawConf, undefined) of
         undefined ->
             badres_gateway(not_found, GwName);
         GwRawConf ->
-            Conf1 = maps:without([<<"listeners">>, ?AUTHN_BIN], Conf),
+            Conf1 = maps:without(?IGNORE_KEYS, Conf),
             NConf = tune_gw_certs(fun convert_certs/2, GwName, Conf1),
             NConf1 = maps:merge(GwRawConf, NConf),
             {ok, emqx_utils_maps:deep_put([GwName], RawConf, NConf1)}
     end;
-pre_config_update(_, {unload_gateway, GwName}, RawConf) ->
+pre_config_update(?GATEWAY, {unload_gateway, GwName}, RawConf) ->
     {ok, maps:remove(GwName, RawConf)};
-pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, <<"listeners">>, LType, LName], RawConf, undefined
-        )
-    of
+pre_config_update(?GATEWAY, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             NConf = convert_certs(certs_dir(GwName), Conf),
             NListener = #{LType => #{LName => NConf}},
@@ -410,12 +408,8 @@ pre_config_update(_, {add_listener, GwName, {LType, LName}, Conf}, RawConf) ->
         _ ->
             badres_listener(already_exist, GwName, LType, LName)
     end;
-pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, <<"listeners">>, LType, LName], RawConf, undefined
-        )
-    of
+pre_config_update(?GATEWAY, {update_listener, GwName, {LType, LName}, Conf}, RawConf) ->
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             badres_listener(not_found, GwName, LType, LName);
         _OldConf ->
@@ -427,20 +421,16 @@ pre_config_update(_, {update_listener, GwName, {LType, LName}, Conf}, RawConf) -
             ),
             {ok, NRawConf}
     end;
-pre_config_update(_, {remove_listener, GwName, {LType, LName}}, RawConf) ->
-    Path = [GwName, <<"listeners">>, LType, LName],
-    case emqx_utils_maps:deep_get(Path, RawConf, undefined) of
+pre_config_update(?GATEWAY, {remove_listener, GwName, {LType, LName}}, RawConf) ->
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             {ok, RawConf};
         _OldConf ->
+            Path = [GwName, <<"listeners">>, LType, LName],
             {ok, emqx_utils_maps:deep_remove(Path, RawConf)}
     end;
-pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, ?AUTHN_BIN], RawConf, undefined
-        )
-    of
+pre_config_update(?GATEWAY, {add_authn, GwName, Conf}, RawConf) ->
+    case get_authn(GwName, RawConf) of
         undefined ->
             CertsDir = authn_certs_dir(GwName, Conf),
             Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf),
@@ -452,14 +442,8 @@ pre_config_update(_, {add_authn, GwName, Conf}, RawConf) ->
         _ ->
             badres_authn(already_exist, GwName)
     end;
-pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
-    case
-        emqx_utils_maps:deep_get(
-            [GwName, <<"listeners">>, LType, LName],
-            RawConf,
-            undefined
-        )
-    of
+pre_config_update(?GATEWAY, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             badres_listener(not_found, GwName, LType, LName);
         Listener ->
@@ -480,9 +464,9 @@ pre_config_update(_, {add_authn, GwName, {LType, LName}, Conf}, RawConf) ->
                     badres_listener_authn(already_exist, GwName, LType, LName)
             end
     end;
-pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
+pre_config_update(?GATEWAY, {update_authn, GwName, Conf}, RawConf) ->
     Path = [GwName, ?AUTHN_BIN],
-    case emqx_utils_maps:deep_get(Path, RawConf, undefined) of
+    case get_authn(GwName, RawConf) of
         undefined ->
             badres_authn(not_found, GwName);
         _OldConf ->
@@ -490,9 +474,9 @@ pre_config_update(_, {update_authn, GwName, Conf}, RawConf) ->
             Conf1 = emqx_authentication_config:convert_certs(CertsDir, Conf),
             {ok, emqx_utils_maps:deep_put(Path, RawConf, Conf1)}
     end;
-pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
+pre_config_update(?GATEWAY, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
     Path = [GwName, <<"listeners">>, LType, LName],
-    case emqx_utils_maps:deep_get(Path, RawConf, undefined) of
+    case get_listener(GwName, LType, LName, RawConf) of
         undefined ->
             badres_listener(not_found, GwName, LType, LName);
         Listener ->
@@ -510,16 +494,190 @@ pre_config_update(_, {update_authn, GwName, {LType, LName}, Conf}, RawConf) ->
                     {ok, emqx_utils_maps:deep_put(Path, RawConf, NListener)}
             end
     end;
-pre_config_update(_, {remove_authn, GwName}, RawConf) ->
+pre_config_update(?GATEWAY, {remove_authn, GwName}, RawConf) ->
     Path = [GwName, ?AUTHN_BIN],
     {ok, emqx_utils_maps:deep_remove(Path, RawConf)};
-pre_config_update(_, {remove_authn, GwName, {LType, LName}}, RawConf) ->
+pre_config_update(?GATEWAY, {remove_authn, GwName, {LType, LName}}, RawConf) ->
     Path = [GwName, <<"listeners">>, LType, LName, ?AUTHN_BIN],
     {ok, emqx_utils_maps:deep_remove(Path, RawConf)};
-pre_config_update(_, UnknownReq, _RawConf) ->
-    logger:error("Unknown configuration update request: ~0p", [UnknownReq]),
+pre_config_update(?GATEWAY, NewRawConf0 = #{}, OldRawConf = #{}) ->
+    %% FIXME don't support gateway's listener's authn update.
+    %% load all authentications
+    NewRawConf1 = pre_load_authentications(NewRawConf0, OldRawConf),
+    %% load all listeners
+    NewRawConf2 = pre_load_listeners(NewRawConf1, OldRawConf),
+    %% load all gateway
+    NewRawConf3 = pre_load_gateways(NewRawConf2, OldRawConf),
+    {ok, NewRawConf3};
+pre_config_update(Path, UnknownReq, _RawConf) ->
+    ?SLOG(error, #{
+        msg => "unknown_gateway_update_request",
+        request => UnknownReq,
+        path => Path
+    }),
     {error, badreq}.
 
+pre_load_gateways(NewConf, OldConf) ->
+    %% unload old gateways
+    maps:foreach(
+        fun(GwName, _OldGwConf) ->
+            case maps:find(GwName, NewConf) of
+                error -> pre_config_update(?GATEWAY, {unload_gateway, GwName}, OldConf);
+                _ -> ok
+            end
+        end,
+        OldConf
+    ),
+    %% load/update gateways
+    maps:map(
+        fun(GwName, NewGwConf) ->
+            case maps:find(GwName, OldConf) of
+                {ok, NewGwConf} ->
+                    NewGwConf;
+                {ok, _OldGwConf} ->
+                    {ok, #{GwName := NewGwConf1}} = pre_config_update(
+                        ?GATEWAY, {update_gateway, GwName, NewGwConf}, OldConf
+                    ),
+                    %% update gateway should pass through ignore keys(listener/authn)
+                    PassThroughConf = maps:with(?IGNORE_KEYS, NewGwConf),
+                    NewGwConf2 = maps:without(?IGNORE_KEYS, NewGwConf1),
+                    maps:merge(NewGwConf2, PassThroughConf);
+                error ->
+                    {ok, #{GwName := NewGwConf1}} = pre_config_update(
+                        ?GATEWAY, {load_gateway, GwName, NewGwConf}, OldConf
+                    ),
+                    NewGwConf1
+            end
+        end,
+        NewConf
+    ).
+
+pre_load_listeners(NewConf, OldConf) ->
+    %% remove listeners
+    maps:foreach(
+        fun(GwName, GwConf) ->
+            Listeners = maps:get(<<"listeners">>, GwConf, #{}),
+            remove_listeners(GwName, NewConf, OldConf, Listeners)
+        end,
+        OldConf
+    ),
+    %% add/update listeners
+    maps:map(
+        fun(GwName, GwConf) ->
+            Listeners = maps:get(<<"listeners">>, GwConf, #{}),
+            NewListeners = create_or_update_listeners(GwName, OldConf, Listeners),
+            maps:put(<<"listeners">>, NewListeners, GwConf)
+        end,
+        NewConf
+    ).
+
+create_or_update_listeners(GwName, OldConf, Listeners) ->
+    maps:map(
+        fun(LType, LConf) ->
+            maps:map(
+                fun(LName, LConf1) ->
+                    NConf =
+                        case get_listener(GwName, LType, LName, OldConf) of
+                            undefined ->
+                                {ok, NConf0} =
+                                    pre_config_update(
+                                        ?GATEWAY,
+                                        {add_listener, GwName, {LType, LName}, LConf1},
+                                        OldConf
+                                    ),
+                                NConf0;
+                            _ ->
+                                {ok, NConf0} =
+                                    pre_config_update(
+                                        ?GATEWAY,
+                                        {update_listener, GwName, {LType, LName}, LConf1},
+                                        OldConf
+                                    ),
+                                NConf0
+                        end,
+                    get_listener(GwName, LType, LName, NConf)
+                end,
+                LConf
+            )
+        end,
+        Listeners
+    ).
+
+remove_listeners(GwName, NewConf, OldConf, Listeners) ->
+    maps:foreach(
+        fun(LType, LConf) ->
+            maps:foreach(
+                fun(LName, _LConf1) ->
+                    case get_listener(GwName, LType, LName, NewConf) of
+                        undefined ->
+                            pre_config_update(
+                                ?GATEWAY, {remove_listener, GwName, {LType, LName}}, OldConf
+                            );
+                        _ ->
+                            ok
+                    end
+                end,
+                LConf
+            )
+        end,
+        Listeners
+    ).
+
+get_listener(GwName, LType, LName, NewConf) ->
+    emqx_utils_maps:deep_get(
+        [GwName, <<"listeners">>, LType, LName], NewConf, undefined
+    ).
+
+get_authn(GwName, Conf) ->
+    emqx_utils_maps:deep_get([GwName, ?AUTHN_BIN], Conf, undefined).
+
+pre_load_authentications(NewConf, OldConf) ->
+    %% remove authentications when not in new config
+    maps:foreach(
+        fun(GwName, OldGwConf) ->
+            case
+                maps:get(?AUTHN_BIN, OldGwConf, undefined) =/= undefined andalso
+                    get_authn(GwName, NewConf) =:= undefined
+            of
+                true ->
+                    pre_config_update(?GATEWAY, {remove_authn, GwName}, OldConf);
+                false ->
+                    ok
+            end
+        end,
+        OldConf
+    ),
+    %% add/update authentications
+    maps:map(
+        fun(GwName, NewGwConf) ->
+            case get_authn(GwName, OldConf) of
+                undefined ->
+                    case maps:get(?AUTHN_BIN, NewGwConf, undefined) of
+                        undefined ->
+                            NewGwConf;
+                        AuthN ->
+                            {ok, #{GwName := #{?AUTHN_BIN := NAuthN}}} =
+                                pre_config_update(?GATEWAY, {add_authn, GwName, AuthN}, OldConf),
+                            maps:put(?AUTHN_BIN, NAuthN, NewGwConf)
+                    end;
+                OldAuthN ->
+                    case maps:get(?AUTHN_BIN, NewGwConf, undefined) of
+                        undefined ->
+                            NewGwConf;
+                        OldAuthN ->
+                            NewGwConf;
+                        NewAuthN ->
+                            {ok, #{GwName := #{?AUTHN_BIN := NAuthN}}} =
+                                pre_config_update(
+                                    ?GATEWAY, {update_authn, GwName, NewAuthN}, OldConf
+                                ),
+                            maps:put(?AUTHN_BIN, NAuthN, NewGwConf)
+                    end
+            end
+        end,
+        NewConf
+    ).
+
 badres_gateway(not_found, GwName) ->
     {error,
         {badres, #{
@@ -593,7 +751,7 @@ badres_listener_authn(already_exist, GwName, LType, LName) ->
 ) ->
     ok | {ok, Result :: any()} | {error, Reason :: term()}.
 
-post_config_update(_, Req, NewConfig, OldConfig, _AppEnvs) when is_tuple(Req) ->
+post_config_update(?GATEWAY, Req, NewConfig, OldConfig, _AppEnvs) when is_tuple(Req) ->
     [_Tag, GwName0 | _] = tuple_to_list(Req),
     GwName = binary_to_existing_atom(GwName0),
 
@@ -608,11 +766,35 @@ post_config_update(_, Req, NewConfig, OldConfig, _AppEnvs) when is_tuple(Req) ->
         {New, Old} when is_map(New), is_map(Old) ->
             emqx_gateway:update(GwName, New)
     end;
-post_config_update(_, _Req, _NewConfig, _OldConfig, _AppEnvs) ->
+post_config_update(?GATEWAY, _Req = #{}, NewConfig, OldConfig, _AppEnvs) ->
+    %% unload gateways
+    maps:foreach(
+        fun(GwName, _OldGwConf) ->
+            case maps:get(GwName, NewConfig, undefined) of
+                undefined ->
+                    emqx_gateway:unload(GwName);
+                _ ->
+                    ok
+            end
+        end,
+        OldConfig
+    ),
+    %% load/update gateways
+    maps:foreach(
+        fun(GwName, NewGwConf) ->
+            case maps:get(GwName, OldConfig, undefined) of
+                undefined ->
+                    emqx_gateway:load(GwName, NewGwConf);
+                _ ->
+                    emqx_gateway:update(GwName, NewGwConf)
+            end
+        end,
+        NewConfig
+    ),
     ok.
 
 %%--------------------------------------------------------------------
-%% Internal funcs
+%% Internal functions
 %%--------------------------------------------------------------------
 
 tune_gw_certs(Fun, GwName, Conf) ->

apps/emqx_gateway/src/emqx_gateway_metrics.erl

@@ -18,7 +18,7 @@
 
 -behaviour(gen_server).
 
--include_lib("emqx_gateway/include/emqx_gateway.hrl").
+-include("emqx_gateway.hrl").
 
 %% APIs
 -export([start_link/1]).

apps/emqx_gateway/src/emqx_gateway_schema.erl

@@ -120,7 +120,10 @@ fields(ssl_listener) ->
             {ssl_options,
                 sc(
                     hoconsc:ref(emqx_schema, "listener_ssl_opts"),
-                    #{desc => ?DESC(ssl_listener_options)}
+                    #{
+                        desc => ?DESC(ssl_listener_options),
+                        validator => fun emqx_schema:validate_server_ssl_opts/1
+                    }
                 )}
         ];
 fields(udp_listener) ->
@@ -132,7 +135,13 @@ fields(udp_listener) ->
 fields(dtls_listener) ->
     [{acceptors, sc(integer(), #{default => 16, desc => ?DESC(dtls_listener_acceptors)})}] ++
         fields(udp_listener) ++
-        [{dtls_options, sc(ref(dtls_opts), #{desc => ?DESC(dtls_listener_dtls_opts)})}];
+        [
+            {dtls_options,
+                sc(ref(dtls_opts), #{
+                    desc => ?DESC(dtls_listener_dtls_opts),
+                    validator => fun emqx_schema:validate_server_ssl_opts/1
+                })}
+        ];
 fields(udp_opts) ->
     [
         {active_n,

apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl

@@ -277,6 +277,48 @@ t_load_unload_gateway(_) ->
         {config_not_found, [<<"gateway">>, stomp]},
         emqx:get_raw_config([gateway, stomp])
     ),
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    #{<<"listeners">> := StompConfL1} = StompConf1,
+    StompConf11 = StompConf1#{
+        <<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL1)
+    },
+    #{<<"listeners">> := StompConfL2} = StompConf2,
+    StompConf22 = StompConf2#{
+        <<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL2)
+    },
+    Raw1 = Raw0#{<<"stomp">> => StompConf11},
+    Raw2 = Raw0#{<<"stomp">> => StompConf22},
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
+    ?assertMatch(
+        #{
+            config := #{
+                authentication := #{backend := built_in_database, enable := true},
+                listeners := #{tcp := #{default := #{bind := 61613}}},
+                mountpoint := <<"t/">>,
+                idle_timeout := 10000
+            }
+        },
+        emqx_gateway:lookup('stomp')
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw2)),
+    assert_confs(StompConf2, emqx:get_raw_config([gateway, stomp])),
+    ?assertMatch(
+        #{
+            config :=
+                #{
+                    authentication := #{backend := built_in_database, enable := true},
+                    listeners := #{tcp := #{default := #{bind := 61613}}},
+                    idle_timeout := 20000,
+                    mountpoint := <<"t2/">>
+                }
+        },
+        emqx_gateway:lookup('stomp')
+    ),
+    %% reset
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    ?assertEqual(undefined, emqx_gateway:lookup('stomp')),
     ok.
 
 t_load_remove_authn(_) ->
@@ -310,6 +352,40 @@ t_load_remove_authn(_) ->
         {config_not_found, [<<"gateway">>, stomp, authentication]},
         emqx:get_raw_config([gateway, stomp, authentication])
     ),
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    #{<<"listeners">> := StompConfL} = StompConf,
+    StompConf1 = StompConf#{
+        <<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL),
+        <<"authentication">> => ?CONF_STOMP_AUTHN_1
+    },
+    Raw1 = maps:put(<<"stomp">>, StompConf1, Raw0),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    assert_confs(StompConf1, emqx:get_raw_config([gateway, stomp])),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"password_based:built_in_database">>,
+                    listeners := [#{authn := <<"undefined">>, type := tcp}],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
+    %% reset(remove authn)
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"undefined">>,
+                    listeners := [#{authn := <<"undefined">>, type := tcp}],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
     ok.
 
 t_load_remove_listeners(_) ->
@@ -324,6 +400,7 @@ t_load_remove_listeners(_) ->
         {<<"tcp">>, <<"default">>},
         ?CONF_STOMP_LISTENER_1
     ),
+
     assert_confs(
         maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_1)),
         emqx:get_raw_config([gateway, stomp])
@@ -355,6 +432,59 @@ t_load_remove_listeners(_) ->
         {config_not_found, [<<"gateway">>, stomp, listeners, tcp, default]},
         emqx:get_raw_config([gateway, stomp, listeners, tcp, default])
     ),
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    Raw1 = emqx_utils_maps:deep_put(
+        [<<"stomp">>, <<"listeners">>, <<"tcp">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_1
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    assert_confs(
+        maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_1)),
+        emqx:get_raw_config([gateway, stomp])
+    ),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"password_based:built_in_database">>,
+                    listeners := [#{authn := <<"undefined">>, type := tcp}],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
+    Raw2 = emqx_utils_maps:deep_put(
+        [<<"stomp">>, <<"listeners">>, <<"tcp">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_2
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw2)),
+    assert_confs(
+        maps:merge(StompConf, listener(?CONF_STOMP_LISTENER_2)),
+        emqx:get_raw_config([gateway, stomp])
+    ),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"password_based:built_in_database">>,
+                    listeners := [#{authn := <<"undefined">>, type := tcp}],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
+    %% reset(remove listener)
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    ?assertMatch(
+        #{
+            stomp :=
+                #{
+                    authn := <<"password_based:built_in_database">>,
+                    listeners := [],
+                    num_clients := 0
+                }
+        },
+        emqx_gateway:get_basic_usage_info()
+    ),
     ok.
 
 t_load_remove_listener_authn(_) ->
@@ -417,6 +547,7 @@ t_load_gateway_with_certs_content(_) ->
         [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
         emqx:get_raw_config([gateway, stomp])
     ),
+    assert_ssl_confs_files_exist(SslConf),
     ok = emqx_gateway_conf:unload_gateway(<<"stomp">>),
     assert_ssl_confs_files_deleted(SslConf),
     ?assertException(
@@ -424,6 +555,25 @@ t_load_gateway_with_certs_content(_) ->
         {config_not_found, [<<"gateway">>, stomp]},
         emqx:get_raw_config([gateway, stomp])
     ),
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    #{<<"listeners">> := StompConfL} = StompConf,
+    StompConf1 = StompConf#{
+        <<"listeners">> => emqx_gateway_conf:unconvert_listeners(StompConfL)
+    },
+    Raw1 = emqx_utils_maps:deep_put([<<"stomp">>], Raw0, StompConf1),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    assert_ssl_confs_files_exist(SslConf),
+    ?assertEqual(
+        SslConf,
+        emqx_utils_maps:deep_get(
+            [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
+            emqx:get_raw_config([gateway, stomp])
+        )
+    ),
+    %% reset
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    assert_ssl_confs_files_deleted(SslConf),
     ok.
 
 %% TODO: Comment out this test case for now, because emqx_tls_lib
@@ -475,6 +625,7 @@ t_add_listener_with_certs_content(_) ->
         [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
         emqx:get_raw_config([gateway, stomp])
     ),
+    assert_ssl_confs_files_exist(SslConf),
     ok = emqx_gateway_conf:remove_listener(
         <<"stomp">>, {<<"ssl">>, <<"default">>}
     ),
@@ -492,6 +643,34 @@ t_add_listener_with_certs_content(_) ->
         {config_not_found, [<<"gateway">>, stomp, listeners, ssl, default]},
         emqx:get_raw_config([gateway, stomp, listeners, ssl, default])
     ),
+
+    %% test update([gateway], Conf)
+    Raw0 = emqx:get_raw_config([gateway]),
+    Raw1 = emqx_utils_maps:deep_put(
+        [<<"stomp">>, <<"listeners">>, <<"ssl">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_SSL
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw1)),
+    SslConf1 = emqx_utils_maps:deep_get(
+        [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
+        emqx:get_raw_config([gateway, stomp])
+    ),
+    assert_ssl_confs_files_exist(SslConf1),
+    %% update
+    Raw2 = emqx_utils_maps:deep_put(
+        [<<"stomp">>, <<"listeners">>, <<"ssl">>, <<"default">>], Raw0, ?CONF_STOMP_LISTENER_SSL_2
+    ),
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw2)),
+    SslConf2 =
+        emqx_utils_maps:deep_get(
+            [<<"listeners">>, <<"ssl">>, <<"default">>, <<"ssl_options">>],
+            emqx:get_raw_config([gateway, stomp])
+        ),
+    assert_ssl_confs_files_exist(SslConf2),
+    %% reset
+    ?assertMatch({ok, _}, emqx:update_config([gateway], Raw0)),
+    assert_ssl_confs_files_deleted(SslConf),
+    assert_ssl_confs_files_deleted(SslConf1),
+    assert_ssl_confs_files_deleted(SslConf2),
     ok.
 
 assert_ssl_confs_files_deleted(SslConf) when is_map(SslConf) ->
@@ -504,6 +683,15 @@ assert_ssl_confs_files_deleted(SslConf) when is_map(SslConf) ->
         end,
         Ks
     ).
+assert_ssl_confs_files_exist(SslConf) when is_map(SslConf) ->
+    Ks = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>],
+    lists:foreach(
+        fun(K) ->
+            Path = maps:get(K, SslConf),
+            {ok, _} = file:read_file(Path)
+        end,
+        Ks
+    ).
 
 %%--------------------------------------------------------------------
 %% Utils

apps/emqx_gateway/test/emqx_gateway_test_utils.erl

@@ -83,13 +83,13 @@ do_assert_confs(Key, Expected, Effected) ->
 
 maybe_unconvert_listeners(Conf) when is_map(Conf) ->
     case maps:take(<<"listeners">>, Conf) of
-        error ->
-            Conf;
-        {Ls, Conf1} ->
+        {Ls, Conf1} when is_list(Ls) ->
             Conf1#{
                 <<"listeners">> =>
                     emqx_gateway_conf:unconvert_listeners(Ls)
-            }
+            };
+        _ ->
+            Conf
     end;
 maybe_unconvert_listeners(Conf) ->
     Conf.

apps/emqx_gateway/test/test_mqtt_broker.erl

@@ -24,6 +24,7 @@
 -record(state, {subscriber}).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 
 -include_lib("emqx/include/emqx_mqtt.hrl").
 
@@ -50,7 +51,7 @@ unsubscribe(Topic) ->
     gen_server:call(?MODULE, {unsubscribe, Topic}).
 
 get_subscrbied_topics() ->
-    [Topic || {_Client, Topic} <- ets:tab2list(emqx_subscription)].
+    [Topic || {_Client, Topic} <- ets:tab2list(?SUBSCRIPTION)].
 
 start_link() ->
     gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).

apps/emqx_gateway_mqttsn/src/emqx_gateway_mqttsn.app.src

@@ -1,6 +1,6 @@
 {application, emqx_gateway_mqttsn, [
     {description, "MQTT-SN Gateway"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {applications, [kernel, stdlib, emqx, emqx_gateway]},
     {env, []},

apps/emqx_gateway_mqttsn/src/emqx_mqttsn_channel.erl

@@ -1111,15 +1111,16 @@ check_pub_authz(
 
 convert_pub_to_msg(
     {TopicName, Flags, Data},
-    Channel = #channel{clientinfo = #{clientid := ClientId}}
+    Channel = #channel{clientinfo = #{clientid := ClientId, mountpoint := Mountpoint}}
 ) ->
     #mqtt_sn_flags{qos = QoS, dup = Dup, retain = Retain} = Flags,
     NewQoS = get_corrected_qos(QoS),
+    NTopicName = emqx_mountpoint:mount(Mountpoint, TopicName),
     Message = put_message_headers(
         emqx_message:make(
             ClientId,
             NewQoS,
-            TopicName,
+            NTopicName,
             Data,
             #{dup => Dup, retain => Retain},
             #{}

apps/emqx_gateway_mqttsn/test/emqx_sn_protocol_SUITE.erl

@@ -120,6 +120,13 @@ restart_mqttsn_with_subs_resume_off() ->
         Conf#{<<"subs_resume">> => <<"false">>}
     ).
 
+restart_mqttsn_with_mountpoint(Mp) ->
+    Conf = emqx:get_raw_config([gateway, mqttsn]),
+    emqx_gateway_conf:update_gateway(
+        mqttsn,
+        Conf#{<<"mountpoint">> => Mp}
+    ).
+
 default_config() ->
     ?CONF_DEFAULT.
 
@@ -990,6 +997,44 @@ t_publish_qos2_case03(_) ->
     ?assertEqual(<<2, ?SN_DISCONNECT>>, receive_response(Socket)),
     gen_udp:close(Socket).
 
+t_publish_mountpoint(_) ->
+    restart_mqttsn_with_mountpoint(<<"mp/">>),
+    Dup = 0,
+    QoS = 1,
+    Retain = 0,
+    Will = 0,
+    CleanSession = 0,
+    MsgId = 1,
+    TopicId1 = ?MAX_PRED_TOPIC_ID + 1,
+    Topic = <<"abc">>,
+    {ok, Socket} = gen_udp:open(0, [binary]),
+    ClientId = ?CLIENTID,
+    send_connect_msg(Socket, ClientId),
+    ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
+    send_subscribe_msg_normal_topic(Socket, QoS, Topic, MsgId),
+    ?assertEqual(
+        <<8, ?SN_SUBACK, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+            TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>,
+        receive_response(Socket)
+    ),
+
+    Payload1 = <<20, 21, 22, 23>>,
+    send_publish_msg_normal_topic(Socket, QoS, MsgId, TopicId1, Payload1),
+    ?assertEqual(
+        <<7, ?SN_PUBACK, TopicId1:16, MsgId:16, ?SN_RC_ACCEPTED>>, receive_response(Socket)
+    ),
+    timer:sleep(100),
+
+    ?assertEqual(
+        <<11, ?SN_PUBLISH, Dup:1, QoS:2, Retain:1, Will:1, CleanSession:1, ?SN_NORMAL_TOPIC:2,
+            TopicId1:16, MsgId:16, <<20, 21, 22, 23>>/binary>>,
+        receive_response(Socket)
+    ),
+
+    send_disconnect_msg(Socket, undefined),
+    restart_mqttsn_with_mountpoint(<<>>),
+    gen_udp:close(Socket).
+
 t_delivery_qos1_register_invalid_topic_id(_) ->
     Dup = 0,
     QoS = 1,

apps/emqx_management/src/emqx_mgmt.erl

@@ -17,6 +17,8 @@
 -module(emqx_mgmt).
 
 -include("emqx_mgmt.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
+
 -elvis([{elvis_style, invalid_dynamic_call, disable}]).
 -elvis([{elvis_style, god_modules, disable}]).
 
@@ -139,7 +141,8 @@ node_info() ->
         max_fds => proplists:get_value(
             max_fds, lists:usort(lists:flatten(erlang:system_info(check_io)))
         ),
-        connections => ets:info(emqx_channel, size),
+        connections => ets:info(?CHAN_TAB, size),
+        live_connections => ets:info(?CHAN_LIVE_TAB, size),
         node_status => 'running',
         uptime => proplists:get_value(uptime, BrokerInfo),
         version => iolist_to_binary(proplists:get_value(version, BrokerInfo)),
@@ -487,7 +490,7 @@ subscribe([], _ClientId, _TopicTables) ->
 -spec do_subscribe(emqx_types:clientid(), emqx_types:topic_filters()) ->
     {subscribe, _} | {error, atom()}.
 do_subscribe(ClientId, TopicTables) ->
-    case ets:lookup(emqx_channel, ClientId) of
+    case ets:lookup(?CHAN_TAB, ClientId) of
         [] -> {error, channel_not_found};
         [{_, Pid}] -> Pid ! {subscribe, TopicTables}
     end.
@@ -514,7 +517,7 @@ unsubscribe([], _ClientId, _Topic) ->
 -spec do_unsubscribe(emqx_types:clientid(), emqx_types:topic()) ->
     {unsubscribe, _} | {error, _}.
 do_unsubscribe(ClientId, Topic) ->
-    case ets:lookup(emqx_channel, ClientId) of
+    case ets:lookup(?CHAN_TAB, ClientId) of
         [] -> {error, channel_not_found};
         [{_, Pid}] -> Pid ! {unsubscribe, [emqx_topic:parse(Topic)]}
     end.
@@ -537,7 +540,7 @@ unsubscribe_batch([], _ClientId, _Topics) ->
 -spec do_unsubscribe_batch(emqx_types:clientid(), [emqx_types:topic()]) ->
     {unsubscribe_batch, _} | {error, _}.
 do_unsubscribe_batch(ClientId, Topics) ->
-    case ets:lookup(emqx_channel, ClientId) of
+    case ets:lookup(?CHAN_TAB, ClientId) of
         [] -> {error, channel_not_found};
         [{_, Pid}] -> Pid ! {unsubscribe, [emqx_topic:parse(Topic) || Topic <- Topics]}
     end.

apps/emqx_management/src/emqx_mgmt_api_clients.erl

@@ -20,6 +20,7 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
 -include_lib("emqx/include/logger.hrl").
@@ -57,7 +58,6 @@
 %% for batch operation
 -export([do_subscribe/3]).
 
--define(CLIENT_QTAB, emqx_channel_info).
 -define(TAGS, [<<"Clients">>]).
 
 -define(CLIENT_QSCHEMA, [
@@ -666,7 +666,7 @@ list_clients(QString) ->
         case maps:get(<<"node">>, QString, undefined) of
             undefined ->
                 emqx_mgmt_api:cluster_query(
-                    ?CLIENT_QTAB,
+                    ?CHAN_INFO_TAB,
                     QString,
                     ?CLIENT_QSCHEMA,
                     fun ?MODULE:qs2ms/2,
@@ -678,7 +678,7 @@ list_clients(QString) ->
                         QStringWithoutNode = maps:without([<<"node">>], QString),
                         emqx_mgmt_api:node_query(
                             Node1,
-                            ?CLIENT_QTAB,
+                            ?CHAN_INFO_TAB,
                             QStringWithoutNode,
                             ?CLIENT_QSCHEMA,
                             fun ?MODULE:qs2ms/2,
@@ -753,7 +753,7 @@ subscribe_batch(#{clientid := ClientID, topics := Topics}) ->
     %% We use emqx_channel instead of emqx_channel_info (used by the emqx_mgmt:lookup_client/2),
     %% as the emqx_channel_info table will only be populated after the hook `client.connected`
     %% has returned. So if one want to subscribe topics in this hook, it will fail.
-    case ets:lookup(emqx_channel, ClientID) of
+    case ets:lookup(?CHAN_TAB, ClientID) of
         [] ->
             {404, ?CLIENTID_NOT_FOUND};
         _ ->

apps/emqx_management/src/emqx_mgmt_api_nodes.erl

@@ -151,6 +151,11 @@ fields(node_info) ->
                 #{desc => <<"Node name">>, example => <<"emqx@127.0.0.1">>}
             )},
         {connections,
+            mk(
+                non_neg_integer(),
+                #{desc => <<"Number of clients session in this node">>, example => 0}
+            )},
+        {live_connections,
             mk(
                 non_neg_integer(),
                 #{desc => <<"Number of clients currently connected to this node">>, example => 0}

apps/emqx_management/src/emqx_mgmt_api_topics.erl

@@ -17,6 +17,7 @@
 -module(emqx_mgmt_api_topics).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
@@ -111,7 +112,7 @@ do_list(Params) ->
     case
         emqx_mgmt_api:node_query(
             node(),
-            emqx_route,
+            ?ROUTE_TAB,
             Params,
             ?TOPICS_QUERY_SCHEMA,
             fun ?MODULE:qs2ms/2,

apps/emqx_management/src/emqx_mgmt_cli.erl

@@ -17,6 +17,8 @@
 -module(emqx_mgmt_cli).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_cm.hrl").
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
 -include_lib("emqx/include/logger.hrl").
 
@@ -168,7 +170,7 @@ sort_map_list_field(Field, Map) ->
 %% @doc Query clients
 
 clients(["list"]) ->
-    dump(emqx_channel, client);
+    dump(?CHAN_TAB, client);
 clients(["show", ClientId]) ->
     if_client(ClientId, fun print/1);
 clients(["kick", ClientId]) ->
@@ -182,7 +184,7 @@ clients(_) ->
     ]).
 
 if_client(ClientId, Fun) ->
-    case ets:lookup(emqx_channel, (bin(ClientId))) of
+    case ets:lookup(?CHAN_TAB, (bin(ClientId))) of
         [] -> emqx_ctl:print("Not Found.~n");
         [Channel] -> Fun({client, Channel})
     end.
@@ -191,9 +193,9 @@ if_client(ClientId, Fun) ->
 %% @doc Topics Command
 
 topics(["list"]) ->
-    dump(emqx_route, emqx_topic);
+    dump(?ROUTE_TAB, emqx_topic);
 topics(["show", Topic]) ->
-    Routes = ets:lookup(emqx_route, bin(Topic)),
+    Routes = ets:lookup(?ROUTE_TAB, bin(Topic)),
     [print({emqx_topic, Route}) || Route <- Routes];
 topics(_) ->
     emqx_ctl:usage([
@@ -204,23 +206,23 @@ topics(_) ->
 subscriptions(["list"]) ->
     lists:foreach(
         fun(Suboption) ->
-            print({emqx_suboption, Suboption})
+            print({?SUBOPTION, Suboption})
         end,
-        ets:tab2list(emqx_suboption)
+        ets:tab2list(?SUBOPTION)
     );
 subscriptions(["show", ClientId]) ->
     case ets:lookup(emqx_subid, bin(ClientId)) of
         [] ->
             emqx_ctl:print("Not Found.~n");
         [{_, Pid}] ->
-            case ets:match_object(emqx_suboption, {{'_', Pid}, '_'}) of
+            case ets:match_object(?SUBOPTION, {{'_', Pid}, '_'}) of
                 [] -> emqx_ctl:print("Not Found.~n");
-                Suboption -> [print({emqx_suboption, Sub}) || Sub <- Suboption]
+                Suboption -> [print({?SUBOPTION, Sub}) || Sub <- Suboption]
             end
     end;
 subscriptions(["add", ClientId, Topic, QoS]) ->
     if_valid_qos(QoS, fun(IntQos) ->
-        case ets:lookup(emqx_channel, bin(ClientId)) of
+        case ets:lookup(?CHAN_TAB, bin(ClientId)) of
             [] ->
                 emqx_ctl:print("Error: Channel not found!");
             [{_, Pid}] ->
@@ -230,7 +232,7 @@ subscriptions(["add", ClientId, Topic, QoS]) ->
         end
     end);
 subscriptions(["del", ClientId, Topic]) ->
-    case ets:lookup(emqx_channel, bin(ClientId)) of
+    case ets:lookup(?CHAN_TAB, bin(ClientId)) of
         [] ->
             emqx_ctl:print("Error: Channel not found!");
         [{_, Pid}] ->
@@ -841,7 +843,7 @@ print({emqx_topic, #route{topic = Topic, dest = {_, Node}}}) ->
     emqx_ctl:print("~ts -> ~ts~n", [Topic, Node]);
 print({emqx_topic, #route{topic = Topic, dest = Node}}) ->
     emqx_ctl:print("~ts -> ~ts~n", [Topic, Node]);
-print({emqx_suboption, {{Topic, Pid}, Options}}) when is_pid(Pid) ->
+print({?SUBOPTION, {{Topic, Pid}, Options}}) when is_pid(Pid) ->
     SubId = maps:get(subid, Options),
     QoS = maps:get(qos, Options, 0),
     NL = maps:get(nl, Options, 0),

apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl

@@ -209,7 +209,7 @@ t_zones(_Config) ->
     ?assertEqual(Mqtt1, NewMqtt),
     %% delete the new zones
     {ok, #{}} = update_config("zones", Zones),
-    ?assertEqual(undefined, emqx_config:get_raw([new_zone, mqtt], undefined)),
+    ?assertEqual(undefined, emqx_config:get_raw([zones, new_zone], undefined)),
     ok.
 
 t_dashboard(_Config) ->

apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl

@@ -60,6 +60,11 @@ t_nodes_api(_) ->
     Edition = maps:get(<<"edition">>, LocalNodeInfo),
     ?assertEqual(emqx_release:edition_longstr(), Edition),
 
+    Conns = maps:get(<<"connections">>, LocalNodeInfo),
+    ?assertEqual(0, Conns),
+    LiveConns = maps:get(<<"live_connections">>, LocalNodeInfo),
+    ?assertEqual(0, LiveConns),
+
     NodePath = emqx_mgmt_api_test_util:api_path(["nodes", atom_to_list(node())]),
     {ok, NodeInfo} = emqx_mgmt_api_test_util:request_api(get, NodePath),
     NodeNameResponse =

apps/emqx_management/test/emqx_mgmt_api_topics_SUITE.erl

@@ -18,11 +18,10 @@
 -compile(export_all).
 -compile(nowarn_export_all).
 
+-include_lib("emqx/include/emqx_router.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
--define(ROUTE_TAB, emqx_route).
-
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 

apps/emqx_resource/test/emqx_resource_schema_tests.erl

@@ -67,7 +67,7 @@ health_check_interval_validator_test_() ->
                 parse_and_check_webhook_bridge(webhook_bridge_health_check_hocon(<<"3_600_000ms">>))
             )},
         ?_assertThrow(
-            #{exception := "timeout value too large" ++ _},
+            #{exception := #{message := "timeout value too large" ++ _}},
             parse_and_check_webhook_bridge(
                 webhook_bridge_health_check_hocon(<<"150000000000000s">>)
             )

apps/emqx_retainer/src/emqx_retainer_index.erl

@@ -31,7 +31,7 @@
 -type index() :: list(pos_integer()).
 
 %% @doc Index key is a term that can be effectively searched in the index table.
--type index_key() :: {index(), {emqx_topic:words(), emqx_topic:words()}}.
+-type index_key() :: {index(), {emqx_types:words(), emqx_types:words()}}.
 
 -type match_pattern_part() :: term().
 
@@ -42,7 +42,7 @@
 %% @doc Given words of a concrete topic (`Tokens') and a list of `Indices',
 %% constructs index keys for the topic and each of the indices.
 %% `Fun' is called with each of these keys.
--spec foreach_index_key(fun((index_key()) -> any()), list(index()), emqx_topic:words()) -> ok.
+-spec foreach_index_key(fun((index_key()) -> any()), list(index()), emqx_types:words()) -> ok.
 foreach_index_key(_Fun, [], _Tokens) ->
     ok;
 foreach_index_key(Fun, [Index | Indices], Tokens) ->
@@ -59,7 +59,7 @@ foreach_index_key(Fun, [Index | Indices], Tokens) ->
 %% returns `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' term.
 %%
 %% @see foreach_index_key/3
--spec to_index_key(index(), emqx_topic:words()) -> index_key().
+-spec to_index_key(index(), emqx_types:words()) -> index_key().
 to_index_key(Index, Tokens) ->
     {Index, split_index_tokens(Index, Tokens, 1, [], [])}.
 
@@ -73,7 +73,7 @@ to_index_key(Index, Tokens) ->
 %%
 %% @see foreach_index_key/3
 %% @see to_index_key/2
--spec index_score(index(), emqx_topic:words()) -> non_neg_integer().
+-spec index_score(index(), emqx_types:words()) -> non_neg_integer().
 index_score(Index, Tokens) ->
     index_score(Index, Tokens, 1, 0).
 
@@ -92,7 +92,7 @@ select_index(Tokens, Indices) ->
 %%
 %% E.g. for `[2, 3]' index and <code>['+', <<"b">>, '+', <<"d">>]</code> wildcard topic
 %% returns <code>{[2, 3], {[<<"b">>, '_'], ['_', <<"d">>]}}</code> pattern.
--spec condition(index(), emqx_topic:words()) -> match_pattern_part().
+-spec condition(index(), emqx_types:words()) -> match_pattern_part().
 condition(Index, Tokens) ->
     {Index, condition(Index, Tokens, 1, [], [])}.
 
@@ -100,7 +100,7 @@ condition(Index, Tokens) ->
 %%
 %% E.g. for <code>['+', <<"b">>, '+', <<"d">>, '#']</code> wildcard topic
 %% returns <code>['_', <<"b">>, '_', <<"d">> | '_']</code> pattern.
--spec condition(emqx_topic:words()) -> match_pattern_part().
+-spec condition(emqx_types:words()) -> match_pattern_part().
 condition(Tokens) ->
     Tokens1 = [
         case W =:= '+' of
@@ -118,7 +118,7 @@ condition(Tokens) ->
 %%
 %% E.g given `{[2, 3], {[<<"b">>, <<"c">>], [<<"a">>, <<"d">>]}}' index key
 %% returns `[<<"a">>, <<"b">>, <<"c">>, <<"d">>]' topic.
--spec restore_topic(index_key()) -> emqx_topic:words().
+-spec restore_topic(index_key()) -> emqx_types:words().
 restore_topic({Index, {IndexTokens, OtherTokens}}) ->
     restore_topic(Index, IndexTokens, OtherTokens, 1, []).
 

apps/emqx_utils/src/emqx_utils.erl

@@ -55,7 +55,8 @@
     safe_to_existing_atom/1,
     safe_to_existing_atom/2,
     pub_props_to_packet/1,
-    safe_filename/1
+    safe_filename/1,
+    diff_lists/3
 ]).
 
 -export([
@@ -753,3 +754,152 @@ safe_filename(Filename) when is_binary(Filename) ->
     binary:replace(Filename, <<":">>, <<"-">>, [global]);
 safe_filename(Filename) when is_list(Filename) ->
     lists:flatten(string:replace(Filename, ":", "-", all)).
+
+%% @doc Compares two lists of maps and returns the differences between them in a
+%% map containing four keys – 'removed', 'added', 'identical', and 'changed' –
+%% each holding a list of maps. Elements are compared using key function KeyFunc
+%% to extract the comparison key used for matching.
+%%
+%% The return value is a map with the following keys and the list of maps as its values:
+%% * 'removed' – a list of maps that were present in the Old list, but not found in the New list.
+%% * 'added' – a list of maps that were present in the New list, but not found in the Old list.
+%% * 'identical' – a list of maps that were present in both lists and have the same comparison key value.
+%% * 'changed' – a list of pairs of maps representing the changes between maps present in the New and Old lists.
+%% The first map in the pair represents the map in the Old list, and the second map
+%% represents the potential modification in the New list.
+
+%% The KeyFunc parameter is a function that extracts the comparison key used
+%% for matching from each map. The function should return a comparable term,
+%% such as an atom, a number, or a string. This is used to determine if each
+%% element is the same in both lists.
+
+-spec diff_lists(list(T), list(T), Func) ->
+    #{
+        added := list(T),
+        identical := list(T),
+        removed := list(T),
+        changed := list({Old :: T, New :: T})
+    }
+when
+    Func :: fun((T) -> any()),
+    T :: any().
+
+diff_lists(New, Old, KeyFunc) when is_list(New) andalso is_list(Old) ->
+    Removed =
+        lists:foldl(
+            fun(E, RemovedAcc) ->
+                case search(KeyFunc(E), KeyFunc, New) of
+                    false -> [E | RemovedAcc];
+                    _ -> RemovedAcc
+                end
+            end,
+            [],
+            Old
+        ),
+    {Added, Identical, Changed} =
+        lists:foldl(
+            fun(E, Acc) ->
+                {Added0, Identical0, Changed0} = Acc,
+                case search(KeyFunc(E), KeyFunc, Old) of
+                    false ->
+                        {[E | Added0], Identical0, Changed0};
+                    E ->
+                        {Added0, [E | Identical0], Changed0};
+                    E1 ->
+                        {Added0, Identical0, [{E1, E} | Changed0]}
+                end
+            end,
+            {[], [], []},
+            New
+        ),
+    #{
+        removed => lists:reverse(Removed),
+        added => lists:reverse(Added),
+        identical => lists:reverse(Identical),
+        changed => lists:reverse(Changed)
+    }.
+
+search(_ExpectValue, _KeyFunc, []) ->
+    false;
+search(ExpectValue, KeyFunc, [Item | List]) ->
+    case KeyFunc(Item) =:= ExpectValue of
+        true -> Item;
+        false -> search(ExpectValue, KeyFunc, List)
+    end.
+
+-ifdef(TEST).
+-include_lib("eunit/include/eunit.hrl").
+
+diff_lists_test() ->
+    KeyFunc = fun(#{name := Name}) -> Name end,
+    ?assertEqual(
+        #{
+            removed => [],
+            added => [],
+            identical => [],
+            changed => []
+        },
+        diff_lists([], [], KeyFunc)
+    ),
+    %% test removed list
+    ?assertEqual(
+        #{
+            removed => [#{name => a, value => 1}],
+            added => [],
+            identical => [],
+            changed => []
+        },
+        diff_lists([], [#{name => a, value => 1}], KeyFunc)
+    ),
+    %% test added list
+    ?assertEqual(
+        #{
+            removed => [],
+            added => [#{name => a, value => 1}],
+            identical => [],
+            changed => []
+        },
+        diff_lists([#{name => a, value => 1}], [], KeyFunc)
+    ),
+    %% test identical list
+    ?assertEqual(
+        #{
+            removed => [],
+            added => [],
+            identical => [#{name => a, value => 1}],
+            changed => []
+        },
+        diff_lists([#{name => a, value => 1}], [#{name => a, value => 1}], KeyFunc)
+    ),
+    Old = [
+        #{name => a, value => 1},
+        #{name => b, value => 4},
+        #{name => e, value => 2},
+        #{name => d, value => 4}
+    ],
+    New = [
+        #{name => a, value => 1},
+        #{name => b, value => 2},
+        #{name => e, value => 2},
+        #{name => c, value => 3}
+    ],
+    Diff = diff_lists(New, Old, KeyFunc),
+    ?assertEqual(
+        #{
+            added => [
+                #{name => c, value => 3}
+            ],
+            identical => [
+                #{name => a, value => 1},
+                #{name => e, value => 2}
+            ],
+            removed => [
+                #{name => d, value => 4}
+            ],
+            changed => [{#{name => b, value => 4}, #{name => b, value => 2}}]
+        },
+        Diff
+    ),
+    ok.
+
+-endif.

changes/ce/feat-10895.en.md

@@ -1,8 +0,0 @@
-Refactored some bridges to avoid leaking resources during crashes at creation, including:
-- TDEngine
-- WebHook
-- LDAP
-- MongoDB
-- MySQL
-- PostgreSQL
-- Redis

changes/ce/feat-10933.en.md

@@ -0,0 +1 @@
+Add support for configuring TCP keep-alive in MQTT/TCP and MQTT/SSL listeners

changes/ce/feat-10948.en.md

@@ -0,0 +1,4 @@
+Add `live_connections` field for some HTTP APIs, i.e:
+- `/monitor_current`, `/monitor_current/nodes/{node}`
+- `/monitor/nodes/{node}`, `/monitor`
+- `/node/{node}`, `/nodes`