Merge remote-tracking branch 'upstream/release-58' into 20240819-sync-release-58

Makefile

@@ -11,7 +11,7 @@ include env.sh
 # Dashboard version
 # from https://github.com/emqx/emqx-dashboard5
 export EMQX_DASHBOARD_VERSION ?= v1.10.0-beta.1
-export EMQX_EE_DASHBOARD_VERSION ?= e1.8.0-beta.1
+export EMQX_EE_DASHBOARD_VERSION ?= e1.8.0-beta.2
 
 export EMQX_RELUP ?= true
 export EMQX_REL_FORM ?= tgz

apps/emqx/rebar.config

@@ -30,8 +30,8 @@
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.12.0"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.5"}}},
-    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.2"}}},
+    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.4.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.43.3"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},

apps/emqx/src/emqx.app.src

@@ -2,7 +2,7 @@
 {application, emqx, [
     {id, "emqx"},
     {description, "EMQX Core"},
-    {vsn, "5.3.4"},
+    {vsn, "5.4.0"},
     {modules, []},
     {registered, []},
     {applications, [

apps/emqx/src/emqx_banned.erl

@@ -519,7 +519,7 @@ on_banned(#banned{who = {clientid, ClientId}}) ->
             clientid => ClientId
         }
     ),
-    emqx_cm:kick_session(ClientId),
+    emqx_cm:try_kick_session(ClientId),
     ok;
 on_banned(_) ->
     ok.

apps/emqx/src/emqx_cm.erl

@@ -54,6 +54,7 @@
     takeover_session_end/1,
     kick_session/1,
     kick_session/2,
+    try_kick_session/1,
     takeover_kick/1
 ]).
 
@@ -600,20 +601,15 @@ kick_session(ClientId) ->
             ),
             ok;
         ChanPids ->
-            case length(ChanPids) > 1 of
-                true ->
-                    ?SLOG(
-                        warning,
-                        #{
-                            msg => "more_than_one_channel_found",
-                            chan_pids => ChanPids
-                        },
-                        #{clientid => ClientId}
-                    );
-                false ->
-                    ok
-            end,
-            lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids)
+            kick_session_chans(ClientId, ChanPids)
+    end.
+
+try_kick_session(ClientId) ->
+    case lookup_channels(ClientId) of
+        [] ->
+            ok;
+        ChanPids ->
+            kick_session_chans(ClientId, ChanPids)
     end.
 
 %% @doc Is clean start?
@@ -814,3 +810,19 @@ get_connected_client_count() ->
         undefined -> 0;
         Size -> Size
     end.
+
+kick_session_chans(ClientId, ChanPids) ->
+    case length(ChanPids) > 1 of
+        true ->
+            ?SLOG(
+                warning,
+                #{
+                    msg => "more_than_one_channel_found",
+                    chan_pids => ChanPids
+                },
+                #{clientid => ClientId}
+            );
+        false ->
+            ok
+    end,
+    lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids).

apps/emqx/src/emqx_config.erl

@@ -621,16 +621,16 @@ save_to_config_map(Conf, RawConf) ->
     ?MODULE:put_raw(RawConf).
 
 -spec save_to_override_conf(boolean(), raw_config(), update_opts()) -> ok | {error, term()}.
-save_to_override_conf(_, undefined, _) ->
+save_to_override_conf(_HasDeprecatedFile, undefined, _) ->
     ok;
-save_to_override_conf(true, RawConf, Opts) ->
+save_to_override_conf(true = _HasDeprecatedFile, RawConf, Opts) ->
     case deprecated_conf_file(Opts) of
         undefined ->
             ok;
         FileName ->
             backup_and_write(FileName, hocon_pp:do(RawConf, Opts))
     end;
-save_to_override_conf(false, RawConf, Opts) ->
+save_to_override_conf(false = _HasDeprecatedFile, RawConf, Opts) ->
     case cluster_hocon_file() of
         undefined ->
             ok;

apps/emqx/src/emqx_ds_schema.erl

@@ -18,7 +18,8 @@
 -module(emqx_ds_schema).
 
 %% API:
--export([schema/0, translate_builtin_raft/1, translate_builtin_local/1]).
+-export([schema/0, storage_schema/1, translate_builtin_raft/1, translate_builtin_local/1]).
+-export([db_config/1]).
 
 %% Behavior callbacks:
 -export([fields/1, desc/1, namespace/0]).
@@ -48,6 +49,11 @@
 %% API
 %%================================================================================
 
+-spec db_config(emqx_config:runtime_config_key_path()) -> emqx_ds:create_db_opts().
+db_config(Path) ->
+    ConfigTree = #{'_config_handler' := {Module, Function}} = emqx_config:get(Path),
+    apply(Module, Function, [ConfigTree]).
+
 translate_builtin_raft(
     Backend = #{
         backend := builtin_raft,
@@ -89,15 +95,22 @@ namespace() ->
 schema() ->
     [
         {messages,
-            ds_schema(#{
-                default =>
-                    #{
-                        <<"backend">> => ?DEFAULT_BACKEND
-                    },
+            storage_schema(#{
                 importance => ?IMPORTANCE_MEDIUM,
                 desc => ?DESC(messages)
             })}
-    ].
+    ] ++ emqx_schema_hooks:injection_point('durable_storage', []).
+
+storage_schema(ExtraOptions) ->
+    Options = #{
+        default => #{<<"backend">> => ?DEFAULT_BACKEND}
+    },
+    sc(
+        hoconsc:union(
+            ?BUILTIN_BACKENDS ++ emqx_schema_hooks:injection_point('durable_storage.backends', [])
+        ),
+        maps:merge(Options, ExtraOptions)
+    ).
 
 fields(builtin_local) ->
     %% Schema for the builtin_raft backend:
@@ -145,26 +158,26 @@ fields(builtin_raft) ->
                     importance => ?IMPORTANCE_HIDDEN
                 }
             )},
-        %% TODO: Deprecate once cluster management and rebalancing is implemented.
-        {"n_sites",
+        {replication_factor,
             sc(
                 pos_integer(),
                 #{
-                    default => 1,
-                    importance => ?IMPORTANCE_HIDDEN,
-                    desc => ?DESC(builtin_n_sites)
+                    default => 3,
+                    importance => ?IMPORTANCE_MEDIUM,
+                    desc => ?DESC(builtin_raft_replication_factor)
                 }
             )},
-        {replication_factor,
+        {n_sites,
             sc(
                 pos_integer(),
                 #{
-                    default => 3,
-                    importance => ?IMPORTANCE_HIDDEN
+                    default => 1,
+                    importance => ?IMPORTANCE_LOW,
+                    desc => ?DESC(builtin_raft_n_sites)
                 }
             )},
         %% TODO: Elaborate.
-        {"replication_options",
+        {replication_options,
             sc(
                 hoconsc:map(name, any()),
                 #{
@@ -375,14 +388,6 @@ translate_layout(
 translate_layout(#{type := reference}) ->
     {emqx_ds_storage_reference, #{}}.
 
-ds_schema(Options) ->
-    sc(
-        hoconsc:union(
-            ?BUILTIN_BACKENDS ++ emqx_schema_hooks:injection_point('durable_storage.backends', [])
-        ),
-        Options
-    ).
-
 builtin_layouts() ->
     %% Reference layout stores everything in one stream, so it's not
     %% suitable for production use. However, it's very simple and

apps/emqx/src/emqx_listeners.erl

@@ -897,7 +897,7 @@ convert_certs(ListenerConf) ->
 
 convert_certs(Type, Name, Conf) ->
     CertsDir = certs_dir(Type, Name),
-    case emqx_tls_lib:ensure_ssl_files(CertsDir, get_ssl_options(Conf)) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(CertsDir, get_ssl_options(Conf)) of
         {ok, undefined} ->
             Conf;
         {ok, SSL} ->

apps/emqx/src/emqx_persistent_message.erl

@@ -45,8 +45,7 @@ init() ->
     persistent_term:put(?PERSISTENCE_ENABLED, IsEnabled),
     ?WITH_DURABILITY_ENABLED(begin
         ?SLOG(notice, #{msg => "Session durability is enabled"}),
-        Backend = storage_backend(),
-        ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, Backend),
+        ok = emqx_ds:open_db(?PERSISTENT_MESSAGE_DB, get_db_config()),
         ok = emqx_persistent_session_ds_router:init_tables(),
         ok = emqx_persistent_session_ds:create_tables(),
         ok
@@ -60,9 +59,9 @@ is_persistence_enabled() ->
 is_persistence_enabled(Zone) ->
     emqx_config:get_zone_conf(Zone, [durable_sessions, enable]).
 
--spec storage_backend() -> emqx_ds:create_db_opts().
-storage_backend() ->
-    storage_backend([durable_storage, messages]).
+-spec get_db_config() -> emqx_ds:create_db_opts().
+get_db_config() ->
+    emqx_ds_schema:db_config([durable_storage, messages]).
 
 %% Dev-only option: force all messages to go through
 %% `emqx_persistent_session_ds':
@@ -70,10 +69,6 @@ storage_backend() ->
 force_ds(Zone) ->
     emqx_config:get_zone_conf(Zone, [durable_sessions, force_persistence]).
 
-storage_backend(Path) ->
-    ConfigTree = #{'_config_handler' := {Module, Function}} = emqx_config:get(Path),
-    apply(Module, Function, [ConfigTree]).
-
 %%--------------------------------------------------------------------
 
 -spec add_handler() -> ok.

apps/emqx/src/emqx_tls_lib.erl

@@ -29,6 +29,8 @@
 
 %% SSL files
 -export([
+    ensure_ssl_files_in_mutable_certs_dir/2,
+    ensure_ssl_files_in_mutable_certs_dir/3,
     ensure_ssl_files/2,
     ensure_ssl_files/3,
     drop_invalid_certs/1,
@@ -310,17 +312,28 @@ trim_space(Bin) ->
 %% or file path.
 %% When PEM format key or certificate is given, it tries to to save them in the given
 %% sub-dir in emqx's data_dir, and replace saved file paths for SSL options.
--spec ensure_ssl_files(file:name_all(), undefined | map()) ->
+-spec ensure_ssl_files_in_mutable_certs_dir(file:name_all(), undefined | map()) ->
     {ok, undefined | map()} | {error, map()}.
-ensure_ssl_files(Dir, SSL) ->
-    ensure_ssl_files(Dir, SSL, #{dry_run => false, required_keys => []}).
+ensure_ssl_files_in_mutable_certs_dir(Dir, SSL) ->
+    ensure_ssl_files_in_mutable_certs_dir(Dir, SSL, #{dry_run => false, required_keys => []}).
 
-ensure_ssl_files(_Dir, undefined, _Opts) ->
+ensure_ssl_files_in_mutable_certs_dir(_Dir, undefined, _Opts) ->
     {ok, undefined};
-ensure_ssl_files(_Dir, #{<<"enable">> := False} = SSL, _Opts) when ?IS_FALSE(False) ->
+ensure_ssl_files_in_mutable_certs_dir(_Dir, #{<<"enable">> := False} = SSL, _Opts) when
+    ?IS_FALSE(False)
+->
     {ok, SSL};
-ensure_ssl_files(_Dir, #{enable := False} = SSL, _Opts) when ?IS_FALSE(False) ->
+ensure_ssl_files_in_mutable_certs_dir(_Dir, #{enable := False} = SSL, _Opts) when
+    ?IS_FALSE(False)
+->
     {ok, SSL};
+ensure_ssl_files_in_mutable_certs_dir(Dir, SSL, Opts) ->
+    %% NOTE:
+    %% Pass Raw Dir to keep the file name hash consistent with the previous version
+    ensure_ssl_files(pem_dir(Dir), SSL, Opts#{raw_dir => Dir}).
+
+ensure_ssl_files(Dir, SSL) ->
+    ensure_ssl_files(Dir, SSL, #{dry_run => false, required_keys => [], raw_dir => Dir}).
 ensure_ssl_files(Dir, SSL, Opts) ->
     RequiredKeys = maps:get(required_keys, Opts, []),
     case ensure_ssl_file_key(SSL, RequiredKeys) of
@@ -356,15 +369,19 @@ ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, Opts) ->
     case is_valid_string(MaybePem) of
         true ->
             DryRun = maps:get(dry_run, Opts, false),
-            do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun);
+            RawDir = maps:get(raw_dir, Opts, Dir),
+            %% RawDir for backward compatibility
+            %% when RawDir is not given, it is the same as Dir
+            %% to keep the file name hash consistent with the previous version (Depends on RawDir)
+            do_ensure_ssl_file(Dir, RawDir, KeyPath, SSL, MaybePem, DryRun);
         false ->
             {error, #{reason => invalid_file_path_or_pem_string}}
     end.
 
-do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun) ->
+do_ensure_ssl_file(Dir, RawDir, KeyPath, SSL, MaybePem, DryRun) ->
     case is_pem(MaybePem) of
         true ->
-            case save_pem_file(Dir, KeyPath, MaybePem, DryRun) of
+            case save_pem_file(Dir, RawDir, KeyPath, MaybePem, DryRun) of
                 {ok, Path} ->
                     NewSSL = emqx_utils_maps:deep_put(KeyPath, SSL, Path),
                     {ok, NewSSL};
@@ -410,8 +427,8 @@ is_pem(MaybePem) ->
 %% To make it simple, the file is always overwritten.
 %% Also a potentially half-written PEM file (e.g. due to power outage)
 %% can be corrected with an overwrite.
-save_pem_file(Dir, KeyPath, Pem, DryRun) ->
-    Path = pem_file_name(Dir, KeyPath, Pem),
+save_pem_file(Dir, RawDir, KeyPath, Pem, DryRun) ->
+    Path = pem_file_path(Dir, RawDir, KeyPath, Pem),
     case filelib:ensure_dir(Path) of
         ok when DryRun ->
             {ok, Path};
@@ -434,16 +451,16 @@ is_managed_ssl_file(Filename) ->
         _ -> false
     end.
 
-pem_file_name(Dir, KeyPath, Pem) ->
+pem_file_path(Dir, RawDir, KeyPath, Pem) ->
     % NOTE
     % Wee need to have the same filename on every cluster node.
     Segments = lists:map(fun ensure_bin/1, KeyPath),
     Filename0 = iolist_to_binary(lists:join(<<"_">>, Segments)),
     Filename1 = binary:replace(Filename0, <<"file">>, <<>>),
-    Fingerprint = crypto:hash(md5, [Dir, Filename1, Pem]),
+    Fingerprint = crypto:hash(md5, [RawDir, Filename1, Pem]),
     Suffix = binary:encode_hex(binary:part(Fingerprint, 0, 8)),
     Filename = <<Filename1/binary, "-", Suffix/binary>>,
-    filename:join([pem_dir(Dir), Filename]).
+    filename:join([Dir, Filename]).
 
 pem_dir(Dir) ->
     filename:join([emqx:mutable_certs_dir(), Dir]).

apps/emqx/test/emqx_takeover_SUITE.erl

@@ -71,6 +71,7 @@ init_per_group(persistence_enabled = Group, Config) ->
             {emqx,
                 "durable_sessions = {\n"
                 "  enable = true\n"
+                "  force_persistence = true\n"
                 "  heartbeat_interval = 100ms\n"
                 "  renew_streams_interval = 100ms\n"
                 "  session_gc_interval = 2s\n"
@@ -113,7 +114,15 @@ end_per_group(_Group, _Config) ->
 
 t_takeover(Config) ->
     process_flag(trap_exit, true),
-    ClientId = atom_to_binary(?FUNCTION_NAME),
+    Vsn = atom_to_list(?config(mqtt_vsn, Config)),
+    Persist =
+        case ?config(persistence_enabled, Config) of
+            true ->
+                "persistent-";
+            false ->
+                "not-persistent-"
+        end,
+    ClientId = iolist_to_binary("t_takeover-" ++ Persist ++ Vsn),
     ClientOpts = [
         {proto_ver, ?config(mqtt_vsn, Config)},
         {clean_start, false}
@@ -176,7 +185,7 @@ t_takeover(Config) ->
 t_takeover_willmsg(Config) ->
     process_flag(trap_exit, true),
     ClientId = atom_to_binary(?FUNCTION_NAME),
-    WillTopic = <<ClientId/binary, <<"willtopic">>/binary>>,
+    WillTopic = <<ClientId/binary, <<"_willtopic">>/binary>>,
     Middle = ?CNT div 2,
     Client1Msgs = messages(ClientId, 0, Middle),
     Client2Msgs = messages(ClientId, Middle, ?CNT div 2),
@@ -890,16 +899,11 @@ t_kick_session(Config) ->
             {fun start_client/5, [
                 <<ClientId/binary, <<"_willsub">>/binary>>, WillTopic, ?QOS_1, []
             ]},
-            %% kick may fail (not found) without this delay
-            {
-                fun(CTX) ->
-                    timer:sleep(300),
-                    CTX
-                end,
-                []
-            },
+            {fun wait_for_chan_reg/2, [ClientId]},
             %% WHEN: client is kicked with kick_session
-            {fun kick_client/2, [ClientId]}
+            {fun kick_client/2, [ClientId]},
+            {fun wait_for_chan_dereg/2, [ClientId]},
+            {fun wait_for_pub_client_down/1, []}
         ]),
     FCtx = lists:foldl(
         fun({Fun, Args}, Ctx) ->
@@ -911,7 +915,7 @@ t_kick_session(Config) ->
     ),
     #{client := [CPidSub, CPid1]} = FCtx,
     assert_client_exit(CPid1, ?config(mqtt_vsn, Config), kicked),
-    Received = [Msg || {publish, Msg} <- ?drainMailbox(?SLEEP)],
+    Received = [Msg || {publish, Msg} <- ?drainMailbox(timer:seconds(1))],
     ct:pal("received: ~p", [[P || #{payload := P} <- Received]]),
     %% THEN: payload <<"willpayload_kick">> should be published
     {IsWill, _ReceivedNoWill} = filter_payload(Received, <<"willpayload_kick">>),
@@ -920,6 +924,30 @@ t_kick_session(Config) ->
     ?assert(not is_process_alive(CPid1)),
     ok.
 
+wait_for_chan_reg(CTX, ClientId) ->
+    ?retry(
+        3_000,
+        100,
+        true = is_map(emqx_cm:get_chan_info(ClientId))
+    ),
+    CTX.
+
+wait_for_chan_dereg(CTX, ClientId) ->
+    ?retry(
+        3_000,
+        100,
+        undefined = emqx_cm:get_chan_info(ClientId)
+    ),
+    CTX.
+
+wait_for_pub_client_down(#{client := [_SubClient, PubClient]} = CTX) ->
+    ?retry(
+        3_000,
+        100,
+        false = is_process_alive(PubClient)
+    ),
+    CTX.
+
 %% t_takover_in_cluster(_) ->
 %%     todo.
 
@@ -929,7 +957,6 @@ start_client(Ctx, ClientId, Topic, Qos, Opts) ->
     {ok, CPid} = emqtt:start_link([{clientid, ClientId} | Opts]),
     _ = erlang:spawn_link(fun() ->
         {ok, _} = emqtt:connect(CPid),
-        ct:pal("CLIENT: connected ~p", [CPid]),
         {ok, _, [Qos]} = emqtt:subscribe(CPid, Topic, Qos)
     end),
     Ctx#{client => [CPid | maps:get(client, Ctx, [])]}.

apps/emqx/test/emqx_tls_certfile_gc_SUITE.erl

@@ -55,8 +55,8 @@ t_no_orphans(Config) ->
         <<"certfile">> => cert(),
         <<"cacertfile">> => cert()
     },
-    {ok, SSL} = emqx_tls_lib:ensure_ssl_files("ssl", SSL0),
-    {ok, SSLUnused} = emqx_tls_lib:ensure_ssl_files("unused", SSL0),
+    {ok, SSL} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("ssl", SSL0),
+    {ok, SSLUnused} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("unused", SSL0),
     SSLKeyfile = maps:get(<<"keyfile">>, SSL),
     ok = load_config(#{
         <<"clients">> => [
@@ -97,8 +97,8 @@ t_collect_orphans(_Config) ->
     SSL1 = SSL0#{
         <<"ocsp">> => #{<<"issuer_pem">> => cert()}
     },
-    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files("client", SSL0),
-    {ok, SSL3} = emqx_tls_lib:ensure_ssl_files("server", SSL1),
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("client", SSL0),
+    {ok, SSL3} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("server", SSL1),
     ok = load_config(#{
         <<"clients">> => [
             #{<<"transport">> => #{<<"ssl">> => SSL2}}
@@ -174,10 +174,10 @@ t_gc_runs_periodically(_Config) ->
         <<"keyfile">> => key(),
         <<"certfile">> => cert()
     },
-    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files("s1", SSL),
+    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s1", SSL),
     SSL1Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL1)),
     SSL1Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL1)),
-    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files("s2", SSL#{
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s2", SSL#{
         <<"ocsp">> => #{<<"issuer_pem">> => cert()}
     }),
     SSL2Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL2)),
@@ -275,10 +275,10 @@ t_gc_spares_recreated_certfiles(_Config) ->
         <<"keyfile">> => key(),
         <<"certfile">> => cert()
     },
-    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files("s1", SSL),
+    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s1", SSL),
     SSL1Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL1)),
     SSL1Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL1)),
-    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files("s2", SSL),
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s2", SSL),
     SSL2Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL2)),
     SSL2Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL2)),
     ok = load_config(#{}),
@@ -306,7 +306,7 @@ t_gc_spares_recreated_certfiles(_Config) ->
     % Recreate the SSL2 certfiles
     ok = file:delete(SSL2Keyfile),
     ok = file:delete(SSL2Certfile),
-    {ok, _} = emqx_tls_lib:ensure_ssl_files("s2", SSL),
+    {ok, _} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s2", SSL),
     % Nothing should have been collected
     ?assertMatch(
         {ok, []},
@@ -324,7 +324,7 @@ t_gc_spares_symlinked_datadir(Config) ->
         <<"certfile">> => cert(),
         <<"ocsp">> => #{<<"issuer_pem">> => cert()}
     },
-    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files("srv", SSL),
+    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("srv", SSL),
     SSL1Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL1)),
 
     ok = load_config(#{

apps/emqx/test/emqx_tls_lib_tests.erl

@@ -90,14 +90,14 @@ ssl_files_failure_test_() ->
         {"undefined_is_undefined", fun() ->
             ?assertEqual(
                 {ok, undefined},
-                emqx_tls_lib:ensure_ssl_files("dir", undefined)
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("dir", undefined)
             )
         end},
         {"no_op_if_disabled", fun() ->
             Disabled = #{<<"enable">> => false, foo => bar},
             ?assertEqual(
                 {ok, Disabled},
-                emqx_tls_lib:ensure_ssl_files("dir", Disabled)
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("dir", Disabled)
             )
         end},
         {"enoent_key_file", fun() ->
@@ -106,7 +106,7 @@ ssl_files_failure_test_() ->
             ),
             ?assertMatch(
                 {error, #{file_read := enoent, pem_check := invalid_pem}},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => NonExistingFile,
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key()
@@ -116,7 +116,7 @@ ssl_files_failure_test_() ->
         {"empty_cacertfile", fun() ->
             ?assertMatch(
                 {ok, _},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => test_key(),
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => <<"">>
@@ -130,7 +130,7 @@ ssl_files_failure_test_() ->
                     reason := pem_file_path_or_string_is_required,
                     which_options := [[<<"keyfile">>]]
                 }},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => <<>>,
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key()
@@ -141,7 +141,7 @@ ssl_files_failure_test_() ->
                 {error, #{
                     reason := invalid_file_path_or_pem_string, which_options := [[<<"keyfile">>]]
                 }},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => <<255, 255>>,
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key()
@@ -152,7 +152,7 @@ ssl_files_failure_test_() ->
                     reason := invalid_file_path_or_pem_string,
                     which_options := [[<<"ocsp">>, <<"issuer_pem">>]]
                 }},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => test_key(),
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key(),
@@ -162,7 +162,7 @@ ssl_files_failure_test_() ->
             %% not printable
             ?assertMatch(
                 {error, #{reason := invalid_file_path_or_pem_string}},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => <<33, 22>>,
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key()
@@ -173,7 +173,7 @@ ssl_files_failure_test_() ->
                 ok = file:write_file(TmpFile, <<"not a valid pem">>),
                 ?assertMatch(
                     {error, #{file_read := not_pem}},
-                    emqx_tls_lib:ensure_ssl_files(
+                    emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
                         "/tmp",
                         #{
                             <<"cacertfile">> => bin(TmpFile),
@@ -205,8 +205,8 @@ ssl_file_replace_test() ->
         <<"ocsp">> => #{<<"issuer_pem">> => Key2}
     },
     Dir = filename:join(["/tmp", "ssl-test-dir2"]),
-    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files(Dir, SSL0),
-    {ok, SSL3} = emqx_tls_lib:ensure_ssl_files(Dir, SSL1),
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir, SSL0),
+    {ok, SSL3} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir, SSL1),
     File1 = maps:get(<<"keyfile">>, SSL2),
     File2 = maps:get(<<"keyfile">>, SSL3),
     IssuerPem1 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2),
@@ -224,17 +224,39 @@ ssl_file_deterministic_names_test() ->
     },
     Dir0 = filename:join(["/tmp", ?FUNCTION_NAME, "ssl0"]),
     Dir1 = filename:join(["/tmp", ?FUNCTION_NAME, "ssl1"]),
-    {ok, SSLFiles0} = emqx_tls_lib:ensure_ssl_files(Dir0, SSL0),
+    {ok, SSLFiles0} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir0, SSL0),
     ?assertEqual(
         {ok, SSLFiles0},
-        emqx_tls_lib:ensure_ssl_files(Dir0, SSL0)
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir0, SSL0)
     ),
     ?assertNotEqual(
         {ok, SSLFiles0},
-        emqx_tls_lib:ensure_ssl_files(Dir1, SSL0)
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir1, SSL0)
     ),
     _ = file:del_dir_r(filename:join(["/tmp", ?FUNCTION_NAME])).
 
+ssl_file_name_hash_test() ->
+    ?assertMatch(
+        {ok, #{
+            <<"cacertfile">> := <<"data/certs/authz/http/cacert-C62234D748AB82B0">>,
+            <<"certfile">> := <<"data/certs/authz/http/cert-0D6E53DBDEF594A4">>,
+            <<"enable">> := true,
+            <<"keyfile">> := <<"data/certs/authz/http/key-D5BB7F027841FA62">>,
+            <<"verify">> := <<"verify_peer">>
+        }},
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
+            <<"authz/http">>,
+            #{
+                <<"cacertfile">> => test_name_hash_cacert(),
+                <<"certfile">> => test_name_hash_cert(),
+                <<"keyfile">> => test_name_hash_key(),
+                <<"enable">> => true,
+                <<"verify">> => <<"verify_peer">>
+            }
+        )
+    ),
+    ok.
+
 to_client_opts_test() ->
     VersionsAll = [tlsv1, 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'],
     Versions13Only = ['tlsv1.3'],
@@ -329,3 +351,81 @@ test_key2() ->
         "AbToUD4JmV9m/XwcSVH06ZaWqNuC5w==\n"
         "-----END EC PRIVATE KEY-----\n"
     >>.
+
+test_name_hash_cacert() ->
+    <<
+        "-----BEGIN CERTIFICATE-----\n"
+        "MIIDUTCCAjmgAwIBAgIJAPPYCjTmxdt/MA0GCSqGSIb3DQEBCwUAMD8xCzAJBgNV\n"
+        "BAYTAkNOMREwDwYDVQQIDAhoYW5nemhvdTEMMAoGA1UECgwDRU1RMQ8wDQYDVQQD\n"
+        "DAZSb290Q0EwHhcNMjAwNTA4MDgwNjUyWhcNMzAwNTA2MDgwNjUyWjA/MQswCQYD\n"
+        "VQQGEwJDTjERMA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UE\n"
+        "AwwGUm9vdENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzcgVLex1\n"
+        "EZ9ON64EX8v+wcSjzOZpiEOsAOuSXOEN3wb8FKUxCdsGrsJYB7a5VM/Jot25Mod2\n"
+        "juS3OBMg6r85k2TWjdxUoUs+HiUB/pP/ARaaW6VntpAEokpij/przWMPgJnBF3Ur\n"
+        "MjtbLayH9hGmpQrI5c2vmHQ2reRZnSFbY+2b8SXZ+3lZZgz9+BaQYWdQWfaUWEHZ\n"
+        "uDaNiViVO0OT8DRjCuiDp3yYDj3iLWbTA/gDL6Tf5XuHuEwcOQUrd+h0hyIphO8D\n"
+        "tsrsHZ14j4AWYLk1CPA6pq1HIUvEl2rANx2lVUNv+nt64K/Mr3RnVQd9s8bK+TXQ\n"
+        "KGHd2Lv/PALYuwIDAQABo1AwTjAdBgNVHQ4EFgQUGBmW+iDzxctWAWxmhgdlE8Pj\n"
+        "EbQwHwYDVR0jBBgwFoAUGBmW+iDzxctWAWxmhgdlE8PjEbQwDAYDVR0TBAUwAwEB\n"
+        "/zANBgkqhkiG9w0BAQsFAAOCAQEAGbhRUjpIred4cFAFJ7bbYD9hKu/yzWPWkMRa\n"
+        "ErlCKHmuYsYk+5d16JQhJaFy6MGXfLgo3KV2itl0d+OWNH0U9ULXcglTxy6+njo5\n"
+        "CFqdUBPwN1jxhzo9yteDMKF4+AHIxbvCAJa17qcwUKR5MKNvv09C6pvQDJLzid7y\n"
+        "E2dkgSuggik3oa0427KvctFf8uhOV94RvEDyqvT5+pgNYZ2Yfga9pD/jjpoHEUlo\n"
+        "88IGU8/wJCx3Ds2yc8+oBg/ynxG8f/HmCC1ET6EHHoe2jlo8FpU/SgGtghS1YL30\n"
+        "IWxNsPrUP+XsZpBJy/mvOhE5QXo6Y35zDqqj8tI7AGmAWu22jg==\n"
+        "-----END CERTIFICATE-----\n"
+    >>.
+
+test_name_hash_cert() ->
+    <<
+        "-----BEGIN CERTIFICATE-----\n"
+        "MIIDEzCCAfugAwIBAgIBAjANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJDTjER\n"
+        "MA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UEAwwGUm9vdENB\n"
+        "MB4XDTIwMDUwODA4MDcwNVoXDTMwMDUwNjA4MDcwNVowPzELMAkGA1UEBhMCQ04x\n"
+        "ETAPBgNVBAgMCGhhbmd6aG91MQwwCgYDVQQKDANFTVExDzANBgNVBAMMBlNlcnZl\n"
+        "cjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALNeWT3pE+QFfiRJzKmn\n"
+        "AMUrWo3K2j/Tm3+Xnl6WLz67/0rcYrJbbKvS3uyRP/stXyXEKw9CepyQ1ViBVFkW\n"
+        "Aoy8qQEOWFDsZc/5UzhXUnb6LXr3qTkFEjNmhj+7uzv/lbBxlUG1NlYzSeOB6/RT\n"
+        "8zH/lhOeKhLnWYPXdXKsa1FL6ij4X8DeDO1kY7fvAGmBn/THh1uTpDizM4YmeI+7\n"
+        "4dmayA5xXvARte5h4Vu5SIze7iC057N+vymToMk2Jgk+ZZFpyXrnq+yo6RaD3ANc\n"
+        "lrc4FbeUQZ5a5s5Sxgs9a0Y3WMG+7c5VnVXcbjBRz/aq2NtOnQQjikKKQA8GF080\n"
+        "BQkCAwEAAaMaMBgwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQEL\n"
+        "BQADggEBAJefnMZpaRDHQSNUIEL3iwGXE9c6PmIsQVE2ustr+CakBp3TZ4l0enLt\n"
+        "iGMfEVFju69cO4oyokWv+hl5eCMkHBf14Kv51vj448jowYnF1zmzn7SEzm5Uzlsa\n"
+        "sqjtAprnLyof69WtLU1j5rYWBuFX86yOTwRAFNjm9fvhAcrEONBsQtqipBWkMROp\n"
+        "iUYMkRqbKcQMdwxov+lHBYKq9zbWRoqLROAn54SRqgQk6c15JdEfgOOjShbsOkIH\n"
+        "UhqcwRkQic7n1zwHVGVDgNIZVgmJ2IdIWBlPEC7oLrRrBD/X1iEEXtKab6p5o22n\n"
+        "KB5mN+iQaE+Oe2cpGKZJiJRdM+IqDDQ=\n"
+        "-----END CERTIFICATE-----\n"
+    >>.
+
+test_name_hash_key() ->
+    <<
+        "-----BEGIN RSA PRIVATE KEY-----\n"
+        "MIIEowIBAAKCAQEAs15ZPekT5AV+JEnMqacAxStajcraP9Obf5eeXpYvPrv/Stxi\n"
+        "sltsq9Le7JE/+y1fJcQrD0J6nJDVWIFUWRYCjLypAQ5YUOxlz/lTOFdSdvotevep\n"
+        "OQUSM2aGP7u7O/+VsHGVQbU2VjNJ44Hr9FPzMf+WE54qEudZg9d1cqxrUUvqKPhf\n"
+        "wN4M7WRjt+8AaYGf9MeHW5OkOLMzhiZ4j7vh2ZrIDnFe8BG17mHhW7lIjN7uILTn\n"
+        "s36/KZOgyTYmCT5lkWnJeuer7KjpFoPcA1yWtzgVt5RBnlrmzlLGCz1rRjdYwb7t\n"
+        "zlWdVdxuMFHP9qrY206dBCOKQopADwYXTzQFCQIDAQABAoIBAQCuvCbr7Pd3lvI/\n"
+        "n7VFQG+7pHRe1VKwAxDkx2t8cYos7y/QWcm8Ptwqtw58HzPZGWYrgGMCRpzzkRSF\n"
+        "V9g3wP1S5Scu5C6dBu5YIGc157tqNGXB+SpdZddJQ4Nc6yGHXYERllT04ffBGc3N\n"
+        "WG/oYS/1cSteiSIrsDy/91FvGRCi7FPxH3wIgHssY/tw69s1Cfvaq5lr2NTFzxIG\n"
+        "xCvpJKEdSfVfS9I7LYiymVjst3IOR/w76/ZFY9cRa8ZtmQSWWsm0TUpRC1jdcbkm\n"
+        "ZoJptYWlP+gSwx/fpMYftrkJFGOJhHJHQhwxT5X/ajAISeqjjwkWSEJLwnHQd11C\n"
+        "Zy2+29lBAoGBANlEAIK4VxCqyPXNKfoOOi5dS64NfvyH4A1v2+KaHWc7lqaqPN49\n"
+        "ezfN2n3X+KWx4cviDD914Yc2JQ1vVJjSaHci7yivocDo2OfZDmjBqzaMp/y+rX1R\n"
+        "/f3MmiTqMa468rjaxI9RRZu7vDgpTR+za1+OBCgMzjvAng8dJuN/5gjlAoGBANNY\n"
+        "uYPKtearBmkqdrSV7eTUe49Nhr0XotLaVBH37TCW0Xv9wjO2xmbm5Ga/DCtPIsBb\n"
+        "yPeYwX9FjoasuadUD7hRvbFu6dBa0HGLmkXRJZTcD7MEX2Lhu4BuC72yDLLFd0r+\n"
+        "Ep9WP7F5iJyagYqIZtz+4uf7gBvUDdmvXz3sGr1VAoGAdXTD6eeKeiI6PlhKBztF\n"
+        "zOb3EQOO0SsLv3fnodu7ZaHbUgLaoTMPuB17r2jgrYM7FKQCBxTNdfGZmmfDjlLB\n"
+        "0xZ5wL8ibU30ZXL8zTlWPElST9sto4B+FYVVF/vcG9sWeUUb2ncPcJ/Po3UAktDG\n"
+        "jYQTTyuNGtSJHpad/YOZctkCgYBtWRaC7bq3of0rJGFOhdQT9SwItN/lrfj8hyHA\n"
+        "OjpqTV4NfPmhsAtu6j96OZaeQc+FHvgXwt06cE6Rt4RG4uNPRluTFgO7XYFDfitP\n"
+        "vCppnoIw6S5BBvHwPP+uIhUX2bsi/dm8vu8tb+gSvo4PkwtFhEr6I9HglBKmcmog\n"
+        "q6waEQKBgHyecFBeM6Ls11Cd64vborwJPAuxIW7HBAFj/BS99oeG4TjBx4Sz2dFd\n"
+        "rzUibJt4ndnHIvCN8JQkjNG14i9hJln+H3mRss8fbZ9vQdqG+2vOWADYSzzsNI55\n"
+        "RFY7JjluKcVkp/zCDeUxTU3O6sS+v6/3VE11Cob6OYQx3lN5wrZ3\n"
+        "-----END RSA PRIVATE KEY-----\n"
+    >>.

apps/emqx_auth/src/emqx_auth.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth, [
     {description, "EMQX Authentication and authorization"},
-    {vsn, "0.3.4"},
+    {vsn, "0.4.0"},
     {modules, []},
     {registered, [emqx_auth_sup]},
     {applications, [

apps/emqx_auth/src/emqx_authn/emqx_authn_chains.erl

@@ -490,9 +490,7 @@ initialize_authentication(Providers) ->
     ProviderTypes = maps:keys(Providers),
     Chains = chain_configs(),
     HasProviders = has_providers_for_configs(Chains, ProviderTypes),
-    Result = do_initialize_authentication(Providers, Chains, HasProviders),
-    ?tp(info, authn_chains_initialization_done, #{providers => Providers}),
-    Result.
+    do_initialize_authentication(Providers, Chains, HasProviders).
 
 do_initialize_authentication(_Providers, _Chains, _HasProviders = false) ->
     false;

apps/emqx_auth/src/emqx_authn/emqx_authn_config.erl

@@ -280,7 +280,7 @@ convert_certs_for_conf_path(ConfPath, NewConfig) ->
 
 convert_certs(CertsDir, NewConfig) ->
     NewSSL = maps:get(<<"ssl">>, NewConfig, undefined),
-    case emqx_tls_lib:ensure_ssl_files(CertsDir, NewSSL) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(CertsDir, NewSSL) of
         {ok, NewSSL1} ->
             new_ssl_config(NewConfig, NewSSL1);
         {error, Reason} ->

apps/emqx_auth/src/emqx_authz/emqx_authz.erl

@@ -745,7 +745,7 @@ maybe_read_source_files_safe(Source0) ->
     end.
 
 maybe_write_certs(#{<<"type">> := Type, <<"ssl">> := SSL = #{}} = Source) ->
-    case emqx_tls_lib:ensure_ssl_files(ssl_file_path(Type), SSL) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(ssl_file_path(Type), SSL) of
         {ok, NSSL} ->
             Source#{<<"ssl">> => NSSL};
         {error, Reason} ->

apps/emqx_auth/test/emqx_authn/emqx_authn_init_SUITE.erl

@@ -68,16 +68,11 @@ t_initialize(_Config) ->
         {error, not_authorized},
         emqx_access_control:authenticate(?CLIENTINFO)
     ),
-
-    ?assertWaitEvent(
-        ok = emqx_authn_test_lib:register_fake_providers([{password_based, built_in_database}]),
-        #{
-            ?snk_kind := authn_chains_initialization_done,
-            providers := #{{password_based, built_in_database} := emqx_authn_fake_provider}
-        },
-        100
-    ),
-
+    %% call emqx_authn_chains:register_providers/1
+    %% which triggers a handle_continue to store the chain in ets
+    ok = emqx_authn_test_lib:register_fake_providers([{password_based, built_in_database}]),
+    %% make another gen_server call to make sure the handle_continue is complete
+    ?assertMatch(#{{password_based, built_in_database} := _}, emqx_authn_chains:get_providers()),
     ?assertMatch(
         {error, bad_username_or_password},
         emqx_access_control:authenticate(?CLIENTINFO)

apps/emqx_auth_http/src/emqx_auth_http.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth_http, [
     {description, "EMQX External HTTP API Authentication and Authorization"},
-    {vsn, "0.3.1"},
+    {vsn, "0.4.0"},
     {registered, []},
     {mod, {emqx_auth_http_app, []}},
     {applications, [

apps/emqx_auth_jwt/src/emqx_authn_jwt.erl

@@ -309,7 +309,7 @@ do_verify(JWT, [JWK | More], VerifyClaims) ->
     try jose_jws:verify(JWK, JWT) of
         {true, Payload, _JWT} ->
             Claims0 = emqx_utils_json:decode(Payload, [return_maps]),
-            Claims = try_convert_to_num(Claims0, [<<"exp">>, <<"iat">>, <<"nbf">>]),
+            Claims = try_convert_to_num(Claims0, [<<"exp">>, <<"nbf">>]),
             case verify_claims(Claims, VerifyClaims) of
                 ok ->
                     {ok, Claims};
@@ -331,9 +331,6 @@ verify_claims(Claims, VerifyClaims0) ->
             {<<"exp">>, fun(ExpireTime) ->
                 is_number(ExpireTime) andalso Now < ExpireTime
             end},
-            {<<"iat">>, fun(IssueAt) ->
-                is_number(IssueAt) andalso IssueAt =< Now
-            end},
             {<<"nbf">>, fun(NotBefore) ->
                 is_number(NotBefore) andalso NotBefore =< Now
             end}

apps/emqx_auth_jwt/test/emqx_authn_jwt_SUITE.erl

@@ -133,7 +133,7 @@ t_hmac_based(_) ->
     Credential4 = Credential#{password => JWS4},
     ?assertMatch({ok, #{is_superuser := false}}, emqx_authn_jwt:authenticate(Credential4, State3)),
 
-    %% Issued At
+    %% Issued At (iat) should not matter
     Payload5 = #{
         <<"username">> => <<"myuser">>,
         <<"iat">> => erlang:system_time(second) - 60,
@@ -149,9 +149,7 @@ t_hmac_based(_) ->
     },
     JWS6 = generate_jws('hmac-based', Payload6, Secret),
     Credential6 = Credential#{password => JWS6},
-    ?assertEqual(
-        {error, bad_username_or_password}, emqx_authn_jwt:authenticate(Credential6, State3)
-    ),
+    ?assertMatch({ok, #{is_superuser := false}}, emqx_authn_jwt:authenticate(Credential6, State3)),
 
     %% Not Before
     Payload7 = #{

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth_mnesia, [
     {description, "EMQX Buitl-in Database Authentication and Authorization"},
-    {vsn, "0.1.7"},
+    {vsn, "0.2.0"},
     {registered, []},
     {mod, {emqx_auth_mnesia_app, []}},
     {applications, [

apps/emqx_auth_mnesia/src/emqx_authz_api_mnesia.erl

@@ -555,8 +555,20 @@ user(put, #{
     bindings := #{username := Username},
     body := #{<<"username">> := Username, <<"rules">> := Rules}
 }) ->
-    emqx_authz_mnesia:store_rules({username, Username}, Rules),
-    {204};
+    case ensure_rules_len(Rules) of
+        ok ->
+            emqx_authz_mnesia:store_rules({username, Username}, Rules),
+            {204};
+        {error, too_many_rules} ->
+            {400, #{
+                code => <<"BAD_REQUEST">>,
+                message =>
+                    binfmt(
+                        <<"The rules length exceeds the maximum limit.">>,
+                        []
+                    )
+            }}
+    end;
 user(delete, #{bindings := #{username := Username}}) ->
     case emqx_authz_mnesia:get_rules({username, Username}) of
         not_found ->
@@ -580,8 +592,20 @@ client(put, #{
     bindings := #{clientid := ClientID},
     body := #{<<"clientid">> := ClientID, <<"rules">> := Rules}
 }) ->
-    emqx_authz_mnesia:store_rules({clientid, ClientID}, Rules),
-    {204};
+    case ensure_rules_len(Rules) of
+        ok ->
+            emqx_authz_mnesia:store_rules({clientid, ClientID}, Rules),
+            {204};
+        {error, too_many_rules} ->
+            {400, #{
+                code => <<"BAD_REQUEST">>,
+                message =>
+                    binfmt(
+                        <<"The rules length exceeds the maximum limit.">>,
+                        []
+                    )
+            }}
+    end;
 client(delete, #{bindings := #{clientid := ClientID}}) ->
     case emqx_authz_mnesia:get_rules({clientid, ClientID}) of
         not_found ->

apps/emqx_auth_mnesia/test/emqx_authz_api_mnesia_SUITE.erl

@@ -136,6 +136,16 @@ t_api(_) ->
             uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]),
             ?USERNAME_RULES_EXAMPLE#{rules => []}
         ),
+
+    %% check length limit
+
+    {ok, 400, _} =
+        request(
+            put,
+            uri(["authorization", "sources", "built_in_database", "rules", "users", "user1"]),
+            dup_rules_example2(?USERNAME_RULES_EXAMPLE)
+        ),
+
     {ok, 200, Request3} =
         request(
             get,
@@ -219,6 +229,16 @@ t_api(_) ->
             uri(["authorization", "sources", "built_in_database", "rules", "clients", "client1"]),
             ?CLIENTID_RULES_EXAMPLE#{rules => []}
         ),
+
+    {ok, 400, _} =
+        request(
+            put,
+            uri(["authorization", "sources", "built_in_database", "rules", "clients", "client1"]),
+            dup_rules_example2(
+                ?CLIENTID_RULES_EXAMPLE
+            )
+        ),
+
     {ok, 200, Request6} =
         request(
             get,
@@ -521,3 +541,6 @@ dup_rules_example(#{clientid := _, rules := Rules}) ->
     #{clientid => client2, rules => Rules ++ Rules};
 dup_rules_example(#{rules := Rules}) ->
     #{rules => Rules ++ Rules}.
+
+dup_rules_example2(#{rules := Rules} = Example) ->
+    Example#{rules := Rules ++ Rules}.

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -601,7 +601,7 @@ schema("/bridges_probe") ->
                     ?NO_CONTENT;
                 {error, #{kind := validation_error} = Reason0} ->
                     Reason = redact(Reason0),
-                    ?BAD_REQUEST('TEST_FAILED', map_to_json(Reason));
+                    ?BAD_REQUEST('TEST_FAILED', emqx_utils_api:to_json(Reason));
                 {error, Reason0} when not is_tuple(Reason0); element(1, Reason0) =/= 'exit' ->
                     Reason1 =
                         case Reason0 of
@@ -681,9 +681,9 @@ create_or_update_bridge(BridgeType0, BridgeName, Conf, HttpStatusCode) ->
         {ok, _} ->
             lookup_from_all_nodes(BridgeType, BridgeName, HttpStatusCode);
         {error, {pre_config_update, _HandlerMod, Reason}} when is_map(Reason) ->
-            ?BAD_REQUEST(map_to_json(redact(Reason)));
+            ?BAD_REQUEST(emqx_utils_api:to_json(redact(Reason)));
         {error, Reason} when is_map(Reason) ->
-            ?BAD_REQUEST(map_to_json(redact(Reason)))
+            ?BAD_REQUEST(emqx_utils_api:to_json(redact(Reason)))
     end.
 
 get_metrics_from_local_node(BridgeType0, BridgeName) ->
@@ -1159,19 +1159,6 @@ bpapi_version_range(From, To) ->
 redact(Term) ->
     emqx_utils:redact(Term).
 
-map_to_json(M0) ->
-    %% When dealing with Hocon validation errors, `value' might contain non-serializable
-    %% values (e.g.: user_lookup_fun), so we try again without that key if serialization
-    %% fails as a best effort.
-    M1 = emqx_utils_maps:jsonable_map(M0, fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end),
-    try
-        emqx_utils_json:encode(M1)
-    catch
-        error:_ ->
-            M2 = maps:without([value, <<"value">>], M1),
-            emqx_utils_json:encode(M2)
-    end.
-
 non_compat_bridge_msg() ->
     <<"bridge already exists as non Bridge V1 compatible action">>.
 

apps/emqx_bridge/src/emqx_bridge_v2_api.erl

@@ -927,7 +927,7 @@ handle_probe(ConfRootKey, Request) ->
                     ?NO_CONTENT;
                 {error, #{kind := validation_error} = Reason0} ->
                     Reason = redact(Reason0),
-                    ?BAD_REQUEST('TEST_FAILED', map_to_json(Reason));
+                    ?BAD_REQUEST('TEST_FAILED', emqx_utils_api:to_json(Reason));
                 {error, Reason0} when not is_tuple(Reason0); element(1, Reason0) =/= 'exit' ->
                     Reason1 =
                         case Reason0 of
@@ -1426,7 +1426,7 @@ create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatusCod
             ok = emqx_resource:validate_name(BridgeName)
         catch
             throw:Error ->
-                ?BAD_REQUEST(map_to_json(Error))
+                ?BAD_REQUEST(emqx_utils_api:to_json(Error))
         end,
     case Check of
         ok ->
@@ -1443,9 +1443,9 @@ do_create_or_update_bridge(ConfRootKey, BridgeType, BridgeName, Conf, HttpStatus
             PreOrPostConfigUpdate =:= pre_config_update;
             PreOrPostConfigUpdate =:= post_config_update
         ->
-            ?BAD_REQUEST(map_to_json(redact(Reason)));
+            ?BAD_REQUEST(emqx_utils_api:to_json(redact(Reason)));
         {error, Reason} when is_map(Reason) ->
-            ?BAD_REQUEST(map_to_json(redact(Reason)))
+            ?BAD_REQUEST(emqx_utils_api:to_json(redact(Reason)))
     end.
 
 enable_func(true) -> enable;
@@ -1471,19 +1471,6 @@ bin(S) when is_atom(S) ->
 bin(S) when is_binary(S) ->
     S.
 
-map_to_json(M0) ->
-    %% When dealing with Hocon validation errors, `value' might contain non-serializable
-    %% values (e.g.: user_lookup_fun), so we try again without that key if serialization
-    %% fails as a best effort.
-    M1 = emqx_utils_maps:jsonable_map(M0, fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end),
-    try
-        emqx_utils_json:encode(M1)
-    catch
-        error:_ ->
-            M2 = maps:without([value, <<"value">>], M1),
-            emqx_utils_json:encode(M2)
-    end.
-
 to_existing_atom(X) ->
     case emqx_utils:safe_to_existing_atom(X, utf8) of
         {ok, A} -> A;

apps/emqx_bridge_azure_blob_storage/mix.exs

@@ -28,7 +28,7 @@ defmodule EMQXBridgeAzureBlobStorage.MixProject do
     [
       {:emqx_resource, in_umbrella: true},
       {:emqx_connector_aggregator, in_umbrella: true},
-      {:erlazure, github: "emqx/erlazure", tag: "0.4.0.0"}
+      {:erlazure, github: "emqx/erlazure", tag: "0.4.0.1"}
     ]
   end
 end

apps/emqx_bridge_azure_blob_storage/rebar.config

@@ -12,5 +12,5 @@
 {deps, [
     {emqx_resource, {path, "../../apps/emqx_resource"}},
     {emqx_connector_aggregator, {path, "../../apps/emqx_connector_aggregator"}},
-    {erlazure, {git, "https://github.com/emqx/erlazure.git", {tag, "0.4.0.0"}}}
+    {erlazure, {git, "https://github.com/emqx/erlazure.git", {tag, "0.4.0.1"}}}
 ]}.

apps/emqx_bridge_couchbase/src/emqx_bridge_couchbase_action_schema.erl

@@ -67,7 +67,7 @@ fields(?ACTION_TYPE) ->
 fields(parameters) ->
     [
         {sql, mk(emqx_schema:template(), #{required => true, desc => ?DESC("sql")})},
-        {max_retries, mk(non_neg_integer(), #{required => false, desc => ?DESC("max_retries")})}
+        {max_retries, mk(non_neg_integer(), #{default => 3, desc => ?DESC("max_retries")})}
     ];
 fields(action_resource_opts) ->
     Fields = emqx_bridge_v2_schema:action_resource_opts_fields(),

apps/emqx_bridge_kafka/src/emqx_bridge_kafka.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_bridge_kafka, [
     {description, "EMQX Enterprise Kafka Bridge"},
-    {vsn, "0.3.4"},
+    {vsn, "0.4.0"},
     {registered, [emqx_bridge_kafka_consumer_sup]},
     {applications, [
         kernel,

apps/emqx_bridge_kafka/src/emqx_bridge_kafka_consumer_schema.erl

@@ -90,9 +90,6 @@ fields(source_parameters) ->
                 binary(),
                 #{
                     required => false,
-                    validator => [
-                        emqx_resource_validator:not_empty("Group id must not be empty")
-                    ],
                     desc => ?DESC(group_id)
                 }
             )}

apps/emqx_bridge_kafka/src/emqx_bridge_kafka_impl_consumer.erl

@@ -627,7 +627,9 @@ log_when_error(Fun, Log) ->
     end.
 
 -spec consumer_group_id(#{group_id => binary(), any() => term()}, atom() | binary()) -> binary().
-consumer_group_id(#{group_id := GroupId}, _BridgeName) when is_binary(GroupId) ->
+consumer_group_id(#{group_id := GroupId}, _BridgeName) when
+    is_binary(GroupId) andalso GroupId =/= <<"">>
+->
     GroupId;
 consumer_group_id(_ConsumerParams, BridgeName0) ->
     BridgeName = to_bin(BridgeName0),

apps/emqx_bridge_kafka/test/emqx_bridge_kafka_tests.erl

@@ -217,13 +217,9 @@ custom_group_id_test() ->
         BaseConfig,
         #{<<"parameters">> => #{<<"group_id">> => <<>>}}
     ),
-    ?assertThrow(
-        {_, [
-            #{
-                path := "sources.kafka_consumer.my_consumer.parameters.group_id",
-                reason := "Group id must not be empty"
-            }
-        ]},
+    %% Empty strings will be treated as absent by the connector.
+    ?assertMatch(
+        #{<<"parameters">> := #{<<"group_id">> := <<"">>}},
         emqx_bridge_v2_testlib:parse_and_check(source, kafka_consumer, my_consumer, BadSourceConfig)
     ),
 

apps/emqx_bridge_kafka/test/emqx_bridge_v2_kafka_consumer_SUITE.erl

@@ -352,6 +352,69 @@ t_bad_bootstrap_host(Config) ->
     ),
     ok.
 
+%% Checks that a group id is automatically generated if a custom one is not provided in
+%% the config.
+t_absent_group_id(Config) ->
+    ?check_trace(
+        begin
+            #{<<"bootstrap_hosts">> := BootstrapHosts} = ?config(connector_config, Config),
+            SourceConfig = ?config(source_config, Config),
+            SourceName = ?config(source_name, Config),
+            ?assertEqual(
+                undefined,
+                emqx_utils_maps:deep_get(
+                    [<<"parameters">>, <<"group_id">>],
+                    SourceConfig,
+                    undefined
+                )
+            ),
+            {ok, {{_, 201, _}, _, _}} = emqx_bridge_v2_testlib:create_bridge_api(Config),
+            [Endpoint] = emqx_bridge_kafka_impl:hosts(BootstrapHosts),
+            GroupId = emqx_bridge_kafka_impl_consumer:consumer_group_id(#{}, SourceName),
+            ct:pal("generated group id: ~p", [GroupId]),
+            ?retry(100, 10, begin
+                {ok, Groups} = brod:list_groups(Endpoint, _ConnOpts = #{}),
+                ?assertMatch(
+                    [_],
+                    [Group || Group = {_, Id, _} <- Groups, Id == GroupId],
+                    #{groups => Groups}
+                )
+            end),
+            ok
+        end,
+        []
+    ),
+    ok.
+
+%% Checks that a group id is automatically generated if an empty string is provided in the
+%% config.
+t_empty_group_id(Config) ->
+    ?check_trace(
+        begin
+            #{<<"bootstrap_hosts">> := BootstrapHosts} = ?config(connector_config, Config),
+            SourceName = ?config(source_name, Config),
+            {ok, {{_, 201, _}, _, _}} =
+                emqx_bridge_v2_testlib:create_bridge_api(
+                    Config,
+                    #{<<"parameters">> => #{<<"group_id">> => <<"">>}}
+                ),
+            [Endpoint] = emqx_bridge_kafka_impl:hosts(BootstrapHosts),
+            GroupId = emqx_bridge_kafka_impl_consumer:consumer_group_id(#{}, SourceName),
+            ct:pal("generated group id: ~p", [GroupId]),
+            ?retry(100, 10, begin
+                {ok, Groups} = brod:list_groups(Endpoint, _ConnOpts = #{}),
+                ?assertMatch(
+                    [_],
+                    [Group || Group = {_, Id, _} <- Groups, Id == GroupId],
+                    #{groups => Groups}
+                )
+            end),
+            ok
+        end,
+        []
+    ),
+    ok.
+
 t_custom_group_id(Config) ->
     ?check_trace(
         begin

apps/emqx_cluster_link/src/emqx_cluster_link_api.erl

@@ -60,7 +60,7 @@ schema("/cluster/links") ->
                 'requestBody' => link_config_schema(),
                 responses =>
                     #{
-                        200 => link_config_schema_response(),
+                        201 => link_config_schema_response(),
                         400 =>
                             emqx_dashboard_swagger:error_codes(
                                 [?BAD_REQUEST, ?ALREADY_EXISTS],
@@ -176,7 +176,7 @@ fields(node_metrics) ->
 '/cluster/links/link/:name'(get, #{bindings := #{name := Name}}) ->
     with_link(Name, fun(Link) -> handle_lookup(Name, Link) end, not_found());
 '/cluster/links/link/:name'(put, #{bindings := #{name := Name}, body := Params0}) ->
-    with_link(Name, fun() -> handle_update(Name, Params0) end, not_found());
+    with_link(Name, fun(OldLink) -> handle_update(Name, Params0, OldLink) end, not_found());
 '/cluster/links/link/:name'(delete, #{bindings := #{name := Name}}) ->
     with_link(
         Name,
@@ -214,23 +214,38 @@ handle_list() ->
         lists:map(
             fun(#{<<"name">> := Name} = Link) ->
                 Status = maps:get(Name, NameToStatus, EmptyStatus),
-                maps:merge(Link, Status)
+                redact(maps:merge(Link, Status))
             end,
             Links
         ),
     ?OK(Response).
 
 handle_create(Name, Params) ->
+    Check =
+        try
+            ok = emqx_resource:validate_name(Name)
+        catch
+            throw:Error ->
+                ?BAD_REQUEST(emqx_utils_api:to_json(redact(Error)))
+        end,
+    case Check of
+        ok ->
+            do_create(Name, Params);
+        BadRequest ->
+            redact(BadRequest)
+    end.
+
+do_create(Name, Params) ->
     case emqx_cluster_link_config:create_link(Params) of
         {ok, Link} ->
-            ?CREATED(add_status(Name, Link));
+            ?CREATED(redact(add_status(Name, Link)));
         {error, Reason} ->
-            Message = list_to_binary(io_lib:format("Create link failed ~p", [Reason])),
+            Message = list_to_binary(io_lib:format("Create link failed ~p", [redact(Reason)])),
             ?BAD_REQUEST(Message)
     end.
 
 handle_lookup(Name, Link) ->
-    ?OK(add_status(Name, Link)).
+    ?OK(redact(add_status(Name, Link))).
 
 handle_metrics(Name) ->
     Results = emqx_cluster_link_metrics:get_metrics(Name),
@@ -325,7 +340,6 @@ format_metrics(Node, RouterMetrics, ResourceMetrics) ->
                 'failed' => Get([counters, 'failed'], ResourceMetrics),
                 'dropped' => Get([counters, 'dropped'], ResourceMetrics),
                 'retried' => Get([counters, 'retried'], ResourceMetrics),
-                'received' => Get([counters, 'received'], ResourceMetrics),
 
                 'queuing' => Get([gauges, 'queuing'], ResourceMetrics),
                 'inflight' => Get([gauges, 'inflight'], ResourceMetrics),
@@ -342,13 +356,14 @@ add_status(Name, Link) ->
     Status = collect_single_status(NodeRPCResults),
     maps:merge(Link, Status).
 
-handle_update(Name, Params0) ->
-    Params = Params0#{<<"name">> => Name},
+handle_update(Name, Params0, OldLinkRaw) ->
+    Params1 = Params0#{<<"name">> => Name},
+    Params = emqx_utils:deobfuscate(Params1, OldLinkRaw),
     case emqx_cluster_link_config:update_link(Params) of
         {ok, Link} ->
-            ?OK(add_status(Name, Link));
+            ?OK(redact(add_status(Name, Link)));
         {error, Reason} ->
-            Message = list_to_binary(io_lib:format("Update link failed ~p", [Reason])),
+            Message = list_to_binary(io_lib:format("Update link failed ~p", [redact(Reason)])),
             ?BAD_REQUEST(Message)
     end.
 
@@ -580,7 +595,7 @@ fill_defaults_single(Link0) ->
     #{<<"cluster">> := #{<<"links">> := [Link]}} =
         emqx_config:fill_defaults(
             #{<<"cluster">> => #{<<"links">> => [Link0]}},
-            #{obfuscate_sensitive_values => true}
+            #{obfuscate_sensitive_values => false}
         ),
     Link.
 
@@ -589,3 +604,6 @@ return(Response) ->
 
 not_found() ->
     return(?NOT_FOUND(<<"Cluster link not found">>)).
+
+redact(Value) ->
+    emqx_utils:redact(Value).

apps/emqx_cluster_link/src/emqx_cluster_link_config.erl

@@ -397,7 +397,7 @@ convert_certs(LinksConf) ->
     ).
 
 do_convert_certs(LinkName, SSLOpts) ->
-    case emqx_tls_lib:ensure_ssl_files(?CERTS_PATH(LinkName), SSLOpts) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(?CERTS_PATH(LinkName), SSLOpts) of
         {ok, undefined} ->
             SSLOpts;
         {ok, SSLOpts1} ->

apps/emqx_cluster_link/test/emqx_cluster_link_api_SUITE.erl

@@ -40,6 +40,7 @@
 >>).
 
 -define(ON(NODE, BODY), erpc:call(NODE, fun() -> BODY end)).
+-define(REDACTED, <<"******">>).
 
 %%------------------------------------------------------------------------------
 %% CT boilerplate
@@ -189,6 +190,7 @@ link_params() ->
 link_params(Overrides) ->
     Default = #{
         <<"clientid">> => <<"linkclientid">>,
+        <<"password">> => <<"my secret password">>,
         <<"pool_size">> => 1,
         <<"server">> => <<"emqxcl_2.nohost:31883">>,
         <<"topics">> => [<<"t/test-topic">>, <<"t/test/#">>]
@@ -253,6 +255,7 @@ t_crud(_Config) ->
     ?assertMatch(
         {201, #{
             <<"name">> := NameA,
+            <<"password">> := ?REDACTED,
             <<"status">> := _,
             <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _]
         }},
@@ -263,6 +266,7 @@ t_crud(_Config) ->
         {200, [
             #{
                 <<"name">> := NameA,
+                <<"password">> := ?REDACTED,
                 <<"status">> := _,
                 <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _]
             }
@@ -272,6 +276,7 @@ t_crud(_Config) ->
     ?assertMatch(
         {200, #{
             <<"name">> := NameA,
+            <<"password">> := ?REDACTED,
             <<"status">> := _,
             <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _]
         }},
@@ -283,6 +288,7 @@ t_crud(_Config) ->
     ?assertMatch(
         {200, #{
             <<"name">> := NameA,
+            <<"password">> := ?REDACTED,
             <<"status">> := _,
             <<"node_status">> := [#{<<"node">> := _, <<"status">> := _} | _]
         }},
@@ -298,6 +304,39 @@ t_crud(_Config) ->
 
     ok.
 
+t_create_invalid(_Config) ->
+    Params = link_params(),
+    EmptyName = <<>>,
+    {400, #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := Message1}} = create_link(
+        EmptyName, Params
+    ),
+    ?assertMatch(
+        #{<<"kind">> := <<"validation_error">>, <<"reason">> := <<"Name cannot be empty string">>},
+        Message1
+    ),
+    LongName = binary:copy(<<$a>>, 256),
+    {400, #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := Message2}} = create_link(
+        LongName, Params
+    ),
+    ?assertMatch(
+        #{
+            <<"kind">> := <<"validation_error">>,
+            <<"reason">> := <<"Name length must be less than 255">>
+        },
+        Message2
+    ),
+    BadName = <<"~!@#$%^&*()_+{}:'<>?|">>,
+    {400, #{<<"code">> := <<"BAD_REQUEST">>, <<"message">> := Message3}} = create_link(
+        BadName, Params
+    ),
+    ?assertMatch(
+        #{
+            <<"kind">> := <<"validation_error">>,
+            <<"reason">> := <<"Invalid name format", _/binary>>
+        },
+        Message3
+    ).
+
 %% Verifies the behavior of reported status under different conditions when listing all
 %% links and when fetching a specific link.
 t_status(Config) ->
@@ -487,7 +526,6 @@ t_metrics(Config) ->
                     <<"failed">> := _,
                     <<"dropped">> := _,
                     <<"retried">> := _,
-                    <<"received">> := _,
                     <<"queuing">> := _,
                     <<"inflight">> := _,
                     <<"rate">> := _,
@@ -508,7 +546,6 @@ t_metrics(Config) ->
                             <<"failed">> := _,
                             <<"dropped">> := _,
                             <<"retried">> := _,
-                            <<"received">> := _,
                             <<"queuing">> := _,
                             <<"inflight">> := _,
                             <<"rate">> := _,
@@ -529,7 +566,6 @@ t_metrics(Config) ->
                             <<"failed">> := _,
                             <<"dropped">> := _,
                             <<"retried">> := _,
-                            <<"received">> := _,
                             <<"queuing">> := _,
                             <<"inflight">> := _,
                             <<"rate">> := _,
@@ -720,3 +756,26 @@ t_metrics(Config) ->
     ),
 
     ok.
+
+%% Checks that we can update a link via the API in the same fashion as the frontend does,
+%% by sending secrets as `******', and the secret is not mangled.
+t_update_password(_Config) ->
+    ?check_trace(
+        begin
+            Name = atom_to_binary(?FUNCTION_NAME),
+            Password = <<"my secret password">>,
+            Params1 = link_params(#{<<"password">> => Password}),
+            {201, Response1} = create_link(Name, Params1),
+            [#{name := Name, password := WrappedPassword0}] = emqx_config:get([cluster, links]),
+            ?assertEqual(Password, emqx_secret:unwrap(WrappedPassword0)),
+            Params2A = maps:without([<<"name">>, <<"node_status">>, <<"status">>], Response1),
+            Params2 = Params2A#{<<"pool_size">> := 2},
+            ?assertEqual(?REDACTED, maps:get(<<"password">>, Params2)),
+            ?assertMatch({200, _}, update_link(Name, Params2)),
+            [#{name := Name, password := WrappedPassword}] = emqx_config:get([cluster, links]),
+            ?assertEqual(Password, emqx_secret:unwrap(WrappedPassword)),
+            ok
+        end,
+        []
+    ),
+    ok.

apps/emqx_conf/src/emqx_conf.app.src

@@ -1,6 +1,6 @@
 {application, emqx_conf, [
     {description, "EMQX configuration management"},
-    {vsn, "0.2.4"},
+    {vsn, "0.3.0"},
     {registered, []},
     {mod, {emqx_conf_app, []}},
     {applications, [kernel, stdlib]},

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -1390,7 +1390,6 @@ log_handler_common_confs(Handler, Default) ->
         {"payload_encode",
             sc(hoconsc:enum([hex, text, hidden]), #{
                 default => text,
-                importance => ?IMPORTANCE_HIDDEN,
                 desc => ?DESC(emqx_schema, fields_trace_payload_encode)
             })}
     ].

apps/emqx_conf/src/emqx_conf_schema_inject.erl

@@ -27,6 +27,7 @@ schemas(Edition) ->
         cluster_linking(Edition) ++
         authn(Edition) ++
         authz() ++
+        shared_subs(Edition) ++
         customized(Edition).
 
 mria(ce) ->
@@ -81,6 +82,11 @@ authz_mods() ->
         emqx_authz_ldap_schema
     ].
 
+shared_subs(ee) ->
+    [emqx_ds_shared_sub_schema];
+shared_subs(ce) ->
+    [].
+
 %% Add more schemas here.
 customized(_Edition) ->
     [].

apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl

@@ -56,6 +56,7 @@ init_per_suite(Config) ->
     ),
     meck:new(mria, [non_strict, passthrough, no_link]),
     meck:expect(mria, running_nodes, 0, [?NODE1, {node(), ?NODE2}, {node(), ?NODE3}]),
+    ok = emqx_cluster_rpc:wait_for_cluster_rpc(),
     [{suite_apps, Apps} | Config].
 
 end_per_suite(Config) ->
@@ -227,29 +228,39 @@ t_catch_up_status_handle_next_commit(_Config) ->
 t_commit_ok_apply_fail_on_other_node_then_recover(_Config) ->
     {atomic, []} = emqx_cluster_rpc:status(),
     ets:new(test, [named_table, public]),
-    ets:insert(test, {other_mfa_result, failed}),
-    ct:pal("111:~p~n", [ets:tab2list(cluster_rpc_commit)]),
-    {M, F, A} = {?MODULE, failed_on_other_recover_after_retry, [erlang:whereis(?NODE1)]},
-    {ok, 1, ok} = multicall(M, F, A, 1, 1000),
-    ct:pal("222:~p~n", [ets:tab2list(cluster_rpc_commit)]),
-    ct:pal("333:~p~n", [emqx_cluster_rpc:status()]),
-    {atomic, [_Status | L]} = emqx_cluster_rpc:status(),
-    ?assertEqual([], L),
-    ets:insert(test, {other_mfa_result, ok}),
-    {ok, 2, ok} = multicall(?MODULE, format, ["format:~p~n", [?FUNCTION_NAME]], 1, 1000),
-    ct:sleep(1000),
-    {atomic, NewStatus} = emqx_cluster_rpc:status(),
-    ?assertEqual(3, length(NewStatus)),
-    Pid = self(),
-    Msg = ?FUNCTION_NAME,
-    MFAEcho = {M1, F1, A1} = {?MODULE, echo, [Pid, Msg]},
-    {ok, TnxId, ok} = multicall(M1, F1, A1),
-    {atomic, Query} = emqx_cluster_rpc:query(TnxId),
-    ?assertEqual(MFAEcho, maps:get(mfa, Query)),
-    ?assertEqual(node(), maps:get(initiator, Query)),
-    ?assert(maps:is_key(created_at, Query)),
-    ?assertEqual(ok, receive_msg(3, Msg)),
-    ok.
+    try
+        %% step1: expect initial commits to be zero for all nodes
+        Commits1 = ets:tab2list(cluster_rpc_commit),
+        ct:pal("step1(expect all tnx_id to be zero):~n~p~n", [Commits1]),
+        ct:pal("step1_inspect_status:~n~p~n", [emqx_cluster_rpc:status()]),
+        ?assertEqual([0, 0, 0], lists:map(fun({_RecordName, _Node, ID}) -> ID end, Commits1)),
+        %% step2: insert stub a failure, and cause one node to fail
+        ets:insert(test, {other_mfa_result, failed}),
+        {M, F, A} = {?MODULE, failed_on_other_recover_after_retry, [erlang:whereis(?NODE1)]},
+        {ok, 1, ok} = multicall(M, F, A, 1, 1000),
+        Commits2 = ets:tab2list(cluster_rpc_commit),
+        ct:pal("step2(expect node1 to have tnx_id=1):~n~p~n", [Commits2]),
+        ct:pal("step2_inspect_status:~n~p~n", [emqx_cluster_rpc:status()]),
+        {atomic, [_Status | L]} = emqx_cluster_rpc:status(),
+        ?assertEqual([], L),
+        ets:insert(test, {other_mfa_result, ok}),
+        {ok, 2, ok} = multicall(?MODULE, format, ["format:~p~n", [?FUNCTION_NAME]], 1, 1000),
+        ct:sleep(1000),
+        {atomic, NewStatus} = emqx_cluster_rpc:status(),
+        ?assertEqual(3, length(NewStatus)),
+        Pid = self(),
+        Msg = ?FUNCTION_NAME,
+        MFAEcho = {M1, F1, A1} = {?MODULE, echo, [Pid, Msg]},
+        {ok, TnxId, ok} = multicall(M1, F1, A1),
+        {atomic, Query} = emqx_cluster_rpc:query(TnxId),
+        ?assertEqual(MFAEcho, maps:get(mfa, Query)),
+        ?assertEqual(node(), maps:get(initiator, Query)),
+        ?assert(maps:is_key(created_at, Query)),
+        ?assertEqual(ok, receive_msg(3, Msg)),
+        ok
+    after
+        ets:delete(test)
+    end.
 
 t_del_stale_mfa(_Config) ->
     {atomic, []} = emqx_cluster_rpc:status(),
@@ -362,8 +373,6 @@ tnx_ids(Status) ->
 start() ->
     {ok, _Pid2} = emqx_cluster_rpc:start_link({node(), ?NODE2}, ?NODE2, 500),
     {ok, _Pid3} = emqx_cluster_rpc:start_link({node(), ?NODE3}, ?NODE3, 500),
-    ok = emqx_cluster_rpc:wait_for_cluster_rpc(),
-    ok = emqx_cluster_rpc:reset(),
     %% Ensure all processes are idle status.
     ok = gen_server:call(?NODE2, test),
     ok = gen_server:call(?NODE3, test),
@@ -382,7 +391,10 @@ stop() ->
             end
         end
      || N <- [?NODE2, ?NODE3]
-    ].
+    ],
+    %% erase all commit history, set commit tnx_id back to 0 for all nodes
+    ok = emqx_cluster_rpc:reset(),
+    ok.
 
 receive_msg(0, _Msg) ->
     ok;

apps/emqx_conf/test/emqx_conf_logger_SUITE.erl

@@ -78,7 +78,8 @@ t_log_conf(_Conf) ->
         <<"rotation_size">> => <<"50MB">>,
         <<"time_offset">> => <<"system">>,
         <<"path">> => <<"log/emqx.log">>,
-        <<"timestamp_format">> => <<"auto">>
+        <<"timestamp_format">> => <<"auto">>,
+        <<"payload_encode">> => <<"text">>
     },
     ExpectLog1 = #{
         <<"console">> =>
@@ -86,6 +87,7 @@ t_log_conf(_Conf) ->
                 <<"enable">> => true,
                 <<"formatter">> => <<"text">>,
                 <<"level">> => <<"debug">>,
+                <<"payload_encode">> => <<"text">>,
                 <<"time_offset">> => <<"system">>,
                 <<"timestamp_format">> => <<"auto">>
             },

apps/emqx_connector/src/emqx_connector_api.erl

@@ -389,7 +389,7 @@ schema("/connectors_probe") ->
                     ?NO_CONTENT;
                 {error, #{kind := validation_error} = Reason0} ->
                     Reason = redact(Reason0),
-                    ?BAD_REQUEST('TEST_FAILED', map_to_json(Reason));
+                    ?BAD_REQUEST('TEST_FAILED', emqx_utils_api:to_json(Reason));
                 {error, Reason0} when not is_tuple(Reason0); element(1, Reason0) =/= 'exit' ->
                     Reason1 =
                         case Reason0 of
@@ -449,7 +449,7 @@ create_or_update_connector(ConnectorType, ConnectorName, Conf, HttpStatusCode) -
             ok = emqx_resource:validate_name(ConnectorName)
         catch
             throw:Error ->
-                ?BAD_REQUEST(map_to_json(Error))
+                ?BAD_REQUEST(emqx_utils_api:to_json(Error))
         end,
     case Check of
         ok ->
@@ -466,9 +466,9 @@ do_create_or_update_connector(ConnectorType, ConnectorName, Conf, HttpStatusCode
             PreOrPostConfigUpdate =:= pre_config_update;
             PreOrPostConfigUpdate =:= post_config_update
         ->
-            ?BAD_REQUEST(map_to_json(redact(Reason)));
+            ?BAD_REQUEST(emqx_utils_api:to_json(redact(Reason)));
         {error, Reason} when is_map(Reason) ->
-            ?BAD_REQUEST(map_to_json(redact(Reason)))
+            ?BAD_REQUEST(emqx_utils_api:to_json(redact(Reason)))
     end.
 
 '/connectors/:id/enable/:enable'(put, #{bindings := #{id := Id, enable := Enable}}) ->
@@ -803,16 +803,3 @@ maybe_unwrap(RpcMulticallResult) ->
 
 redact(Term) ->
     emqx_utils:redact(Term).
-
-map_to_json(M0) ->
-    %% When dealing with Hocon validation errors, `value' might contain non-serializable
-    %% values (e.g.: user_lookup_fun), so we try again without that key if serialization
-    %% fails as a best effort.
-    M1 = emqx_utils_maps:jsonable_map(M0, fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end),
-    try
-        emqx_utils_json:encode(M1)
-    catch
-        error:_ ->
-            M2 = maps:without([value, <<"value">>], M1),
-            emqx_utils_json:encode(M2)
-    end.

apps/emqx_connector/src/emqx_connector_ssl.erl

@@ -31,7 +31,7 @@ convert_certs(_RltvDir, Config) ->
     {ok, Config}.
 
 new_ssl_config(RltvDir, Config, SSL) ->
-    case emqx_tls_lib:ensure_ssl_files(RltvDir, SSL) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(RltvDir, SSL) of
         {ok, NewSSL} ->
             {ok, new_ssl_config(Config, NewSSL)};
         {error, Reason} ->

apps/emqx_dashboard/src/emqx_dashboard_listener.erl

@@ -189,7 +189,7 @@ ensure_ssl_cert(#{<<"listeners">> := #{<<"https">> := #{<<"bind">> := Bind} = Ht
     Conf1 = emqx_utils_maps:deep_put([<<"listeners">>, <<"https">>], Conf0, Https1),
     Ssl = maps:get(<<"ssl_options">>, Https1, undefined),
     Opts = #{required_keys => [[<<"keyfile">>], [<<"certfile">>], [<<"cacertfile">>]]},
-    case emqx_tls_lib:ensure_ssl_files(?DIR, Ssl, Opts) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(?DIR, Ssl, Opts) of
         {ok, undefined} ->
             {error, <<"ssl_cert_not_found">>};
         {ok, NewSsl} ->

apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl

@@ -163,7 +163,7 @@ ensure_user_exists(Username) ->
 
 convert_certs(Dir, Conf) ->
     case
-        emqx_tls_lib:ensure_ssl_files(
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
             Dir, maps:get(<<"ssl">>, Conf, undefined)
         )
     of

apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl

@@ -162,7 +162,7 @@ convert_certs(
         Conf
 ) ->
     case
-        emqx_tls_lib:ensure_ssl_files(
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
             Dir, #{enable => true, certfile => Cert, keyfile => Key}, #{}
         )
     of

apps/emqx_ds_backends/src/emqx_ds_backends.app.src.script

@@ -20,7 +20,7 @@ Backends = case Profile of
                [emqx_ds_builtin_local, emqx_ds_builtin_raft, emqx_fdb_ds]
            end,
 
-io:format(user, "DS backends available for this release (~p): ~p~n", [Profile, Backends]),
+io:format(user, "DS backends available for this release (~p): ~0p~n", [Profile, Backends]),
 
 {application, emqx_ds_backends, [
     {description, "A placeholder application that depends on all available DS backends"},

apps/emqx_ds_backends/test/emqx_ds_backends_SUITE.erl

@@ -689,18 +689,9 @@ all() ->
 
 groups() ->
     TCs = emqx_common_test_helpers:all(?MODULE),
-    %% TODO: Remove once builtin-local supports preconditions + atomic batches.
-    BuiltinLocalTCs =
-        TCs --
-            [
-                t_09_atomic_store_batch,
-                t_11_batch_preconditions,
-                t_12_batch_precondition_conflicts
-            ],
-    BuiltinRaftTCs = TCs,
     [
-        {builtin_local, BuiltinLocalTCs},
-        {builtin_raft, BuiltinRaftTCs}
+        {builtin_local, TCs},
+        {builtin_raft, TCs}
     ].
 
 init_per_group(builtin_local, Config) ->

apps/emqx_ds_builtin_local/src/emqx_ds_builtin_local.erl

@@ -49,7 +49,9 @@
 %% Internal exports:
 -export([
     do_next/3,
-    do_delete_next/4
+    do_delete_next/4,
+    %% Used by batch serializer
+    make_batch/3
 ]).
 
 -export_type([db_opts/0, shard/0, iterator/0, delete_iterator/0]).
@@ -88,7 +90,10 @@
     #{
         backend := builtin_local,
         storage := emqx_ds_storage_layer:prototype(),
-        n_shards := pos_integer()
+        n_shards := pos_integer(),
+        %% Inherited from `emqx_ds:generic_db_opts()`.
+        force_monotonic_timestamps => boolean(),
+        atomic_batches => boolean()
     }.
 
 -type generation_rank() :: {shard(), emqx_ds_storage_layer:gen_id()}.
@@ -193,9 +198,17 @@ drop_db(DB) ->
     ),
     emqx_ds_builtin_local_meta:drop_db(DB).
 
--spec store_batch(emqx_ds:db(), [emqx_types:message()], emqx_ds:message_store_opts()) ->
+-spec store_batch(emqx_ds:db(), emqx_ds:batch(), emqx_ds:message_store_opts()) ->
     emqx_ds:store_batch_result().
-store_batch(DB, Messages, Opts) ->
+store_batch(DB, Batch, Opts) ->
+    case emqx_ds_builtin_local_meta:db_config(DB) of
+        #{atomic_batches := true} ->
+            store_batch_atomic(DB, Batch, Opts);
+        _ ->
+            store_batch_buffered(DB, Batch, Opts)
+    end.
+
+store_batch_buffered(DB, Messages, Opts) ->
     try
         emqx_ds_buffer:store_batch(DB, Messages, Opts)
     catch
@@ -203,6 +216,34 @@ store_batch(DB, Messages, Opts) ->
             {error, recoverable, Reason}
     end.
 
+store_batch_atomic(DB, Batch, Opts) ->
+    Shards = shards_of_batch(DB, Batch),
+    case Shards of
+        [Shard] ->
+            emqx_ds_builtin_local_batch_serializer:store_batch_atomic(DB, Shard, Batch, Opts);
+        [] ->
+            ok;
+        [_ | _] ->
+            {error, unrecoverable, atomic_batch_spans_multiple_shards}
+    end.
+
+shards_of_batch(DB, #dsbatch{operations = Operations, preconditions = Preconditions}) ->
+    shards_of_batch(DB, Preconditions, shards_of_batch(DB, Operations, []));
+shards_of_batch(DB, Operations) ->
+    shards_of_batch(DB, Operations, []).
+
+shards_of_batch(DB, [Operation | Rest], Acc) ->
+    case shard_of_operation(DB, Operation, clientid, #{}) of
+        Shard when Shard =:= hd(Acc) ->
+            shards_of_batch(DB, Rest, Acc);
+        Shard when Acc =:= [] ->
+            shards_of_batch(DB, Rest, [Shard]);
+        ShardAnother ->
+            [ShardAnother | Acc]
+    end;
+shards_of_batch(_DB, [], Acc) ->
+    Acc.
+
 -record(bs, {options :: emqx_ds:create_db_opts()}).
 -type buffer_state() :: #bs{}.
 

apps/emqx_ds_builtin_local/src/emqx_ds_builtin_local_batch_serializer.erl

@@ -0,0 +1,122 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_ds_builtin_local_batch_serializer).
+
+-include_lib("emqx_durable_storage/include/emqx_ds.hrl").
+
+%% API
+-export([
+    start_link/3,
+
+    store_batch_atomic/4
+]).
+
+%% `gen_server' API
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2
+]).
+
+%%------------------------------------------------------------------------------
+%% Type declarations
+%%------------------------------------------------------------------------------
+
+-define(name(DB, SHARD), {n, l, {?MODULE, DB, SHARD}}).
+-define(via(DB, SHARD), {via, gproc, ?name(DB, SHARD)}).
+
+-record(store_batch_atomic, {batch :: emqx_ds:batch(), opts :: emqx_ds:message_store_opts()}).
+
+%%------------------------------------------------------------------------------
+%% API
+%%------------------------------------------------------------------------------
+
+start_link(DB, Shard, _Opts) ->
+    gen_server:start_link(?via(DB, Shard), ?MODULE, [DB, Shard], []).
+
+store_batch_atomic(DB, Shard, Batch, Opts) ->
+    gen_server:call(?via(DB, Shard), #store_batch_atomic{batch = Batch, opts = Opts}, infinity).
+
+%%------------------------------------------------------------------------------
+%% `gen_server' API
+%%------------------------------------------------------------------------------
+
+init([DB, Shard]) ->
+    process_flag(message_queue_data, off_heap),
+    State = #{
+        db => DB,
+        shard => Shard
+    },
+    {ok, State}.
+
+handle_call(#store_batch_atomic{batch = Batch, opts = StoreOpts}, _From, State) ->
+    ShardId = shard_id(State),
+    DBOpts = db_config(State),
+    Result = do_store_batch_atomic(ShardId, Batch, DBOpts, StoreOpts),
+    {reply, Result, State};
+handle_call(Call, _From, State) ->
+    {reply, {error, {unknown_call, Call}}, State}.
+
+handle_cast(_Cast, State) ->
+    {noreply, State}.
+
+%%------------------------------------------------------------------------------
+%% Internal fns
+%%------------------------------------------------------------------------------
+
+shard_id(#{db := DB, shard := Shard}) ->
+    {DB, Shard}.
+
+db_config(#{db := DB}) ->
+    emqx_ds_builtin_local_meta:db_config(DB).
+
+-spec do_store_batch_atomic(
+    emqx_ds_storage_layer:shard_id(),
+    emqx_ds:dsbatch(),
+    emqx_ds_builtin_local:db_opts(),
+    emqx_ds:message_store_opts()
+) ->
+    emqx_ds:store_batch_result().
+do_store_batch_atomic(ShardId, #dsbatch{} = Batch, DBOpts, StoreOpts) ->
+    #dsbatch{
+        operations = Operations0,
+        preconditions = Preconditions
+    } = Batch,
+    case emqx_ds_precondition:verify(emqx_ds_storage_layer, ShardId, Preconditions) of
+        ok ->
+            do_store_operations(ShardId, Operations0, DBOpts, StoreOpts);
+        {precondition_failed, _} = PreconditionFailed ->
+            {error, unrecoverable, PreconditionFailed};
+        Error ->
+            Error
+    end;
+do_store_batch_atomic(ShardId, Operations, DBOpts, StoreOpts) ->
+    do_store_operations(ShardId, Operations, DBOpts, StoreOpts).
+
+do_store_operations(ShardId, Operations0, DBOpts, _StoreOpts) ->
+    ForceMonotonic = maps:get(force_monotonic_timestamps, DBOpts),
+    {Latest, Operations} =
+        emqx_ds_builtin_local:make_batch(
+            ForceMonotonic,
+            current_timestamp(ShardId),
+            Operations0
+        ),
+    Result = emqx_ds_storage_layer:store_batch(ShardId, Operations, _Options = #{}),
+    emqx_ds_builtin_local_meta:set_current_timestamp(ShardId, Latest),
+    Result.
+
+current_timestamp(ShardId) ->
+    emqx_ds_builtin_local_meta:current_timestamp(ShardId).

apps/emqx_ds_builtin_local/src/emqx_ds_builtin_local_db_sup.erl

@@ -158,7 +158,8 @@ init({#?shard_sup{db = DB, shard = Shard}, _}) ->
     Opts = emqx_ds_builtin_local_meta:db_config(DB),
     Children = [
         shard_storage_spec(DB, Shard, Opts),
-        shard_buffer_spec(DB, Shard, Opts)
+        shard_buffer_spec(DB, Shard, Opts),
+        shard_batch_serializer_spec(DB, Shard, Opts)
     ],
     {ok, {SupFlags, Children}}.
 
@@ -208,6 +209,15 @@ shard_buffer_spec(DB, Shard, Options) ->
         type => worker
     }.
 
+shard_batch_serializer_spec(DB, Shard, Opts) ->
+    #{
+        id => {Shard, batch_serializer},
+        start => {emqx_ds_builtin_local_batch_serializer, start_link, [DB, Shard, Opts]},
+        shutdown => 5_000,
+        restart => permanent,
+        type => worker
+    }.
+
 ensure_started(Res) ->
     case Res of
         {ok, _Pid} ->

apps/emqx_ds_builtin_raft/src/emqx_ds_replication_layer.erl

@@ -479,10 +479,10 @@ shards_of_batch(_DB, [], Acc) ->
 %% TODO
 %% There's a possibility of race condition: storage may shut down right after we
 %% ask for its status.
--define(IF_STORAGE_RUNNING(SHARDID, EXPR),
-    case emqx_ds_storage_layer:shard_info(SHARDID, status) of
-        running -> EXPR;
-        down -> {error, recoverable, storage_down}
+-define(IF_SHARD_READY(DB, SHARD, EXPR),
+    case emqx_ds_replication_layer_shard:shard_info(DB, SHARD, ready) of
+        true -> EXPR;
+        false -> {error, recoverable, shard_unavailable}
     end
 ).
 
@@ -525,8 +525,9 @@ do_get_streams_v1(_DB, _Shard, _TopicFilter, _StartTime) ->
     [{integer(), emqx_ds_storage_layer:stream()}] | emqx_ds:error(storage_down).
 do_get_streams_v2(DB, Shard, TopicFilter, StartTime) ->
     ShardId = {DB, Shard},
-    ?IF_STORAGE_RUNNING(
-        ShardId,
+    ?IF_SHARD_READY(
+        DB,
+        Shard,
         emqx_ds_storage_layer:get_streams(ShardId, TopicFilter, StartTime)
     ).
 
@@ -552,8 +553,9 @@ do_make_iterator_v1(_DB, _Shard, _Stream, _TopicFilter, _StartTime) ->
     emqx_ds:make_iterator_result(emqx_ds_storage_layer:iterator()).
 do_make_iterator_v2(DB, Shard, Stream, TopicFilter, StartTime) ->
     ShardId = {DB, Shard},
-    ?IF_STORAGE_RUNNING(
-        ShardId,
+    ?IF_SHARD_READY(
+        DB,
+        Shard,
         emqx_ds_storage_layer:make_iterator(ShardId, Stream, TopicFilter, StartTime)
     ).
 
@@ -587,8 +589,9 @@ do_update_iterator_v2(DB, Shard, OldIter, DSKey) ->
     emqx_ds:next_result(emqx_ds_storage_layer:iterator()).
 do_next_v1(DB, Shard, Iter, BatchSize) ->
     ShardId = {DB, Shard},
-    ?IF_STORAGE_RUNNING(
-        ShardId,
+    ?IF_SHARD_READY(
+        DB,
+        Shard,
         emqx_ds_storage_layer:next(
             ShardId, Iter, BatchSize, emqx_ds_replication_layer:current_timestamp(DB, Shard)
         )
@@ -620,8 +623,9 @@ do_add_generation_v2(_DB) ->
     | emqx_ds:error(storage_down).
 do_list_generations_with_lifetimes_v3(DB, Shard) ->
     ShardId = {DB, Shard},
-    ?IF_STORAGE_RUNNING(
-        ShardId,
+    ?IF_SHARD_READY(
+        DB,
+        Shard,
         emqx_ds_storage_layer:list_generations_with_lifetimes(ShardId)
     ).
 

apps/emqx_ds_builtin_raft/src/emqx_ds_replication_layer_shard.erl

@@ -18,7 +18,8 @@
 
 %% Dynamic server location API
 -export([
-    servers/3
+    servers/3,
+    shard_info/3
 ]).
 
 %% Safe Process Command API
@@ -38,8 +39,10 @@
 -behaviour(gen_server).
 -export([
     init/1,
+    handle_continue/2,
     handle_call/3,
     handle_cast/2,
+    handle_info/2,
     terminate/2
 ]).
 
@@ -52,6 +55,9 @@
     | {error, servers_unreachable}.
 
 -define(MEMBERSHIP_CHANGE_TIMEOUT, 30_000).
+-define(MAX_BOOSTRAP_RETRY_TIMEOUT, 1_000).
+
+-define(PTERM(DB, SHARD, KEY), {?MODULE, DB, SHARD, KEY}).
 
 %%
 
@@ -160,6 +166,12 @@ local_site() ->
 
 %%
 
+-spec shard_info(emqx_ds:db(), emqx_ds_replication_layer:shard_id(), _Info) -> _Value.
+shard_info(DB, Shard, ready) ->
+    get_shard_info(DB, Shard, ready, false).
+
+%%
+
 -spec process_command([server()], _Command, timeout()) ->
     {ok, _Result, _Leader :: server()} | server_error().
 process_command(Servers, Command, Timeout) ->
@@ -324,10 +336,45 @@ ra_overview(Server) ->
 
 %%
 
+-record(st, {
+    db :: emqx_ds:db(),
+    shard :: emqx_ds_replication_layer:shard_id(),
+    server :: server(),
+    bootstrapped :: boolean(),
+    stage :: term()
+}).
+
 init({DB, Shard, Opts}) ->
     _ = process_flag(trap_exit, true),
-    ok = start_server(DB, Shard, Opts),
-    {ok, {DB, Shard}}.
+    case start_server(DB, Shard, Opts) of
+        {_New = true, Server} ->
+            NextStage = trigger_election;
+        {_New = false, Server} ->
+            NextStage = wait_leader
+    end,
+    St = #st{
+        db = DB,
+        shard = Shard,
+        server = Server,
+        bootstrapped = false,
+        stage = NextStage
+    },
+    {ok, St, {continue, bootstrap}}.
+
+handle_continue(bootstrap, St = #st{bootstrapped = true}) ->
+    {noreply, St};
+handle_continue(bootstrap, St0 = #st{db = DB, shard = Shard, stage = Stage}) ->
+    ?tp(emqx_ds_replshard_bootstrapping, #{db => DB, shard => Shard, stage => Stage}),
+    case bootstrap(St0) of
+        St = #st{bootstrapped = true} ->
+            ?tp(emqx_ds_replshard_bootstrapped, #{db => DB, shard => Shard}),
+            {noreply, St};
+        St = #st{bootstrapped = false} ->
+            {noreply, St, {continue, bootstrap}};
+        {retry, Timeout, St} ->
+            _TRef = erlang:start_timer(Timeout, self(), bootstrap),
+            {noreply, St}
+    end.
 
 handle_call(_Call, _From, State) ->
     {reply, ignored, State}.
@@ -335,7 +382,14 @@ handle_call(_Call, _From, State) ->
 handle_cast(_Msg, State) ->
     {noreply, State}.
 
-terminate(_Reason, {DB, Shard}) ->
+handle_info({timeout, _TRef, bootstrap}, St) ->
+    {noreply, St, {continue, bootstrap}};
+handle_info(_Info, State) ->
+    {noreply, State}.
+
+terminate(_Reason, #st{db = DB, shard = Shard}) ->
+    %% NOTE: Mark as not ready right away.
+    ok = erase_shard_info(DB, Shard),
     %% NOTE: Timeouts are ignored, it's a best effort attempt.
     catch prep_stop_server(DB, Shard),
     LocalServer = get_local_server(DB, Shard),
@@ -343,6 +397,40 @@ terminate(_Reason, {DB, Shard}) ->
 
 %%
 
+bootstrap(St = #st{stage = trigger_election, server = Server}) ->
+    ok = trigger_election(Server),
+    St#st{stage = wait_leader};
+bootstrap(St = #st{stage = wait_leader, server = Server}) ->
+    case current_leader(Server) of
+        Leader = {_, _} ->
+            St#st{stage = {wait_log, Leader}};
+        unknown ->
+            St
+    end;
+bootstrap(St = #st{stage = {wait_log, Leader}}) ->
+    case ra_overview(Leader) of
+        #{commit_index := RaftIdx} ->
+            St#st{stage = {wait_log_index, RaftIdx}};
+        #{} ->
+            St#st{stage = wait_leader}
+    end;
+bootstrap(St = #st{stage = {wait_log_index, RaftIdx}, db = DB, shard = Shard, server = Server}) ->
+    Overview = ra_overview(Server),
+    case maps:get(last_applied, Overview, 0) of
+        LastApplied when LastApplied >= RaftIdx ->
+            ok = announce_shard_ready(DB, Shard),
+            St#st{bootstrapped = true, stage = undefined};
+        LastApplied ->
+            %% NOTE
+            %% Blunt estimate of time shard needs to catch up. If this proves to be too long in
+            %% practice, it's could be augmented with handling `recover` -> `follower` Ra
+            %% member state transition.
+            Timeout = min(RaftIdx - LastApplied, ?MAX_BOOSTRAP_RETRY_TIMEOUT),
+            {retry, Timeout, St}
+    end.
+
+%%
+
 start_server(DB, Shard, #{replication_options := ReplicationOpts}) ->
     ClusterName = cluster_name(DB, Shard),
     LocalServer = local_server(DB, Shard),
@@ -350,7 +438,6 @@ start_server(DB, Shard, #{replication_options := ReplicationOpts}) ->
     MutableConfig = #{tick_timeout => 100},
     case ra:restart_server(DB, LocalServer, MutableConfig) of
         {error, name_not_registered} ->
-            Bootstrap = true,
             Machine = {module, emqx_ds_replication_layer, #{db => DB, shard => Shard}},
             LogOpts = maps:with(
                 [
@@ -366,30 +453,34 @@ start_server(DB, Shard, #{replication_options := ReplicationOpts}) ->
                 initial_members => Servers,
                 machine => Machine,
                 log_init_args => LogOpts
-            });
+            }),
+            {_NewServer = true, LocalServer};
         ok ->
-            Bootstrap = false;
+            {_NewServer = false, LocalServer};
         {error, {already_started, _}} ->
-            Bootstrap = false
-    end,
+            {_NewServer = false, LocalServer}
+    end.
+
+trigger_election(Server) ->
     %% NOTE
     %% Triggering election is necessary when a new consensus group is being brought up.
     %% TODO
     %% It's probably a good idea to rebalance leaders across the cluster from time to
     %% time. There's `ra:transfer_leadership/2` for that.
-    try Bootstrap andalso ra:trigger_election(LocalServer, _Timeout = 1_000) of
-        false ->
-            ok;
-        ok ->
-            ok
+    try ra:trigger_election(Server) of
+        ok -> ok
     catch
-        %% TODO
+        %% NOTE
         %% Tolerating exceptions because server might be occupied with log replay for
         %% a while.
-        exit:{timeout, _} when not Bootstrap ->
+        exit:{timeout, _} ->
+            ?tp(emqx_ds_replshard_trigger_election, #{server => Server, error => timeout}),
             ok
     end.
 
+announce_shard_ready(DB, Shard) ->
+    set_shard_info(DB, Shard, ready, true).
+
 server_uid(_DB, Shard) ->
     %% NOTE
     %% Each new "instance" of a server should have a unique identifier. Otherwise,
@@ -402,6 +493,22 @@ server_uid(_DB, Shard) ->
 
 %%
 
+get_shard_info(DB, Shard, K, Default) ->
+    persistent_term:get(?PTERM(DB, Shard, K), Default).
+
+set_shard_info(DB, Shard, K, V) ->
+    persistent_term:put(?PTERM(DB, Shard, K), V).
+
+erase_shard_info(DB, Shard) ->
+    lists:foreach(fun(K) -> erase_shard_info(DB, Shard, K) end, [
+        ready
+    ]).
+
+erase_shard_info(DB, Shard, K) ->
+    persistent_term:erase(?PTERM(DB, Shard, K)).
+
+%%
+
 prep_stop_server(DB, Shard) ->
     prep_stop_server(DB, Shard, 5_000).
 

apps/emqx_ds_builtin_raft/test/emqx_ds_replication_SUITE.erl

@@ -131,7 +131,6 @@ t_replication_transfers_snapshots(Config) ->
             %% Initialize DB on all nodes and wait for it to be online.
             Opts = opts(Config, #{n_shards => 1, n_sites => 3}),
             assert_db_open(Nodes, ?DB, Opts),
-            assert_db_stable(Nodes, ?DB),
 
             %% Stop the DB on the "offline" node.
             ?wait_async_action(
@@ -207,7 +206,6 @@ t_rebalance(Config) ->
             %% 1. Initialize DB on the first node.
             Opts = opts(Config, #{n_shards => 16, n_sites => 1, replication_factor => 3}),
             assert_db_open(Nodes, ?DB, Opts),
-            assert_db_stable(Nodes, ?DB),
 
             %% 1.1 Kick all sites except S1 from the replica set as
             %% the initial condition:
@@ -419,7 +417,6 @@ t_rebalance_chaotic_converges(Config) ->
 
             %% Open DB:
             assert_db_open(Nodes, ?DB, Opts),
-            assert_db_stable(Nodes, ?DB),
 
             %% Kick N3 from the replica set as the initial condition:
             ?assertMatch(
@@ -503,7 +500,6 @@ t_rebalance_offline_restarts(Config) ->
     %% Initialize DB on all 3 nodes.
     Opts = opts(Config, #{n_shards => 8, n_sites => 3, replication_factor => 3}),
     assert_db_open(Nodes, ?DB, Opts),
-    assert_db_stable(Nodes, ?DB),
 
     ?retry(
         1000,
@@ -845,13 +841,11 @@ t_crash_restart_recover(Config) ->
     ?check_trace(
         begin
             %% Initialize DB on all nodes.
-            ?assertEqual(
-                [{ok, ok} || _ <- Nodes],
-                erpc:multicall(Nodes, emqx_ds, open_db, [?DB, DBOpts])
-            ),
+            assert_db_open(Nodes, ?DB, DBOpts),
 
             %% Apply the test events, including simulated node crashes.
             NodeStream = emqx_utils_stream:const(N1),
+            StartedAt = erlang:monotonic_time(millisecond),
             emqx_ds_test_helpers:apply_stream(?DB, NodeStream, Stream, 0),
 
             %% It's expected to lose few messages when leaders are abruptly killed.
@@ -865,6 +859,10 @@ t_crash_restart_recover(Config) ->
             ct:pal("Some messages were lost: ~p", [LostMessages]),
             ?assert(length(LostMessages) < NMsgs div 20),
 
+            %% Wait until crashed nodes are ready.
+            SinceStarted = erlang:monotonic_time(millisecond) - StartedAt,
+            wait_db_bootstrapped([N2, N3], ?DB, infinity, SinceStarted),
+
             %% Verify that all the successfully persisted messages are there.
             VerifyClient = fun({ClientId, ExpectedStream}) ->
                 Topic = emqx_ds_test_helpers:client_topic(?FUNCTION_NAME, ClientId),
@@ -926,7 +924,8 @@ assert_db_open(Nodes, DB, Opts) ->
     ?assertEqual(
         [{ok, ok} || _ <- Nodes],
         erpc:multicall(Nodes, emqx_ds, open_db, [DB, Opts])
-    ).
+    ),
+    wait_db_bootstrapped(Nodes, ?DB).
 
 assert_db_stable([Node | _], DB) ->
     Shards = ds_repl_meta(Node, shards, [DB]),
@@ -935,6 +934,32 @@ assert_db_stable([Node | _], DB) ->
         db_leadership(Node, DB, Shards)
     ).
 
+wait_db_bootstrapped(Nodes, DB) ->
+    wait_db_bootstrapped(Nodes, DB, infinity, infinity).
+
+wait_db_bootstrapped(Nodes, DB, Timeout, BackInTime) ->
+    SRefs = [
+        snabbkaffe:subscribe(
+            ?match_event(#{
+                ?snk_kind := emqx_ds_replshard_bootstrapped,
+                ?snk_meta := #{node := Node},
+                db := DB,
+                shard := Shard
+            }),
+            1,
+            Timeout,
+            BackInTime
+        )
+     || Node <- Nodes,
+        Shard <- ds_repl_meta(Node, my_shards, [DB])
+    ],
+    lists:foreach(
+        fun({ok, SRef}) ->
+            ?assertMatch({ok, [_]}, snabbkaffe:receive_events(SRef))
+        end,
+        SRefs
+    ).
+
 %%
 
 db_leadership(Node, DB, Shards) ->

apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_elector.erl

@@ -0,0 +1,124 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+%% @doc Shared subscription group elector process.
+%% Hosted under the _shared subscription registry_ supervisor.
+%% Responsible for starting the leader election process that eventually
+%% finishes with 2 outcomes:
+%% 1. The elector wins the leadership.
+%%    In this case the elector _becomes_ the leader, by entering the
+%%    `emqx_ds_shared_sub_leader` process loop.
+%% 2. The elector finds the active leader.
+%%    In this case the elector idles while the leader is considered active
+%%    and redirects any connect requests to the active leader.
+-module(emqx_ds_shared_sub_elector).
+
+-include("emqx_ds_shared_sub_proto.hrl").
+
+-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/trace.hrl").
+
+%% Internal API
+-export([
+    start_link/2
+]).
+
+-behaviour(gen_server).
+-export([
+    init/1,
+    handle_continue/2,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+%%--------------------------------------------------------------------
+%% Internal API
+%%--------------------------------------------------------------------
+
+start_link(ShareTopic, StartTime) ->
+    gen_server:start_link(?MODULE, {elect, ShareTopic, StartTime}, []).
+
+%%--------------------------------------------------------------------
+%% gen_server callbacks
+%%--------------------------------------------------------------------
+
+-record(follower, {
+    topic :: emqx_persistent_session_ds:share_topic_filter(),
+    leader :: pid(),
+    alive_until :: non_neg_integer()
+}).
+
+init(Elect = {elect, _ShareTopic, _StartTime}) ->
+    %% NOTE
+    %% Important to have it here, because this process can become
+    %% `emqx_ds_shared_sub_leader`, which has `terminate/2` logic.
+    _ = erlang:process_flag(trap_exit, true),
+    {ok, #{}, {continue, Elect}}.
+
+handle_continue({elect, ShareTopic, StartTime}, _State) ->
+    elect(ShareTopic, StartTime).
+
+handle_call(_Request, _From, State) ->
+    {reply, {error, unknown_request}, State}.
+
+handle_cast(_Cast, State) ->
+    {noreply, State}.
+
+handle_info(?agent_connect_leader_match(Agent, AgentMetadata, _ShareTopic), State) ->
+    %% NOTE: Redirecting to the known leader.
+    ok = connect_leader(Agent, AgentMetadata, State),
+    {noreply, State};
+handle_info({timeout, _TRef, invalidate}, State) ->
+    {stop, {shutdown, invalidate}, State}.
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%%--------------------------------------------------------------------
+
+elect(ShareTopic, TS) ->
+    Group = emqx_ds_shared_sub_leader:group_name(ShareTopic),
+    case emqx_ds_shared_sub_leader_store:claim_leadership(Group, _Leader = self(), TS) of
+        {ok, LeaderClaim} ->
+            %% Become the leader.
+            ?tp(debug, shared_sub_elector_becomes_leader, #{
+                id => ShareTopic,
+                group => Group,
+                leader => LeaderClaim
+            }),
+            emqx_ds_shared_sub_leader:become(ShareTopic, TS, LeaderClaim);
+        {exists, LeaderClaim} ->
+            %% Turn into the follower that redirects connect requests to the leader
+            %% while it's considered alive. Note that the leader may in theory decide
+            %% to let go of leadership earlier than that.
+            AliveUntil = emqx_ds_shared_sub_leader_store:alive_until(LeaderClaim),
+            ?tp(debug, shared_sub_elector_becomes_follower, #{
+                id => ShareTopic,
+                group => Group,
+                leader => LeaderClaim,
+                until => AliveUntil
+            }),
+            TTL = AliveUntil - TS,
+            _TRef = erlang:start_timer(max(0, TTL), self(), invalidate),
+            St = #follower{
+                topic = ShareTopic,
+                leader = emqx_ds_shared_sub_leader_store:leader_id(LeaderClaim),
+                alive_until = AliveUntil
+            },
+            {noreply, St};
+        {error, Class, Reason} = Error ->
+            ?tp(warning, "Shared subscription leader election failed", #{
+                id => ShareTopic,
+                group => Group,
+                error => Error
+            }),
+            case Class of
+                recoverable -> StopReason = {shutdown, Reason};
+                unrecoverable -> StopReason = Error
+            end,
+            {stop, StopReason, ShareTopic}
+    end.
+
+connect_leader(Agent, AgentMetadata, #follower{topic = ShareTopic, leader = Pid}) ->
+    emqx_ds_shared_sub_proto:agent_connect_leader(Pid, Agent, AgentMetadata, ShareTopic).

apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_group_sm.erl

@@ -180,7 +180,7 @@ handle_connecting(#{agent := Agent, share_topic_filter := ShareTopicFilter} = GS
         agent => Agent,
         share_topic_filter => ShareTopicFilter
     }),
-    ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM), ShareTopicFilter),
+    ok = emqx_ds_shared_sub_registry:leader_wanted(Agent, agent_metadata(GSM), ShareTopicFilter),
     ensure_state_timeout(GSM, find_leader_timeout, ?dq_config(session_find_leader_timeout_ms)).
 
 handle_leader_lease_streams(
@@ -211,7 +211,7 @@ handle_find_leader_timeout(#{agent := Agent, share_topic_filter := ShareTopicFil
         agent => Agent,
         share_topic_filter => ShareTopicFilter
     }),
-    ok = emqx_ds_shared_sub_registry:lookup_leader(Agent, agent_metadata(GSM0), ShareTopicFilter),
+    ok = emqx_ds_shared_sub_registry:leader_wanted(Agent, agent_metadata(GSM0), ShareTopicFilter),
     GSM1 = ensure_state_timeout(
         GSM0, find_leader_timeout, ?dq_config(session_find_leader_timeout_ms)
     ),

apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader.erl

@@ -4,8 +4,6 @@
 
 -module(emqx_ds_shared_sub_leader).
 
--behaviour(gen_statem).
-
 -include("emqx_ds_shared_sub_proto.hrl").
 -include("emqx_ds_shared_sub_config.hrl").
 
@@ -15,12 +13,16 @@
 -include_lib("snabbkaffe/include/trace.hrl").
 
 -export([
-    register/2,
-
     start_link/1,
-    child_spec/1,
-    id/1,
+    become/3
+]).
 
+-export([
+    group_name/1
+]).
+
+-behaviour(gen_statem).
+-export([
     callback_mode/0,
     init/1,
     handle_event/4,
@@ -53,14 +55,6 @@
     revoked_streams := list(emqx_ds:stream())
 }.
 
--type progress() :: emqx_persistent_session_ds_shared_subs:progress().
-
--type stream_state() :: #{
-    progress => progress(),
-    rank => emqx_ds:stream_rank()
-}.
-
-%% TODO https://emqx.atlassian.net/browse/EMQX-12307
 %% Some data should be persisted
 -type data() :: #{
     %%
@@ -68,12 +62,8 @@
     %%
     group_id := group_id(),
     topic := emqx_types:topic(),
-    %% TODO https://emqx.atlassian.net/browse/EMQX-12575
     %% Implement some stats to assign evenly?
-    stream_states := #{
-        emqx_ds:stream() => stream_state()
-    },
-    rank_progress := emqx_ds_shared_sub_leader_rank_progress:t(),
+    store := emqx_ds_shared_sub_leader_store:t(),
 
     %%
     %% Ephemeral data, should not be persisted
@@ -88,49 +78,44 @@
 
 -export_type([
     options/0,
-    data/0,
-    progress/0
+    data/0
 ]).
 
 %% States
 
--define(leader_waiting_registration, leader_waiting_registration).
 -define(leader_active, leader_active).
 
 %% Events
 
--record(register, {
-    register_fun :: fun(() -> pid())
-}).
 -record(renew_streams, {}).
 -record(renew_leases, {}).
 -record(drop_timeout, {}).
+-record(renew_leader_claim, {}).
 
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
 
-register(Pid, Fun) ->
-    gen_statem:call(Pid, #register{register_fun = Fun}).
-
-%%--------------------------------------------------------------------
-%% Internal API
-%%--------------------------------------------------------------------
-
-child_spec(#{share_topic_filter := ShareTopicFilter} = Options) ->
-    #{
-        id => id(ShareTopicFilter),
-        start => {?MODULE, start_link, [Options]},
-        restart => temporary,
-        shutdown => 5000,
-        type => worker
-    }.
-
 start_link(Options) ->
     gen_statem:start_link(?MODULE, [Options], []).
 
-id(ShareTopicFilter) ->
-    {?MODULE, ShareTopicFilter}.
+become(ShareTopicFilter, StartTime, Claim) ->
+    Data0 = init_data(ShareTopicFilter, StartTime),
+    Data1 = attach_claim(Claim, Data0),
+    case store_is_dirty(Data1) of
+        true ->
+            Actions = force_claim_renewal(Data1);
+        false ->
+            Actions = init_claim_renewal(Data1)
+    end,
+    gen_statem:enter_loop(?MODULE, [], ?leader_active, Data1, Actions).
+
+%%--------------------------------------------------------------------
+
+group_name(#share{group = ShareGroup, topic = Topic}) ->
+    %% NOTE: Should not contain `/`s.
+    %% TODO: More observable encoding.
+    iolist_to_binary([ShareGroup, $:, binary:encode_hex(Topic)]).
 
 %%--------------------------------------------------------------------
 %% gen_statem callbacks
@@ -138,29 +123,42 @@ id(ShareTopicFilter) ->
 
 callback_mode() -> [handle_event_function, state_enter].
 
-init([#{share_topic_filter := #share{topic = Topic} = ShareTopicFilter} = _Options]) ->
-    Data = #{
+init([#{share_topic_filter := ShareTopicFilter} = _Options]) ->
+    _ = erlang:process_flag(trap_exit, true),
+    Data = init_data(ShareTopicFilter, now_ms()),
+    {ok, ?leader_active, Data}.
+
+init_data(#share{topic = Topic} = ShareTopicFilter, StartTime) ->
+    Group = group_name(ShareTopicFilter),
+    case emqx_ds_shared_sub_leader_store:open(Group) of
+        Store when Store =/= false ->
+            ?tp(warning, shared_sub_leader_store_open, #{topic => ShareTopicFilter, store => Store}),
+            ok;
+        false ->
+            ?tp(warning, shared_sub_leader_store_init, #{topic => ShareTopicFilter}),
+            RankProgress = emqx_ds_shared_sub_leader_rank_progress:init(),
+            Store0 = emqx_ds_shared_sub_leader_store:init(Group),
+            Store1 = emqx_ds_shared_sub_leader_store:set(start_time, StartTime, Store0),
+            Store = emqx_ds_shared_sub_leader_store:set(rank_progress, RankProgress, Store1)
+    end,
+    #{
         group_id => ShareTopicFilter,
         topic => Topic,
-        start_time => now_ms(),
-        stream_states => #{},
+        store => Store,
         stream_owners => #{},
-        agents => #{},
-        rank_progress => emqx_ds_shared_sub_leader_rank_progress:init()
-    },
-    {ok, ?leader_waiting_registration, Data}.
+        agents => #{}
+    }.
+
+attach_claim(Claim, Data) ->
+    Data#{leader_claim => Claim}.
+
+force_claim_renewal(_Data = #{}) ->
+    [{{timeout, #renew_leader_claim{}}, 0, #renew_leader_claim{}}].
+
+init_claim_renewal(_Data = #{leader_claim := Claim}) ->
+    Interval = emqx_ds_shared_sub_leader_store:heartbeat_interval(Claim),
+    [{{timeout, #renew_leader_claim{}}, Interval, #renew_leader_claim{}}].
 
-%%--------------------------------------------------------------------
-%% waiting_registration state
-
-handle_event({call, From}, #register{register_fun = Fun}, ?leader_waiting_registration, Data) ->
-    Self = self(),
-    case Fun() of
-        Self ->
-            {next_state, ?leader_active, Data, {reply, From, {ok, Self}}};
-        OtherPid ->
-            {stop_and_reply, normal, {reply, From, {ok, OtherPid}}}
-    end;
 %%--------------------------------------------------------------------
 %% repalying state
 handle_event(enter, _OldState, ?leader_active, #{topic := Topic} = _Data) ->
@@ -193,6 +191,14 @@ handle_event({timeout, #drop_timeout{}}, #drop_timeout{}, ?leader_active, Data0)
     Data1 = drop_timeout_agents(Data0),
     {keep_state, Data1,
         {{timeout, #drop_timeout{}}, ?dq_config(leader_drop_timeout_interval_ms), #drop_timeout{}}};
+handle_event({timeout, #renew_leader_claim{}}, #renew_leader_claim{}, ?leader_active, Data0) ->
+    case renew_leader_claim(Data0) of
+        Data1 = #{} ->
+            Actions = init_claim_renewal(Data1),
+            {keep_state, Data1, Actions};
+        Error ->
+            {stop, Error}
+    end;
 %%--------------------------------------------------------------------
 %% agent events
 handle_event(
@@ -243,13 +249,57 @@ handle_event(Event, Content, State, _Data) ->
     }),
     keep_state_and_data.
 
-terminate(_Reason, _State, _Data) ->
-    ok.
+terminate(
+    _Reason,
+    _State,
+    #{group_id := ShareTopicFilter, leader_claim := Claim, store := Store}
+) ->
+    %% NOTE
+    %% Call to `commit_dirty/2` will currently block.
+    %% On the other hand, call to `disown_leadership/1` should be non-blocking.
+    Group = group_name(ShareTopicFilter),
+    Result = emqx_ds_shared_sub_leader_store:commit_dirty(Claim, Store),
+    ok = emqx_ds_shared_sub_leader_store:disown_leadership(Group, Claim),
+    ?tp(shared_sub_leader_store_committed_dirty, #{
+        id => ShareTopicFilter,
+        group => Group,
+        claim => Claim,
+        result => Result
+    }).
 
 %%--------------------------------------------------------------------
 %% Event handlers
 %%--------------------------------------------------------------------
 
+%%--------------------------------------------------------------------
+
+renew_leader_claim(Data = #{group_id := ShareTopicFilter, store := Store0, leader_claim := Claim}) ->
+    TS = emqx_message:timestamp_now(),
+    Group = group_name(ShareTopicFilter),
+    case emqx_ds_shared_sub_leader_store:commit_renew(Claim, TS, Store0) of
+        {ok, RenewedClaim, CommittedStore} ->
+            ?tp(shared_sub_leader_store_committed, #{
+                id => ShareTopicFilter,
+                group => Group,
+                claim => Claim,
+                renewed => RenewedClaim
+            }),
+            attach_claim(RenewedClaim, Data#{store := CommittedStore});
+        {error, Class, Reason} = Error ->
+            ?tp(warning, "Shared subscription leader store commit failed", #{
+                id => ShareTopicFilter,
+                group => Group,
+                claim => Claim,
+                reason => Reason
+            }),
+            case Class of
+                %% Will retry.
+                recoverable -> Data;
+                %% Will have to crash.
+                unrecoverable -> Error
+            end
+    end.
+
 %%--------------------------------------------------------------------
 %% Renew streams
 
@@ -257,60 +307,50 @@ terminate(_Reason, _State, _Data) ->
 %% * Revoke streams from agents having too many streams
 %% * Assign streams to agents having too few streams
 
-renew_streams(
-    #{
-        start_time := StartTime,
-        stream_states := StreamStates,
-        topic := Topic,
-        rank_progress := RankProgress0
-    } = Data0
-) ->
+renew_streams(#{topic := Topic} = Data0) ->
     TopicFilter = emqx_topic:words(Topic),
-    StreamsWRanks = emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime),
+    StartTime = store_get_start_time(Data0),
+    StreamsWRanks = get_streams(TopicFilter, StartTime),
 
     %% Discard streams that are already replayed and init new
-    {NewStreamsWRanks, RankProgress1} = emqx_ds_shared_sub_leader_rank_progress:add_streams(
-        StreamsWRanks, RankProgress0
-    ),
-    {NewStreamStates, VanishedStreamStates} = update_progresses(
-        StreamStates, NewStreamsWRanks, TopicFilter, StartTime
+    {NewStreamsWRanks, RankProgress} = emqx_ds_shared_sub_leader_rank_progress:add_streams(
+        StreamsWRanks,
+        store_get_rank_progress(Data0)
     ),
-    Data1 = removed_vanished_streams(Data0, VanishedStreamStates),
-    Data2 = Data1#{stream_states => NewStreamStates, rank_progress => RankProgress1},
-    Data3 = revoke_streams(Data2),
-    Data4 = assign_streams(Data3),
-    ?SLOG(debug, #{
+    {Data1, VanishedStreams} = update_progresses(Data0, NewStreamsWRanks, TopicFilter, StartTime),
+    Data2 = store_put_rank_progress(Data1, RankProgress),
+    Data3 = removed_vanished_streams(Data2, VanishedStreams),
+    Data4 = revoke_streams(Data3),
+    Data5 = assign_streams(Data4),
+    ?SLOG(info, #{
         msg => leader_renew_streams,
         topic_filter => TopicFilter,
         new_streams => length(NewStreamsWRanks)
     }),
-    Data4.
+    Data5.
 
-update_progresses(StreamStates, NewStreamsWRanks, TopicFilter, StartTime) ->
-    lists:foldl(
-        fun({Rank, Stream}, {NewStreamStatesAcc, OldStreamStatesAcc}) ->
-            case OldStreamStatesAcc of
-                #{Stream := StreamData} ->
-                    {
-                        NewStreamStatesAcc#{Stream => StreamData},
-                        maps:remove(Stream, OldStreamStatesAcc)
-                    };
-                _ ->
-                    {ok, It} = emqx_ds:make_iterator(
-                        ?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime
-                    ),
-                    Progress = #{
-                        iterator => It
-                    },
-                    {
-                        NewStreamStatesAcc#{Stream => #{progress => Progress, rank => Rank}},
-                        OldStreamStatesAcc
-                    }
+update_progresses(Data0, NewStreamsWRanks, TopicFilter, StartTime) ->
+    ExistingStreams = store_setof_streams(Data0),
+    Data = lists:foldl(
+        fun({Rank, Stream}, DataAcc) ->
+            case sets:is_element(Stream, ExistingStreams) of
+                true ->
+                    DataAcc;
+                false ->
+                    {ok, It} = make_iterator(Stream, TopicFilter, StartTime),
+                    StreamData = #{progress => #{iterator => It}, rank => Rank},
+                    store_put_stream(DataAcc, Stream, StreamData)
             end
         end,
-        {#{}, StreamStates},
+        Data0,
         NewStreamsWRanks
-    ).
+    ),
+    VanishedStreams = lists:foldl(
+        fun({_Rank, Stream}, Acc) -> sets:del_element(Stream, Acc) end,
+        ExistingStreams,
+        NewStreamsWRanks
+    ),
+    {Data, sets:to_list(VanishedStreams)}.
 
 %% We just remove disappeared streams from anywhere.
 %%
@@ -320,8 +360,7 @@ update_progresses(StreamStates, NewStreamsWRanks, TopicFilter, StartTime) ->
 %%
 %% If streams disappear after long leader sleep, it is a normal situation.
 %% This removal will be a part of initialization before any agents connect.
-removed_vanished_streams(Data0, VanishedStreamStates) ->
-    VanishedStreams = maps:keys(VanishedStreamStates),
+removed_vanished_streams(Data0, VanishedStreams) ->
     Data1 = lists:foldl(
         fun(Stream, #{stream_owners := StreamOwners0} = DataAcc) ->
             case StreamOwners0 of
@@ -601,47 +640,61 @@ update_agent_stream_states(Data0, Agent, AgentStreamProgresses, Version) ->
     end.
 
 update_stream_progresses(
-    #{stream_states := StreamStates0, stream_owners := StreamOwners} = Data0,
+    #{stream_owners := StreamOwners} = Data0,
     Agent,
     AgentState0,
     ReceivedStreamProgresses
 ) ->
-    {StreamStates1, ReplayedStreams} = lists:foldl(
-        fun(#{stream := Stream, progress := Progress}, {StreamStatesAcc, ReplayedStreamsAcc}) ->
+    ReplayedStreams = lists:foldl(
+        fun(#{stream := Stream, progress := Progress}, Acc) ->
             case StreamOwners of
                 #{Stream := Agent} ->
-                    StreamData0 = maps:get(Stream, StreamStatesAcc),
                     case Progress of
                         #{iterator := end_of_stream} ->
-                            Rank = maps:get(rank, StreamData0),
-                            {maps:remove(Stream, StreamStatesAcc), ReplayedStreamsAcc#{
-                                Stream => Rank
-                            }};
+                            #{rank := Rank} = store_get_stream(Data0, Stream),
+                            Acc#{Stream => Rank};
                         _ ->
-                            StreamData1 = StreamData0#{progress => Progress},
-                            {StreamStatesAcc#{Stream => StreamData1}, ReplayedStreamsAcc}
+                            Acc
                     end;
                 _ ->
-                    {StreamStatesAcc, ReplayedStreamsAcc}
+                    Acc
             end
         end,
-        {StreamStates0, #{}},
+        #{},
         ReceivedStreamProgresses
     ),
-    Data1 = update_rank_progress(Data0, ReplayedStreams),
-    Data2 = Data1#{stream_states => StreamStates1},
+    Data1 = lists:foldl(
+        fun(#{stream := Stream, progress := Progress}, DataAcc) ->
+            case StreamOwners of
+                #{Stream := Agent} ->
+                    StreamData0 = store_get_stream(DataAcc, Stream),
+                    case Progress of
+                        #{iterator := end_of_stream} ->
+                            store_delete_stream(DataAcc, Stream);
+                        _ ->
+                            StreamData = StreamData0#{progress => Progress},
+                            store_put_stream(DataAcc, Stream, StreamData)
+                    end;
+                _ ->
+                    DataAcc
+            end
+        end,
+        Data0,
+        ReceivedStreamProgresses
+    ),
+    Data2 = update_rank_progress(Data1, ReplayedStreams),
     AgentState1 = filter_replayed_streams(AgentState0, ReplayedStreams),
     {Data2, AgentState1}.
 
-update_rank_progress(#{rank_progress := RankProgress0} = Data, ReplayedStreams) ->
-    RankProgress1 = maps:fold(
+update_rank_progress(Data, ReplayedStreams) ->
+    RankProgress = maps:fold(
         fun(Stream, Rank, RankProgressAcc) ->
             emqx_ds_shared_sub_leader_rank_progress:set_replayed({Rank, Stream}, RankProgressAcc)
         end,
-        RankProgress0,
+        store_get_rank_progress(Data),
         ReplayedStreams
     ),
-    Data#{rank_progress => RankProgress1}.
+    store_put_rank_progress(Data, RankProgress).
 
 %% No need to revoke fully replayed streams. We do not assign them anymore.
 %% The agent's session also will drop replayed streams itself.
@@ -899,10 +952,9 @@ renew_no_replaying_deadline(#{} = AgentState) ->
             ?dq_config(leader_session_not_replaying_timeout_ms)
     }.
 
-unassigned_streams(#{stream_states := StreamStates, stream_owners := StreamOwners}) ->
-    Streams = maps:keys(StreamStates),
-    AssignedStreams = maps:keys(StreamOwners),
-    Streams -- AssignedStreams.
+unassigned_streams(#{stream_owners := StreamOwners} = Data) ->
+    Streams = store_setof_streams(Data),
+    sets:to_list(sets:subtract(Streams, StreamOwners)).
 
 %% Those who are not connecting or updating, i.e. not in a transient state.
 replaying_agents(#{agents := AgentStates}) ->
@@ -922,12 +974,12 @@ desired_stream_count_per_agent(#{agents := AgentStates} = Data) ->
 desired_stream_count_for_new_agent(#{agents := AgentStates} = Data) ->
     desired_stream_count_per_agent(Data, maps:size(AgentStates) + 1).
 
-desired_stream_count_per_agent(#{stream_states := StreamStates}, AgentCount) ->
+desired_stream_count_per_agent(Data, AgentCount) ->
     case AgentCount of
         0 ->
             0;
         _ ->
-            StreamCount = maps:size(StreamStates),
+            StreamCount = store_num_streams(Data),
             case StreamCount rem AgentCount of
                 0 ->
                     StreamCount div AgentCount;
@@ -936,10 +988,10 @@ desired_stream_count_per_agent(#{stream_states := StreamStates}, AgentCount) ->
             end
     end.
 
-stream_progresses(#{stream_states := StreamStates} = _Data, Streams) ->
+stream_progresses(Data, Streams) ->
     lists:map(
         fun(Stream) ->
-            StreamData = maps:get(Stream, StreamStates),
+            StreamData = store_get_stream(Data, Stream),
             #{
                 stream => Stream,
                 progress => maps:get(progress, StreamData)
@@ -1013,3 +1065,45 @@ with_agent(#{agents := Agents} = Data, Agent, Fun) ->
         _ ->
             Data
     end.
+
+%%
+
+get_streams(TopicFilter, StartTime) ->
+    emqx_ds:get_streams(?PERSISTENT_MESSAGE_DB, TopicFilter, StartTime).
+
+make_iterator(Stream, TopicFilter, StartTime) ->
+    emqx_ds:make_iterator(?PERSISTENT_MESSAGE_DB, Stream, TopicFilter, StartTime).
+
+%% Leader store
+
+store_is_dirty(#{store := Store}) ->
+    emqx_ds_shared_sub_leader_store:dirty(Store).
+
+store_get_stream(#{store := Store}, ID) ->
+    emqx_ds_shared_sub_leader_store:get(stream, ID, Store).
+
+store_put_stream(Data = #{store := Store0}, ID, StreamData) ->
+    Store = emqx_ds_shared_sub_leader_store:put(stream, ID, StreamData, Store0),
+    Data#{store := Store}.
+
+store_delete_stream(Data = #{store := Store0}, ID) ->
+    Store = emqx_ds_shared_sub_leader_store:delete(stream, ID, Store0),
+    Data#{store := Store}.
+
+store_get_rank_progress(#{store := Store}) ->
+    emqx_ds_shared_sub_leader_store:get(rank_progress, Store).
+
+store_put_rank_progress(Data = #{store := Store0}, RankProgress) ->
+    Store = emqx_ds_shared_sub_leader_store:set(rank_progress, RankProgress, Store0),
+    Data#{store := Store}.
+
+store_get_start_time(#{store := Store}) ->
+    emqx_ds_shared_sub_leader_store:get(start_time, Store).
+
+store_num_streams(#{store := Store}) ->
+    emqx_ds_shared_sub_leader_store:size(stream, Store).
+
+store_setof_streams(#{store := Store}) ->
+    Acc0 = sets:new([{version, 2}]),
+    FoldFun = fun(Stream, _StreamData, Acc) -> sets:add_element(Stream, Acc) end,
+    emqx_ds_shared_sub_leader_store:fold(stream, FoldFun, Acc0, Store).

apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_store.erl

@@ -0,0 +1,656 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_ds_shared_sub_leader_store).
+
+-include_lib("emqx_utils/include/emqx_message.hrl").
+-include_lib("emqx_durable_storage/include/emqx_ds.hrl").
+-include_lib("snabbkaffe/include/trace.hrl").
+
+-export([
+    open/0,
+    close/0
+]).
+
+%% Leadership API
+-export([
+    %% Leadership claims
+    claim_leadership/3,
+    renew_leadership/3,
+    disown_leadership/2,
+    %% Accessors
+    leader_id/1,
+    alive_until/1,
+    heartbeat_interval/1
+]).
+
+%% Store API
+-export([
+    %% Lifecycle
+    init/1,
+    open/1,
+    %% TODO
+    %% destroy/1,
+    %% Managing records
+    get/3,
+    get/4,
+    fold/4,
+    size/2,
+    put/4,
+    get/2,
+    set/3,
+    delete/3,
+    dirty/1,
+    commit_dirty/2,
+    commit_renew/3
+]).
+
+-export_type([
+    t/0,
+    leader_claim/1
+]).
+
+-type group() :: binary().
+-type leader_claim(ID) :: {ID, _Heartbeat :: emqx_message:timestamp()}.
+
+-define(DS_DB, dqleader).
+
+-define(LEADER_TTL, 30_000).
+-define(LEADER_HEARTBEAT_INTERVAL, 10_000).
+
+-define(LEADER_TOPIC_PREFIX, <<"$leader">>).
+-define(LEADER_HEADER_HEARTBEAT, <<"$leader.ts">>).
+
+-define(STORE_TOPIC_PREFIX, <<"$s">>).
+
+-define(STORE_SK(SPACE, KEY), [SPACE | KEY]).
+-define(STORE_STAGE_ENTRY(SEQNUM, VALUE), {SEQNUM, VALUE}).
+-define(STORE_TOMBSTONE, '$tombstone').
+-define(STORE_PAYLOAD(ID, VALUE), [ID, VALUE]).
+-define(STORE_HEADER_CHANGESEQNUM, '$store.seqnum').
+
+-define(STORE_BATCH_SIZE, 500).
+-define(STORE_SLURP_RETRIES, 2).
+-define(STORE_SLURP_RETRY_TIMEOUT, 1000).
+
+-define(STORE_IS_ROOTSET(VAR), (VAR == seqnum)).
+
+-ifdef(TEST).
+-undef(LEADER_TTL).
+-undef(LEADER_HEARTBEAT_INTERVAL).
+-define(LEADER_TTL, 3_000).
+-define(LEADER_HEARTBEAT_INTERVAL, 1_000).
+-endif.
+
+%%
+
+open() ->
+    emqx_ds:open_db(?DS_DB, db_config()).
+
+close() ->
+    emqx_ds:close_db(?DS_DB).
+
+db_config() ->
+    Config = emqx_ds_schema:db_config([durable_storage, queues]),
+    tune_db_config(Config).
+
+tune_db_config(Config0 = #{backend := Backend}) ->
+    Config = Config0#{
+        %% We need total control over timestamp assignment.
+        force_monotonic_timestamps => false
+    },
+    case Backend of
+        B when B == builtin_raft; B == builtin_local ->
+            tune_db_storage_layout(Config);
+        _ ->
+            Config
+    end.
+
+tune_db_storage_layout(Config = #{storage := {Layout, Opts0}}) when
+    Layout == emqx_ds_storage_skipstream_lts;
+    Layout == emqx_ds_storage_bitfield_lts
+->
+    Opts = Opts0#{
+        %% Since these layouts impose somewhat strict requirements on message
+        %% timestamp uniqueness, we need to additionally ensure that LTS always
+        %% keeps different groups under separate indices.
+        lts_threshold_spec => {simple, {inf, inf, inf, 0}}
+    },
+    Config#{storage := {Layout, Opts}};
+tune_db_storage_layout(Config = #{storage := _}) ->
+    Config.
+
+%%
+
+-spec claim_leadership(group(), ID, emqx_message:timestamp()) ->
+    {ok | exists, leader_claim(ID)} | emqx_ds:error(_).
+claim_leadership(Group, LeaderID, TS) ->
+    LeaderClaim = {LeaderID, TS},
+    case try_replace_leader(Group, LeaderClaim, undefined) of
+        ok ->
+            {ok, LeaderClaim};
+        {exists, ExistingClaim = {_, LastHeartbeat}} when LastHeartbeat > TS - ?LEADER_TTL ->
+            {exists, ExistingClaim};
+        {exists, ExistingClaim = {_LeaderDead, _}} ->
+            case try_replace_leader(Group, LeaderClaim, ExistingClaim) of
+                ok ->
+                    {ok, LeaderClaim};
+                {exists, ConcurrentClaim} ->
+                    {exists, ConcurrentClaim};
+                Error ->
+                    Error
+            end;
+        Error ->
+            Error
+    end.
+
+-spec renew_leadership(group(), leader_claim(ID), emqx_message:timestamp()) ->
+    {ok | exists, leader_claim(ID)} | emqx_ds:error(_).
+renew_leadership(Group, LeaderClaim, TS) ->
+    RenewedClaim = renew_claim(LeaderClaim, TS),
+    case RenewedClaim =/= false andalso try_replace_leader(Group, RenewedClaim, LeaderClaim) of
+        ok ->
+            {ok, RenewedClaim};
+        {exists, NewestClaim} ->
+            {exists, NewestClaim};
+        false ->
+            {error, unrecoverable, leader_claim_outdated};
+        Error ->
+            Error
+    end.
+
+-spec renew_claim(leader_claim(ID), emqx_message:timestamp()) -> leader_claim(ID) | false.
+renew_claim({LeaderID, LastHeartbeat}, TS) ->
+    RenewedClaim = {LeaderID, TS},
+    IsRenewable = (LastHeartbeat > TS - ?LEADER_TTL),
+    IsRenewable andalso RenewedClaim.
+
+-spec disown_leadership(group(), leader_claim(_ID)) ->
+    ok | emqx_ds:error(_).
+disown_leadership(Group, LeaderClaim) ->
+    try_delete_leader(Group, LeaderClaim).
+
+-spec leader_id(leader_claim(ID)) ->
+    ID.
+leader_id({LeaderID, _}) ->
+    LeaderID.
+
+-spec alive_until(leader_claim(_)) ->
+    emqx_message:timestamp().
+alive_until({_LeaderID, LastHeartbeatTS}) ->
+    LastHeartbeatTS + ?LEADER_TTL.
+
+-spec heartbeat_interval(leader_claim(_)) ->
+    _Milliseconds :: pos_integer().
+heartbeat_interval(_) ->
+    ?LEADER_HEARTBEAT_INTERVAL.
+try_replace_leader(Group, LeaderClaim, ExistingClaim) ->
+    Batch = #dsbatch{
+        preconditions = [mk_precondition(Group, ExistingClaim)],
+        operations = [encode_leader_claim(Group, LeaderClaim)]
+    },
+    case emqx_ds:store_batch(?DS_DB, Batch, #{sync => true}) of
+        ok ->
+            ok;
+        {error, unrecoverable, {precondition_failed, Mismatch}} ->
+            {exists, decode_leader_msg(Mismatch)};
+        Error ->
+            Error
+    end.
+
+try_delete_leader(Group, LeaderClaim) ->
+    {_Cond, Matcher} = mk_precondition(Group, LeaderClaim),
+    emqx_ds:store_batch(?DS_DB, #dsbatch{operations = [{delete, Matcher}]}, #{sync => false}).
+
+mk_precondition(Group, undefined) ->
+    {unless_exists, #message_matcher{
+        from = Group,
+        topic = mk_leader_topic(Group),
+        timestamp = 0,
+        payload = '_'
+    }};
+mk_precondition(Group, {Leader, HeartbeatTS}) ->
+    {if_exists, #message_matcher{
+        from = Group,
+        topic = mk_leader_topic(Group),
+        timestamp = 0,
+        payload = encode_leader(Leader),
+        headers = #{?LEADER_HEADER_HEARTBEAT => HeartbeatTS}
+    }}.
+
+encode_leader_claim(Group, {Leader, HeartbeatTS}) ->
+    #message{
+        id = <<>>,
+        qos = 0,
+        from = Group,
+        topic = mk_leader_topic(Group),
+        timestamp = 0,
+        payload = encode_leader(Leader),
+        headers = #{?LEADER_HEADER_HEARTBEAT => HeartbeatTS}
+    }.
+
+decode_leader_msg(#message{from = _Group, payload = Payload, headers = Headers}) ->
+    Leader = decode_leader(Payload),
+    Heartbeat = maps:get(?LEADER_HEADER_HEARTBEAT, Headers, 0),
+    {Leader, Heartbeat}.
+
+encode_leader(Leader) ->
+    %% NOTE: Lists are compact but easy to extend later.
+    term_to_binary([Leader]).
+
+decode_leader(Payload) ->
+    [Leader | _Extra] = binary_to_term(Payload),
+    Leader.
+
+mk_leader_topic(GroupName) ->
+    emqx_topic:join([?LEADER_TOPIC_PREFIX, GroupName]).
+
+%%
+
+-type space_name() :: stream.
+-type var_name() :: start_time | rank_progress.
+-type space_key() :: nonempty_improper_list(space_name(), _Key).
+
+%% NOTE
+%% Instances of `emqx_ds:stream()` type are persisted in durable storage.
+%% Given that streams are opaque and identity of a stream is stream itself (i.e.
+%% if S1 =:= S2 then both are the same stream), it's critical to keep the "shape"
+%% of the term intact between releases. Otherwise, if it changes then we will
+%% need an additional API to deal with that (e.g. `emqx_ds:term_to_stream/2`).
+%% Instances of `emqx_ds:iterator()` are also persisted in durable storage,
+%% but those already has similar requirement because in some backends they travel
+%% in RPCs between different nodes of potentially different releases.
+-type t() :: #{
+    %% General.
+    group := group(),
+    %% Spaces and variables: most up-to-date in-memory state.
+    stream := #{emqx_ds:stream() => stream_state()},
+    start_time => _SubsriptionStartTime :: emqx_message:timestamp(),
+    rank_progress => _RankProgress,
+    %% Internal _sequence number_ that tracks every change.
+    seqnum := integer(),
+    %% Mapping between complex keys and seqnums.
+    seqmap := #{space_key() => _SeqNum :: integer()},
+    %% Stage: uncommitted changes.
+    stage := #{space_key() | var_name() => _Value},
+    dirty => true
+}.
+
+-type stream_state() :: #{
+    progress => emqx_persistent_session_ds_shared_subs:progress(),
+    rank => emqx_ds:stream_rank()
+}.
+
+-spec init(group()) -> t().
+init(Group) ->
+    %% NOTE: Empty store is dirty because rootset needs to be persisted.
+    mark_dirty(mk_store(Group)).
+
+-spec open(group()) -> t() | false.
+open(Group) ->
+    open_store(mk_store(Group)).
+
+mk_store(Group) ->
+    #{
+        group => Group,
+        stream => #{},
+        seqnum => 0,
+        seqmap => #{},
+        stage => #{}
+    }.
+
+open_store(Store = #{group := Group}) ->
+    ReadRootset = mk_read_root_batch(Group),
+    case emqx_ds:store_batch(?DS_DB, ReadRootset, #{sync => true}) of
+        ok ->
+            false;
+        {error, unrecoverable, {precondition_failed, RootMessage}} ->
+            Rootset = open_root_message(RootMessage),
+            slurp_store(Rootset, Store)
+    end.
+
+slurp_store(Rootset, Acc) ->
+    slurp_store(Rootset, #{}, ?STORE_SLURP_RETRIES, ?STORE_SLURP_RETRY_TIMEOUT, Acc).
+
+slurp_store(Rootset, StreamIts0, Retries, RetryTimeout, Acc = #{group := Group}) ->
+    TopicFilter = mk_store_wildcard(Group),
+    StreamIts1 = ds_refresh_streams(TopicFilter, _StartTime = 0, StreamIts0),
+    {StreamIts, Store} = ds_streams_fold(
+        fun(Message, StoreAcc) -> open_message(Message, StoreAcc) end,
+        Acc,
+        StreamIts1
+    ),
+    case map_get(seqnum, Store) of
+        SeqNum when SeqNum >= map_get(seqnum, Rootset) ->
+            maps:merge(Store, Rootset);
+        _Mismatch when Retries > 0 ->
+            ok = timer:sleep(RetryTimeout),
+            slurp_store(Rootset, StreamIts, Retries - 1, RetryTimeout, Store);
+        _Mismatch ->
+            {error, unrecoverable, {leader_store_inconsistent, Store, Rootset}}
+    end.
+
+-spec get(space_name(), _ID, t()) -> _Value.
+get(SpaceName, ID, Store) ->
+    Space = maps:get(SpaceName, Store),
+    maps:get(ID, Space).
+
+-spec get(space_name(), _ID, Default, t()) -> _Value | Default.
+get(SpaceName, ID, Default, Store) ->
+    Space = maps:get(SpaceName, Store),
+    maps:get(ID, Space, Default).
+
+-spec fold(space_name(), fun((_ID, _Value, Acc) -> Acc), Acc, t()) -> Acc.
+fold(SpaceName, Fun, Acc, Store) ->
+    Space = maps:get(SpaceName, Store),
+    maps:fold(Fun, Acc, Space).
+
+-spec size(space_name(), t()) -> non_neg_integer().
+size(SpaceName, Store) ->
+    map_size(maps:get(SpaceName, Store)).
+
+-spec put(space_name(), _ID, _Value, t()) -> t().
+put(SpaceName, ID, Value, Store0 = #{stage := Stage, seqnum := SeqNum0, seqmap := SeqMap}) ->
+    Space0 = maps:get(SpaceName, Store0),
+    Space1 = maps:put(ID, Value, Space0),
+    SeqNum = SeqNum0 + 1,
+    SK = ?STORE_SK(SpaceName, ID),
+    Store = Store0#{
+        SpaceName := Space1,
+        seqnum := SeqNum,
+        stage := Stage#{SK => ?STORE_STAGE_ENTRY(SeqNum, Value)}
+    },
+    case map_size(Space1) of
+        S when S > map_size(Space0) ->
+            Store#{seqmap := maps:put(SK, SeqNum, SeqMap)};
+        _ ->
+            Store
+    end.
+
+get_seqnum(?STORE_SK(_SpaceName, _) = SK, SeqMap) ->
+    maps:get(SK, SeqMap);
+get_seqnum(_VarName, _SeqMap) ->
+    0.
+
+-spec get(var_name(), t()) -> _Value.
+get(VarName, Store) ->
+    maps:get(VarName, Store).
+
+-spec set(var_name(), _Value, t()) -> t().
+set(VarName, Value, Store = #{stage := Stage, seqnum := SeqNum0}) ->
+    SeqNum = SeqNum0 + 1,
+    Store#{
+        VarName => Value,
+        seqnum := SeqNum,
+        stage := Stage#{VarName => ?STORE_STAGE_ENTRY(SeqNum, Value)}
+    }.
+
+-spec delete(space_name(), _ID, t()) -> t().
+delete(SpaceName, ID, Store = #{stage := Stage, seqmap := SeqMap}) ->
+    Space0 = maps:get(SpaceName, Store),
+    Space1 = maps:remove(ID, Space0),
+    case map_size(Space1) of
+        S when S < map_size(Space0) ->
+            %% NOTE
+            %% We do not bump seqnum on deletions because tracking them does
+            %% not make a lot of sense, assuming batches are atomic.
+            SK = ?STORE_SK(SpaceName, ID),
+            Store#{
+                SpaceName := Space1,
+                stage := Stage#{SK => ?STORE_TOMBSTONE},
+                seqmap := maps:remove(SK, SeqMap)
+            };
+        _ ->
+            Store
+    end.
+
+mark_dirty(Store) ->
+    Store#{dirty => true}.
+
+mark_clean(Store) ->
+    maps:remove(dirty, Store).
+
+-spec dirty(t()) -> boolean().
+dirty(#{dirty := Dirty}) ->
+    Dirty;
+dirty(#{stage := Stage}) ->
+    map_size(Stage) > 0.
+
+%% @doc Commit staged changes to the storage.
+%% Does nothing if there are no staged changes.
+-spec commit_dirty(leader_claim(_), t()) ->
+    {ok, t()} | emqx_ds:error(_).
+commit_dirty(LeaderClaim, Store = #{dirty := true}) ->
+    commit(LeaderClaim, Store);
+commit_dirty(LeaderClaim, Store = #{stage := Stage}) when map_size(Stage) > 0 ->
+    commit(LeaderClaim, Store).
+
+commit(LeaderClaim, Store = #{group := Group}) ->
+    Operations = mk_store_operations(Store),
+    Batch = mk_store_batch(Group, LeaderClaim, Operations),
+    case emqx_ds:store_batch(?DS_DB, Batch, #{sync => true}) of
+        ok ->
+            {ok, mark_clean(Store#{stage := #{}})};
+        {error, unrecoverable, {precondition_failed, Mismatch}} ->
+            {error, unrecoverable, {leadership_lost, decode_leader_msg(Mismatch)}};
+        Error ->
+            Error
+    end.
+
+%% @doc Commit staged changes and renew leadership at the same time.
+%% Goes to the storage even if there are no staged changes.
+-spec commit_renew(leader_claim(ID), emqx_message:timestamp(), t()) ->
+    {ok, leader_claim(ID), t()} | emqx_ds:error(_).
+commit_renew(LeaderClaim, TS, Store = #{group := Group}) ->
+    case renew_claim(LeaderClaim, TS) of
+        RenewedClaim when RenewedClaim =/= false ->
+            Operations = mk_store_operations(Store),
+            Batch = mk_store_batch(Group, LeaderClaim, RenewedClaim, Operations),
+            case emqx_ds:store_batch(?DS_DB, Batch, #{sync => true}) of
+                ok ->
+                    {ok, RenewedClaim, mark_clean(Store#{stage := #{}})};
+                {error, unrecoverable, {precondition_failed, Mismatch}} ->
+                    {error, unrecoverable, {leadership_lost, decode_leader_msg(Mismatch)}};
+                Error ->
+                    Error
+            end;
+        false ->
+            {error, unrecoverable, leader_claim_outdated}
+    end.
+
+mk_store_batch(Group, LeaderClaim, Operations) ->
+    #dsbatch{
+        preconditions = [mk_precondition(Group, LeaderClaim)],
+        operations = Operations
+    }.
+
+mk_store_batch(Group, ExistingClaim, RenewedClaim, Operations) ->
+    #dsbatch{
+        preconditions = [mk_precondition(Group, ExistingClaim)],
+        operations = [encode_leader_claim(Group, RenewedClaim) | Operations]
+    }.
+
+mk_store_operations(Store = #{group := Group, stage := Stage, seqmap := SeqMap}) ->
+    %% NOTE: Always persist rootset.
+    RootOperation = mk_store_root(Store),
+    maps:fold(
+        fun(SK, Value, Acc) ->
+            [mk_store_operation(Group, SK, Value, SeqMap) | Acc]
+        end,
+        [RootOperation],
+        Stage
+    ).
+
+mk_store_root(Store = #{group := Group}) ->
+    Payload = maps:filter(fun(V, _) -> ?STORE_IS_ROOTSET(V) end, Store),
+    #message{
+        id = <<>>,
+        qos = 0,
+        from = Group,
+        topic = mk_store_root_topic(Group),
+        payload = term_to_binary(Payload),
+        timestamp = 0
+    }.
+
+mk_store_operation(Group, SK, ?STORE_TOMBSTONE, SeqMap) ->
+    {delete, #message_matcher{
+        from = Group,
+        topic = mk_store_topic(Group, SK, SeqMap),
+        payload = '_',
+        timestamp = get_seqnum(SK, SeqMap)
+    }};
+mk_store_operation(Group, SK, ?STORE_STAGE_ENTRY(ChangeSeqNum, Value), SeqMap) ->
+    %% NOTE
+    %% Using `SeqNum` as timestamp to further disambiguate one record (message) from
+    %% another in the DS DB keyspace. As an example, Skipstream-LTS storage layout
+    %% _requires_ messages in the same stream to have unique timestamps.
+    %% TODO
+    %% Do we need to have wall-clock timestamp here?
+    Payload = mk_store_payload(SK, Value),
+    #message{
+        id = <<>>,
+        qos = 0,
+        from = Group,
+        topic = mk_store_topic(Group, SK, SeqMap),
+        payload = term_to_binary(Payload),
+        timestamp = get_seqnum(SK, SeqMap),
+        %% NOTE: Preserving the seqnum when this change has happened.
+        headers = #{?STORE_HEADER_CHANGESEQNUM => ChangeSeqNum}
+    }.
+
+open_root_message(#message{payload = Payload, timestamp = 0}) ->
+    #{} = binary_to_term(Payload).
+
+open_message(
+    Msg = #message{topic = Topic, payload = Payload, timestamp = SeqNum, headers = Headers}, Store
+) ->
+    Entry =
+        try
+            ChangeSeqNum = maps:get(?STORE_HEADER_CHANGESEQNUM, Headers),
+            case emqx_topic:tokens(Topic) of
+                [_Prefix, _Group, SpaceTok, _SeqTok] ->
+                    SpaceName = token_to_space(SpaceTok),
+                    ?STORE_PAYLOAD(ID, Value) = binary_to_term(Payload),
+                    %% TODO: Records.
+                    Record = {SpaceName, ID, Value, SeqNum};
+                [_Prefix, _Group, VarTok] ->
+                    VarName = token_to_varname(VarTok),
+                    Value = binary_to_term(Payload),
+                    Record = {VarName, Value}
+            end,
+            {ChangeSeqNum, Record}
+        catch
+            error:_ ->
+                ?tp(warning, "dssubs_leader_store_unrecognized_message", #{
+                    group => maps:get(group, Store),
+                    message => Msg
+                }),
+                unrecognized
+        end,
+    open_entry(Entry, Store).
+
+open_entry({ChangeSeqNum, Record}, Store = #{seqnum := SeqNum}) ->
+    open_record(Record, Store#{seqnum := max(ChangeSeqNum, SeqNum)}).
+
+open_record({SpaceName, ID, Value, SeqNum}, Store = #{seqmap := SeqMap}) ->
+    Space0 = maps:get(SpaceName, Store),
+    Space1 = maps:put(ID, Value, Space0),
+    SK = ?STORE_SK(SpaceName, ID),
+    Store#{
+        SpaceName := Space1,
+        seqmap := SeqMap#{SK => SeqNum}
+    };
+open_record({VarName, Value}, Store) ->
+    Store#{VarName => Value}.
+
+mk_store_payload(?STORE_SK(_SpaceName, ID), Value) ->
+    ?STORE_PAYLOAD(ID, Value);
+mk_store_payload(_VarName, Value) ->
+    Value.
+
+mk_store_root_topic(GroupName) ->
+    emqx_topic:join([?STORE_TOPIC_PREFIX, GroupName]).
+
+mk_store_topic(GroupName, ?STORE_SK(SpaceName, _) = SK, SeqMap) ->
+    SeqNum = get_seqnum(SK, SeqMap),
+    SeqTok = integer_to_binary(SeqNum),
+    emqx_topic:join([?STORE_TOPIC_PREFIX, GroupName, space_to_token(SpaceName), SeqTok]);
+mk_store_topic(GroupName, VarName, _SeqMap) ->
+    emqx_topic:join([?STORE_TOPIC_PREFIX, GroupName, varname_to_token(VarName)]).
+
+mk_store_wildcard(GroupName) ->
+    [?STORE_TOPIC_PREFIX, GroupName, '+', '#'].
+
+mk_read_root_batch(Group) ->
+    %% NOTE
+    %% Construct batch that essentially does nothing but reads rootset in a consistent
+    %% manner.
+    Matcher = #message_matcher{
+        from = Group,
+        topic = mk_store_root_topic(Group),
+        payload = '_',
+        timestamp = 0
+    },
+    #dsbatch{
+        preconditions = [{unless_exists, Matcher}],
+        operations = [{delete, Matcher#message_matcher{payload = <<>>}}]
+    }.
+
+ds_refresh_streams(TopicFilter, StartTime, StreamIts) ->
+    Streams = emqx_ds:get_streams(?DS_DB, TopicFilter, StartTime),
+    lists:foldl(
+        fun({_Rank, Stream}, Acc) ->
+            case StreamIts of
+                #{Stream := _It} ->
+                    Acc;
+                #{} ->
+                    %% TODO: Gracefully handle `emqx_ds:error(_)`?
+                    {ok, It} = emqx_ds:make_iterator(?DS_DB, Stream, TopicFilter, StartTime),
+                    Acc#{Stream => It}
+            end
+        end,
+        StreamIts,
+        Streams
+    ).
+
+ds_streams_fold(Fun, AccIn, StreamItsIn) ->
+    maps:fold(
+        fun(Stream, It0, {StreamIts, Acc0}) ->
+            {It, Acc} = ds_stream_fold(Fun, Acc0, It0),
+            {StreamIts#{Stream := It}, Acc}
+        end,
+        {StreamItsIn, AccIn},
+        StreamItsIn
+    ).
+
+ds_stream_fold(_Fun, Acc0, end_of_stream) ->
+    %% NOTE: Assuming atom `end_of_stream` is not a valid `emqx_ds:iterator()`.
+    {end_of_stream, Acc0};
+ds_stream_fold(Fun, Acc0, It0) ->
+    %% TODO: Gracefully handle `emqx_ds:error(_)`?
+    case emqx_ds:next(?DS_DB, It0, ?STORE_BATCH_SIZE) of
+        {ok, It, Messages = [_ | _]} ->
+            Acc1 = lists:foldl(fun({_Key, Msg}, Acc) -> Fun(Msg, Acc) end, Acc0, Messages),
+            ds_stream_fold(Fun, Acc1, It);
+        {ok, It, []} ->
+            {It, Acc0};
+        {ok, end_of_stream} ->
+            {end_of_stream, Acc0}
+    end.
+
+%%
+
+space_to_token(stream) -> <<"s">>;
+space_to_token(progress) -> <<"prog">>;
+space_to_token(sequence) -> <<"seq">>.
+
+token_to_space(<<"s">>) -> stream;
+token_to_space(<<"prog">>) -> progress;
+token_to_space(<<"seq">>) -> sequence.
+
+varname_to_token(rank_progress) -> <<"rankp">>;
+varname_to_token(start_time) -> <<"stime">>.
+
+token_to_varname(<<"rankp">>) -> rank_progress;
+token_to_varname(<<"stime">>) -> start_time.

apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_leader_sup.erl

@@ -1,59 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%--------------------------------------------------------------------
-
--module(emqx_ds_shared_sub_leader_sup).
-
--behaviour(supervisor).
-
-%% API
--export([
-    start_link/0,
-    child_spec/0,
-
-    start_leader/1,
-    stop_leader/1
-]).
-
-%% supervisor behaviour callbacks
--export([init/1]).
-
-%%------------------------------------------------------------------------------
-%% API
-%%------------------------------------------------------------------------------
-
--spec start_link() -> supervisor:startlink_ret().
-start_link() ->
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-
--spec child_spec() -> supervisor:child_spec().
-child_spec() ->
-    #{
-        id => ?MODULE,
-        start => {?MODULE, start_link, []},
-        restart => permanent,
-        shutdown => 5000,
-        type => supervisor
-    }.
-
--spec start_leader(emqx_ds_shared_sub_leader:options()) -> supervisor:startchild_ret().
-start_leader(Options) ->
-    ChildSpec = emqx_ds_shared_sub_leader:child_spec(Options),
-    supervisor:start_child(?MODULE, ChildSpec).
-
--spec stop_leader(emqx_persistent_session_ds:share_topic_filter()) -> ok | {error, term()}.
-stop_leader(TopicFilter) ->
-    supervisor:terminate_child(?MODULE, emqx_ds_shared_sub_leader:id(TopicFilter)).
-
-%%------------------------------------------------------------------------------
-%% supervisor behaviour callbacks
-%%------------------------------------------------------------------------------
-
-init([]) ->
-    SupFlags = #{
-        strategy => one_for_one,
-        intensity => 10,
-        period => 10
-    },
-    ChildSpecs = [],
-    {ok, {SupFlags, ChildSpecs}}.

apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_registry.erl

@@ -4,122 +4,77 @@
 
 -module(emqx_ds_shared_sub_registry).
 
--behaviour(gen_server).
-
--include_lib("emqx/include/logger.hrl").
-
+%% API
 -export([
     start_link/0,
-    child_spec/0,
-
-    init/1,
-    handle_call/3,
-    handle_cast/2,
-    handle_info/2,
-    terminate/2
+    child_spec/0
 ]).
 
 -export([
-    lookup_leader/3
+    leader_wanted/3,
+    start_elector/2
 ]).
 
--record(lookup_leader, {
-    agent :: emqx_ds_shared_sub_proto:agent(),
-    agent_metadata :: emqx_ds_shared_sub_proto:agent_metadata(),
-    share_topic_filter :: emqx_persistent_session_ds:share_topic_filter()
-}).
+-behaviour(supervisor).
+-export([init/1]).
 
--define(gproc_id(ID), {n, l, ID}).
-
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 %% API
-%%--------------------------------------------------------------------
-
--spec lookup_leader(
-    emqx_ds_shared_sub_proto:agent(),
-    emqx_ds_shared_sub_proto:agent_metadata(),
-    emqx_persistent_session_ds:share_topic_filter()
-) -> ok.
-lookup_leader(Agent, AgentMetadata, ShareTopicFilter) ->
-    gen_server:cast(?MODULE, #lookup_leader{
-        agent = Agent, agent_metadata = AgentMetadata, share_topic_filter = ShareTopicFilter
-    }).
-
-%%--------------------------------------------------------------------
-%% Internal API
-%%--------------------------------------------------------------------
+%%------------------------------------------------------------------------------
 
+-spec start_link() -> supervisor:startlink_ret().
 start_link() ->
-    gen_server:start_link({local, ?MODULE}, ?MODULE, [], []).
+    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
+-spec child_spec() -> supervisor:child_spec().
 child_spec() ->
     #{
         id => ?MODULE,
         start => {?MODULE, start_link, []},
         restart => permanent,
-        shutdown => 5000,
-        type => worker
+        type => supervisor
     }.
 
-%%--------------------------------------------------------------------
-%% gen_server callbacks
-%%--------------------------------------------------------------------
-
-init([]) ->
-    {ok, #{}}.
+-spec leader_wanted(
+    emqx_ds_shared_sub_proto:agent(),
+    emqx_ds_shared_sub_proto:agent_metadata(),
+    emqx_persistent_session_ds:share_topic_filter()
+) -> ok.
+leader_wanted(Agent, AgentMetadata, ShareTopic) ->
+    {ok, Pid} = ensure_elector_started(ShareTopic),
+    emqx_ds_shared_sub_proto:agent_connect_leader(Pid, Agent, AgentMetadata, ShareTopic).
+
+-spec ensure_elector_started(emqx_persistent_session_ds:share_topic_filter()) ->
+    {ok, pid()}.
+ensure_elector_started(ShareTopic) ->
+    case start_elector(ShareTopic, _StartTime = emqx_message:timestamp_now()) of
+        {ok, Pid} ->
+            {ok, Pid};
+        {error, {already_started, Pid}} when is_pid(Pid) ->
+            {ok, Pid}
+    end.
+
+-spec start_elector(emqx_persistent_session_ds:share_topic_filter(), emqx_message:timestamp()) ->
+    supervisor:startchild_ret().
+start_elector(ShareTopic, StartTime) ->
+    supervisor:start_child(?MODULE, #{
+        id => ShareTopic,
+        start => {emqx_ds_shared_sub_elector, start_link, [ShareTopic, StartTime]},
+        restart => temporary,
+        type => worker,
+        shutdown => 5000
+    }).
 
-handle_call(_Request, _From, State) ->
-    {reply, {error, unknown_request}, State}.
+%%------------------------------------------------------------------------------
+%% supervisor behaviour callbacks
+%%------------------------------------------------------------------------------
 
-handle_cast(
-    #lookup_leader{
-        agent = Agent,
-        agent_metadata = AgentMetadata,
-        share_topic_filter = ShareTopicFilter
+init([]) ->
+    ok = emqx_ds_shared_sub_leader_store:open(),
+    SupFlags = #{
+        strategy => one_for_one,
+        intensity => 10,
+        period => 1
     },
-    State
-) ->
-    State1 = do_lookup_leader(Agent, AgentMetadata, ShareTopicFilter, State),
-    {noreply, State1}.
-
-handle_info(_Info, State) ->
-    {noreply, State}.
-
-terminate(_Reason, _State) ->
-    ok.
-
-%%--------------------------------------------------------------------
-%% Internal functions
-%%--------------------------------------------------------------------
-
-do_lookup_leader(Agent, AgentMetadata, ShareTopicFilter, State) ->
-    %% TODO https://emqx.atlassian.net/browse/EMQX-12309
-    %% Cluster-wide unique leader election should be implemented
-    Id = emqx_ds_shared_sub_leader:id(ShareTopicFilter),
-    LeaderPid =
-        case gproc:where(?gproc_id(Id)) of
-            undefined ->
-                {ok, Pid} = emqx_ds_shared_sub_leader_sup:start_leader(#{
-                    share_topic_filter => ShareTopicFilter
-                }),
-                {ok, NewLeaderPid} = emqx_ds_shared_sub_leader:register(
-                    Pid,
-                    fun() ->
-                        {LPid, _} = gproc:reg_or_locate(?gproc_id(Id)),
-                        LPid
-                    end
-                ),
-                NewLeaderPid;
-            Pid ->
-                Pid
-        end,
-    ?SLOG(debug, #{
-        msg => lookup_leader,
-        agent => Agent,
-        share_topic_filter => ShareTopicFilter,
-        leader => LeaderPid
-    }),
-    ok = emqx_ds_shared_sub_proto:agent_connect_leader(
-        LeaderPid, Agent, AgentMetadata, ShareTopicFilter
-    ),
-    State.
+    ChildSpecs = [],
+    {ok, {SupFlags, ChildSpecs}}.

apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_schema.erl

@@ -13,6 +13,10 @@
     desc/1
 ]).
 
+-export([
+    injected_fields/0
+]).
+
 namespace() -> emqx_shared_subs.
 
 roots() ->
@@ -42,6 +46,17 @@ fields(durable_queues) ->
         duration(leader_session_not_replaying_timeout_ms, 5000)
     ].
 
+injected_fields() ->
+    #{
+        'durable_storage' => [
+            {queues,
+                emqx_ds_schema:storage_schema(#{
+                    importance => ?IMPORTANCE_HIDDEN,
+                    desc => ?DESC(durable_queues_storage)
+                })}
+        ]
+    }.
+
 duration(MsFieldName, Default) ->
     {MsFieldName,
         ?HOCON(

apps/emqx_ds_shared_sub/src/emqx_ds_shared_sub_sup.erl

@@ -30,7 +30,6 @@ init([]) ->
         period => 10
     },
     ChildSpecs = [
-        emqx_ds_shared_sub_registry:child_spec(),
-        emqx_ds_shared_sub_leader_sup:child_spec()
+        emqx_ds_shared_sub_registry:child_spec()
     ],
     {ok, {SupFlags, ChildSpecs}}.

apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_SUITE.erl

@@ -18,7 +18,7 @@ all() ->
 init_per_suite(Config) ->
     Apps = emqx_cth_suite:start(
         [
-            {emqx, #{
+            {emqx_conf, #{
                 config => #{
                     <<"durable_sessions">> => #{
                         <<"enable">> => true,
@@ -27,10 +27,17 @@ init_per_suite(Config) ->
                     <<"durable_storage">> => #{
                         <<"messages">> => #{
                             <<"backend">> => <<"builtin_raft">>
+                        },
+                        <<"queues">> => #{
+                            <<"backend">> => <<"builtin_raft">>,
+                            <<"local_write_buffer">> => #{
+                                <<"flush_interval">> => <<"10ms">>
+                            }
                         }
                     }
                 }
             }},
+            emqx,
             emqx_ds_shared_sub
         ],
         #{work_dir => ?config(priv_dir, Config)}
@@ -183,6 +190,44 @@ t_graceful_disconnect(_Config) ->
     ok = emqtt:disconnect(ConnShared2),
     ok = emqtt:disconnect(ConnPub).
 
+t_leader_state_preserved(_Config) ->
+    ?check_trace(
+        begin
+            ConnShared1 = emqtt_connect_sub(<<"client1">>),
+            {ok, _, _} = emqtt:subscribe(ConnShared1, <<"$share/lsp/topic42/#">>, 1),
+
+            ConnShared2 = emqtt_connect_sub(<<"client2">>),
+            {ok, _, _} = emqtt:subscribe(ConnShared2, <<"$share/lsp/topic42/#">>, 1),
+
+            ConnPub = emqtt_connect_pub(<<"client_pub">>),
+
+            {ok, _} = emqtt:publish(ConnPub, <<"topic42/1/2">>, <<"hello1">>, 1),
+            {ok, _} = emqtt:publish(ConnPub, <<"topic42/3/4">>, <<"hello2">>, 1),
+            ?assertReceive({publish, #{payload := <<"hello1">>}}, 2_000),
+            ?assertReceive({publish, #{payload := <<"hello2">>}}, 2_000),
+
+            ok = emqtt:disconnect(ConnShared1),
+            ok = emqtt:disconnect(ConnShared2),
+
+            %% Equivalent to node restart.
+            ok = terminate_leaders(),
+            ok = timer:sleep(1_000),
+
+            {ok, _} = emqtt:publish(ConnPub, <<"topic42/1/2">>, <<"hello3">>, 1),
+            {ok, _} = emqtt:publish(ConnPub, <<"topic42/3/4">>, <<"hello4">>, 1),
+
+            ConnShared3 = emqtt_connect_sub(<<"client3">>),
+            {ok, _, _} = emqtt:subscribe(ConnShared3, <<"$share/lsp/topic42/#">>, 1),
+
+            ?assertReceive({publish, #{payload := <<"hello3">>}}, 2_000),
+            ?assertReceive({publish, #{payload := <<"hello4">>}}, 2_000),
+
+            ok = emqtt:disconnect(ConnShared3),
+            ok = emqtt:disconnect(ConnPub)
+        end,
+        []
+    ).
+
 t_intensive_reassign(_Config) ->
     ConnPub = emqtt_connect_pub(<<"client_pub">>),
 
@@ -405,15 +450,16 @@ t_disconnect_no_double_replay2(_Config) ->
     %     3000
     % ),
 
-    ok = emqtt:disconnect(ConnShared12).
+    ok = emqtt:disconnect(ConnShared12),
+    ok = emqtt:disconnect(ConnPub).
 
 t_lease_reconnect(_Config) ->
     ConnPub = emqtt_connect_pub(<<"client_pub">>),
 
     ConnShared = emqtt_connect_sub(<<"client_shared">>),
 
-    %% Stop registry to simulate unability to find leader.
-    ok = supervisor:terminate_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_registry),
+    %% Simulate unability to find leader.
+    ok = emqx_ds_shared_sub_leader_store:close(),
 
     ?assertWaitEvent(
         {ok, _, _} = emqtt:subscribe(ConnShared, <<"$share/gr2/topic2/#">>, 1),
@@ -421,9 +467,9 @@ t_lease_reconnect(_Config) ->
         5_000
     ),
 
-    %% Start registry, agent should retry after some time and find the leader.
+    %% Agent should retry after some time and find the leader.
     ?assertWaitEvent(
-        {ok, _} = supervisor:restart_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_registry),
+        ok = emqx_ds_shared_sub_leader_store:open(),
         #{?snk_kind := leader_lease_streams},
         5_000
     ),
@@ -490,8 +536,8 @@ emqtt_connect_pub(ClientId) ->
     C.
 
 terminate_leaders() ->
-    ok = supervisor:terminate_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup),
-    {ok, _} = supervisor:restart_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup),
+    ok = supervisor:terminate_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_registry),
+    {ok, _} = supervisor:restart_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_registry),
     ok.
 
 publish_n(_Conn, _Topics, From, To) when From > To ->

apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_api_SUITE.erl

@@ -25,7 +25,7 @@ all() ->
 init_per_suite(Config) ->
     Apps = emqx_cth_suite:start(
         [
-            {emqx, #{
+            {emqx_conf, #{
                 config => #{
                     <<"durable_sessions">> => #{
                         <<"enable">> => true,
@@ -34,10 +34,14 @@ init_per_suite(Config) ->
                     <<"durable_storage">> => #{
                         <<"messages">> => #{
                             <<"backend">> => <<"builtin_raft">>
+                        },
+                        <<"queues">> => #{
+                            <<"backend">> => <<"builtin_raft">>
                         }
                     }
                 }
             }},
+            emqx,
             emqx_ds_shared_sub,
             emqx_management,
             emqx_mgmt_api_test_util:emqx_dashboard()
@@ -135,6 +139,6 @@ api(Method, Path, Data) ->
     end.
 
 terminate_leaders() ->
-    ok = supervisor:terminate_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup),
-    {ok, _} = supervisor:restart_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_leader_sup),
+    ok = supervisor:terminate_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_registry),
+    {ok, _} = supervisor:restart_child(emqx_ds_shared_sub_sup, emqx_ds_shared_sub_registry),
     ok.

apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_config_SUITE.erl

@@ -18,8 +18,7 @@ all() ->
 init_per_suite(Config) ->
     Apps = emqx_cth_suite:start(
         [
-            emqx_conf,
-            {emqx, #{
+            {emqx_conf, #{
                 config => #{
                     <<"durable_sessions">> => #{
                         <<"enable">> => true,
@@ -28,6 +27,9 @@ init_per_suite(Config) ->
                     <<"durable_storage">> => #{
                         <<"messages">> => #{
                             <<"backend">> => <<"builtin_raft">>
+                        },
+                        <<"queues">> => #{
+                            <<"backend">> => <<"builtin_raft">>
                         }
                     }
                 }
@@ -39,7 +41,8 @@ init_per_suite(Config) ->
                         <<"session_find_leader_timeout_ms">> => "1200ms"
                     }
                 }
-            }}
+            }},
+            emqx
         ],
         #{work_dir => ?config(priv_dir, Config)}
     ),

apps/emqx_ds_shared_sub/test/emqx_ds_shared_sub_mgmt_api_subscription_SUITE.erl

@@ -18,11 +18,11 @@ all() -> emqx_common_test_helpers:all(?MODULE).
 init_per_suite(Config) ->
     Apps = emqx_cth_suite:start(
         [
-            {emqx,
-                "durable_sessions {\n"
-                "    enable = true\n"
-                "    renew_streams_interval = 10ms\n"
-                "}"},
+            {emqx_conf,
+                "durable_sessions {"
+                "\n     enable = true"
+                "\n     renew_streams_interval = 10ms"
+                "\n }"},
             {emqx_ds_shared_sub, #{
                 config => #{
                     <<"durable_queues">> => #{
@@ -31,6 +31,7 @@ init_per_suite(Config) ->
                     }
                 }
             }},
+            emqx,
             emqx_management,
             emqx_mgmt_api_test_util:emqx_dashboard()
         ],

apps/emqx_durable_storage/src/emqx_ds.erl

@@ -55,6 +55,7 @@
     topic_filter/0,
     topic/0,
     batch/0,
+    dsbatch/0,
     operation/0,
     deletion/0,
     precondition/0,
@@ -104,7 +105,9 @@
 -type message_matcher(Payload) :: #message_matcher{payload :: Payload}.
 
 %% A batch of storage operations.
--type batch() :: [operation()] | #dsbatch{}.
+-type batch() :: [operation()] | dsbatch().
+
+-type dsbatch() :: #dsbatch{}.
 
 -type operation() ::
     %% Store a message.

apps/emqx_durable_storage/src/emqx_ds_buffer.erl

@@ -192,7 +192,8 @@ handle_info(?flush, S) ->
 handle_info(_Info, S) ->
     {noreply, S}.
 
-terminate(_Reason, #s{db = DB}) ->
+terminate(_Reason, S = #s{db = DB}) ->
+    _ = flush(S),
     persistent_term:erase(?cbm(DB)),
     ok.
 

apps/emqx_durable_storage/src/emqx_ds_lts.erl

@@ -31,6 +31,8 @@
     info/2,
     info/1,
 
+    threshold_fun/1,
+
     compress_topic/3,
     decompress_topic/2
 ]).
@@ -44,7 +46,9 @@
     static_key/0,
     trie/0,
     msg_storage_key/0,
-    learned_structure/0
+    learned_structure/0,
+    threshold_spec/0,
+    threshold_fun/0
 ]).
 
 -include_lib("stdlib/include/ms_transform.hrl").
@@ -83,6 +87,12 @@
 
 -type msg_storage_key() :: {static_key(), varying()}.
 
+-type threshold_spec() ::
+    %% Simple spec that maps level (depth) to a threshold.
+    %% For example, `{simple, {inf, 20}}` means that 0th level has infinite
+    %% threshold while all other levels' threshold is 20.
+    {simple, tuple()}.
+
 -type threshold_fun() :: fun((non_neg_integer()) -> non_neg_integer()).
 
 -type persist_callback() :: fun((_Key, _Val) -> ok).
@@ -313,6 +323,13 @@ info(Trie) ->
         {topics, info(Trie, topics)}
     ].
 
+-spec threshold_fun(threshold_spec()) -> threshold_fun().
+threshold_fun({simple, Thresholds}) ->
+    S = tuple_size(Thresholds),
+    fun(Depth) ->
+        element(min(Depth + 1, S), Thresholds)
+    end.
+
 %%%%%%%% Topic compression %%%%%%%%%%
 
 %% @doc Given topic structure for the static LTS index (as returned by

apps/emqx_durable_storage/src/emqx_ds_storage_bitfield_lts.erl

@@ -81,7 +81,8 @@
     #{
         bits_per_wildcard_level => pos_integer(),
         topic_index_bytes => pos_integer(),
-        epoch_bits => non_neg_integer()
+        epoch_bits => non_neg_integer(),
+        lts_threshold_spec => emqx_ds_lts:threshold_spec()
     }.
 
 %% Permanent state:
@@ -90,7 +91,8 @@
         bits_per_wildcard_level := pos_integer(),
         topic_index_bytes := pos_integer(),
         ts_bits := non_neg_integer(),
-        ts_offset_bits := non_neg_integer()
+        ts_offset_bits := non_neg_integer(),
+        lts_threshold_spec => emqx_ds_lts:threshold_spec()
     }.
 
 %% Runtime state:
@@ -102,6 +104,7 @@
     keymappers :: array:array(emqx_ds_bitmask_keymapper:keymapper()),
     ts_bits :: non_neg_integer(),
     ts_offset :: non_neg_integer(),
+    threshold_fun :: emqx_ds_lts:threshold_fun(),
     gvars :: ets:table()
 }).
 
@@ -141,6 +144,9 @@
 %% Limit on the number of wildcard levels in the learned topic trie:
 -define(WILDCARD_LIMIT, 10).
 
+%% Default LTS thresholds: 0th level = 100 entries max, other levels = 20 entries.
+-define(DEFAULT_LTS_THRESHOLD, {simple, {100, 20}}).
+
 %% Persistent (durable) term representing `#message{}' record. Must
 %% not change.
 -type value_v1() ::
@@ -195,6 +201,7 @@ create(_ShardId, DBHandle, GenId, Options, SPrev) ->
     TopicIndexBytes = maps:get(topic_index_bytes, Options, 4),
     %% 20 bits -> 1048576 us -> ~1 sec
     TSOffsetBits = maps:get(epoch_bits, Options, 20),
+    ThresholdSpec = maps:get(lts_threshold_spec, Options, ?DEFAULT_LTS_THRESHOLD),
     %% Create column families:
     DataCFName = data_cf(GenId),
     TrieCFName = trie_cf(GenId),
@@ -213,7 +220,8 @@ create(_ShardId, DBHandle, GenId, Options, SPrev) ->
         bits_per_wildcard_level => BitsPerTopicLevel,
         topic_index_bytes => TopicIndexBytes,
         ts_bits => 64,
-        ts_offset_bits => TSOffsetBits
+        ts_offset_bits => TSOffsetBits,
+        lts_threshold_spec => ThresholdSpec
     },
     {Schema, [{DataCFName, DataCFHandle}, {TrieCFName, TrieCFHandle}]}.
 
@@ -245,6 +253,7 @@ open(_Shard, DBHandle, GenId, CFRefs, Schema) ->
          || N <- lists:seq(0, MaxWildcardLevels)
         ]
     ),
+    ThresholdSpec = maps:get(lts_threshold_spec, Schema, ?DEFAULT_LTS_THRESHOLD),
     #s{
         db = DBHandle,
         data = DataCF,
@@ -253,6 +262,7 @@ open(_Shard, DBHandle, GenId, CFRefs, Schema) ->
         keymappers = KeymapperCache,
         ts_offset = TSOffsetBits,
         ts_bits = TSBits,
+        threshold_fun = emqx_ds_lts:threshold_fun(ThresholdSpec),
         gvars = ets:new(?MODULE, [public, set, {read_concurrency, true}])
     }.
 
@@ -841,9 +851,9 @@ format_key(KeyMapper, Key) ->
     lists:flatten(io_lib:format("~.16B (~s)", [Key, string:join(Vec, ",")])).
 
 -spec make_key(s(), emqx_ds:time(), emqx_types:topic()) -> {binary(), [binary()]}.
-make_key(#s{keymappers = KeyMappers, trie = Trie}, Timestamp, Topic) ->
+make_key(#s{keymappers = KeyMappers, trie = Trie, threshold_fun = TFun}, Timestamp, Topic) ->
     Tokens = emqx_topic:words(Topic),
-    {TopicIndex, Varying} = emqx_ds_lts:topic_key(Trie, fun threshold_fun/1, Tokens),
+    {TopicIndex, Varying} = emqx_ds_lts:topic_key(Trie, TFun, Tokens),
     VaryingHashes = [hash_topic_level(I) || I <- Varying],
     KeyMapper = array:get(length(Varying), KeyMappers),
     KeyBin = make_key(KeyMapper, TopicIndex, Timestamp, VaryingHashes),
@@ -861,12 +871,6 @@ make_key(KeyMapper, TopicIndex, Timestamp, Varying) ->
         ])
     ).
 
-%% TODO: don't hardcode the thresholds
-threshold_fun(0) ->
-    100;
-threshold_fun(_) ->
-    20.
-
 hash_topic_level('') ->
     hash_topic_level(<<>>);
 hash_topic_level(TopicLevel) ->

apps/emqx_durable_storage/src/emqx_ds_storage_skipstream_lts.erl

@@ -87,9 +87,13 @@
         topic_index_bytes := pos_integer(),
         keep_message_id := boolean(),
         serialization_schema := emqx_ds_msg_serializer:schema(),
-        with_guid := boolean()
+        with_guid := boolean(),
+        lts_threshold_spec => emqx_ds_lts:threshold_spec()
     }.
 
+%% Default LTS thresholds: 0th level = 100 entries max, other levels = 10 entries.
+-define(DEFAULT_LTS_THRESHOLD, {simple, {100, 10}}).
+
 %% Runtime state:
 -record(s, {
     db :: rocksdb:db_handle(),
@@ -98,6 +102,7 @@
     trie_cf :: rocksdb:cf_handle(),
     serialization_schema :: emqx_ds_msg_serializer:schema(),
     hash_bytes :: pos_integer(),
+    threshold_fun :: emqx_ds_lts:threshold_fun(),
     with_guid :: boolean()
 }).
 
@@ -136,7 +141,8 @@ create(_ShardId, DBHandle, GenId, Schema0, SPrev) ->
         wildcard_hash_bytes => 8,
         topic_index_bytes => 8,
         serialization_schema => asn1,
-        with_guid => false
+        with_guid => false,
+        lts_threshold_spec => ?DEFAULT_LTS_THRESHOLD
     },
     Schema = maps:merge(Defaults, Schema0),
     ok = emqx_ds_msg_serializer:check_schema(maps:get(serialization_schema, Schema)),
@@ -154,15 +160,22 @@ create(_ShardId, DBHandle, GenId, Schema0, SPrev) ->
     end,
     {Schema, [{DataCFName, DataCFHandle}, {TrieCFName, TrieCFHandle}]}.
 
-open(_Shard, DBHandle, GenId, CFRefs, #{
-    topic_index_bytes := TIBytes,
-    wildcard_hash_bytes := WCBytes,
-    serialization_schema := SSchema,
-    with_guid := WithGuid
-}) ->
+open(
+    _Shard,
+    DBHandle,
+    GenId,
+    CFRefs,
+    Schema = #{
+        topic_index_bytes := TIBytes,
+        wildcard_hash_bytes := WCBytes,
+        serialization_schema := SSchema,
+        with_guid := WithGuid
+    }
+) ->
     {_, DataCF} = lists:keyfind(data_cf(GenId), 1, CFRefs),
     {_, TrieCF} = lists:keyfind(trie_cf(GenId), 1, CFRefs),
     Trie = restore_trie(TIBytes, DBHandle, TrieCF),
+    ThresholdSpec = maps:get(lts_threshold_spec, Schema, ?DEFAULT_LTS_THRESHOLD),
     #s{
         db = DBHandle,
         data_cf = DataCF,
@@ -170,6 +183,7 @@ open(_Shard, DBHandle, GenId, CFRefs, #{
         trie = Trie,
         hash_bytes = WCBytes,
         serialization_schema = SSchema,
+        threshold_fun = emqx_ds_lts:threshold_fun(ThresholdSpec),
         with_guid = WithGuid
     }.
 
@@ -181,7 +195,7 @@ drop(_ShardId, DBHandle, _GenId, _CFRefs, #s{data_cf = DataCF, trie_cf = TrieCF,
 
 prepare_batch(
     _ShardId,
-    S = #s{trie = Trie},
+    S = #s{trie = Trie, threshold_fun = TFun},
     Operations,
     _Options
 ) ->
@@ -190,7 +204,7 @@ prepare_batch(
         fun
             ({Timestamp, Msg = #message{topic = Topic}}) ->
                 Tokens = words(Topic),
-                {Static, Varying} = emqx_ds_lts:topic_key(Trie, fun threshold_fun/1, Tokens),
+                {Static, Varying} = emqx_ds_lts:topic_key(Trie, TFun, Tokens),
                 ?cooked_msg_op(Timestamp, Static, Varying, serialize(S, Varying, Msg));
             ({delete, #message_matcher{topic = Topic, timestamp = Timestamp}}) ->
                 case emqx_ds_lts:lookup_topic_key(Trie, words(Topic)) of
@@ -692,12 +706,6 @@ hash(HashBytes, TopicLevel) ->
 
 %%%%%%%% LTS %%%%%%%%%%
 
-%% TODO: don't hardcode the thresholds
-threshold_fun(0) ->
-    100;
-threshold_fun(_) ->
-    10.
-
 -spec restore_trie(pos_integer(), rocksdb:db_handle(), rocksdb:cf_handle()) -> emqx_ds_lts:trie().
 restore_trie(StaticIdxBytes, DB, CF) ->
     PersistCallback = fun(Key, Val) ->

apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl

@@ -79,7 +79,8 @@ t_audit_log_conf(_Config) ->
         <<"rotation_size">> => <<"50MB">>,
         <<"time_offset">> => <<"system">>,
         <<"path">> => <<"log/emqx.log">>,
-        <<"timestamp_format">> => <<"auto">>
+        <<"timestamp_format">> => <<"auto">>,
+        <<"payload_encode">> => <<"text">>
     },
     ExpectLog1 = #{
         <<"console">> =>
@@ -88,10 +89,13 @@ t_audit_log_conf(_Config) ->
                 <<"formatter">> => <<"text">>,
                 <<"level">> => <<"warning">>,
                 <<"time_offset">> => <<"system">>,
-                <<"timestamp_format">> => <<"auto">>
+                <<"timestamp_format">> => <<"auto">>,
+                <<"payload_encode">> => <<"text">>
             },
         <<"file">> =>
-            #{<<"default">> => FileExpect},
+            #{
+                <<"default">> => FileExpect
+            },
         <<"audit">> =>
             #{
                 <<"enable">> => false,
@@ -102,7 +106,8 @@ t_audit_log_conf(_Config) ->
                 <<"rotation_count">> => 10,
                 <<"rotation_size">> => <<"50MB">>,
                 <<"time_offset">> => <<"system">>,
-                <<"timestamp_format">> => <<"auto">>
+                <<"timestamp_format">> => <<"auto">>,
+                <<"payload_encode">> => <<"text">>
             }
     },
     %% The default value of throttling.msgs can be frequently updated,

apps/emqx_exhook/src/emqx_exhook_mgr.erl

@@ -658,7 +658,7 @@ hooks(Name) ->
 
 maybe_write_certs(#{<<"name">> := Name} = Conf) ->
     case
-        emqx_tls_lib:ensure_ssl_files(
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
             ssl_file_path(Name), maps:get(<<"ssl">>, Conf, undefined)
         )
     of

apps/emqx_gateway/src/emqx_gateway_conf.erl

@@ -887,7 +887,7 @@ convert_certs(SubDir, Conf) ->
 
 convert_certs(Type, SubDir, Conf) ->
     SSL = maps:get(Type, Conf, undefined),
-    case is_map(SSL) andalso emqx_tls_lib:ensure_ssl_files(SubDir, SSL) of
+    case is_map(SSL) andalso emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(SubDir, SSL) of
         false ->
             Conf;
         {ok, NSSL = #{}} ->

apps/emqx_gateway_coap/include/emqx_coap.hrl

@@ -91,4 +91,10 @@
     {<<"r">>, <<"retain">>}
 ]).
 
+-type sub_data() :: #{
+    topic := emqx_types:topic(),
+    token := binary(),
+    subopts := emqx_types:subopts()
+}.
+
 -endif.

apps/emqx_gateway_coap/src/emqx_coap_channel.erl

@@ -264,7 +264,7 @@ handle_call(
         [ClientInfo, MountedTopic, NSubOpts]
     ),
     %% modify session state
-    SubReq = {Topic, Token},
+    SubReq = #{topic => Topic, token => Token, subopts => NSubOpts},
     TempMsg = #coap_message{type = non},
     %% FIXME: The subopts is not used for emqx_coap_session
     Result = emqx_coap_session:process_subscribe(
@@ -438,14 +438,13 @@ check_token(
             <<"token">> := Token
         } ->
             call_session(handle_request, Msg, Channel);
-        Any ->
+        _ ->
             %% This channel is create by this DELETE command, so here can safely close this channel
             case Token =:= undefined andalso is_delete_connection_request(Msg) of
                 true ->
                     Reply = emqx_coap_message:piggyback({ok, deleted}, Msg),
                     {shutdown, normal, Reply, Channel};
                 false ->
-                    io:format(">>> C1:~p, T1:~p~nC2:~p~n", [ClientId, Token, Any]),
                     ErrMsg = <<"Missing token or clientid in connection mode">>,
                     Reply = emqx_coap_message:piggyback({error, bad_request}, ErrMsg, Msg),
                     {ok, {outgoing, Reply}, Channel}

apps/emqx_gateway_coap/src/emqx_coap_observe_res.erl

@@ -16,10 +16,12 @@
 
 -module(emqx_coap_observe_res).
 
+-include("emqx_coap.hrl").
+
 %% API
 -export([
     new_manager/0,
-    insert/3,
+    insert/2,
     remove/2,
     res_changed/2,
     foreach/2,
@@ -34,7 +36,8 @@
 
 -type res() :: #{
     token := token(),
-    seq_id := seq_id()
+    seq_id := seq_id(),
+    subopts := emqx_types:subopts()
 }.
 
 -type manager() :: #{emqx_types:topic() => res()}.
@@ -46,12 +49,12 @@
 new_manager() ->
     #{}.
 
--spec insert(emqx_types:topic(), token(), manager()) -> {seq_id(), manager()}.
-insert(Topic, Token, Manager) ->
+-spec insert(sub_data(), manager()) -> {seq_id(), manager()}.
+insert(#{topic := Topic, token := Token, subopts := SubOpts}, Manager) ->
     Res =
         case maps:get(Topic, Manager, undefined) of
             undefined ->
-                new_res(Token);
+                new_res(Token, SubOpts);
             Any ->
                 Any
         end,
@@ -84,18 +87,24 @@ foreach(F, Manager) ->
     ),
     ok.
 
--spec subscriptions(manager()) -> [emqx_types:topic()].
+-spec subscriptions(manager()) -> _.
 subscriptions(Manager) ->
-    maps:keys(Manager).
+    maps:map(
+        fun(_Topic, #{subopts := SubOpts}) ->
+            SubOpts
+        end,
+        Manager
+    ).
 
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------
--spec new_res(token()) -> res().
-new_res(Token) ->
+-spec new_res(token(), emqx_types:subopts()) -> res().
+new_res(Token, SubOpts) ->
     #{
         token => Token,
-        seq_id => 0
+        seq_id => 0,
+        subopts => SubOpts
     }.
 
 -spec res_changed(res()) -> res().

apps/emqx_gateway_coap/src/emqx_coap_pubsub_handler.erl

@@ -28,7 +28,16 @@
 -import(emqx_coap_channel, [run_hooks/3]).
 
 -define(UNSUB(Topic, Msg), #{subscribe => {Topic, Msg}}).
--define(SUB(Topic, Token, Msg), #{subscribe => {{Topic, Token}, Msg}}).
+-define(SUB(Topic, Token, Opts, Msg), #{
+    subscribe => {
+        #{
+            topic => Topic,
+            token => Token,
+            subopts => Opts
+        },
+        Msg
+    }
+}).
 -define(SUBOPTS, #{qos => 0, rh => 1, rap => 0, nl => 0, is_new => false}).
 
 %% TODO maybe can merge this code into emqx_coap_session, simplify the call chain
@@ -172,7 +181,7 @@ subscribe(#coap_message{token = Token} = Msg, Topic, Ctx, CInfo) ->
             MountTopic = mount(CInfo, Topic),
             emqx_broker:subscribe(MountTopic, ClientId, SubOpts),
             run_hooks(Ctx, 'session.subscribed', [CInfo, MountTopic, SubOpts]),
-            ?SUB(MountTopic, Token, Msg);
+            ?SUB(MountTopic, Token, SubOpts, Msg);
         _ ->
             reply({error, unauthorized}, Msg)
     end.

apps/emqx_gateway_coap/src/emqx_coap_session.erl

@@ -100,14 +100,9 @@ info(Session) ->
 info(Keys, Session) when is_list(Keys) ->
     [{Key, info(Key, Session)} || Key <- Keys];
 info(subscriptions, #session{observe_manager = OM}) ->
-    Topics = emqx_coap_observe_res:subscriptions(OM),
-    lists:foldl(
-        fun(T, Acc) -> Acc#{T => emqx_gateway_utils:default_subopts()} end,
-        #{},
-        Topics
-    );
+    emqx_coap_observe_res:subscriptions(OM);
 info(subscriptions_cnt, #session{observe_manager = OM}) ->
-    erlang:length(emqx_coap_observe_res:subscriptions(OM));
+    maps:size(emqx_coap_observe_res:subscriptions(OM));
 info(subscriptions_max, _) ->
     infinity;
 info(upgrade_qos, _) ->
@@ -229,8 +224,10 @@ process_subscribe(
     case Sub of
         undefined ->
             Result;
-        {Topic, Token} ->
-            {SeqId, OM2} = emqx_coap_observe_res:insert(Topic, Token, OM),
+        #{
+            topic := _Topic
+        } = SubData ->
+            {SeqId, OM2} = emqx_coap_observe_res:insert(SubData, OM),
             Replay = emqx_coap_message:piggyback({ok, content}, Msg),
             Replay2 = Replay#coap_message{options = #{observe => SeqId}},
             Result#{

apps/emqx_management/src/emqx_management.app.src

@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.2.4"},
+    {vsn, "5.3.0"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [

apps/emqx_management/src/emqx_mgmt_api_clients.erl

@@ -1013,13 +1013,9 @@ format_results(Acc, Cursor) ->
     Meta =
         case Cursor of
             done ->
-                #{
-                    hasnext => false,
-                    count => N
-                };
+                #{count => N};
             _ ->
                 #{
-                    hasnext => true,
                     count => N,
                     cursor => serialize_cursor(Cursor)
                 }
@@ -1043,7 +1039,8 @@ do_ets_select(Nodes, QString0, #{node := Node, node_idx := NodeIdx, cont := Cont
     {Rows, next_ets_cursor(Nodes, NewNodeIdx, NewCont)}.
 
 maybe_run_fuzzy_filter(Rows, QString0) ->
-    {_, {_, FuzzyQString}} = emqx_mgmt_api:parse_qstring(QString0, ?CLIENT_QSCHEMA),
+    {_NClauses, {QString1, FuzzyQString1}} = emqx_mgmt_api:parse_qstring(QString0, ?CLIENT_QSCHEMA),
+    {_QString, FuzzyQString} = adapt_custom_filters(QString1, FuzzyQString1),
     FuzzyFilterFn = fuzzy_filter_fun(FuzzyQString),
     case FuzzyFilterFn of
         undefined ->
@@ -1055,6 +1052,27 @@ maybe_run_fuzzy_filter(Rows, QString0) ->
             )
     end.
 
+%% These filters, while they are able to be adapted to efficient ETS match specs, must be
+%% used as fuzzy filters when iterating over offlient persistent clients, which live
+%% outside ETS.
+adapt_custom_filters(Qs, Fuzzy) ->
+    lists:foldl(
+        fun
+            ({Field, '=:=', X}, {QsAcc, FuzzyAcc}) when
+                Field =:= username
+            ->
+                Xs = wrap(X),
+                {QsAcc, [{Field, in, Xs} | FuzzyAcc]};
+            (Clause, {QsAcc, FuzzyAcc}) ->
+                {[Clause | QsAcc], FuzzyAcc}
+        end,
+        {[], Fuzzy},
+        Qs
+    ).
+
+wrap(Xs) when is_list(Xs) -> Xs;
+wrap(X) -> [X].
+
 initial_ets_cursor([Node | _Rest] = _Nodes) ->
     #{
         type => ?CURSOR_TYPE_ETS,
@@ -1611,8 +1629,8 @@ qs2ms(_Tab, {QString, FuzzyQString}) ->
 
 -spec qs2ms(list()) -> ets:match_spec().
 qs2ms(Qs) ->
-    {MtchHead, Conds} = qs2ms(Qs, 2, {#{}, []}),
-    [{{{'$1', '_'}, MtchHead, '_'}, Conds, ['$_']}].
+    {MatchHead, Conds} = qs2ms(Qs, 2, {#{}, []}),
+    [{{{'$1', '_'}, MatchHead, '_'}, Conds, ['$_']}].
 
 qs2ms([], _, {MtchHead, Conds}) ->
     {MtchHead, lists:reverse(Conds)};
@@ -1685,6 +1703,20 @@ run_fuzzy_filter(
     %% Row from DS
     run_fuzzy_filter1(ClientInfo, Key, SubStr) andalso
         run_fuzzy_filter(Row, RestArgs);
+run_fuzzy_filter(
+    Row = {_, #{metadata := #{clientinfo := ClientInfo}}},
+    [{Key, in, Xs} | RestArgs]
+) ->
+    %% Row from DS
+    IsMatch =
+        case maps:find(Key, ClientInfo) of
+            {ok, X} ->
+                lists:member(X, Xs);
+            error ->
+                false
+        end,
+    IsMatch andalso
+        run_fuzzy_filter(Row, RestArgs);
 run_fuzzy_filter(Row = {_, #{clientinfo := ClientInfo}, _}, [{Key, like, SubStr} | RestArgs]) ->
     %% Row from ETS
     run_fuzzy_filter1(ClientInfo, Key, SubStr) andalso

apps/emqx_management/src/emqx_mgmt_api_relup.erl

@@ -20,7 +20,7 @@
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx/include/logger.hrl").
 
--export([get_upgrade_status/0, emqx_relup_upgrade/1]).
+-export([get_upgrade_status/0, emqx_relup_upgrade/0]).
 
 -export([
     api_spec/0,
@@ -431,7 +431,18 @@ validate_name(Name) ->
 '/relup/package'(get, _) ->
     case get_installed_packages() of
         [PluginInfo] ->
-            {200, format_package_info(PluginInfo)};
+            case get_package_info(PluginInfo) of
+                {error, Reason} ->
+                    ?SLOG(error, #{
+                        msg => get_package_info_failed,
+                        reason => Reason,
+                        details => <<"the corrupted plugin will be deleted">>
+                    }),
+                    delete_installed_packages(),
+                    return_internal_error(Reason);
+                Info when is_map(Info) ->
+                    {200, Info}
+            end;
         [] ->
             return_not_found(<<"No relup package is installed">>)
     end;
@@ -474,24 +485,18 @@ validate_name(Name) ->
 
 '/relup/upgrade'(post, _) ->
     ?ASSERT_PKG_READY(
-        upgrade_with_targe_vsn(fun(TargetVsn) ->
-            run_upgrade_on_nodes(emqx:running_nodes(), TargetVsn)
-        end)
+        run_upgrade_on_nodes(emqx:running_nodes())
     ).
 
 '/relup/upgrade/:node'(post, #{bindings := #{node := NodeNameStr}}) ->
     ?ASSERT_PKG_READY(
-        upgrade_with_targe_vsn(
-            fun(TargetVsn) ->
-                emqx_utils_api:with_node(
-                    NodeNameStr,
-                    fun
-                        (Node) when node() =:= Node ->
-                            run_upgrade(TargetVsn);
-                        (Node) when is_atom(Node) ->
-                            run_upgrade_on_nodes([Node], TargetVsn)
-                    end
-                )
+        emqx_utils_api:with_node(
+            NodeNameStr,
+            fun
+                (Node) when node() =:= Node ->
+                    run_upgrade();
+                (Node) when is_atom(Node) ->
+                    run_upgrade_on_nodes([Node])
             end
         )
     ).
@@ -541,18 +546,8 @@ call_emqx_relup_main(Fun, Args, Default) ->
             Default
     end.
 
-upgrade_with_targe_vsn(Fun) ->
-    case get_target_vsn() of
-        {ok, TargetVsn} ->
-            Fun(TargetVsn);
-        {error, no_relup_package_installed} ->
-            return_package_not_installed();
-        {error, multiple_relup_packages_installed} ->
-            return_internal_error(<<"Multiple relup package installed">>)
-    end.
-
-run_upgrade_on_nodes(Nodes, TargetVsn) ->
-    {[_ | _] = Res, []} = emqx_mgmt_api_relup_proto_v1:run_upgrade(Nodes, TargetVsn),
+run_upgrade_on_nodes(Nodes) ->
+    {[_ | _] = Res, []} = emqx_mgmt_api_relup_proto_v1:run_upgrade(Nodes),
     case lists:filter(fun(R) -> R =/= ok end, Res) of
         [] ->
             {204};
@@ -565,22 +560,15 @@ run_upgrade_on_nodes(Nodes, TargetVsn) ->
             end
     end.
 
-run_upgrade(TargetVsn) ->
-    case emqx_relup_upgrade(TargetVsn) of
+run_upgrade() ->
+    case emqx_relup_upgrade() of
         no_pkg_installed -> return_package_not_installed();
         ok -> {204};
         {error, Reason} -> upgrade_return(Reason)
     end.
 
-emqx_relup_upgrade(TargetVsn) ->
-    call_emqx_relup_main(upgrade, [TargetVsn], no_pkg_installed).
-
-get_target_vsn() ->
-    case get_installed_packages() of
-        [PackageInfo] -> {ok, target_vsn_from_rel_vsn(maps_get(rel_vsn, PackageInfo))};
-        [] -> {error, no_relup_package_installed};
-        _ -> {error, multiple_relup_packages_installed}
-    end.
+emqx_relup_upgrade() ->
+    call_emqx_relup_main(upgrade, [], no_pkg_installed).
 
 get_installed_packages() ->
     lists:filtermap(
@@ -593,12 +581,6 @@ get_installed_packages() ->
         emqx_plugins:list(hidden)
     ).
 
-target_vsn_from_rel_vsn(Vsn) ->
-    case string:split(binary_to_list(Vsn), "-") of
-        [_] -> throw({invalid_vsn, Vsn});
-        [VsnStr | _] -> VsnStr
-    end.
-
 delete_installed_packages() ->
     lists:foreach(
         fun(PackageInfo) ->
@@ -609,14 +591,13 @@ delete_installed_packages() ->
         get_installed_packages()
     ).
 
-format_package_info(PluginInfo) when is_map(PluginInfo) ->
+get_package_info(PluginInfo) when is_map(PluginInfo) ->
     Vsn = maps_get(rel_vsn, PluginInfo),
-    TargetVsn = target_vsn_from_rel_vsn(Vsn),
-    case call_emqx_relup_main(get_package_info, [TargetVsn], no_pkg_installed) of
+    case call_emqx_relup_main(get_package_info, [], no_pkg_installed) of
         no_pkg_installed ->
-            throw({get_pkg_info_failed, <<"No relup package is installed">>});
+            {error, <<"No relup package is installed">>};
         {error, Reason} ->
-            throw({get_pkg_info_failed, Reason});
+            {error, Reason};
         {ok, #{base_vsns := BaseVsns, change_logs := ChangeLogs}} ->
             #{
                 name => name_vsn(?PLUGIN_NAME, Vsn),

apps/emqx_management/src/emqx_mgmt_cli.erl

@@ -97,7 +97,7 @@ broker(_) ->
 %% @doc Cluster with other nodes
 
 cluster(["join", SNode]) ->
-    case mria:join(ekka_node:parse_name(SNode)) of
+    case ekka:join(ekka_node:parse_name(SNode)) of
         ok ->
             emqx_ctl:print("Join the cluster successfully.~n"),
             %% FIXME: running status on the replicant immediately
@@ -112,7 +112,7 @@ cluster(["join", SNode]) ->
     end;
 cluster(["leave"]) ->
     _ = maybe_disable_autocluster(),
-    case mria:leave() of
+    case ekka:leave() of
         ok ->
             emqx_ctl:print("Leave the cluster successfully.~n"),
             cluster(["status"]);
@@ -121,7 +121,7 @@ cluster(["leave"]) ->
     end;
 cluster(["force-leave", SNode]) ->
     Node = ekka_node:parse_name(SNode),
-    case mria:force_leave(Node) of
+    case ekka:force_leave(Node) of
         ok ->
             case emqx_cluster_rpc:force_leave_clean(Node) of
                 ok ->
@@ -674,6 +674,7 @@ listeners([]) ->
     lists:foreach(
         fun({ID, Conf}) ->
             Bind = maps:get(bind, Conf),
+            Enable = maps:get(enable, Conf),
             Acceptors = maps:get(acceptors, Conf),
             ProxyProtocol = maps:get(proxy_protocol, Conf, undefined),
             Running = maps:get(running, Conf),
@@ -704,6 +705,7 @@ listeners([]) ->
                     {listen_on, {string, emqx_listeners:format_bind(Bind)}},
                     {acceptors, Acceptors},
                     {proxy_protocol, ProxyProtocol},
+                    {enbale, Enable},
                     {running, Running}
                 ] ++ CurrentConns ++ MaxConn ++ ShutdownCount,
             emqx_ctl:print("~ts~n", [ID]),
@@ -747,12 +749,37 @@ listeners(["restart", ListenerId]) ->
         _ ->
             emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId])
     end;
+listeners(["enable", ListenerId, Enable0]) ->
+    Enable = Enable0 =:= "true",
+    Action =
+        case Enable of
+            true ->
+                start;
+            _ ->
+                stop
+        end,
+    case emqx_listeners:parse_listener_id(ListenerId) of
+        {ok, #{type := Type, name := Name}} ->
+            RawConf = emqx_mgmt_listeners_conf:get_raw(Type, Name),
+            Conf = RawConf#{<<"enable">> := Enable},
+            case emqx_mgmt_listeners_conf:action(Type, Name, Action, Conf) of
+                {ok, _} ->
+                    emqx_ctl:print("Updated 'enable' to: '~0p' successfully.~n", [Enable]);
+                {error, Reason} ->
+                    emqx_ctl:print("Update listener: ~0p failed, Reason: ~0p~n", [
+                        ListenerId, Reason
+                    ])
+            end;
+        _ ->
+            emqx_ctl:print("Invalid listener: ~0p~n", [ListenerId])
+    end;
 listeners(_) ->
     emqx_ctl:usage([
         {"listeners", "List listeners"},
         {"listeners stop    <Identifier>", "Stop a listener"},
         {"listeners start   <Identifier>", "Start a listener"},
-        {"listeners restart <Identifier>", "Restart a listener"}
+        {"listeners restart <Identifier>", "Restart a listener"},
+        {"listeners enable <Identifier> <Bool>", "Enable or disable a listener"}
     ]).
 
 %%--------------------------------------------------------------------

apps/emqx_management/src/proto/emqx_mgmt_api_relup_proto_v1.erl

@@ -21,7 +21,7 @@
 
 -export([
     introduced_in/0,
-    run_upgrade/2,
+    run_upgrade/1,
     get_upgrade_status_from_all_nodes/0,
     get_upgrade_status/1
 ]).
@@ -32,9 +32,9 @@
 introduced_in() ->
     "5.8.0".
 
--spec run_upgrade([node()], string()) -> emqx_rpc:multicall_result().
-run_upgrade(Nodes, TargetVsn) ->
-    rpc:multicall(Nodes, emqx_mgmt_api_relup, emqx_relup_upgrade, [TargetVsn], ?RPC_TIMEOUT_OP).
+-spec run_upgrade([node()]) -> emqx_rpc:multicall_result().
+run_upgrade(Nodes) ->
+    rpc:multicall(Nodes, emqx_mgmt_api_relup, emqx_relup_upgrade, [], ?RPC_TIMEOUT_OP).
 
 -spec get_upgrade_status_from_all_nodes() -> emqx_rpc:multicall_result().
 get_upgrade_status_from_all_nodes() ->

apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl

@@ -31,6 +31,9 @@
 -define(HTTP400, {"HTTP/1.1", 400, "Bad Request"}).
 -define(HTTP404, {"HTTP/1.1", 404, "Not Found"}).
 
+-define(ON(NODE, BODY), erpc:call(NODE, fun() -> BODY end)).
+-define(assertContainsClientids(RES, EXPECTED), assert_contains_clientids(RES, EXPECTED, ?LINE)).
+
 all() ->
     [
         {group, general},
@@ -65,7 +68,8 @@ persistent_session_testcases() ->
         t_persistent_sessions5,
         t_persistent_sessions6,
         t_persistent_sessions_subscriptions1,
-        t_list_clients_v2
+        t_list_clients_v2,
+        t_list_clients_v2_exact_filters
     ].
 non_persistent_cluster_testcases() ->
     [
@@ -474,8 +478,7 @@ t_persistent_sessions5(Config) ->
                     {?HTTP200, _, #{
                         <<"data">> := [_, _, _],
                         <<"meta">> := #{
-                            <<"count">> := 4,
-                            <<"hasnext">> := true
+                            <<"count">> := 4
                         }
                     }}},
                 P1
@@ -485,8 +488,7 @@ t_persistent_sessions5(Config) ->
                     {?HTTP200, _, #{
                         <<"data">> := [_],
                         <<"meta">> := #{
-                            <<"count">> := 4,
-                            <<"hasnext">> := false
+                            <<"count">> := 4
                         }
                     }}},
                 P2
@@ -502,8 +504,7 @@ t_persistent_sessions5(Config) ->
                     {?HTTP200, _, #{
                         <<"data">> := [_, _],
                         <<"meta">> := #{
-                            <<"count">> := 4,
-                            <<"hasnext">> := true
+                            <<"count">> := 4
                         }
                     }}},
                 list_request(#{limit => 2, page => 1}, Config)
@@ -519,8 +520,7 @@ t_persistent_sessions5(Config) ->
                             {?HTTP200, _, #{
                                 <<"data">> := [_, _, _],
                                 <<"meta">> := #{
-                                    <<"count">> := 4,
-                                    <<"hasnext">> := true
+                                    <<"count">> := 4
                                 }
                             }}},
                         P3_
@@ -535,8 +535,7 @@ t_persistent_sessions5(Config) ->
                             {?HTTP200, _, #{
                                 <<"data">> := [_],
                                 <<"meta">> := #{
-                                    <<"count">> := 4,
-                                    <<"hasnext">> := false
+                                    <<"count">> := 4
                                 }
                             }}},
                         P4_
@@ -1631,7 +1630,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 1,
                                 <<"cursor">> := _
                             }
@@ -1640,7 +1638,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 1,
                                 <<"cursor">> := _
                             }
@@ -1649,7 +1646,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 1,
                                 <<"cursor">> := _
                             }
@@ -1658,7 +1654,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 1,
                                 <<"cursor">> := _
                             }
@@ -1667,7 +1662,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 1,
                                 <<"cursor">> := _
                             }
@@ -1676,14 +1670,13 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := false,
                                 <<"count">> := 1
                             }
                     }
                 ],
                 Res1
             ),
-            assert_contains_clientids(Res1, AllClientIds),
+            ?assertContainsClientids(Res1, AllClientIds),
 
             %% Reusing the same cursors yield the same pages
             traverse_in_reverse_v2(QueryParams1, Res1, Config),
@@ -1697,7 +1690,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_, _, _, _],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 4,
                                 <<"cursor">> := _
                             }
@@ -1706,14 +1698,13 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_, _],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := false,
                                 <<"count">> := 2
                             }
                     }
                 ],
                 Res2
             ),
-            assert_contains_clientids(Res2, AllClientIds),
+            ?assertContainsClientids(Res2, AllClientIds),
             traverse_in_reverse_v2(QueryParams2, Res2, Config),
 
             QueryParams3 = #{limit => "2"},
@@ -1724,7 +1715,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_, _],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 2,
                                 <<"cursor">> := _
                             }
@@ -1733,7 +1723,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_, _],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 2,
                                 <<"cursor">> := _
                             }
@@ -1742,14 +1731,13 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_, _],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := false,
                                 <<"count">> := 2
                             }
                     }
                 ],
                 Res3
             ),
-            assert_contains_clientids(Res3, AllClientIds),
+            ?assertContainsClientids(Res3, AllClientIds),
             traverse_in_reverse_v2(QueryParams3, Res3, Config),
 
             %% fuzzy filters
@@ -1761,14 +1749,13 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_, _, _],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := false,
                                 <<"count">> := 3
                             }
                     }
                 ],
                 Res4
             ),
-            assert_contains_clientids(Res4, [ClientId1, ClientId4, ClientId5]),
+            ?assertContainsClientids(Res4, [ClientId1, ClientId4, ClientId5]),
             traverse_in_reverse_v2(QueryParams4, Res4, Config),
             QueryParams5 = #{limit => "1", like_clientid => "ca"},
             Res5 = list_all_v2(QueryParams5, Config),
@@ -1778,7 +1765,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 1,
                                 <<"cursor">> := _
                             }
@@ -1787,7 +1773,6 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := true,
                                 <<"count">> := 1,
                                 <<"cursor">> := _
                             }
@@ -1796,14 +1781,13 @@ t_list_clients_v2(Config) ->
                         <<"data">> := [_],
                         <<"meta">> :=
                             #{
-                                <<"hasnext">> := false,
                                 <<"count">> := 1
                             }
                     }
                 ],
                 Res5
             ),
-            assert_contains_clientids(Res5, [ClientId1, ClientId4, ClientId5]),
+            ?assertContainsClientids(Res5, [ClientId1, ClientId4, ClientId5]),
             traverse_in_reverse_v2(QueryParams5, Res5, Config),
 
             lists:foreach(
@@ -1845,6 +1829,82 @@ t_list_clients_v2(Config) ->
     ),
     ok.
 
+%% Checks that exact match filters (username) works in clients_v2 API.
+t_list_clients_v2_exact_filters(Config) ->
+    [N1, N2] = ?config(nodes, Config),
+    Port1 = get_mqtt_port(N1, tcp),
+    Port2 = get_mqtt_port(N2, tcp),
+    Id = fun(Bin) -> iolist_to_binary([atom_to_binary(?FUNCTION_NAME), <<"-">>, Bin]) end,
+    ?check_trace(
+        begin
+            ClientId1 = Id(<<"ps1">>),
+            ClientId2 = Id(<<"ps2">>),
+            ClientId3 = Id(<<"ps3-offline">>),
+            ClientId4 = Id(<<"ps4-offline">>),
+            ClientId5 = Id(<<"mem2">>),
+            ClientId6 = Id(<<"mem3">>),
+            Username1 = Id(<<"u1">>),
+            Username2 = Id(<<"u2">>),
+            C1 = connect_client(#{
+                port => Port1,
+                clientid => ClientId1,
+                clean_start => true,
+                username => Username1
+            }),
+            C2 = connect_client(#{
+                port => Port2,
+                clientid => ClientId2,
+                clean_start => true,
+                username => Username2
+            }),
+            C3 = connect_client(#{port => Port1, clientid => ClientId3, clean_start => true}),
+            C4 = connect_client(#{port => Port2, clientid => ClientId4, clean_start => true}),
+            %% in-memory clients
+            C5 = connect_client(#{
+                port => Port1,
+                clientid => ClientId5,
+                expiry => 0,
+                clean_start => true,
+                username => Username1
+            }),
+            C6 = connect_client(#{
+                port => Port2,
+                clientid => ClientId6,
+                expiry => 0,
+                clean_start => true,
+                username => Username2
+            }),
+            %% offline persistent clients
+            lists:foreach(fun stop_and_commit/1, [C3, C4]),
+
+            %% Username query
+            QueryParams1 = [
+                {"limit", "100"},
+                {"username", Username1}
+            ],
+            Res1 = list_all_v2(QueryParams1, Config),
+            ?assertContainsClientids(Res1, [ClientId1, ClientId5]),
+
+            QueryParams2 = [
+                {"limit", "100"},
+                {"username", Username1},
+                {"username", Username2}
+            ],
+            Res2 = list_all_v2(QueryParams2, Config),
+            ?assertContainsClientids(Res2, [ClientId1, ClientId2, ClientId5, ClientId6]),
+
+            C3B = connect_client(#{port => Port1, clientid => ClientId3}),
+            C4B = connect_client(#{port => Port2, clientid => ClientId4}),
+
+            lists:foreach(fun stop_and_commit/1, [C1, C2, C3B, C4B]),
+            lists:foreach(fun emqtt:stop/1, [C5, C6]),
+
+            ok
+        end,
+        []
+    ),
+    ok.
+
 t_cursor_serde_prop(_Config) ->
     ?assert(proper:quickcheck(cursor_serde_prop(), [{numtests, 100}, {to_file, user}])).
 
@@ -1989,18 +2049,18 @@ simplify_result(Res) ->
             {Status, Body}
     end.
 
-list_v2_request(QueryParams = #{}, Config) ->
+list_v2_request(QueryParams, Config) ->
     Path = emqx_mgmt_api_test_util:api_path(["clients_v2"]),
     request(get, Path, [], compose_query_string(QueryParams), Config).
 
-list_all_v2(QueryParams = #{}, Config) ->
+list_all_v2(QueryParams, Config) ->
     do_list_all_v2(QueryParams, Config, _Acc = []).
 
 do_list_all_v2(QueryParams, Config, Acc) ->
     case list_v2_request(QueryParams, Config) of
         {ok, {{_, 200, _}, _, Resp = #{<<"meta">> := #{<<"cursor">> := Cursor}}}} ->
             do_list_all_v2(QueryParams#{cursor => Cursor}, Config, [Resp | Acc]);
-        {ok, {{_, 200, _}, _, Resp = #{<<"meta">> := #{<<"hasnext">> := false}}}} ->
+        {ok, {{_, 200, _}, _, Resp}} ->
             lists:reverse([Resp | Acc]);
         Other ->
             error(
@@ -2018,8 +2078,10 @@ lookup_request(ClientId, Config) ->
 
 compose_query_string(QueryParams = #{}) ->
     QPList = maps:to_list(QueryParams),
+    compose_query_string(QPList);
+compose_query_string([{_, _} | _] = QueryParams) ->
     uri_string:compose_query(
-        [{emqx_utils_conv:bin(K), emqx_utils_conv:bin(V)} || {K, V} <- QPList]
+        [{emqx_utils_conv:bin(K), emqx_utils_conv:bin(V)} || {K, V} <- QueryParams]
     );
 compose_query_string(QueryString) when is_list(QueryString) ->
     QueryString.
@@ -2081,22 +2143,23 @@ connect_client(Opts) ->
         clean_start => false
     },
     #{
-        port := Port,
-        clientid := ClientId,
-        clean_start := CleanStart,
+        port := _Port,
+        clientid := _ClientId,
         expiry := EI
-    } = maps:merge(Defaults, Opts),
-    {ok, C} = emqtt:start_link([
-        {port, Port},
-        {proto_ver, v5},
-        {clientid, ClientId},
-        {clean_start, CleanStart},
-        {properties, #{'Session-Expiry-Interval' => EI}}
-    ]),
+    } = ConnOpts0 = maps:merge(Defaults, Opts),
+    ConnOpts1 = maps:without([expiry], ConnOpts0),
+    ConnOpts = emqx_utils_maps:deep_merge(
+        #{
+            proto_ver => v5,
+            properties => #{'Session-Expiry-Interval' => EI}
+        },
+        ConnOpts1
+    ),
+    {ok, C} = emqtt:start_link(ConnOpts),
     {ok, _} = emqtt:connect(C),
     C.
 
-assert_contains_clientids(Results, ExpectedClientIds) ->
+assert_contains_clientids(Results, ExpectedClientIds, Line) ->
     ContainedClientIds = [
         ClientId
      || #{<<"data">> := Rows} <- Results,
@@ -2105,7 +2168,7 @@ assert_contains_clientids(Results, ExpectedClientIds) ->
     ?assertEqual(
         lists:sort(ExpectedClientIds),
         lists:sort(ContainedClientIds),
-        #{results => Results}
+        #{results => Results, line => Line}
     ).
 
 traverse_in_reverse_v2(QueryParams0, Results, Config) ->

apps/emqx_message_transformation/src/emqx_message_transformation.app.src

@@ -1,6 +1,6 @@
 {application, emqx_message_transformation, [
     {description, "EMQX Message Transformation"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, [emqx_message_transformation_sup, emqx_message_transformation_registry]},
     {mod, {emqx_message_transformation_app, []}},
     {applications, [

apps/emqx_message_transformation/src/emqx_message_transformation.erl

@@ -29,7 +29,7 @@
 ]).
 
 %% Internal exports
--export([run_transformation/2, trace_failure_context_to_map/1]).
+-export([run_transformation/2, trace_failure_context_to_map/1, prettify_operation/1]).
 
 %%------------------------------------------------------------------------------
 %% Type declarations
@@ -173,6 +173,19 @@ run_transformation(Transformation, MessageIn) ->
             {FailureAction, TraceFailureContext}
     end.
 
+prettify_operation(Operation0) ->
+    %% TODO: remove injected bif module
+    Operation = maps:update_with(
+        value,
+        fun(V) -> iolist_to_binary(emqx_variform:decompile(V)) end,
+        Operation0
+    ),
+    maps:update_with(
+        key,
+        fun(Path) -> iolist_to_binary(lists:join(".", Path)) end,
+        Operation
+    ).
+
 %%------------------------------------------------------------------------------
 %% Internal functions
 %%------------------------------------------------------------------------------
@@ -181,17 +194,32 @@ run_transformation(Transformation, MessageIn) ->
     {ok, eval_context()} | {error, trace_failure_context()}.
 eval_operation(Operation, Transformation, Context) ->
     #{key := K, value := V} = Operation,
-    case eval_variform(K, V, Context) of
-        {error, Reason} ->
-            FailureContext = #trace_failure_context{
+    try
+        case eval_variform(K, V, Context) of
+            {error, Reason} ->
+                FailureContext = #trace_failure_context{
+                    transformation = Transformation,
+                    tag = "transformation_eval_operation_failure",
+                    context = #{reason => Reason}
+                },
+                {error, FailureContext};
+            {ok, Rendered} ->
+                NewContext = put_value(K, Rendered, Context),
+                {ok, NewContext}
+        end
+    catch
+        Class:Error:Stacktrace ->
+            FailureContext1 = #trace_failure_context{
                 transformation = Transformation,
-                tag = "transformation_eval_operation_failure",
-                context = #{reason => Reason}
+                tag = "transformation_eval_operation_exception",
+                context = #{
+                    kind => Class,
+                    reason => Error,
+                    stacktrace => Stacktrace,
+                    operation => prettify_operation(Operation)
+                }
             },
-            {error, FailureContext};
-        {ok, Rendered} ->
-            NewContext = put_value(K, Rendered, Context),
-            {ok, NewContext}
+            {error, FailureContext1}
     end.
 
 -spec eval_variform([binary(), ...], _, eval_context()) ->
@@ -280,12 +308,22 @@ run_transformations(Transformations, Message = #message{headers = Headers}) ->
     end.
 
 do_run_transformations(Transformations, Message) ->
+    LastTransformation = #{name := LastTransformationName} = lists:last(Transformations),
     Fun = fun(Transformation, MessageAcc) ->
         #{name := Name} = Transformation,
         emqx_message_transformation_registry:inc_matched(Name),
         case run_transformation(Transformation, MessageAcc) of
             {ok, #message{} = NewAcc} ->
-                emqx_message_transformation_registry:inc_succeeded(Name),
+                %% If this is the last transformation, we can't bump its success counter
+                %% yet.  We perform a check to see if the final payload is encoded as a
+                %% binary after all transformations have run, and it's the last
+                %% transformation's responsibility to properly encode it.
+                case Name =:= LastTransformationName of
+                    true ->
+                        ok;
+                    false ->
+                        emqx_message_transformation_registry:inc_succeeded(Name)
+                end,
                 {cont, NewAcc};
             {ignore, TraceFailureContext} ->
                 trace_failure_from_context(TraceFailureContext),
@@ -307,11 +345,12 @@ do_run_transformations(Transformations, Message) ->
         #message{} = FinalMessage ->
             case is_payload_properly_encoded(FinalMessage) of
                 true ->
+                    emqx_message_transformation_registry:inc_succeeded(LastTransformationName),
                     FinalMessage;
                 false ->
                     %% Take the last validation's failure action, as it's the one
                     %% responsible for getting the right encoding.
-                    LastTransformation = lists:last(Transformations),
+                    emqx_message_transformation_registry:inc_failed(LastTransformationName),
                     #{failure_action := FailureAction} = LastTransformation,
                     trace_failure(LastTransformation, "transformation_bad_encoding", #{
                         action => FailureAction,

apps/emqx_message_transformation/src/emqx_message_transformation_http_api.erl

@@ -715,17 +715,7 @@ transformation_out(Transformation) ->
     ).
 
 operation_out(Operation0) ->
-    %% TODO: remove injected bif module
-    Operation = maps:update_with(
-        value,
-        fun(V) -> iolist_to_binary(emqx_variform:decompile(V)) end,
-        Operation0
-    ),
-    maps:update_with(
-        key,
-        fun(Path) -> iolist_to_binary(lists:join(".", Path)) end,
-        Operation
-    ).
+    emqx_message_transformation:prettify_operation(Operation0).
 
 dryrun_input_message_in(Params) ->
     %% We already check the params against the schema at the API boundary, so we can

apps/emqx_message_transformation/src/emqx_message_transformation_schema.erl

@@ -63,7 +63,10 @@ fields(transformation) ->
     [
         {tags, emqx_schema:tags_schema()},
         {description, emqx_schema:description_schema()},
-        {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
+        {enable,
+            mk(boolean(), #{
+                desc => ?DESC("config_enable"), default => true, importance => ?IMPORTANCE_NO_DOC
+            })},
         {name,
             mk(
                 binary(),

apps/emqx_message_transformation/test/emqx_message_transformation_http_api_SUITE.erl

@@ -1635,23 +1635,114 @@ t_load_config(_Config) ->
 t_final_payload_must_be_binary(_Config) ->
     ?check_trace(
         begin
-            Name = <<"foo">>,
+            Name1 = <<"foo">>,
             Operations = [operation(<<"payload.hello">>, <<"concat(['world'])">>)],
-            Transformation = transformation(Name, Operations, #{
+            Transformation1 = transformation(Name1, Operations, #{
                 <<"payload_decoder">> => #{<<"type">> => <<"json">>},
                 <<"payload_encoder">> => #{<<"type">> => <<"none">>}
             }),
-            {201, _} = insert(Transformation),
+            {201, _} = insert(Transformation1),
 
             C = connect(<<"c1">>),
             {ok, _, [_]} = emqtt:subscribe(C, <<"t/#">>),
             ok = publish(C, <<"t/1">>, #{x => 1, y => true}),
             ?assertNotReceive({publish, _}),
+
+            ?retry(
+                100,
+                10,
+                ?assertMatch(
+                    {200, #{
+                        <<"metrics">> :=
+                            #{
+                                <<"matched">> := 1,
+                                <<"succeeded">> := 0,
+                                <<"failed">> := 1
+                            },
+                        <<"node_metrics">> :=
+                            [
+                                #{
+                                    <<"node">> := _,
+                                    <<"metrics">> := #{
+                                        <<"matched">> := 1,
+                                        <<"succeeded">> := 0,
+                                        <<"failed">> := 1
+                                    }
+                                }
+                            ]
+                    }},
+                    get_metrics(Name1)
+                )
+            ),
+
+            %% When there are multiple transformations for a topic, the last one is
+            %% responsible for properly encoding the payload to a binary.
+            Name2 = <<"bar">>,
+            Transformation2 = transformation(Name2, _Operations = [], #{
+                <<"payload_decoder">> => #{<<"type">> => <<"none">>},
+                <<"payload_encoder">> => #{<<"type">> => <<"none">>}
+            }),
+            {201, _} = insert(Transformation2),
+
+            ok = publish(C, <<"t/1">>, #{x => 1, y => true}),
+            ?assertNotReceive({publish, _}),
+
+            %% The old, first transformation succeeds.
+            ?assertMatch(
+                {200, #{
+                    <<"metrics">> :=
+                        #{
+                            <<"matched">> := 2,
+                            <<"succeeded">> := 1,
+                            <<"failed">> := 1
+                        },
+                    <<"node_metrics">> :=
+                        [
+                            #{
+                                <<"node">> := _,
+                                <<"metrics">> := #{
+                                    <<"matched">> := 2,
+                                    <<"succeeded">> := 1,
+                                    <<"failed">> := 1
+                                }
+                            }
+                        ]
+                }},
+                get_metrics(Name1)
+            ),
+
+            %% The last transformation gets the failure metric bump.
+            ?assertMatch(
+                {200, #{
+                    <<"metrics">> :=
+                        #{
+                            <<"matched">> := 1,
+                            <<"succeeded">> := 0,
+                            <<"failed">> := 1
+                        },
+                    <<"node_metrics">> :=
+                        [
+                            #{
+                                <<"node">> := _,
+                                <<"metrics">> := #{
+                                    <<"matched">> := 1,
+                                    <<"succeeded">> := 0,
+                                    <<"failed">> := 1
+                                }
+                            }
+                        ]
+                }},
+                get_metrics(Name2)
+            ),
+
             ok
         end,
         fun(Trace) ->
             ?assertMatch(
-                [#{message := "transformation_bad_encoding"}],
+                [
+                    #{message := "transformation_bad_encoding"},
+                    #{message := "transformation_bad_encoding"}
+                ],
                 ?of_kind(message_transformation_failed, Trace)
             ),
             ok
@@ -1659,6 +1750,55 @@ t_final_payload_must_be_binary(_Config) ->
     ),
     ok.
 
+%% Checks that an input value that does not respect the declared encoding bumps the
+%% failure metric as expected.  Also, such a crash does not lead to the message continuing
+%% the publication process.
+t_bad_decoded_value_failure_metric(_Config) ->
+    ?check_trace(
+        begin
+            Name = <<"bar">>,
+            Operations = [operation(<<"payload.msg">>, <<"payload">>)],
+            Transformation = transformation(Name, Operations, #{
+                <<"payload_decoder">> => #{<<"type">> => <<"none">>},
+                <<"payload_encoder">> => #{<<"type">> => <<"json">>}
+            }),
+            {201, _} = insert(Transformation),
+            C = connect(<<"c1">>),
+            {ok, _, [_]} = emqtt:subscribe(C, <<"t/#">>),
+            ok = publish(C, <<"t/1">>, {raw, <<"aaa">>}),
+            ?assertNotReceive({publish, _}),
+            ?retry(
+                100,
+                10,
+                ?assertMatch(
+                    {200, #{
+                        <<"metrics">> :=
+                            #{
+                                <<"matched">> := 1,
+                                <<"succeeded">> := 0,
+                                <<"failed">> := 1
+                            },
+                        <<"node_metrics">> :=
+                            [
+                                #{
+                                    <<"node">> := _,
+                                    <<"metrics">> := #{
+                                        <<"matched">> := 1,
+                                        <<"succeeded">> := 0,
+                                        <<"failed">> := 1
+                                    }
+                                }
+                            ]
+                    }},
+                    get_metrics(Name)
+                )
+            ),
+            ok
+        end,
+        []
+    ),
+    ok.
+
 %% Smoke test for the `json_encode' and `json_decode' BIFs.
 t_json_encode_decode_smoke_test(_Config) ->
     ?check_trace(

apps/emqx_opentelemetry/src/emqx_otel_config.erl

@@ -108,7 +108,7 @@ convert_exporter_certs(ExporterConf) ->
     ExporterConf.
 
 do_convert_certs(SSLOpts) ->
-    case emqx_tls_lib:ensure_ssl_files(?CERTS_PATH, SSLOpts) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(?CERTS_PATH, SSLOpts) of
         {ok, undefined} ->
             SSLOpts;
         {ok, SSLOpts1} ->

apps/emqx_plugins/src/emqx_plugins.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_plugins, [
     {description, "EMQX Plugin Management"},
-    {vsn, "0.2.3"},
+    {vsn, "0.3.0"},
     {modules, []},
     {mod, {emqx_plugins_app, []}},
     {applications, [kernel, stdlib, emqx, erlavro]},

apps/emqx_plugins/src/emqx_plugins.erl

@@ -75,7 +75,9 @@
     install_dir/0,
     avsc_file_path/1,
     md5sum_file/1,
-    with_plugin_avsc/1
+    with_plugin_avsc/1,
+    ensure_ssl_files/2,
+    ensure_ssl_files/3
 ]).
 
 %% `emqx_config_handler' API
@@ -514,6 +516,12 @@ get_tar(NameVsn) ->
             end
     end.
 
+ensure_ssl_files(NameVsn, SSL) ->
+    emqx_tls_lib:ensure_ssl_files(plugin_certs_dir(NameVsn), SSL).
+
+ensure_ssl_files(NameVsn, SSL, Opts) ->
+    emqx_tls_lib:ensure_ssl_files(plugin_certs_dir(NameVsn), SSL, Opts).
+
 %%--------------------------------------------------------------------
 %% Internal
 %%--------------------------------------------------------------------
@@ -1290,7 +1298,7 @@ maybe_create_config_dir(NameVsn, Mode) ->
         do_create_config_dir(NameVsn, Mode).
 
 do_create_config_dir(NameVsn, Mode) ->
-    case plugin_config_dir(NameVsn) of
+    case plugin_data_dir(NameVsn) of
         {error, Reason} ->
             {error, {gen_config_dir_failed, Reason}};
         ConfigDir ->
@@ -1332,7 +1340,7 @@ ensure_plugin_config({NameVsn, ?fresh_install}) ->
 -spec ensure_plugin_config(name_vsn(), list()) -> ok.
 ensure_plugin_config(NameVsn, []) ->
     ?SLOG(debug, #{
-        msg => "default_plugin_config_used",
+        msg => "local_plugin_config_used",
         name_vsn => NameVsn,
         reason => "no_other_running_nodes"
     }),
@@ -1358,7 +1366,13 @@ cp_default_config_file(NameVsn) ->
     maybe
         true ?= filelib:is_regular(Source),
         %% destination path not existed (not configured)
-        true ?= (not filelib:is_regular(Destination)),
+        false ?=
+            case filelib:is_regular(Destination) of
+                true ->
+                    ?SLOG(debug, #{msg => "plugin_config_file_already_existed", name_vsn => NameVsn});
+                false ->
+                    false
+            end,
         ok = filelib:ensure_dir(Destination),
         case file:copy(Source, Destination) of
             {ok, _} ->
@@ -1509,8 +1523,8 @@ plugin_priv_dir(NameVsn) ->
         _ -> wrap_to_list(filename:join([install_dir(), NameVsn, "priv"]))
     end.
 
--spec plugin_config_dir(name_vsn()) -> string() | {error, Reason :: string()}.
-plugin_config_dir(NameVsn) ->
+-spec plugin_data_dir(name_vsn()) -> string() | {error, Reason :: string()}.
+plugin_data_dir(NameVsn) ->
     case parse_name_vsn(NameVsn) of
         {ok, NameAtom, _Vsn} ->
             wrap_to_list(filename:join([emqx:data_dir(), "plugins", atom_to_list(NameAtom)]));
@@ -1523,6 +1537,9 @@ plugin_config_dir(NameVsn) ->
             {error, Reason}
     end.
 
+plugin_certs_dir(NameVsn) ->
+    wrap_to_list(filename:join([plugin_data_dir(NameVsn), "certs"])).
+
 %% Files
 -spec pkg_file_path(name_vsn()) -> string().
 pkg_file_path(NameVsn) ->
@@ -1538,7 +1555,7 @@ avsc_file_path(NameVsn) ->
 
 -spec plugin_config_file(name_vsn()) -> string().
 plugin_config_file(NameVsn) ->
-    wrap_to_list(filename:join([plugin_config_dir(NameVsn), "config.hocon"])).
+    wrap_to_list(filename:join([plugin_data_dir(NameVsn), "config.hocon"])).
 
 %% should only used when plugin installing
 -spec default_plugin_config_file(name_vsn()) -> string().

apps/emqx_postgresql/src/emqx_postgresql.app.src

@@ -1,6 +1,6 @@
 {application, emqx_postgresql, [
     {description, "EMQX PostgreSQL Database Connector"},
-    {vsn, "0.2.3"},
+    {vsn, "0.2.4"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_postgresql/src/emqx_postgresql.erl

@@ -339,8 +339,7 @@ on_query(
     {TypeOrKey, NameOrMap, Params},
     #{pool_name := PoolName} = State
 ) ->
-    ?SLOG(debug, #{
-        msg => "postgresql_connector_received_sql_query",
+    ?TRACE("QUERY", "postgresql_connector_received_sql_query", #{
         connector => InstId,
         type => TypeOrKey,
         sql => NameOrMap,

apps/emqx_rule_engine/src/emqx_rule_engine_api.erl

@@ -20,6 +20,7 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("typerefl/include/types.hrl").
+-include_lib("emqx_utils/include/emqx_utils_api.hrl").
 
 -behaviour(minirest_api).
 
@@ -385,16 +386,16 @@ param_path_id() ->
     case maps:get(<<"id">>, Params0, list_to_binary(emqx_utils:gen_id(8))) of
         <<>> ->
             {400, #{code => 'BAD_REQUEST', message => <<"empty rule id is not allowed">>}};
-        Id ->
+        Id when is_binary(Id) ->
             Params = filter_out_request_body(add_metadata(Params0)),
             case emqx_rule_engine:get_rule(Id) of
                 {ok, _Rule} ->
-                    {400, #{code => 'BAD_REQUEST', message => <<"rule id already exists">>}};
+                    ?BAD_REQUEST(<<"rule id already exists">>);
                 not_found ->
                     ConfPath = ?RULE_PATH(Id),
                     case emqx_conf:update(ConfPath, Params, #{override_to => cluster}) of
                         {ok, #{post_config_update := #{emqx_rule_engine := Rule}}} ->
-                            {201, format_rule_info_resp(Rule)};
+                            ?CREATED(format_rule_info_resp(Rule));
                         {error, Reason} ->
                             ?SLOG(
                                 info,
@@ -405,9 +406,11 @@ param_path_id() ->
                                 },
                                 #{tag => ?TAG}
                             ),
-                            {400, #{code => 'BAD_REQUEST', message => ?ERR_BADARGS(Reason)}}
+                            ?BAD_REQUEST(?ERR_BADARGS(Reason))
                     end
-            end
+            end;
+        _BadId ->
+            ?BAD_REQUEST(<<"rule id must be a string">>)
     end.
 
 '/rule_test'(post, #{body := Params}) ->

apps/emqx_rule_engine/src/emqx_rule_events.erl

@@ -334,13 +334,14 @@ eventmsg_publish(
             qos => QoS,
             flags => Flags,
             pub_props => printable_maps(emqx_message:get_header(properties, Message, #{})),
-            publish_received_at => Timestamp
+            publish_received_at => Timestamp,
+            client_attrs => emqx_message:get_header(client_attrs, Message, #{})
         },
         #{headers => Headers}
     ).
 
 eventmsg_connected(
-    _ClientInfo = #{
+    ClientInfo = #{
         clientid := ClientId,
         username := Username,
         is_bridge := IsBridge,
@@ -375,13 +376,14 @@ eventmsg_connected(
             expiry_interval => ExpiryInterval div 1000,
             is_bridge => IsBridge,
             conn_props => printable_maps(ConnProps),
-            connected_at => ConnectedAt
+            connected_at => ConnectedAt,
+            client_attrs => maps:get(client_attrs, ClientInfo, #{})
         },
         #{}
     ).
 
 eventmsg_disconnected(
-    _ClientInfo = #{
+    ClientInfo = #{
         clientid := ClientId,
         username := Username
     },
@@ -405,7 +407,8 @@ eventmsg_disconnected(
             proto_name => ProtoName,
             proto_ver => ProtoVer,
             disconn_props => printable_maps(maps:get(disconn_props, ConnInfo, #{})),
-            disconnected_at => DisconnectedAt
+            disconnected_at => DisconnectedAt,
+            client_attrs => maps:get(client_attrs, ClientInfo, #{})
         },
         #{}
     ).
@@ -444,7 +447,7 @@ eventmsg_connack(
     ).
 
 eventmsg_check_authz_complete(
-    _ClientInfo = #{
+    ClientInfo = #{
         clientid := ClientId,
         username := Username,
         peerhost := PeerHost,
@@ -465,13 +468,14 @@ eventmsg_check_authz_complete(
             topic => Topic,
             action => PubSub,
             authz_source => AuthzSource,
-            result => Result
+            result => Result,
+            client_attrs => maps:get(client_attrs, ClientInfo, #{})
         },
         #{}
     ).
 
 eventmsg_check_authn_complete(
-    _ClientInfo = #{
+    ClientInfo = #{
         clientid := ClientId,
         username := Username,
         peername := PeerName
@@ -493,14 +497,15 @@ eventmsg_check_authn_complete(
             peername => ntoa(PeerName),
             reason_code => force_to_bin(Reason),
             is_anonymous => IsAnonymous,
-            is_superuser => IsSuperuser
+            is_superuser => IsSuperuser,
+            client_attrs => maps:get(client_attrs, ClientInfo, #{})
         },
         #{}
     ).
 
 eventmsg_sub_or_unsub(
     Event,
-    _ClientInfo = #{
+    ClientInfo = #{
         clientid := ClientId,
         username := Username,
         peerhost := PeerHost,
@@ -519,7 +524,8 @@ eventmsg_sub_or_unsub(
             peername => ntoa(PeerName),
             PropKey => printable_maps(maps:get(PropKey, SubOpts, #{})),
             topic => Topic,
-            qos => QoS
+            qos => QoS,
+            client_attrs => maps:get(client_attrs, ClientInfo, #{})
         },
         #{}
     ).
@@ -1028,7 +1034,8 @@ columns_with_exam('message.publish') ->
         {<<"publish_received_at">>, erlang:system_time(millisecond)},
         columns_example_props(pub_props),
         {<<"timestamp">>, erlang:system_time(millisecond)},
-        {<<"node">>, node()}
+        {<<"node">>, node()},
+        columns_example_client_attrs()
     ];
 columns_with_exam('message.delivered') ->
     columns_message_ack_delivered('message.delivered');
@@ -1125,7 +1132,8 @@ columns_with_exam('client.connected') ->
         columns_example_props(conn_props),
         {<<"connected_at">>, erlang:system_time(millisecond)},
         {<<"timestamp">>, erlang:system_time(millisecond)},
-        {<<"node">>, node()}
+        {<<"node">>, node()},
+        columns_example_client_attrs()
     ];
 columns_with_exam('client.disconnected') ->
     [
@@ -1140,7 +1148,8 @@ columns_with_exam('client.disconnected') ->
         columns_example_props(disconn_props),
         {<<"disconnected_at">>, erlang:system_time(millisecond)},
         {<<"timestamp">>, erlang:system_time(millisecond)},
-        {<<"node">>, node()}
+        {<<"node">>, node()},
+        columns_example_client_attrs()
     ];
 columns_with_exam('client.connack') ->
     [
@@ -1172,7 +1181,8 @@ columns_with_exam('client.check_authz_complete') ->
         {<<"authz_source">>, <<"cache">>},
         {<<"result">>, <<"allow">>},
         {<<"timestamp">>, erlang:system_time(millisecond)},
-        {<<"node">>, node()}
+        {<<"node">>, node()},
+        columns_example_client_attrs()
     ];
 columns_with_exam('client.check_authn_complete') ->
     [
@@ -1184,12 +1194,15 @@ columns_with_exam('client.check_authn_complete') ->
         {<<"is_superuser">>, true},
         {<<"is_anonymous">>, false},
         {<<"timestamp">>, erlang:system_time(millisecond)},
-        {<<"node">>, node()}
+        {<<"node">>, node()},
+        columns_example_client_attrs()
     ];
 columns_with_exam('session.subscribed') ->
-    [columns_example_props(sub_props)] ++ columns_message_sub_unsub('session.subscribed');
+    [columns_example_props(sub_props), columns_example_client_attrs()] ++
+        columns_message_sub_unsub('session.subscribed');
 columns_with_exam('session.unsubscribed') ->
-    [columns_example_props(unsub_props)] ++ columns_message_sub_unsub('session.unsubscribed');
+    [columns_example_props(unsub_props), columns_example_client_attrs()] ++
+        columns_message_sub_unsub('session.unsubscribed');
 columns_with_exam(<<"$bridges/mqtt", _/binary>> = EventName) ->
     [
         {<<"event">>, EventName},
@@ -1274,6 +1287,13 @@ columns_example_props_specific(sub_props) ->
 columns_example_props_specific(unsub_props) ->
     #{}.
 
+columns_example_client_attrs() ->
+    {<<"client_attrs">>, #{
+        <<"client_attrs">> => #{
+            <<"test">> => <<"example">>
+        }
+    }}.
+
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------

apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl

@@ -112,7 +112,8 @@ groups() ->
             t_sqlparse_undefined_variable,
             t_sqlparse_new_map,
             t_sqlparse_invalid_json,
-            t_sqlselect_as_put
+            t_sqlselect_as_put,
+            t_sqlselect_client_attr
         ]},
         {events, [], [
             t_events,
@@ -3891,6 +3892,57 @@ t_trace_rule_id(_Config) ->
     ?assertEqual([], emqx_trace_handler:running()),
     emqtt:disconnect(T).
 
+t_sqlselect_client_attr(_) ->
+    ClientId = atom_to_binary(?FUNCTION_NAME),
+    {ok, Compiled} = emqx_variform:compile("user_property.group"),
+    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], [
+        #{
+            expression => Compiled,
+            set_as_attr => <<"group">>
+        },
+        #{
+            expression => Compiled,
+            set_as_attr => <<"group2">>
+        }
+    ]),
+
+    SQL =
+        "SELECT client_attrs as payload FROM \"t/1\" ",
+    Repub = republish_action(<<"t/2">>),
+    {ok, _TopicRule} = emqx_rule_engine:create_rule(
+        #{
+            sql => SQL,
+            id => ?TMP_RULEID,
+            actions => [Repub]
+        }
+    ),
+
+    {ok, Client} = emqtt:start_link([
+        {clientid, ClientId},
+        {proto_ver, v5},
+        {properties, #{'User-Property' => [{<<"group">>, <<"g1">>}]}}
+    ]),
+    {ok, _} = emqtt:connect(Client),
+
+    {ok, _, _} = emqtt:subscribe(Client, <<"t/2">>, 0),
+    ct:sleep(100),
+    emqtt:publish(Client, <<"t/1">>, <<"Hello">>),
+
+    receive
+        {publish, #{topic := Topic, payload := Payload}} ->
+            ?assertEqual(<<"t/2">>, Topic),
+            ?assertMatch(
+                #{<<"group">> := <<"g1">>, <<"group2">> := <<"g1">>},
+                emqx_utils_json:decode(Payload)
+            )
+    after 1000 ->
+        ct:fail(wait_for_t_2)
+    end,
+
+    emqtt:disconnect(Client),
+    emqx_rule_engine:delete_rule(?TMP_RULEID),
+    emqx_config:put_zone_conf(default, [mqtt, client_attrs_init], []).
+
 %%------------------------------------------------------------------------------
 %% Internal helpers
 %%------------------------------------------------------------------------------
@@ -3990,7 +4042,8 @@ verify_event_fields('message.publish', Fields) ->
         flags := Flags,
         pub_props := Properties,
         timestamp := Timestamp,
-        publish_received_at := EventAt
+        publish_received_at := EventAt,
+        client_attrs := ClientAttrs
     } = Fields,
     Now = erlang:system_time(millisecond),
     TimestampElapse = Now - Timestamp,
@@ -4007,7 +4060,8 @@ verify_event_fields('message.publish', Fields) ->
     ?assertMatch(#{'Message-Expiry-Interval' := 60}, Properties),
     ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60 * 1000),
     ?assert(0 =< RcvdAtElapse andalso RcvdAtElapse =< 60 * 1000),
-    ?assert(EventAt =< Timestamp);
+    ?assert(EventAt =< Timestamp),
+    ?assert(is_map(ClientAttrs));
 verify_event_fields('client.connected', Fields) ->
     #{
         clientid := ClientId,
@@ -4023,7 +4077,8 @@ verify_event_fields('client.connected', Fields) ->
         is_bridge := IsBridge,
         conn_props := Properties,
         timestamp := Timestamp,
-        connected_at := EventAt
+        connected_at := EventAt,
+        client_attrs := ClientAttrs
     } = Fields,
     Now = erlang:system_time(millisecond),
     TimestampElapse = Now - Timestamp,
@@ -4042,7 +4097,8 @@ verify_event_fields('client.connected', Fields) ->
     ?assertMatch(#{'Session-Expiry-Interval' := 60}, Properties),
     ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60 * 1000),
     ?assert(0 =< RcvdAtElapse andalso RcvdAtElapse =< 60 * 1000),
-    ?assert(EventAt =< Timestamp);
+    ?assert(EventAt =< Timestamp),
+    ?assert(is_map(ClientAttrs));
 verify_event_fields('client.disconnected', Fields) ->
     #{
         reason := Reason,
@@ -4052,7 +4108,8 @@ verify_event_fields('client.disconnected', Fields) ->
         sockname := SockName,
         disconn_props := Properties,
         timestamp := Timestamp,
-        disconnected_at := EventAt
+        disconnected_at := EventAt,
+        client_attrs := ClientAttrs
     } = Fields,
     Now = erlang:system_time(millisecond),
     TimestampElapse = Now - Timestamp,
@@ -4065,7 +4122,8 @@ verify_event_fields('client.disconnected', Fields) ->
     ?assertMatch(#{'User-Property' := #{<<"reason">> := <<"normal">>}}, Properties),
     ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60 * 1000),
     ?assert(0 =< RcvdAtElapse andalso RcvdAtElapse =< 60 * 1000),
-    ?assert(EventAt =< Timestamp);
+    ?assert(EventAt =< Timestamp),
+    ?assert(is_map(ClientAttrs));
 verify_event_fields(SubUnsub, Fields) when
     SubUnsub == 'session.subscribed';
     SubUnsub == 'session.unsubscribed'
@@ -4077,7 +4135,8 @@ verify_event_fields(SubUnsub, Fields) when
         peername := PeerName,
         topic := Topic,
         qos := QoS,
-        timestamp := Timestamp
+        timestamp := Timestamp,
+        client_attrs := ClientAttrs
     } = Fields,
     Now = erlang:system_time(millisecond),
     TimestampElapse = Now - Timestamp,
@@ -4097,7 +4156,8 @@ verify_event_fields(SubUnsub, Fields) when
         #{'User-Property' := #{<<"topic_name">> := <<"t1">>}},
         maps:get(PropKey, Fields)
     ),
-    ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60 * 1000);
+    ?assert(0 =< TimestampElapse andalso TimestampElapse =< 60 * 1000),
+    ?assert(is_map(ClientAttrs));
 verify_event_fields('delivery.dropped', Fields) ->
     #{
         event := 'delivery.dropped',
@@ -4282,7 +4342,8 @@ verify_event_fields('client.check_authz_complete', Fields) ->
         peername := PeerName,
         topic := Topic,
         authz_source := AuthzSource,
-        username := Username
+        username := Username,
+        client_attrs := ClientAttrs
     } = Fields,
     ?assertEqual(<<"t1">>, Topic),
     ?assert(lists:member(Action, [subscribe, publish])),
@@ -4302,20 +4363,23 @@ verify_event_fields('client.check_authz_complete', Fields) ->
         ])
     ),
     ?assert(lists:member(ClientId, [<<"c_event">>, <<"c_event2">>])),
-    ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>]));
+    ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>])),
+    ?assert(is_map(ClientAttrs));
 verify_event_fields('client.check_authn_complete', Fields) ->
     #{
         clientid := ClientId,
         peername := PeerName,
         username := Username,
         is_anonymous := IsAnonymous,
-        is_superuser := IsSuperuser
+        is_superuser := IsSuperuser,
+        client_attrs := ClientAttrs
     } = Fields,
     verify_peername(PeerName),
     ?assert(lists:member(ClientId, [<<"c_event">>, <<"c_event2">>])),
     ?assert(lists:member(Username, [<<"u_event">>, <<"u_event2">>])),
     ?assert(erlang:is_boolean(IsAnonymous)),
-    ?assert(erlang:is_boolean(IsSuperuser));
+    ?assert(erlang:is_boolean(IsSuperuser)),
+    ?assert(is_map(ClientAttrs));
 verify_event_fields('schema.validation_failed', Fields) ->
     #{
         validation := ValidationName,