Merge branch 'release-57' into 0617-release-57-sync

* release-57:
  chore(auth,http): cache REs for parsing URIs
  fix(auth,http): improve URI handling
  chore: revert ULOG/ELOG
  test: generate dispatch.eterm in dashboard test
  docs: refine change log
  feat: make the dashboard restart quicker
  chore: fix typo
  fix(http authz): handle unknown content types in responses
  chore: change types of mysql and mongodb fields to `template()`
  fix(client mgmt api): allow projecting `client_attrs` from client fields
  fix(emqx_rule_funcs): expose regex_extract function to rule engine

README-CN.md

@@ -1,3 +1,5 @@
+简体中文 | [English](./README.md) | [Русский](./README-RU.md)
+
 # EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)

README-RU.md

@@ -1,3 +1,5 @@
+Русский | [简体中文](./README-CN.md) | [English](./README.md)
+
 # Брокер EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)

README.md

@@ -1,3 +1,5 @@
+English | [简体中文](./README-CN.md) | [Русский](./README-RU.md)
+
 # EMQX
 
 [![GitHub Release](https://img.shields.io/github/release/emqx/emqx?color=brightgreen&label=Release)](https://github.com/emqx/emqx/releases)

apps/emqx/include/emqx_metrics.hrl

@@ -119,12 +119,12 @@
     %% All Messages received
     {counter, 'messages.received', <<
         "Number of messages received from the client, equal to the sum of "
-        "messages.qos0.received\fmessages.qos1.received and messages.qos2.received"
+        "messages.qos0.received, messages.qos1.received and messages.qos2.received"
     >>},
     %% All Messages sent
     {counter, 'messages.sent', <<
         "Number of messages sent to the client, equal to the sum of "
-        "messages.qos0.sent\fmessages.qos1.sent and messages.qos2.sent"
+        "messages.qos0.sent, messages.qos1.sent and messages.qos2.sent"
     >>},
     %% QoS0 Messages received
     {counter, 'messages.qos0.received', <<"Number of QoS 0 messages received from clients">>},

apps/emqx_auth/src/emqx_authz/emqx_authz_utils.erl

@@ -100,7 +100,7 @@ update_config(Path, ConfigRequest) ->
         override_to => cluster
     }).
 
--spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error.
+-spec parse_http_resp_body(binary(), binary()) -> allow | deny | ignore | error | {error, term()}.
 parse_http_resp_body(<<"application/x-www-form-urlencoded", _/binary>>, Body) ->
     try
         result(maps:from_list(cow_qs:parse_qs(Body)))
@@ -112,7 +112,9 @@ parse_http_resp_body(<<"application/json", _/binary>>, Body) ->
         result(emqx_utils_json:decode(Body, [return_maps]))
     catch
         _:_ -> error
-    end.
+    end;
+parse_http_resp_body(ContentType = <<_/binary>>, _Body) ->
+    {error, <<"unsupported content-type: ", ContentType/binary>>}.
 
 result(#{<<"result">> := <<"allow">>}) -> allow;
 result(#{<<"result">> := <<"deny">>}) -> deny;

apps/emqx_auth_http/src/emqx_authz_http.erl

@@ -106,6 +106,9 @@ authorize(
                                 body => Body
                             }),
                             nomatch;
+                        {error, Reason} ->
+                            ?tp(error, bad_authz_http_response, #{reason => Reason}),
+                            nomatch;
                         Result ->
                             {matched, Result}
                     end;

apps/emqx_auth_http/test/emqx_authz_http_SUITE.erl

@@ -463,6 +463,72 @@ t_placeholder_and_body(_Config) ->
         emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
     ).
 
+%% Checks that we don't crash when receiving an unsupported content-type back.
+t_bad_response_content_type(_Config) ->
+    ok = setup_handler_and_config(
+        fun(Req0, State) ->
+            ?assertEqual(
+                <<"/authz/users/">>,
+                cowboy_req:path(Req0)
+            ),
+
+            {ok, _PostVars, Req1} = cowboy_req:read_urlencoded_body(Req0),
+
+            Req = cowboy_req:reply(
+                200,
+                #{<<"content-type">> => <<"text/csv">>},
+                "hi",
+                Req1
+            ),
+            {ok, Req, State}
+        end,
+        #{
+            <<"method">> => <<"post">>,
+            <<"body">> => #{
+                <<"username">> => <<"${username}">>,
+                <<"clientid">> => <<"${clientid}">>,
+                <<"peerhost">> => <<"${peerhost}">>,
+                <<"proto_name">> => <<"${proto_name}">>,
+                <<"mountpoint">> => <<"${mountpoint}">>,
+                <<"topic">> => <<"${topic}">>,
+                <<"action">> => <<"${action}">>,
+                <<"access">> => <<"${access}">>,
+                <<"CN">> => ?PH_CERT_CN_NAME,
+                <<"CS">> => ?PH_CERT_SUBJECT
+            },
+            <<"headers">> => #{
+                <<"accept">> => <<"text/plain">>,
+                <<"content-type">> => <<"application/json">>
+            }
+        }
+    ),
+
+    ClientInfo = #{
+        clientid => <<"client id">>,
+        username => <<"user name">>,
+        peerhost => {127, 0, 0, 1},
+        protocol => <<"MQTT">>,
+        mountpoint => <<"MOUNTPOINT">>,
+        zone => default,
+        listener => {tcp, default},
+        cn => ?PH_CERT_CN_NAME,
+        dn => ?PH_CERT_SUBJECT
+    },
+
+    ?check_trace(
+        ?assertEqual(
+            deny,
+            emqx_access_control:authorize(ClientInfo, ?AUTHZ_PUBLISH, <<"t">>)
+        ),
+        fun(Trace) ->
+            ?assertMatch(
+                [#{reason := <<"unsupported content-type", _/binary>>}],
+                ?of_kind(bad_authz_http_response, Trace)
+            ),
+            ok
+        end
+    ).
+
 t_no_value_for_placeholder(_Config) ->
     ok = setup_handler_and_config(
         fun(Req0, State) ->

apps/emqx_bridge/src/emqx_bridge_v2_api.erl

@@ -32,6 +32,7 @@
 %% Swagger specs from hocon schema
 -export([
     api_spec/0,
+    check_api_schema/2,
     paths/0,
     schema/1,
     namespace/0
@@ -96,7 +97,7 @@
 namespace() -> "actions_and_sources".
 
 api_spec() ->
-    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun check_api_schema/2}).
+    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => fun ?MODULE:check_api_schema/2}).
 
 paths() ->
     [

apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_mongodb, [
     {description, "EMQX Enterprise MongoDB Bridge"},
-    {vsn, "0.3.0"},
+    {vsn, "0.3.1"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.erl

@@ -100,8 +100,10 @@ fields(mongodb_action) ->
     );
 fields(action_parameters) ->
     [
-        {collection, mk(binary(), #{desc => ?DESC("collection"), default => <<"mqtt">>})},
-        {payload_template, mk(binary(), #{required => false, desc => ?DESC("payload_template")})}
+        {collection,
+            mk(emqx_schema:template(), #{desc => ?DESC("collection"), default => <<"mqtt">>})},
+        {payload_template,
+            mk(emqx_schema:template(), #{required => false, desc => ?DESC("payload_template")})}
     ];
 fields(connector_resource_opts) ->
     emqx_connector_schema:resource_opts_fields();

apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src

@@ -1,6 +1,6 @@
 {application, emqx_bridge_mysql, [
     {description, "EMQX Enterprise MySQL Bridge"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {applications, [
         kernel,

apps/emqx_bridge_mysql/src/emqx_bridge_mysql.erl

@@ -146,7 +146,7 @@ fields(action_parameters) ->
     [
         {sql,
             mk(
-                binary(),
+                emqx_schema:template(),
                 #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>}
             )}
     ];

apps/emqx_conf/src/emqx_conf.erl

@@ -144,17 +144,27 @@ reset(Node, KeyPath, Opts) ->
 %% @doc Called from build script.
 %% TODO: move to a external escript after all refactoring is done
 dump_schema(Dir, SchemaModule) ->
-    %% TODO: Load all apps instead of only emqx_dashboard
+    %% Load all apps in ERL_LIBS
     %% as this will help schemas that searches for apps with
     %% relevant schema definitions
-    _ = application:load(emqx_dashboard),
+    lists:foreach(
+        fun(LibPath) ->
+            Lib = list_to_atom(lists:last(filename:split(LibPath))),
+            load(SchemaModule, Lib)
+        end,
+        string:lexemes(os:getenv("ERL_LIBS"), ":;")
+    ),
     ok = emqx_dashboard_desc_cache:init(),
     lists:foreach(
         fun(Lang) ->
             ok = gen_schema_json(Dir, SchemaModule, Lang)
         end,
         ["en", "zh"]
-    ).
+    ),
+    emqx_dashboard:save_dispatch_eterm(SchemaModule).
+
+load(emqx_enterprise_schema, emqx_telemetry) -> ignore;
+load(_, Lib) -> ok = application:load(Lib).
 
 %% for scripts/spellcheck.
 gen_schema_json(Dir, SchemaModule, Lang) ->

apps/emqx_dashboard/src/emqx_dashboard.erl

@@ -28,11 +28,15 @@
 %% Authorization
 -export([authorize/1]).
 
+-export([save_dispatch_eterm/1]).
+
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx/include/http_api.hrl").
 -include_lib("emqx/include/emqx_release.hrl").
+-dialyzer({[no_opaque, no_match, no_return], [init_cache_dispatch/2, start_listeners/1]}).
 
 -define(EMQX_MIDDLE, emqx_dashboard_middleware).
+-define(DISPATCH_FILE, "dispatch.eterm").
 
 %%--------------------------------------------------------------------
 %% Start/Stop Listeners
@@ -46,43 +50,19 @@ stop_listeners() ->
 
 start_listeners(Listeners) ->
     {ok, _} = application:ensure_all_started(minirest),
-    Authorization = {?MODULE, authorize},
-    GlobalSpec = #{
-        openapi => "3.0.0",
-        info => #{title => emqx_api_name(), version => emqx_release_version()},
-        servers => [#{url => emqx_dashboard_swagger:base_path()}],
-        components => #{
-            schemas => #{},
-            'securitySchemes' => #{
-                'basicAuth' => #{
-                    type => http,
-                    scheme => basic,
-                    description =>
-                        <<"Authorize with [API Keys](https://www.emqx.io/docs/en/v5.0/admin/api.html#api-keys)">>
-                },
-                'bearerAuth' => #{
-                    type => http,
-                    scheme => bearer,
-                    description => <<"Authorize with Bearer Token">>
-                }
-            }
-        }
-    },
-    BaseMinirest = #{
-        base_path => emqx_dashboard_swagger:base_path(),
-        modules => minirest_api:find_api_modules(apps()),
-        authorization => Authorization,
-        log => audit_log_fun(),
-        security => [#{'basicAuth' => []}, #{'bearerAuth' => []}],
-        swagger_global_spec => GlobalSpec,
-        dispatch => dispatch(),
-        middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler],
-        swagger_support => emqx:get_config([dashboard, swagger_support], true)
-    },
+    SwaggerSupport = emqx:get_config([dashboard, swagger_support], true),
+    InitDispatch = dispatch(),
     {OkListeners, ErrListeners} =
         lists:foldl(
             fun({Name, Protocol, Bind, RanchOptions, ProtoOpts}, {OkAcc, ErrAcc}) ->
-                Minirest = BaseMinirest#{protocol => Protocol, protocol_options => ProtoOpts},
+                ok = init_cache_dispatch(Name, InitDispatch),
+                Options = #{
+                    dispatch => InitDispatch,
+                    swagger_support => SwaggerSupport,
+                    protocol => Protocol,
+                    protocol_options => ProtoOpts
+                },
+                Minirest = minirest_option(Options),
                 case minirest:start(Name, RanchOptions, Minirest) of
                     {ok, _} ->
                         ?ULOG("Listener ~ts on ~ts started.~n", [
@@ -105,6 +85,57 @@ start_listeners(Listeners) ->
             {error, ErrListeners}
     end.
 
+minirest_option(Options) ->
+    Authorization = {?MODULE, authorize},
+    GlobalSpec = #{
+        openapi => "3.0.0",
+        info => #{title => emqx_api_name(), version => emqx_release_version()},
+        servers => [#{url => emqx_dashboard_swagger:base_path()}],
+        components => #{
+            schemas => #{},
+            'securitySchemes' => #{
+                'basicAuth' => #{
+                    type => http,
+                    scheme => basic,
+                    description =>
+                        <<"Authorize with [API Keys](https://www.emqx.io/docs/en/v5.0/admin/api.html#api-keys)">>
+                },
+                'bearerAuth' => #{
+                    type => http,
+                    scheme => bearer,
+                    description => <<"Authorize with Bearer Token">>
+                }
+            }
+        }
+    },
+    Base =
+        #{
+            base_path => emqx_dashboard_swagger:base_path(),
+            modules => minirest_api:find_api_modules(apps()),
+            authorization => Authorization,
+            log => audit_log_fun(),
+            security => [#{'basicAuth' => []}, #{'bearerAuth' => []}],
+            swagger_global_spec => GlobalSpec,
+            dispatch => static_dispatch(),
+            middlewares => [?EMQX_MIDDLE, cowboy_router, cowboy_handler],
+            swagger_support => true
+        },
+    maps:merge(Base, Options).
+
+%% save dispatch to priv dir.
+save_dispatch_eterm(SchemaMod) ->
+    Dir = code:priv_dir(emqx_dashboard),
+    emqx_config:put([dashboard], #{i18n_lang => en, swagger_support => false}),
+    os:putenv("SCHEMA_MOD", atom_to_list(SchemaMod)),
+    DispatchFile = filename:join([Dir, ?DISPATCH_FILE]),
+    io:format(user, "===< Generating: ~s~n", [DispatchFile]),
+    #{dispatch := Dispatch} = generate_dispatch(),
+    IoData = io_lib:format("~p.~n", [Dispatch]),
+    ok = file:write_file(DispatchFile, IoData),
+    {ok, [SaveDispatch]} = file:consult(DispatchFile),
+    SaveDispatch =/= Dispatch andalso erlang:error("bad dashboard dispatch.eterm file generated"),
+    ok.
+
 stop_listeners(Listeners) ->
     optvar:unset(emqx_dashboard_listeners_ready),
     [
@@ -127,6 +158,34 @@ wait_for_listeners() ->
 
 %%--------------------------------------------------------------------
 %% internal
+%%--------------------------------------------------------------------
+
+init_cache_dispatch(Name, Dispatch0) ->
+    Dispatch1 = [{_, _, Rules}] = trails:single_host_compile(Dispatch0),
+    FileName = filename:join(code:priv_dir(emqx_dashboard), ?DISPATCH_FILE),
+    Dispatch2 =
+        case file:consult(FileName) of
+            {ok, [[{Host, Path, CacheRules}]]} ->
+                Trails = trails:trails([{cowboy_swagger_handler, #{server => 'http:dashboard'}}]),
+                [{_, _, SwaggerRules}] = trails:single_host_compile(Trails),
+                [{Host, Path, CacheRules ++ SwaggerRules ++ Rules}];
+            {error, _} ->
+                Dispatch1
+        end,
+    persistent_term:put(Name, Dispatch2).
+
+generate_dispatch() ->
+    Options = #{
+        dispatch => [],
+        swagger_support => false,
+        protocol => http,
+        protocol_options => proto_opts(#{})
+    },
+    Minirest = minirest_option(Options),
+    minirest:generate_dispatch(Minirest).
+
+dispatch() ->
+    static_dispatch() ++ dynamic_dispatch().
 
 apps() ->
     [
@@ -287,9 +346,6 @@ ensure_ssl_cert(Listeners = #{https := Https0 = #{ssl_options := SslOpts}}) ->
 ensure_ssl_cert(Listeners) ->
     Listeners.
 
-dispatch() ->
-    static_dispatch() ++ dynamic_dispatch().
-
 static_dispatch() ->
     StaticFiles = ["/editor.worker.js", "/json.worker.js", "/version"],
     [

apps/emqx_dashboard/src/emqx_dashboard_app.erl

@@ -25,6 +25,8 @@
 
 -include("emqx_dashboard.hrl").
 
+-dialyzer({nowarn_function, [start/2]}).
+
 start(_StartType, _StartArgs) ->
     Tables = lists:append([
         emqx_dashboard_admin:create_tables(),

apps/emqx_dashboard/src/emqx_dashboard_middleware.erl

@@ -21,28 +21,14 @@
 -export([execute/2]).
 
 execute(Req, Env) ->
-    case check_dispatch_ready(Env) of
-        true -> add_cors_flag(Req, Env);
-        false -> {stop, cowboy_req:reply(503, #{<<"retry-after">> => <<"15">>}, Req)}
-    end.
+    add_cors_flag(Req, Env).
 
 add_cors_flag(Req, Env) ->
     CORS = emqx_conf:get([dashboard, cors], false),
-    Origin = cowboy_req:header(<<"origin">>, Req, undefined),
-    case CORS andalso Origin =/= undefined of
+    case CORS andalso cowboy_req:header(<<"origin">>, Req, undefined) =/= undefined of
         false ->
             {ok, Req, Env};
         true ->
             Req2 = cowboy_req:set_resp_header(<<"Access-Control-Allow-Origin">>, <<"*">>, Req),
             {ok, Req2, Env}
     end.
-
-check_dispatch_ready(Env) ->
-    case maps:is_key(options, Env) of
-        false ->
-            true;
-        true ->
-            %% dashboard should always ready, if not, is_ready/1 will block until ready.
-            %% if not ready, dashboard will return 503.
-            emqx_dashboard_listener:is_ready(timer:seconds(20))
-    end.

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -348,14 +348,7 @@ compose_filters(undefined, Filter2) ->
 compose_filters(Filter1, undefined) ->
     Filter1;
 compose_filters(Filter1, Filter2) ->
-    fun(Request, RequestMeta) ->
-        case Filter1(Request, RequestMeta) of
-            {ok, Request1} ->
-                Filter2(Request1, RequestMeta);
-            Response ->
-                Response
-        end
-    end.
+    [Filter1, Filter2].
 
 %%------------------------------------------------------------------------------
 %% Private functions

apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl

@@ -69,6 +69,9 @@ init_per_suite(Config) ->
         ],
         #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
+    _ = emqx_conf_schema:roots(),
+    ok = emqx_dashboard_desc_cache:init(),
+    emqx_dashboard:save_dispatch_eterm(emqx_conf:schema_module()),
     emqx_common_test_http:create_default_app(),
     [{suite_apps, SuiteApps} | Config].
 
@@ -87,6 +90,79 @@ t_overview(_) ->
      || Overview <- ?OVERVIEWS
     ].
 
+t_dashboard_restart(Config) ->
+    emqx_config:put([dashboard], #{
+        i18n_lang => en,
+        swagger_support => true,
+        listeners =>
+            #{
+                http =>
+                    #{
+                        inet6 => false,
+                        bind => 18083,
+                        ipv6_v6only => false,
+                        send_timeout => 10000,
+                        num_acceptors => 8,
+                        max_connections => 512,
+                        backlog => 1024,
+                        proxy_header => false
+                    }
+            }
+    }),
+    application:stop(emqx_dashboard),
+    application:start(emqx_dashboard),
+    Name = 'http:dashboard',
+    t_overview(Config),
+    [{'_', [], Rules}] = Dispatch = persistent_term:get(Name),
+    %% complete dispatch has more than 150 rules.
+    ?assertNotMatch([{[], [], cowboy_static, _} | _], Rules),
+    ?assert(erlang:length(Rules) > 150),
+    CheckRules = fun(Tag) ->
+        [{'_', [], NewRules}] = persistent_term:get(Name, Tag),
+        ?assertEqual(length(Rules), length(NewRules), Tag),
+        ?assertEqual(lists:sort(Rules), lists:sort(NewRules), Tag)
+    end,
+    ?check_trace(
+        ?wait_async_action(
+            begin
+                ok = application:stop(emqx_dashboard),
+                ?assertEqual(Dispatch, persistent_term:get(Name)),
+                ok = application:start(emqx_dashboard),
+                %% After we restart the dashboard, the dispatch rules should be the same.
+                CheckRules(step_1)
+            end,
+            #{?snk_kind := regenerate_minirest_dispatch},
+            30_000
+        ),
+        fun(Trace) ->
+            ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)),
+            %% The dispatch is updated after being regenerated.
+            CheckRules(step_2)
+        end
+    ),
+    t_overview(Config),
+    ?check_trace(
+        ?wait_async_action(
+            begin
+                %% erase to mock the initial dashboard startup.
+                persistent_term:erase(Name),
+                ok = application:stop(emqx_dashboard),
+                ok = application:start(emqx_dashboard),
+                ct:sleep(800),
+                %% regenerate the dispatch rules again
+                CheckRules(step_3)
+            end,
+            #{?snk_kind := regenerate_minirest_dispatch},
+            30_000
+        ),
+        fun(Trace) ->
+            ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)),
+            CheckRules(step_4)
+        end
+    ),
+    t_overview(Config),
+    ok.
+
 t_admins_add_delete(_) ->
     mnesia:clear_table(?ADMIN),
     Desc = <<"simple description">>,
@@ -196,28 +272,41 @@ t_disable_swagger_json(_Config) ->
         {ok, {{"HTTP/1.1", 200, "OK"}, __, _}},
         httpc:request(get, {Url, []}, [], [{body_format, binary}])
     ),
-
     DashboardCfg = emqx:get_raw_config([dashboard]),
-    DashboardCfg2 = DashboardCfg#{<<"swagger_support">> => false},
-    emqx:update_config([dashboard], DashboardCfg2),
-    ?retry(
-        _Sleep = 1000,
-        _Attempts = 5,
-        ?assertMatch(
-            {ok, {{"HTTP/1.1", 404, "Not Found"}, _, _}},
-            httpc:request(get, {Url, []}, [], [{body_format, binary}])
-        )
-    ),
 
-    DashboardCfg3 = DashboardCfg#{<<"swagger_support">> => true},
-    emqx:update_config([dashboard], DashboardCfg3),
-    ?retry(
-        _Sleep0 = 1000,
-        _Attempts0 = 5,
-        ?assertMatch(
-            {ok, {{"HTTP/1.1", 200, "OK"}, __, _}},
-            httpc:request(get, {Url, []}, [], [{body_format, binary}])
-        )
+    ?check_trace(
+        ?wait_async_action(
+            begin
+                DashboardCfg2 = DashboardCfg#{<<"swagger_support">> => false},
+                emqx:update_config([dashboard], DashboardCfg2)
+            end,
+            #{?snk_kind := regenerate_minirest_dispatch},
+            30_000
+        ),
+        fun(Trace) ->
+            ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)),
+            ?assertMatch(
+                {ok, {{"HTTP/1.1", 404, "Not Found"}, _, _}},
+                httpc:request(get, {Url, []}, [], [{body_format, binary}])
+            )
+        end
+    ),
+    ?check_trace(
+        ?wait_async_action(
+            begin
+                DashboardCfg3 = DashboardCfg#{<<"swagger_support">> => true},
+                emqx:update_config([dashboard], DashboardCfg3)
+            end,
+            #{?snk_kind := regenerate_minirest_dispatch},
+            30_000
+        ),
+        fun(Trace) ->
+            ?assertMatch([#{i18n_lang := en}], ?of_kind(regenerate_minirest_dispatch, Trace)),
+            ?assertMatch(
+                {ok, {{"HTTP/1.1", 200, "OK"}, __, _}},
+                httpc:request(get, {Url, []}, [], [{body_format, binary}])
+            )
+        end
     ),
     ok.
 

apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml_api.erl

@@ -20,6 +20,7 @@
 
 -export([
     api_spec/0,
+    check_api_schema/2,
     paths/0,
     schema/1,
     namespace/0
@@ -40,11 +41,12 @@ namespace() -> "dashboard_sso".
 api_spec() ->
     emqx_dashboard_swagger:spec(?MODULE, #{
         translate_body => false,
-        check_schema => fun(Params, Meta) ->
-            emqx_dashboard_swagger:validate_content_type(Params, Meta, <<"application/xml">>)
-        end
+        check_schema => fun ?MODULE:check_api_schema/2
     }).
 
+check_api_schema(Params, Meta) ->
+    emqx_dashboard_swagger:validate_content_type(Params, Meta, <<"application/xml">>).
+
 paths() ->
     [
         "/sso/saml/acs",

apps/emqx_management/src/emqx_mgmt_api_clients.erl

@@ -815,7 +815,8 @@ fields(mqueue_message) ->
 fields(requested_client_fields) ->
     %% NOTE: some Client fields actually returned in response are missing in schema:
     %%  enable_authn, is_persistent, listener, peerport
-    ClientFields = [element(1, F) || F <- fields(client)],
+    ClientFields0 = [element(1, F) || F <- fields(client)],
+    ClientFields = [client_attrs | ClientFields0],
     [
         {fields,
             hoconsc:mk(

apps/emqx_management/test/emqx_mgmt_api_clients_SUITE.erl

@@ -1032,6 +1032,7 @@ t_query_multiple_clients_urlencode(_) ->
 t_query_clients_with_fields(_) ->
     process_flag(trap_exit, true),
     TCBin = atom_to_binary(?FUNCTION_NAME),
+    APIPort = 18083,
     ClientId = <<TCBin/binary, "_client">>,
     Username = <<TCBin/binary, "_user">>,
     {ok, C} = emqtt:start_link(#{clientid => ClientId, username => Username}),
@@ -1040,6 +1041,13 @@ t_query_clients_with_fields(_) ->
 
     Auth = emqx_mgmt_api_test_util:auth_header_(),
     ?assertEqual([#{<<"clientid">> => ClientId}], get_clients_all_fields(Auth, "fields=clientid")),
+    ?assertMatch(
+        {ok,
+            {{_, 200, _}, _, #{
+                <<"data">> := [#{<<"client_attrs">> := #{}}]
+            }}},
+        list_request(APIPort, "fields=client_attrs")
+    ),
     ?assertEqual(
         [#{<<"clientid">> => ClientId, <<"username">> => Username}],
         get_clients_all_fields(Auth, "fields=clientid,username")
@@ -1072,6 +1080,7 @@ get_clients(Auth, Qs, ExpectError, ClientIdOnly) ->
     Resp = emqx_mgmt_api_test_util:request_api(get, ClientsPath, Qs, Auth),
     case ExpectError of
         false ->
+            ct:pal("get clients response:\n  ~p", [Resp]),
             {ok, Body} = Resp,
             #{<<"data">> := Clients} = emqx_utils_json:decode(Body),
             case ClientIdOnly of

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -155,6 +155,7 @@
     replace/4,
     regex_match/2,
     regex_replace/3,
+    regex_extract/2,
     ascii/1,
     find/2,
     find/3,
@@ -805,6 +806,8 @@ regex_match(Str, RE) -> emqx_variform_bif:regex_match(Str, RE).
 
 regex_replace(SrcStr, RE, RepStr) -> emqx_variform_bif:regex_replace(SrcStr, RE, RepStr).
 
+regex_extract(SrcStr, RE) -> emqx_variform_bif:regex_extract(SrcStr, RE).
+
 ascii(Char) -> emqx_variform_bif:ascii(Char).
 
 find(S, P) -> emqx_variform_bif:find(S, P).

changes/ce/feat-13242.en.md

@@ -0,0 +1 @@
+Significantly increased the startup speed of EMQX dashboard listener.

changes/ce/fix-13216.en.md

@@ -1,10 +1,10 @@
 Respcet `clientid_prefix` config for MQTT bridges.
 
-As of version 5.4.1, EMQX limits MQTT Client ID lengths to 23 bytes.
-Previously, the system included the `clientid_prefix` in the hash calculation of the original, excessively long Client ID, thereby impacting the resulting shortened ID.
+As of version 5.4.1, EMQX limits MQTT client ID lengths to 23 bytes.
+Previously, the system included the `clientid_prefix` in the hash calculation of the original unique, but long client ID, thereby impacting the resulting shortened ID.
 
 Change Details:
-- Without Prefix: Behavior remains unchanged; EMQX will hash the entire Client ID into a 23-byte space (when longer than 23 bytes).
+- Without Prefix: Behavior remains unchanged; EMQX will hash the long (> 23 bytes) client ID into a 23-byte space.
 - With Prefix:
-  - Prefix no more than 19 bytes: The prefix is preserved, and the remaining suffix is hashed into a 4-byte space.
-  - Prefix is 20 or more bytes: EMQX no longer attempts to shorten the Client ID, respecting the configured prefix in its entirety.
+  - Prefix no more than 19 bytes: The prefix is preserved, and the client ID is hashed into a 4-byte space capping the length within 23 bytes.
+  - Prefix is 20 or more bytes: EMQX no longer attempts to shorten the client ID, respecting the configured prefix in its entirety.

changes/ce/fix-13238.en.md

@@ -0,0 +1 @@
+Improved the logged error messages when an HTTP authorization request with an unsupported content-type header is returned.

mix.exs

@@ -58,7 +58,7 @@ defmodule EMQXUmbrella.MixProject do
       {:ekka, github: "emqx/ekka", tag: "0.19.4", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true},
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true},
-      {:minirest, github: "emqx/minirest", tag: "1.4.1", override: true},
+      {:minirest, github: "emqx/minirest", tag: "1.4.3", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true},
       {:replayq, github: "emqx/replayq", tag: "0.3.8", override: true},
       {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},

rebar.config

@@ -86,7 +86,7 @@
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.4"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
     {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.12"}}},
-    {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.4.1"}}},
+    {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.4.3"}}},
     {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.7"}}},
     {replayq, {git, "https://github.com/emqx/replayq.git", {tag, "0.3.8"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},