Merge remote-tracking branch 'origin/release-54' into sync-r54-m-20231221

Makefile

@@ -20,8 +20,8 @@ endif
 
 # Dashboard version
 # from https://github.com/emqx/emqx-dashboard5
-export EMQX_DASHBOARD_VERSION ?= v1.5.2
-export EMQX_EE_DASHBOARD_VERSION ?= e1.4.0-beta.8
+export EMQX_DASHBOARD_VERSION ?= v1.6.0
+export EMQX_EE_DASHBOARD_VERSION ?= e1.4.0
 
 PROFILE ?= emqx
 REL_PROFILES := emqx emqx-enterprise

apps/emqx/include/emqx_release.hrl

@@ -32,10 +32,10 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Opensource edition
--define(EMQX_RELEASE_CE, "5.4.0-rc.1").
+-define(EMQX_RELEASE_CE, "5.4.0").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.4.0-rc.1").
+-define(EMQX_RELEASE_EE, "5.4.0").
 
 %% The HTTP API version
 -define(EMQX_API_VERSION, "5.0").

apps/emqx_bridge/src/emqx_bridge_v2.erl

@@ -416,7 +416,7 @@ uninstall_bridge_v2(
         {error, _} ->
             ok;
         ok ->
-            %% Deinstall from connector
+            %% uninstall from connector
             ConnectorId = emqx_connector_resource:resource_id(
                 connector_type(BridgeV2Type), ConnectorName
             ),
@@ -869,6 +869,8 @@ config_key_path() ->
 config_key_path_leaf() ->
     [?ROOT_KEY, '?', '?'].
 
+pre_config_update(_, {force_update, Conf}, _OldConf) ->
+    {ok, Conf};
 %% NOTE: We depend on the `emqx_bridge:pre_config_update/3` to restart/stop the
 %%       underlying resources.
 pre_config_update(_, {_Oper, _, _}, undefined) ->
@@ -882,15 +884,57 @@ pre_config_update(_Path, Conf, _OldConfig) when is_map(Conf) ->
 operation_to_enable(disable) -> false;
 operation_to_enable(enable) -> true.
 
+%% A public API that can trigger this is:
+%% bin/emqx ctl conf load data/configs/cluster.hocon
+post_config_update([?ROOT_KEY], {force_update, _Req}, NewConf, OldConf, _AppEnv) ->
+    do_post_config_update(NewConf, OldConf, #{validate_referenced_connectors => false});
 %% This top level handler will be triggered when the actions path is updated
 %% with calls to emqx_conf:update([actions], BridgesConf, #{}).
 %%
-%% A public API that can trigger this is:
-%% bin/emqx ctl conf load data/configs/cluster.hocon
 post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) ->
+    do_post_config_update(NewConf, OldConf, #{validate_referenced_connectors => true});
+post_config_update([?ROOT_KEY, BridgeType, BridgeName], '$remove', _, _OldConf, _AppEnvs) ->
+    Conf = emqx:get_config([?ROOT_KEY, BridgeType, BridgeName]),
+    ok = uninstall_bridge_v2(BridgeType, BridgeName, Conf),
+    Bridges = emqx_utils_maps:deep_remove([BridgeType, BridgeName], emqx:get_config([?ROOT_KEY])),
+    reload_message_publish_hook(Bridges),
+    ?tp(bridge_post_config_update_done, #{}),
+    ok;
+post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, undefined, _AppEnvs) ->
+    %% N.B.: all bridges must use the same field name (`connector`) to define the
+    %% connector name.
+    ConnectorName = maps:get(connector, NewConf),
+    case validate_referenced_connectors(BridgeType, ConnectorName, BridgeName) of
+        ok ->
+            ok = install_bridge_v2(BridgeType, BridgeName, NewConf),
+            Bridges = emqx_utils_maps:deep_put(
+                [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf
+            ),
+            reload_message_publish_hook(Bridges),
+            ?tp(bridge_post_config_update_done, #{}),
+            ok;
+        {error, Error} ->
+            {error, Error}
+    end;
+post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, OldConf, _AppEnvs) ->
+    ConnectorName = maps:get(connector, NewConf),
+    case validate_referenced_connectors(BridgeType, ConnectorName, BridgeName) of
+        ok ->
+            ok = uninstall_bridge_v2(BridgeType, BridgeName, OldConf),
+            ok = install_bridge_v2(BridgeType, BridgeName, NewConf),
+            Bridges = emqx_utils_maps:deep_put(
+                [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf
+            ),
+            reload_message_publish_hook(Bridges),
+            ?tp(bridge_post_config_update_done, #{}),
+            ok;
+        {error, Error} ->
+            {error, Error}
+    end.
+
+do_post_config_update(NewConf, OldConf, #{validate_referenced_connectors := NeedValidate}) ->
     #{added := Added, removed := Removed, changed := Updated} =
         diff_confs(NewConf, OldConf),
-    %% new and updated bridges must have their connector references validated
     UpdatedConfigs =
         lists:map(
             fun({{Type, BridgeName}, {_Old, New}}) ->
@@ -906,7 +950,7 @@ post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) ->
             maps:to_list(Added)
         ),
     ToValidate = UpdatedConfigs ++ AddedConfigs,
-    case multi_validate_referenced_connectors(ToValidate) of
+    case multi_validate_referenced_connectors(NeedValidate, ToValidate) of
         ok ->
             %% The config update will be failed if any task in `perform_bridge_changes` failed.
             RemoveFun = fun uninstall_bridge_v2/3,
@@ -930,44 +974,6 @@ post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) ->
             Result;
         {error, Error} ->
             {error, Error}
-    end;
-post_config_update([?ROOT_KEY, BridgeType, BridgeName], '$remove', _, _OldConf, _AppEnvs) ->
-    Conf = emqx:get_config([?ROOT_KEY, BridgeType, BridgeName]),
-    ok = uninstall_bridge_v2(BridgeType, BridgeName, Conf),
-    Bridges = emqx_utils_maps:deep_remove([BridgeType, BridgeName], emqx:get_config([?ROOT_KEY])),
-    reload_message_publish_hook(Bridges),
-    ?tp(bridge_post_config_update_done, #{}),
-    ok;
-post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, undefined, _AppEnvs) ->
-    %% N.B.: all bridges must use the same field name (`connector`) to define the
-    %% connector name.
-    ConnectorName = maps:get(connector, NewConf),
-    case validate_referenced_connectors(BridgeType, ConnectorName, BridgeName) of
-        ok ->
-            ok = install_bridge_v2(BridgeType, BridgeName, NewConf),
-            Bridges = emqx_utils_maps:deep_put(
-                [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf
-            ),
-            reload_message_publish_hook(Bridges),
-            ?tp(bridge_post_config_update_done, #{}),
-            ok;
-        {error, Error} ->
-            {error, Error}
-    end;
-post_config_update([?ROOT_KEY, BridgeType, BridgeName], _Req, NewConf, OldConf, _AppEnvs) ->
-    ConnectorName = maps:get(connector, NewConf),
-    case validate_referenced_connectors(BridgeType, ConnectorName, BridgeName) of
-        ok ->
-            ok = uninstall_bridge_v2(BridgeType, BridgeName, OldConf),
-            ok = install_bridge_v2(BridgeType, BridgeName, NewConf),
-            Bridges = emqx_utils_maps:deep_put(
-                [BridgeType, BridgeName], emqx:get_config([?ROOT_KEY]), NewConf
-            ),
-            reload_message_publish_hook(Bridges),
-            ?tp(bridge_post_config_update_done, #{}),
-            ok;
-        {error, Error} ->
-            {error, Error}
     end.
 
 diff_confs(NewConfs, OldConfs) ->
@@ -1600,7 +1606,9 @@ to_connector(ConnectorNameBin, BridgeType) ->
             throw(not_found)
     end.
 
-multi_validate_referenced_connectors(Configs) ->
+multi_validate_referenced_connectors(false, _Configs) ->
+    ok;
+multi_validate_referenced_connectors(true, Configs) ->
     Pipeline =
         lists:map(
             fun({Type, BridgeName, #{connector := ConnectorName}}) ->

apps/emqx_bridge_redis/src/emqx_bridge_redis_action_info.erl

@@ -11,8 +11,8 @@
     action_type_name/0,
     connector_type_name/0,
     schema_module/0,
-    bridge_v1_config_to_action_config/2,
     connector_action_config_to_bridge_v1_config/2,
+    bridge_v1_config_to_action_config/2,
     bridge_v1_config_to_connector_config/1,
     bridge_v1_type_name_fun/1
 ]).
@@ -28,14 +28,25 @@ connector_type_name() -> redis.
 
 schema_module() -> ?SCHEMA_MODULE.
 
+%% redis_cluster don't have batch options
 connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) ->
-    maps:merge(
+    Config0 = emqx_utils_maps:deep_merge(
         maps:without(
             [<<"connector">>],
-            map_unindent(<<"parameters">>, ActionConfig)
+            emqx_utils_maps:unindent(<<"parameters">>, ActionConfig)
         ),
-        map_unindent(<<"parameters">>, ConnectorConfig)
-    ).
+        emqx_utils_maps:unindent(<<"parameters">>, ConnectorConfig)
+    ),
+    Config1 =
+        case Config0 of
+            #{<<"resource_opts">> := ResOpts0, <<"redis_type">> := Type} ->
+                Schema = emqx_bridge_redis:fields("creation_opts_redis_" ++ binary_to_list(Type)),
+                ResOpts = maps:with(schema_keys(Schema), ResOpts0),
+                Config0#{<<"resource_opts">> => ResOpts};
+            _ ->
+                Config0
+        end,
+    maps:without([<<"description">>], Config1).
 
 bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) ->
     ActionTopLevelKeys = schema_keys(?SCHEMA_MODULE:fields(redis_action)),
@@ -81,22 +92,9 @@ v1_type(<<"cluster">>) -> redis_cluster.
 
 bridge_v1_type_names() -> [redis_single, redis_sentinel, redis_cluster].
 
-map_unindent(Key, Map) ->
-    maps:merge(
-        maps:get(Key, Map),
-        maps:remove(Key, Map)
-    ).
-
-map_indent(IndentKey, PickKeys, Map) ->
-    maps:put(
-        IndentKey,
-        maps:with(PickKeys, Map),
-        maps:without(PickKeys, Map)
-    ).
-
 schema_keys(Schema) ->
     [bin(Key) || {Key, _} <- Schema].
 
 make_config_map(PickKeys, IndentKeys, Config) ->
     Conf0 = maps:with(PickKeys, Config),
-    map_indent(<<"parameters">>, IndentKeys, Conf0).
+    emqx_utils_maps:indent(<<"parameters">>, IndentKeys, Conf0).

apps/emqx_conf/src/emqx_conf.erl

@@ -28,7 +28,7 @@
 -export([remove/2, remove/3]).
 -export([tombstone/2]).
 -export([reset/2, reset/3]).
--export([dump_schema/2, reformat_schema_dump/1]).
+-export([dump_schema/2, reformat_schema_dump/2]).
 -export([schema_module/0]).
 
 %% TODO: move to emqx_dashboard when we stop building api schema at build time
@@ -186,7 +186,7 @@ gen_schema_json(Dir, SchemaModule, Lang) ->
     ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang).
 
 gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
-    NestedStruct = reformat_schema_dump(StructsJsonArray),
+    NestedStruct = reformat_schema_dump(StructsJsonArray, Lang),
     %% write to files
     NestedJsonFile = filename:join([Dir, "schema-v2-" ++ Lang ++ ".json"]),
     io:format(user, "===< Generating: ~s~n", [NestedJsonFile]),
@@ -196,15 +196,17 @@ gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
     ok.
 
 %% @doc This function is exported for scripts/schema-dump-reformat.escript
-reformat_schema_dump(StructsJsonArray0) ->
+reformat_schema_dump(StructsJsonArray0, Lang) ->
     %% prepare
+    DescResolver = make_desc_resolver(Lang),
     StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0),
     #{fields := RootFields} = hd(StructsJsonArray),
     RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields),
     RootNames = lists:map(fun to_bin/1, RootNames0),
     %% reformat
     [Root | FlatStructs0] = lists:map(
-        fun(Struct) -> gen_flat_doc(RootNames, Struct) end, StructsJsonArray
+        fun(Struct) -> gen_flat_doc(RootNames, Struct, DescResolver) end,
+        StructsJsonArray
     ),
     FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0],
     gen_nested_doc(FlatStructs).
@@ -302,7 +304,7 @@ expand_ref(#{hash := FullName}, FindFn, Path) ->
 
 %% generate flat docs for each struct.
 %% using references to link to other structs.
-gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
+gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S, DescResolver) ->
     ShortName = short_name(FullName),
     case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of
         true ->
@@ -314,18 +316,19 @@ gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
         text => short_name(FullName),
         hash => format_hash(FullName),
         doc => maps:get(desc, S, <<"">>),
-        fields => format_fields(Fields)
+        fields => format_fields(Fields, DescResolver)
     }.
 
-format_fields([]) ->
-    [];
-format_fields([Field | Fields]) ->
-    [format_field(Field) | format_fields(Fields)].
+format_fields(Fields, DescResolver) ->
+    [format_field(F, DescResolver) || F <- Fields].
 
-format_field(#{name := Name, aliases := Aliases, type := Type} = F) ->
+format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver) ->
+    TypeDoc = format_type_desc(Type, DescResolver),
     L = [
         {text, Name},
         {type, format_type(Type)},
+        %% TODO: Make it into a separate field.
+        %% {typedoc, format_type_desc(Type, DescResolver)},
         {refs, format_refs(Type)},
         {aliases,
             case Aliases of
@@ -333,7 +336,7 @@ format_field(#{name := Name, aliases := Aliases, type := Type} = F) ->
                 _ -> Aliases
             end},
         {default, maps:get(hocon, maps:get(default, F, #{}), undefined)},
-        {doc, maps:get(desc, F, undefined)}
+        {doc, join_format([maps:get(desc, F, undefined), TypeDoc])}
     ],
     maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]).
 
@@ -393,10 +396,26 @@ format_union_members([Member | Members], Acc) ->
     NewAcc = [format_type(Member) | Acc],
     format_union_members(Members, NewAcc).
 
+format_type_desc(#{kind := primitive, name := Name}, DescResolver) ->
+    format_primitive_type_desc(Name, DescResolver);
+format_type_desc(#{}, _DescResolver) ->
+    undefined.
+
 format_primitive_type(TypeStr) ->
-    Spec = emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr),
+    Spec = get_primitive_typespec(TypeStr),
     to_bin(maps:get(type, Spec)).
 
+format_primitive_type_desc(TypeStr, DescResolver) ->
+    case get_primitive_typespec(TypeStr) of
+        #{desc := Desc} ->
+            DescResolver(Desc);
+        #{} ->
+            undefined
+    end.
+
+get_primitive_typespec(TypeStr) ->
+    emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr).
+
 %% All types should have a namespace to avlid name clashing.
 is_missing_namespace(ShortName, FullName, RootNames) ->
     case lists:member(ShortName, RootNames) of
@@ -560,6 +579,14 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
 typename_to_spec(TypeStr, Module) ->
     emqx_conf_schema_types:readable_dashboard(Module, TypeStr).
 
+join_format(Snippets) ->
+    case [S || S <- Snippets, S =/= undefined] of
+        [] ->
+            undefined;
+        NonEmpty ->
+            to_bin(lists:join("<br/>", NonEmpty))
+    end.
+
 to_bin(List) when is_list(List) -> iolist_to_binary(List);
 to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
 to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);

apps/emqx_conf/src/emqx_conf_cli.erl

@@ -286,9 +286,16 @@ update_config_cluster(
     check_res(Key, emqx_authn:merge_config(Conf), Conf, Opts);
 update_config_cluster(Key, NewConf, #{mode := merge} = Opts) ->
     Merged = merge_conf(Key, NewConf),
-    check_res(Key, emqx_conf:update([Key], Merged, ?OPTIONS), NewConf, Opts);
+    Request = make_request(Key, Merged),
+    check_res(Key, emqx_conf:update([Key], Request, ?OPTIONS), NewConf, Opts);
 update_config_cluster(Key, Value, #{mode := replace} = Opts) ->
-    check_res(Key, emqx_conf:update([Key], Value, ?OPTIONS), Value, Opts).
+    Request = make_request(Key, Value),
+    check_res(Key, emqx_conf:update([Key], Request, ?OPTIONS), Value, Opts).
+
+make_request(Key, Value) when Key =:= <<"connectors">> orelse Key =:= <<"actions">> ->
+    {force_update, Value};
+make_request(_Key, Value) ->
+    Value.
 
 -define(LOCAL_OPTIONS, #{rawconf_with_defaults => true, persistent => false}).
 update_config_local(
@@ -305,9 +312,11 @@ update_config_local(
     check_res(node(), Key, emqx_authn:merge_config_local(Conf, ?LOCAL_OPTIONS), Conf, Opts);
 update_config_local(Key, NewConf, #{mode := merge} = Opts) ->
     Merged = merge_conf(Key, NewConf),
-    check_res(node(), Key, emqx:update_config([Key], Merged, ?LOCAL_OPTIONS), NewConf, Opts);
+    Request = make_request(Key, Merged),
+    check_res(node(), Key, emqx:update_config([Key], Request, ?LOCAL_OPTIONS), NewConf, Opts);
 update_config_local(Key, Value, #{mode := replace} = Opts) ->
-    check_res(node(), Key, emqx:update_config([Key], Value, ?LOCAL_OPTIONS), Value, Opts).
+    Request = make_request(Key, Value),
+    check_res(node(), Key, emqx:update_config([Key], Request, ?LOCAL_OPTIONS), Value, Opts).
 
 check_res(Key, Res, Conf, Opts) -> check_res(cluster, Key, Res, Conf, Opts).
 check_res(Node, Key, {ok, _}, _Conf, Opts) ->

apps/emqx_conf/src/emqx_conf_schema_types.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_conf_schema_types).
 
+-include_lib("hocon/include/hocon_types.hrl").
+
 -export([readable/2]).
 -export([readable_swagger/2, readable_dashboard/2, readable_docgen/2]).
 
@@ -165,37 +167,37 @@ readable("duration()") ->
     #{
         swagger => #{type => string, example => <<"12m">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"12m">>}
+        docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)}
     };
 readable("duration_s()") ->
     #{
         swagger => #{type => string, example => <<"1h">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"1h">>}
+        docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)}
     };
 readable("duration_ms()") ->
     #{
         swagger => #{type => string, example => <<"32s">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"32s">>}
+        docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)}
     };
 readable("timeout_duration()") ->
     #{
         swagger => #{type => string, example => <<"12m">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"12m">>}
+        docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)}
     };
 readable("timeout_duration_s()") ->
     #{
         swagger => #{type => string, example => <<"1h">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"1h">>}
+        docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)}
     };
 readable("timeout_duration_ms()") ->
     #{
         swagger => #{type => string, example => <<"32s">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"32s">>}
+        docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)}
     };
 readable("percent()") ->
     #{
@@ -219,13 +221,13 @@ readable("bytesize()") ->
     #{
         swagger => #{type => string, example => <<"32MB">>},
         dashboard => #{type => 'byteSize'},
-        docgen => #{type => "String", example => <<"32MB">>}
+        docgen => #{type => "Bytesize", example => <<"32MB">>, desc => ?DESC(bytesize)}
     };
 readable("wordsize()") ->
     #{
         swagger => #{type => string, example => <<"1024KB">>},
         dashboard => #{type => 'wordSize'},
-        docgen => #{type => "String", example => <<"1024KB">>}
+        docgen => #{type => "Bytesize", example => <<"1024KB">>, desc => ?DESC(bytesize)}
     };
 readable("map(" ++ Map) ->
     [$) | _MapArgs] = lists:reverse(Map),
@@ -287,7 +289,11 @@ readable("secret()") ->
     #{
         swagger => #{type => string, example => <<"R4ND0M/S∃CЯ∃T"/utf8>>},
         dashboard => #{type => string},
-        docgen => #{type => "String", example => <<"R4ND0M/S∃CЯ∃T"/utf8>>}
+        docgen => #{
+            type => "Secret",
+            example => <<"R4ND0M/S∃CЯ∃T"/utf8>>,
+            desc => ?DESC(secret)
+        }
     };
 readable(TypeStr0) ->
     case string:split(TypeStr0, ":") of

apps/emqx_connector/src/emqx_connector.erl

@@ -107,6 +107,8 @@ config_key_path() ->
 
 pre_config_update([?ROOT_KEY], RawConf, RawConf) ->
     {ok, RawConf};
+pre_config_update([?ROOT_KEY], {force_update, NewConf}, RawConf) ->
+    pre_config_update([?ROOT_KEY], NewConf, RawConf);
 pre_config_update([?ROOT_KEY], NewConf, _RawConf) ->
     case multi_validate_connector_names(NewConf) of
         ok ->
@@ -135,23 +137,16 @@ pre_config_update(Path, Conf, _OldConfig) when is_map(Conf) ->
 operation_to_enable(disable) -> false;
 operation_to_enable(enable) -> true.
 
+post_config_update([?ROOT_KEY], {force_update, _}, NewConf, OldConf, _AppEnv) ->
+    #{added := Added, removed := Removed, changed := Updated} =
+        diff_confs(NewConf, OldConf),
+    perform_connector_changes(Removed, Added, Updated);
 post_config_update([?ROOT_KEY], _Req, NewConf, OldConf, _AppEnv) ->
     #{added := Added, removed := Removed, changed := Updated} =
         diff_confs(NewConf, OldConf),
     case ensure_no_channels(Removed) of
         ok ->
-            %% The config update will be failed if any task in `perform_connector_changes` failed.
-            Result = perform_connector_changes([
-                #{action => fun emqx_connector_resource:remove/4, data => Removed},
-                #{
-                    action => fun emqx_connector_resource:create/4,
-                    data => Added,
-                    on_exception_fn => fun emqx_connector_resource:remove/4
-                },
-                #{action => fun emqx_connector_resource:update/4, data => Updated}
-            ]),
-            ?tp(connector_post_config_update_done, #{}),
-            Result;
+            perform_connector_changes(Removed, Added, Updated);
         {error, Error} ->
             {error, Error}
     end;
@@ -175,6 +170,20 @@ post_config_update([?ROOT_KEY, Type, Name], _Req, NewConf, OldConf, _AppEnvs) ->
     ?tp(connector_post_config_update_done, #{}),
     ok.
 
+%% The config update will be failed if any task in `perform_connector_changes` failed.
+perform_connector_changes(Removed, Added, Updated) ->
+    Result = perform_connector_changes([
+        #{action => fun emqx_connector_resource:remove/4, data => Removed},
+        #{
+            action => fun emqx_connector_resource:create/4,
+            data => Added,
+            on_exception_fn => fun emqx_connector_resource:remove/4
+        },
+        #{action => fun emqx_connector_resource:update/4, data => Updated}
+    ]),
+    ?tp(connector_post_config_update_done, #{}),
+    Result.
+
 list() ->
     maps:fold(
         fun(Type, NameAndConf, Connectors) ->

apps/emqx_management/src/emqx_mgmt_api_configs.erl

@@ -356,10 +356,13 @@ configs(put, #{body := Conf, query_string := #{<<"mode">> := Mode}}, _Req) ->
     case emqx_conf_cli:load_config(Conf, #{mode => Mode, log => none}) of
         ok ->
             {200};
-        {error, MsgList} ->
+        %% bad hocon format
+        {error, MsgList = [{_, _} | _]} ->
             JsonFun = fun(K, V) -> {K, emqx_utils_maps:binary_string(V)} end,
             JsonMap = emqx_utils_maps:jsonable_map(maps:from_list(MsgList), JsonFun),
-            {400, #{<<"content-type">> => <<"text/plain">>}, JsonMap}
+            {400, #{<<"content-type">> => <<"text/plain">>}, JsonMap};
+        {error, Msg} ->
+            {400, #{<<"content-type">> => <<"text/plain">>}, Msg}
     end.
 
 find_suitable_accept(Headers, Preferences) when is_list(Preferences), length(Preferences) > 0 ->

apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl

@@ -378,8 +378,13 @@ t_get_configs_in_different_accept(_Config) ->
     ?assertMatch({400, "application/json", _}, Request(<<"application/xml">>)).
 
 t_create_webhook_v1_bridges_api({'init', Config}) ->
-    application:ensure_all_started(emqx_connector),
-    application:ensure_all_started(emqx_bridge),
+    lists:foreach(
+        fun(App) ->
+            _ = application:stop(App),
+            {ok, [App]} = application:ensure_all_started(App)
+        end,
+        [emqx_connector, emqx_bridge]
+    ),
     Config;
 t_create_webhook_v1_bridges_api({'end', _}) ->
     application:stop(emqx_bridge),

deploy/charts/emqx-enterprise/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.4.0-rc.1
+version: 5.4.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.4.0-rc.1
+appVersion: 5.4.0

deploy/charts/emqx/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.4.0-rc.1
+version: 5.4.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.4.0-rc.1
+appVersion: 5.4.0

rel/i18n/emqx_conf_schema_types.hocon

@@ -0,0 +1,12 @@
+emqx_conf_schema_types {
+
+    duration.desc:
+    """A string that represents a time duration, for example: <code>10s</code>, <code>2.5m</code>, <code>1h30m</code>, <code>1W2D</code>, or <code>2345ms</code>, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing <code>1200ms</code> for <code>Duration(s)</code> is equivalent to writing <code>1s</code>. The unit part is case-insensitive."""
+
+    bytesize.desc:
+    """A string that represents a number of bytes, for example: <code>10B</code>, <code>640kb</code>, <code>4MB</code>, <code>1GB</code>. Units are interpreted as powers of 1024, and the unit part is case-insensitive."""
+
+    secret.desc:
+    """A string holding some sensitive information, such as a password. When secret starts with <code>file://</code>, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value. Note: when clustered, all EMQX nodes should have the same file present before using <code>file://</code> secrets."""
+
+}

scripts/schema-dump-reformat.escript

@@ -1,132 +0,0 @@
-#!/usr/bin/env escript
-
-%% This script translates the hocon_schema_json's schema dump to a new format.
-%% It is used to convert older version EMQX's schema dumps to the new format
-%% after all files are upgraded to the new format, this script can be removed.
-
--mode(compile).
-
-main([Input]) ->
-    ok = add_libs(),
-    _ = atoms(),
-    {ok, Data} = file:read_file(Input),
-    Json = jsx:decode(Data),
-    NewJson = reformat(Json),
-    io:format("~s~n", [jsx:encode(NewJson)]);
-main(_) ->
-    io:format("Usage: schema-dump-reformat.escript <input.json>~n"),
-    halt(1).
-
-reformat(Json) ->
-    emqx_conf:reformat_schema_dump(fix(Json)).
-
-%% fix old type specs to make them compatible with new type specs
-fix(#{
-    <<"kind">> := <<"union">>,
-    <<"members">> := [#{<<"name">> := <<"string()">>}, #{<<"name">> := <<"function()">>}]
-}) ->
-    %% s3_exporter.secret_access_key
-    #{
-        kind => primitive,
-        name => <<"string()">>
-    };
-fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_conf_schema:log_level()">>}) ->
-    #{
-        kind => enum,
-        symbols => [emergency, alert, critical, error, warning, notice, info, debug, none, all]
-    };
-fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_connector_http:pool_type()">>}) ->
-    #{kind => enum, symbols => [random, hash]};
-fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_bridge_http_connector:pool_type()">>}) ->
-    #{kind => enum, symbols => [random, hash]};
-fix(Map) when is_map(Map) ->
-    maps:from_list(fix(maps:to_list(Map)));
-fix(List) when is_list(List) ->
-    lists:map(fun fix/1, List);
-fix({<<"kind">>, Kind}) ->
-    {kind, binary_to_atom(Kind, utf8)};
-fix({<<"name">>, Type}) ->
-    {name, fix_type(Type)};
-fix({K, V}) ->
-    {binary_to_atom(K, utf8), fix(V)};
-fix(V) when is_number(V) ->
-    V;
-fix(V) when is_atom(V) ->
-    V;
-fix(V) when is_binary(V) ->
-    V.
-
-%% ensure below ebin dirs are added to code path:
-%% _build/default/lib/*/ebin
-%% _build/emqx/lib/*/ebin
-%% _build/emqx-enterprise/lib/*/ebin
-add_libs() ->
-    Profile = os:getenv("PROFILE"),
-    case Profile of
-        "emqx" ->
-            ok;
-        "emqx-enterprise" ->
-            ok;
-        _ ->
-            io:format("PROFILE is not set~n"),
-            halt(1)
-    end,
-    Dirs =
-        filelib:wildcard("_build/default/lib/*/ebin") ++
-            filelib:wildcard("_build/" ++ Profile ++ "/lib/*/ebin"),
-    lists:foreach(fun add_lib/1, Dirs).
-
-add_lib(Dir) ->
-    code:add_patha(Dir),
-    Beams = filelib:wildcard(Dir ++ "/*.beam"),
-    _ = spawn(fun() -> lists:foreach(fun load_beam/1, Beams) end),
-    ok.
-
-load_beam(BeamFile) ->
-    ModuleName = filename:basename(BeamFile, ".beam"),
-    Module = list_to_atom(ModuleName),
-    %% load the beams to make sure the atoms are existing
-    code:ensure_loaded(Module),
-    ok.
-
-fix_type(<<"[{string(), string()}]">>) ->
-    <<"map()">>;
-fix_type(<<"[{binary(), binary()}]">>) ->
-    <<"map()">>;
-fix_type(<<"emqx_limiter_schema:rate()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_limiter_schema:burst_rate()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_limiter_schema:capacity()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_limiter_schema:initial()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_limiter_schema:failure_strategy()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_conf_schema:file()">>) ->
-    <<"string()">>;
-fix_type(<<"#{term() => binary()}">>) ->
-    <<"map()">>;
-fix_type(<<"[term()]">>) ->
-    %% jwt claims
-    <<"map()">>;
-fix_type(<<"emqx_ee_bridge_influxdb:write_syntax()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_bridge_influxdb:write_syntax()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_schema:mqtt_max_packet_size()">>) ->
-    <<"non_neg_integer()">>;
-fix_type(<<"emqx_s3_schema:secret_access_key()">>) ->
-    <<"string()">>;
-fix_type(Type) ->
-    Type.
-
-%% ensure atoms are loaded
-%% these atoms are from older version of emqx
-atoms() ->
-    [
-        emqx_ee_connector_clickhouse,
-        emqx_ee_bridge_gcp_pubsub,
-        emqx_ee_bridge_influxdb,
-        emqx_connector_http
-    ].