feat(emqx_config): add emqx_config:fill_defaults/1

apps/emqx/rebar.config

@@ -15,7 +15,7 @@
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.2"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.10.4"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.1"}}}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}
     , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}}

apps/emqx/src/emqx_config.erl

@@ -20,14 +20,20 @@
 -export([ init_load/2
         , read_override_conf/0
         , check_config/2
+        , fill_defaults/1
+        , fill_defaults/2
         , save_configs/4
         , save_to_app_env/1
         , save_to_config_map/2
         , save_to_override_conf/1
         ]).
 
--export([get_root/1,
-         get_root_raw/1]).
+-export([ get_root/1
+        , get_root_raw/1
+        ]).
+
+-export([ get_default_value/1
+        ]).
 
 -export([ get/1
         , get/2
@@ -37,6 +43,12 @@
         , put/2
         ]).
 
+-export([ save_schema_mod/1
+        , get_schema_mod/0
+        , get_schema_mod/1
+        , get_root_names/0
+        ]).
+
 -export([ get_zone_conf/2
         , get_zone_conf/3
         , put_zone_conf/3
@@ -53,6 +65,7 @@
         , update/3
         , remove/1
         , remove/2
+        , reset/1
         ]).
 
 -export([ get_raw/1
@@ -63,6 +76,7 @@
 
 -define(CONF, conf).
 -define(RAW_CONF, raw_conf).
+-define(PERSIS_MOD_ROOTNAMES, {?MODULE, default_conf}).
 -define(PERSIS_KEY(TYPE, ROOT), {?MODULE, TYPE, ROOT}).
 -define(ZONE_CONF_PATH(ZONE, PATH), [zones, ZONE | PATH]).
 -define(LISTENER_CONF_PATH(ZONE, LISTENER, PATH), [zones, ZONE, listeners, LISTENER | PATH]).
@@ -170,20 +184,49 @@ put(KeyPath, Config) -> do_put(?CONF, KeyPath, Config).
 
 -spec update(emqx_map_lib:config_key_path(), update_request()) ->
     ok | {error, term()}.
-update(KeyPath, UpdateReq) ->
-    update(emqx_schema, KeyPath, UpdateReq).
+update([RootName | _] = KeyPath, UpdateReq) ->
+    update(get_schema_mod(RootName), KeyPath, UpdateReq).
 
 -spec update(module(), emqx_map_lib:config_key_path(), update_request()) ->
     ok | {error, term()}.
-update(SchemaModule, KeyPath, UpdateReq) ->
-    emqx_config_handler:update_config(SchemaModule, KeyPath, {update, UpdateReq}).
+update(SchemaMod, KeyPath, UpdateReq) ->
+    emqx_config_handler:update_config(SchemaMod, KeyPath, {update, UpdateReq}).
 
 -spec remove(emqx_map_lib:config_key_path()) -> ok | {error, term()}.
-remove(KeyPath) ->
-    remove(emqx_schema, KeyPath).
+remove([RootName | _] = KeyPath) ->
+    remove(get_schema_mod(RootName), KeyPath).
+
+remove(SchemaMod, KeyPath) ->
+    emqx_config_handler:update_config(SchemaMod, KeyPath, remove).
+
+-spec reset(emqx_map_lib:config_key_path()) -> ok | {error, term()}.
+reset([RootName | _] = KeyPath) ->
+    case get_default_value(KeyPath) of
+        {ok, Default} ->
+            emqx_config_handler:update_config(get_schema_mod(RootName), KeyPath,
+                {update, Default});
+        {error, _} = Error ->
+            Error
+    end.
 
-remove(SchemaModule, KeyPath) ->
-    emqx_config_handler:update_config(SchemaModule, KeyPath, remove).
+-spec get_default_value(emqx_map_lib:config_key_path()) -> ok | {error, term()}.
+get_default_value([RootName | _] = KeyPath) ->
+    BinKeyPath = [bin(Key) || Key <- KeyPath],
+    case find_raw([RootName]) of
+        {ok, RawConf} ->
+            RawConf1 = emqx_map_lib:deep_remove(BinKeyPath, RawConf),
+            SchemaMod = get_schema_mod(RootName),
+            try fill_defaults(SchemaMod, RawConf1) of FullConf ->
+                case emqx_map_lib:deep_find(BinKeyPath, FullConf) of
+                    {not_found, _, _} -> {error, no_default_value};
+                    {ok, Val} -> {ok, Val}
+                end
+            catch error:_ ->
+                {error, required_conf}
+            end;
+        {not_found, _, _} ->
+            {error, {rootname_not_found, RootName}}
+    end.
 
 -spec get_raw(emqx_map_lib:config_key_path()) -> term().
 get_raw(KeyPath) -> do_get(?RAW_CONF, KeyPath).
@@ -208,7 +251,7 @@ put_raw(KeyPath, Config) -> do_put(?RAW_CONF, KeyPath, Config).
 %% NOTE: The order of the files is significant, configs from files orderd
 %% in the rear of the list overrides prior values.
 -spec init_load(module(), [string()] | binary() | hocon:config()) -> ok.
-init_load(SchemaModule, Conf) when is_list(Conf) orelse is_binary(Conf) ->
+init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) ->
     ParseOptions = #{format => richmap},
     Parser = case is_binary(Conf) of
               true -> fun hocon:binary/2;
@@ -216,39 +259,78 @@ init_load(SchemaModule, Conf) when is_list(Conf) orelse is_binary(Conf) ->
              end,
     case Parser(Conf, ParseOptions) of
         {ok, RawRichConf} ->
-            init_load(SchemaModule, RawRichConf);
+            init_load(SchemaMod, RawRichConf);
         {error, Reason} ->
             logger:error(#{msg => failed_to_load_hocon_conf,
                            reason => Reason
                           }),
             error(failed_to_load_hocon_conf)
     end;
-init_load(SchemaModule, RawRichConf) when is_map(RawRichConf) ->
+init_load(SchemaMod, RawRichConf) when is_map(RawRichConf) ->
     %% check with richmap for line numbers in error reports (future enhancement)
     Opts = #{return_plain => true,
              nullable => true
             },
     %% this call throws exception in case of check failure
-    {_AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaModule, RawRichConf, Opts),
-    ok = save_to_config_map(emqx_map_lib:unsafe_atom_key_map(CheckedConf),
-            hocon_schema:richmap_to_map(RawRichConf)).
+    {_AppEnvs, CheckedConf} = hocon_schema:map_translate(SchemaMod, RawRichConf, Opts),
+    ok = save_schema_mod(SchemaMod),
+    ok = save_to_config_map(emqx_map_lib:unsafe_atom_key_map(normalize_conf(CheckedConf)),
+            normalize_conf(hocon_schema:richmap_to_map(RawRichConf))).
+
+normalize_conf(Conf) ->
+    maps:with(get_root_names(), Conf).
 
 -spec check_config(module(), raw_config()) -> {AppEnvs, CheckedConf}
     when AppEnvs :: app_envs(), CheckedConf :: config().
-check_config(SchemaModule, RawConf) ->
+check_config(SchemaMod, RawConf) ->
     Opts = #{return_plain => true,
              nullable => true,
              format => map
             },
     {AppEnvs, CheckedConf} =
-        hocon_schema:map_translate(SchemaModule, RawConf, Opts),
+        hocon_schema:map_translate(SchemaMod, RawConf, Opts),
     Conf = maps:with(maps:keys(RawConf), CheckedConf),
     {AppEnvs, emqx_map_lib:unsafe_atom_key_map(Conf)}.
 
+-spec fill_defaults(raw_config()) -> map().
+fill_defaults(RawConf) ->
+    RootNames = get_root_names(),
+    maps:fold(fun(Key, Conf, Acc) ->
+            SubMap = #{Key => Conf},
+            WithDefaults = case lists:member(Key, RootNames) of
+                true -> fill_defaults(get_schema_mod(Key), SubMap);
+                false -> SubMap
+            end,
+            maps:merge(Acc, WithDefaults)
+        end, #{}, RawConf).
+
+-spec fill_defaults(module(), raw_config()) -> map().
+fill_defaults(SchemaMod, RawConf) ->
+    hocon_schema:check_plain(SchemaMod, RawConf,
+        #{nullable => true, no_conversion => true}, [str(K) || K <- maps:keys(RawConf)]).
+
 -spec read_override_conf() -> raw_config().
 read_override_conf() ->
     load_hocon_file(emqx_override_conf_name(), map).
 
+-spec save_schema_mod(module()) -> ok.
+save_schema_mod(SchemaMod) ->
+    OldMods = get_schema_mod(),
+    NewMods = maps:from_list([{bin(RootName), SchemaMod} || RootName <- SchemaMod:structs()]),
+    persistent_term:put(?PERSIS_MOD_ROOTNAMES, maps:merge(OldMods, NewMods)).
+
+-spec get_schema_mod() -> #{binary() => atom()}.
+get_schema_mod() ->
+    persistent_term:get(?PERSIS_MOD_ROOTNAMES, #{}).
+
+-spec get_schema_mod(atom() | binary()) -> [module()].
+get_schema_mod(RootName) ->
+    maps:get(bin(RootName), get_schema_mod()).
+
+-spec get_root_names() -> [binary()].
+get_root_names() ->
+    maps:keys(get_schema_mod()).
+
 -spec save_configs(app_envs(), config(), raw_config(), raw_config()) -> ok | {error, term()}.
 save_configs(_AppEnvs, Conf, RawConf, OverrideConf) ->
     %% We may need also support hot config update for the apps that use application envs.
@@ -338,10 +420,20 @@ do_deep_put(?RAW_CONF, KeyPath, Map, Value) ->
 
 atom(Bin) when is_binary(Bin) ->
     binary_to_existing_atom(Bin, latin1);
+atom(Str) when is_list(Str) ->
+    list_to_existing_atom(Str);
 atom(Atom) when is_atom(Atom) ->
     Atom.
 
+str(Bin) when is_binary(Bin) ->
+    binary_to_list(Bin);
+str(Str) when is_list(Str) ->
+    Str;
+str(Atom) when is_atom(Atom) ->
+    atom_to_list(Atom).
+
 bin(Bin) when is_binary(Bin) -> Bin;
+bin(Str) when is_list(Str) -> list_to_binary(Str);
 bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
 
 conf_key(?CONF, RootName) ->

apps/emqx/src/emqx_map_lib.erl

@@ -23,6 +23,8 @@
         , deep_merge/2
         , safe_atom_key_map/1
         , unsafe_atom_key_map/1
+        , jsonable_map/1
+        , deep_convert/2
         ]).
 
 -export_type([config_key/0, config_key_path/0]).
@@ -97,21 +99,42 @@ deep_merge(BaseMap, NewMap) ->
         end, #{}, BaseMap),
     maps:merge(MergedBase, maps:with(NewKeys, NewMap)).
 
+-spec deep_convert(map(), fun((K::any(), V::any()) -> {K1::any(), V1::any()})) -> map().
+deep_convert(Map, ConvFun) when is_map(Map) ->
+    maps:fold(fun(K, V, Acc) ->
+            {K1, V1} = ConvFun(K, deep_convert(V, ConvFun)),
+            Acc#{K1 => V1}
+        end, #{}, Map);
+deep_convert(ListV, ConvFun) when is_list(ListV) ->
+    [deep_convert(V, ConvFun) || V <- ListV];
+deep_convert(Val, _) -> Val.
+
+-spec unsafe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}.
 unsafe_atom_key_map(Map) ->
     covert_keys_to_atom(Map, fun(K) -> binary_to_atom(K, utf8) end).
 
+-spec safe_atom_key_map(#{binary() | atom() => any()}) -> #{atom() => any()}.
 safe_atom_key_map(Map) ->
     covert_keys_to_atom(Map, fun(K) -> binary_to_existing_atom(K, utf8) end).
 
+-spec jsonable_map(map()) -> map().
+jsonable_map(Map) ->
+    deep_convert(Map, fun(K, V) ->
+            {jsonable_value(K), jsonable_value(V)}
+        end).
+
+jsonable_value([]) -> [];
+jsonable_value(Val) when is_list(Val) ->
+    case io_lib:printable_unicode_list(Val) of
+        true -> unicode:characters_to_binary(Val);
+        false -> Val
+    end;
+jsonable_value(Val) ->
+    Val.
+
 %%---------------------------------------------------------------------------
-covert_keys_to_atom(BinKeyMap, Conv) when is_map(BinKeyMap) ->
-    maps:fold(
-        fun(K, V, Acc) when is_binary(K) ->
-              Acc#{Conv(K) => covert_keys_to_atom(V, Conv)};
-           (K, V, Acc) when is_atom(K) ->
-              %% richmap keys
-              Acc#{K => covert_keys_to_atom(V, Conv)}
-        end, #{}, BinKeyMap);
-covert_keys_to_atom(ListV, Conv) when is_list(ListV) ->
-    [covert_keys_to_atom(V, Conv) || V <- ListV];
-covert_keys_to_atom(Val, _) -> Val.
+covert_keys_to_atom(BinKeyMap, Conv) ->
+    deep_convert(BinKeyMap, fun
+            (K, V) when is_atom(K) -> {K, V};
+            (K, V) when is_binary(K) -> {Conv(K), V}
+        end).

apps/emqx_machine/src/emqx_machine_schema.erl

@@ -54,6 +54,7 @@
         , emqx_dashboard_schema
         , emqx_gateway_schema
         , emqx_prometheus_schema
+        , emqx_rule_engine_schema
         , emqx_exhook_schema
         ]).
 

rebar.config

@@ -61,7 +61,7 @@
     , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
     , {getopt, "1.0.2"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.14.1"}}}
-    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.11.1"}}}
     , {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.4.0"}}}
     , {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.1.0"}}}
     ]}.