refactor(config): load emqx.conf before all other tasks in emqx_app

apps/emqx/src/emqx_app.erl

@@ -42,6 +42,7 @@
 %%--------------------------------------------------------------------
 
 start(_Type, _Args) ->
+    emqx_config:load(),
     set_backtrace_depth(),
     print_otp_version_warning(),
     print_banner(),

apps/emqx/src/emqx_config.erl

@@ -17,6 +17,15 @@
 
 -compile({no_auto_import, [get/0, get/1]}).
 
+-export([ load/0
+        , save_configs/2
+        , save_to_app_env/1
+        , save_to_emqx_config/2
+        , save_to_override_conf/1
+        , to_richmap/1
+        , to_plainmap/1
+        ]).
+
 -export([ get/0
         , get/1
         , get/2
@@ -141,3 +150,75 @@ put_raw(Config) ->
 -spec put_raw(emqx_map_lib:config_key_path(), term()) -> ok.
 put_raw(KeyPath, Config) ->
     put_raw(emqx_map_lib:deep_put(KeyPath, get_raw(), Config)).
+
+%%============================================================================
+%% Load/Update configs From/To files
+%%============================================================================
+load() ->
+    %% the app env 'config_files' should be set before emqx get started.
+    ConfFiles = application:get_env(emqx, config_files, []),
+    RawRichConf = lists:foldl(fun(ConfFile, Acc) ->
+        Raw = load_hocon_file(ConfFile),
+        emqx_map_lib:deep_merge(Acc, Raw)
+    end, #{}, ConfFiles),
+    {_MappedEnvs, RichConf} = hocon_schema:map_translate(emqx_schema, RawRichConf, #{}),
+    ok = save_to_emqx_config(to_plainmap(RichConf), to_plainmap(RawRichConf)).
+
+-spec save_configs(raw_config(), Opts) -> ok | {error, term()}
+  when Opts :: #{overridden_keys => all | [binary()]}.
+save_configs(RawConf, Opts) ->
+    {_MappedEnvs, RichConf} = hocon_schema:map_translate(emqx_schema, to_richmap(RawConf), #{}),
+    save_to_emqx_config(to_plainmap(RichConf), RawConf),
+
+    %% We may need also support hot config update for the apps that use application envs.
+    %% If that is the case uncomment the following line to update the configs to application env
+    %save_to_app_env(_MappedEnvs),
+
+    %% We don't save the entire config to emqx_override.conf, but only the sub configs
+    %% specified by RootKeys
+    case maps:get(overridden_keys, Opts, all) of
+        all -> save_to_override_conf(RawConf);
+        RootKeys -> save_to_override_conf(maps:with(RootKeys, RawConf))
+    end.
+
+-spec save_to_app_env([tuple()]) -> ok.
+save_to_app_env(AppEnvs) ->
+    lists:foreach(fun({AppName, Envs}) ->
+            [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs]
+        end, AppEnvs).
+
+-spec save_to_emqx_config(config(), raw_config()) -> ok.
+save_to_emqx_config(Conf, RawConf) ->
+    emqx_config:put(emqx_map_lib:unsafe_atom_key_map(Conf)),
+    emqx_config:put_raw(RawConf).
+
+-spec save_to_override_conf(config()) -> ok | {error, term()}.
+save_to_override_conf(Conf) ->
+    FileName = emqx_override_conf_name(),
+    OldConf = load_hocon_file(FileName),
+    MergedConf = maps:merge(OldConf, Conf),
+    ok = filelib:ensure_dir(FileName),
+    case file:write_file(FileName, jsx:prettify(jsx:encode(MergedConf))) of
+        ok -> ok;
+        {error, Reason} ->
+            logger:error("write to ~s failed, ~p", [FileName, Reason]),
+            {error, Reason}
+    end.
+
+load_hocon_file(FileName) ->
+    case filelib:is_regular(FileName) of
+        true ->
+            {ok, Raw0} = hocon:load(FileName, #{format => richmap}),
+            Raw0;
+        false -> #{}
+    end.
+
+emqx_override_conf_name() ->
+    filename:join([emqx_config:get([node, data_dir]), "emqx_override.conf"]).
+
+to_richmap(Map) ->
+    {ok, RichMap} = hocon:binary(jsx:encode(Map), #{format => richmap}),
+    RichMap.
+
+to_plainmap(RichMap) ->
+    hocon_schema:richmap_to_map(RichMap).

apps/emqx/src/emqx_config_handler.erl

@@ -76,10 +76,8 @@ add_handler(ConfKeyPath, HandlerName) ->
 
 -spec init(term()) -> {ok, state()}.
 init(_) ->
-    RawConf = load_config_file(),
-    {_MappedEnvs, Conf} = hocon_schema:map_translate(emqx_schema, RawConf, #{}),
-    ok = save_config_to_emqx(to_plainmap(Conf), to_plainmap(RawConf)),
     {ok, #{handlers => #{?MOD => ?MODULE}}}.
+
 handle_call({add_child, ConfKeyPath, HandlerName}, _From,
             State = #{handlers := Handlers}) ->
     {reply, ok, State#{handlers =>
@@ -89,7 +87,7 @@ handle_call({update_config, ConfKeyPath, UpdateReq, RawConf}, _From,
             #{handlers := Handlers} = State) ->
     OldConf = emqx_config:get(),
     try {RootKeys, Conf} = do_update_config(ConfKeyPath, Handlers, RawConf, UpdateReq),
-        Result = save_configs(RootKeys, Conf),
+        Result = emqx_config:save_configs(Conf, #{overridden_keys => RootKeys}),
         do_post_update_config(ConfKeyPath, Handlers, OldConf, emqx_config:get()),
         {reply, Result, State}
     catch
@@ -165,71 +163,6 @@ merge_to_old_config(UpdateReq, RawConf) when is_map(UpdateReq), is_map(RawConf)
 merge_to_old_config(UpdateReq, _RawConf) ->
     UpdateReq.
 
-%%============================================================================
-save_configs(RootKeys, RawConf) ->
-    {_MappedEnvs, Conf} = hocon_schema:map_translate(emqx_schema, to_richmap(RawConf), #{}),
-    %% We may need also support hot config update for the apps that use application envs.
-    %% If so uncomment the following line to update the configs to application env
-    %save_config_to_app_env(_MappedEnvs),
-    save_config_to_emqx(to_plainmap(Conf), RawConf),
-    save_config_to_disk(RootKeys, RawConf).
-
-% save_config_to_app_env(MappedEnvs) ->
-%     lists:foreach(fun({AppName, Envs}) ->
-%             [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs]
-%         end, MappedEnvs).
-
-save_config_to_emqx(Conf, RawConf) ->
-    ?LOG(debug, "set config: ~p", [Conf]),
-    emqx_config:put(emqx_map_lib:unsafe_atom_key_map(Conf)),
-    emqx_config:put_raw(RawConf).
-
-save_config_to_disk(RootKeys, Conf) ->
-    FileName = emqx_override_conf_name(),
-    OldConf = read_old_config(FileName),
-    %% We don't save the overall config to file, but only the sub configs
-    %% under RootKeys
-    write_new_config(FileName,
-        maps:merge(OldConf, maps:with(RootKeys, Conf))).
-
-write_new_config(FileName, Conf) ->
-    case file:write_file(FileName, jsx:prettify(jsx:encode(Conf))) of
-        ok -> ok;
-        {error, Reason} ->
-            logger:error("write to ~s failed, ~p", [FileName, Reason]),
-            {error, Reason}
-    end.
-
-read_old_config(FileName) ->
-    case file:read_file(FileName) of
-        {ok, Text} ->
-            try jsx:decode(Text, [{return_maps, true}]) of
-                Conf when is_map(Conf) -> Conf;
-                _ -> #{}
-            catch _Err : _Reason ->
-                #{}
-            end;
-        _ -> #{}
-    end.
-
-load_config_file() ->
-    lists:foldl(fun(ConfFile, Acc) ->
-            {ok, RawConf} = hocon:load(ConfFile, #{format => richmap}),
-            emqx_map_lib:deep_merge(Acc, RawConf)
-        end, #{}, application:get_env(emqx, config_files, [])).
-
-emqx_override_conf_name() ->
-    File = filename:join([emqx_config:get([node, data_dir]), "emqx_override.conf"]),
-    ok = filelib:ensure_dir(File),
-    File.
-
-to_richmap(Map) ->
-    {ok, RichMap} = hocon:binary(jsx:encode(Map), #{format => richmap}),
-    RichMap.
-
-to_plainmap(RichMap) ->
-    hocon_schema:richmap_to_map(RichMap).
-
 bin(A) when is_atom(A) -> list_to_binary(atom_to_list(A));
 bin(B) when is_binary(B) -> B;
 bin(S) when is_list(S) -> list_to_binary(S).

apps/emqx/src/emqx_plugins.erl

@@ -44,7 +44,7 @@
 %% @doc Load all plugins when the broker started.
 -spec(load() -> ok | ignore | {error, term()}).
 load() ->
-    ok = load_ext_plugins(emqx_config:get([plugins, expand_plugins_dir])).
+    ok = load_ext_plugins(emqx_config:get([plugins, expand_plugins_dir], undefined)).
 
 %% @doc Load a Plugin
 -spec(load(atom()) -> ok | {error, term()}).