feat: add base.hocon

apps/emqx/src/emqx_config.erl

@@ -440,12 +440,13 @@ do_parse_hocon(true, Conf, IncDirs) ->
 do_parse_hocon(false, Conf, IncDirs) ->
     Opts = #{format => map, include_dirs => IncDirs},
     case is_binary(Conf) of
-        %% only use in test
         true ->
+            %% only used in test
             hocon:binary(Conf, Opts);
         false ->
+            BaseHocon = base_hocon_file(),
             ClusterFile = cluster_hocon_file(),
-            hocon:files([ClusterFile | Conf], Opts)
+            hocon:files([BaseHocon, ClusterFile | Conf], Opts)
     end.
 
 include_dirs() ->
@@ -541,12 +542,12 @@ ensure_file_deleted(F) ->
 
 -spec read_override_conf(map()) -> raw_config().
 read_override_conf(#{} = Opts) ->
-    File =
+    Files =
         case has_deprecated_file() of
-            true -> deprecated_conf_file(Opts);
-            false -> cluster_hocon_file()
+            true -> [deprecated_conf_file(Opts)];
+            false -> [base_hocon_file(), cluster_hocon_file()]
         end,
-    load_hocon_file(File, map).
+    load_hocon_files(Files, map).
 
 %% @doc Return `true' if this node is upgraded from older version which used cluster-override.conf for
 %% cluster-wide config persistence.
@@ -564,6 +565,9 @@ deprecated_conf_file(Opts) when is_map(Opts) ->
 deprecated_conf_file(Which) when is_atom(Which) ->
     application:get_env(emqx, Which, undefined).
 
+base_hocon_file() ->
+    emqx:etc_file("base.hocon").
+
 %% The newer version cluster-wide config persistence file.
 cluster_hocon_file() ->
     application:get_env(emqx, cluster_hocon_file, undefined).
@@ -633,16 +637,29 @@ save_to_override_conf(true = _HasDeprecatedFile, RawConf, Opts) ->
         undefined ->
             ok;
         FileName ->
-            backup_and_write(FileName, hocon_pp:do(RawConf, Opts))
+            backup_and_write(FileName, generate_hocon_content(RawConf, Opts))
     end;
 save_to_override_conf(false = _HasDeprecatedFile, RawConf, Opts) ->
     case cluster_hocon_file() of
         undefined ->
             ok;
         FileName ->
-            backup_and_write(FileName, hocon_pp:do(RawConf, Opts))
+            backup_and_write(FileName, generate_hocon_content(RawConf, Opts))
     end.
 
+generate_hocon_content(RawConf, Opts) ->
+    [
+        cluster_dot_hocon_header(),
+        hocon_pp:do(RawConf, Opts)
+    ].
+
+cluster_dot_hocon_header() ->
+    [
+        "# This file is generated. Do not edit.\n",
+        "# The configs are results of online config changes from UI/API/CLI.\n",
+        "# To persist configs in this file, copy the content to etc/base.hocon.\n"
+    ].
+
 %% @private This is the same human-readable timestamp format as
 %% hocon-cli generated app.<time>.config file name.
 now_time() ->
@@ -730,22 +747,17 @@ remove_handlers() ->
     emqx_sys_mon:remove_handler(),
     ok.
 
-load_hocon_file(FileName, LoadType) ->
-    case filelib:is_regular(FileName) of
-        true ->
-            Opts = #{include_dirs => include_dirs(), format => LoadType},
-            case hocon:load(FileName, Opts) of
-                {ok, Raw0} ->
-                    Raw0;
-                {error, Reason} ->
-                    throw(#{
-                        msg => failed_to_load_conf,
-                        reason => Reason,
-                        file => FileName
-                    })
-            end;
-        false ->
-            #{}
+load_hocon_files(FileNames, LoadType) ->
+    Opts = #{include_dirs => include_dirs(), format => LoadType},
+    case hocon:files(FileNames, Opts) of
+        {ok, Raw0} ->
+            Raw0;
+        {error, Reason} ->
+            throw(#{
+                msg => failed_to_load_conf,
+                reason => Reason,
+                files => FileNames
+            })
     end.
 
 do_get_raw(Path) ->

apps/emqx/test/emqx_common_test_helpers.erl

@@ -547,7 +547,7 @@ force_set_config_file_paths(emqx, Paths) ->
     %% we need init cluster conf, so we can save the cluster conf to the file
     application:set_env(emqx, local_override_conf_file, "local_override.conf"),
     application:set_env(emqx, cluster_override_conf_file, "cluster_override.conf"),
-    application:set_env(emqx, cluster_conf_file, "cluster.hocon"),
+    application:set_env(emqx, cluster_hocon_file, "cluster.hocon"),
     application:set_env(emqx, config_files, Paths);
 force_set_config_file_paths(_, _) ->
     ok.

apps/emqx_conf/etc/base.hocon

@@ -0,0 +1,24 @@
+## Define configurations that can later be overridden through UI/API/CLI.
+##
+## Config precedence order:
+##   etc/base.hocon < cluster.hocon < emqx.conf < environment variables
+
+## Logging configs
+## EMQX provides support for two primary log handlers: `file` and `console`,
+## with an additional `audit` handler specifically designed to always direct logs to files.
+## The system's default log handling behavior can be configured via the environment
+## variable `EMQX_DEFAULT_LOG_HANDLER`, which accepts the following settings:
+##  - `file`: Directs log output exclusively to files.
+##  - `console`: Channels log output solely to the console.
+## It's noteworthy that `EMQX_DEFAULT_LOG_HANDLER` is set to `file`
+## when EMQX is initiated via systemd `emqx.service` file.
+## In scenarios outside systemd initiation, `console` serves as the default log handler.
+## Read more about configs here: {{ emqx_configuration_doc_log }}
+log {
+    file {
+        level = warning
+    }
+    console {
+        level = warning
+    }
+}

apps/emqx_conf/etc/emqx_conf.conf

@@ -1,12 +1,10 @@
-## NOTE:
-## This config file overrides data/configs/cluster.hocon,
-## and is merged with environment variables which start with 'EMQX_' prefix.
+## Place read-only configurations in this file.
+## To define configurations that can later be overridden through UI/API/CLI, add them to `etc/base.hocon`.
 ##
-## Config changes made from EMQX dashboard UI, management HTTP API, or CLI
-## are stored in data/configs/cluster.hocon.
-## To avoid confusion, please do not store the same configs in both files.
+## Config precedence order:
+##   etc/base.hocon < cluster.hocon < emqx.conf < environment variables
 ##
-## See {{ emqx_configuration_doc }} for more details.
+## See {{ emqx_configuration_doc }} for more information.
 ## Configuration full example can be found in etc/examples
 
 node {
@@ -19,23 +17,3 @@ cluster {
   name = emqxcl
   discovery_strategy = manual
 }
-
-## EMQX provides support for two primary log handlers: `file` and `console`, with an additional `audit` handler specifically designed to always direct logs to files.
-## The system's default log handling behavior can be configured via the environment variable `EMQX_DEFAULT_LOG_HANDLER`, which accepts the following settings:
-##
-##   - `file`: Directs log output exclusively to files.
-##   - `console`: Channels log output solely to the console.
-##
-## It's noteworthy that `EMQX_DEFAULT_LOG_HANDLER` is set to `file` when EMQX is initiated via systemd `emqx.service` file.
-## In scenarios outside systemd initiation, `console` serves as the default log handler.
-
-## Read more about configs here: {{ emqx_configuration_doc_log }}
-
-# log {
-#     file {
-#         level = warning
-#     }
-#     console {
-#         level = warning
-#     }
-# }

bin/emqx

@@ -784,7 +784,8 @@ check_config() {
     ## this command checks the configs without generating any files
     call_hocon -v \
         -s "$SCHEMA_MOD" \
-        -c "$DATA_DIR"/configs/cluster.hocon \
+        -c "$EMQX_ETC_DIR"/base.hocon \
+        -c "$CONFIGS_DIR"/cluster.hocon \
         -c "$EMQX_ETC_DIR"/emqx.conf \
         check_schema
 }
@@ -804,7 +805,8 @@ generate_config() {
 
     ## This command populates two files: app.<time>.config and vm.<time>.args
     ## It takes input sources and overlays values in below order:
-    ##   - $DATA_DIR/cluster.hocon (if exists)
+    ##   - etc/base.hocon
+    ##   - $CONFIGS_DIR/cluster.hocon
     ##   - etc/emqx.conf
     ##   - environment variables starts with EMQX_ e.g. EMQX_NODE__ROLE
     ##
@@ -812,7 +814,8 @@ generate_config() {
     ##       because it has to sync cluster.hocon from other nodes.
     call_hocon -v -t "$NOW_TIME" \
         -s "$SCHEMA_MOD" \
-        -c "$DATA_DIR"/configs/cluster.hocon \
+        -c "$EMQX_ETC_DIR"/base.hocon \
+        -c "$CONFIGS_DIR"/cluster.hocon \
         -c "$EMQX_ETC_DIR"/emqx.conf \
         -d "$DATA_DIR"/configs generate
 

changes/ce/feat-14269.en.md

@@ -0,0 +1,20 @@
+Added `etc/base.hocon` config file.
+
+In this release, we introduced a new configuration file, `etc/base.hocon`, to enhance configuration management and clarity.
+
+Previously, since emqx.conf was the only place for manually crafted configurations, and because it sits at the top-most layer
+of the configuration override system, it caused some confusion.
+While mutable (not read-only) configurations set in `emqx.conf` could be changed through the UI, API, or CLI and take effect immediately,
+those changes would not persist after a node restart — leading to inconsistent behavior.
+
+To address this, we’ve added etc/base.hocon as a foundational configuration layer.
+The updated configuration precedence order, from top to bottom, is as follows:
+
+1. Environment variables
+2. `etc/emqx.conf`
+3. `data/configs/cluster.hocon`
+4. `etc/base.hocon`
+
+The `etc/base.hocon` file serves as the base layer for configurations.
+While settings defined here can still be modified after the node starts,
+this layer ensures consistent override behavior.

dev

@@ -318,6 +318,7 @@ generate_app_conf() {
 
     ## This command populates two files: app.<time>.config and vm.<time>.args
     ## It takes input sources and overlays values in below order:
+    ##   - etc/base.hocon
     ##   - $DATA_DIR/cluster.hocon (if exists)
     ##   - etc/emqx.conf
     ##   - environment variables starts with EMQX_ e.g. EMQX_NODE__ROLE
@@ -325,6 +326,7 @@ generate_app_conf() {
     ## NOTE: it's a known issue that cluster.hocon may change right after the node boots up
     ##       because it has to sync cluster.hocon from other nodes.
     call_hocon -v -t "$NOW_TIME" -s "$SCHEMA_MOD" \
+        -c "$EMQX_ETC_DIR"/base.hocon \
         -c "$EMQX_DATA_DIR"/configs/cluster.hocon \
         -c "$EMQX_ETC_DIR"/emqx.conf \
         -d "$EMQX_DATA_DIR"/configs generate
@@ -358,6 +360,7 @@ EOF
 # copy cert files and acl.conf to etc
 copy_other_conf_files() {
     cp -r apps/emqx/etc/certs "$EMQX_ETC_DIR"/
+    cp -r apps/emqx/etc/base.hocon "$EMQX_ETC_DIR"/
     cp apps/emqx_auth/etc/acl.conf "$EMQX_ETC_DIR"/
 }
 

mix.exs

@@ -950,6 +950,12 @@ defmodule EMQXUmbrella.MixProject do
       Path.join(etc, "emqx.conf")
     )
 
+    render_template(
+      "apps/emqx_conf/etc/base.hocon",
+      assigns,
+      Path.join(etc, "base.hocon")
+    )
+
     render_template(
       "rel/emqx_vars",
       assigns,

rebar.config.erl

@@ -552,7 +552,8 @@ emqx_etc_overlay_per_rel(_RelType) ->
 emqx_etc_overlay() ->
     [
         {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"},
-        {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}
+        {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"},
+        {"{{base_dir}}/lib/emqx_conf/etc/base.hocon", "etc/base.hocon"}
     ].
 
 get_vsn(Profile) ->