fix(data_dir): get data_dir config in the right way

now data_dir config is respected by boot script as well as
EMQX application after booted

apps/emqx_conf/etc/emqx_conf.conf

@@ -1,8 +1,8 @@
 ## NOTE:
 ## Configs in this file might be overridden by:
 ## 1. Environment variables which start with 'EMQX_' prefix
-## 2. File {{ platform_data_dir }}/configs/cluster-override.conf
-## 3. File {{ platform_data_dir }}/configs/local-override.conf
+## 2. File $EMQX_NODE__DATA_DIR/configs/cluster-override.conf
+## 3. File $EMQX_NODE__DATA_DIR/configs/local-override.conf
 ##
 ## The *-override.conf files are overwritten at runtime when changes
 ## are made from EMQX dashboard UI, management HTTP API, or CLI.
@@ -30,8 +30,8 @@ node {
   ##
   ## @doc node.data_dir
   ## ValueType: Folder
-  ## Default: "{{ platform_data_dir }}/"
-  data_dir = "{{ platform_data_dir }}/"
+  ## Default: "{{ platform_data_dir }}"
+  data_dir = "{{ platform_data_dir }}"
 
   ## Location of crash dump file.
   ##

apps/emqx_conf/etc/emqx_conf.md

@@ -9,14 +9,18 @@ From bottom up:
 
 1. Immutable base: `emqx.conf` + `EMQX_` prefixed environment variables.<br>
    Changes in this layer require a full node restart to take effect.
-1. Cluster overrides: `${data_dir}/cluster-override.conf`
-1. Local node overrides: `${data_dir}/configs/local-override.conf`
+1. Cluster overrides: `$EMQX_NODE__DATA_DIR/configs/cluster-override.conf`
+1. Local node overrides: `$EMQX_NODE__DATA_DIR/configs/local-override.conf`
 
-Where `${data_dir}` is configurable from `node.data_dir`.
+When environment variable `$EMQX_NODE__DATA_DIR` is not set, config `node.data_dir`
+is used.
 
 The `*-override.conf` files are overwritten at runtime when changes
 are made from dashboard UI, management HTTP API, or CLI.
 
+**NOTE** Config values from `*-override.conf` are **not** mapped to boot configs for
+the config feilds attributed with `mapping: path.to.boot.config.key`
+
 For detailed override rules, see [Config Overlay Rules](#config-overlay-rules).
 
 ## Syntax

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -264,7 +264,21 @@ fields("node") ->
        sc(string(),
           #{ required => true,
              mapping => "emqx.data_dir",
-             desc => "Path to the persistent data directory. It must be unique per broker instance."
+             desc =>
+"""
+Path to the persistent data directory.
+Possible auto-created sub-directories are:
+  - `mnesia/\<node_name>`: EMQX's built-in database directory.
+    For example, `mnesia/emqx@127.0.0.1`.
+    There should be only one one such sub dirrectory.
+    Meaning, in case the node is to be renamed (to e.g. `emqx@10.0.1.1`),
+    the old dir should be deleted first.
+  - `configs`: Generated configs at boot time, and cluster/local override configs.
+  - `patches`: Hot-patch beam files are to be placed here.
+  - `trace`: Trace log files.
+
+**NOTE**: One data dir can not be shared by two or more EMQX nodes.
+"""
            })}
     , {"config_files",
        sc(list(string()),

bin/emqx

@@ -28,13 +28,6 @@ WHOAMI=$(whoami)
 # Make sure log directory exists
 mkdir -p "$RUNNER_LOG_DIR"
 
-# Make sure data directory exists
-mkdir -p "$RUNNER_DATA_DIR"
-
-# Make sure data/configs exists
-CONFIGS_DIR="$RUNNER_DATA_DIR/configs"
-mkdir -p "$CONFIGS_DIR"
-
 # hocon try to read environment variables starting with "EMQX_"
 export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'
 
@@ -360,7 +353,7 @@ call_hocon() {
 
 get_config_value() {
     path_to_value="$1"
-    call_hocon -s "$SCHEMA_MOD" -I "$CONFIGS_DIR/" -c "$RUNNER_ETC_DIR"/emqx.conf get "$path_to_value" | tr -d \"
+    call_hocon -s "$SCHEMA_MOD" -c "$RUNNER_ETC_DIR"/emqx.conf get "$path_to_value" | tr -d \"
 }
 
 check_license() {
@@ -398,6 +391,15 @@ relx_start_command() {
            "$START_OPTION"
 }
 
+DATA_DIR="$(get_config_value 'node.data_dir')"
+DATA_DIR="${DATA_DIR%/}"
+if [[ $DATA_DIR != /* ]]; then
+    # relative
+    DATA_DIR="${RUNNER_ROOT_DIR}/${DATA_DIR}"
+fi
+CONFIGS_DIR="$DATA_DIR/configs"
+mkdir -p "$CONFIGS_DIR"
+
 # Function to generate app.config and vm.args
 generate_config() {
     local name_type="$1"
@@ -414,7 +416,7 @@ generate_config() {
     ## NOTE: the generate command merges environment variables to the base config (emqx.conf),
     ## but does not include the cluster-override.conf and local-override.conf
     ## meaning, certain overrides will not be mapped to app.<time>.config file
-    call_hocon -v -t "$NOW_TIME" -I "$CONFIGS_DIR/" -s "$SCHEMA_MOD" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate
+    call_hocon -v -t "$NOW_TIME" -s "$SCHEMA_MOD" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$DATA_DIR"/configs generate
 
     ## filenames are per-hocon convention
     local CONF_FILE="$CONFIGS_DIR/app.$NOW_TIME.config"
@@ -573,7 +575,7 @@ fi
 
 # force to use 'emqx' short name
 [ -z "$NAME" ] && NAME='emqx'
-MNESIA_DATA_DIR="$RUNNER_DATA_DIR/mnesia/$NAME"
+MNESIA_DATA_DIR="$DATA_DIR/mnesia/$NAME"
 
 case "$NAME" in
     *@*)
@@ -585,7 +587,7 @@ esac
 SHORT_NAME="$(echo "$NAME" | awk -F'@' '{print $1}')"
 export ESCRIPT_NAME="$SHORT_NAME"
 
-PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"
+PIPE_DIR="${PIPE_DIR:-/$DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"
 
 ## make EMQX_NODE_COOKIE right
 if [ -n "${EMQX_NODE_COOKIE:-}" ]; then

rel/emqx_vars

@@ -11,7 +11,6 @@ RUNNER_BIN_DIR="{{ runner_bin_dir }}"
 RUNNER_LOG_DIR="{{ runner_log_dir }}"
 RUNNER_LIB_DIR="{{ runner_lib_dir }}"
 RUNNER_ETC_DIR="{{ runner_etc_dir }}"
-RUNNER_DATA_DIR="{{ runner_data_dir }}"
 RUNNER_USER="{{ runner_user }}"
 IS_ELIXIR="{{ is_elixir }}"
 SCHEMA_MOD="{{ emqx_schema_mod }}"
@@ -22,6 +21,5 @@ export EMQX_DESCRIPTION='{{ emqx_description }}'
 ## computed vars
 REL_NAME="emqx"
 ERTS_PATH="$RUNNER_ROOT_DIR/erts-$ERTS_VSN/bin"
-RUNNER_DATA_DIR="${EMQX_NODE__DATA_DIR:-$RUNNER_DATA_DIR}"
 
 ## updated vars here