feat(conf): use hocon schema

.ci/docker-compose-file/conf.env

@@ -11,3 +11,4 @@ EMQX_AUTH__PGSQL__DATABASE=mqtt
 EMQX_AUTH__REDIS__SERVER=redis_server:6379
 EMQX_AUTH__REDIS__PASSWORD=public
 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+HOCON_ENV_OVERRIDE_PREFIX=EMQX_

bin/emqx

@@ -20,8 +20,8 @@ mkdir -p "$RUNNER_LOG_DIR"
 # Make sure data directory exists
 mkdir -p "$RUNNER_DATA_DIR"
 
-# cuttlefish try to read environment variables starting with "EMQX_"
-export CUTTLEFISH_ENV_OVERRIDE_PREFIX='EMQX_'
+# hocon try to read environment variables starting with "EMQX_"
+export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'
 
 relx_usage() {
     command="$1"
@@ -123,8 +123,8 @@ fi
 # Echo to stderr on errors
 echoerr() { echo "$@" 1>&2; }
 
-# By default, use cuttlefish to generate app.config and vm.args
-CUTTLEFISH="${USE_CUTTLEFISH:-yes}"
+# By default, use hocon to generate app.config and vm.args
+HOCON="${USE_HOCON:-yes}"
 
 SED_REPLACE="sed -i "
 case $(sed --help 2>&1) in
@@ -202,7 +202,7 @@ generate_config() {
     ## changing the config 'log.rotation.size'
     rm -rf "${RUNNER_LOG_DIR}"/*.siz
 
-    if [ "$CUTTLEFISH" != "yes" ]; then
+    if [ "$HOCON" != "yes" ]; then
         # Note: we have added a parameter '-vm_args' to this. It
         # appears redundant but it is not! the erlang vm allows us to
         # access all arguments to the erl command EXCEPT '-args_file',
@@ -217,25 +217,26 @@ generate_config() {
 
         set +e
         # shellcheck disable=SC2086
-        CUTTLEFISH_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -v -i "$REL_DIR"/emqx.schema $EMQX_LICENSE_CONF_OPTION -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)"
+        HOCON_OUTPUT="$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/hocon -s emqx_schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)"
+        echo $HOCON_OUTPUT
         # shellcheck disable=SC2181
         RESULT=$?
         set -e
         if [ $RESULT -gt 0 ]; then
-            echo "$CUTTLEFISH_OUTPUT"
+            echo "$HOCON_OUTPUT"
             exit $RESULT
         fi
         # print override from environment variables (EMQX_*)
-        echo "$CUTTLEFISH_OUTPUT" | sed -e '$d'
-        CONFIG_ARGS=$(echo "$CUTTLEFISH_OUTPUT" | tail -n 1)
+        echo "$HOCON_OUTPUT" | sed -e '$d'
+        CONFIG_ARGS=$(echo "$HOCON_OUTPUT" | tail -n 1)
 
-        ## Merge cuttlefish generated *.args into the vm.args
-        CUTTLE_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}')
+        ## Merge hocon generated *.args into the vm.args
+        HOCON_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}')
         TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp"
         cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE"
         echo "" >> "$TMP_ARG_FILE"
         echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE"
-        sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
+        sed '/^#/d' "$HOCON_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
             ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}')
             ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}')
             TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}')
@@ -247,7 +248,7 @@ generate_config() {
                 fi
             fi
         done
-        mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE"
+        mv -f "$TMP_ARG_FILE" "$HOCON_GEN_ARG_FILE"
     fi
 
     # shellcheck disable=SC2086
@@ -303,7 +304,7 @@ if [ -z "$NAME_ARG" ]; then
             NODENAME="$(grep -E '^-name' "$LATEST_VM_ARGS" | awk '{print $2}')"
         else
             # for boot commands, inspect emqx.conf for node name
-            NODENAME=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.name)
+            NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}'| tr -d \")
         fi
     fi
     if [ -z "$NODENAME" ]; then
@@ -329,7 +330,7 @@ PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"
 COOKIE="${EMQX_NODE_COOKIE:-}"
 if [ -z "$COOKIE" ]; then
     if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
-        COOKIE=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie)
+        COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | awk -F"= " '{print $NF}')
     else
         # shellcheck disable=SC2012,SC2086
         LATEST_VM_ARGS="$(ls -t $RUNNER_DATA_DIR/configs/vm.*.args | head -1)"

etc/emqx.conf

@@ -598,42 +598,42 @@ log.rotation.count = 5
 ## Notice: Disable the option in production deployment!
 ##
 ## Value: true | false
-allow_anonymous = true
+acl.allow_anonymous = true
 
 ## Allow or deny if no ACL rules matched.
 ##
 ## Value: allow | deny
-acl_nomatch = allow
+acl.acl_nomatch = allow
 
 ## Default ACL File.
 ##
 ## Value: File Name
-acl_file = "{{ platform_etc_dir }}/acl.conf"
+acl.acl_file = "{{ platform_etc_dir }}/acl.conf"
 
 ## Whether to enable ACL cache.
 ##
 ## If enabled, ACLs roles for each client will be cached in the memory
 ##
 ## Value: on | off
-enable_acl_cache = on
+acl.enable_acl_cache = on
 
 ## The maximum count of ACL entries can be cached for a client.
 ##
 ## Value: Integer greater than 0
 ## Default: 32
-acl_cache_max_size = 32
+acl.acl_cache_max_size = 32
 
 ## The time after which an ACL cache entry will be deleted
 ##
 ## Value: Duration
 ## Default: 1 minute
-acl_cache_ttl = 1m
+acl.acl_cache_ttl = 1m
 
 ## The action when acl check reject current operation
 ##
 ## Value: ignore | disconnect
 ## Default: ignore
-acl_deny_action = ignore
+acl.acl_deny_action = ignore
 
 ## Specify the global flapping detect policy.
 ## The value is a string composed of flapping threshold, duration and banned interval.
@@ -642,7 +642,7 @@ acl_deny_action = ignore
 ## 3. banned interval: the banned interval if a flapping is detected.
 ##
 ## Value: Integer,Duration,Duration
-flapping_detect_policy = "30, 1m, 5m"
+acl.flapping_detect_policy = "30, 1m, 5m"
 
 ##--------------------------------------------------------------------
 ## MQTT Protocol
@@ -2155,7 +2155,7 @@ listener.wss.external.check_origins = "https://localhost:8084, https://127.0.0.1
 ## The file to store loaded module names.
 ##
 ## Value: File
-modules.loaded_file = "{{ platform_data_dir }}/loaded_modules"
+module.loaded_file = "{{ platform_data_dir }}/loaded_modules"
 
 ##--------------------------------------------------------------------
 ## Presence Module
@@ -2204,8 +2204,8 @@ module.presence.qos = 1
 ## Rewrite Module
 
 ## {rewrite, Topic, Re, Dest}
-## module.rewrite.pub.rule.1 = "x/# ^x/y/(.+)$ z/y/$1"
-## module.rewrite.sub.rule.1 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2"
+## module.rewrite.pub_rule.1 = "x/# ^x/y/(.+)$ z/y/$1"
+## module.rewrite.sub_rule.1 = "y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2"
 
 ## CONFIG_SECTION_END=modules ==================================================
 

rebar.config

@@ -7,7 +7,7 @@
 %% rebar.config.rendered if environment DEBUG is set.
 
 {edoc_opts, [{preprocess,true}]}.
-{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import,
+{erl_opts, [warn_unused_vars,warn_shadow_vars,
             warn_obsolete_guard,compressed,
             {d, snk_kind, msg}]}.
 
@@ -56,6 +56,7 @@
     , {observer_cli, "1.6.1"} % NOTE: depends on recon 2.5.1
     , {getopt, "1.0.1"}
     , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.2.0"}}}
     ]}.
 
 {xref_ignores,

rebar.config.erl

@@ -336,9 +336,8 @@ relx_overlay(ReleaseType) ->
     , {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}
     , {copy, "bin/nodetool", "bin/nodetool"}
     , {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
-    , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish"}
-    , {copy, "_build/default/lib/cuttlefish/cuttlefish", "bin/cuttlefish-{{release_version}}"}
-    , {copy, "priv/emqx.schema", "releases/{{release_version}}/"}
+    , {copy, "_build/default/lib/hocon/hocon", "bin/hocon"}
+    , {copy, "_build/default/lib/hocon/hocon", "bin/hocon-{{release_version}}"}
     ] ++ case is_enterprise() of
              true -> ee_etc_overlay(ReleaseType);
              false -> etc_overlay(ReleaseType)

test/emqx_listeners_SUITE.erl

@@ -65,10 +65,9 @@ mustache_vars() ->
     ].
 
 generate_config() ->
-    Schema = cuttlefish_schema:files([local_path(["priv", "emqx.schema"])]),
     ConfFile = render_config_file(),
-    {ok, Conf} = hocon:load(ConfFile, #{format => proplists}),
-    cuttlefish_generator:map(Schema, Conf).
+    {ok, Conf} = hocon:load(ConfFile, #{format => richmap}),
+    hocon_schema:generate(emqx_schema, Conf).
 
 set_app_env({App, Lists}) ->
     lists:foreach(fun({acl_file, _Var}) ->