Merge branch 'dev/v4.3.0' into unify_pgsql_conf

.ci/apps_tests/docker-compose.yaml

@@ -12,6 +12,14 @@ services:
       - ldap_server
     networks:
       - emqx_bridge
+    environment:
+      GITHUB_ACTIONS: ${GITHUB_ACTIONS}
+      GITHUB_TOKEN: ${GITHUB_TOKEN}
+      GITHUB_RUN_ID: ${GITHUB_RUN_ID}
+      GITHUB_SHA: ${GITHUB_SHA}
+      GITHUB_RUN_NUMBER: ${GITHUB_RUN_NUMBER}
+      GITHUB_EVENT_NAME: ${GITHUB_EVENT_NAME}
+      GITHUB_REF: ${GITHUB_REF}
     volumes:
       - ../../.:/emqx
     working_dir: /emqx

.ci/build_packages/tests.sh

@@ -7,7 +7,7 @@ export RELUP_PACKAGE_PATH="/emqx/relup_packages/${EMQX_NAME}"
 # export EMQX_NODE_COOKIE=$(date +%s%N)
 
 emqx_prepare(){
-    mkdir -p ${PACKAGE_PATH}
+    mkdir -p "${PACKAGE_PATH}"
 
     if [ ! -d "/paho-mqtt-testing" ]; then
         git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho-mqtt-testing
@@ -16,28 +16,28 @@ emqx_prepare(){
 }
 
 emqx_test(){
-    cd ${PACKAGE_PATH}
+    cd "${PACKAGE_PATH}"
 
-    for var in $(ls $PACKAGE_PATH/${EMQX_NAME}-*);do
+    for var in "$PACKAGE_PATH"/"${EMQX_NAME}"-*;do
         case ${var##*.} in
             "zip")
-                packagename=`basename ${PACKAGE_PATH}/${EMQX_NAME}-*.zip`
-                unzip -q ${PACKAGE_PATH}/$packagename
-                sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" ${PACKAGE_PATH}/emqx/etc/emqx.conf
-                sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" ${PACKAGE_PATH}/emqx/etc/emqx.conf
-                sed -i '/emqx_telemetry/d' ${PACKAGE_PATH}/emqx/data/loaded_plugins
-
-                if [ ! -z $(echo ${EMQX_DEPS_DEFAULT_VSN#v} | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?-(alpha|beta|rc)\.[0-9]") ]; then
-                    if [ ! -d ${PACKAGE_PATH}/emqx/lib/emqx-${EMQX_DEPS_DEFAULT_VSN#v} ] || [ ! -d ${PACKAGE_PATH}/emqx/releases/${EMQX_DEPS_DEFAULT_VSN#v} ] ;then
+                packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip)
+                unzip -q "${PACKAGE_PATH}/${packagename}"
+                sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" "${PACKAGE_PATH}"/emqx/etc/emqx.conf
+                sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" "${PACKAGE_PATH}"/emqx/etc/emqx.conf
+                sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
+
+                if echo "${EMQX_DEPS_DEFAULT_VSN#v}" | grep -qE "[0-9]+\.[0-9]+(\.[0-9]+)?-(alpha|beta|rc)\.[0-9]"; then
+                    if [ ! -d "${PACKAGE_PATH}/emqx/lib/emqx-${EMQX_DEPS_DEFAULT_VSN#v}" ] || [ ! -d "${PACKAGE_PATH}/emqx/releases/${EMQX_DEPS_DEFAULT_VSN#v}" ] ;then
                         echo "emqx zip version error"
                         exit 1
                     fi
                 fi
 
                 echo "running ${packagename} start"
-                ${PACKAGE_PATH}/emqx/bin/emqx start || tail ${PACKAGE_PATH}/emqx/log/erlang.log.1
+                "${PACKAGE_PATH}"/emqx/bin/emqx start || tail "${PACKAGE_PATH}"/emqx/log/erlang.log.1
                 IDLE_TIME=0
-                while [ -z "$(${PACKAGE_PATH}/emqx/bin/emqx_ctl status |grep 'is running'|awk '{print $1}')" ]
+                while [ -z "$("${PACKAGE_PATH}"/emqx/bin/emqx_ctl status |grep 'is running'|awk '{print $1}')" ]
                 do
                     if [ $IDLE_TIME -gt 10 ]
                     then
@@ -48,54 +48,54 @@ emqx_test(){
                     IDLE_TIME=$((IDLE_TIME+1))
                 done
                 pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
-                ${PACKAGE_PATH}/emqx/bin/emqx stop
+                "${PACKAGE_PATH}"/emqx/bin/emqx stop
                 echo "running ${packagename} stop"
-                rm -rf ${PACKAGE_PATH}/emqx
+                rm -rf "${PACKAGE_PATH}"/emqx
             ;;
             "deb")
-                packagename=`basename ${PACKAGE_PATH}/${EMQX_NAME}-*.deb`
-                dpkg -i ${PACKAGE_PATH}/$packagename
-                if [ $(dpkg -l |grep emqx |awk '{print $1}') != "ii" ]
+                packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.deb)
+                dpkg -i "${PACKAGE_PATH}/${packagename}"
+                if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "ii" ]
                 then
                     echo "package install error"
                     exit 1
                 fi
-                
+
                 echo "running ${packagename} start"
-                running_test 
+                running_test
                 echo "running ${packagename} stop"
 
-                dpkg -r ${EMQX_NAME} 
-                if [ $(dpkg -l |grep emqx |awk '{print $1}') != "rc" ]
+                dpkg -r "${EMQX_NAME}"
+                if [ "$(dpkg -l |grep emqx |awk '{print $1}')" != "rc" ]
                 then
                     echo "package remove error"
                     exit 1
                 fi
 
-                dpkg -P ${EMQX_NAME}
-                if [ ! -z "$(dpkg -l |grep emqx)" ]
+                dpkg -P "${EMQX_NAME}"
+                if dpkg -l |grep -q emqx
                 then
                     echo "package uninstall error"
                     exit 1
                 fi
             ;;
             "rpm")
-                packagename=`basename ${PACKAGE_PATH}/${EMQX_NAME}-*.rpm`
-                rpm -ivh ${PACKAGE_PATH}/$packagename
-                if [ -z $(rpm -q emqx | grep -o emqx) ];then
+                packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.rpm)
+                rpm -ivh "${PACKAGE_PATH}/${packagename}"
+                if ! rpm -q emqx | grep -q emqx; then
                     echo "package install error"
                     exit 1
                 fi
-                
+
                 echo "running ${packagename} start"
-                running_test 
+                running_test
                 echo "running ${packagename} stop"
-                
-                rpm -e ${EMQX_NAME}
+
+                rpm -e "${EMQX_NAME}"
                 if [ "$(rpm -q emqx)" != "package emqx is not installed" ];then
                     echo "package uninstall error"
                     exit 1
-                fi  
+                fi
             ;;
 
         esac
@@ -103,8 +103,8 @@ emqx_test(){
 }
 
 running_test(){
-    if [ ! -z $(echo ${EMQX_DEPS_DEFAULT_VSN#v} | grep -oE "[0-9]+\.[0-9]+(\.[0-9]+)?-(alpha|beta|rc)\.[0-9]") ]; then
-        if [ ! -d /usr/lib/emqx/lib/emqx-${EMQX_DEPS_DEFAULT_VSN#v} ] || [ ! -d /usr/lib/emqx/releases/${EMQX_DEPS_DEFAULT_VSN#v} ];then
+    if echo "${EMQX_DEPS_DEFAULT_VSN#v}" | grep -qE "[0-9]+\.[0-9]+(\.[0-9]+)?-(alpha|beta|rc)\.[0-9]"; then
+        if [ ! -d /usr/lib/emqx/lib/emqx-"${EMQX_DEPS_DEFAULT_VSN#v}" ] || [ ! -d /usr/lib/emqx/releases/"${EMQX_DEPS_DEFAULT_VSN#v}" ];then
             echo "emqx package version error"
             exit 1
         fi
@@ -127,11 +127,12 @@ running_test(){
         IDLE_TIME=$((IDLE_TIME+1))
     done
     pytest -v /paho-mqtt-testing/interoperability/test_client/V5/test_connect.py::test_basic
-    emqx stop || kill $(ps -ef | grep -E '\-progname\s.+emqx\s' |awk '{print $2}')
+    # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
+    emqx stop || kill "$(ps -ef | grep -E '\-progname\s.+emqx\s' |awk '{print $2}')"
 
-    if [ $(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g') = ubuntu ] \
-    || [ $(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g') = debian ] \
-    || [ $(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g') = raspbian ];then
+    if [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = ubuntu ] \
+    || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = debian ] \
+    || [ "$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')" = raspbian ];then
         service emqx start || tail /var/log/emqx/erlang.log.1
         IDLE_TIME=0
         while [ -z "$(emqx_ctl status |grep 'is running'|awk '{print $1}')" ]
@@ -149,18 +150,18 @@ running_test(){
 }
 
 relup_test(){
-    if [ -d ${RELUP_PACKAGE_PATH} ];then
-        cd ${RELUP_PACKAGE_PATH }
+    if [ -d "${RELUP_PACKAGE_PATH}" ];then
+        cd "${RELUP_PACKAGE_PATH }"
 
-        for var in $(ls ${EMQX_NAME}-*-$(uname -m).zip);do
-            packagename=`basename ${var}`
-            unzip $packagename
+        for var in "${EMQX_NAME}"-*-"$(uname -m)".zip;do
+            packagename=$(basename "${var}")
+            unzip "$packagename"
             ./emqx/bin/emqx start
             ./emqx/bin/emqx_ctl status
             ./emqx/bin/emqx versions
-            cp ${PACKAGE_PATH}/${EMQX_NAME}-*-${EMQX_DEPS_DEFAULT_VSN#v}-$(uname -m).zip ./emqx/releases
-            ./emqx/bin/emqx install ${EMQX_DEPS_DEFAULT_VSN#v}
-            [ $(./emqx/bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]") = ${EMQX_DEPS_DEFAULT_VSN#v} ] || exit 1
+            cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${EMQX_DEPS_DEFAULT_VSN#v}-$(uname -m)".zip ./emqx/releases
+            ./emqx/bin/emqx install "${EMQX_DEPS_DEFAULT_VSN#v}"
+            [ "$(./emqx/bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]")" = "${EMQX_DEPS_DEFAULT_VSN#v}" ] || exit 1
             ./emqx/bin/emqx_ctl status
             ./emqx/bin/emqx stop
             rm -rf emqx

.ci/compatibility_tests/redis/redis.sh

@@ -31,7 +31,7 @@ rm -f \
     /data/conf/nodes.7001.conf \
     /data/conf/nodes.7002.conf ;
 
-if [ ${node} = "cluster" ] ; then
+if [ "${node}" = "cluster" ] ; then
   if $tls ; then
     redis-server /data/conf/redis-tls.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
                                            --tls-port 8000 --cluster-enabled yes ;
@@ -44,7 +44,7 @@ if [ ${node} = "cluster" ] ; then
     redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf --cluster-enabled yes;
     redis-server /data/conf/redis.conf --port 7002 --cluster-config-file /data/conf/nodes.7002.conf --cluster-enabled yes;
   fi
-elif [ ${node} = "sentinel" ] ; then
+elif [ "${node}" = "sentinel" ] ; then
     redis-server /data/conf/redis.conf --port 7000 --cluster-config-file /data/conf/nodes.7000.conf \
                                        --cluster-enabled no;
     redis-server /data/conf/redis.conf --port 7001 --cluster-config-file /data/conf/nodes.7001.conf \
@@ -75,9 +75,9 @@ do
     else
         continue;
     fi
-    if [ ${node} = "cluster" ] ; then
+    if [ "${node}" = "cluster" ] ; then
       yes "yes" | redis-cli --cluster create 172.16.239.10:7000 172.16.239.10:7001 172.16.239.10:7002;
-    elif [ ${node} = "sentinel" ] ; then
+    elif [ "${node}" = "sentinel" ] ; then
       cp /data/conf/sentinel.conf /_sentinel.conf
       redis-server /_sentinel.conf --sentinel;
     fi

.github/workflows/run_test_cases.yaml

@@ -25,6 +25,7 @@ jobs:
             MONGO_TAG: 4
             PGSQL_TAG: 13
             LDAP_TAG: 2.4.50
+            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           run: |
             docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache
             docker-compose -f .ci/apps_tests/docker-compose.yaml up -d
@@ -48,6 +49,7 @@ jobs:
             docker exec -i erlang bash -c "make xref"
             docker exec -i erlang bash -c "make ct"
             docker exec -i erlang bash -c "make cover"
+            docker exec -i erlang bash -c "make coveralls"
         - uses: actions/upload-artifact@v1
           if: failure()
           with:

.github/workflows/shellcheck.yaml

@@ -0,0 +1,18 @@
+name: Shellcheck
+
+on: [pull_request]
+
+jobs:
+  shellcheck:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@master
+      - name: Install shellcheck
+        run: |
+          sudo apt-get update
+          sudo apt install shellcheck
+      - name: Run shellcheck
+        run: |
+          ./scripts/shellcheck.sh
+          echo "success"

Makefile

@@ -31,6 +31,10 @@ get-dashboard:
 eunit: $(REBAR)
 	$(REBAR) eunit
 
+.PHONY: proper
+proper: $(REBAR)
+	$(REBAR) as test proper -d test/props -c
+
 .PHONY: ct
 ct: $(REBAR)
 	$(REBAR) ct --name 'test@127.0.0.1' -c -v
@@ -39,6 +43,10 @@ ct: $(REBAR)
 cover: $(REBAR)
 	$(REBAR) cover
 
+.PHONY: coveralls
+coveralls: $(REBAR)
+	$(REBAR) as test coveralls send
+
 .PHONY: $(REL_PROFILES)
 $(REL_PROFILES:%=%): $(REBAR) get-dashboard
 ifneq ($(shell echo $(@) |grep edge),)

apps/emqx_auth_http/src/emqx_auth_http_app.erl

@@ -74,11 +74,10 @@ translate_env(EnvName) ->
                                                         (_) ->
                                                         true
                                                     end, [{keyfile, KeyFile}, {certfile, CertFile}, {cacertfile, CACertFile}]),
-                            TlsVers = ['tlsv1.2','tlsv1.1',tlsv1],
-                            NTLSOpts = [{versions, TlsVers},
-                                        {ciphers, lists:foldl(fun(TlsVer, Ciphers) ->
-                                                                    Ciphers ++ ssl:cipher_suites(all, TlsVer)
-                                                                end, [], TlsVers)} | TLSOpts],
+                            NTLSOpts = [ {versions, emqx_tls_lib:default_versions()}
+                                       , {ciphers, emqx_tls_lib:default_ciphers()}
+                                       | TLSOpts
+                                       ],
                             [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}]
                         end,
             PoolOpts = [{host, Host},

apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl

@@ -44,12 +44,13 @@ groups() ->
 
 init_per_group(GrpName, Cfg) ->
     Fun = fun(App) -> set_special_configs(GrpName, App) end,
+    emqx_ct_helpers:start_apps([emqx_modules]),
     emqx_ct_helpers:start_apps([emqx_auth_ldap], Fun),
     emqx_mod_acl_internal:unload([]),
     Cfg.
 
 end_per_group(_GrpName, _Cfg) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_ldap]).
+    emqx_ct_helpers:stop_apps([emqx_auth_ldap, emqx_modules]).
 
 %%--------------------------------------------------------------------
 %% Cases

apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl

@@ -36,12 +36,12 @@ all() ->
      check_acl].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx, emqx_auth_ldap], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_ldap], fun set_special_configs/1),
     emqx_mod_acl_internal:unload([]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_ldap, emqx]).
+    emqx_ct_helpers:stop_apps([emqx_auth_ldap, emqx_modules]).
 
 check_auth(_) ->
     MqttUser1 = #{clientid => <<"mqttuser1">>,

apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl

@@ -50,14 +50,14 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Cfg) ->
-    emqx_ct_helpers:start_apps([emqx_auth_mongo], fun set_special_confs/1),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_mongo], fun set_special_confs/1),
     emqx_modules:load_module(emqx_mod_acl_internal, false),
     init_mongo_data(),
     Cfg.
 
 end_per_suite(_Cfg) ->
     deinit_mongo_data(),
-    emqx_ct_helpers:stop_apps([emqx_auth_mongo]).
+    emqx_ct_helpers:stop_apps([emqx_auth_mongo, emqx_modules]).
 
 set_special_confs(emqx) ->
     application:set_env(emqx, acl_nomatch, deny),

apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl

@@ -70,7 +70,7 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_auth_pgsql]),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_pgsql]),
     drop_acl(),
     drop_auth(),
     init_auth(),
@@ -79,7 +79,7 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_pgsql]),
+    emqx_ct_helpers:stop_apps([emqx_auth_pgsql, emqx_modules]),
     Config.
 
 set_special_configs() ->

apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl

@@ -49,13 +49,13 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Cfg) ->
-    emqx_ct_helpers:start_apps([emqx_auth_redis], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_redis], fun set_special_configs/1),
     init_redis_rows(),
     Cfg.
 
 end_per_suite(_Cfg) ->
     deinit_redis_rows(),
-    emqx_ct_helpers:stop_apps([emqx_auth_redis]).
+    emqx_ct_helpers:stop_apps([emqx_auth_redis, emqx_modules]).
 
 set_special_configs(emqx) ->
     application:set_env(emqx, allow_anonymous, false),
@@ -187,4 +187,4 @@ q(Cmd) ->
         _ ->
             {ok, Connection} = ?POOL(?APP),
             eredis:q(Connection, Cmd)
-    end.
+    end.

apps/emqx_bridge_mqtt/README.md

@@ -126,7 +126,7 @@ bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-S
 bridge.mqtt.emqx2.keepalive = 60s
 
 ## Supported TLS version
-bridge.mqtt.emqx2.tls_versions = tlsv1.2,tlsv1.1,tlsv1
+bridge.mqtt.emqx2.tls_versions = tlsv1.3,tlsv1.2,tlsv1.1,tlsv1
 
 ## Forwarding topics of the message
 bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/#

apps/emqx_bridge_mqtt/docs/guide.rst

@@ -133,9 +133,6 @@ EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and co
    ## Key file of Client SSL connection 
    bridge.mqtt.emqx2.keyfile = etc/certs/client-key.pem
 
-   ## SSL encryption
-   bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384
-
    ## TTLS PSK password
    ## Note 'listener.ssl.external.ciphers' and 'listener.ssl.external.psk_ciphers' cannot be configured at the same time
    ##
@@ -146,7 +143,10 @@ EMQ X MQTT bridging principle: Create an MQTT client on the EMQ X broker, and co
    bridge.mqtt.emqx2.keepalive = 60s
 
    ## Supported TLS version
-   bridge.mqtt.emqx2.tls_versions = tlsv1.2,tlsv1.1,tlsv1
+   bridge.mqtt.emqx2.tls_versions = tlsv1.2
+
+   ## SSL encryption
+   bridge.mqtt.emqx2.ciphers = ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384
 
    ## Forwarding topics of the message
    bridge.mqtt.emqx2.forwards = sensor1/#,sensor2/#

apps/emqx_bridge_mqtt/priv/emqx_bridge_mqtt.schema

@@ -90,7 +90,7 @@
 
 {mapping, "bridge.mqtt.$name.tls_versions", "emqx_bridge_mqtt.bridges", [
   {datatype, string},
-  {default, "tlsv1,tlsv1.1,tlsv1.2"}
+  {default, "tlsv1.3,tlsv1.2,tlsv1.1,tlsv1"}
 ]}.
 
 {mapping, "bridge.mqtt.$name.reconnect_interval", "emqx_bridge_mqtt.bridges", [

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl

@@ -671,12 +671,6 @@ format_data([], Msg) ->
 format_data(Tokens, Msg) ->
     emqx_rule_utils:proc_tmpl(Tokens, Msg).
 
-tls_versions() ->
-    ['tlsv1.2','tlsv1.1', tlsv1].
-
-ciphers(Ciphers) ->
-    string:tokens(str(Ciphers), ", ").
-
 subscriptions(Subscriptions) ->
     scan_binary(<<"[", Subscriptions/binary, "].">>).
 
@@ -749,6 +743,8 @@ options(Options, PoolName) ->
                      Topic ->
                          [{subscriptions, [{Topic, Get(<<"qos">>)}]} | Subscriptions]
                  end,
+                 %% TODO check why only ciphers are configurable but not versions
+                 TlsVersions = emqx_tls_lib:default_versions(),
                  [{address, binary_to_list(Address)},
                   {bridge_mode, GetD(<<"bridge_mode">>, true)},
                   {clean_start, true},
@@ -761,12 +757,13 @@ options(Options, PoolName) ->
                   {proto_ver, mqtt_ver(Get(<<"proto_ver">>))},
                   {retry_interval, cuttlefish_duration:parse(str(GetD(<<"retry_interval">>, "30s")), s)},
                   {ssl, cuttlefish_flag:parse(str(Get(<<"ssl">>)))},
-                  {ssl_opts, [{versions, tls_versions()},
-                              {ciphers, ciphers(Get(<<"ciphers">>))},
-                              {keyfile, str(Get(<<"keyfile">>))},
-                              {certfile, str(Get(<<"certfile">>))},
-                              {cacertfile, str(Get(<<"cacertfile">>))}
-                             ]}] ++ Subscriptions1
+                  {ssl_opts, [ {keyfile, str(Get(<<"keyfile">>))}
+                             , {certfile, str(Get(<<"certfile">>))}
+                             , {cacertfile, str(Get(<<"cacertfile">>))}
+                             , {versions, TlsVersions}
+                             , {ciphers, emqx_tls_lib:integral_ciphers(TlsVersions, Get(<<"ciphers">>))}
+                             ]}
+                 ] ++ Subscriptions1
          end.
 
 

apps/emqx_coap/priv/emqx_coap.schema

@@ -75,10 +75,7 @@ end}.
   Ciphers =
       case cuttlefish:conf_get("coap.dtls.ciphers", Conf, undefined) of
           undefined ->
-              lists:foldl(
-                  fun(TlsVer, Ciphers) ->
-                      Ciphers ++ ssl:cipher_suites(all, TlsVer)
-                  end, [], ['dtlsv1', 'dtlsv1.2']);
+              lists:append([ssl:cipher_suites(all, V, openssl) || V <- ['dtlsv1.2', 'dtlsv1']]);
           C ->
               SplitFun(C)
       end,

apps/emqx_exproto/test/emqx_exproto_SUITE.erl

@@ -424,30 +424,28 @@ udp_opts() ->
      {reuseaddr, true}].
 
 ssl_opts() ->
-    Path = emqx_ct_helpers:deps_path(emqx, "etc/certs"),
-    [{versions, ['tlsv1.2','tlsv1.1',tlsv1]},
-     {ciphers, ciphers()},
-     {keyfile, Path ++ "/key.pem"},
-     {certfile, Path ++ "/cert.pem"},
-     {cacertfile, Path ++ "/cacert.pem"},
+    Certs = certs("key.pem", "cert.pem", "cacert.pem"),
+    [{versions, emqx_tls_lib:default_versions()},
+     {ciphers, emqx_tls_lib:default_ciphers()},
      {verify, verify_peer},
      {fail_if_no_peer_cert, true},
      {secure_renegotiate, false},
      {reuse_sessions, true},
-     {honor_cipher_order, true}].
+     {honor_cipher_order, true}]++Certs.
 
 dtls_opts() ->
     Opts = ssl_opts(),
     lists:keyreplace(versions, 1, Opts, {versions, ['dtlsv1.2', 'dtlsv1']}).
 
-ciphers() ->
-    proplists:get_value(ciphers, emqx_ct_helpers:client_ssl()).
-
 %%--------------------------------------------------------------------
 %% Client-Opts
 
 client_ssl_opts() ->
-    Path = emqx_ct_helpers:deps_path(emqx, "etc/certs"),
-    [{keyfile, Path ++ "/client-key.pem"},
-     {certfile, Path ++ "/client-cert.pem"},
-     {cacertfile, Path ++ "/cacert.pem"}].
+    certs( "client-key.pem", "client-cert.pem", "cacert.pem" ).
+
+certs( Key, Cert, CACert ) ->
+    CertsPath = emqx_ct_helpers:deps_path(emqx, "etc/certs"),
+    [ { keyfile,    filename:join([ CertsPath, Key    ]) },
+      { certfile,   filename:join([ CertsPath, Cert   ]) },
+      { cacertfile, filename:join([ CertsPath, CACert ]) } ].
+

apps/emqx_lua_hook/test/emqx_lua_hook_SUITE.erl

@@ -41,11 +41,11 @@ all() ->
     ].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_lua_hook], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_lua_hook], fun set_special_configs/1),
     Config.
 
 end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_lua_hook]),
+    emqx_ct_helpers:stop_apps([emqx_lua_hook, emqx_modules]),
     Config.
 
 set_special_configs(emqx) ->

apps/emqx_management/src/emqx_management.app.src

@@ -3,7 +3,7 @@
   {vsn, "4.3.0"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
-  {applications, [kernel,stdlib,minirest]},
+  {applications, [kernel,stdlib,minirest,emqx_modules]},
   {mod, {emqx_mgmt_app,[]}},
   {env, []},
   {licenses, ["Apache-2.0"]},

apps/emqx_management/test/emqx_mgmt_SUITE.erl

@@ -58,11 +58,11 @@ apps() ->
 init_per_suite(Config) ->
     ekka_mnesia:start(),
     emqx_mgmt_auth:mnesia(boot),
-    emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia]),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]).
+    emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia, emqx_modules]).
 
 t_app(_Config) ->
     {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>),

apps/emqx_management/test/emqx_mgmt_api_SUITE.erl

@@ -58,13 +58,13 @@ groups() ->
     }].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx, emqx_management, emqx_auth_mnesia]),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia]),
     ekka_mnesia:start(),
     emqx_mgmt_auth:mnesia(boot),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_mnesia, emqx_management, emqx]),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia, emqx_management, emqx_modules]),
     ekka_mnesia:ensure_stopped().
 
 init_per_testcase(data, Config) ->

apps/emqx_plugin_template/.gitignore

@@ -1,19 +0,0 @@
-.eunit
-deps
-*.o
-*.beam
-*.plt
-erl_crash.dump
-ebin
-rel/example_project
-.concrete/DEV_MODE
-.rebar
-.erlang.mk/
-data/
-emqx_plugin_template.d
-.DS_Store
-erlang.mk
-_build/
-rebar.lock
-test/ct.cover.spec
-.rebar3

apps/emqx_plugin_template/README.md

@@ -1,34 +0,0 @@
-emqx-plugin-template
-====================
-
-This is a template plugin for the EMQ X broker. And you can see [Plugin Development Guide](https://docs.emqx.io/broker/v3/en/plugins.html#plugin-development-template) to learning how to use it.
-
-Plugin Config
--------------
-
-Each plugin should have a 'etc/{plugin_name}.conf|config' file to store application config.
-
-Authentication and ACL
-----------------------
-
-```
-emqx:hook('client.authenticate', fun ?MODULE:on_client_authenticate/3, [Env]).
-emqx:hook('client.check_acl', fun ?MODULE:on_client_check_acl/5, [Env]).
-```
-
-Plugin and Hooks
------------------
-
-[Plugin Design](https://docs.emqx.io/broker/v3/en/design.html#plugin-design)
-
-[Hooks Design](https://docs.emqx.io/broker/v3/en/design.html#hooks-design)
-
-License
--------
-
-Apache License Version 2.0
-
-Author
-------
-
-EMQ X Team.

apps/emqx_plugin_template/TODO

@@ -1,3 +0,0 @@
-1. Add a script to generate plugin project
-2. Upgrade the README.md
-3. Add the plugin development guide

apps/emqx_plugin_template/etc/emqx_plugin_template.config

@@ -1,5 +0,0 @@
-
-[
-  {emqx_plugin_template, []}
-].
-

apps/emqx_plugin_template/rebar.config

@@ -1,3 +0,0 @@
-{deps, []}.
-
-{erl_opts, [debug_info]}.

apps/emqx_plugin_template/src/emqx_cli_demo.erl

@@ -1,26 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_cli_demo).
-
--export([cmd/1]).
-
-cmd(["arg1", "arg2"]) ->
-    emqx_ctl:print("ok");
-
-cmd(_) ->
-    emqx_ctl:usage([{"cmd arg1 arg2", "cmd demo"}]).
-

apps/emqx_plugin_template/src/emqx_plugin_template.app.src

@@ -1,14 +0,0 @@
-{application, emqx_plugin_template,
- [{description, "EMQ X Plugin Template"},
-  {vsn, "4.3.0"}, % strict semver, bump manually!
-  {modules, []},
-  {registered, [emqx_plugin_template_sup]},
-  {applications, [kernel,stdlib]},
-  {mod, {emqx_plugin_template_app,[]}},
-  {env, []},
-  {licenses, ["Apache-2.0"]},
-  {maintainers, ["EMQ X Team <contact@emqx.io>"]},
-  {links, [{"Homepage", "https://emqx.io/"},
-           {"Github", "https://github.com/emqx/emqx-plugin-template"}
-          ]}
- ]}.

apps/emqx_plugin_template/src/emqx_plugin_template.erl

@@ -1,193 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_plugin_template).
-
--include_lib("emqx/include/emqx.hrl").
-
--export([ load/1
-        , unload/0
-        ]).
-
-%% Client Lifecircle Hooks
--export([ on_client_connect/3
-        , on_client_connack/4
-        , on_client_connected/3
-        , on_client_disconnected/4
-        , on_client_authenticate/3
-        , on_client_check_acl/5
-        , on_client_subscribe/4
-        , on_client_unsubscribe/4
-        ]).
-
-%% Session Lifecircle Hooks
--export([ on_session_created/3
-        , on_session_subscribed/4
-        , on_session_unsubscribed/4
-        , on_session_resumed/3
-        , on_session_discarded/3
-        , on_session_takeovered/3
-        , on_session_terminated/4
-        ]).
-
-%% Message Pubsub Hooks
--export([ on_message_publish/2
-        , on_message_delivered/3
-        , on_message_acked/3
-        , on_message_dropped/4
-        ]).
-
-%% Called when the plugin application start
-load(Env) ->
-    hook('client.connect',      {?MODULE, on_client_connect, [Env]}),
-    hook('client.connack',      {?MODULE, on_client_connack, [Env]}),
-    hook('client.connected',    {?MODULE, on_client_connected, [Env]}),
-    hook('client.disconnected', {?MODULE, on_client_disconnected, [Env]}),
-    hook('client.authenticate', {?MODULE, on_client_authenticate, [Env]}),
-    hook('client.check_acl',    {?MODULE, on_client_check_acl, [Env]}),
-    hook('client.subscribe',    {?MODULE, on_client_subscribe, [Env]}),
-    hook('client.unsubscribe',  {?MODULE, on_client_unsubscribe, [Env]}),
-    hook('session.created',     {?MODULE, on_session_created, [Env]}),
-    hook('session.subscribed',  {?MODULE, on_session_subscribed, [Env]}),
-    hook('session.unsubscribed',{?MODULE, on_session_unsubscribed, [Env]}),
-    hook('session.resumed',     {?MODULE, on_session_resumed, [Env]}),
-    hook('session.discarded',   {?MODULE, on_session_discarded, [Env]}),
-    hook('session.takeovered',  {?MODULE, on_session_takeovered, [Env]}),
-    hook('session.terminated',  {?MODULE, on_session_terminated, [Env]}),
-    hook('message.publish',     {?MODULE, on_message_publish, [Env]}),
-    hook('message.delivered',   {?MODULE, on_message_delivered, [Env]}),
-    hook('message.acked',       {?MODULE, on_message_acked, [Env]}),
-    hook('message.dropped',     {?MODULE, on_message_dropped, [Env]}).
-
-%%--------------------------------------------------------------------
-%% Client Lifecircle Hooks
-%%--------------------------------------------------------------------
-
-on_client_connect(ConnInfo = #{clientid := ClientId}, Props, _Env) ->
-    io:format("Client(~s) connect, ConnInfo: ~p, Props: ~p~n",
-              [ClientId, ConnInfo, Props]),
-    {ok, Props}.
-
-on_client_connack(ConnInfo = #{clientid := ClientId}, Rc, Props, _Env) ->
-    io:format("Client(~s) connack, ConnInfo: ~p, Rc: ~p, Props: ~p~n",
-              [ClientId, ConnInfo, Rc, Props]),
-    {ok, Props}.
-
-on_client_connected(ClientInfo = #{clientid := ClientId}, ConnInfo, _Env) ->
-    io:format("Client(~s) connected, ClientInfo:~n~p~n, ConnInfo:~n~p~n",
-              [ClientId, ClientInfo, ConnInfo]).
-
-on_client_disconnected(ClientInfo = #{clientid := ClientId}, ReasonCode, ConnInfo, _Env) ->
-    io:format("Client(~s) disconnected due to ~p, ClientInfo:~n~p~n, ConnInfo:~n~p~n",
-              [ClientId, ReasonCode, ClientInfo, ConnInfo]).
-
-on_client_authenticate(_ClientInfo = #{clientid := ClientId}, Result, _Env) ->
-    io:format("Client(~s) authenticate, Result:~n~p~n", [ClientId, Result]),
-    {ok, Result}.
-
-on_client_check_acl(_ClientInfo = #{clientid := ClientId}, Topic, PubSub, Result, _Env) ->
-    io:format("Client(~s) check_acl, PubSub:~p, Topic:~p, Result:~p~n",
-              [ClientId, PubSub, Topic, Result]),
-    {ok, Result}.
-
-on_client_subscribe(#{clientid := ClientId}, _Properties, TopicFilters, _Env) ->
-    io:format("Client(~s) will subscribe: ~p~n", [ClientId, TopicFilters]),
-    {ok, TopicFilters}.
-
-on_client_unsubscribe(#{clientid := ClientId}, _Properties, TopicFilters, _Env) ->
-    io:format("Client(~s) will unsubscribe ~p~n", [ClientId, TopicFilters]),
-    {ok, TopicFilters}.
-
-%%--------------------------------------------------------------------
-%% Session Lifecircle Hooks
-%%--------------------------------------------------------------------
-
-on_session_created(#{clientid := ClientId}, SessInfo, _Env) ->
-    io:format("Session(~s) created, Session Info:~n~p~n", [ClientId, SessInfo]).
-
-on_session_subscribed(#{clientid := ClientId}, Topic, SubOpts, _Env) ->
-    io:format("Session(~s) subscribed ~s with subopts: ~p~n", [ClientId, Topic, SubOpts]).
-
-on_session_unsubscribed(#{clientid := ClientId}, Topic, Opts, _Env) ->
-    io:format("Session(~s) unsubscribed ~s with opts: ~p~n", [ClientId, Topic, Opts]).
-
-on_session_resumed(#{clientid := ClientId}, SessInfo, _Env) ->
-    io:format("Session(~s) resumed, Session Info:~n~p~n", [ClientId, SessInfo]).
-
-on_session_discarded(_ClientInfo = #{clientid := ClientId}, SessInfo, _Env) ->
-    io:format("Session(~s) is discarded. Session Info: ~p~n", [ClientId, SessInfo]).
-
-on_session_takeovered(_ClientInfo = #{clientid := ClientId}, SessInfo, _Env) ->
-    io:format("Session(~s) is takeovered. Session Info: ~p~n", [ClientId, SessInfo]).
-
-on_session_terminated(_ClientInfo = #{clientid := ClientId}, Reason, SessInfo, _Env) ->
-    io:format("Session(~s) is terminated due to ~p~nSession Info: ~p~n",
-              [ClientId, Reason, SessInfo]).
-
-%%--------------------------------------------------------------------
-%% Message PubSub Hooks
-%%--------------------------------------------------------------------
-
-%% Transform message and return
-on_message_publish(Message = #message{topic = <<"$SYS/", _/binary>>}, _Env) ->
-    {ok, Message};
-
-on_message_publish(Message, _Env) ->
-    io:format("Publish ~s~n", [emqx_message:format(Message)]),
-    {ok, Message}.
-
-on_message_dropped(#message{topic = <<"$SYS/", _/binary>>}, _By, _Reason, _Env) ->
-    ok;
-on_message_dropped(Message, _By = #{node := Node}, Reason, _Env) ->
-    io:format("Message dropped by node ~s due to ~s: ~s~n",
-              [Node, Reason, emqx_message:format(Message)]).
-
-on_message_delivered(_ClientInfo = #{clientid := ClientId}, Message, _Env) ->
-    io:format("Message delivered to client(~s): ~s~n",
-              [ClientId, emqx_message:format(Message)]),
-    {ok, Message}.
-
-on_message_acked(_ClientInfo = #{clientid := ClientId}, Message, _Env) ->
-    io:format("Message acked by client(~s): ~s~n",
-              [ClientId, emqx_message:format(Message)]).
-
-%% Called when the plugin application stop
-unload() ->
-    emqx:unhook('client.connect',      {?MODULE, on_client_connect}),
-    emqx:unhook('client.connack',      {?MODULE, on_client_connack}),
-    emqx:unhook('client.connected',    {?MODULE, on_client_connected}),
-    emqx:unhook('client.disconnected', {?MODULE, on_client_disconnected}),
-    emqx:unhook('client.authenticate', {?MODULE, on_client_authenticate}),
-    emqx:unhook('client.check_acl',    {?MODULE, on_client_check_acl}),
-    emqx:unhook('client.subscribe',    {?MODULE, on_client_subscribe}),
-    emqx:unhook('client.unsubscribe',  {?MODULE, on_client_unsubscribe}),
-    emqx:unhook('session.created',     {?MODULE, on_session_created}),
-    emqx:unhook('session.subscribed',  {?MODULE, on_session_subscribed}),
-    emqx:unhook('session.unsubscribed',{?MODULE, on_session_unsubscribed}),
-    emqx:unhook('session.resumed',     {?MODULE, on_session_resumed}),
-    emqx:unhook('session.discarded',   {?MODULE, on_session_discarded}),
-    emqx:unhook('session.takeovered',  {?MODULE, on_session_takeovered}),
-    emqx:unhook('session.terminated',  {?MODULE, on_session_terminated}),
-    emqx:unhook('message.publish',     {?MODULE, on_message_publish}),
-    emqx:unhook('message.delivered',   {?MODULE, on_message_delivered}),
-    emqx:unhook('message.acked',       {?MODULE, on_message_acked}),
-    emqx:unhook('message.dropped',     {?MODULE, on_message_dropped}).
-
-hook(Name, MFA) ->
-    case emqx:hook(Name, MFA) of
-        ok -> ok;
-        {error, already_exists} -> ok
-    end.

apps/emqx_plugin_template/src/emqx_plugin_template_app.erl

@@ -1,34 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_plugin_template_app).
-
--behaviour(application).
-
--emqx_plugin(?MODULE).
-
--export([ start/2
-        , stop/1
-        ]).
-
-start(_StartType, _StartArgs) ->
-    {ok, Sup} = emqx_plugin_template_sup:start_link(),
-    emqx_plugin_template:load(application:get_all_env()),
-    {ok, Sup}.
-
-stop(_State) ->
-    emqx_plugin_template:unload().
-

apps/emqx_plugin_template/src/emqx_plugin_template_sup.erl

@@ -1,30 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_plugin_template_sup).
-
--behaviour(supervisor).
-
--export([start_link/0]).
-
--export([init/1]).
-
-start_link() ->
-    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
-
-init([]) ->
-    {ok, { {one_for_all, 0, 1}, []} }.
-

apps/emqx_plugin_template/test/emqx_plugin_template_SUITE.erl

@@ -1,24 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_plugin_template_SUITE).
-
--compile(nowarn_export_all).
--compile(export_all).
-
-all() -> [].
-
-groups() -> [].

apps/emqx_rule_engine/src/emqx_rule_utils.erl

@@ -27,6 +27,7 @@
         , preproc_sql/2
         , proc_sql/2
         , proc_sql_param_str/2
+        , proc_cql_param_str/2
         ]).
 
 %% type converting
@@ -145,8 +146,15 @@ proc_sql(Tokens, Data) ->
 
 -spec(proc_sql_param_str(tmpl_token(), map()) -> binary()).
 proc_sql_param_str(Tokens, Data) ->
+    proc_param_str(Tokens, Data, fun quote_sql/1).
+
+-spec(proc_cql_param_str(tmpl_token(), map()) -> binary()).
+proc_cql_param_str(Tokens, Data) ->
+    proc_param_str(Tokens, Data, fun quote_cql/1).
+
+proc_param_str(Tokens, Data, Quote) ->
     iolist_to_binary(
-      proc_tmpl(Tokens, Data, #{return => rawlist, var_trans => fun quote/1})).
+      proc_tmpl(Tokens, Data, #{return => rawlist, var_trans => Quote})).
 
 %% backward compatibility for hot upgrading from =< e4.2.1
 get_phld_var(Fun, Data) when is_function(Fun) ->
@@ -238,12 +246,23 @@ sql_data(Bool) when is_boolean(Bool) -> Bool;
 sql_data(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
 sql_data(Map) when is_map(Map) -> emqx_json:encode(Map).
 
-quote(List) when is_list(List) -> [$', List, $'];
-quote(Bin) when is_binary(Bin) -> [$', Bin, $'];
-quote(Num) when is_number(Num) -> bin(Num);
-quote(Bool) when is_boolean(Bool) -> bin(Bool);
-quote(Atom) when is_atom(Atom) -> [$', atom_to_binary(Atom, utf8), $'];
-quote(Map) when is_map(Map) -> [$', emqx_json:encode(Map), $'].
+quote_sql(Str) ->
+    quote(Str, <<"\\\\'">>).
+
+quote_cql(Str) ->
+    quote(Str, <<"''">>).
+
+quote(Str, ReplaceWith) when
+        is_list(Str);
+        is_binary(Str);
+        is_atom(Str);
+        is_map(Str) ->
+    [$', escape_apo(bin(Str), ReplaceWith), $'];
+quote(Val, _) ->
+    bin(Val).
+
+escape_apo(Str, ReplaceWith) ->
+    re:replace(Str, <<"'">>, ReplaceWith, [{return, binary}, global]).
 
 str(Bin) when is_binary(Bin) -> binary_to_list(Bin);
 str(Num) when is_number(Num) -> number_to_list(Num);

apps/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl

@@ -116,3 +116,21 @@ t_preproc_sql3(_) ->
     ParamsTokens = emqx_rule_utils:preproc_tmpl(<<"a:${a},b:${b},c:${c},d:${d}">>),
     ?assertEqual(<<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}'">>,
                  emqx_rule_utils:proc_sql_param_str(ParamsTokens, Selected)).
+
+t_preproc_sql4(_) ->
+    %% with apostrophes
+    %% https://github.com/emqx/emqx/issues/4135
+    Selected = #{a => <<"1''2">>, b => 1, c => 1.0,
+                 d => #{d1 => <<"someone's phone">>}},
+    ParamsTokens = emqx_rule_utils:preproc_tmpl(<<"a:${a},b:${b},c:${c},d:${d}">>),
+    ?assertEqual(<<"a:'1\\'\\'2',b:1,c:1.0,d:'{\"d1\":\"someone\\'s phone\"}'">>,
+                 emqx_rule_utils:proc_sql_param_str(ParamsTokens, Selected)).
+
+t_preproc_sql5(_) ->
+    %% with apostrophes for cassandra
+    %% https://github.com/emqx/emqx/issues/4148
+    Selected = #{a => <<"1''2">>, b => 1, c => 1.0,
+                 d => #{d1 => <<"someone's phone">>}},
+    ParamsTokens = emqx_rule_utils:preproc_tmpl(<<"a:${a},b:${b},c:${c},d:${d}">>),
+    ?assertEqual(<<"a:'1''''2',b:1,c:1.0,d:'{\"d1\":\"someone''s phone\"}'">>,
+                 emqx_rule_utils:proc_cql_param_str(ParamsTokens, Selected)).

apps/emqx_telemetry/src/emqx_telemetry.app.src

@@ -3,7 +3,7 @@
   {vsn, "4.3.0"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_telemetry_sup]},
-  {applications, [kernel,stdlib]},
+  {applications, [kernel,stdlib,emqx_modules]},
   {mod, {emqx_telemetry_app,[]}},
   {env, []},
   {licenses, ["Apache-2.0"]},

apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl

@@ -28,11 +28,11 @@ all() -> emqx_ct:all(?MODULE).
 
 init_per_testcase(_, Config) ->
     emqx_ct_helpers:boot_modules(all),
-    emqx_ct_helpers:start_apps([emqx_telemetry]),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_telemetry]),
     Config.
 
 end_per_testcase(_, _Config) ->
-    emqx_ct_helpers:stop_apps([emqx_telemetry]).
+    emqx_ct_helpers:stop_apps([emqx_telemetry, emqx_modules]).
 
 t_uuid(_) ->
     UUID = emqx_telemetry:generate_uuid(),
@@ -63,4 +63,4 @@ t_enable(_) ->
 bin(L) when is_list(L) ->
     list_to_binary(L);
 bin(B) when is_binary(B) ->
-    B.
+    B.

apps/emqx_web_hook/src/emqx_web_hook_actions.erl

@@ -354,12 +354,11 @@ pool_opts(Params = #{<<"url">> := URL}) ->
                                                  (_) ->
                                                   true
                                               end, [{keyfile, KeyFile}, {certfile, CertFile}, {cacertfile, CACertFile}]),
-                       TlsVers = ['tlsv1.2', 'tlsv1.1', tlsv1],
-                       NTLSOpts = [{verify, VerifyType},
-                                   {versions, TlsVers},
-                                   {ciphers, lists:foldl(fun(TlsVer, Ciphers) ->
-                                                             Ciphers ++ ssl:cipher_suites(all, TlsVer)
-                                                         end, [], TlsVers)} | TLSOpts],
+                       NTLSOpts = [ {verify, VerifyType}
+                                  , {versions, emqx_tls_lib:default_versions()}
+                                  , {ciphers, emqx_tls_lib:default_ciphers()}
+                                  | TLSOpts
+                                  ],
                        [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}]
               end,
     [{host, Host},
@@ -397,4 +396,4 @@ test_http_connect(Conf) ->
         Err:Reason:ST ->
            ?LOG(error, "check http_connectivity failed: ~p, ~0p", [Conf, {Err, Reason, ST}]),
            false
-    end.
+    end.

apps/emqx_web_hook/src/emqx_web_hook_app.erl

@@ -75,12 +75,11 @@ translate_env() ->
                        TLSOpts = lists:filter(fun({_K, V}) ->
                                                 V /= <<>> andalso V /= undefined andalso V /= "" andalso true
                                               end, [{keyfile, KeyFile}, {certfile, CertFile}, {cacertfile, CACertFile}]),
-                       TlsVers = ['tlsv1.2','tlsv1.1',tlsv1],
-                       NTLSOpts = [{verify, VerifyType},
-                                   {versions, TlsVers},
-                                   {ciphers, lists:foldl(fun(TlsVer, Ciphers) ->
-                                                               Ciphers ++ ssl:cipher_suites(all, TlsVer)
-                                                           end, [], TlsVers)} | TLSOpts],
+                       NTLSOpts = [ {verify, VerifyType}
+                                  , {versions, emqx_tls_lib:default_versions()}
+                                  , {ciphers, emqx_tls_lib:default_ciphers()}
+                                  | TLSOpts
+                                  ],
                        [{transport, ssl}, {transport_opts, [Inet | NTLSOpts]}]
                 end,
     PoolOpts = [{host, Host},
@@ -114,4 +113,4 @@ parse_host(Host) ->
                 {ok, _} -> {inet6, Host};
                 {error, _} -> {inet, Host}
             end
-    end.
+    end.

apps/emqx_web_hook/test/props/prop_webhook_confs.erl

@@ -62,7 +62,6 @@ do_teardown(_) ->
 
 set_special_cfgs(_) ->
     application:set_env(emqx, plugins_loaded_file, undefined),
-    application:set_env(emqx, modules_loaded_file, undefined),
     ok.
 
 assert_confs([{"web.hook.api.url", Url}|More], Envs) ->

apps/emqx_web_hook/test/props/prop_webhook_hooks.erl

@@ -290,58 +290,22 @@ prop_message_acked() ->
             true
         end).
 
-prop_try_again() ->
-    Setup = fun() ->
-                logger:set_module_level(emqx_web_hook, emergency),
-                meck:new(httpc, [passthrough, no_history]),
-                meck:expect(httpc, request,
-                    fun(Method, {Url, [], ContentType, Body}, _HttpOpts, _Opt) ->
-                        self() ! {Method, Url, ContentType, Body}, {error, get(code)}
-                    end),
-                meck:new(emqx_metrics, [passthrough, no_history]),
-                meck:expect(emqx_metrics, inc, fun(_) -> ok end)
-            end,
-    Teardown = fun() ->
-                   meck:unload(httpc),
-                   meck:unload(emqx_metrics),
-                   logger:set_module_level(emqx_web_hook, debug)
-               end,
-    ?SETUP(fun() -> Setup(), Teardown end,
-        ?FORALL({ConnInfo, ConnProps, Env, Code},
-                {conninfo(), conn_properties(), empty_env(), http_code()},
-                begin
-                    %% pre-set error code
-                    put(code, Code),
-                    %% run hook
-                    ok = emqx_web_hook:on_client_connect(ConnInfo, ConnProps, Env),
-
-                    Bodys = receive_http_request_bodys(),
-                    Body = emqx_json:encode(
-                             #{action => client_connect,
-                               node => stringfy(node()),
-                               clientid => maps:get(clientid, ConnInfo),
-                               username => maybe(maps:get(username, ConnInfo)),
-                               ipaddress => peer2addr(maps:get(peername, ConnInfo)),
-                               keepalive => maps:get(keepalive, ConnInfo),
-                               proto_ver => maps:get(proto_ver, ConnInfo)
-                              }),
-                    [ B = Body || B <- Bodys],
-                    if Code == socket_closed_remotely ->
-                           4 = length(Bodys);
-                       true -> ok
-                    end,
-                    true
-                end)).
-
 %%--------------------------------------------------------------------
 %% Helper
 %%--------------------------------------------------------------------
 do_setup() ->
+    %% Pre-defined envs
+    application:set_env(emqx_web_hook, path, "path"),
+    application:set_env(emqx_web_hook, headers, []),
+
+    meck:new(ehttpc_pool, [passthrough, no_history]),
+    meck:expect(ehttpc_pool, pick_worker, fun(_, _) -> ok end),
+
     Self = self(),
-    meck:new(httpc, [passthrough, no_history]),
-    meck:expect(httpc, request,
-                fun(Method, {Url, [], ContentType, Body}, _HttpOpts, _Opt) ->
-                    Self ! {Method, Url, ContentType, Body}, {ok, ok}
+    meck:new(ehttpc, [passthrough, no_history]),
+    meck:expect(ehttpc, request,
+                fun(_ClientId, Method, {Path, Headers, Body}) ->
+                    Self ! {Method, Path, Headers, Body}, {ok, ok, ok}
                 end),
 
     meck:new(emqx_metrics, [passthrough, no_history]),
@@ -349,7 +313,8 @@ do_setup() ->
     ok.
 
 do_teardown(_) ->
-    meck:unload(httpc),
+    meck:unload(ehttpc_pool),
+    meck:unload(ehttpc),
     meck:unload(emqx_metrics).
 
 maybe(undefined) -> null;
@@ -372,7 +337,7 @@ stringfy(Term) ->
 
 receive_http_request_body() ->
     receive
-        {post, "http://127.0.0.1", "application/json", Body} ->
+        {post, _, _, Body} ->
             Body
     after 100 ->
         exit(waiting_message_timeout)
@@ -383,7 +348,7 @@ receive_http_request_bodys() ->
 
 receive_http_request_bodys_(Acc) ->
     receive
-        {post, "http://127.0.0.1", "application/json", Body} ->
+        {post, _, _, Body} ->
            receive_http_request_bodys_([Body|Acc])
     after 1000 ->
           lists:reverse(Acc)

bin/emqx

@@ -1,11 +1,12 @@
-#!/bin/sh
+#!/bin/bash
 # -*- tab-width:4;indent-tabs-mode:nil -*-
 # ex: ts=4 sw=4 et
 
 set -e
 
-ROOT_DIR="$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"
-. $ROOT_DIR/releases/emqx_vars
+ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
+# shellcheck disable=SC1090
+. "$ROOT_DIR"/releases/emqx_vars
 
 RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME"
 CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}"
@@ -79,7 +80,7 @@ relx_usage() {
 check_user() {
     # Validate that the user running the script is the owner of the
     # RUN_DIR.
-    if ([ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]); then
+    if [ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]; then
         if [ "x$WHOAMI" != "xroot" ]; then
             echo "You need to be root or use sudo to run this command"
             exit 1
@@ -90,13 +91,13 @@ check_user() {
         done
         # This will drop priviledges into the runner user
         # It exec's in a new shell and the current shell will exit
-        exec su - $RUNNER_USER -c "$CMD"
+        exec su - "$RUNNER_USER" -c "$CMD"
     fi
 }
 
 
 # Make sure the user running this script is the owner and/or su to that user
-check_user $@
+check_user "$@"
 ES=$?
 if [ "$ES" -ne 0 ]; then
     exit $ES
@@ -109,7 +110,7 @@ else
 fi
 
 # Warn the user if ulimit -n is less than 1024
-ULIMIT_F=`ulimit -n`
+ULIMIT_F=$(ulimit -n)
 if [ "$ULIMIT_F" -lt 1024 ]; then
     echo "!!!!"
     echo "!!!! WARNING: ulimit -n is ${ULIMIT_F}; 1024 is the recommended minimum."
@@ -133,6 +134,7 @@ esac
 relx_get_pid() {
     if output="$(relx_nodetool rpcterms os getpid)"
     then
+        # shellcheck disable=SC2001 # Escaped quote taken as closing quote in editor
         echo "$output" | sed -e 's/"//g'
         return 0
     else
@@ -143,7 +145,7 @@ relx_get_pid() {
 
 relx_get_nodename() {
     id="longname$(relx_gen_id)-${NAME}"
-    "$BINDIR/erl" -boot "$REL_DIR/start_clean" -eval '[Host] = tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n", [Host]), halt()' -noshell ${NAME_TYPE} $id
+    "$BINDIR/erl" -boot "$REL_DIR/start_clean" -eval '[Host] = tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n", [Host]), halt()' -noshell "${NAME_TYPE}" "$id"
 }
 
 # Connect to a remote node
@@ -154,10 +156,11 @@ relx_rem_sh() {
     # Get the node's ticktime so that we use the same thing.
     TICKTIME="$(relx_nodetool rpcterms net_kernel get_net_ticktime)"
 
+    # shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace
     # Setup remote shell command to control node
     exec "$BINDIR/erl" "$NAME_TYPE" "$id" -remsh "$NAME" -boot "$REL_DIR/start_clean" \
          -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
-         -setcookie "$COOKIE" -hidden -kernel net_ticktime $TICKTIME $EPMD_ARG
+         -setcookie "$COOKIE" -hidden -kernel net_ticktime "$TICKTIME" $EPMD_ARG
 }
 
 # Generate a random id
@@ -171,7 +174,7 @@ relx_nodetool() {
 
     ERL_FLAGS="$ERL_FLAGS $EPMD_ARG" \
     "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \
-                                -setcookie "$COOKIE" "$command" $@
+                                -setcookie "$COOKIE" "$command" "$@"
 }
 
 # Run an escript in the node's environment
@@ -179,7 +182,7 @@ relx_escript() {
     shift; scriptpath="$1"; shift
     export RUNNER_ROOT_DIR
 
-    "$ERTS_DIR/bin/escript" "$ROOTDIR/$scriptpath" $@
+    "$ERTS_DIR/bin/escript" "$ROOTDIR/$scriptpath" "$@"
 }
 
 # Output a start command for the last argument of run_erl
@@ -202,19 +205,19 @@ generate_config() {
         # the vm, we need to pass it in twice.
         CONFIG_ARGS=" -config $RUNNER_ETC_DIR/app.config -args_file $RUNNER_ETC_DIR/vm.args -vm_args $RUNNER_ETC_DIR/vm.args "
     else
-        CONFIG_ARGS=`$ERTS_PATH/escript $RUNNER_ROOT_DIR/bin/cuttlefish -i $REL_DIR/emqx.schema -c $RUNNER_ETC_DIR/emqx.conf -d $RUNNER_DATA_DIR/configs generate`
+        CONFIG_ARGS=$("$ERTS_PATH"/escript "$RUNNER_ROOT_DIR"/bin/cuttlefish -i "$REL_DIR"/emqx.schema -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate)
 
         ## Merge cuttlefish generated *.args into the vm.args
-        CUTTLE_GEN_ARG_FILE=`echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}'`
+        CUTTLE_GEN_ARG_FILE=$(echo "$CONFIG_ARGS" | sed -n 's/^.*\(vm_args[[:space:]]\)//p' | awk '{print $1}')
         TMP_ARG_FILE="$RUNNER_DATA_DIR/configs/vm.args.tmp"
         cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE"
         echo "" >> "$TMP_ARG_FILE"
-        sed '/^#/d' $CUTTLE_GEN_ARG_FILE | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
-            ARG_KEY=`echo "$ARG_LINE" | awk '{$NF="";print}'`
-            ARG_VALUE=`echo "$ARG_LINE" | awk '{print $NF}'`
-            TMP_ARG_VALUE=`grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}'`
+        sed '/^#/d' "$CUTTLE_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
+            ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}')
+            ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}')
+            TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" | awk '{print $NF}')
             if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then
-                if [ ! -z $TMP_ARG_VALUE ]; then
+                if [ -n "$TMP_ARG_VALUE" ]; then
                     sh -c "$SED_REPLACE 's/^$ARG_KEY.*$/$ARG_LINE/' $TMP_ARG_FILE"
                 else
                     echo "$ARG_LINE" >> "$TMP_ARG_FILE"
@@ -224,6 +227,7 @@ generate_config() {
         mv -f "$TMP_ARG_FILE" "$CUTTLE_GEN_ARG_FILE"
     fi
 
+    # shellcheck disable=SC2086
     if ! relx_nodetool chkconfig $CONFIG_ARGS; then
         echoerr "Error reading $CONFIG_ARGS"
         exit 1
@@ -233,7 +237,7 @@ generate_config() {
 # Call bootstrapd for daemon commands like start/stop/console
 bootstrapd() {
     if [ -e "$RUNNER_DATA_DIR/.erlang.cookie" ]; then
-        chown $RUNNER_USER $RUNNER_DATA_DIR/.erlang.cookie
+        chown "$RUNNER_USER" "$RUNNER_DATA_DIR"/.erlang.cookie
     fi
 }
 
@@ -249,8 +253,9 @@ fi
 if [ -z "$NAME_ARG" ]; then
     NODENAME="${EMQX_NODE_NAME:-}"
     # check if there is a node running, inspect its name
-    [ -z "$NODENAME" ] && NODENAME=`ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-name (\S*)' | awk '{print $2}'`
-    [ -z "$NODENAME" ] && NODENAME=`egrep '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-`
+    # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
+    [ -z "$NODENAME" ] && NODENAME=$(ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-name (\S*)' | awk '{print $2}')
+    [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-)
     if [ -z "$NODENAME" ]; then
         echoerr "vm.args needs to have a -name parameter."
         echoerr "  -sname is not supported."
@@ -273,8 +278,9 @@ PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"
 if [ -z "$COOKIE_ARG" ]; then
     COOKIE="${EMQX_NODE_COOKIE:-}"
     # check if there is a node running, steal its cookie
-    [ -z "$COOKIE" ] && COOKIE=`ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-setcookie (\S*)' | awk '{print $2}'`
-    [ -z "$COOKIE" ] && COOKIE=`egrep '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-`
+    # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
+    [ -z "$COOKIE" ] && COOKIE=$(ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-setcookie (\S*)' | awk '{print $2}')
+    [ -z "$COOKIE" ] && COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-)
     if [ -z "$COOKIE" ]; then
         echoerr "vm.args needs to have a -setcookie parameter."
         echoerr "please check $RUNNER_ETC_DIR/emqx.conf"
@@ -288,7 +294,7 @@ fi
 COOKIE="$(echo "$COOKIE_ARG" | awk '{print $2}')"
 
 # Support for IPv6 Dist. See: https://github.com/emqtt/emqttd/issues/1460
-PROTO_DIST=`egrep '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-`
+PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-)
 if [ -z "$PROTO_DIST" ]; then
     PROTO_DIST_ARG=""
 else
@@ -327,7 +333,7 @@ case "$1" in
             exit 1
         fi
         # Bootstrap daemon command (check perms & drop to $RUNNER_USER)
-        bootstrapd $@
+        bootstrapd
 
         # Save this for later.
         CMD=$1
@@ -343,7 +349,7 @@ case "$1" in
                 HEART_OPTION="start_boot"
                 ;;
         esac
-        RUN_PARAM="$@"
+        RUN_PARAM="$*"
 
         # Set arguments for the heart command
         set -- "$RUNNER_SCRIPT" "$HEART_OPTION"
@@ -366,9 +372,9 @@ case "$1" in
                           "$(relx_start_command)"
 
         WAIT_TIME=${WAIT_FOR_ERLANG:-15}
-        while [ $WAIT_TIME -gt 0 ]; do
+        while [ "$WAIT_TIME" -gt 0 ]; do
             if ! relx_nodetool "ping" >/dev/null 2>&1; then
-                WAIT_TIME=`expr $WAIT_TIME - 1`
+                WAIT_TIME=$((WAIT_TIME - 1))
                 sleep 1
                 continue
             fi
@@ -390,14 +396,14 @@ case "$1" in
         if ! relx_nodetool "stop"; then
             exit 1
         fi
-        while $(kill -s 0 "$PID" 2>/dev/null); do
+        while kill -s 0 "$PID" 2>/dev/null; do
             sleep 1
         done
         ;;
 
     restart|reboot)
-        echo "$EMQX_DISCR $REL_VSN is stopped: $($RUNNER_BIN_DIR/emqx stop)"
-        $RUNNER_BIN_DIR/emqx start
+        echo "$EMQX_DISCR $REL_VSN is stopped: $("$RUNNER_BIN_DIR"/emqx stop)"
+        "$RUNNER_BIN_DIR"/emqx start
         ;;
 
     pid)
@@ -416,7 +422,7 @@ case "$1" in
 
     escript)
         ## Run an escript under the node's environment
-        if ! relx_escript $@; then
+        if ! relx_escript "$@"; then
             exit 1
         fi
         ;;
@@ -429,7 +435,7 @@ case "$1" in
         fi
 
         # Bootstrap daemon command (check perms & drop to $RUNNER_USER)
-        bootstrapd $@
+        bootstrapd
 
         shift
         exec "$BINDIR/to_erl" "$PIPE_DIR"
@@ -443,7 +449,7 @@ case "$1" in
         fi
 
         # Bootstrap daemon command (check perms & drop to $RUNNER_USER)
-        bootstrapd $@
+        bootstrapd
 
         shift
         relx_rem_sh
@@ -485,7 +491,7 @@ case "$1" in
 
     console|console_clean|console_boot)
         # Bootstrap daemon command (check perms & drop to $RUNNER_USER)
-        bootstrapd $@
+        bootstrapd
 
         # .boot file typically just $REL_NAME (ie, the app name)
         # however, for debugging, sometimes start_clean.boot is useful.
@@ -519,8 +525,9 @@ case "$1" in
         export PROGNAME
 
         # Store passed arguments since they will be erased by `set`
-        ARGS="$@"
+        ARGS="$*"
 
+        # shellcheck disable=SC2086 # $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
         # Build an array of arguments to pass to exec later on
         # Build it here because this command will be used for logging.
         set -- "$BINDIR/erlexec" -boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \
@@ -529,12 +536,12 @@ case "$1" in
             $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG
 
         # Dump environment info for logging purposes
-        echo "Exec: $@" -- ${1+$ARGS}
+        echo "Exec: $*" -- ${1+$ARGS}
         echo "Root: $ROOTDIR"
 
         # Log the startup
         echo "$RUNNER_ROOT_DIR"
-        logger -t "$REL_NAME[$$]" "Starting up"
+        logger -t "${REL_NAME[$$]}" "Starting up"
 
         # Start the VM
         exec "$@" -- ${1+$ARGS}
@@ -542,7 +549,7 @@ case "$1" in
 
     foreground)
         # Bootstrap daemon command (check perms & drop to $RUNNER_USER)
-        bootstrapd $@
+        bootstrapd
         # start up the release in the foreground for use by runit
         # or other supervision services
 
@@ -560,8 +567,9 @@ case "$1" in
         export PROGNAME
 
         # Store passed arguments since they will be erased by `set`
-        ARGS="$@"
+        ARGS="$*"
 
+        # shellcheck disable=SC2086 # $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
         # Build an array of arguments to pass to exec later on
         # Build it here because this command will be used for logging.
         set -- "$BINDIR/erlexec" $FOREGROUNDOPTIONS \
@@ -571,14 +579,14 @@ case "$1" in
             $RELX_CONFIG_PATH $CONFIG_ARGS $EPMD_ARG
 
         # Dump environment info for logging purposes
-        echo "Exec: $@" -- ${1+$ARGS}
+        echo "Exec: $*" -- ${1+$ARGS}
         echo "Root: $ROOTDIR"
 
         # Start the VM
         exec "$@" -- ${1+$ARGS}
         ;;
     ertspath)
-        echo $ERTS_PATH
+        echo "$ERTS_PATH"
         ;;
     rpc)
         # Make sure a node IS running
@@ -589,7 +597,7 @@ case "$1" in
 
         shift
 
-        relx_nodetool rpc $@
+        relx_nodetool rpc "$@"
         ;;
     rpcterms)
         # Make sure a node IS running
@@ -600,7 +608,7 @@ case "$1" in
 
         shift
 
-        relx_nodetool rpcterms $@
+        relx_nodetool rpcterms "$@"
         ;;
     root_dir)
         # Make sure a node IS running
@@ -620,10 +628,10 @@ case "$1" in
         fi
 
         shift
-        relx_nodetool "eval" $@
+        relx_nodetool "eval" "$@"
         ;;
     *)
-        relx_usage $1
+        relx_usage "$1"
         exit 1
         ;;
 esac

bin/emqx_ctl

@@ -4,8 +4,9 @@
 
 set -e
 
-ROOT_DIR="$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"
-. $ROOT_DIR/releases/emqx_vars
+ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
+# shellcheck disable=SC1090
+. "$ROOT_DIR"/releases/emqx_vars
 
 # Echo to stderr on errors
 echoerr() { echo "$@" 1>&2; }
@@ -18,7 +19,7 @@ fi
 
 relx_get_nodename() {
     id="longname$(relx_gen_id)-${NAME}"
-    "$BINDIR/erl" -boot start_clean -eval '[Host] = tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n", [Host]), halt()' -noshell ${NAME_TYPE} $id
+    "$BINDIR/erl" -boot start_clean -eval '[Host] = tl(string:tokens(atom_to_list(node()),"@")), io:format("~s~n", [Host]), halt()' -noshell "${NAME_TYPE}" "$id"
 }
 
 # Control a node
@@ -34,8 +35,9 @@ relx_nodetool() {
 if [ -z "$NAME_ARG" ]; then
     NODENAME="${EMQX_NODE_NAME:-}"
     # check if there is a node running, inspect its name
-    [ -z "$NODENAME" ] && NODENAME=`ps -ef | grep -E '\progname\s.*emqx\s' | grep -o -E '\-name (\S*)' | awk '{print $2}'`
-    [ -z "$NODENAME" ] && NODENAME=`egrep '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-`
+    # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
+    [ -z "$NODENAME" ] && NODENAME=$(ps -ef | grep -E '\progname\s.*emqx\s' | grep -o -E '\-name (\S*)' | awk '{print $2}')
+    [ -z "$NODENAME" ] && NODENAME=$(grep -E '^[ \t]*node.name[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-)
     if [ -z "$NODENAME" ]; then
         echoerr "vm.args needs to have a -name parameter."
         echoerr "  -sname is not supported."
@@ -54,8 +56,9 @@ NAME="$(echo "$NAME_ARG" | awk '{print $2}')"
 if [ -z "$COOKIE_ARG" ]; then
     COOKIE="${EMQX_NODE_COOKIE:-}"
     # check if there is a node running, steal its cookie
-    [ -z "$COOKIE" ] && COOKIE=`ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-setcookie (\S*)' | awk '{print $2}'`
-    [ -z "$COOKIE" ] && COOKIE=`egrep '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-`
+    # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
+    [ -z "$COOKIE" ] && COOKIE=$(ps -ef | grep -E '\-progname\s.*emqx\s' | grep -o -E '\-setcookie (\S*)' | awk '{print $2}')
+    [ -z "$COOKIE" ] && COOKIE=$(grep -E '^[ \t]*node.cookie[ \t]*=[ \t]*' "$RUNNER_ETC_DIR/emqx.conf" 2> /dev/null | tail -1 | cut -d = -f 2-)
     if [ -z "$COOKIE" ]; then
         echoerr "vm.args needs to have a -setcookie parameter."
         echoerr "please check $RUNNER_ETC_DIR/emqx.conf"
@@ -69,7 +72,7 @@ fi
 COOKIE="$(echo "$COOKIE_ARG" | awk '{print $2}')"
 
 # Support for IPv6 Dist. See: https://github.com/emqtt/emqttd/issues/1460
-PROTO_DIST=`egrep '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' $RUNNER_ETC_DIR/emqx.conf 2> /dev/null | tail -1 | cut -d = -f 2-`
+PROTO_DIST=$(grep -E '^[ \t]*cluster.proto_dist[ \t]*=[ \t]*' "$RUNNER_ETC_DIR"/emqx.conf 2> /dev/null | tail -1 | cut -d = -f 2-)
 if [ -z "$PROTO_DIST" ]; then
     PROTO_DIST_ARG=""
 else

build

@@ -71,7 +71,8 @@ make_relup() {
     local releases=()
     if [ -d "$releases_dir" ]; then
         while read -r dir; do
-            local version="$(basename "$dir")"
+            local version
+            version="$(basename "$dir")"
             # skip current version
             if [ "$version" != "$PKG_VSN" ]; then
                 releases+=( "$version" )

deploy/docker/docker-entrypoint.sh

@@ -128,9 +128,9 @@ try_fill_config() {
     if grep -qE "^[#[:space:]]*$escaped_key\s*=" "$file"; then
         echo_value "$key" "$value"
         if [[ -z "$value" ]]; then
-            echo "$(sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/# \1 = \2/" "$file")" > "$file"
+            sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/# \1 = \2/" "$file" > tmpfile && cat tmpfile > "$file"
         else
-            echo "$(sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/\1 = $escaped_value/" "$file")" > "$file"
+            sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/\1 = $escaped_value/" "$file" > tmpfile && cat tmpfile > "$file"
         fi
     # Check if config has a numbering system, but no existing configuration line in file
     elif echo "$key" | grep -qE '\.\d+|\d+\.'; then
@@ -139,7 +139,7 @@ try_fill_config() {
             template="$(echo "$escaped_key" | sed -r -e 's/\\\.[0-9]+/\\.[0-9]+/g' -e 's/[0-9]+\\\./[0-9]+\\./g')"
             if grep -qE "^[#[:space:]]*$template\s*=" "$file"; then
                 echo_value "$key" "$value"
-                echo "$(sed '$a'\\ "$file")" > "$file"
+                sed '$a'\\ "$file" > tmpfile && cat tmpfile > "$file"
                 echo "$key = $value" >> "$file"
             fi
         fi
@@ -171,12 +171,12 @@ fill_tuples() {
     local elements=${*:2}
     for var in $elements; do
         if grep -qE "\{\s*$var\s*,\s*(true|false)\s*\}\s*\." "$file"; then
-            echo "$(sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file")" > "$file"
+            sed -r "s/\{\s*($var)\s*,\s*(true|false)\s*\}\s*\./{\1, true}./1" "$file" > tmpfile && mv tmpfile "$file"
         elif grep -q "$var\s*\." "$file"; then
             # backward compatible.
-            echo "$(sed -r "s/($var)\s*\./{\1, true}./1" "$file")" > "$file"
+            sed -r "s/($var)\s*\./{\1, true}./1" "$file" > tmpfile && cat tmpfile > "$file"
         else
-            echo "$(sed '$a'\\ "$file")" > "$file"
+            sed '$a'\\ "$file" > tmpfile && cat tmpfile > "$file"
             echo "{$var, true}." >> "$file"
         fi
     done

deploy/packages/rpm/init.script

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 #
 # emqx
 #
@@ -8,11 +8,11 @@
 #
 
 # Source function library.
+# shellcheck disable=SC1091
 . /etc/rc.d/init.d/functions
 
 RETVAL=0
 PATH=/sbin:/usr/sbin:/bin:/usr/bin
-DESC="EMQX, a distributed, massively scalable, highly extensible MQTT message broker written in Erlang/OTP"
 NAME=emqx
 DAEMON=/usr/bin/$NAME
 lockfile=/var/lock/subsys/$NAME
@@ -25,6 +25,7 @@ pidfile=/var/run/$NAME/$NAME.pid
 [ -d /var/lib/$NAME ] || exit 0
 
 # Read configuration variable file if it is present and readable
+# shellcheck disable=SC1090
 [ -r /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME
 
 # `service` strips all environmental VARS so
@@ -34,15 +35,17 @@ if [ -z "$HOME" ]; then
     export HOME=
 fi
 
-status -p $pidfile -l $(basename $lockfile) $NAME >/dev/null 2>&1
+status -p $pidfile -l "$(basename $lockfile)" $NAME >/dev/null 2>&1
 running=$?
 
 find_pid() {
+    # shellcheck disable=SC2009 # pgrep does not support Extended Regular Expressions
     ps ax | grep -E "\-progname\s+$NAME\s" | awk '{print $1}'
 }
 
 check_pid_status() {
-    local pid="$(find_pid)"
+    local pid
+    pid="$(find_pid)"
     if [ "$pid" = "" ]; then
         # prog not running?
         return 1
@@ -72,7 +75,7 @@ stop() {
     # Stop daemon.
     echo -n $"Shutting down emqx: "
     $DAEMON stop 2>/dev/null
-    for n in $(seq 1 10); do
+    for _ in $(seq 1 10); do
         sleep 1
         check_pid_status
         RETVAL=$?
@@ -93,7 +96,7 @@ stop() {
 hardstop() {
     echo -n $"Shutting down $NAME: "
     su - emqx -c "ps -ef | grep -E '\-progname\s+$NAME\s' | awk '{print \$2}' | xargs kill -9"
-    for n in $(seq 1 10); do
+    for _ in $(seq 1 10); do
         sleep 1
         check_pid_status
         RETVAL=$?
@@ -133,7 +136,7 @@ case "$1" in
         restart
         ;;
     status)
-        status -p $pidfile -l $(basename $lockfile) $NAME
+        status -p $pidfile -l "$(basename $lockfile)" $NAME
         ;;
     ping)
         $DAEMON ping || exit $?

etc/emqx.d/acl.conf

@@ -1,26 +0,0 @@
-%%--------------------------------------------------------------------
-%% [ACL](https://docs.emqx.io/broker/v3/en/config.html)
-%%
-%% -type(who() :: all | binary() |
-%%                {ipaddr, esockd_access:cidr()} |
-%%                {client, binary()} |
-%%                {user, binary()}).
-%%
-%% -type(access() :: subscribe | publish | pubsub).
-%%
-%% -type(topic() :: binary()).
-%%
-%% -type(rule() :: {allow, all} |
-%%                 {allow, who(), access(), list(topic())} |
-%%                 {deny, all} |
-%%                 {deny, who(), access(), list(topic())}).
-%%--------------------------------------------------------------------
-
-{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}.
-
-{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}.
-
-{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}.
-
-{allow, all}.
-

etc/emqx.d/ssl_dist.conf

@@ -1,11 +0,0 @@
-%% The options in the {server, Opts} tuple are used when calling ssl:ssl_accept/3,
-%% and the options in the {client, Opts} tuple are used when calling ssl:connect/4.
-%%
-%% More information at: http://erlang.org/doc/apps/ssl/ssl_distribution.html
-[{server,
-  [{certfile, "{{ platform_etc_dir }}/certs/cert.pem"},
-   {keyfile, "{{ platform_etc_dir }}/certs/key.pem"},
-   {secure_renegotiate, true},
-   {depth, 0}]},
- {client,
-  [{secure_renegotiate, true}]}].

lib-opensource/emqx_modules/etc/emqx_modules.conf

@@ -0,0 +1,57 @@
+##--------------------------------------------------------------------
+## Modules
+##--------------------------------------------------------------------
+## The file to store loaded module names.
+##
+## Value: File
+modules.loaded_file = {{ platform_data_dir }}/loaded_modules
+
+##--------------------------------------------------------------------
+## Presence Module
+
+## Sets the QoS for presence MQTT message.
+##
+## Value: 0 | 1 | 2
+module.presence.qos = 1
+
+##--------------------------------------------------------------------
+## Subscription Module
+
+## Subscribe the Topics automatically when client connected.
+##
+## Value: String
+## module.subscription.1.topic = connected/%c/%u
+
+## Qos of the proxy subscription.
+##
+## Value: 0 | 1 | 2
+## Default: 0
+## module.subscription.1.qos = 0
+
+## No Local of the proxy subscription options.
+## This configuration only takes effect in the MQTT V5 protocol.
+##
+## Value: 0 | 1
+## Default: 0
+## module.subscription.1.nl = 0
+
+## Retain As Published of the proxy subscription options.
+## This configuration only takes effect in the MQTT V5 protocol.
+##
+## Value: 0 | 1
+## Default: 0
+## module.subscription.1.rap = 0
+
+## Retain Handling of the proxy subscription options.
+## This configuration only takes effect in the MQTT V5 protocol.
+##
+## Value: 0 | 1 | 2
+## Default: 0
+## module.subscription.1.rh = 0
+
+##--------------------------------------------------------------------
+## Rewrite Module
+
+## {rewrite, Topic, Re, Dest}
+## module.rewrite.pub.rule.1 = x/# ^x/y/(.+)$ z/y/$1
+## module.rewrite.sub.rule.1 = y/+/z/# ^y/(.+)/z/(.+)$ y/z/$2

lib-opensource/emqx_modules/priv/emqx_modules.schema

@@ -0,0 +1,89 @@
+%%--------------------------------------------------------------------
+%% Modules
+%%--------------------------------------------------------------------
+
+{mapping, "modules.loaded_file", "emqx_modules.modules_loaded_file", [
+  {datatype, string}
+]}.
+
+{mapping, "module.presence.qos", "emqx_modules.modules", [
+  {default, 1},
+  {datatype, integer},
+  {validators, ["range:0-2"]}
+]}.
+
+{mapping, "module.subscription.$id.topic", "emqx_modules.modules", [
+  {datatype, string}
+]}.
+
+{mapping, "module.subscription.$id.qos", "emqx_modules.modules", [
+  {default, 1},
+  {datatype, integer},
+  {validators, ["range:0-2"]}
+]}.
+
+{mapping, "module.subscription.$id.nl", "emqx_modules.modules", [
+  {default, 0},
+  {datatype, integer},
+  {validators, ["range:0-1"]}
+]}.
+
+{mapping, "module.subscription.$id.rap", "emqx_modules.modules", [
+  {default, 0},
+  {datatype, integer},
+  {validators, ["range:0-1"]}
+]}.
+
+{mapping, "module.subscription.$id.rh", "emqx_modules.modules", [
+  {default, 0},
+  {datatype, integer},
+  {validators, ["range:0-2"]}
+]}.
+
+{mapping, "module.rewrite.rule.$id", "emqx_modules.modules", [
+  {datatype, string}
+]}.
+
+{mapping, "module.rewrite.pub.rule.$id", "emqx_modules.modules", [
+  {datatype, string}
+]}.
+
+{mapping, "module.rewrite.sub.rule.$id", "emqx_modules.modules", [
+  {datatype, string}
+]}.
+
+{translation, "emqx_modules.modules", fun(Conf, _, Conf1) ->
+  Subscriptions = fun() ->
+      List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf),
+      TopicList = [{N, Topic}|| {[_,"subscription",N,"topic"], Topic} <- List],
+      [{iolist_to_binary(T), #{ qos => cuttlefish:conf_get("module.subscription." ++ N ++ ".qos", Conf, 0),
+                                nl  => cuttlefish:conf_get("module.subscription." ++ N ++ ".nl", Conf, 0),
+                                rap => cuttlefish:conf_get("module.subscription." ++ N ++ ".rap", Conf, 0),
+                                rh  => cuttlefish:conf_get("module.subscription." ++ N ++ ".rh", Conf, 0)
+                                }} || {N, T} <- TopicList]
+  end,
+  Rewrites = fun() ->
+      Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf),
+      PubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.pub.rule", Conf),
+      SubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.sub.rule", Conf),
+      TotalRules = lists:append(
+                     [ {["module", "rewrite", "pub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ PubRules,
+                     [ {["module", "rewrite", "sub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ SubRules
+                    ),
+      lists:map(fun({[_, "rewrite", PubOrSub, "rule", I], Rule}) ->
+                    [Topic, Re, Dest] = string:tokens(Rule, " "),
+                    {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)}
+                end, TotalRules)
+  end,
+  lists:append([
+    [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}],
+    [{emqx_mod_subscription, Subscriptions()}],
+    [{emqx_mod_rewrite, Rewrites()}],
+    [{emqx_mod_topic_metrics, []}],
+    [{emqx_mod_delayed, []}],
+    %% TODO: acl_file config should be moved to emqx_modules.conf
+    %% when all the plubin tests stops using it in the old way.
+    [{emqx_mod_acl_internal, [{acl_file, {emqx, get_env, [acl_file]}}]}]
+    %[{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}]
+  ])
+end}.

lib-opensource/emqx_modules/rebar.config

@@ -0,0 +1 @@
+{deps, []}.

src/emqx_mod_acl_internal.erl

@@ -18,8 +18,8 @@
 
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("logger.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -logger_header("[ACL_INTERNAL]").
 
@@ -43,7 +43,13 @@
 %%--------------------------------------------------------------------
 
 load(Env) ->
-    Rules = rules_from_file(proplists:get_value(acl_file, Env)),
+    %% TODO: acl_file config should be moved to emqx_modules.conf
+    %% when all the plubin tests stops using it in the old way.
+    File = case proplists:get_value(acl_file, Env) of
+               {emqx, get_env, _} -> emqx:get_env(acl_file);
+               F -> F
+           end,
+    Rules = rules_from_file(File),
     emqx_hooks:add('client.check_acl', {?MODULE, check_acl, [Rules]},  -1).
 
 unload(_Env) ->

src/emqx_mod_delayed.erl

@@ -19,8 +19,8 @@
 -behaviour(gen_server).
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("logger.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).

src/emqx_mod_presence.erl

@@ -18,8 +18,8 @@
 
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("logger.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -logger_header("[Presence]").
 

src/emqx_mod_rewrite.erl

@@ -18,8 +18,8 @@
 
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("emqx_mqtt.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 -ifdef(TEST).
 -export([ compile/1

src/emqx_mod_subscription.erl

@@ -18,8 +18,8 @@
 
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("emqx_mqtt.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 %% emqx_gen_mod callbacks
 -export([ load/1

src/emqx_mod_sup.erl

@@ -18,7 +18,7 @@
 
 -behaviour(supervisor).
 
--include("types.hrl").
+-include_lib("emqx/include/types.hrl").
 
 -export([ start_link/0
         , start_child/1

src/emqx_mod_topic_metrics.erl

@@ -19,9 +19,9 @@
 -behaviour(gen_server).
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("logger.hrl").
--include("emqx_mqtt.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 -logger_header("[TOPIC_METRICS]").
 

lib-opensource/emqx_modules/src/emqx_modules.app.src

@@ -0,0 +1,9 @@
+{application, emqx_modules,
+ [{description, "EMQ X Module Management"},
+  {vsn, "4.3.0"},
+  {modules, []},
+  {applications, [kernel,stdlib]},
+  {mod, {emqx_modules_app, []}},
+  {registered, [emqx_mod_sup]},
+  {env, []}
+ ]}.

src/emqx_modules.erl

@@ -16,7 +16,7 @@
 
 -module(emqx_modules).
 
--include("logger.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -logger_header("[Modules]").
 
@@ -30,6 +30,8 @@
         , load_module/2
         ]).
 
+-define(APP, ?MODULE).
+
 %% @doc List all available plugins
 -spec(list() -> [{atom(), boolean()}]).
 list() ->
@@ -38,7 +40,7 @@ list() ->
 %% @doc Load all the extended modules.
 -spec(load() -> ok).
 load() ->
-    case emqx:get_env(modules_loaded_file) of
+    case get_env(modules_loaded_file) of
         undefined -> ok;
         File ->
             load_modules(File)
@@ -59,7 +61,7 @@ load(ModuleName) ->
 %% @doc Unload all the extended modules.
 -spec(unload() -> ok).
 unload() ->
-    case emqx:get_env(modules_loaded_file) of
+    case get_env(modules_loaded_file) of
         undefined -> ignore;
         File ->
             unload_modules(File)
@@ -79,7 +81,7 @@ unload(ModuleName) ->
 
 -spec(reload(module()) -> ok | ignore | {error, any()}).
 reload(emqx_mod_acl_internal) ->
-    Modules = emqx:get_env(modules, []),
+    Modules = get_env(modules, []),
     Env = proplists:get_value(emqx_mod_acl_internal, Modules, undefined),
     case emqx_mod_acl_internal:reload(Env) of
         ok ->
@@ -96,7 +98,7 @@ find_module(ModuleName) ->
     ets:lookup(?MODULE, ModuleName).
 
 filter_module(ModuleNames) ->
-    filter_module(ModuleNames, emqx:get_env(modules, [])).
+    filter_module(ModuleNames, get_env(modules, [])).
 filter_module([], Acc) ->
     Acc;
 filter_module([{ModuleName, true} | ModuleNames], Acc) ->
@@ -123,7 +125,7 @@ load_module(ModuleName) ->
     load_module({ModuleName, true}).
 
 load_module(ModuleName, Persistent) ->
-    Modules = emqx:get_env(modules, []),
+    Modules = get_env(modules, []),
     Env = proplists:get_value(ModuleName, Modules, undefined),
     case ModuleName:load(Env) of
         ok ->
@@ -150,7 +152,7 @@ unload_module(ModuleName) ->
     unload_module({ModuleName, true}).
 
 unload_module(ModuleName, Persistent) ->
-    Modules = emqx:get_env(modules, []),
+    Modules = get_env(modules, []),
     Env = proplists:get_value(ModuleName, Modules, undefined),
     case ModuleName:unload(Env) of
         ok ->
@@ -162,7 +164,7 @@ unload_module(ModuleName, Persistent) ->
     end.
 
 write_loaded(true) ->
-    FilePath = emqx:get_env(modules_loaded_file),
+    FilePath = get_env(modules_loaded_file),
     case file:write_file(FilePath, [io_lib:format("~p.~n", [Name]) || Name <- list()]) of
         ok -> ok;
         {error, Error} ->
@@ -170,3 +172,7 @@ write_loaded(true) ->
             ok
     end;
 write_loaded(false) -> ok.
+
+get_env(Key) -> get_env(Key, undefined).
+
+get_env(Key, Default) -> application:get_env(?APP, Key, Default).

lib-opensource/emqx_modules/src/emqx_modules_app.erl

@@ -0,0 +1,51 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_modules_app).
+
+-behaviour(application).
+
+-emqx_plugin(?MODULE).
+
+-export([start/2]).
+
+-export([stop/1]).
+
+-define(APP, emqx_modules).
+
+start(_Type, _Args) ->
+    % the configs for emqx_modules is so far still in emqx application
+    % Ensure it's loaded
+    application:load(emqx),
+    ok = load_app_env(),
+    {ok, Pid} = emqx_mod_sup:start_link(),
+    ok = emqx_modules:load(),
+    {ok, Pid}.
+
+stop(_State) ->
+    emqx_modules:unload().
+
+load_app_env() ->
+    Schema = filename:join([code:priv_dir(?APP), "emqx_modules.schema"]),
+    Conf1 = filename:join([code:lib_dir(?APP), "etc", "emqx_modules.conf"]),
+    Conf2 = filename:join([emqx:get_env(plugins_etc_dir), "emqx_modules.conf"]),
+    [ConfFile | _] = lists:filter(fun filelib:is_regular/1, [Conf1, Conf2]),
+    Conf = cuttlefish_conf:file(ConfFile),
+    AppEnv = cuttlefish_generator:map(cuttlefish_schema:files([Schema]), Conf),
+    lists:foreach(fun({AppName, Envs}) ->
+        [application:set_env(AppName, Par, Val) || {Par, Val} <- Envs]
+    end, AppEnv).
+

test/emqx_mod_delayed_SUITE.erl

@@ -35,11 +35,11 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps([emqx_modules], fun set_special_configs/1),
     Config.
 
 end_per_suite(_) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 set_special_configs(emqx) ->
     application:set_env(emqx, modules, [{emqx_mod_delayed, []}]),

test/emqx_mod_presence_SUITE.erl

@@ -26,13 +26,13 @@ all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
     emqx_ct_helpers:boot_modules(all),
-    emqx_ct_helpers:start_apps([]),
+    emqx_ct_helpers:start_apps([emqx_modules]),
     %% Ensure all the modules unloaded.
     ok = emqx_modules:unload(),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 %% Test case for emqx_mod_presence
 t_mod_presence(_) ->

test/emqx_mod_rewrite_SUITE.erl

@@ -30,13 +30,13 @@ all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
     emqx_ct_helpers:boot_modules(all),
-    emqx_ct_helpers:start_apps([]),
+    emqx_ct_helpers:start_apps([emqx_modules]),
     %% Ensure all the modules unloaded.
     ok = emqx_modules:unload(),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 %% Test case for emqx_mod_write
 t_mod_rewrite(_Config) ->

test/emqx_mod_topic_metrics_SUITE.erl

@@ -25,11 +25,11 @@ all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
     emqx_ct_helpers:boot_modules(all),
-    emqx_ct_helpers:start_apps([]),
+    emqx_ct_helpers:start_apps([emqx_modules]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 t_nonexistent_topic_metrics(_) ->
     emqx_mod_topic_metrics:load([]),

test/emqx_modules_SUITE.erl

@@ -24,20 +24,20 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([], fun set_sepecial_cfg/1),
+    emqx_ct_helpers:start_apps([emqx_modules]),
+    File = emqx_ct_helpers:deps_path(emqx_modules, "test/emqx_modules_SUITE_data/loaded_modules"),
+    application:set_env(emqx_modules, modules_loaded_file, File),
+    ok = emqx_modules:unload(),
+    ok = emqx_modules:load(),
     Config.
 
-set_sepecial_cfg(_) ->
-    application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")),
-    ok.
-
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 t_load(_) ->
     ?assertEqual(ok, emqx_modules:unload()),
     ?assertEqual(ok, emqx_modules:load()),
-    ?assertEqual({error, not_found}, emqx_modules:load(not_existed_module)),
+    ?assertEqual({error, not_found}, emqx_modules:load(foo)),
     ?assertEqual({error, not_started}, emqx_modules:unload(emqx_mod_rewrite)),
     ?assertEqual(ignore, emqx_modules:reload(emqx_mod_rewrite)),
     ?assertEqual(ok, emqx_modules:reload(emqx_mod_acl_internal)).

priv/emqx.schema

@@ -2020,93 +2020,6 @@ end}.
                                                ++ cuttlefish_variable:filter_by_prefix("listener.wss", Conf)])
 end}.
 
-%%--------------------------------------------------------------------
-%% Modules
-%%--------------------------------------------------------------------
-
-{mapping, "modules.loaded_file", "emqx.modules_loaded_file", [
-  {datatype, string}
-]}.
-
-{mapping, "module.presence.qos", "emqx.modules", [
-  {default, 1},
-  {datatype, integer},
-  {validators, ["range:0-2"]}
-]}.
-
-{mapping, "module.subscription.$id.topic", "emqx.modules", [
-  {datatype, string}
-]}.
-
-{mapping, "module.subscription.$id.qos", "emqx.modules", [
-  {default, 1},
-  {datatype, integer},
-  {validators, ["range:0-2"]}
-]}.
-
-{mapping, "module.subscription.$id.nl", "emqx.modules", [
-  {default, 0},
-  {datatype, integer},
-  {validators, ["range:0-1"]}
-]}.
-
-{mapping, "module.subscription.$id.rap", "emqx.modules", [
-  {default, 0},
-  {datatype, integer},
-  {validators, ["range:0-1"]}
-]}.
-
-{mapping, "module.subscription.$id.rh", "emqx.modules", [
-  {default, 0},
-  {datatype, integer},
-  {validators, ["range:0-2"]}
-]}.
-
-{mapping, "module.rewrite.rule.$id", "emqx.modules", [
-  {datatype, string}
-]}.
-
-{mapping, "module.rewrite.pub.rule.$id", "emqx.modules", [
-  {datatype, string}
-]}.
-
-{mapping, "module.rewrite.sub.rule.$id", "emqx.modules", [
-  {datatype, string}
-]}.
-
-{translation, "emqx.modules", fun(Conf, _, Conf1) ->
-  Subscriptions = fun() ->
-      List = cuttlefish_variable:filter_by_prefix("module.subscription", Conf),
-      TopicList = [{N, Topic}|| {[_,"subscription",N,"topic"], Topic} <- List],
-      [{iolist_to_binary(T), #{ qos => cuttlefish:conf_get("module.subscription." ++ N ++ ".qos", Conf, 0),
-                                nl  => cuttlefish:conf_get("module.subscription." ++ N ++ ".nl", Conf, 0),
-                                rap => cuttlefish:conf_get("module.subscription." ++ N ++ ".rap", Conf, 0),
-                                rh  => cuttlefish:conf_get("module.subscription." ++ N ++ ".rh", Conf, 0)
-                                }} || {N, T} <- TopicList]
-  end,
-  Rewrites = fun() ->
-      Rules = cuttlefish_variable:filter_by_prefix("module.rewrite.rule", Conf),
-      PubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.pub.rule", Conf),
-      SubRules = cuttlefish_variable:filter_by_prefix("module.rewrite.sub.rule", Conf),
-      TotalRules = lists:append(
-                     [ {["module", "rewrite", "pub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ PubRules,
-                     [ {["module", "rewrite", "sub", "rule", I], Rule} || {["module", "rewrite", "rule", I], Rule} <- Rules] ++ SubRules
-                    ),
-      lists:map(fun({[_, "rewrite", PubOrSub, "rule", I], Rule}) ->
-                    [Topic, Re, Dest] = string:tokens(Rule, " "),
-                    {rewrite, list_to_atom(PubOrSub), list_to_binary(Topic), list_to_binary(Re), list_to_binary(Dest)}
-                end, TotalRules)
-  end,
-  lists:append([
-    [{emqx_mod_presence, [{qos, cuttlefish:conf_get("module.presence.qos", Conf, 1)}]}],
-    [{emqx_mod_subscription, Subscriptions()}],
-    [{emqx_mod_rewrite, Rewrites()}],
-    [{emqx_mod_topic_metrics, []}],
-    [{emqx_mod_delayed, []}],
-    [{emqx_mod_acl_internal, [{acl_file, cuttlefish:conf_get("acl_file", Conf1)}]}]
-  ])
-end}.
-
 %%-------------------------------------------------------------------
 %% Plugins
 %%-------------------------------------------------------------------

rebar.config

@@ -11,8 +11,7 @@
 {erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import,
             warn_obsolete_guard,compressed]}.
 
-{overrides,[{add,[{erl_opts,[compressed,deterministic]}]}
-           ,{add,[{extra_src_dirs, [{"etc", [{recursive,true}]}]}]}
+{overrides,[{add,[{extra_src_dirs, [{"etc", [{recursive,true}]}]}]}
            ]}.
 {extra_src_dirs, [{"etc", [{recursive,true}]}]}.
 

rebar.config.erl

@@ -4,7 +4,9 @@
 
 do(Dir, CONFIG) ->
     ok = compile_and_load_pase_transforms(Dir),
-    dump(deps(CONFIG) ++ dialyzer(CONFIG) ++ config()).
+    C1 = deps(CONFIG),
+    Config = dialyzer(C1),
+    dump(Config ++ coveralls() ++ config()).
 
 bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}.
@@ -20,35 +22,55 @@ deps(Config) ->
 config() ->
     [ {plugins, plugins()}
     , {profiles, profiles()}
+    , {project_app_dirs, project_app_dirs()}
     ].
 
+extra_lib_dir() ->
+    EnterpriseFlag = os:getenv("EMQX_ENTERPRISE"),
+    case EnterpriseFlag =:= "true" orelse EnterpriseFlag =:= "1" of
+        true -> "lib-enterprise";
+        false -> "lib-opensource"
+    end.
+
+project_app_dirs() ->
+    ["apps/*", extra_lib_dir() ++ "/*", "."].
+
 plugins() ->
     [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {branch,"master"}}},
       {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}
     ].
 
+test_plugins() ->
+    [ rebar3_proper,
+      {coveralls, {git, "https://github.com/emqx/coveralls-erl", {branch, "github"}}}
+    ].
+
 test_deps() ->
     [ {bbmustache, "1.10.0"}
-    , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.4"}}}
-    , meck
+     , {emqx_ct_helpers, {git, "https://github.com/emqx/emqx-ct-helpers", {tag, "1.3.5"}}}
+     , meck
     ].
 
+default_compile_opts() ->
+    [compressed, deterministic, no_debug_info, warnings_as_errors, {parse_transform, mod_vsn}].
+
 profiles() ->
-    [ {'emqx',          [ {erl_opts, [no_debug_info, warnings_as_errors, {parse_transform, mod_vsn}]}
+    [ {'emqx',          [ {erl_opts, default_compile_opts()}
                         , {relx, relx('emqx')}
                         ]}
-    , {'emqx-pkg',      [ {erl_opts, [no_debug_info, warnings_as_errors, {parse_transform, mod_vsn}]}
+    , {'emqx-pkg',      [ {erl_opts, default_compile_opts()}
                         , {relx, relx('emqx-pkg')}
                         ]}
-    , {'emqx-edge',     [ {erl_opts, [no_debug_info, warnings_as_errors, {parse_transform, mod_vsn}]}
+    , {'emqx-edge',     [ {erl_opts, default_compile_opts()}
                         , {relx, relx('emqx-edge')}
                         ]}
-    , {'emqx-edge-pkg', [ {erl_opts, [no_debug_info, warnings_as_errors, {parse_transform, mod_vsn}]}
+    , {'emqx-edge-pkg', [ {erl_opts, default_compile_opts()}
                         , {relx, relx('emqx-edge-pkg')}
                         ]}
     , {check,           [ {erl_opts, [debug_info, warnings_as_errors, {parse_transform, mod_vsn}]}
                         ]}
     , {test,            [ {deps, test_deps()}
+                        , {plugins, test_plugins()}
                         , {erl_opts, [debug_info, {parse_transform, mod_vsn}] ++ erl_opts_i()}
                         ]}
     ].
@@ -133,6 +155,7 @@ relx_plugin_apps(ReleaseType) ->
     , emqx_rule_engine
     , emqx_sasl
     , emqx_telemetry
+    , emqx_modules
     ] ++ relx_plugin_apps_per_rel(ReleaseType).
 
 relx_plugin_apps_per_rel(cloud) ->
@@ -146,7 +169,6 @@ relx_plugin_apps_per_rel(cloud) ->
     , emqx_exproto
     , emqx_prometheus
     , emqx_psk_file
-    , emqx_plugin_template
     ];
 relx_plugin_apps_per_rel(edge) ->
     [].
@@ -221,8 +243,9 @@ plugin_etc_overlays(App0) ->
 %% NOTE: for apps fetched as rebar dependency (there is so far no such an app)
 %% the overlay should be hand-coded but not to rely on build-time wildcards.
 find_conf_files(App) ->
-    Dir = filename:join(["apps", App, "etc"]),
-    filelib:wildcard("*.conf", Dir).
+    Dir1 = filename:join(["apps", App, "etc"]),
+    Dir2 = filename:join([extra_lib_dir(), App, "etc"]),
+    filelib:wildcard("*.conf", Dir1) ++ filelib:wildcard("*.conf", Dir2).
 
 env(Name, Default) ->
     case os:getenv(Name) of
@@ -267,7 +290,9 @@ str(L) when is_list(L) -> L;
 str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8).
 
 erl_opts_i() ->
-    [{i, "apps"}] ++ [{i, Dir}  || Dir <- filelib:wildcard(filename:join(["apps", "**", "include"]))].
+    [{i, "apps"}] ++
+    [{i, Dir}  || Dir <- filelib:wildcard(filename:join(["apps", "**", "include"]))] ++
+    [{i, Dir}  || Dir <- filelib:wildcard(filename:join([extra_lib_dir(), "**", "include"]))].
 
 dialyzer(Config) ->
     {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
@@ -279,12 +304,9 @@ dialyzer(Config) ->
             [ list_to_atom(App) || App <- string:tokens(Value, ",")]
     end,
 
-    AppsDir = "apps",
-    AppNames = [emqx | list_dir(AppsDir)],
+    AppNames = [emqx | list_dir("apps")] ++ list_dir(extra_lib_dir()),
 
     KnownApps = [Name ||  Name <- AppsToAnalyse, lists:member(Name, AppNames)],
-    UnknownApps = AppsToAnalyse -- KnownApps,
-    io:format("Unknown Apps ~p ~n", [UnknownApps]),
 
     AppsToExclude = AppNames -- KnownApps,
 
@@ -295,6 +317,26 @@ dialyzer(Config) ->
             Config
     end.
 
+coveralls() ->
+    case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of
+      {"true", Token} when is_list(Token) ->
+        Cfgs = [{coveralls_repo_token, Token},
+                {coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")},
+                {coveralls_commit_sha, os:getenv("GITHUB_SHA")},
+                {coveralls_service_number, os:getenv("GITHUB_RUN_NUMBER")},
+                {coveralls_coverdata, "_build/test/cover/*.coverdata"},
+                {coveralls_service_name, "github"}],
+        case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request"
+            andalso string:tokens(os:getenv("GITHUB_REF"), "/") of
+            [_, "pull", PRNO, _] ->
+                [{coveralls_service_pull_request, PRNO} | Cfgs];
+            _ ->
+                Cfgs
+        end;
+      _ ->
+        []
+    end.
+
 list_dir(Dir) ->
     {ok, Names} = file:list_dir(Dir),
     [list_to_atom(Name) || Name <- Names, filelib:is_dir(filename:join([Dir, Name]))].

scripts/shellcheck.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+
+target_files=()
+while IFS='' read -r line; do target_files+=("$line"); done < <(grep -r -l --exclude-dir=.git --exclude-dir=_build "#!/bin/" .)
+return_code=0
+for i in "${target_files[@]}"; do
+  echo checking "$i" ...
+  if ! shellcheck "$i"; then
+    return_code=1
+  fi
+done
+
+exit $return_code

scripts/start-two-nodes-in-docker.sh

@@ -39,8 +39,8 @@ docker run -d -t --restart=always --name "$NODE1" \
   -e EMQX_NODE_COOKIE="$COOKIE" \
   -e WAIT_FOR_ERLANG=60 \
   -p 18083:18083 \
-  -v $PROJ_DIR/_build/emqx/rel/emqx:/built \
-  $IMAGE sh -c 'cp -r /built /emqx && /emqx/bin/emqx console'
+  -v "$PROJ_DIR"/_build/emqx/rel/emqx:/built \
+  "$IMAGE" sh -c 'cp -r /built /emqx && /emqx/bin/emqx console'
 
 docker run -d -t --restart=always --name "$NODE2" \
   --net "$NET" \
@@ -48,8 +48,8 @@ docker run -d -t --restart=always --name "$NODE2" \
   -e EMQX_NODE_COOKIE="$COOKIE" \
   -e WAIT_FOR_ERLANG=60 \
   -p 18084:18083 \
-  -v $PROJ_DIR/_build/emqx/rel/emqx:/built \
-  $IMAGE sh -c 'cp -r /built /emqx && /emqx/bin/emqx console'
+  -v "$PROJ_DIR"/_build/emqx/rel/emqx:/built \
+  "$IMAGE" sh -c 'cp -r /built /emqx && /emqx/bin/emqx console'
 
 wait (){
   container="$1"

src/emqx_app.erl

@@ -32,7 +32,6 @@ start(_Type, _Args) ->
     print_banner(),
     ekka:start(),
     {ok, Sup} = emqx_sup:start_link(),
-    ok = emqx_modules:load(),
     ok = emqx_plugins:init(),
     _ = emqx_plugins:load(),
     emqx_boot:is_enabled(listeners)
@@ -47,8 +46,7 @@ start(_Type, _Args) ->
 stop(_State) ->
     ok = emqx_alarm_handler:unload(),
     emqx_boot:is_enabled(listeners)
-      andalso emqx_listeners:stop(),
-    emqx_modules:unload().
+      andalso emqx_listeners:stop().
 
 %%--------------------------------------------------------------------
 %% Print Banner

src/emqx_sup.erl

@@ -67,12 +67,11 @@ init([]) ->
     BrokerSup = child_spec(emqx_broker_sup, supervisor),
     CMSup = child_spec(emqx_cm_sup, supervisor),
     SysSup = child_spec(emqx_sys_sup, supervisor),
-    ModSup = child_spec(emqx_mod_sup, supervisor),
     Childs = [KernelSup] ++
              [RouterSup || emqx_boot:is_enabled(router)] ++
              [BrokerSup || emqx_boot:is_enabled(broker)] ++
              [CMSup || emqx_boot:is_enabled(broker)] ++
-             [SysSup] ++ [ModSup],
+             [SysSup],
     SupFlags = #{strategy => one_for_all,
                  intensity => 0,
                  period => 1

src/emqx_tls_lib.erl

@@ -0,0 +1,98 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_tls_lib).
+
+-export([ default_versions/0
+        , integral_versions/1
+        , default_ciphers/0
+        , default_ciphers/1
+        , integral_ciphers/2
+        ]).
+
+-define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso is_list(hd(L)))).
+
+%% @doc Returns the default supported tls versions.
+-spec default_versions() -> [atom()].
+default_versions() ->
+    OtpRelease = list_to_integer(erlang:system_info(otp_release)),
+    integral_versions(default_versions(OtpRelease)).
+
+%% @doc Validate a given list of desired tls versions.
+%% raise an error exception if non of them are available.
+-spec integral_versions([ssl:tls_version()]) -> [ssl:tls_version()].
+integral_versions(Desired) ->
+    {_, Available} = lists:keyfind(available, 1, ssl:versions()),
+    case lists:filter(fun(V) -> lists:member(V, Available) end, Desired) of
+        [] -> erlang:error(#{ reason => no_available_tls_version
+                            , desired => Desired
+                            , available => Available
+                            });
+        Filtered ->
+            Filtered
+    end.
+
+%% @doc Return a list of default (openssl string format) cipher suites.
+-spec default_ciphers() -> [string()].
+default_ciphers() -> default_ciphers(default_versions()).
+
+%% @doc Return a list of (openssl string format) cipher suites.
+-spec default_ciphers([ssl:tls_version()]) -> [string()].
+default_ciphers(['tlsv1.3']) ->
+    %% When it's only tlsv1.3 wanted, use 'exclusive' here
+    %% because 'all' returns legacy cipher suites too,
+    %% which does not make sense since tlsv1.3 can not use
+    %% legacy cipher suites.
+    ssl:cipher_suites(exclusive, 'tlsv1.3', openssl);
+default_ciphers(Versions) ->
+    %% assert non-empty
+    [_ | _] = dedup(lists:append([ssl:cipher_suites(all, V, openssl) || V <- Versions])).
+
+%% @doc Ensure version & cipher-suites integrity.
+-spec integral_ciphers([ssl:tls_version()], binary() | string() | [string()]) -> [string()].
+integral_ciphers(Versions, Ciphers) when Ciphers =:= [] orelse Ciphers =:= undefined ->
+    %% not configured
+    integral_ciphers(Versions, default_ciphers(Versions));
+integral_ciphers(Versions, Ciphers) when ?IS_STRING_LIST(Ciphers) ->
+    %% ensure tlsv1.3 ciphers if none of them is found in Ciphers
+    dedup(ensure_tls13_cipher(lists:member('tlsv1.3', Versions), Ciphers));
+integral_ciphers(Versions, Ciphers) when is_binary(Ciphers) ->
+    %% parse binary
+    integral_ciphers(Versions, binary_to_list(Ciphers));
+integral_ciphers(Versions, Ciphers) ->
+    %% parse comma separated cipher suite names
+    integral_ciphers(Versions, string:tokens(Ciphers, ", ")).
+
+%% In case tlsv1.3 is present, ensure tlsv1.3 cipher is added if user
+%% did not provide it from config --- which is a common mistake
+ensure_tls13_cipher(true, Ciphers) ->
+    Tls13Ciphers = default_ciphers(['tlsv1.3']),
+    case lists:any(fun(C) -> lists:member(C, Tls13Ciphers) end, Ciphers) of
+        true  -> Ciphers;
+        false -> Tls13Ciphers ++ Ciphers
+    end;
+ensure_tls13_cipher(false, Ciphers) ->
+    Ciphers.
+
+%% tlsv1.3 is available from OTP-22 but we do not want to use until 23.
+default_versions(OtpRelease) when OtpRelease >= 23 ->
+    ['tlsv1.3' | default_versions(22)];
+default_versions(_) ->
+    ['tlsv1.2', 'tlsv1.1', tlsv1].
+
+%% Deduplicate a list without re-ordering the elements.
+dedup([]) -> [];
+dedup([H | T]) -> [H | dedup([I || I <- T, I =/= H])].

sync-apps.sh

@@ -21,7 +21,6 @@ apps=(
 "emqx_lua_hook"
 "emqx_lwm2m"
 "emqx_management"
-"emqx_plugin_template"
 "emqx_prometheus"
 "emqx_psk_file"
 "emqx_recon"
@@ -43,12 +42,14 @@ mkdir -p tmp/
 download_zip() {
     local app="$1"
     local ref="$2"
-    local vsn="$(echo "$ref" | tr '/' '-')"
+    local vsn
+    vsn="$(echo "$ref" | tr '/' '-')"
     local file="tmp/${app}-${vsn}.zip"
     if [ -f "$file" ] && [ "$force" != "force" ]; then
         return 0
     fi
-    local repo="$(echo "$app" | sed 's#_#-#g')"
+    local repo
+    repo=${app//_/-}
     local url="https://github.com/emqx/$repo/archive/$ref.zip"
     echo "downloading ${url}"
     curl -fLsS -o "$file" "$url"
@@ -56,7 +57,7 @@ download_zip() {
 
 default_vsn="dev/v4.3.0"
 download_zip "emqx_auth_mnesia" "e4.2.3"
-for app in ${apps[@]}; do
+for app in "${apps[@]}"; do
     download_zip "$app" "$default_vsn"
 done
 
@@ -64,7 +65,8 @@ extract_zip(){
     local app="$1"
     local ref="$2"
     local vsn_arg="${3:-}"
-    local vsn_dft="$(echo "$ref" | tr '/' '-')"
+    local vsn_dft
+    vsn_dft="$(echo "$ref" | tr '/' '-')"
     local vsn
     if [ -n "$vsn_arg" ]; then
         vsn="$vsn_arg"
@@ -72,14 +74,15 @@ extract_zip(){
         vsn="$vsn_dft"
     fi
     local file="tmp/${app}-${vsn_dft}.zip"
-    local repo="$(echo "$app" | sed 's#_#-#g')"
+    local repo
+    repo=${app//_/-}
     rm -rf "apps/${app}/"
     unzip "$file" -d apps/
     mv "apps/${repo}-${vsn}/" "apps/$app/"
 }
 
 extract_zip "emqx_auth_mnesia" "e4.2.3" "e4.2.3"
-for app in ${apps[@]}; do
+for app in "${apps[@]}"; do
     extract_zip "$app" "$default_vsn"
 done
 
@@ -95,6 +98,6 @@ cleanup_app(){
 }
 
 apps+=( "emqx_auth_mnesia" )
-for app in ${apps[@]}; do
-    cleanup_app $app
+for app in "${apps[@]}"; do
+    cleanup_app "$app"
 done

test/emqx_acl_cache_SUITE.erl

@@ -56,13 +56,14 @@ t_clean_acl_cache(_) ->
     emqtt:stop(Client).
 
 % optimize??
-t_reload_aclfile_and_cleanall(Config) ->
+t_reload_aclfile_and_cleanall(_Config) ->
 
     RasieMsg = fun() -> Self = self(), #{puback => fun(Msg) -> Self ! {puback, Msg} end,
                                          disconnected => fun(_) ->  ok end,
                                          publish => fun(_) -> ok end } end,
 
-    {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}, {proto_ver, v5}, {msg_handler, RasieMsg()}]),
+    {ok, Client} = emqtt:start_link([{clientid, <<"emqx_c">>}, {proto_ver, v5},
+                                     {msg_handler, RasieMsg()}]),
     {ok, _} = emqtt:connect(Client),
 
     {ok, PktId} = emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, qos1),
@@ -78,27 +79,6 @@ t_reload_aclfile_and_cleanall(Config) ->
     %% Check acl cache list
     [ClientPid] = emqx_cm:lookup_channels(<<"emqx_c">>),
     ?assert(length(gen_server:call(ClientPid, list_acl_cache)) > 0),
-
-    %% Update acl file and reload mod_acl_internal
-    Path = filename:join([testdir(proplists:get_value(data_dir, Config)), "acl2.conf"]),
-    ok = file:write_file(Path, <<"{deny, all}.">>),
-    OldPath = emqx:get_env(acl_file),
-    % application:set_env(emqx, acl_file, Path),
-    emqx_mod_acl_internal:reload([{acl_file, Path}]),
-
-    ?assert(length(gen_server:call(ClientPid, list_acl_cache)) == 0),
-    {ok, PktId2} = emqtt:publish(Client, <<"t1">>, <<"{\"x\":1}">>, qos1),
-
-    receive
-        {puback, #{packet_id := PktId2, reason_code := Rc2}} ->
-            %% Not authorized
-            ?assertEqual(16#87, Rc2);
-        _ ->
-            ?assert(false)
-    end,
-    application:set_env(emqx, acl_file, OldPath),
-    file:delete(Path),
-    emqx_mod_acl_internal:reload([{acl_file, OldPath}]),
     emqtt:stop(Client).
 
 %% @private

test/emqx_client_SUITE.erl

@@ -70,7 +70,9 @@ groups() ->
       ]},
      {others, [non_parallel_tests],
       [t_username_as_clientid,
-       t_certcn_as_clientid
+       t_certcn_as_clientid_default_config_tls,
+       t_certcn_as_clientid_tlsv1_3,
+       t_certcn_as_clientid_tlsv1_2
       ]}
     ].
 
@@ -278,14 +280,18 @@ t_username_as_clientid(_) ->
     #{clientinfo := #{clientid := Username}} = emqx_cm:get_chan_info(Username),
     emqtt:disconnect(C).
 
-t_certcn_as_clientid(_) ->
-    CN = <<"Client">>,
-    emqx_zone:set_env(external, use_username_as_clientid, true),
-    SslConf = emqx_ct_helpers:client_ssl_twoway(),
-    {ok, C} = emqtt:start_link([{port, 8883}, {ssl, true}, {ssl_opts, SslConf}]),
-    {ok, _} = emqtt:connect(C),
-    #{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN),
-    emqtt:disconnect(C).
+
+
+t_certcn_as_clientid_default_config_tls(_) ->
+    tls_certcn_as_clientid(default).
+
+t_certcn_as_clientid_tlsv1_3(_) ->
+    tls_certcn_as_clientid('tlsv1.3').
+
+t_certcn_as_clientid_tlsv1_2(_) ->
+    tls_certcn_as_clientid('tlsv1.2').
+
+
 
 %%--------------------------------------------------------------------
 %% Helper functions
@@ -304,3 +310,29 @@ recv_msgs(Count, Msgs) ->
     after 100 ->
         Msgs
     end.
+
+
+confirm_tls_version( Client, RequiredProtocol ) ->
+    Info = emqtt:info(Client),
+    SocketInfo = proplists:get_value( socket, Info ),
+    %% emqtt_sock has #ssl_socket.ssl
+    SSLSocket = element( 3, SocketInfo ),
+    { ok, SSLInfo } = ssl:connection_information(SSLSocket),
+    Protocol = proplists:get_value( protocol, SSLInfo ),
+    RequiredProtocol = Protocol.
+
+
+tls_certcn_as_clientid(default = TLSVsn) ->
+    tls_certcn_as_clientid(TLSVsn, 'tlsv1.3');
+tls_certcn_as_clientid(TLSVsn) ->
+    tls_certcn_as_clientid(TLSVsn, TLSVsn).
+
+tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) ->
+    CN = <<"Client">>,
+    emqx_zone:set_env(external, use_username_as_clientid, true),
+    SslConf = emqx_ct_helpers:client_ssl_twoway(TLSVsn),
+    {ok, Client} = emqtt:start_link([{port, 8883}, {ssl, true}, {ssl_opts, SslConf}]),
+    {ok, _} = emqtt:connect(Client),
+    #{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN),
+    confirm_tls_version( Client, RequiredTLSVsn ),
+    emqtt:disconnect(Client).

test/emqx_tls_lib_tests.erl

@@ -0,0 +1,64 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_tls_lib_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+%% one of the cipher suite from tlsv1.2 and tlsv1.3 each
+-define(TLS_12_CIPHER, "ECDHE-ECDSA-AES256-GCM-SHA384").
+-define(TLS_13_CIPHER, "TLS_AES_256_GCM_SHA384").
+
+ensure_tls13_ciphers_added_test() ->
+    Ciphers = emqx_tls_lib:integral_ciphers(['tlsv1.3'], [?TLS_12_CIPHER]),
+    ?assert(lists:member(?TLS_12_CIPHER, Ciphers)),
+    ?assert(lists:member(?TLS_13_CIPHER, Ciphers)).
+
+legacy_cipher_suites_test() ->
+    Ciphers = emqx_tls_lib:integral_ciphers(['tlsv1.2'], [?TLS_12_CIPHER]),
+    ?assertEqual([?TLS_12_CIPHER], Ciphers).
+
+use_default_ciphers_test() ->
+    Ciphers = emqx_tls_lib:integral_ciphers(['tlsv1.3', 'tlsv1.2'], ""),
+    ?assert(lists:member(?TLS_12_CIPHER, Ciphers)),
+    ?assert(lists:member(?TLS_13_CIPHER, Ciphers)).
+
+ciphers_format_test_() ->
+    String = ?TLS_13_CIPHER ++ "," ++ ?TLS_12_CIPHER,
+    Binary = iolist_to_binary(String),
+    List = [?TLS_13_CIPHER, ?TLS_12_CIPHER],
+    [ {"string", fun() -> test_cipher_format(String) end}
+    , {"binary", fun() -> test_cipher_format(Binary) end}
+    , {"string-list", fun() -> test_cipher_format(List) end}
+    ].
+
+test_cipher_format(Input) ->
+    Ciphers = emqx_tls_lib:integral_ciphers(['tlsv1.3', 'tlsv1.2'], Input),
+    ?assertEqual([?TLS_13_CIPHER, ?TLS_12_CIPHER], Ciphers).
+
+tls_versions_test() ->
+    ?assert(lists:member('tlsv1.3', emqx_tls_lib:default_versions())).
+
+tls_version_unknown_test() ->
+    ?assertError(#{reason := no_available_tls_version},
+                 emqx_tls_lib:integral_versions([])),
+    ?assertError(#{reason := no_available_tls_version},
+                 emqx_tls_lib:integral_versions([foo])).
+
+cipher_suites_no_duplication_test() ->
+    AllCiphers = emqx_tls_lib:default_ciphers(),
+    ?assertEqual(length(AllCiphers), length(lists:usort(AllCiphers))).
+

test/mqtt_protocol_v5_SUITE.erl

@@ -111,11 +111,13 @@ t_basic_test(_) ->
 
 t_connect_clean_start(_) ->
     process_flag(trap_exit, true),
-    {ok, Client1} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>},{proto_ver, v5},{clean_start, true}]),
+    {ok, Client1} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>},
+                                      {proto_ver, v5},{clean_start, true}]),
     {ok, _} = emqtt:connect(Client1),
     ?assertEqual(0, client_info(session_present, Client1)),  %% [MQTT-3.1.2-4]
     ok = emqtt:pause(Client1),
-    {ok, Client2} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>},{proto_ver, v5},{clean_start, false}]),
+    {ok, Client2} = emqtt:start_link([{clientid, <<"t_connect_clean_start">>},
+                                      {proto_ver, v5},{clean_start, false}]),
     {ok, _} = emqtt:connect(Client2),
     ?assertEqual(1, client_info(session_present, Client2)),  %% [MQTT-3.1.2-5]
     ?assertEqual(142, receive_disconnect_reasoncode()),
@@ -124,7 +126,8 @@ t_connect_clean_start(_) ->
     ok = emqtt:disconnect(Client2),
     waiting_client_process_exit(Client2),
 
-    {ok, Client3} = emqtt:start_link([{clientid, <<"new_client">>},{proto_ver, v5},{clean_start, false}]),
+    {ok, Client3} = emqtt:start_link([{clientid, <<"new_client">>},
+                                      {proto_ver, v5},{clean_start, false}]),
     {ok, _} = emqtt:connect(Client3),
     ?assertEqual(0, client_info(session_present, Client3)),  %% [MQTT-3.1.2-6]
     ok = emqtt:disconnect(Client3),
@@ -145,7 +148,8 @@ t_connect_will_message(_) ->
                                         ]),
     {ok, _} = emqtt:connect(Client1),
     [ClientPid] = emqx_cm:lookup_channels(client_info(clientid, Client1)),
-    ?assertNotEqual(undefined, maps:find(will_msg, emqx_connection:info(sys:get_state(ClientPid)))),  %% [MQTT-3.1.2-7]
+    Info = emqx_connection:info(sys:get_state(ClientPid)),
+    ?assertNotEqual(undefined, maps:find(will_msg, Info)),  %% [MQTT-3.1.2-7]
 
     {ok, Client2} = emqtt:start_link([{proto_ver, v5}]),
     {ok, _} = emqtt:connect(Client2),
@@ -179,10 +183,7 @@ t_batch_subscribe(_) ->
     {ok, Client} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"batch_test">>}]),
     {ok, _} = emqtt:connect(Client),
     application:set_env(emqx, enable_acl_cache, false),
-    TempAcl = emqx_ct_helpers:deps_path(emqx, "test/emqx_access_SUITE_data/acl_temp.conf"),
-    file:write_file(TempAcl, "{deny, {client, \"batch_test\"}, subscribe, [\"t1\", \"t2\", \"t3\"]}.\n"),
-    timer:sleep(10),
-    emqx_mod_acl_internal:reload([{acl_file, TempAcl}]),
+    application:set_env(emqx, acl_nomatch, deny),
     {ok, _, [?RC_NOT_AUTHORIZED,
              ?RC_NOT_AUTHORIZED,
              ?RC_NOT_AUTHORIZED]} = emqtt:subscribe(Client, [{<<"t1">>, qos1},
@@ -193,7 +194,7 @@ t_batch_subscribe(_) ->
              ?RC_NO_SUBSCRIPTION_EXISTED]} = emqtt:unsubscribe(Client, [<<"t1">>,
                                                                         <<"t2">>,
                                                                         <<"t3">>]),
-    file:delete(TempAcl),
+    application:set_env(emqx, acl_nomatch, allow),
     emqtt:disconnect(Client).
 
 t_connect_will_retain(_) ->
@@ -261,9 +262,10 @@ t_connect_limit_timeout(_) ->
     [ClientPid] = emqx_cm:lookup_channels(client_info(clientid, Client)),
 
     ?assertEqual(undefined, emqx_connection:info(limit_timer, sys:get_state(ClientPid))),
-    ok = emqtt:publish(Client, Topic, <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 0),
-    ok = emqtt:publish(Client, Topic, <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 0),
-    ok = emqtt:publish(Client, Topic, <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 0),
+    Payload = <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>,
+    ok = emqtt:publish(Client, Topic, Payload, 0),
+    ok = emqtt:publish(Client, Topic, Payload, 0),
+    ok = emqtt:publish(Client, Topic, Payload, 0),
     timer:sleep(200),
     ?assert(is_reference(emqx_connection:info(limit_timer, sys:get_state(ClientPid)))),
 
@@ -523,7 +525,8 @@ t_publish_rap(_) ->
     {ok, Client1} = emqtt:start_link([{proto_ver, v5}]),
     {ok, _} = emqtt:connect(Client1),
     {ok, _, [2]} = emqtt:subscribe(Client1, #{}, [{Topic, [{rap, true}, {qos, 2}]}]),
-    {ok, _} = emqtt:publish(Client1, Topic, #{}, <<"retained message">>, [{qos, ?QOS_1}, {retain, true}]),
+    {ok, _} = emqtt:publish(Client1, Topic, #{}, <<"retained message">>,
+                            [{qos, ?QOS_1}, {retain, true}]),
     [Msg1 | _] = receive_messages(1),
     ?assertEqual(true, maps:get(retain, Msg1)),  %% [MQTT-3.3.1-12]
     ok = emqtt:disconnect(Client1),
@@ -531,7 +534,8 @@ t_publish_rap(_) ->
     {ok, Client2} = emqtt:start_link([{proto_ver, v5}]),
     {ok, _} = emqtt:connect(Client2),
     {ok, _, [2]} = emqtt:subscribe(Client2, #{}, [{Topic, [{rap, false}, {qos, 2}]}]),
-    {ok, _} = emqtt:publish(Client2, Topic, #{}, <<"retained message">>, [{qos, ?QOS_1}, {retain, true}]),
+    {ok, _} = emqtt:publish(Client2, Topic, #{}, <<"retained message">>,
+                            [{qos, ?QOS_1}, {retain, true}]),
     [Msg2 | _] = receive_messages(1),
     ?assertEqual(false, maps:get(retain, Msg2)),  %% [MQTT-3.3.1-13]
     ok = emqtt:disconnect(Client2),
@@ -575,8 +579,10 @@ t_publish_topic_alias(_) ->
     {ok, Client2} = emqtt:start_link([{proto_ver, v5}]),
     {ok, _} = emqtt:connect(Client2),
     {ok, _, [2]} = emqtt:subscribe(Client2, Topic, qos2),
-    ok = emqtt:publish(Client2, Topic, #{'Topic-Alias' => 233}, <<"Topic-Alias">>, [{qos, ?QOS_0}]),
-    ok = emqtt:publish(Client2, <<"">>, #{'Topic-Alias' => 233}, <<"Topic-Alias">>, [{qos, ?QOS_0}]),
+    ok = emqtt:publish(Client2, Topic, #{'Topic-Alias' => 233},
+                       <<"Topic-Alias">>, [{qos, ?QOS_0}]),
+    ok = emqtt:publish(Client2, <<"">>, #{'Topic-Alias' => 233},
+                       <<"Topic-Alias">>, [{qos, ?QOS_0}]),
     ?assertEqual(2, length(receive_messages(2))),   %% [MQTT-3.3.2-12]
     ok = emqtt:disconnect(Client2),
     waiting_client_process_exit(Client2),
@@ -589,7 +595,8 @@ t_publish_response_topic(_) ->
 
     {ok, Client1} = emqtt:start_link([{proto_ver, v5}]),
     {ok, _} = emqtt:connect(Client1),
-    ok = emqtt:publish(Client1, Topic, #{'Response-Topic' => nth(1, ?WILD_TOPICS)}, <<"Response-Topic">>, [{qos, ?QOS_0}]),
+    ok = emqtt:publish(Client1, Topic, #{'Response-Topic' => nth(1, ?WILD_TOPICS)},
+                       <<"Response-Topic">>, [{qos, ?QOS_0}]),
     ?assertEqual(130, receive_disconnect_reasoncode()),  %% [MQTT-3.3.2-14]
     waiting_client_process_exit(Client1),
 
@@ -620,7 +627,8 @@ t_publish_overlapping_subscriptions(_) ->
     {ok, _} = emqtt:connect(Client1),
     {ok, _, [1]} = emqtt:subscribe(Client1, Properties, nth(1, ?WILD_TOPICS), qos1),
     {ok, _, [0]} = emqtt:subscribe(Client1, Properties, nth(3, ?WILD_TOPICS), qos0),
-    {ok, _} = emqtt:publish(Client1, Topic, #{}, <<"t_publish_overlapping_subscriptions">>, [{qos, ?QOS_2}]),
+    {ok, _} = emqtt:publish(Client1, Topic, #{},
+                            <<"t_publish_overlapping_subscriptions">>, [{qos, ?QOS_2}]),
 
     [Msg1 | _ ] = receive_messages(2),
     ?assert( maps:get(qos, Msg1) < 2 ),  %% [MQTT-3.3.4-2]
@@ -684,8 +692,9 @@ t_subscribe_actions(_) ->
     {ok, _} = emqtt:publish(Client1, Topic, <<"t_subscribe_actions">>, 2),
     [Msg1 | _ ] = receive_messages(1),
     ?assertEqual(1, maps:get(qos, Msg1)),   %% [MQTT-3.8.4-3] [MQTT-3.8.4-8]
-
-    {ok, _, [2,2]} = emqtt:subscribe(Client1, [{nth(1, ?TOPICS), qos2}, {nth(2, ?TOPICS), qos2}] ), %% [MQTT-3.8.4-5] [MQTT-3.8.4-6] [MQTT-3.8.4-7]
+    %% [MQTT-3.8.4-5] [MQTT-3.8.4-6] [MQTT-3.8.4-7]
+    {ok, _, [2,2]} = emqtt:subscribe(Client1, [{nth(1, ?TOPICS), qos2},
+                                               {nth(2, ?TOPICS), qos2}] ),
     ok = emqtt:disconnect(Client1).
 %%--------------------------------------------------------------------
 %% Unsubsctibe Unsuback
@@ -702,7 +711,8 @@ t_unscbsctibe(_) ->
     {ok, _, [17]} = emqtt:unsubscribe(Client1, <<"noExistTopic">>),  %% [MQTT-3.10.4-5]
 
     {ok, _, [2, 2]} = emqtt:subscribe(Client1, [{Topic1, qos2}, {Topic2, qos2}]),
-    {ok, _, [0, 0, 17]} = emqtt:unsubscribe(Client1, [Topic1, Topic2, <<"noExistTopic">>]), %% [[MQTT-3.10.4-6]]  [MQTT-3.11.3-1]  [MQTT-3.11.3-2]
+    %% [[MQTT-3.10.4-6]]  [MQTT-3.11.3-1]  [MQTT-3.11.3-2]
+    {ok, _, [0, 0, 17]} = emqtt:unsubscribe(Client1, [Topic1, Topic2, <<"noExistTopic">>]),
     ok = emqtt:disconnect(Client1).
 
 %%--------------------------------------------------------------------
@@ -748,7 +758,8 @@ t_shared_subscriptions_client_terminates_when_qos_eq_2(_) ->
 
     {ok, Pub} = emqtt:start_link([{proto_ver, v5}, {clientid, <<"pub_client">>}]),
     {ok, _} = emqtt:connect(Pub),
-    {ok, _} = emqtt:publish(Pub, Topic, <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 2),
+    {ok, _} = emqtt:publish(Pub, Topic,
+                            <<"t_shared_subscriptions_client_terminates_when_qos_eq_2">>, 2),
 
     receive
         {'EXIT', _,{shutdown, for_testiong}} ->