feat(conf): merge all conf to emqx.conf

.ci/build_packages/tests.sh

@@ -39,7 +39,7 @@ emqx_test(){
                 unzip -q "${PACKAGE_PATH}/${packagename}"
                 export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \
                        EMQX_MQTT__MAX_TOPIC_ALIAS=10
-                sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
+                # sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
 
                 echo "running ${packagename} start"
                 if ! "${PACKAGE_PATH}"/emqx/bin/emqx start; then
@@ -115,7 +115,7 @@ emqx_test(){
 running_test(){
     export EMQX_ZONE__EXTERNAL__SERVER_KEEPALIVE=60 \
            EMQX_MQTT__MAX_TOPIC_ALIAS=10
-    sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
+    # sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
 
     if ! emqx start; then
         cat /var/log/emqx/erlang.log.1 || true

.ci/docker-compose-file/docker-compose-emqx-cluster.yaml

@@ -38,7 +38,7 @@ services:
         - -c
         - |
           sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
-          sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
+          # sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
           /opt/emqx/bin/emqx foreground
     healthcheck:
       test: ["CMD", "/opt/emqx/bin/emqx_ctl", "status"]
@@ -62,7 +62,7 @@ services:
       - -c
       - |
         sed -i "s 127.0.0.1 $$(ip route show |grep "link" |awk '{print $$1}') g" /opt/emqx/etc/acl.conf
-        sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
+        # sed -i '/emqx_telemetry/d' /opt/emqx/data/loaded_plugins
         /opt/emqx/bin/emqx foreground
     healthcheck:
       test: ["CMD", "/opt/emqx/bin/emqx", "ping"]

.github/workflows/build_packages.yaml

@@ -179,7 +179,7 @@ jobs:
         cd source
         pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip)
         unzip -q _packages/${{ matrix.profile }}/$pkg_name
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
+        # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
         ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
         ready='no'
         for i in {1..10}; do

.github/workflows/build_slim_packages.yaml

@@ -108,7 +108,7 @@ jobs:
       run: |
         pkg_name=$(basename _packages/${EMQX_NAME}/emqx-*.zip)
         unzip -q _packages/${EMQX_NAME}/$pkg_name
-        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
+        # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
         ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
         ready='no'
         for i in {1..10}; do

.github/workflows/run_fvt_tests.yaml

@@ -48,13 +48,13 @@ jobs:
                 echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:waiting emqx";
                 sleep 5;
             done
-        - name: verify EMQX_LOADED_PLUGINS override working
-          run: |
-            expected="{emqx_sn, true}."
-            output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1)
-            if [ "$expected" != "$output" ]; then
-                exit 1
-            fi
+        # - name: verify EMQX_LOADED_PLUGINS override working
+        #   run: |
+        #     expected="{emqx_sn, true}."
+        #     output=$(docker exec -i node1.emqx.io bash -c "cat data/loaded_plugins" | tail -n1)
+        #     if [ "$expected" != "$output" ]; then
+        #         exit 1
+        #     fi
         - name: make paho tests
           run: |
             if ! docker exec -i python /scripts/pytest.sh; then

.gitignore

@@ -45,6 +45,6 @@ emqx_dialyzer_*_plt
 */emqx_dashboard/priv/www
 dist.zip
 scripts/git-token
-etc/*.seg
+apps/*/etc/*.all
 _upgrade_base/
 TAGS

Makefile

@@ -73,7 +73,8 @@ coveralls: $(REBAR)
 	@ENABLE_COVER_COMPILE=1 $(REBAR) as test coveralls send
 
 .PHONY: $(REL_PROFILES)
-$(REL_PROFILES:%=%): $(REBAR) get-dashboard
+
+$(REL_PROFILES:%=%): $(REBAR) get-dashboard conf-segs
 	@$(REBAR) as $(@) do compile,release
 
 ## Not calling rebar3 clean because
@@ -111,7 +112,7 @@ xref: $(REBAR)
 dialyzer: $(REBAR)
 	@$(REBAR) as check dialyzer
 
-COMMON_DEPS := $(REBAR) get-dashboard $(CONF_SEGS)
+COMMON_DEPS := $(REBAR) get-dashboard conf-segs
 
 ## rel target is to create release package without relup
 .PHONY: $(REL_PROFILES:%=%-rel) $(PKG_PROFILES:%=%-rel)
@@ -152,3 +153,6 @@ quickrun:
 	./_build/$(PROFILE)/rel/emqx/bin/emqx console
 
 include docker.mk
+
+conf-segs:
+	@scripts/merge-config.escript

apps/emqx/etc/acl.conf

@@ -1,26 +0,0 @@
-%%--------------------------------------------------------------------
-%% [ACL](https://docs.emqx.io/broker/v3/en/config.html)
-%%
-%% -type(who() :: all | binary() |
-%%                {ipaddr, esockd_access:cidr()} |
-%%                {client, binary()} |
-%%                {user, binary()}).
-%%
-%% -type(access() :: subscribe | publish | pubsub).
-%%
-%% -type(topic() :: binary()).
-%%
-%% -type(rule() :: {allow, all} |
-%%                 {allow, who(), access(), list(topic())} |
-%%                 {deny, all} |
-%%                 {deny, who(), access(), list(topic())}).
-%%--------------------------------------------------------------------
-
-{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}.
-
-{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}.
-
-{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}.
-
-{allow, all}.
-

apps/emqx/etc/acl.conf.paho

@@ -1,14 +0,0 @@
-%%--------------------------------------------------------------------
-%% For paho interoperability test cases
-%%--------------------------------------------------------------------
-
-{deny, {client, "myclientid"}, subscribe, ["test/nosubscribe"]}.
-
-{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}.
-
-{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}.
-
-{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}.
-
-{allow, all}.
-

apps/emqx/include/emqx.hrl

@@ -108,8 +108,7 @@
           descr          :: string(),
           vendor         :: string() | undefined,
           active = false :: boolean(),
-          info   = #{}   :: map(),
-          type           :: atom()
+          info   = #{}   :: map()
         }).
 
 %%--------------------------------------------------------------------

apps/emqx/src/emqx.appup.src

@@ -1,111 +0,0 @@
-%% -*- mode: erlang -*-
-{VSN,
-  [
-   {"4.3.4",
-    [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.3",
-    [{load_module,emqx_packet,brutal_purge,soft_purge,[]},
-     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.2",
-    [{load_module,emqx_packet,brutal_purge,soft_purge,[]},
-     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.1",
-    [{load_module,emqx_packet,brutal_purge,soft_purge,[]},
-     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_frame,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
-     {load_module,emqx_congestion,brutal_purge,soft_purge,[]},
-     {load_module,emqx_node_dump,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
-     {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
-     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]},
-   {"4.3.0",
-    [{load_module,emqx_packet,brutal_purge,soft_purge,[]},
-     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]},
-     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_congestion,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_frame,brutal_purge,soft_purge,[]},
-     {load_module,emqx_trie,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
-     {load_module,emqx_node_dump,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
-     {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
-     {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
-     {apply,{emqx_metrics,upgrade_retained_delayed_counter_type,[]}},
-     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]},
-   {<<".*">>,[]}],
-  [
-   {"4.3.4",
-    [{load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.3",
-    [{load_module,emqx_packet,brutal_purge,soft_purge,[]},
-     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.2",
-    [{load_module,emqx_packet,brutal_purge,soft_purge,[]},
-     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]}
-    ]},
-   {"4.3.1",
-    [{load_module,emqx_packet,brutal_purge,soft_purge,[]},
-     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_frame,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
-     {load_module,emqx_congestion,brutal_purge,soft_purge,[]},
-     {load_module,emqx_node_dump,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
-     {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
-     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]},
-   {"4.3.0",
-    [{load_module,emqx_packet,brutal_purge,soft_purge,[]},
-     {load_module,emqx_shared_sub,brutal_purge,soft_purge,[]},
-     {load_module,emqx_logger_jsonfmt,brutal_purge,soft_purge,[]},
-     {load_module,emqx_ws_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_connection,brutal_purge,soft_purge,[]},
-     {load_module,emqx_congestion,brutal_purge,soft_purge,[]},
-     {load_module,emqx_frame,brutal_purge,soft_purge,[]},
-     {load_module,emqx_trie,brutal_purge,soft_purge,[]},
-     {load_module,emqx_cm,brutal_purge,soft_purge,[]},
-     {load_module,emqx_node_dump,brutal_purge,soft_purge,[]},
-     {load_module,emqx_channel,brutal_purge,soft_purge,[]},
-     {load_module,emqx_app,brutal_purge,soft_purge,[]},
-     {load_module,emqx_plugins,brutal_purge,soft_purge,[]},
-     {load_module,emqx_logger_textfmt,brutal_purge,soft_purge,[]},
-     {load_module,emqx_metrics,brutal_purge,soft_purge,[]},
-     {load_module,emqx_http_lib,brutal_purge,soft_purge,[]}]},
-   {<<".*">>,[]}]}.

apps/emqx/src/emqx.erl

@@ -227,7 +227,6 @@ shutdown() ->
 shutdown(Reason) ->
     ?LOG(critical, "emqx shutdown for ~s", [Reason]),
     _ = emqx_alarm_handler:unload(),
-    _ = emqx_plugins:unload(),
     lists:foreach(fun application:stop/1
                  , lists:reverse(default_started_applications())
                  ).

apps/emqx/src/emqx_app.erl

@@ -51,7 +51,7 @@ start(_Type, _Args) ->
     ok = ekka_rlog:wait_for_shards(?EMQX_SHARDS, infinity),
     {ok, Sup} = emqx_sup:start_link(),
     ok = start_autocluster(),
-    ok = emqx_plugins:init(),
+    % ok = emqx_plugins:init(),
     _ = emqx_plugins:load(),
     _ = start_ce_modules(),
     emqx_boot:is_enabled(listeners) andalso (ok = emqx_listeners:start()),

apps/emqx/src/emqx_plugins.erl

@@ -21,8 +21,6 @@
 
 -logger_header("[Plugins]").
 
--export([init/0]).
-
 -export([ load/0
         , load/1
         , unload/0
@@ -30,8 +28,6 @@
         , reload/1
         , list/0
         , find_plugin/1
-        , generate_configs/1
-        , apply_configs/1
         ]).
 
 -export([funlog/2]).
@@ -41,35 +37,14 @@
 -compile(nowarn_export_all).
 -endif.
 
--dialyzer({no_match, [ plugin_loaded/2
-                     , plugin_unloaded/2
-                     ]}).
-
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
 
-%% @doc Init plugins' config
--spec(init() -> ok).
-init() ->
-    case emqx:get_env(plugins_etc_dir) of
-        undefined  -> ok;
-        PluginsEtc ->
-            CfgFiles = [filename:join(PluginsEtc, File) ||
-                        File <- filelib:wildcard("*.config", PluginsEtc)],
-            lists:foreach(fun init_config/1, CfgFiles)
-    end.
-
 %% @doc Load all plugins when the broker started.
 -spec(load() -> ok | ignore | {error, term()}).
 load() ->
-    ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)),
-    case emqx:get_env(plugins_loaded_file) of
-        undefined -> ignore; %% No plugins available
-        File ->
-            _ = ensure_file(File),
-            with_loaded_file(File, fun(Names) -> load_plugins(Names, false) end)
-    end.
+    ok = load_ext_plugins(emqx:get_env(expand_plugins_dir)).
 
 %% @doc Load a Plugin
 -spec(load(atom()) -> ok | {error, term()}).
@@ -82,17 +57,13 @@ load(PluginName) when is_atom(PluginName) ->
             ?LOG(notice, "Plugin ~s is already started", [PluginName]),
             {error, already_started};
         {_, false} ->
-            load_plugin(PluginName, true)
+            load_plugin(PluginName)
     end.
 
 %% @doc Unload all plugins before broker stopped.
--spec(unload() -> list() | {error, term()}).
+-spec(unload() -> ok).
 unload() ->
-    case emqx:get_env(plugins_loaded_file) of
-        undefined -> ignore;
-        File ->
-            with_loaded_file(File, fun stop_plugins/1)
-    end.
+    stop_plugins(list()).
 
 %% @doc UnLoad a Plugin
 -spec(unload(atom()) -> ok | {error, term()}).
@@ -105,7 +76,7 @@ unload(PluginName) when is_atom(PluginName) ->
             ?LOG(error, "Plugin ~s is not started", [PluginName]),
             {error, not_started};
         {_, _} ->
-            unload_plugin(PluginName, true)
+            unload_plugin(PluginName)
     end.
 
 reload(PluginName) when is_atom(PluginName)->
@@ -126,8 +97,8 @@ reload(PluginName) when is_atom(PluginName)->
 -spec(list() -> [emqx_types:plugin()]).
 list() ->
     StartedApps = names(started_app),
-    lists:map(fun({Name, _, [Type| _]}) ->
-        Plugin = plugin(Name, Type),
+    lists:map(fun({Name, _, _}) ->
+        Plugin = plugin(Name),
         case lists:member(Name, StartedApps) of
             true  -> Plugin#plugin{active = true};
             false -> Plugin
@@ -144,12 +115,6 @@ find_plugin(Name, Plugins) ->
 %% Internal functions
 %%--------------------------------------------------------------------
 
-init_config(CfgFile) ->
-    {ok, [AppsEnv]} = file:consult(CfgFile),
-    lists:foreach(fun({App, Envs}) ->
-                      [application:set_env(App, Par, Val) || {Par, Val} <- Envs]
-                  end, AppsEnv).
-
 %% load external plugins which are placed in etc/plugins dir
 load_ext_plugins(undefined) -> ok;
 load_ext_plugins(Dir) ->
@@ -173,15 +138,15 @@ load_ext_plugin(PluginDir) ->
                       ?LOG(alert, "plugin_app_file_not_found: ~s", [AppFile]),
                       error({plugin_app_file_not_found, AppFile})
               end,
-    ok = load_plugin_app(AppName, Ebin),
-    try
-        ok = generate_configs(AppName, PluginDir)
-    catch
-        throw : {conf_file_not_found, ConfFile} ->
-            %% this is maybe a dependency of an external plugin
-            ?LOG(debug, "config_load_error_ignored for app=~p, path=~s", [AppName, ConfFile]),
-            ok
-    end.
+    ok = load_plugin_app(AppName, Ebin).
+    % try
+    %     ok = generate_configs(AppName, PluginDir)
+    % catch
+    %     throw : {conf_file_not_found, ConfFile} ->
+    %         %% this is maybe a dependency of an external plugin
+    %         ?LOG(debug, "config_load_error_ignored for app=~p, path=~s", [AppName, ConfFile]),
+    %         ok
+    % end.
 
 load_plugin_app(AppName, Ebin) ->
     _ = code:add_patha(Ebin),
@@ -199,57 +164,24 @@ load_plugin_app(AppName, Ebin) ->
         {error, {already_loaded, _}} -> ok
     end.
 
-ensure_file(File) ->
-    case filelib:is_file(File) of false -> write_loaded([]); true -> ok end.
-
-with_loaded_file(File, SuccFun) ->
-    case read_loaded(File) of
-        {ok, Names0} ->
-            Names = filter_plugins(Names0),
-            SuccFun(Names);
-        {error, Error} ->
-            ?LOG(alert, "Failed to read: ~p, error: ~p", [File, Error]),
-            {error, Error}
-    end.
-
-filter_plugins(Names) ->
-    lists:filtermap(fun(Name1) when is_atom(Name1) -> {true, Name1};
-                       ({Name1, true}) -> {true, Name1};
-                       ({_Name1, false}) -> false
-                    end, Names).
-
-load_plugins(Names, Persistent) ->
-    Plugins = list(),
-    NotFound = Names -- names(Plugins),
-    case NotFound of
-        []       -> ok;
-        NotFound -> ?LOG(alert, "cannot_find_plugins: ~p", [NotFound])
-    end,
-    NeedToLoad = Names -- NotFound -- names(started_app),
-    lists:foreach(fun(Name) ->
-                      Plugin = find_plugin(Name, Plugins),
-                      load_plugin(Plugin#plugin.name, Persistent)
-                  end, NeedToLoad).
-
 %% Stop plugins
-stop_plugins(Names) ->
-    _ = [stop_app(App) || App <- Names],
+stop_plugins(Plugins) ->
+    _ = [stop_app(Plugin#plugin.name) || Plugin <- Plugins],
     ok.
 
-plugin(AppName, Type) ->
+plugin(AppName) ->
     case application:get_all_key(AppName) of
         {ok, Attrs} ->
             Descr = proplists:get_value(description, Attrs, ""),
-            #plugin{name = AppName, descr = Descr, type = plugin_type(Type)};
+            #plugin{name = AppName, descr = Descr};
         undefined -> error({plugin_not_found, AppName})
     end.
 
-load_plugin(Name, Persistent) ->
+load_plugin(Name) ->
     try
-        ok = ?MODULE:generate_configs(Name),
         case load_app(Name) of
             ok ->
-                start_app(Name, fun(App) -> plugin_loaded(App, Persistent) end);
+                start_app(Name);
             {error, Error0} ->
                 {error, Error0}
         end
@@ -268,22 +200,21 @@ load_app(App) ->
             {error, Error}
     end.
 
-start_app(App, SuccFun) ->
+start_app(App) ->
     case application:ensure_all_started(App) of
         {ok, Started} ->
             ?LOG(info, "Started plugins: ~p", [Started]),
             ?LOG(info, "Load plugin ~s successfully", [App]),
-            _ = SuccFun(App),
             ok;
         {error, {ErrApp, Reason}} ->
             ?LOG(error, "Load plugin ~s failed, cannot start plugin ~s for ~0p", [App, ErrApp, Reason]),
             {error, {ErrApp, Reason}}
     end.
 
-unload_plugin(App, Persistent) ->
+unload_plugin(App) ->
     case stop_app(App) of
         ok ->
-            _ = plugin_unloaded(App, Persistent), ok;
+            ok;
         {error, Reason} ->
             {error, Reason}
     end.
@@ -307,133 +238,5 @@ names(started_app) ->
 names(Plugins) ->
     [Name || #plugin{name = Name} <- Plugins].
 
-plugin_loaded(_Name, false) ->
-    ok;
-plugin_loaded(Name, true) ->
-    case read_loaded() of
-        {ok, Names} ->
-            case lists:member(Name, Names) of
-                false ->
-                    %% write file if plugin is loaded
-                    write_loaded(lists:append(Names, [{Name, true}]));
-                true ->
-                    ignore
-            end;
-        {error, Error} ->
-            ?LOG(error, "Cannot read loaded plugins: ~p", [Error])
-    end.
-
-plugin_unloaded(_Name, false) ->
-    ok;
-plugin_unloaded(Name, true) ->
-    case read_loaded() of
-        {ok, Names0} ->
-            Names = filter_plugins(Names0),
-            case lists:member(Name, Names) of
-                true ->
-                    write_loaded(lists:delete(Name, Names));
-                false ->
-                    ?LOG(error, "Cannot find ~s in loaded_file", [Name])
-            end;
-        {error, Error} ->
-            ?LOG(error, "Cannot read loaded_plugins: ~p", [Error])
-    end.
-
-read_loaded() ->
-    case emqx:get_env(plugins_loaded_file) of
-        undefined -> {error, not_found};
-        File      -> read_loaded(File)
-    end.
-
-read_loaded(File) -> file:consult(File).
-
-write_loaded(AppNames) ->
-    FilePath = emqx:get_env(plugins_loaded_file),
-    case file:write_file(FilePath, [io_lib:format("~p.~n", [Name]) || Name <- AppNames]) of
-        ok -> ok;
-        {error, Error} ->
-            ?LOG(error, "Write File ~p Error: ~p", [FilePath, Error]),
-            {error, Error}
-    end.
-
-plugin_type(auth) -> auth;
-plugin_type(protocol) -> protocol;
-plugin_type(backend) -> backend;
-plugin_type(bridge) -> bridge;
-plugin_type(_) -> feature.
-
-
 funlog(Key, Value) ->
     ?LOG(info, "~s = ~p", [string:join(Key, "."), Value]).
-
-generate_configs(App) ->
-    PluginConfDir = emqx:get_env(plugins_etc_dir),
-    PluginSchemaDir = code:priv_dir(App),
-    generate_configs(App, PluginConfDir, PluginSchemaDir).
-
-generate_configs(App, PluginDir) ->
-    PluginConfDir = filename:join([PluginDir, "etc"]),
-    PluginSchemaDir = filename:join([PluginDir, "priv"]),
-    generate_configs(App, PluginConfDir, PluginSchemaDir).
-
-generate_configs(App, PluginConfDir, PluginSchemaDir) ->
-    ConfigFile = filename:join([PluginConfDir, App]) ++ ".config",
-    case filelib:is_file(ConfigFile) of
-        true ->
-            {ok, [Configs]} = file:consult(ConfigFile),
-            apply_configs(Configs);
-        false ->
-            SchemaFile = filename:join([PluginSchemaDir, App]) ++ ".schema",
-            case filelib:is_file(SchemaFile) of
-                true ->
-                    AppsEnv = do_generate_configs(App),
-                    apply_configs(AppsEnv);
-                false ->
-                    SchemaMod = lists:concat([App, "_schema"]),
-                    ConfName = filename:join([PluginConfDir, App]) ++ ".conf",
-                    SchemaFile1 = filename:join([code:lib_dir(App), "ebin", SchemaMod]) ++ ".beam",
-                    do_generate_hocon_configs(App, ConfName, SchemaFile1)
-            end
-    end.
-
-do_generate_configs(App) ->
-    Name1 = filename:join([emqx:get_env(plugins_etc_dir), App]) ++ ".conf",
-    Name2 = filename:join([code:lib_dir(App), "etc", App]) ++ ".conf",
-    ConfFile = case {filelib:is_file(Name1), filelib:is_file(Name2)} of
-                   {true, _} -> Name1;
-                   {false, true} -> Name2;
-                   {false, false} -> error({config_not_found, [Name1, Name2]})
-               end,
-    SchemaFile = filename:join([code:priv_dir(App), App]) ++ ".schema",
-    case filelib:is_file(SchemaFile) of
-        true ->
-            Schema = cuttlefish_schema:files([SchemaFile]),
-            Conf = cuttlefish_conf:file(ConfFile),
-            cuttlefish_generator:map(Schema, Conf, undefined, fun ?MODULE:funlog/2);
-        false ->
-            error({schema_not_found, SchemaFile})
-    end.
-
-do_generate_hocon_configs(App, ConfName, SchemaFile) ->
-    SchemaMod = lists:concat([App, "_schema"]),
-    case {filelib:is_file(ConfName), filelib:is_file(SchemaFile)} of
-        {true, true} ->
-            {ok, RawConfig} = hocon:load(ConfName, #{format => richmap}),
-            _ = hocon_schema:check(list_to_atom(SchemaMod), RawConfig, #{atom_key => true,
-                                                                         return_plain => true}),
-            ok;
-            % emqx_config:update_config([App], Config);
-        {true, false} ->
-            error({schema_not_found, [SchemaFile]});
-        {false, true} ->
-            error({config_not_found, [ConfName]});
-        {false, false} ->
-            error({conf_and_schema_not_found, [ConfName, SchemaFile]})
-    end.
-
-apply_configs([]) ->
-    ok;
-apply_configs([{App, Config} | More]) ->
-    lists:foreach(fun({Key, _}) -> application:unset_env(App, Key) end, application:get_all_env(App)),
-    lists:foreach(fun({Key, Val}) -> application:set_env(App, Key, Val) end, Config),
-    apply_configs(More).

apps/emqx/src/emqx_schema.erl

@@ -65,6 +65,12 @@ includes() ->
     [ "emqx_data_bridge"
     , "emqx_telemetry"
     , "emqx_retainer"
+    , "emqx_statsd"
+    , "emqx_authn"
+    , "emqx_authz"
+    , "emqx_bridge_mqtt"
+    , "emqx_modules"
+    , "emqx_management"
     ].
 -endif.
 

apps/emqx/test/emqx_plugins_SUITE.erl

@@ -63,64 +63,28 @@ t_load(_) ->
     ?assertEqual({error, not_started}, emqx_plugins:unload(emqx_hocon_plugin)),
 
     application:set_env(emqx, expand_plugins_dir, undefined),
-    application:set_env(emqx, plugins_loaded_file, undefined),
-    ?assertEqual(ignore, emqx_plugins:load()),
-    ?assertEqual(ignore, emqx_plugins:unload()).
-
-
-t_init_config(_) ->
-    ConfFile = "emqx_mini_plugin.config",
-    Data = "[{emqx_mini_plugin,[{mininame ,test}]}].",
-    file:write_file(ConfFile, list_to_binary(Data)),
-    ?assertEqual(ok, emqx_plugins:init_config(ConfFile)),
-    file:delete(ConfFile),
-    ?assertEqual({ok,test}, application:get_env(emqx_mini_plugin, mininame)).
+    application:set_env(emqx, plugins_loaded_file, undefined).
 
 t_load_ext_plugin(_) ->
     ?assertError({plugin_app_file_not_found, _},
                  emqx_plugins:load_ext_plugin("./not_existed_path/")).
 
 t_list(_) ->
-    ?assertMatch([{plugin, _, _, _, _, _, _, _} | _ ], emqx_plugins:list()).
+    ?assertMatch([{plugin, _, _, _, _, _, _} | _ ], emqx_plugins:list()).
 
 t_find_plugin(_) ->
-    ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)),
-    ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_hocon_plugin)).
-
-t_plugin_type(_) ->
-    ?assertEqual(auth, emqx_plugins:plugin_type(auth)),
-    ?assertEqual(protocol, emqx_plugins:plugin_type(protocol)),
-    ?assertEqual(backend, emqx_plugins:plugin_type(backend)),
-    ?assertEqual(bridge, emqx_plugins:plugin_type(bridge)),
-    ?assertEqual(feature, emqx_plugins:plugin_type(undefined)).
-
-t_with_loaded_file(_) ->
-    ?assertMatch({error, _}, emqx_plugins:with_loaded_file("./not_existed_path/", fun(_) -> ok end)).
-
-t_plugin_loaded(_) ->
-    ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, false)),
-    ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_mini_plugin, true)),
-    ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, false)),
-    ?assertEqual(ok, emqx_plugins:plugin_loaded(emqx_hocon_plugin, true)).
-
-t_plugin_unloaded(_) ->
-    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, false)),
-    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_mini_plugin, true)),
-    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, false)),
-    ?assertEqual(ok, emqx_plugins:plugin_unloaded(emqx_hocon_plugin, true)).
+    ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_mini_plugin)),
+    ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _}, emqx_plugins:find_plugin(emqx_hocon_plugin)).
 
 t_plugin(_) ->
     try
-        emqx_plugins:plugin(not_existed_plugin, undefined)
+        emqx_plugins:plugin(not_existed_plugin)
     catch
         _Error:Reason:_Stacktrace ->
             ?assertEqual({plugin_not_found,not_existed_plugin}, Reason)
     end,
-    ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin, undefined)),
-    ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _, _}, emqx_plugins:plugin(emqx_hocon_plugin, undefined)).
-
-t_filter_plugins(_) ->
-    ?assertEqual([name1, name2], emqx_plugins:filter_plugins([name1, {name2,true}, {name3, false}])).
+    ?assertMatch({plugin, emqx_mini_plugin, _, _, _, _, _}, emqx_plugins:plugin(emqx_mini_plugin)),
+    ?assertMatch({plugin, emqx_hocon_plugin, _, _, _, _, _}, emqx_plugins:plugin(emqx_hocon_plugin)).
 
 t_load_plugin(_) ->
     ok = meck:new(application, [unstick, non_strict, passthrough, no_history]),
@@ -133,9 +97,9 @@ t_load_plugin(_) ->
     ok = meck:new(emqx_plugins, [unstick, non_strict, passthrough, no_history]),
     ok = meck:expect(emqx_plugins, generate_configs, fun(_) -> ok end),
     ok = meck:expect(emqx_plugins, apply_configs, fun(_) -> ok end),
-    ?assertMatch({error, _}, emqx_plugins:load_plugin(already_loaded_app, true)),
-    ?assertMatch(ok, emqx_plugins:load_plugin(normal, true)),
-    ?assertMatch({error,_}, emqx_plugins:load_plugin(error_app, true)),
+    ?assertMatch({error, _}, emqx_plugins:load_plugin(already_loaded_app)),
+    ?assertMatch(ok, emqx_plugins:load_plugin(normal)),
+    ?assertMatch({error,_}, emqx_plugins:load_plugin(error_app)),
 
     ok = meck:unload(emqx_plugins),
     ok = meck:unload(application).
@@ -146,8 +110,8 @@ t_unload_plugin(_) ->
                                            (error_app) -> {error, error};
                                            (_) -> ok end),
 
-    ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app, true)),
-    ?assertEqual(ok, emqx_plugins:unload_plugin(normal, true)),
-    ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app, true)),
+    ?assertEqual(ok, emqx_plugins:unload_plugin(not_started_app)),
+    ?assertEqual(ok, emqx_plugins:unload_plugin(normal)),
+    ?assertEqual({error,error}, emqx_plugins:unload_plugin(error_app)),
 
     ok = meck:unload(application).

apps/emqx_authn/etc/emqx_authn.conf

@@ -1,4 +1,4 @@
-authn: {
+emqx_authn: {
     chains: [
         # {
         #     id: "chain1"

apps/emqx_authn/src/emqx_authn_app.erl

@@ -38,11 +38,8 @@ stop(_State) ->
     ok.
 
 initialize() ->
-    ConfFile = filename:join([emqx:get_env(plugins_etc_dir), ?APP]) ++ ".conf",
-    {ok, RawConfig} = hocon:load(ConfFile),
-    #{authn := #{chains := Chains,
-                 bindings := Bindings}}
-        = hocon_schema:check_plain(emqx_authn_schema, RawConfig, #{atom_key => true, nullable => true}),
+    #{chains := Chains,
+     bindings := Bindings} = emqx_config:get([authn], #{chains => [], bindings => []}),
     initialize_chains(Chains),
     initialize_bindings(Bindings).
 

apps/emqx_authn/src/emqx_authn_schema.erl

@@ -27,9 +27,9 @@
               , authenticator_name/0
               ]).
 
-structs() -> [authn].
+structs() -> ["emqx_authn"].
 
-fields(authn) ->
+fields("emqx_authn") ->
     [ {chains, fun chains/1}
     , {bindings, fun bindings/1}];
 
@@ -80,7 +80,7 @@ fields(pgsql) ->
     , {config, hoconsc:t(hoconsc:ref(emqx_authn_pgsql, config))}
     ].
 
-chains(type) -> hoconsc:array({union, [hoconsc:ref('simple-chain')]});
+chains(type) -> hoconsc:array({union, [hoconsc:ref(?MODULE, 'simple-chain')]});
 chains(default) -> [];
 chains(_) -> undefined.
 
@@ -89,10 +89,10 @@ chain_id(nullable) -> false;
 chain_id(_) -> undefined.
 
 simple_authenticators(type) ->
-    hoconsc:array({union, [ hoconsc:ref('built-in-database')
-                          , hoconsc:ref(jwt)
-                          , hoconsc:ref(mysql)
-                          , hoconsc:ref(pgsql)]});
+    hoconsc:array({union, [ hoconsc:ref(?MODULE, 'built-in-database')
+                          , hoconsc:ref(?MODULE, jwt)
+                          , hoconsc:ref(?MODULE, mysql)
+                          , hoconsc:ref(?MODULE, pgsql)]});
 simple_authenticators(default) -> [];
 simple_authenticators(_) -> undefined.
 
@@ -105,7 +105,7 @@ authenticator_name(type) -> authenticator_name();
 authenticator_name(nullable) -> false;
 authenticator_name(_) -> undefined.
 
-bindings(type) -> hoconsc:array(hoconsc:ref(binding));
+bindings(type) -> hoconsc:array(hoconsc:ref(?MODULE, binding));
 bindings(default) -> [];
 bindings(_) -> undefined.
 

apps/emqx_authz/src/emqx_authz.erl

@@ -36,11 +36,7 @@ register_metrics() ->
 
 init() ->
     ok = register_metrics(),
-    Conf = filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'),
-    {ok, RawConf} = hocon:load(Conf),
-    #{emqx_authz := #{rules := Rules}} = hocon_schema:check_plain(emqx_authz_schema, RawConf, #{atom_key => true}),
-    emqx_config:put([emqx_authz], #{rules => Rules}),
-    % Rules = emqx_config:get([emqx_authz, rules], []),
+    Rules = emqx_config:get([emqx_authz, rules], []),
     NRules = [compile(Rule) || Rule <- Rules],
     ok = emqx_hooks:add('client.authorize', {?MODULE, authorize, [NRules]},  -1).
 

apps/emqx_authz/test/emqx_authz_SUITE.erl

@@ -41,11 +41,7 @@ set_special_configs(emqx) ->
     application:set_env(emqx, enable_acl_cache, false),
     ok;
 set_special_configs(emqx_authz) ->
-    application:set_env(emqx, plugins_etc_dir,
-                        emqx_ct_helpers:deps_path(emqx_authz, "test")),
-    Conf = #{<<"emqx_authz">> => #{<<"rules">> => []}},
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)),
-    % emqx_config:put([emqx_authz], #{rules => []}),
+    emqx_config:put([emqx_authz], #{rules => []}),
     ok;
 set_special_configs(_App) ->
     ok.

apps/emqx_authz/test/emqx_authz_api_SUITE.erl

@@ -55,22 +55,13 @@ set_special_configs(emqx) ->
     application:set_env(emqx, enable_acl_cache, false),
     ok;
 set_special_configs(emqx_authz) ->
-    application:set_env(emqx, plugins_etc_dir,
-                        emqx_ct_helpers:deps_path(emqx_authz, "test")),
-    Conf = #{<<"emqx_authz">> => #{<<"rules">> => []}},
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)),
-    % emqx_config:put([emqx_authz], #{rules => []}),
+    emqx_config:put([emqx_authz], #{rules => []}),
     ok;
 
 set_special_configs(emqx_management) ->
-    application:set_env(emqx, plugins_etc_dir,
-        emqx_ct_helpers:deps_path(emqx_management, "test")),
-    Conf = #{<<"emqx_management">> => #{
-        <<"listeners">> => [#{
-            <<"protocol">> => <<"http">>
-        }]}
-    },
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}],
+                                         default_application_id => <<"admin">>,
+                                         default_application_secret => <<"public">>}),
     ok;
 
 set_special_configs(_App) ->

apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl

@@ -47,22 +47,12 @@ set_special_configs(emqx) ->
                         emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")),
     ok;
 set_special_configs(emqx_authz) ->
-    application:set_env(emqx, plugins_etc_dir,
-                        emqx_ct_helpers:deps_path(emqx_authz, "test")),
-    Conf = #{<<"emqx_authz">> =>
-             #{<<"rules">> =>
-               [#{<<"config">> =>#{<<"meck">> => <<"fake">>},
-                  <<"principal">> => all,
-                  <<"sql">> => <<"fake sql">>,
-                  <<"type">> => mysql}
-               ]}},
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)),
-    % Rules = [#{config =>#{<<"meck">> => <<"fake">>},
-    %            principal => all,
-    %            sql => <<"fake sql">>,
-    %            type => mysql}
-    %         ],
-    % emqx_config:put([emqx_authz], #{rules => Rules}),
+    Rules = [#{config =>#{<<"meck">> => <<"fake">>},
+               principal => all,
+               sql => <<"fake sql">>,
+               type => mysql}
+            ],
+    emqx_config:put([emqx_authz], #{rules => Rules}),
     ok;
 set_special_configs(_App) ->
     ok.

apps/emqx_authz/test/emqx_authz_pgsql_SUITE.erl

@@ -47,22 +47,12 @@ set_special_configs(emqx) ->
                         emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")),
     ok;
 set_special_configs(emqx_authz) ->
-    application:set_env(emqx, plugins_etc_dir,
-                        emqx_ct_helpers:deps_path(emqx_authz, "test")),
-    Conf = #{<<"emqx_authz">> =>
-             #{<<"rules">> =>
-               [#{<<"config">> =>#{<<"meck">> => <<"fake">>},
-                  <<"principal">> => all,
-                  <<"sql">> => <<"fake sql">>,
-                  <<"type">> => pgsql}
-               ]}},
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)),
-    % Rules = [#{config =>#{<<"meck">> => <<"fake">>},
-    %            principal => all,
-    %            sql => <<"fake sql">>,
-    %            type => pgsql}
-    %         ],
-    % emqx_config:put([emqx_authz], #{rules => Rules}),
+    Rules = [#{config =>#{<<"meck">> => <<"fake">>},
+               principal => all,
+               sql => <<"fake sql">>,
+               type => pgsql}
+            ],
+    emqx_config:put([emqx_authz], #{rules => Rules}),
     ok;
 set_special_configs(_App) ->
     ok.

apps/emqx_authz/test/emqx_authz_redis_SUITE.erl

@@ -47,28 +47,12 @@ set_special_configs(emqx) ->
                         emqx_ct_helpers:deps_path(emqx, "test/loaded_plguins")),
     ok;
 set_special_configs(emqx_authz) ->
-    application:set_env(emqx, plugins_etc_dir,
-                        emqx_ct_helpers:deps_path(emqx_authz, "test")),
-    Conf = #{<<"emqx_authz">> =>
-             #{<<"rules">> =>
-               [#{<<"config">> =>#{
-                    <<"server">> => <<"127.0.0.1:6379">>,
-                    <<"password">> => <<"public">>,
-                    <<"pool_size">> => 1,
-                    <<"auto_reconnect">> => true,
-                    <<"ssl">> => #{<<"enable">> => false}
-                  },
-                  <<"principal">> => all,
-                  <<"cmd">> => <<"fake cmd">>,
-                  <<"type">> => redis}
-               ]}},
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)),
-    % Rules = [#{config =>#{<<"meck">> => <<"fake">>},
-    %            principal => all,
-    %            cmd => <<"fake cmd">>,
-    %            type => redis}
-    %         ],
-    % emqx_config:put([emqx_authz], #{rules => Rules}),
+    Rules = [#{config =>#{<<"meck">> => <<"fake">>},
+               principal => all,
+               cmd => <<"fake cmd">>,
+               type => redis}
+            ],
+    emqx_config:put([emqx_authz], #{rules => Rules}),
     ok;
 set_special_configs(_App) ->
     ok.

apps/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf

@@ -3,54 +3,56 @@
 ##====================================================================
 
 emqx_bridge_mqtt:{
-    bridges:[{
-        name: "mqtt1"
-        start_type: auto
-        forwards: ["test/#"],
-        forward_mountpoint: ""
-        reconnect_interval: "30s"
-        batch_size: 100
-        queue:{
-            replayq_dir: false
-            # replayq_seg_bytes: "100MB"
-            # replayq_offload_mode: false
-            # replayq_max_total_bytes: "1GB"
-        },
-        config:{
-            conn_type: mqtt
-            address: "127.0.0.1:1883"
-            proto_ver: v4
-            bridge_mode: true
-            clientid: "client1"
-            clean_start: true
-            username: "username1"
-            password: ""
-            keepalive: 300
-            subscriptions: [{
-                topic: "t/#"
-                qos: 1
-            }]
-            receive_mountpoint: ""
-            retry_interval: "30s"
-            max_inflight: 32
-        }
-    },
-    {
-        name: "rpc1"
-        start_type: auto
-        forwards: ["test/#"],
-        forward_mountpoint: ""
-        reconnect_interval: "30s"
-        batch_size: 100
-        queue:{
-            replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/"
-            replayq_seg_bytes: "100MB"
-            replayq_offload_mode: false
-            replayq_max_total_bytes: "1GB"
-        },
-        config:{
-            conn_type: rpc
-            node: "emqx@127.0.0.1"
-        }
-    }]
+    bridges:[
+        # {
+        #     name: "mqtt1"
+        #     start_type: auto
+        #     forwards: ["test/#"],
+        #     forward_mountpoint: ""
+        #     reconnect_interval: "30s"
+        #     batch_size: 100
+        #     queue:{
+        #         replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/"
+        #         replayq_seg_bytes: "100MB"
+        #         replayq_offload_mode: false
+        #         replayq_max_total_bytes: "1GB"
+        #     },
+        #     config:{
+        #         conn_type: mqtt
+        #         address: "127.0.0.1:1883"
+        #         proto_ver: v4
+        #         bridge_mode: true
+        #         clientid: "client1"
+        #         clean_start: true
+        #         username: "username1"
+        #         password: ""
+        #         keepalive: 300
+        #         subscriptions: [{
+        #             topic: "t/#"
+        #             qos: 1
+        #         }]
+        #         receive_mountpoint: ""
+        #         retry_interval: "30s"
+        #         max_inflight: 32
+        #     }
+        # },
+        # {
+        #     name: "rpc1"
+        #     start_type: auto
+        #     forwards: ["test/#"],
+        #     forward_mountpoint: ""
+        #     reconnect_interval: "30s"
+        #     batch_size: 100
+        #     queue:{
+        #         replayq_dir: "{{ platform_data_dir }}/replayq/bridge_mqtt/"
+        #         replayq_seg_bytes: "100MB"
+        #         replayq_offload_mode: false
+        #         replayq_max_total_bytes: "1GB"
+        #     },
+        #     config:{
+        #         conn_type: rpc
+        #         node: "emqx@127.0.0.1"
+        #     }
+        # }
+    ]
 }

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_schema.erl

@@ -26,7 +26,7 @@
 structs() -> ["emqx_bridge_mqtt"].
 
 fields("emqx_bridge_mqtt") ->
-    [ {bridges, hoconsc:array("bridges")}
+    [ {bridges, hoconsc:array(hoconsc:ref(?MODULE, "bridges"))}
     ];
 
 fields("bridges") ->
@@ -36,8 +36,8 @@ fields("bridges") ->
     , {forward_mountpoint, emqx_schema:t(string())}
     , {reconnect_interval, emqx_schema:t(emqx_schema:duration_ms(), undefined, "30s")}
     , {batch_size, emqx_schema:t(integer(), undefined, 100)}
-    , {queue, emqx_schema:t(hoconsc:ref("queue"))}
-    , {config, hoconsc:union([hoconsc:ref("mqtt"), hoconsc:ref("rpc")])}
+    , {queue, emqx_schema:t(hoconsc:ref(?MODULE, "queue"))}
+    , {config, hoconsc:union([hoconsc:ref(?MODULE, "mqtt"), hoconsc:ref(?MODULE, "rpc")])}
     ];
 
 fields("mqtt") ->

apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl

@@ -40,18 +40,7 @@
 -define(OVERVIEWS, ['alarms/activated', 'alarms/deactivated', banned, brokers, stats, metrics, listeners, clients, subscriptions, routes, plugins]).
 
 all() ->
-    [{group, overview},
-     {group, admins},
-     {group, rest},
-     {group, cli}
-     ].
-
-groups() ->
-    [{overview, [sequence], [t_overview]},
-     {admins, [sequence], [t_admins_add_delete]},
-     {rest, [sequence], [t_rest_api]},
-     {cli, [sequence], [t_cli]}
-    ].
+    emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
     emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard],fun set_special_configs/1),
@@ -62,29 +51,27 @@ end_per_suite(_Config) ->
     ekka_mnesia:ensure_stopped().
 
 set_special_configs(emqx_management) ->
-    application:set_env(emqx, plugins_etc_dir,
-        emqx_ct_helpers:deps_path(emqx_management, "test")),
-    Conf = #{<<"emqx_management">> => #{
-        <<"listeners">> => [#{
-            <<"protocol">> => <<"http">>
-        }]}
-    },
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}],
+                                         default_application_id => <<"admin">>,
+                                         default_application_secret => <<"public">>}),
     ok;
 set_special_configs(_) ->
     ok.
 
 t_overview(_) ->
+    mnesia:clear_table(mqtt_admin),
+    emqx_dashboard_admin:add_user(<<"admin">>, <<"public">>, <<"tag">>),
     [?assert(request_dashboard(get, api_path(erlang:atom_to_list(Overview)), auth_header_()))|| Overview <- ?OVERVIEWS].
 
 t_admins_add_delete(_) ->
+    mnesia:clear_table(mqtt_admin),
     ok = emqx_dashboard_admin:add_user(<<"username">>, <<"password">>, <<"tag">>),
     ok = emqx_dashboard_admin:add_user(<<"username1">>, <<"password1">>, <<"tag1">>),
     Admins = emqx_dashboard_admin:all_users(),
-    ?assertEqual(3, length(Admins)),
+    ?assertEqual(2, length(Admins)),
     ok = emqx_dashboard_admin:remove_user(<<"username1">>),
     Users = emqx_dashboard_admin:all_users(),
-    ?assertEqual(2, length(Users)),
+    ?assertEqual(1, length(Users)),
     ok = emqx_dashboard_admin:change_password(<<"username">>, <<"password">>, <<"pwd">>),
     timer:sleep(10),
     ?assert(request_dashboard(get, api_path("brokers"), auth_header_("username", "pwd"))),
@@ -93,6 +80,8 @@ t_admins_add_delete(_) ->
     ?assertNotEqual(true, request_dashboard(get, api_path("brokers"), auth_header_("username", "pwd"))).
 
 t_rest_api(_Config) ->
+    mnesia:clear_table(mqtt_admin),
+    emqx_dashboard_admin:add_user(<<"admin">>, <<"public">>, <<"administrator">>),
     {ok, Res0} = http_get("users"),
 
     ?assertEqual([#{<<"username">> => <<"admin">>,

apps/emqx_data_bridge/etc/emqx_data_bridge.conf

@@ -2,125 +2,128 @@
 ## EMQ X Bridge Plugin
 ##--------------------------------------------------------------------
 
-emqx_data_bridge.bridges: [
-#    {name: "mysql_bridge_1"
-#     type: mysql
-#     config: {
-#        server: "192.168.0.172:3306"
-#        database: mqtt
-#        pool_size: 1
-#        username: root
-#        password: public
-#        auto_reconnect: true
-#        ssl: false
-#     }
-#    }
-#   , {name: "pgsql_bridge_1"
-#     type: pgsql
-#     config: {
-#        server: "192.168.0.172:5432"
-#        database: mqtt
-#        pool_size: 1
-#        username: root
-#        password: public
-#        auto_reconnect: true
-#        ssl: false
-#     }
-#    }
-#   , {name: "mongodb_bridge_single"
-#     type: mongo
-#     config: {
-#        servers: "192.168.0.172:27017"
-#        mongo_type: single
-#        pool_size: 1
-#        login: root
-#        password: public
-#        auth_source: mqtt
-#        database: mqtt
-#        ssl: false
-#     }
-#    }
-#  ,{name: "mongodb_bridge_rs"
-#   type: mongo
-#   config: {
-#      servers: "127.0.0.1:27017"
-#      mongo_type: rs
-#      rs_set_name: rs_name
-#      pool_size: 1
-#      login: root
-#      password: public
-#      auth_source: mqtt
-#      database: mqtt
-#      ssl: false
-#   }
-#  }
-#  ,{name: "mongodb_bridge_shared"
-#   type: mongo
-#   config: {
-#      servers: "127.0.0.1:27017"
-#      mongo_type: shared
-#      pool_size: 1
-#      login: root
-#      password: public
-#      auth_source: mqtt
-#      database: mqtt
-#      ssl: false
-#      max_overflow: 1
-#      overflow_ttl:
-#      overflow_check_period: 10s
-#      local_threshold_ms: 10s
-#      connect_timeout_ms: 10s
-#      socket_timeout_ms: 10s
-#      server_selection_timeout_ms: 10s
-#      wait_queue_timeout_ms: 10s
-#      heartbeat_frequency_ms: 10s
-#      min_heartbeat_frequency_ms: 10s
-#   }
-#  }
-#  , {name: "redis_bridge_single"
-#    type: redis
-#    config: {
-#       servers: "192.168.0.172:6379"
-#       redis_type: single
-#       pool_size: 1
-#       database: 0
-#       password: public
-#       auto_reconnect: true
-#       ssl: false
-#    }
-#   }
-#  ,{name: "redis_bridge_sentinel"
-#   type: redis
-#   config: {
-#      servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379"
-#      redis_type: sentinel
-#      sentinel_name: mymaster
-#      pool_size: 1
-#      database: 0
-#      ssl: false
-#   }
-#  }
-#  ,{name: "redis_bridge_cluster"
-#   type: redis
-#   config: {
-#      servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379"
-#      redis_type: cluster
-#      pool_size: 1
-#      database: 0
-#      password: "public"
-#      ssl: false
-#   }
-#  }
-#  , {name: "ldap_bridge_1"
-#    type: ldap
-#    config: {
-#       servers: "192.168.0.172"
-#       port: 389
-#       bind_dn: "cn=root,dc=emqx,dc=io"
-#       bind_password: "public"
-#       timeout: 30s
-#       pool_size: 1
-#       ssl: false
-#    }
-#   }
-]
+emqx_data_bridge:{
+    bridges:[
+        #    {name: "mysql_bridge_1"
+        #     type: mysql
+        #     config: {
+        #        server: "192.168.0.172:3306"
+        #        database: mqtt
+        #        pool_size: 1
+        #        username: root
+        #        password: public
+        #        auto_reconnect: true
+        #        ssl: false
+        #     }
+        #    }
+        #   , {name: "pgsql_bridge_1"
+        #     type: pgsql
+        #     config: {
+        #        server: "192.168.0.172:5432"
+        #        database: mqtt
+        #        pool_size: 1
+        #        username: root
+        #        password: public
+        #        auto_reconnect: true
+        #        ssl: false
+        #     }
+        #    }
+        #   , {name: "mongodb_bridge_single"
+        #     type: mongo
+        #     config: {
+        #        servers: "192.168.0.172:27017"
+        #        mongo_type: single
+        #        pool_size: 1
+        #        login: root
+        #        password: public
+        #        auth_source: mqtt
+        #        database: mqtt
+        #        ssl: false
+        #     }
+        #    }
+        #  ,{name: "mongodb_bridge_rs"
+        #   type: mongo
+        #   config: {
+        #      servers: "127.0.0.1:27017"
+        #      mongo_type: rs
+        #      rs_set_name: rs_name
+        #      pool_size: 1
+        #      login: root
+        #      password: public
+        #      auth_source: mqtt
+        #      database: mqtt
+        #      ssl: false
+        #   }
+        #  }
+        #  ,{name: "mongodb_bridge_shared"
+        #   type: mongo
+        #   config: {
+        #      servers: "127.0.0.1:27017"
+        #      mongo_type: shared
+        #      pool_size: 1
+        #      login: root
+        #      password: public
+        #      auth_source: mqtt
+        #      database: mqtt
+        #      ssl: false
+        #      max_overflow: 1
+        #      overflow_ttl:
+        #      overflow_check_period: 10s
+        #      local_threshold_ms: 10s
+        #      connect_timeout_ms: 10s
+        #      socket_timeout_ms: 10s
+        #      server_selection_timeout_ms: 10s
+        #      wait_queue_timeout_ms: 10s
+        #      heartbeat_frequency_ms: 10s
+        #      min_heartbeat_frequency_ms: 10s
+        #   }
+        #  }
+        #  , {name: "redis_bridge_single"
+        #    type: redis
+        #    config: {
+        #       servers: "192.168.0.172:6379"
+        #       redis_type: single
+        #       pool_size: 1
+        #       database: 0
+        #       password: public
+        #       auto_reconnect: true
+        #       ssl: false
+        #    }
+        #   }
+        #  ,{name: "redis_bridge_sentinel"
+        #   type: redis
+        #   config: {
+        #      servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379"
+        #      redis_type: sentinel
+        #      sentinel_name: mymaster
+        #      pool_size: 1
+        #      database: 0
+        #      ssl: false
+        #   }
+        #  }
+        #  ,{name: "redis_bridge_cluster"
+        #   type: redis
+        #   config: {
+        #      servers: "127.0.0.1:6379, 127.0.0.2:6379, 127.0.0.3:6379"
+        #      redis_type: cluster
+        #      pool_size: 1
+        #      database: 0
+        #      password: "public"
+        #      ssl: false
+        #   }
+        #  }
+        #  , {name: "ldap_bridge_1"
+        #    type: ldap
+        #    config: {
+        #       servers: "192.168.0.172"
+        #       port: 389
+        #       bind_dn: "cn=root,dc=emqx,dc=io"
+        #       bind_password: "public"
+        #       timeout: 30s
+        #       pool_size: 1
+        #       ssl: false
+        #    }
+        #   }
+
+    ]
+}

apps/emqx_data_bridge/src/emqx_data_bridge_schema.erl

@@ -23,4 +23,4 @@ fields(mysql) -> ?BRIDGE_FIELDS(mysql);
 fields(pgsql) -> ?BRIDGE_FIELDS(pgsql);
 fields(mongo) -> ?BRIDGE_FIELDS(mongo);
 fields(redis) -> ?BRIDGE_FIELDS(redis);
-fields(ldap) -> ?BRIDGE_FIELDS(ldap).
+fields(ldap) -> ?BRIDGE_FIELDS(ldap).

apps/emqx_management/src/emqx_management.app.src

@@ -1,6 +1,6 @@
 {application, emqx_management,
  [{description, "EMQ X Management API and CLI"},
-  {vsn, "4.4.0"}, % strict semver, bump manually!
+  {vsn, "5.0.0"}, % strict semver, bump manually!
   {modules, []},
   {registered, [emqx_management_sup]},
   {applications, [kernel,stdlib,minirest]},

apps/emqx_management/src/emqx_management.appup.src

@@ -1,13 +0,0 @@
-%% -*- mode: erlang -*-
-{VSN,
- [ {<<"4.3.[0-2]">>,
-    [ {restart_application, emqx_management}
-    ]},
-   {<<".*">>, []}
- ],
- [ {<<"4.3.[0-2]">>,
-    [ {restart_application, emqx_management}
-    ]},
-   {<<".*">>, []}
- ]
-}.

apps/emqx_management/src/emqx_management_schema.erl

@@ -28,7 +28,7 @@ fields("emqx_management") ->
     [ {default_application_id, fun default_application_id/1}
     , {default_application_secret, fun default_application_secret/1}
     , {max_row_limit, fun max_row_limit/1}
-    , {listeners, hoconsc:array(hoconsc:union([hoconsc:ref("http"), hoconsc:ref("https")]))}
+    , {listeners, hoconsc:array(hoconsc:union([hoconsc:ref(?MODULE, "http"), hoconsc:ref(?MODULE, "https")]))}
     ];
 
 fields("http") ->

apps/emqx_management/src/emqx_mgmt.erl

@@ -525,7 +525,7 @@ check_row_limit([Tab|Tables], Limit) ->
     end.
 
 max_row_limit() ->
-    application:get_env(?APP, max_row_limit, ?MAX_ROW_LIMIT).
+    emqx_config:get([?APP, max_row_limit], ?MAX_ROW_LIMIT).
 
 table_size(Tab) -> ets:info(Tab, size).
 

apps/emqx_management/src/emqx_mgmt_api_plugins.erl

@@ -106,10 +106,8 @@ format({Node, Plugins}) ->
 
 format(#plugin{name = Name,
                descr = Descr,
-               active = Active,
-               type = Type}) ->
+               active = Active}) ->
     #{name => Name,
       description => iolist_to_binary(Descr),
-      active => Active,
-      type => Type}.
+      active => Active}.
 

apps/emqx_management/src/emqx_mgmt_app.erl

@@ -29,11 +29,6 @@
 -include("emqx_mgmt.hrl").
 
 start(_Type, _Args) ->
-    Conf = filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'),
-    {ok, RawConf} = hocon:load(Conf),
-    #{emqx_management := Config} =
-        hocon_schema:check_plain(emqx_management_schema, RawConf, #{atom_key => true}),
-    [application:set_env(?APP, Key, maps:get(Key, Config)) || Key <- maps:keys(Config)],
     {ok, Sup} = emqx_mgmt_sup:start_link(),
     ok = ekka_rlog:wait_for_shards([?MANAGEMENT_SHARD], infinity),
     _ = emqx_mgmt_auth:add_default_app(),

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -68,14 +68,14 @@ mnesia(copy) ->
 %%--------------------------------------------------------------------
 -spec(add_default_app() -> ok | {ok, appsecret()} | {error, term()}).
 add_default_app() ->
-    AppId = application:get_env(?APP, default_application_id, undefined),
-    AppSecret = application:get_env(?APP, default_application_secret, undefined),
+    AppId = emqx_config:get([?APP, default_application_id], undefined),
+    AppSecret = emqx_config:get([?APP, default_application_secret], undefined),
     case {AppId, AppSecret} of
         {undefined, _} -> ok;
         {_, undefined} -> ok;
         {_, _} ->
-            AppId1 = erlang:list_to_binary(AppId),
-            AppSecret1 = erlang:list_to_binary(AppSecret),
+            AppId1 = to_binary(AppId),
+            AppSecret1 = to_binary(AppSecret),
             add_app(AppId1, <<"Default">>, AppSecret1, <<"Application user">>, true, undefined)
     end.
 
@@ -210,3 +210,6 @@ is_authorized(AppId, AppSecret) ->
 
 is_expired(undefined) -> true;
 is_expired(Expired)   -> Expired >= erlang:system_time(second).
+
+to_binary(L) when is_list(L) -> list_to_binary(L);
+to_binary(B) when is_binary(B) -> B.

apps/emqx_management/src/emqx_mgmt_http.erl

@@ -80,7 +80,7 @@ stop_listener({Proto, Port, _}) ->
 listeners() ->
     [{list_to_atom(Protocol), Port, maps:to_list(maps:without([protocol, port], Map))}
         || Map = #{protocol := Protocol,port := Port}
-        <- application:get_env(?APP, listeners, [])].
+        <- emqx_config:get([emqx_management, listeners], [])].
 
 listener_name(Proto) ->
     list_to_atom(atom_to_list(Proto) ++ ":management").

apps/emqx_management/test/emqx_mgmt_SUITE.erl

@@ -29,50 +29,19 @@
 -define(LOG_HANDLER_ID, [file, default]).
 
 all() ->
-    [{group, manage_apps},
-     {group, check_cli}].
-
-groups() ->
-    [{manage_apps, [sequence],
-      [t_app
-      ]},
-      {check_cli, [sequence],
-       [t_cli,
-        t_log_cmd,
-        t_mgmt_cmd,
-        t_status_cmd,
-        t_clients_cmd,
-        t_vm_cmd,
-        t_plugins_cmd,
-        t_trace_cmd,
-        t_broker_cmd,
-        t_router_cmd,
-        t_subscriptions_cmd,
-        t_listeners_cmd_old,
-        t_listeners_cmd_new
-       ]}].
-
-apps() ->
-    [emqx_management, emqx_retainer].
+    emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
     ekka_mnesia:start(),
     emqx_mgmt_auth:mnesia(boot),
-    emqx_ct_helpers:start_apps(apps(), fun set_special_configs/1),
+    emqx_ct_helpers:start_apps([emqx_retainer, emqx_management], fun set_special_configs/1),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps(apps()).
+    emqx_ct_helpers:stop_apps([emqx_management, emqx_retainer]).
 
 set_special_configs(emqx_management) ->
-    application:set_env(emqx, plugins_etc_dir,
-        emqx_ct_helpers:deps_path(emqx_management, "test")),
-    Conf = #{<<"emqx_management">> => #{
-        <<"listeners">> => [#{
-            <<"protocol">> => <<"http">>
-        }]}
-    },
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}]}),
     ok;
 set_special_configs(_App) ->
     ok.

apps/emqx_management/test/emqx_mgmt_api_SUITE.erl

@@ -51,14 +51,9 @@ end_per_testcase(_, Config) ->
     Config.
 
 set_special_configs(emqx_management) ->
-    application:set_env(emqx, plugins_etc_dir,
-        emqx_ct_helpers:deps_path(emqx_management, "test")),
-    Conf = #{<<"emqx_management">> => #{
-        <<"listeners">> => [#{
-            <<"protocol">> => <<"http">>
-        }]}
-    },
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}],
+                                         default_application_id => <<"admin">>,
+                                         default_application_secret => <<"public">>}),
     ok;
 set_special_configs(_App) ->
     ok.

apps/emqx_modules/src/emqx_modules.erl

@@ -143,6 +143,8 @@ cli(_) ->
                     {"modules reload <Module>", "Reload module"}
                    ]).
 
+name(Name) when is_binary(Name) ->
+    name(binary_to_atom(Name, utf8));
 name(delayed) -> emqx_mod_delayed;
 name(presence) -> emqx_mod_presence;
 name(recon) -> emqx_mod_recon;

apps/emqx_modules/test/emqx_modules_SUITE.erl

@@ -37,15 +37,9 @@ init_per_suite(Config) ->
     Config.
 
 set_special_configs(emqx_management) ->
-    application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")),
-    application:set_env(emqx, plugins_etc_dir,
-        emqx_ct_helpers:deps_path(emqx_management, "test")),
-    Conf = #{<<"emqx_management">> => #{
-        <<"listeners">> => [#{
-            <<"protocol">> => <<"http">>
-        }]}
-    },
-    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    emqx_config:put([emqx_management], #{listeners => [#{protocol => "http", port => 8081}],
+                                         default_application_id => <<"admin">>,
+                                         default_application_secret => <<"public">>}),
     ok;
 set_special_configs(_) ->
     ok.

apps/emqx_sn/vars

@@ -5,4 +5,4 @@
 {platform_etc_dir,  "etc"}.
 {platform_lib_dir,  "lib"}.
 {platform_log_dir,  "log"}.
-{platform_plugins_dir,  "plugins"}.
+{platform_plugins_dir,  "data/plugins"}.

data/loaded_plugins.tmpl

@@ -1,3 +0,0 @@
-{emqx_management, true}.
-{emqx_dashboard, true}.
-{emqx_retainer, {{enable_plugin_emqx_retainer}}}.

rebar.config.erl

@@ -127,8 +127,9 @@ prod_compile_opts() ->
 prod_overrides() ->
     [{add, [ {erl_opts, [deterministic]}]}].
 
-relup_deps(Profile) ->
-    {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}.
+relup_deps(_Profile) ->
+    % {post_hooks, [{"(linux|darwin|solaris|freebsd|netbsd|openbsd)", compile, "scripts/inject-deps.escript " ++ atom_to_list(Profile)}]}.
+    {post_hooks, []}.
 
 profiles() ->
     Vsn = get_vsn(),
@@ -192,9 +193,8 @@ overlay_vars_rel(RelType) ->
                  cloud -> "vm.args";
                  edge -> "vm.args.edge"
              end,
-    [
-      {enable_plugin_emqx_retainer, true}
-    , {vm_args_file, VmArgs}
+
+    [ {vm_args_file, VmArgs}
     ].
 
 %% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb)
@@ -204,7 +204,7 @@ overlay_vars_pkg(bin) ->
     , {platform_etc_dir, "etc"}
     , {platform_lib_dir, "lib"}
     , {platform_log_dir, "log"}
-    , {platform_plugins_dir,  "etc/plugins"}
+    , {platform_plugins_dir, "plugins"}
     , {runner_root_dir, "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)"}
     , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"}
     , {runner_etc_dir, "$RUNNER_ROOT_DIR/etc"}
@@ -219,7 +219,7 @@ overlay_vars_pkg(pkg) ->
     , {platform_etc_dir, "/etc/emqx"}
     , {platform_lib_dir, ""}
     , {platform_log_dir, "/var/log/emqx"}
-    , {platform_plugins_dir, "/var/lib/emqx/plugins"}
+    , {platform_plugins_dir, "/var/lib/enqx/plugins"}
     , {runner_root_dir, "/usr/lib/emqx"}
     , {runner_bin_dir, "/usr/bin"}
     , {runner_etc_dir, "/etc/emqx"}
@@ -255,8 +255,12 @@ relx_apps(ReleaseType) ->
     , emqx_connector
     , emqx_data_bridge
     , emqx_rule_engine
+    , emqx_rule_actions
     , emqx_bridge_mqtt
     , emqx_modules
+    , emqx_management
+    , emqx_retainer
+    , emqx_statsd
     ]
     ++ [emqx_telemetry || not is_enterprise()]
     ++ [emqx_license || is_enterprise()]
@@ -279,25 +283,13 @@ is_app(Name) ->
     end.
 
 relx_plugin_apps(ReleaseType) ->
-    [ emqx_retainer
-    , emqx_management
-    , emqx_dashboard
-    , emqx_sn
-    , emqx_coap
-    , emqx_stomp
-    , emqx_statsd
-    , emqx_rule_actions
-    ]
+    []
     ++ relx_plugin_apps_per_rel(ReleaseType)
     ++ relx_plugin_apps_enterprise(is_enterprise())
     ++ relx_plugin_apps_extra().
 
 relx_plugin_apps_per_rel(cloud) ->
-    [ emqx_lwm2m
-    , emqx_exhook
-    , emqx_exproto
-    , emqx_prometheus
-    ];
+    [];
 relx_plugin_apps_per_rel(edge) ->
     [].
 
@@ -313,11 +305,11 @@ relx_plugin_apps_extra() ->
 relx_overlay(ReleaseType) ->
     [ {mkdir, "log/"}
     , {mkdir, "data/"}
+    , {mkdir, "plugins"}
     , {mkdir, "data/mnesia"}
     , {mkdir, "data/configs"}
     , {mkdir, "data/patches"}
     , {mkdir, "data/scripts"}
-    , {template, "data/loaded_plugins.tmpl", "data/loaded_plugins"}
     , {template, "data/emqx_vars", "releases/emqx_vars"}
     , {template, "data/BUILT_ON", "releases/{{release_version}}/BUILT_ON"}
     , {copy, "bin/emqx", "bin/emqx"}
@@ -337,12 +329,8 @@ relx_overlay(ReleaseType) ->
          end.
 
 etc_overlay(ReleaseType) ->
-    PluginApps = relx_plugin_apps(ReleaseType),
-    Templates = emqx_etc_overlay(ReleaseType) ++
-                lists:append([plugin_etc_overlays(App) || App <- PluginApps]) ++
-                [community_plugin_etc_overlays(App) || App <- relx_plugin_apps_extra()],
+    Templates = emqx_etc_overlay(ReleaseType),
     [ {mkdir, "etc/"}
-    , {mkdir, "etc/plugins"}
     , {copy, "{{base_dir}}/lib/emqx/etc/certs","etc/"}
     ] ++
     lists:map(
@@ -352,7 +340,7 @@ etc_overlay(ReleaseType) ->
     ++ extra_overlay(ReleaseType).
 
 extra_overlay(cloud) ->
-    [ {copy,"{{base_dir}}/lib/emqx_lwm2m/lwm2m_xml","etc/"}
+    [
     ];
 extra_overlay(edge) ->
     [].
@@ -366,42 +354,9 @@ emqx_etc_overlay(edge) ->
     ].
 
 emqx_etc_overlay_common() ->
-    [{"{{base_dir}}/lib/emqx/etc/acl.conf", "etc/acl.conf"},
-     {"{{base_dir}}/lib/emqx/etc/emqx.conf", "etc/emqx.conf"},
-     {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"},
-     {"{{base_dir}}/lib/emqx_data_bridge/etc/emqx_data_bridge.conf", "etc/plugins/emqx_data_bridge.conf"},
-     {"{{base_dir}}/lib/emqx_telemetry/etc/emqx_telemetry.conf", "etc/plugins/emqx_telemetry.conf"},
-     {"{{base_dir}}/lib/emqx_authn/etc/emqx_authn.conf", "etc/plugins/emqx_authn.conf"},
-     {"{{base_dir}}/lib/emqx_authz/etc/emqx_authz.conf", "etc/plugins/authz.conf"},
-     {"{{base_dir}}/lib/emqx_rule_engine/etc/emqx_rule_engine.conf", "etc/plugins/emqx_rule_engine.conf"},
-     {"{{base_dir}}/lib/emqx_bridge_mqtt/etc/emqx_bridge_mqtt.conf", "etc/plugins/emqx_bridge_mqtt.conf"},
-     {"{{base_dir}}/lib/emqx_modules/etc/emqx_modules.conf", "etc/plugins/emqx_modules.conf"},
-     %% TODO: check why it has to end with .paho
-     %% and why it is put to etc/plugins dir
-     {"{{base_dir}}/lib/emqx/etc/acl.conf.paho", "etc/plugins/acl.conf.paho"}].
-
-plugin_etc_overlays(App0) ->
-    App = atom_to_list(App0),
-    ConfFiles = find_conf_files(App),
-    %% NOTE: not filename:join here since relx translates it for windows
-    [{"{{base_dir}}/lib/"++ App ++"/etc/" ++ F, "etc/plugins/" ++ F}
-     || F <- ConfFiles].
-
-community_plugin_etc_overlays(App0) ->
-    App = atom_to_list(App0),
-    {"{{base_dir}}/lib/"++ App ++"/etc/" ++ App ++ ".conf", "etc/plugins/" ++ App ++ ".conf"}.
-
-%% NOTE: for apps fetched as rebar dependency (there is so far no such an app)
-%% the overlay should be hand-coded but not to rely on build-time wildcards.
-find_conf_files(App) ->
-    Dir1 = filename:join(["apps", App, "etc"]),
-    filelib:wildcard("*.conf", Dir1) ++
-    case is_enterprise() of
-        true ->
-            Dir2 = filename:join(["lib-ee", App, "etc"]),
-            filelib:wildcard("*.conf", Dir2);
-        false -> []
-    end.
+    [ {"{{base_dir}}/lib/emqx/etc/emqx.conf.all", "etc/emqx.conf"}
+    , {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}
+    ].
 
 get_vsn() ->
     PkgVsn = os:cmd("./pkg-vsn.sh"),

scripts/find-apps.sh

@@ -7,7 +7,7 @@ cd -P -- "$(dirname -- "$0")/.."
 
 find_app() {
     local appdir="$1"
-    find "${appdir}" -mindepth 1 -maxdepth 1 -type d | grep -vE "emqx_exhook|emqx_exproto|emqx_lwm2m|emqx_sn|emqx_coap|emqx_stomp"
+    find "${appdir}" -mindepth 1 -maxdepth 1 -type d | grep -vE "emqx_exhook|emqx_exproto|emqx_lwm2m|emqx_sn|emqx_coap|emqx_stomp|emqx_dashboard"
 }
 
 find_app 'apps'

scripts/merge-config.escript

@@ -0,0 +1,36 @@
+#!/usr/bin/env escript
+
+%% This script reads up emqx.conf and split the sections
+%% and dump sections to separate files.
+%% Sections are grouped between CONFIG_SECTION_BGN and
+%% CONFIG_SECTION_END pairs
+%%
+%% NOTE: this feature is so far not used in opensource
+%% edition due to backward-compatibility reasons.
+
+-mode(compile).
+
+main(_) ->
+    BaseConf = "apps/emqx/etc/emqx.conf",
+    {ok, Bin} = file:read_file(BaseConf),
+    Apps = filelib:wildcard("emqx_*", "apps/"),
+    Conf = lists:foldl(fun(App, Acc) ->
+        case lists:member(App, ["emqx_exhook",
+                                "emqx_exproto",
+                                "emqx_lwm2m",
+                                "emqx_sn",
+                                "emqx_coap",
+                                "emqx_stomp",
+                                "emqx_dashboard"]) of
+            true -> Acc;
+            false ->
+                Filename = filename:join([apps, App, "etc", App]) ++ ".conf",
+                case filelib:is_regular(Filename) of
+                    true ->
+                        {ok, Bin1} = file:read_file(Filename),
+                        <<Acc/binary, "\r\n", Bin1/binary>>;
+                    false -> Acc
+                end
+        end
+    end, Bin, Apps),
+    ok = file:write_file("apps/emqx/etc/emqx.conf.all", Conf).