refactor(emqx_modules): Move emqx_modules to lib-opensource

emqx_moduels for enterprise is refactored as a lib/plugin
in order to avoid merge conflicts, we make sure they can
co-exist in the same branch, because lib-opensource is compiled
in opensource project and lib-enterprise (to be added) is
compiled in enterprise project

apps/emqx_auth_ldap/test/emqx_auth_ldap_SUITE.erl

@@ -44,12 +44,13 @@ groups() ->
 
 init_per_group(GrpName, Cfg) ->
     Fun = fun(App) -> set_special_configs(GrpName, App) end,
+    emqx_ct_helpers:start_apps([emqx_modules]),
     emqx_ct_helpers:start_apps([emqx_auth_ldap], Fun),
     emqx_mod_acl_internal:unload([]),
     Cfg.
 
 end_per_group(_GrpName, _Cfg) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_ldap]).
+    emqx_ct_helpers:stop_apps([emqx_auth_ldap, emqx_modules]).
 
 %%--------------------------------------------------------------------
 %% Cases

apps/emqx_auth_ldap/test/emqx_auth_ldap_bind_as_user_SUITE.erl

@@ -36,12 +36,12 @@ all() ->
      check_acl].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx, emqx_auth_ldap], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_ldap], fun set_special_configs/1),
     emqx_mod_acl_internal:unload([]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_ldap, emqx]).
+    emqx_ct_helpers:stop_apps([emqx_auth_ldap, emqx_modules]).
 
 check_auth(_) ->
     MqttUser1 = #{clientid => <<"mqttuser1">>,

apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE.erl

@@ -50,14 +50,14 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Cfg) ->
-    emqx_ct_helpers:start_apps([emqx_auth_mongo], fun set_special_confs/1),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_mongo], fun set_special_confs/1),
     emqx_modules:load_module(emqx_mod_acl_internal, false),
     init_mongo_data(),
     Cfg.
 
 end_per_suite(_Cfg) ->
     deinit_mongo_data(),
-    emqx_ct_helpers:stop_apps([emqx_auth_mongo]).
+    emqx_ct_helpers:stop_apps([emqx_auth_mongo, emqx_modules]).
 
 set_special_confs(emqx) ->
     application:set_env(emqx, acl_nomatch, deny),

apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE.erl

@@ -70,7 +70,7 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_auth_pgsql]),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_pgsql]),
     drop_acl(),
     drop_auth(),
     init_auth(),
@@ -79,7 +79,7 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_pgsql]),
+    emqx_ct_helpers:stop_apps([emqx_auth_pgsql, emqx_modules]),
     Config.
 
 set_special_configs() ->

apps/emqx_auth_redis/test/emqx_auth_redis_SUITE.erl

@@ -49,13 +49,13 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Cfg) ->
-    emqx_ct_helpers:start_apps([emqx_auth_redis], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_auth_redis], fun set_special_configs/1),
     init_redis_rows(),
     Cfg.
 
 end_per_suite(_Cfg) ->
     deinit_redis_rows(),
-    emqx_ct_helpers:stop_apps([emqx_auth_redis]).
+    emqx_ct_helpers:stop_apps([emqx_auth_redis, emqx_modules]).
 
 set_special_configs(emqx) ->
     application:set_env(emqx, allow_anonymous, false),
@@ -187,4 +187,4 @@ q(Cmd) ->
         _ ->
             {ok, Connection} = ?POOL(?APP),
             eredis:q(Connection, Cmd)
-    end.
+    end.

apps/emqx_lua_hook/test/emqx_lua_hook_SUITE.erl

@@ -41,11 +41,11 @@ all() ->
     ].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_lua_hook], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_lua_hook], fun set_special_configs/1),
     Config.
 
 end_per_suite(Config) ->
-    emqx_ct_helpers:stop_apps([emqx_lua_hook]),
+    emqx_ct_helpers:stop_apps([emqx_lua_hook, emqx_modules]),
     Config.
 
 set_special_configs(emqx) ->

apps/emqx_management/test/emqx_mgmt_SUITE.erl

@@ -58,11 +58,11 @@ apps() ->
 init_per_suite(Config) ->
     ekka_mnesia:start(),
     emqx_mgmt_auth:mnesia(boot),
-    emqx_ct_helpers:start_apps([emqx_management, emqx_auth_mnesia]),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia]).
+    emqx_ct_helpers:stop_apps([emqx_management, emqx_auth_mnesia, emqx_modules]).
 
 t_app(_Config) ->
     {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>),

apps/emqx_management/test/emqx_mgmt_api_SUITE.erl

@@ -58,13 +58,13 @@ groups() ->
     }].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx, emqx_management, emqx_auth_mnesia]),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_auth_mnesia]),
     ekka_mnesia:start(),
     emqx_mgmt_auth:mnesia(boot),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_auth_mnesia, emqx_management, emqx]),
+    emqx_ct_helpers:stop_apps([emqx_auth_mnesia, emqx_management, emqx_modules]),
     ekka_mnesia:ensure_stopped().
 
 init_per_testcase(data, Config) ->

apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl

@@ -28,11 +28,11 @@ all() -> emqx_ct:all(?MODULE).
 
 init_per_testcase(_, Config) ->
     emqx_ct_helpers:boot_modules(all),
-    emqx_ct_helpers:start_apps([emqx_telemetry]),
+    emqx_ct_helpers:start_apps([emqx_modules, emqx_telemetry]),
     Config.
 
 end_per_testcase(_, _Config) ->
-    emqx_ct_helpers:stop_apps([emqx_telemetry]).
+    emqx_ct_helpers:stop_apps([emqx_telemetry, emqx_modules]).
 
 t_uuid(_) ->
     UUID = emqx_telemetry:generate_uuid(),
@@ -63,4 +63,4 @@ t_enable(_) ->
 bin(L) when is_list(L) ->
     list_to_binary(L);
 bin(B) when is_binary(B) ->
-    B.
+    B.

lib-opensource/emqx_modules/etc/emqx_modules.conf

@@ -0,0 +1 @@
+# empty

lib-opensource/emqx_modules/priv/emqx_modules.schema

@@ -0,0 +1 @@
+% empty

lib-opensource/emqx_modules/rebar.config

@@ -0,0 +1 @@
+{deps, []}.

src/emqx_mod_acl_internal.erl

@@ -18,8 +18,8 @@
 
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("logger.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -logger_header("[ACL_INTERNAL]").
 

src/emqx_mod_delayed.erl

@@ -19,8 +19,8 @@
 -behaviour(gen_server).
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("logger.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 %% Mnesia bootstrap
 -export([mnesia/1]).

src/emqx_mod_presence.erl

@@ -18,8 +18,8 @@
 
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("logger.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -logger_header("[Presence]").
 

src/emqx_mod_rewrite.erl

@@ -18,8 +18,8 @@
 
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("emqx_mqtt.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 -ifdef(TEST).
 -export([ compile/1

src/emqx_mod_subscription.erl

@@ -18,8 +18,8 @@
 
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("emqx_mqtt.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 %% emqx_gen_mod callbacks
 -export([ load/1

src/emqx_mod_sup.erl

@@ -18,7 +18,7 @@
 
 -behaviour(supervisor).
 
--include("types.hrl").
+-include_lib("emqx/include/types.hrl").
 
 -export([ start_link/0
         , start_child/1

src/emqx_mod_topic_metrics.erl

@@ -19,9 +19,9 @@
 -behaviour(gen_server).
 -behaviour(emqx_gen_mod).
 
--include("emqx.hrl").
--include("logger.hrl").
--include("emqx_mqtt.hrl").
+-include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx/include/emqx_mqtt.hrl").
 
 -logger_header("[TOPIC_METRICS]").
 

lib-opensource/emqx_modules/src/emqx_modules.app.src

@@ -0,0 +1,9 @@
+{application, emqx_modules,
+ [{description, "EMQ X Module Management"},
+  {vsn, "4.3.0"},
+  {modules, []},
+  {applications, [kernel,stdlib]},
+  {mod, {emqx_modules_app, []}},
+  {registered, [emqx_mod_sup]},
+  {env, []}
+ ]}.

src/emqx_modules.erl

@@ -16,7 +16,7 @@
 
 -module(emqx_modules).
 
--include("logger.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -logger_header("[Modules]").
 

lib-opensource/emqx_modules/src/emqx_modules_app.erl

@@ -0,0 +1,36 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_modules_app).
+
+-behaviour(application).
+
+-emqx_plugin(?MODULE).
+
+-export([start/2]).
+
+-export([stop/1]).
+
+start(_Type, _Args) ->
+    % the configs for emqx_modules is so far still in emqx application
+    % Ensure it's loaded
+    application:load(emqx),
+    {ok, Pid} = emqx_mod_sup:start_link(),
+    ok = emqx_modules:load(),
+    {ok, Pid}.
+
+stop(_State) ->
+    emqx_modules:unload().

test/emqx_mod_delayed_SUITE.erl

@@ -35,11 +35,11 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([], fun set_special_configs/1),
+    emqx_ct_helpers:start_apps([emqx_modules], fun set_special_configs/1),
     Config.
 
 end_per_suite(_) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 set_special_configs(emqx) ->
     application:set_env(emqx, modules, [{emqx_mod_delayed, []}]),

test/emqx_mod_presence_SUITE.erl

@@ -26,13 +26,13 @@ all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
     emqx_ct_helpers:boot_modules(all),
-    emqx_ct_helpers:start_apps([]),
+    emqx_ct_helpers:start_apps([emqx_modules]),
     %% Ensure all the modules unloaded.
     ok = emqx_modules:unload(),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 %% Test case for emqx_mod_presence
 t_mod_presence(_) ->

test/emqx_mod_rewrite_SUITE.erl

@@ -30,13 +30,13 @@ all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
     emqx_ct_helpers:boot_modules(all),
-    emqx_ct_helpers:start_apps([]),
+    emqx_ct_helpers:start_apps([emqx_modules]),
     %% Ensure all the modules unloaded.
     ok = emqx_modules:unload(),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 %% Test case for emqx_mod_write
 t_mod_rewrite(_Config) ->

test/emqx_mod_topic_metrics_SUITE.erl

@@ -25,11 +25,11 @@ all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
     emqx_ct_helpers:boot_modules(all),
-    emqx_ct_helpers:start_apps([]),
+    emqx_ct_helpers:start_apps([emqx_modules]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 t_nonexistent_topic_metrics(_) ->
     emqx_mod_topic_metrics:load([]),

test/emqx_modules_SUITE.erl

@@ -24,7 +24,7 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([], fun set_sepecial_cfg/1),
+    emqx_ct_helpers:start_apps([emqx_modules], fun set_sepecial_cfg/1),
     Config.
 
 set_sepecial_cfg(_) ->
@@ -32,7 +32,7 @@ set_sepecial_cfg(_) ->
     ok.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([]).
+    emqx_ct_helpers:stop_apps([emqx_modules]).
 
 t_load(_) ->
     ?assertEqual(ok, emqx_modules:unload()),

rebar.config.erl

@@ -4,7 +4,9 @@
 
 do(Dir, CONFIG) ->
     ok = compile_and_load_pase_transforms(Dir),
-    dump(deps(CONFIG) ++ dialyzer(CONFIG) ++ coveralls() ++ config()).
+    C1 = deps(CONFIG),
+    Config = dialyzer(C1),
+    dump(Config ++ coveralls() ++ config()).
 
 bcrypt() ->
     {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}}.
@@ -20,8 +22,19 @@ deps(Config) ->
 config() ->
     [ {plugins, plugins()}
     , {profiles, profiles()}
+    , {project_app_dirs, project_app_dirs()}
     ].
 
+extra_lib_dir() ->
+    EnterpriseFlag = os:getenv("EMQX_ENTERPRISE"),
+    case EnterpriseFlag =:= "true" orelse EnterpriseFlag =:= "1" of
+        true -> "lib-enterprise";
+        false -> "lib-opensource"
+    end.
+
+project_app_dirs() ->
+    ["apps/*", extra_lib_dir() ++ "/*", "."].
+
 plugins() ->
     [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {branch,"master"}}},
       {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0"}}}
@@ -142,6 +155,7 @@ relx_plugin_apps(ReleaseType) ->
     , emqx_rule_engine
     , emqx_sasl
     , emqx_telemetry
+    , emqx_modules
     ] ++ relx_plugin_apps_per_rel(ReleaseType).
 
 relx_plugin_apps_per_rel(cloud) ->
@@ -229,8 +243,9 @@ plugin_etc_overlays(App0) ->
 %% NOTE: for apps fetched as rebar dependency (there is so far no such an app)
 %% the overlay should be hand-coded but not to rely on build-time wildcards.
 find_conf_files(App) ->
-    Dir = filename:join(["apps", App, "etc"]),
-    filelib:wildcard("*.conf", Dir).
+    Dir1 = filename:join(["apps", App, "etc"]),
+    Dir2 = filename:join([extra_lib_dir(), App, "etc"]),
+    filelib:wildcard("*.conf", Dir1) ++ filelib:wildcard("*.conf", Dir2).
 
 env(Name, Default) ->
     case os:getenv(Name) of
@@ -275,7 +290,9 @@ str(L) when is_list(L) -> L;
 str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8).
 
 erl_opts_i() ->
-    [{i, "apps"}] ++ [{i, Dir}  || Dir <- filelib:wildcard(filename:join(["apps", "**", "include"]))].
+    [{i, "apps"}] ++
+    [{i, Dir}  || Dir <- filelib:wildcard(filename:join(["apps", "**", "include"]))] ++
+    [{i, Dir}  || Dir <- filelib:wildcard(filename:join([extra_lib_dir(), "**", "include"]))].
 
 dialyzer(Config) ->
     {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
@@ -287,12 +304,9 @@ dialyzer(Config) ->
             [ list_to_atom(App) || App <- string:tokens(Value, ",")]
     end,
 
-    AppsDir = "apps",
-    AppNames = [emqx | list_dir(AppsDir)],
+    AppNames = [emqx | list_dir("apps")] ++ list_dir(extra_lib_dir()),
 
     KnownApps = [Name ||  Name <- AppsToAnalyse, lists:member(Name, AppNames)],
-    UnknownApps = AppsToAnalyse -- KnownApps,
-    io:format("Unknown Apps ~p ~n", [UnknownApps]),
 
     AppsToExclude = AppNames -- KnownApps,
 

src/emqx_app.erl

@@ -32,7 +32,6 @@ start(_Type, _Args) ->
     print_banner(),
     ekka:start(),
     {ok, Sup} = emqx_sup:start_link(),
-    ok = emqx_modules:load(),
     ok = emqx_plugins:init(),
     _ = emqx_plugins:load(),
     emqx_boot:is_enabled(listeners)
@@ -47,8 +46,7 @@ start(_Type, _Args) ->
 stop(_State) ->
     ok = emqx_alarm_handler:unload(),
     emqx_boot:is_enabled(listeners)
-      andalso emqx_listeners:stop(),
-    emqx_modules:unload().
+      andalso emqx_listeners:stop().
 
 %%--------------------------------------------------------------------
 %% Print Banner

src/emqx_sup.erl

@@ -67,12 +67,11 @@ init([]) ->
     BrokerSup = child_spec(emqx_broker_sup, supervisor),
     CMSup = child_spec(emqx_cm_sup, supervisor),
     SysSup = child_spec(emqx_sys_sup, supervisor),
-    ModSup = child_spec(emqx_mod_sup, supervisor),
     Childs = [KernelSup] ++
              [RouterSup || emqx_boot:is_enabled(router)] ++
              [BrokerSup || emqx_boot:is_enabled(broker)] ++
              [CMSup || emqx_boot:is_enabled(broker)] ++
-             [SysSup] ++ [ModSup],
+             [SysSup],
     SupFlags = #{strategy => one_for_all,
                  intensity => 0,
                  period => 1