chore: mgmt hoconf support (#5153)

apps/emqx_authz/test/emqx_authz_api_SUITE.erl

@@ -61,6 +61,18 @@ set_special_configs(emqx_authz) ->
     ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'authz.conf'), jsx:encode(Conf)),
     % emqx_config:put([emqx_authz], #{rules => []}),
     ok;
+
+set_special_configs(emqx_management) ->
+    application:set_env(emqx, plugins_etc_dir,
+        emqx_ct_helpers:deps_path(emqx_management, "test")),
+    Conf = #{<<"emqx_management">> => #{
+        <<"listeners">> => [#{
+            <<"protocol">> => <<"http">>
+        }]}
+    },
+    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    ok;
+
 set_special_configs(_App) ->
     ok.
 

apps/emqx_dashboard/test/emqx_dashboard_SUITE.erl

@@ -54,13 +54,26 @@ groups() ->
     ].
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_modules, emqx_management, emqx_dashboard]),
+    emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard],fun set_special_configs/1),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management, emqx_modules]),
+    emqx_ct_helpers:stop_apps([emqx_dashboard, emqx_management]),
     ekka_mnesia:ensure_stopped().
 
+set_special_configs(emqx_management) ->
+    application:set_env(emqx, plugins_etc_dir,
+        emqx_ct_helpers:deps_path(emqx_management, "test")),
+    Conf = #{<<"emqx_management">> => #{
+        <<"listeners">> => [#{
+            <<"protocol">> => <<"http">>
+        }]}
+    },
+    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    ok;
+set_special_configs(_) ->
+    ok.
+
 t_overview(_) ->
     [?assert(request_dashboard(get, api_path(erlang:atom_to_list(Overview)), auth_header_()))|| Overview <- ?OVERVIEWS].
 

apps/emqx_management/priv/emqx_management.schema

@@ -1,239 +0,0 @@
-%%-*- mode: erlang -*-
-%% emqx_management config mapping
-
-{mapping, "management.max_row_limit", "emqx_management.max_row_limit", [
-  {default, 10000},
-  {datatype, integer}
-]}.
-
-{mapping, "management.default_application.id", "emqx_management.default_application_id", [
-  {default, undefined},
-  {datatype, string}
-]}.
-
-{mapping, "management.default_application.secret", "emqx_management.default_application_secret", [
-  {default, undefined},
-  {datatype, string}
-]}.
-
-{mapping, "management.application.default_secret", "emqx_management.application", [
-  {default, undefined},
-  {datatype, string}
-]}.
-
-{mapping, "management.listener.http.port", "emqx_management.listeners", [
-  {datatype, [integer, ip]}
-]}.
-
-{mapping, "management.listener.http.acceptors", "emqx_management.listeners", [
-  {default, 4},
-  {datatype, integer}
-]}.
-
-{mapping, "management.listener.http.max_clients", "emqx_management.listeners", [
-  {default, 512},
-  {datatype, integer}
-]}.
-
-{mapping, "management.listener.http.backlog", "emqx_management.listeners", [
-  {default, 1024},
-  {datatype, integer}
-]}.
-
-{mapping, "management.listener.http.send_timeout", "emqx_management.listeners", [
-  {datatype, {duration, ms}},
-  {default, "15s"}
-]}.
-
-{mapping, "management.listener.http.send_timeout_close", "emqx_management.listeners", [
-  {datatype, flag},
-  {default, on}
-]}.
-
-{mapping, "management.listener.http.recbuf", "emqx_management.listeners", [
-  {datatype, bytesize},
-  hidden
-]}.
-
-{mapping, "management.listener.http.sndbuf", "emqx_management.listeners", [
-  {datatype, bytesize},
-  hidden
-]}.
-
-{mapping, "management.listener.http.buffer", "emqx_management.listeners", [
-  {datatype, bytesize},
-  hidden
-]}.
-
-{mapping, "management.listener.http.tune_buffer", "emqx_management.listeners", [
-  {datatype, flag},
-  hidden
-]}.
-
-{mapping, "management.listener.http.nodelay", "emqx_management.listeners", [
-  {datatype, {enum, [true, false]}},
-  hidden
-]}.
-
-{mapping, "management.listener.http.inet6", "emqx_management.listeners", [
-  {default, false},
-  {datatype, {enum, [true, false]}}
-]}.
-
-{mapping, "management.listener.http.ipv6_v6only", "emqx_management.listeners", [
-  {default, false},
-  {datatype, {enum, [true, false]}}
-]}.
-
-{mapping, "management.listener.https.port", "emqx_management.listeners", [
-  {datatype, [integer, ip]}
-]}.
-
-{mapping, "management.listener.https.acceptors", "emqx_management.listeners", [
-  {default, 8},
-  {datatype, integer}
-]}.
-
-{mapping, "management.listener.https.max_clients", "emqx_management.listeners", [
-  {default, 64},
-  {datatype, integer}
-]}.
-
-{mapping, "management.listener.https.backlog", "emqx_management.listeners", [
-  {default, 1024},
-  {datatype, integer}
-]}.
-
-{mapping, "management.listener.https.send_timeout", "emqx_management.listeners", [
-  {datatype, {duration, ms}},
-  {default, "15s"}
-]}.
-
-{mapping, "management.listener.https.send_timeout_close", "emqx_management.listeners", [
-  {datatype, flag},
-  {default, on}
-]}.
-
-{mapping, "management.listener.https.recbuf", "emqx_management.listeners", [
-  {datatype, bytesize},
-  hidden
-]}.
-
-{mapping, "management.listener.https.sndbuf", "emqx_management.listeners", [
-  {datatype, bytesize},
-  hidden
-]}.
-
-{mapping, "management.listener.https.buffer", "emqx_management.listeners", [
-  {datatype, bytesize},
-  hidden
-]}.
-
-{mapping, "management.listener.https.tune_buffer", "emqx_management.listeners", [
-  {datatype, flag},
-  hidden
-]}.
-
-{mapping, "management.listener.https.nodelay", "emqx_management.listeners", [
-  {datatype, {enum, [true, false]}},
-  hidden
-]}.
-
-{mapping, "management.listener.https.keyfile", "emqx_management.listeners", [
-  {datatype, string}
-]}.
-
-{mapping, "management.listener.https.certfile", "emqx_management.listeners", [
-  {datatype, string}
-]}.
-
-{mapping, "management.listener.https.cacertfile", "emqx_management.listeners", [
-  {datatype, string}
-]}.
-
-{mapping, "management.listener.https.verify", "emqx_management.listeners", [
-  {datatype, atom}
-]}.
-
-{mapping, "management.listener.https.ciphers", "emqx_management.listeners", [
-  {datatype, string}
-]}.
-
-{mapping, "management.listener.https.tls_versions", "emqx_management.listeners", [
-  {datatype, string}
-]}.
-
-{mapping, "management.listener.https.fail_if_no_peer_cert", "emqx_management.listeners", [
-  {datatype, {enum, [true, false]}}
-]}.
-
-{mapping, "management.listener.https.inet6", "emqx_management.listeners", [
-  {default, false},
-  {datatype, {enum, [true, false]}}
-]}.
-
-{mapping, "management.listener.https.ipv6_v6only", "emqx_management.listeners", [
-  {default, false},
-  {datatype, {enum, [true, false]}}
-]}.
-
-{translation, "emqx_management.application", fun(Conf) ->
-  Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end,
-  Opts = fun(Prefix) ->
-            Filter([{default_secret, cuttlefish:conf_get(Prefix ++ ".default_secret", Conf)}])
-         end,
-  Prefix = "management.application",
-  Transfer = fun(default_secret, V) -> list_to_binary(V);
-                (_, V) -> V
-             end,
-  [{K, Transfer(K, V)}|| {K, V} <- Opts(Prefix)]
-end}.
-
-{translation, "emqx_management.listeners", fun(Conf) ->
-  Filter = fun(Opts) -> [{K, V} || {K, V} <- Opts, V =/= undefined] end,
-  Opts = fun(Prefix) ->
-             Filter([{num_acceptors,   cuttlefish:conf_get(Prefix ++ ".acceptors", Conf)},
-                     {max_connections, cuttlefish:conf_get(Prefix ++ ".max_clients", Conf)}])
-         end,
-  TcpOpts = fun(Prefix) ->
-                Filter([{backlog, cuttlefish:conf_get(Prefix ++ ".backlog", Conf, undefined)},
-                        {send_timeout, cuttlefish:conf_get(Prefix ++ ".send_timeout", Conf, undefined)},
-                        {send_timeout_close, cuttlefish:conf_get(Prefix ++ ".send_timeout_close", Conf, undefined)},
-                        {recbuf,  cuttlefish:conf_get(Prefix ++ ".recbuf", Conf, undefined)},
-                        {sndbuf,  cuttlefish:conf_get(Prefix ++ ".sndbuf", Conf, undefined)},
-                        {buffer,  cuttlefish:conf_get(Prefix ++ ".buffer", Conf, undefined)},
-                        {nodelay, cuttlefish:conf_get(Prefix ++ ".nodelay", Conf, true)},
-                        {inet6, cuttlefish:conf_get(Prefix ++ ".inet6", Conf)},
-                        {ipv6_v6only, cuttlefish:conf_get(Prefix ++ ".ipv6_v6only", Conf)}])
-            end,
-
-  SplitFun = fun(undefined) -> undefined; (S) -> string:tokens(S, ",") end,
-
-  SslOpts = fun(Prefix) ->
-               Versions = case SplitFun(cuttlefish:conf_get(Prefix ++ ".tls_versions", Conf, undefined)) of
-                              undefined -> undefined;
-                              L -> [list_to_atom(V) || V <- L]
-                          end,
-                Filter([{versions, Versions},
-                        {ciphers, SplitFun(cuttlefish:conf_get(Prefix ++ ".ciphers", Conf, undefined))},
-                        {keyfile,    cuttlefish:conf_get(Prefix ++ ".keyfile", Conf, undefined)},
-                        {certfile,   cuttlefish:conf_get(Prefix ++ ".certfile", Conf, undefined)},
-                        {cacertfile, cuttlefish:conf_get(Prefix ++ ".cacertfile", Conf, undefined)},
-                        {verify,     cuttlefish:conf_get(Prefix ++ ".verify", Conf, undefined)},
-                        {fail_if_no_peer_cert, cuttlefish:conf_get(Prefix ++ ".fail_if_no_peer_cert", Conf, undefined)}])
-              end,
-    lists:foldl(
-      fun(Proto, Acc) ->
-          Prefix = "management.listener." ++ atom_to_list(Proto),
-          case cuttlefish:conf_get(Prefix ++ ".port", Conf, undefined) of
-              undefined -> Acc;
-              Port ->
-                  [{Proto, Port, TcpOpts(Prefix) ++ Opts(Prefix)
-                    ++ case Proto of
-                         http -> [];
-                         https -> SslOpts(Prefix)
-                       end} | Acc]
-          end
-      end, [], [http, https])
-end}.
-

apps/emqx_management/src/emqx_management_schema.erl

@@ -0,0 +1,62 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_management_schema).
+
+-include_lib("typerefl/include/types.hrl").
+
+-behaviour(hocon_schema).
+
+-export([ structs/0
+    , fields/1]).
+
+structs() -> ["emqx_management"].
+
+fields("emqx_management") ->
+    [ {default_application_id, fun default_application_id/1}
+    , {default_application_secret, fun default_application_secret/1}
+    , {max_row_limit, fun max_row_limit/1}
+    , {listeners, hoconsc:array(hoconsc:union([hoconsc:ref("http"), hoconsc:ref("https")]))}
+    ];
+
+fields("http") ->
+    [ {"protocol", emqx_schema:t(string(), undefined, "http")}
+    , {"port", emqx_schema:t(integer(), undefined, 8081)}
+    , {"num_acceptors", emqx_schema:t(integer(), undefined, 4)}
+    , {"max_connections", emqx_schema:t(integer(), undefined, 512)}
+    , {"backlog", emqx_schema:t(integer(), undefined, 1024)}
+    , {"send_timeout", emqx_schema:t(emqx_schema:duration(), undefined, "15s")}
+    , {"send_timeout_close", emqx_schema:t(emqx_schema:flag(), undefined, true)}
+    , {"inet6", emqx_schema:t(boolean(), undefined, false)}
+    , {"ipv6_v6only", emqx_schema:t(boolean(), undefined, false)}
+    ];
+
+fields("https") ->
+    emqx_schema:ssl(undefined, #{enable => true}) ++ fields("http").
+
+default_application_id(type) -> string();
+default_application_id(default) -> "admin";
+default_application_id(nullable) -> true;
+default_application_id(_) -> undefined.
+
+default_application_secret(type) -> string();
+default_application_secret(default) -> "public";
+default_application_secret(nullable) -> true;
+default_application_secret(_) -> undefined.
+
+max_row_limit(type) -> integer();
+max_row_limit(default) -> 1000;
+max_row_limit(nullable) -> false;
+max_row_limit(_) -> undefined.

apps/emqx_management/src/emqx_mgmt_app.erl

@@ -20,6 +20,8 @@
 
 -emqx_plugin(?MODULE).
 
+-define(APP, emqx_management).
+
 -export([ start/2
         , stop/1
         ]).
@@ -27,6 +29,11 @@
 -include("emqx_mgmt.hrl").
 
 start(_Type, _Args) ->
+    Conf = filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'),
+    {ok, RawConf} = hocon:load(Conf),
+    #{emqx_management := Config} =
+        hocon_schema:check_plain(emqx_management_schema, RawConf, #{atom_key => true}),
+    [application:set_env(?APP, Key, maps:get(Key, Config)) || Key <- maps:keys(Config)],
     {ok, Sup} = emqx_mgmt_sup:start_link(),
     ok = ekka_rlog:wait_for_shards([?MANAGEMENT_SHARD], infinity),
     _ = emqx_mgmt_auth:add_default_app(),

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -129,11 +129,7 @@ force_add_app(AppId, Name, Secret, Desc, Status, Expired) ->
 generate_appsecret_if_need(InSecrt) when is_binary(InSecrt), byte_size(InSecrt) > 0 ->
     InSecrt;
 generate_appsecret_if_need(_) ->
-    AppConf = application:get_env(?APP, application, []),
-    case proplists:get_value(default_secret,  AppConf) of
-       undefined -> emqx_guid:to_base62(emqx_guid:gen());
-       Secret when is_binary(Secret) -> Secret
-    end.
+    emqx_guid:to_base62(emqx_guid:gen()).
 
 -spec(get_appsecret(appid()) -> {appsecret() | undefined}).
 get_appsecret(AppId) when is_binary(AppId) ->

apps/emqx_management/src/emqx_mgmt_http.erl

@@ -78,7 +78,9 @@ stop_listener({Proto, Port, _}) ->
     minirest:stop_http(listener_name(Proto)).
 
 listeners() ->
-    application:get_env(?APP, listeners, []).
+    [{list_to_atom(Protocol), Port, maps:to_list(maps:without([protocol, port], Map))}
+        || Map = #{protocol := Protocol,port := Port}
+        <- application:get_env(?APP, listeners, [])].
 
 listener_name(Proto) ->
     list_to_atom(atom_to_list(Proto) ++ ":management").

apps/emqx_management/test/emqx_mgmt_SUITE.erl

@@ -53,18 +53,30 @@ groups() ->
        ]}].
 
 apps() ->
-    [emqx_management, emqx_retainer, emqx_modules].
+    [emqx_management, emqx_retainer].
 
 init_per_suite(Config) ->
-    application:set_env(ekka, strict_mode, true),
     ekka_mnesia:start(),
     emqx_mgmt_auth:mnesia(boot),
-    emqx_ct_helpers:start_apps(apps()),
+    emqx_ct_helpers:start_apps(apps(), fun set_special_configs/1),
     Config.
 
 end_per_suite(_Config) ->
     emqx_ct_helpers:stop_apps(apps()).
 
+set_special_configs(emqx_management) ->
+    application:set_env(emqx, plugins_etc_dir,
+        emqx_ct_helpers:deps_path(emqx_management, "test")),
+    Conf = #{<<"emqx_management">> => #{
+        <<"listeners">> => [#{
+            <<"protocol">> => <<"http">>
+        }]}
+    },
+    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    ok;
+set_special_configs(_App) ->
+    ok.
+
 t_app(_Config) ->
     {ok, AppSecret} = emqx_mgmt_auth:add_app(<<"app_id">>, <<"app_name">>),
     ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret)),
@@ -74,16 +86,6 @@ t_app(_Config) ->
                   true, undefined},
                  lists:keyfind(<<"app_id">>, 1, emqx_mgmt_auth:list_apps())),
     emqx_mgmt_auth:del_app(<<"app_id">>),
-    %% Use the default application secret
-    application:set_env(emqx_management, application, [{default_secret, <<"public">>}]),
-    {ok, AppSecret1} = emqx_mgmt_auth:add_app(
-                         <<"app_id">>, <<"app_name">>, <<"app_desc">>, true, undefined),
-    ?assert(emqx_mgmt_auth:is_authorized(<<"app_id">>, AppSecret1)),
-    ?assertEqual(AppSecret1, emqx_mgmt_auth:get_appsecret(<<"app_id">>)),
-    ?assertEqual(AppSecret1, <<"public">>),
-    ?assertEqual({<<"app_id">>, AppSecret1, <<"app_name">>, <<"app_desc">>, true, undefined},
-                 lists:keyfind(<<"app_id">>, 1, emqx_mgmt_auth:list_apps())),
-    emqx_mgmt_auth:del_app(<<"app_id">>),
     application:set_env(emqx_management, application, []),
     %% Specify the application secret
     {ok, AppSecret2} = emqx_mgmt_auth:add_app(

apps/emqx_management/test/emqx_mgmt_api_SUITE.erl

@@ -37,8 +37,7 @@ all() ->
     emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    application:load(emqx_modules),
-    emqx_ct_helpers:start_apps([emqx_management]),
+    emqx_ct_helpers:start_apps([emqx_management], fun set_special_configs/1),
     Config.
 
 end_per_suite(Config) ->
@@ -51,6 +50,19 @@ init_per_testcase(_, Config) ->
 end_per_testcase(_, Config) ->
     Config.
 
+set_special_configs(emqx_management) ->
+    application:set_env(emqx, plugins_etc_dir,
+        emqx_ct_helpers:deps_path(emqx_management, "test")),
+    Conf = #{<<"emqx_management">> => #{
+        <<"listeners">> => [#{
+            <<"protocol">> => <<"http">>
+        }]}
+    },
+    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    ok;
+set_special_configs(_App) ->
+    ok.
+
 get(Key, ResponseBody) ->
    maps:get(Key, jiffy:decode(list_to_binary(ResponseBody), [return_maps])).
 

apps/emqx_modules/test/emqx_modules_SUITE.erl

@@ -32,11 +32,22 @@
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_special_cfg/1),
+    emqx_ct_helpers:start_apps([emqx_management, emqx_modules], fun set_special_configs/1),
     emqx_ct_http:create_default_app(),
     Config.
 
-set_special_cfg(_) ->
+set_special_configs(emqx_management) ->
+    application:set_env(emqx, modules_loaded_file, emqx_ct_helpers:deps_path(emqx, "test/emqx_SUITE_data/loaded_modules")),
+    application:set_env(emqx, plugins_etc_dir,
+        emqx_ct_helpers:deps_path(emqx_management, "test")),
+    Conf = #{<<"emqx_management">> => #{
+        <<"listeners">> => [#{
+            <<"protocol">> => <<"http">>
+        }]}
+    },
+    ok = file:write_file(filename:join(emqx:get_env(plugins_etc_dir), 'emqx_management.conf'), jsx:encode(Conf)),
+    ok;
+set_special_configs(_) ->
     ok.
 
 end_per_suite(_Config) ->