Merge pull request #7628 from JimMoen/refine-authn-authz-api-fields

refine authn/authz api fields

apps/emqx/include/emqx_access_control.hrl

@@ -0,0 +1,20 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% config root name all auth providers have to agree on.
+-define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME, "authorization").
+-define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM, authorization).
+-define(EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY, <<"authorization">>).

apps/emqx/src/emqx_schema.erl

@@ -24,6 +24,7 @@
 -elvis([{elvis_style, invalid_dynamic_call, disable}]).
 
 -include("emqx_authentication.hrl").
+-include("emqx_access_control.hrl").
 -include_lib("typerefl/include/types.hrl").
 
 -type duration() :: integer().
@@ -159,9 +160,9 @@ roots(high) ->
             )},
         %% NOTE: authorization schema here is only to keep emqx app prue
         %% the full schema for EMQX node is injected in emqx_conf_schema.
-        {"authorization",
+        {?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME,
             sc(
-                ref("authorization"),
+                ref(?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME),
                 #{}
             )}
     ];
@@ -315,6 +316,7 @@ fields("authorization") ->
                 hoconsc:enum([allow, deny]),
                 #{
                     default => allow,
+                    required => true,
                     %% TODO: make sources a reference link
                     desc =>
                         "Default access control action if the user or client matches no ACL rules,\n"
@@ -328,6 +330,7 @@ fields("authorization") ->
                 hoconsc:enum([ignore, disconnect]),
                 #{
                     default => ignore,
+                    required => true,
                     desc => "The action when the authorization check rejects an operation."
                 }
             )},

apps/emqx_authn/src/emqx_authn.erl

@@ -29,14 +29,14 @@
 
 providers() ->
     [
-        {{'password_based', 'built_in_database'}, emqx_authn_mnesia},
-        {{'password_based', mysql}, emqx_authn_mysql},
-        {{'password_based', postgresql}, emqx_authn_pgsql},
-        {{'password_based', mongodb}, emqx_authn_mongodb},
-        {{'password_based', redis}, emqx_authn_redis},
-        {{'password_based', 'http'}, emqx_authn_http},
+        {{password_based, built_in_database}, emqx_authn_mnesia},
+        {{password_based, mysql}, emqx_authn_mysql},
+        {{password_based, postgresql}, emqx_authn_pgsql},
+        {{password_based, mongodb}, emqx_authn_mongodb},
+        {{password_based, redis}, emqx_authn_redis},
+        {{password_based, http}, emqx_authn_http},
         {jwt, emqx_authn_jwt},
-        {{scram, 'built_in_database'}, emqx_enhanced_authn_scram_mnesia}
+        {{scram, built_in_database}, emqx_enhanced_authn_scram_mnesia}
     ].
 
 check_configs(C) when is_map(C) ->

apps/emqx_authn/src/emqx_authn_api.erl

@@ -24,7 +24,7 @@
 -include_lib("emqx/include/logger.hrl").
 -include_lib("emqx/include/emqx_authentication.hrl").
 
--import(hoconsc, [mk/2, ref/1]).
+-import(hoconsc, [mk/2, ref/1, ref/2]).
 -import(emqx_dashboard_swagger, [error_codes/2]).
 
 -define(BAD_REQUEST, 'BAD_REQUEST').
@@ -128,31 +128,26 @@ roots() ->
 
 fields(request_user_create) ->
     [
-        {user_id, binary()}
+        {user_id, mk(binary(), #{required => true})}
         | fields(request_user_update)
     ];
 fields(request_user_update) ->
     [
-        {password, binary()},
+        {password, mk(binary(), #{required => true})},
         {is_superuser, mk(boolean(), #{default => false, required => false})}
     ];
 fields(request_move) ->
-    [{position, binary()}];
+    [{position, mk(binary(), #{required => true})}];
 fields(request_import_users) ->
-    [{filename, binary()}];
+    %% TODO: add file update
+    [{filename, mk(binary(), #{required => true})}];
 fields(response_user) ->
     [
-        {user_id, binary()},
+        {user_id, mk(binary(), #{required => true})},
         {is_superuser, mk(boolean(), #{default => false, required => false})}
     ];
 fields(response_users) ->
-    paginated_list_type(ref(response_user));
-fields(pagination_meta) ->
-    [
-        {page, pos_integer()},
-        {limit, pos_integer()},
-        {count, non_neg_integer()}
-    ].
+    paginated_list_type(ref(response_user)).
 
 schema("/authentication") ->
     #{
@@ -431,10 +426,8 @@ schema("/authentication/:id/users") ->
             description => <<"List users in authenticator in global authentication chain">>,
             parameters => [
                 param_auth_id(),
-                {page,
-                    mk(pos_integer(), #{in => query, desc => <<"Page Index">>, required => false})},
-                {limit,
-                    mk(pos_integer(), #{in => query, desc => <<"Page Limit">>, required => false})},
+                ref(emqx_dashboard_swagger, page),
+                ref(emqx_dashboard_swagger, limit),
                 {like_username,
                     mk(binary(), #{
                         in => query,
@@ -483,10 +476,8 @@ schema("/listeners/:listener_id/authentication/:id/users") ->
             parameters => [
                 param_listener_id(),
                 param_auth_id(),
-                {page,
-                    mk(pos_integer(), #{in => query, desc => <<"Page Index">>, required => false})},
-                {limit,
-                    mk(pos_integer(), #{in => query, desc => <<"Page Limit">>, required => false})}
+                ref(emqx_dashboard_swagger, page),
+                ref(emqx_dashboard_swagger, limit)
             ],
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_example(
@@ -587,7 +578,8 @@ param_auth_id() ->
         id,
         mk(binary(), #{
             in => path,
-            desc => <<"Authenticator ID">>
+            desc => <<"Authenticator ID">>,
+            required => true
         })
     }.
 
@@ -597,6 +589,7 @@ param_listener_id() ->
         mk(binary(), #{
             in => path,
             desc => <<"Listener ID">>,
+            required => true,
             example => emqx_listeners:id_example()
         })
     }.
@@ -1182,7 +1175,7 @@ update_config(Path, ConfigRequest) ->
 
 get_raw_config_with_defaults(ConfKeyPath) ->
     NConfKeyPath = [atom_to_binary(Key, utf8) || Key <- ConfKeyPath],
-    RawConfig = emqx_map_lib:deep_get(NConfKeyPath, emqx_config:get_raw([]), []),
+    RawConfig = emqx:get_raw_config(NConfKeyPath, []),
     ensure_list(fill_defaults(RawConfig)).
 
 find_config(AuthenticatorID, AuthenticatorsConfig) ->
@@ -1200,7 +1193,24 @@ find_config(AuthenticatorID, AuthenticatorsConfig) ->
 fill_defaults(Configs) when is_list(Configs) ->
     lists:map(fun fill_defaults/1, Configs);
 fill_defaults(Config) ->
-    emqx_authn:check_config(Config, #{only_fill_defaults => true}).
+    emqx_authn:check_config(merge_default_headers(Config), #{only_fill_defaults => true}).
+
+merge_default_headers(Config) ->
+    case maps:find(<<"headers">>, Config) of
+        {ok, Headers} ->
+            NewHeaders =
+                case Config of
+                    #{<<"method">> := <<"get">>} ->
+                        (emqx_authn_http:headers_no_content_type(converter))(Headers);
+                    #{<<"method">> := <<"post">>} ->
+                        (emqx_authn_http:headers(converter))(Headers);
+                    _ ->
+                        Headers
+                end,
+            Config#{<<"headers">> => NewHeaders};
+        error ->
+            Config
+    end.
 
 convert_certs(#{ssl := SSL} = Config) when SSL =/= undefined ->
     Config#{ssl := emqx_tls_lib:drop_invalid_certs(SSL)};
@@ -1316,7 +1326,7 @@ binfmt(Fmt, Args) -> iolist_to_binary(io_lib:format(Fmt, Args)).
 paginated_list_type(Type) ->
     [
         {data, hoconsc:array(Type)},
-        {meta, ref(pagination_meta)}
+        {meta, ref(emqx_dashboard_swagger, meta)}
     ].
 
 authenticator_array_example() ->

apps/emqx_authn/src/emqx_authn_password_hashing.erl

@@ -68,21 +68,31 @@ roots() -> [pbkdf2, bcrypt, bcrypt_rw, other_algorithms].
 
 fields(bcrypt_rw) ->
     fields(bcrypt) ++
-        [{salt_rounds, fun salt_rounds/1}];
+        [
+            {salt_rounds,
+                sc(
+                    integer(),
+                    #{
+                        default => 10,
+                        example => 10,
+                        desc => "Salt rounds for BCRYPT password generation."
+                    }
+                )}
+        ];
 fields(bcrypt) ->
-    [{name, sc(bcrypt, #{desc => "BCRYPT password hashing."})}];
+    [{name, sc(bcrypt, #{required => true, desc => "BCRYPT password hashing."})}];
 fields(pbkdf2) ->
     [
-        {name, sc(pbkdf2, #{desc => "PBKDF2 password hashing."})},
+        {name, sc(pbkdf2, #{required => true, desc => "PBKDF2 password hashing."})},
         {mac_fun,
             sc(
                 hoconsc:enum([md4, md5, ripemd160, sha, sha224, sha256, sha384, sha512]),
-                #{desc => "Specifies mac_fun for PBKDF2 hashing algorithm."}
+                #{required => true, desc => "Specifies mac_fun for PBKDF2 hashing algorithm."}
             )},
         {iterations,
             sc(
                 integer(),
-                #{desc => "Iteration count for PBKDF2 hashing algorithm."}
+                #{required => true, desc => "Iteration count for PBKDF2 hashing algorithm."}
             )},
         {dk_length, fun dk_length/1}
     ];
@@ -91,10 +101,7 @@ fields(other_algorithms) ->
         {name,
             sc(
                 hoconsc:enum([plain, md5, sha, sha256, sha512]),
-                #{
-                    desc =>
-                        "Simple password hashing algorithm."
-                }
+                #{required => true, desc => "Simple password hashing algorithm."}
             )},
         {salt_position, fun salt_position/1}
     ].
@@ -115,11 +122,6 @@ salt_position(default) -> prefix;
 salt_position(desc) -> "Salt position for PLAIN, MD5, SHA, SHA256 and SHA512 algorithms.";
 salt_position(_) -> undefined.
 
-salt_rounds(type) -> integer();
-salt_rounds(default) -> 10;
-salt_rounds(desc) -> "Salt rounds for BCRYPT password generation.";
-salt_rounds(_) -> undefined.
-
 dk_length(type) ->
     integer();
 dk_length(required) ->
@@ -130,6 +132,7 @@ dk_length(desc) ->
 dk_length(_) ->
     undefined.
 
+%% for simple_authn/emqx_authn_mnesia
 type_rw(type) ->
     hoconsc:union(rw_refs());
 type_rw(default) ->
@@ -139,6 +142,7 @@ type_rw(desc) ->
 type_rw(_) ->
     undefined.
 
+%% for other authn resources
 type_ro(type) ->
     hoconsc:union(ro_refs());
 type_ro(default) ->

apps/emqx_authn/src/simple_authn/emqx_authn_http.erl

@@ -32,6 +32,11 @@
     validations/0
 ]).
 
+-export([
+    headers_no_content_type/1,
+    headers/1
+]).
+
 -export([
     refs/0,
     create/2,
@@ -57,12 +62,12 @@ roots() ->
 
 fields(get) ->
     [
-        {method, #{type => get, default => post, desc => "HTTP method."}},
+        {method, #{type => get, required => true, default => post, desc => "HTTP method."}},
         {headers, fun headers_no_content_type/1}
     ] ++ common_fields();
 fields(post) ->
     [
-        {method, #{type => post, default => post, desc => "HTTP method."}},
+        {method, #{type => post, required => true, default => post, desc => "HTTP method."}},
         {headers, fun headers/1}
     ] ++ common_fields().
 
@@ -75,7 +80,7 @@ desc(_) ->
 
 common_fields() ->
     [
-        {mechanism, emqx_authn_schema:mechanism('password_based')},
+        {mechanism, emqx_authn_schema:mechanism(password_based)},
         {backend, emqx_authn_schema:backend(http)},
         {url, fun url/1},
         {body,

apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl

@@ -55,20 +55,22 @@ roots() ->
 
 fields('hmac-based') ->
     [
-        {use_jwks, sc(hoconsc:enum([false]), #{desc => ""})},
-        {algorithm, sc(hoconsc:enum(['hmac-based']), #{desc => "Signing algorithm."})},
+        {use_jwks, sc(hoconsc:enum([false]), #{required => true, desc => ""})},
+        {algorithm,
+            sc(hoconsc:enum(['hmac-based']), #{required => true, desc => "Signing algorithm."})},
         {secret, fun secret/1},
         {secret_base64_encoded, fun secret_base64_encoded/1}
     ] ++ common_fields();
 fields('public-key') ->
     [
-        {use_jwks, sc(hoconsc:enum([false]), #{desc => ""})},
-        {algorithm, sc(hoconsc:enum(['public-key']), #{desc => "Signing algorithm."})},
+        {use_jwks, sc(hoconsc:enum([false]), #{required => true, desc => ""})},
+        {algorithm,
+            sc(hoconsc:enum(['public-key']), #{required => true, desc => "Signing algorithm."})},
         {certificate, fun certificate/1}
     ] ++ common_fields();
 fields('jwks') ->
     [
-        {use_jwks, sc(hoconsc:enum([true]), #{desc => ""})},
+        {use_jwks, sc(hoconsc:enum([true]), #{required => true, desc => ""})},
         {endpoint, fun endpoint/1},
         {pool_size, fun pool_size/1},
         {refresh_interval, fun refresh_interval/1},
@@ -78,7 +80,8 @@ fields('jwks') ->
                 hoconsc:ref(?MODULE, ssl_disable)
             ]),
             desc => "Enable/disable SSL.",
-            default => #{<<"enable">> => false}
+            default => #{<<"enable">> => false},
+            required => false
         }}
     ] ++ common_fields();
 fields(ssl_enable) ->
@@ -114,6 +117,7 @@ common_fields() ->
 
 secret(type) -> binary();
 secret(desc) -> "The key to verify the JWT Token using HMAC algorithm.";
+secret(required) -> true;
 secret(_) -> undefined.
 
 secret_base64_encoded(type) -> boolean();
@@ -123,10 +127,12 @@ secret_base64_encoded(_) -> undefined.
 
 certificate(type) -> string();
 certificate(desc) -> "The certificate used for signing the token.";
+certificate(required) -> ture;
 certificate(_) -> undefined.
 
 endpoint(type) -> string();
 endpoint(desc) -> "JWKs endpoint.";
+endpoint(required) -> true;
 endpoint(_) -> undefined.
 
 refresh_interval(type) -> integer();
@@ -168,6 +174,8 @@ verify_claims(converter) ->
     fun(VerifyClaims) ->
         [{to_binary(K), V} || {K, V} <- maps:to_list(VerifyClaims)]
     end;
+verify_claims(required) ->
+    false;
 verify_claims(_) ->
     undefined.
 

apps/emqx_authn/src/simple_authn/emqx_authn_mnesia.erl

@@ -103,8 +103,8 @@ roots() -> [?CONF_NS].
 
 fields(?CONF_NS) ->
     [
-        {mechanism, emqx_authn_schema:mechanism('password_based')},
-        {backend, emqx_authn_schema:backend('built_in_database')},
+        {mechanism, emqx_authn_schema:mechanism(password_based)},
+        {backend, emqx_authn_schema:backend(built_in_database)},
         {user_id_type, fun user_id_type/1},
         {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1}
     ] ++ emqx_authn_schema:common_fields().
@@ -117,6 +117,7 @@ desc(_) ->
 user_id_type(type) -> user_id_type();
 user_id_type(desc) -> "Authenticate by client ID or username.";
 user_id_type(default) -> <<"username">>;
+user_id_type(required) -> true;
 user_id_type(_) -> undefined.
 
 %%------------------------------------------------------------------------------

apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl

@@ -83,6 +83,7 @@ common_fields() ->
 
 collection(type) -> binary();
 collection(desc) -> "Collection used to store authentication data.";
+collection(required) -> true;
 collection(_) -> undefined.
 
 selector(type) ->
@@ -97,6 +98,7 @@ selector(_) ->
 
 password_hash_field(type) -> binary();
 password_hash_field(desc) -> "Document field that contains password hash.";
+password_hash_field(required) -> false;
 password_hash_field(_) -> undefined.
 
 salt_field(type) -> binary();

apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl

@@ -48,7 +48,7 @@ roots() -> [?CONF_NS].
 
 fields(?CONF_NS) ->
     [
-        {mechanism, emqx_authn_schema:mechanism('password_based')},
+        {mechanism, emqx_authn_schema:mechanism(password_based)},
         {backend, emqx_authn_schema:backend(mysql)},
         {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1},
         {query, fun query/1},
@@ -63,6 +63,7 @@ desc(_) ->
 
 query(type) -> string();
 query(desc) -> "SQL query used to lookup client data.";
+query(required) -> true;
 query(_) -> undefined.
 
 query_timeout(type) -> emqx_schema:duration_ms();

apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl

@@ -54,7 +54,7 @@ roots() -> [?CONF_NS].
 
 fields(?CONF_NS) ->
     [
-        {mechanism, emqx_authn_schema:mechanism('password_based')},
+        {mechanism, emqx_authn_schema:mechanism(password_based)},
         {backend, emqx_authn_schema:backend(postgresql)},
         {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1},
         {query, fun query/1}
@@ -69,6 +69,7 @@ desc(_) ->
 
 query(type) -> string();
 query(desc) -> "`SQL` query for looking up authentication data.";
+query(required) -> true;
 query(_) -> undefined.
 
 %%------------------------------------------------------------------------------

apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl

@@ -79,6 +79,7 @@ common_fields() ->
 
 cmd(type) -> string();
 cmd(desc) -> "Redis query.";
+cmd(required) -> true;
 cmd(_) -> undefined.
 
 %%------------------------------------------------------------------------------

apps/emqx_authz/include/emqx_authz.hrl

@@ -14,6 +14,8 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
+-include_lib("emqx/include/emqx_access_control.hrl").
+
 -define(APP, emqx_authz).
 
 -define(ALLOW_DENY(A),
@@ -45,6 +47,11 @@
 
 -define(RE_PLACEHOLDER, "\\$\\{[a-z0-9_]+\\}").
 
+%% has to be the same as the root field name defined in emqx_schema
+-define(CONF_NS, ?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME).
+-define(CONF_NS_ATOM, ?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_ATOM).
+-define(CONF_NS_BINARY, ?EMQX_AUTHORIZATION_CONFIG_ROOT_NAME_BINARY).
+
 %% API examples
 -define(USERNAME_RULES_EXAMPLE, #{
     username => user1,

apps/emqx_authz/src/emqx_authz_api_mnesia.erl

@@ -383,7 +383,7 @@ fields(rules_for_username) ->
 fields(username_response_data) ->
     [
         {data, mk(array(ref(rules_for_username)), #{})},
-        {meta, ref(meta)}
+        {meta, ref(emqx_dashboard_swagger, meta)}
     ];
 fields(rules_for_clientid) ->
     fields(rules) ++
@@ -391,14 +391,10 @@ fields(rules_for_clientid) ->
 fields(clientid_response_data) ->
     [
         {data, mk(array(ref(rules_for_clientid)), #{})},
-        {meta, ref(meta)}
+        {meta, ref(emqx_dashboard_swagger, meta)}
     ];
 fields(rules) ->
-    [{rules, mk(array(ref(rule_item)))}];
-fields(meta) ->
-    emqx_dashboard_swagger:fields(page) ++
-        emqx_dashboard_swagger:fields(limit) ++
-        [{count, mk(integer(), #{example => 1})}].
+    [{rules, mk(array(ref(rule_item)))}].
 
 %%--------------------------------------------------------------------
 %% HTTP API

apps/emqx_authz/src/emqx_authz_api_schema.erl

@@ -16,37 +16,34 @@
 
 -module(emqx_authz_api_schema).
 
+-include("emqx_authz.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx_connector/include/emqx_connector.hrl").
 
 -import(hoconsc, [mk/2, enum/1]).
 -import(emqx_schema, [mk_duration/2]).
 
--export([fields/1, authz_sources_types/1]).
+-export([
+    fields/1,
+    authz_sources_types/1
+]).
 
-fields(http) ->
-    authz_common_fields(http) ++
-        [
-            {url, fun url/1},
-            {method, #{
-                type => enum([get, post]),
-                default => get
-            }},
-            {headers, fun headers/1},
-            {body, map([{fuzzy, term(), binary()}])},
-            {request_timeout, mk_duration("Request timeout", #{default => "30s"})}
-        ] ++
-        maps:to_list(
-            maps:without(
-                [
-                    base_url,
-                    pool_type
-                ],
-                maps:from_list(emqx_connector_http:fields(config))
-            )
-        );
-fields('built_in_database') ->
-    authz_common_fields('built_in_database');
+%%------------------------------------------------------------------------------
+%% Hocon Schema
+%%------------------------------------------------------------------------------
+
+fields(http_get) ->
+    [
+        {method, #{type => get, default => get, required => true}},
+        {headers, fun headers_no_content_type/1}
+    ] ++ authz_http_common_fields();
+fields(http_post) ->
+    [
+        {method, #{type => post, default => post, required => true}},
+        {headers, fun headers/1}
+    ] ++ authz_http_common_fields();
+fields(built_in_database) ->
+    authz_common_fields(built_in_database);
 fields(mongo_single) ->
     authz_mongo_common_fields() ++
         emqx_connector_mongo:fields(single);
@@ -58,11 +55,11 @@ fields(mongo_sharded) ->
         emqx_connector_mongo:fields(sharded);
 fields(mysql) ->
     authz_common_fields(mysql) ++
-        [{query, #{type => binary()}}] ++
+        [{query, mk(binary(), #{required => true})}] ++
         emqx_connector_mysql:fields(config);
 fields(postgresql) ->
     authz_common_fields(postgresql) ++
-        [{query, #{type => binary()}}] ++
+        [{query, mk(binary(), #{required => true})}] ++
         proplists:delete(named_queries, emqx_connector_pgsql:fields(config));
 fields(redis_single) ->
     authz_redis_common_fields() ++
@@ -100,6 +97,23 @@ fields(position) ->
 %%------------------------------------------------------------------------------
 %% http type funcs
 
+authz_http_common_fields() ->
+    authz_common_fields(http) ++
+        [
+            {url, fun url/1},
+            {body, map([{fuzzy, term(), binary()}])},
+            {request_timeout, mk_duration("Request timeout", #{default => "30s"})}
+        ] ++
+        maps:to_list(
+            maps:without(
+                [
+                    base_url,
+                    pool_type
+                ],
+                maps:from_list(emqx_connector_http:fields(config))
+            )
+        ).
+
 url(type) -> binary();
 url(validator) -> [?NOT_EMPTY("the value of the field 'url' cannot be empty")];
 url(required) -> true;
@@ -107,6 +121,8 @@ url(_) -> undefined.
 
 headers(type) ->
     map();
+headers(desc) ->
+    "List of HTTP headers.";
 headers(converter) ->
     fun(Headers) ->
         maps:merge(default_headers(), transform_header_name(Headers))
@@ -116,6 +132,19 @@ headers(default) ->
 headers(_) ->
     undefined.
 
+headers_no_content_type(type) ->
+    map();
+headers_no_content_type(desc) ->
+    "List of HTTP headers.";
+headers_no_content_type(converter) ->
+    fun(Headers) ->
+        maps:merge(default_headers_no_content_type(), transform_header_name(Headers))
+    end;
+headers_no_content_type(default) ->
+    default_headers_no_content_type();
+headers_no_content_type(_) ->
+    undefined.
+
 %% headers
 default_headers() ->
     maps:put(
@@ -153,10 +182,19 @@ authz_mongo_common_fields() ->
         ].
 
 collection(type) -> binary();
+collection(desc) -> "Collection used to store authentication data.";
+collection(required) -> true;
 collection(_) -> undefined.
 
-selector(type) -> map();
-selector(_) -> undefined.
+selector(type) ->
+    map();
+selector(desc) ->
+    "Statement that is executed during the authentication process. "
+    "Commands can support following wildcards:\n"
+    " - `${username}`: substituted with client's username\n"
+    " - `${clientid}`: substituted with the clientid";
+selector(_) ->
+    undefined.
 
 %%------------------------------------------------------------------------------
 %% Redis type funcs
@@ -164,10 +202,11 @@ selector(_) -> undefined.
 authz_redis_common_fields() ->
     authz_common_fields(redis) ++
         [
-            {cmd, #{
-                type => binary(),
-                example => <<"HGETALL mqtt_authz">>
-            }}
+            {cmd,
+                mk(binary(), #{
+                    required => true,
+                    example => <<"HGETALL mqtt_authz">>
+                })}
         ].
 
 %%------------------------------------------------------------------------------
@@ -179,6 +218,7 @@ authz_common_fields(Type) when is_atom(Type) ->
         {type, #{
             type => enum([Type]),
             default => Type,
+            required => true,
             in => body
         }}
     ].
@@ -194,9 +234,11 @@ enable(_) -> undefined.
 authz_sources_types(Type) ->
     case Type of
         simple ->
-            [mongodb, redis];
+            [http, mongodb, redis];
         detailed ->
             [
+                http_get,
+                http_post,
                 mongo_single,
                 mongo_rs,
                 mongo_sharded,
@@ -206,8 +248,7 @@ authz_sources_types(Type) ->
             ]
     end ++
         [
-            http,
-            'built_in_database',
+            built_in_database,
             mysql,
             postgresql,
             file

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -16,6 +16,7 @@
 
 -module(emqx_authz_schema).
 
+-include("emqx_authz.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("emqx_connector/include/emqx_connector.hrl").
 
@@ -40,9 +41,6 @@
     headers/1
 ]).
 
--import(emqx_schema, [mk_duration/2]).
--include_lib("hocon/include/hoconsc.hrl").
-
 %%--------------------------------------------------------------------
 %% Hocon Schema
 %%--------------------------------------------------------------------
@@ -96,7 +94,7 @@ fields("authorization") ->
     ];
 fields(file) ->
     [
-        {type, #{type => file, desc => "Backend type."}},
+        {type, #{type => file, required => true, desc => "Backend type."}},
         {enable, #{
             type => boolean(),
             default => true,
@@ -118,17 +116,17 @@ fields(file) ->
     ];
 fields(http_get) ->
     [
-        {method, #{type => get, default => get, desc => "HTTP method."}},
+        {method, #{type => get, default => get, required => true, desc => "HTTP method."}},
         {headers, fun headers_no_content_type/1}
     ] ++ http_common_fields();
 fields(http_post) ->
     [
-        {method, #{type => post, default => post, desc => "HTTP method."}},
+        {method, #{type => post, default => post, required => true, desc => "HTTP method."}},
         {headers, fun headers/1}
     ] ++ http_common_fields();
 fields(mnesia) ->
     [
-        {type, #{type => 'built_in_database', desc => "Backend type."}},
+        {type, #{type => 'built_in_database', required => true, desc => "Backend type."}},
         {enable, #{
             type => boolean(),
             default => true,
@@ -147,7 +145,7 @@ fields(mysql) ->
 fields(postgresql) ->
     [
         {query, query()},
-        {type, #{type => postgresql, desc => "Backend type."}},
+        {type, #{type => postgresql, required => true, desc => "Backend type."}},
         {enable, #{
             type => boolean(),
             desc => "Enable this backend.",
@@ -197,7 +195,9 @@ http_common_fields() ->
     [
         {url, fun url/1},
         {request_timeout,
-            mk_duration("Request timeout", #{default => "30s", desc => "Request timeout."})},
+            emqx_schema:mk_duration("Request timeout", #{
+                default => "30s", desc => "Request timeout."
+            })},
         {body, #{type => map(), required => false, desc => "HTTP request body."}}
     ] ++
         maps:to_list(
@@ -213,10 +213,16 @@ http_common_fields() ->
 mongo_common_fields() ->
     [
         {collection, #{
-            type => atom(), desc => "`MongoDB` collection containing the authorization data."
+            type => atom(),
+            required => true,
+            desc => "`MongoDB` collection containing the authorization data."
+        }},
+        {selector, #{
+            type => map(),
+            required => true,
+            desc => "MQL query used to select the authorization record."
         }},
-        {selector, #{type => map(), desc => "MQL query used to select the authorization record."}},
-        {type, #{type => mongodb, desc => "Database backend."}},
+        {type, #{type => mongodb, required => true, desc => "Database backend."}},
         {enable, #{
             type => boolean(),
             default => true,
@@ -226,8 +232,7 @@ mongo_common_fields() ->
 
 validations() ->
     [
-        {check_ssl_opts, fun check_ssl_opts/1},
-        {check_headers, fun check_headers/1}
+        {check_ssl_opts, fun check_ssl_opts/1}
     ].
 
 headers(type) ->
@@ -253,6 +258,13 @@ headers_no_content_type(converter) ->
     end;
 headers_no_content_type(default) ->
     default_headers_no_content_type();
+headers_no_content_type(validator) ->
+    fun(Headers) ->
+        case lists:keyfind(<<"content-type">>, 1, Headers) of
+            false -> ok;
+            _ -> {error, do_not_include_content_type}
+        end
+    end;
 headers_no_content_type(_) ->
     undefined.
 
@@ -291,6 +303,7 @@ transform_header_name(Headers) ->
         Headers
     ).
 
+%% TODO: fix me, not work
 check_ssl_opts(Conf) ->
     case hocon_maps:get("config.url", Conf) of
         undefined ->
@@ -309,25 +322,6 @@ check_ssl_opts(Conf) ->
             end
     end.
 
-check_headers(Conf) ->
-    case hocon_maps:get("config.method", Conf) of
-        undefined ->
-            true;
-        Method0 ->
-            Method = to_bin(Method0),
-            Headers = hocon_maps:get("config.headers", Conf),
-            case Method of
-                <<"post">> ->
-                    true;
-                _ when Headers =:= undefined -> true;
-                _ when is_list(Headers) ->
-                    case lists:member(<<"content-type">>, Headers) of
-                        false -> true;
-                        true -> {Method0, do_not_include_content_type}
-                    end
-            end
-    end.
-
 union_array(Item) when is_list(Item) ->
     hoconsc:array(hoconsc:union(Item)).
 
@@ -335,6 +329,7 @@ query() ->
     #{
         type => binary(),
         desc => "Database query used to retrieve authorization data.",
+        required => true,
         validator => fun(S) ->
             case size(S) > 0 of
                 true -> ok;
@@ -369,10 +364,3 @@ to_list(A) when is_atom(A) ->
     atom_to_list(A);
 to_list(B) when is_binary(B) ->
     binary_to_list(B).
-
-to_bin(A) when is_atom(A) ->
-    atom_to_binary(A);
-to_bin(B) when is_binary(B) ->
-    B;
-to_bin(L) when is_list(L) ->
-    list_to_binary(L).

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -56,6 +56,7 @@ roots() ->
 fields(single) ->
     [ {mongo_type, #{type => single,
                      default => single,
+                     required => true,
                      desc => ?DESC("single_mongo_type")}}
     , {server, fun server/1}
     , {w_mode, fun w_mode/1}
@@ -63,6 +64,7 @@ fields(single) ->
 fields(rs) ->
     [ {mongo_type, #{type => rs,
                      default => rs,
+                     required => true,
                      desc => ?DESC("rs_mongo_type")}}
     , {servers, fun servers/1}
     , {w_mode, fun w_mode/1}
@@ -72,6 +74,7 @@ fields(rs) ->
 fields(sharded) ->
     [ {mongo_type, #{type => sharded,
                      default => sharded,
+                     required => true,
                      desc => ?DESC("sharded_mongo_type")}}
     , {servers, fun servers/1}
     , {w_mode, fun w_mode/1}
@@ -336,7 +339,7 @@ max_overflow(_) -> undefined.
 
 replica_set_name(type) -> binary();
 replica_set_name(desc) -> ?DESC("replica_set_name");
-replica_set_name(required) -> false;
+replica_set_name(required) -> true;
 replica_set_name(_) -> undefined.
 
 srv_record(type) -> boolean();

apps/emqx_connector/src/emqx_connector_redis.erl

@@ -57,6 +57,7 @@ fields(single) ->
     [ {server, fun server/1}
     , {redis_type, #{type => hoconsc:enum([single]),
                      default => single,
+                     required => true,
                      desc => ?DESC("single")
                     }}
     ] ++
@@ -66,6 +67,7 @@ fields(cluster) ->
     [ {servers, fun servers/1}
     , {redis_type, #{type => hoconsc:enum([cluster]),
                      default => cluster,
+                     required => true,
                      desc => ?DESC("cluster")
                     }}
     ] ++
@@ -75,6 +77,7 @@ fields(sentinel) ->
     [ {servers, fun servers/1}
     , {redis_type, #{type => hoconsc:enum([sentinel]),
                      default => sentinel,
+                     required => true,
                      desc => ?DESC("sentinel")
                     }}
     , {sentinel, #{type => string(), desc => ?DESC("sentinel_desc")
@@ -210,6 +213,7 @@ redis_fields() ->
     , {password, fun emqx_connector_schema_lib:password/1}
     , {database, #{type => integer(),
                    default => 0,
+                   required => true,
                    desc => ?DESC("database")
                   }}
     , {auto_reconnect, fun emqx_connector_schema_lib:auto_reconnect/1}

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -135,7 +135,12 @@ fields(limit) ->
         <<")">>
     ]),
     Meta = #{in => query, desc => Desc, default => ?DEFAULT_ROW, example => 50},
-    [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}].
+    [{limit, hoconsc:mk(range(1, ?MAX_ROW_LIMIT), Meta)}];
+fields(count) ->
+    Meta = #{desc => <<"Results count.">>, required => true},
+    [{count, hoconsc:mk(range(0, inf), Meta)}];
+fields(meta) ->
+    fields(page) ++ fields(limit) ++ fields(count).
 
 -spec schema_with_example(hocon_schema:type(), term()) -> hocon_schema:field_schema_map().
 schema_with_example(Type, Example) ->
@@ -574,7 +579,7 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
 typename_to_spec("user_id_type()", _Mod) ->
     #{type => string, enum => [clientid, username]};
 typename_to_spec("term()", _Mod) ->
-    #{type => string};
+    #{type => string, example => <<"any">>};
 typename_to_spec("boolean()", _Mod) ->
     #{type => boolean};
 typename_to_spec("binary()", _Mod) ->

apps/emqx_management/src/emqx_mgmt_api_alarms.erl

@@ -53,7 +53,7 @@ schema("/alarms") ->
             responses => #{
                 200 => [
                     {data, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, alarm)), #{})},
-                    {meta, hoconsc:mk(hoconsc:ref(?MODULE, meta), #{})}
+                    {meta, hoconsc:mk(hoconsc:ref(emqx_dashboard_swagger, meta), #{})}
                 ]
             }
         },
@@ -98,11 +98,8 @@ fields(alarm) ->
                 desc => ?DESC(deactivate_at),
                 example => <<"2021-10-31T10:52:52.548+08:00">>
             })}
-    ];
-fields(meta) ->
-    emqx_dashboard_swagger:fields(page) ++
-        emqx_dashboard_swagger:fields(limit) ++
-        [{count, hoconsc:mk(integer(), #{example => 1})}].
+    ].
+
 %%%==============================================================================================
 %% parameters trans
 alarms(get, #{query_string := QString}) ->

apps/emqx_management/src/emqx_mgmt_api_banned.erl

@@ -62,7 +62,7 @@ schema("/banned") ->
             responses => #{
                 200 => [
                     {data, hoconsc:mk(hoconsc:array(hoconsc:ref(ban)), #{})},
-                    {meta, hoconsc:mk(hoconsc:ref(meta), #{})}
+                    {meta, hoconsc:mk(hoconsc:ref(emqx_dashboard_swagger, meta), #{})}
                 ]
             }
         },
@@ -147,11 +147,7 @@ fields(ban) ->
                 required => false,
                 example => <<"2021-10-25T21:53:47+08:00">>
             })}
-    ];
-fields(meta) ->
-    emqx_dashboard_swagger:fields(page) ++
-        emqx_dashboard_swagger:fields(limit) ++
-        [{count, hoconsc:mk(integer(), #{example => 1})}].
+    ].
 
 banned(get, #{query_string := Params}) ->
     Response = emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN),

apps/emqx_management/src/emqx_mgmt_api_clients.erl

@@ -203,7 +203,7 @@ schema("/clients") ->
             responses => #{
                 200 => [
                     {data, hoconsc:mk(hoconsc:array(hoconsc:ref(?MODULE, client)), #{})},
-                    {meta, hoconsc:mk(hoconsc:ref(?MODULE, meta), #{})}
+                    {meta, hoconsc:mk(hoconsc:ref(emqx_dashboard_swagger, meta), #{})}
                 ],
                 400 =>
                     emqx_dashboard_swagger:error_codes(
@@ -518,11 +518,7 @@ fields(subscribe) ->
 fields(unsubscribe) ->
     [
         {topic, hoconsc:mk(binary(), #{desc => <<"Topic">>})}
-    ];
-fields(meta) ->
-    emqx_dashboard_swagger:fields(page) ++
-        emqx_dashboard_swagger:fields(limit) ++
-        [{count, hoconsc:mk(integer(), #{example => 1})}].
+    ].
 
 %%%==============================================================================================
 %% parameters trans

apps/emqx_management/src/emqx_mgmt_api_configs.erl

@@ -106,9 +106,11 @@ schema("/configs_reset/:rootname") ->
         post => #{
             tags => [conf],
             description =>
-                <<"Reset the config entry specified by the query string parameter `conf_path`.<br/>\n"
-                "- For a config entry that has default value, this resets it to the default value;\n"
-                "- For a config entry that has no default value, an error 400 will be returned">>,
+                <<
+                    "Reset the config entry specified by the query string parameter `conf_path`.<br/>\n"
+                    "- For a config entry that has default value, this resets it to the default value;\n"
+                    "- For a config entry that has no default value, an error 400 will be returned"
+                >>,
             %% We only return "200" rather than the new configs that has been changed, as
             %% the schema of the changed configs is depends on the request parameter
             %% `conf_path`, it cannot be defined here.

apps/emqx_management/src/emqx_mgmt_api_topics.erl

@@ -60,7 +60,7 @@ schema("/topics") ->
             responses => #{
                 200 => [
                     {data, hoconsc:mk(hoconsc:array(hoconsc:ref(topic)), #{})},
-                    {meta, hoconsc:mk(hoconsc:ref(meta), #{})}
+                    {meta, hoconsc:mk(hoconsc:ref(emqx_dashboard_swagger, meta), #{})}
                 ]
             }
         }
@@ -91,11 +91,7 @@ fields(topic) ->
                 desc => <<"Node">>,
                 required => true
             })}
-    ];
-fields(meta) ->
-    emqx_dashboard_swagger:fields(page) ++
-        emqx_dashboard_swagger:fields(limit) ++
-        [{count, hoconsc:mk(integer(), #{example => 1})}].
+    ].
 
 %%%==============================================================================================
 %% parameters trans

git-blame-ignore-revs

@@ -23,5 +23,6 @@ acb3544d4b112121b5d9414237d2af7860ccc2a3
 f1acfece6b79ed69b491da03783a7adaa7627b96
 # reformat apps/emqx_management
 aa7807baebfa5d8678025e43f386bcd9b3259d6a
+bf54f571fb8b27e76ada4ca75137d96ce4211d60
 # reformat apps/emqx_slow_subs
 83511f8a4c1570a2c89d9c6c5b6f462520199ed8

scripts/check-format.sh

@@ -13,6 +13,7 @@ APPS+=( 'apps/emqx_authn' 'apps/emqx_authz' )
 APPS+=( 'lib-ee/emqx_enterprise_conf' 'lib-ee/emqx_license' )
 APPS+=( 'apps/emqx_exhook')
 APPS+=( 'apps/emqx_retainer' 'apps/emqx_slow_subs')
+APPS+=( 'apps/emqx_management')
 
 for app in "${APPS[@]}"; do
     echo "$app ..."