Mgmt http api banned (#5998)

* fix(swagger): don't transform [log,publish] to a list.pwd

* chore: replace banned-api by hocon schema

* fix(api): code style warning

apps/emqx/src/emqx_banned.erl

@@ -37,6 +37,7 @@
         , info/1
         , format/1
         , parse/1
+        , to_timestamp/1
         ]).
 
 %% gen_server callbacks
@@ -108,8 +109,8 @@ parse(Params) ->
     Who    = pares_who(Params),
     By     = maps:get(<<"by">>, Params, <<"mgmt_api">>),
     Reason = maps:get(<<"reason">>, Params, <<"">>),
-    At     = pares_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)),
-    Until  = pares_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60),
+    At     = parse_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)),
+    Until  = parse_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60),
     #banned{
         who    = Who,
         by     = By,
@@ -120,15 +121,15 @@ parse(Params) ->
 
 pares_who(#{as := As, who := Who}) ->
     pares_who(#{<<"as">> => As, <<"who">> => Who});
-pares_who(#{<<"as">> := <<"peerhost">>, <<"who">> := Peerhost0}) ->
+pares_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) ->
     {ok, Peerhost} = inet:parse_address(binary_to_list(Peerhost0)),
     {peerhost, Peerhost};
 pares_who(#{<<"as">> := As, <<"who">> := Who}) ->
-    {binary_to_atom(As, utf8), Who}.
+    {As, Who}.
 
-pares_time(undefined, Default) ->
+parse_time(undefined, Default) ->
     Default;
-pares_time(Rfc3339, _Default) ->
+parse_time(Rfc3339, _Default) ->
     to_timestamp(Rfc3339).
 
 maybe_format_host({peerhost, Host}) ->

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -22,11 +22,14 @@
 -define(DEFAULT_FIELDS, [example, allowReserved, style,
     explode, maxLength, allowEmptyValue, deprecated, minimum, maximum]).
 
--define(INIT_SCHEMA, #{fields => #{}, translations => #{}, validations => [], namespace => undefined}).
+-define(INIT_SCHEMA, #{fields => #{}, translations => #{},
+                       validations => [], namespace => undefined}).
 
 -define(TO_REF(_N_, _F_), iolist_to_binary([to_bin(_N_), ".", to_bin(_F_)])).
--define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>, ?TO_REF(namespace(_M_), _F_)])).
--define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>, ?TO_REF(namespace(_M_), _F_)])).
+-define(TO_COMPONENTS_SCHEMA(_M_, _F_), iolist_to_binary([<<"#/components/schemas/">>,
+                                                         ?TO_REF(namespace(_M_), _F_)])).
+-define(TO_COMPONENTS_PARAM(_M_, _F_), iolist_to_binary([<<"#/components/parameters/">>,
+                                                         ?TO_REF(namespace(_M_), _F_)])).
 
 -define(MAX_ROW_LIMIT, 100).
 
@@ -116,9 +119,9 @@ translate_req(Request, #{module := Module, path := Path, method := Method}, Chec
     #{Method := Spec} = apply(Module, schema, [Path]),
     try
         Params = maps:get(parameters, Spec, []),
-        Body = maps:get(requestBody, Spec, []),
+        Body = maps:get('requestBody', Spec, []),
         {Bindings, QueryStr} = check_parameters(Request, Params, Module),
-        NewBody = check_requestBody(Request, Body, Module, CheckFun, hoconsc:is_schema(Body)),
+        NewBody = check_request_body(Request, Body, Module, CheckFun, hoconsc:is_schema(Body)),
         {ok, Request#{bindings => Bindings, query_string => QueryStr, body => NewBody}}
     catch throw:Error ->
         {_, [{validation_error, ValidErr}]} = Error,
@@ -155,34 +158,41 @@ parse_spec_ref(Module, Path) ->
         {Spec, SubRefs} = meta_to_spec(Meta, Module),
         {Acc#{Method => Spec}, SubRefs ++ RefsAcc}
                               end, {#{}, []},
-        maps:without([operationId], Schema)),
-    {maps:get(operationId, Schema), Specs, Refs}.
+        maps:without(['operationId'], Schema)),
+    {maps:get('operationId', Schema), Specs, Refs}.
 
 check_parameters(Request, Spec, Module) ->
     #{bindings := Bindings, query_string := QueryStr} = Request,
-    BindingsBin = maps:fold(fun(Key, Value, Acc) -> Acc#{atom_to_binary(Key) => Value} end, #{}, Bindings),
+    BindingsBin = maps:fold(fun(Key, Value, Acc) ->
+        Acc#{atom_to_binary(Key) => Value}
+                            end, #{}, Bindings),
     check_parameter(Spec, BindingsBin, QueryStr, Module, #{}, #{}).
 
 check_parameter([?REF(Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
-    check_parameter([?R_REF(LocalMod, Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
-check_parameter([?R_REF(Module, Fields) | Spec], Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
+    check_parameter([?R_REF(LocalMod, Fields) | Spec],
+        Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
+check_parameter([?R_REF(Module, Fields) | Spec],
+    Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc) ->
     Params = apply(Module, fields, [Fields]),
     check_parameter(Params ++ Spec, Bindings, QueryStr, LocalMod, BindingsAcc, QueryStrAcc);
-check_parameter([], _Bindings, _QueryStr, _Module, NewBindings, NewQueryStr) -> {NewBindings, NewQueryStr};
+check_parameter([], _Bindings, _QueryStr, _Module, NewBindings, NewQueryStr) ->
+    {NewBindings, NewQueryStr};
 check_parameter([{Name, Type} | Spec], Bindings, QueryStr, Module, BindingsAcc, QueryStrAcc) ->
     Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
     case hocon_schema:field_schema(Type, in) of
         path ->
-            NewBindings = hocon_schema:check_plain(Schema, Bindings, #{atom_key => true, override_env => false}),
+            Option = #{atom_key => true, override_env => false},
+            NewBindings = hocon_schema:check_plain(Schema, Bindings, Option),
             NewBindingsAcc = maps:merge(BindingsAcc, NewBindings),
             check_parameter(Spec, Bindings, QueryStr, Module, NewBindingsAcc, QueryStrAcc);
         query ->
-            NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, #{override_env => false}),
+            Option = #{override_env => false},
+            NewQueryStr = hocon_schema:check_plain(Schema, QueryStr, Option),
             NewQueryStrAcc = maps:merge(QueryStrAcc, NewQueryStr),
             check_parameter(Spec, Bindings, QueryStr, Module,BindingsAcc, NewQueryStrAcc)
     end.
 
-check_requestBody(#{body := Body}, Schema, Module, CheckFun, true) ->
+check_request_body(#{body := Body}, Schema, Module, CheckFun, true) ->
     Type0 = hocon_schema:field_schema(Schema, type),
     Type =
         case Type0 of
@@ -190,16 +200,17 @@ check_requestBody(#{body := Body}, Schema, Module, CheckFun, true) ->
             _ -> Type0
         end,
     NewSchema = ?INIT_SCHEMA#{roots => [{root, Type}]},
-    #{<<"root">> := NewBody} = CheckFun(NewSchema, #{<<"root">> => Body}, #{override_env => false}),
+    Option = #{override_env => false},
+    #{<<"root">> := NewBody} = CheckFun(NewSchema, #{<<"root">> => Body}, Option),
     NewBody;
 %% TODO not support nest object check yet, please use ref!
-%% RequestBody = [ {per_page, mk(integer(), #{}},
+%% 'requestBody' = [ {per_page, mk(integer(), #{}},
 %%                 {nest_object, [
 %%                   {good_nest_1, mk(integer(), #{})},
 %%                   {good_nest_2, mk(ref(?MODULE, good_ref), #{})}
 %%                ]}
 %% ]
-check_requestBody(#{body := Body}, Spec, _Module, CheckFun, false) ->
+check_request_body(#{body := Body}, Spec, _Module, CheckFun, false) ->
     lists:foldl(fun({Name, Type}, Acc) ->
         Schema = ?INIT_SCHEMA#{roots => [{Name, Type}]},
         maps:merge(Acc, CheckFun(Schema, Body, #{}))
@@ -208,7 +219,7 @@ check_requestBody(#{body := Body}, Spec, _Module, CheckFun, false) ->
 %% tags, description, summary, security, deprecated
 meta_to_spec(Meta, Module) ->
     {Params, Refs1} = parameters(maps:get(parameters, Meta, []), Module),
-    {RequestBody, Refs2} = requestBody(maps:get(requestBody, Meta, []), Module),
+    {RequestBody, Refs2} = request_body(maps:get('requestBody', Meta, []), Module),
     {Responses, Refs3} = responses(maps:get(responses, Meta, #{}), Module),
     {
         to_spec(Meta, Params, RequestBody, Responses),
@@ -216,25 +227,22 @@ meta_to_spec(Meta, Module) ->
     }.
 
 to_spec(Meta, Params, [], Responses) ->
-    Spec = maps:without([parameters, requestBody, responses], Meta),
+    Spec = maps:without([parameters, 'requestBody', responses], Meta),
     Spec#{parameters => Params, responses => Responses};
 to_spec(Meta, Params, RequestBody, Responses) ->
     Spec = to_spec(Meta, Params, [], Responses),
-    maps:put(requestBody, RequestBody, Spec).
+    maps:put('requestBody', RequestBody, Spec).
 
 parameters(Params, Module) ->
     {SpecList, AllRefs} =
         lists:foldl(fun(Param, {Acc, RefsAcc}) ->
             case Param of
-                ?REF(StructName) ->
-                    {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(Module, StructName)} | Acc],
-                        [{Module, StructName, parameter} | RefsAcc]};
-                ?R_REF(RModule, StructName) ->
-                    {[#{<<"$ref">> => ?TO_COMPONENTS_PARAM(RModule, StructName)} | Acc],
-                        [{RModule, StructName, parameter} | RefsAcc]};
+                ?REF(StructName) -> to_ref(Module, StructName, Acc, RefsAcc);
+                ?R_REF(RModule, StructName) -> to_ref(RModule, StructName, Acc, RefsAcc);
                 {Name, Type} ->
                     In = hocon_schema:field_schema(Type, in),
-                    In =:= undefined andalso throw({error, <<"missing in:path/query field in parameters">>}),
+                    In =:= undefined andalso
+                        throw({error, <<"missing in:path/query field in parameters">>}),
                     Nullable = hocon_schema:field_schema(Type, nullable),
                     Default = hocon_schema:field_schema(Type, default),
                     HoconType = hocon_schema:field_schema(Type, type),
@@ -278,8 +286,8 @@ trans_desc(Spec, Hocon) ->
         Desc -> Spec#{description => to_bin(Desc)}
     end.
 
-requestBody([], _Module) -> {[], []};
-requestBody(Schema, Module) ->
+request_body([], _Module) -> {[], []};
+request_body(Schema, Module) ->
     {{Props, Refs}, Examples} =
         case hoconsc:is_schema(Schema) of
             true ->
@@ -311,7 +319,10 @@ response(Status, Schema, {Acc, RefsAcc, Module}) ->
             {Spec, Refs} = hocon_schema_to_spec(Hocon, Module),
             Init = trans_desc(#{}, Schema),
             Content = content(Spec, Examples),
-            {Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}}, Refs ++ RefsAcc, Module};
+            {
+                Acc#{integer_to_binary(Status) => Init#{<<"content">> => Content}},
+                    Refs ++ RefsAcc, Module
+            };
         false ->
             {Props, Refs} = parse_object(Schema, Module),
             Content = #{<<"content">> => content(Props)},
@@ -401,11 +412,16 @@ typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, examp
 typename_to_spec("bytesize()", _Mod) -> #{type => string, example => <<"32MB">>};
 typename_to_spec("wordsize()", _Mod) -> #{type => string, example => <<"1024KB">>};
 typename_to_spec("map()", _Mod) -> #{type => object, example => #{}};
-typename_to_spec("comma_separated_list()", _Mod) -> #{type => string, example => <<"item1,item2">>};
-typename_to_spec("comma_separated_atoms()", _Mod) -> #{type => string, example => <<"item1,item2">>};
-typename_to_spec("pool_type()", _Mod) -> #{type => string, enum => [random, hash], example => hash};
+typename_to_spec("comma_separated_list()", _Mod) ->
+    #{type => string, example => <<"item1,item2">>};
+typename_to_spec("comma_separated_atoms()", _Mod) ->
+    #{type => string, example => <<"item1,item2">>};
+typename_to_spec("pool_type()", _Mod) ->
+    #{type => string, enum => [random, hash], example => hash};
 typename_to_spec("log_level()", _Mod) ->
-    #{type => string, enum => [debug, info, notice, warning, error, critical, alert, emergency, all]};
+    #{ type => string,
+       enum => [debug, info, notice, warning, error, critical, alert, emergency, all]
+    };
 typename_to_spec("rate()", _Mod) ->
     #{type => string, example => <<"10M/s">>};
 typename_to_spec("bucket_rate()", _Mod) ->
@@ -465,9 +481,12 @@ add_integer_prop(Schema, Key, Value) ->
         {Int, []} -> Schema#{Key => Int}
     end.
 
-to_bin([Atom | _] = List) when is_atom(Atom) -> iolist_to_binary(io_lib:format("~p", [List]));
-to_bin(List) when is_list(List) -> unicode:characters_to_binary(List);
-to_bin(B) when is_boolean(B) -> B;
+to_bin(List) when is_list(List) ->
+    case io_lib:printable_list(List) of
+        true -> unicode:characters_to_binary(List);
+        false -> List
+    end;
+to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
 to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
 to_bin(X) -> X.
 
@@ -513,3 +532,7 @@ content(ApiSpec, undefined) ->
     #{<<"application/json">> => #{<<"schema">> => ApiSpec}};
 content(ApiSpec, Examples) when is_map(Examples) ->
     #{<<"application/json">> => Examples#{<<"schema">> => ApiSpec}}.
+
+to_ref(Mod, StructName, Acc, RefsAcc) ->
+    Ref = #{<<"$ref">> => ?TO_COMPONENTS_PARAM(Mod, StructName)},
+    {[Ref | Acc], [{Mod, StructName, parameter} | RefsAcc]}.

apps/emqx_management/src/emqx_mgmt_api_banned.erl

@@ -17,87 +17,119 @@
 -module(emqx_mgmt_api_banned).
 
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("typerefl/include/types.hrl").
 
 -include("emqx_mgmt.hrl").
 
 -behaviour(minirest_api).
 
--export([api_spec/0]).
+-export([api_spec/0, paths/0, schema/1, fields/1]).
+-export([format/1]).
 
 -export([ banned/2
         , delete_banned/2
         ]).
 
--import(emqx_mgmt_util, [ page_params/0
-                        , schema/1
-                        , object_schema/1
-                        , page_object_schema/1
-                        , properties/1
-                        , error_schema/1
-                        ]).
+-define(TAB, emqx_banned).
 
--export([format/1]).
+-define(BANNED_TYPES, [clientid, username, peerhost]).
 
--define(TAB, emqx_banned).
 -define(FORMAT_FUN, {?MODULE, format}).
 
-
 api_spec() ->
-    {[banned_api(), delete_banned_api()], []}.
+    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
 
--define(BANNED_TYPES, [clientid, username, peerhost]).
+paths() ->
+    ["/banned", "/banned/:as/:who"].
 
-properties() ->
-    properties([
-        {as, string, <<"Banned type clientid, username, peerhost">>, [clientid, username, peerhost]},
-        {who, string, <<"Client info as banned type">>},
-        {by, integer, <<"Commander">>},
-        {reason, string, <<"Banned reason">>},
-        {at, integer, <<"Create banned time. Nullable, rfc3339, default is now">>},
-        {until, string, <<"Cancel banned time. Nullable, rfc3339, default is now + 5 minute">>}
-    ]).
-
-banned_api() ->
-    Path = "/banned",
-    MetaData = #{
+schema("/banned") ->
+    #{
+        'operationId' =>  banned,
         get => #{
             description => <<"List banned">>,
-            parameters => page_params(),
+            parameters => [
+                hoconsc:ref(emqx_dashboard_swagger, page),
+                hoconsc:ref(emqx_dashboard_swagger, limit)
+            ],
             responses => #{
-                <<"200">> =>
-                    page_object_schema(properties())}},
+                200 =>[
+                    {data, hoconsc:mk(hoconsc:array(hoconsc:ref(ban)), #{})},
+                    {meta, hoconsc:mk(hoconsc:ref(meta), #{})}
+                ]
+            }
+        },
         post => #{
             description => <<"Create banned">>,
-            'requestBody' => object_schema(properties()),
+            'requestBody' => hoconsc:mk(hoconsc:ref(ban)),
             responses => #{
-                <<"200">> => schema(<<"Create success">>)}}},
-    {Path, MetaData, banned}.
-
-delete_banned_api() ->
-    Path = "/banned/:as/:who",
-    MetaData = #{
+                200 => <<"Create success">>
+            }
+        }
+    };
+schema("/banned/:as/:who") ->
+    #{
+        'operationId' => delete_banned,
         delete => #{
             description => <<"Delete banned">>,
             parameters => [
-                #{
-                    name => as,
+                {as, hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{
+                    desc => <<"Banned type">>,
                     in => path,
-                    required => true,
-                    description => <<"Banned type">>,
-                    schema => #{type => string, enum => ?BANNED_TYPES}
-                },
-                #{
-                    name => who,
+                    example => username})},
+                {who, hoconsc:mk(binary(), #{
+                    desc => <<"Client info as banned type">>,
                     in => path,
-                    required => true,
-                    description => <<"Client info as banned type">>,
-                    schema => #{type => string}
-                }
-            ],
+                    example => <<"Badass">>})}
+                ],
             responses => #{
-                <<"200">> => schema(<<"Delete banned success">>),
-                <<"404">> => error_schema(<<"Banned not found">>)}}},
-    {Path, MetaData, delete_banned}.
+                200 => <<"Delete banned success">>,
+                404 => emqx_dashboard_swagger:error_codes(['RESOURCE_NOT_FOUND'],
+                                                          <<"Banned not found">>)
+            }
+        }
+    }.
+
+fields(ban) ->
+    [
+        {as, hoconsc:mk(hoconsc:enum(?BANNED_TYPES), #{
+            desc => <<"Banned type clientid, username, peerhost">>,
+            nullable => false,
+            example => username})},
+        {who, hoconsc:mk(binary(), #{
+            desc => <<"Client info as banned type">>,
+            nullable => false,
+            example => <<"Badass">>})},
+        {by, hoconsc:mk(binary(), #{
+            desc => <<"Commander">>,
+            nullable => true,
+            example => <<"mgmt_api">>})},
+        {reason, hoconsc:mk(binary(), #{
+            desc => <<"Banned reason">>,
+            nullable => true,
+            example => <<"Too many requests">>})},
+        {at, hoconsc:mk(binary(), #{
+            desc => <<"Create banned time, rfc3339, now if not specified">>,
+            nullable => true,
+            validator => fun is_rfc3339/1,
+            example => <<"2021-10-25T21:48:47+08:00">>})},
+        {until, hoconsc:mk(binary(), #{
+            desc => <<"Cancel banned time, rfc3339, now + 5 minute if not specified">>,
+            nullable => true,
+            validator => fun is_rfc3339/1,
+            example => <<"2021-10-25T21:53:47+08:00">>})
+        }
+    ];
+fields(meta) ->
+    emqx_dashboard_swagger:fields(page) ++
+        emqx_dashboard_swagger:fields(limit) ++
+        [{count, hoconsc:mk(integer(), #{example => 1})}].
+
+is_rfc3339(Time) ->
+    try
+        emqx_banned:to_timestamp(Time),
+        ok
+    catch _:_ -> {error, Time}
+    end.
 
 banned(get, #{query_string := Params}) ->
     Response = emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN),
@@ -109,9 +141,8 @@ banned(post, #{body := Body}) ->
 delete_banned(delete, #{bindings := Params}) ->
     case emqx_banned:look_up(Params) of
         [] ->
-            As0 = maps:get(as, Params),
-            Who0 = maps:get(who, Params),
-            Message = list_to_binary(io_lib:format("~p: ~p not found", [As0, Who0])),
+            #{as := As0, who := Who0} = Params,
+            Message = list_to_binary(io_lib:format("~p: ~s not found", [As0, Who0])),
             {404, #{code => 'RESOURCE_NOT_FOUND', message => Message}};
         _ ->
             ok = emqx_banned:delete(Params),

rebar.config

@@ -54,7 +54,7 @@
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.11.1"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
-    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.5"}}}
+    , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.2.6"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.1"}}}
     , {replayq, "0.3.3"}
     , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}}