chore(authz mnesia api): get method supports paging

Signed-off-by: zhanghongtong <rory-z@outlook.com>

apps/emqx_authz/src/emqx_authz_api_mnesia.erl

@@ -118,7 +118,7 @@ definitions() ->
                      }
                    }
                  , #{type => object,
-                     required => [cleitnid, rules],
+                     required => [clientid, rules],
                      properties => #{
                         username => #{
                             type => string,
@@ -164,6 +164,20 @@ records_api() ->
                        enum => [<<"username">>, <<"clientid">>, <<"all">>]
                     },
                     required => true
+                },
+                #{
+                    name => page,
+                    in => query,
+                    required => false,
+                    description => <<"Page Index">>,
+                    schema => #{type => integer}
+                },
+                #{
+                    name => limit,
+                    in => query,
+                    required => false,
+                    description => <<"Page limit">>,
+                    schema => #{type => integer}
                 }
             ],
             responses => #{
@@ -391,28 +405,59 @@ purge(delete, _) ->
     [ ekka_mnesia:dirty_delete(?ACL_TABLE, K) || K <- mnesia:dirty_all_keys(?ACL_TABLE)],
     {204}.
 
-records(get, #{bindings := #{type := <<"username">>}}) ->
+records(get, #{bindings := #{type := <<"username">>},
+               query_string := Qs
+              }) ->
     MatchSpec = ets:fun2ms(
                   fun({?ACL_TABLE, {username, Username}, Rules}) ->
                           [{username, Username}, {rules, Rules}]
                   end),
-    {200, [ #{username => Username,
-              rules => [ #{topic => Topic,
-                           action => Action,
-                           permission => Permission
-                          } || {Permission, Action, Topic} <- Rules]
-             } || [{username, Username}, {rules, Rules}] <- ets:select(?ACL_TABLE, MatchSpec)]};
-records(get, #{bindings := #{type := <<"clientid">>}}) ->
+    Format = fun ([{username, Username}, {rules, Rules}]) ->
+                #{username => Username,
+                  rules => [ #{topic => Topic,
+                               action => Action,
+                               permission => Permission
+                              } || {Permission, Action, Topic} <- Rules]
+                 }
+             end,
+    case Qs of
+        #{<<"limit">> := _, <<"page">> := _} = Page ->
+            {200, emqx_mgmt_api:paginate(?ACL_TABLE, MatchSpec, Page, Format)};
+        #{<<"limit">> := Limit} ->
+            case ets:select(?ACL_TABLE, MatchSpec, binary_to_integer(Limit)) of
+                {Rows, _Continuation} -> {200, [Format(Row) || Row <- Rows ]};
+                '$end_of_table' -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}
+            end;
+        _ ->
+            {200, [Format(Row) || Row <- ets:select(?ACL_TABLE, MatchSpec)]}
+    end;
+
+records(get, #{bindings := #{type := <<"clientid">>},
+               query_string := Qs
+              }) ->
     MatchSpec = ets:fun2ms(
                   fun({?ACL_TABLE, {clientid, Clientid}, Rules}) ->
                           [{clientid, Clientid}, {rules, Rules}]
                   end),
-    {200, [ #{clientid => Clientid,
-              rules => [ #{topic => Topic,
-                           action => Action,
-                           permission => Permission
-                          } || {Permission, Action, Topic} <- Rules]
-             } || [{clientid, Clientid}, {rules, Rules}] <- ets:select(?ACL_TABLE, MatchSpec)]};
+    Format = fun ([{clientid, Clientid}, {rules, Rules}]) ->
+                #{clientid => Clientid,
+                  rules => [ #{topic => Topic,
+                               action => Action,
+                               permission => Permission
+                              } || {Permission, Action, Topic} <- Rules]
+                 }
+             end,
+    case Qs of
+        #{<<"limit">> := _, <<"page">> := _} = Page ->
+            {200, emqx_mgmt_api:paginate(?ACL_TABLE, MatchSpec, Page, Format)};
+        #{<<"limit">> := Limit} ->
+            case ets:select(?ACL_TABLE, MatchSpec, binary_to_integer(Limit)) of
+                {Rows, _Continuation} -> {200, [Format(Row) || Row <- Rows ]};
+                '$end_of_table' -> {404, #{code => <<"NOT_FOUND">>, message => <<"Not Found">>}}
+            end;
+        _ ->
+            {200, [Format(Row) || Row <- ets:select(?ACL_TABLE, MatchSpec)]}
+    end;
 records(get, #{bindings := #{type := <<"all">>}}) ->
     MatchSpec = ets:fun2ms(
                   fun({?ACL_TABLE, all, Rules}) ->

apps/emqx_authz/test/emqx_authz_api_mnesia_SUITE.erl

@@ -174,7 +174,17 @@ t_api(_) ->
     [#{<<"rules">> := Rules6}] = jsx:decode(Request8),
     ?assertEqual(0, length(Rules6)),
 
+    {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "username"]), [ #{username => N, rules => []} || N <- lists:seq(1, 20) ]),
+    {ok, 200, Request9} = request(get, uri(["authorization", "sources", "built-in-database", "username?page=2&limit=5"]), []),
+    #{<<"data">> := Data1} = jsx:decode(Request9),
+    ?assertEqual(5, length(Data1)),
+
+    {ok, 204, _} = request(post, uri(["authorization", "sources", "built-in-database", "clientid"]), [ #{clientid => N, rules => []} || N <- lists:seq(1, 20) ]),
+    {ok, 200, Request10} = request(get, uri(["authorization", "sources", "built-in-database", "clientid?limit=5"]), []),
+    ?assertEqual(5, length(jsx:decode(Request10))),
+
     {ok, 204, _} = request(delete, uri(["authorization", "sources", "built-in-database", "purge-all"]), []),
+    ?assertEqual([], mnesia:dirty_all_keys(?ACL_TABLE)),
 
     ok.
 

apps/emqx_management/src/emqx_mgmt_api.erl

@@ -18,7 +18,9 @@
 
 -include_lib("stdlib/include/qlc.hrl").
 
--export([paginate/3]).
+-export([ paginate/3
+        , paginate/4
+        ]).
 
 %% first_next query APIs
 -export([ params2qs/2
@@ -47,6 +49,23 @@ paginate(Tables, Params, RowFun) ->
     #{meta  => #{page => Page, limit => Limit, count => Count},
       data  => [RowFun(Row) || Row <- Rows]}.
 
+paginate(Tables, MatchSpec, Params, RowFun) ->
+    Qh = query_handle(Tables, MatchSpec),
+    Count = count(Tables, MatchSpec),
+    Page = b2i(page(Params)),
+    Limit = b2i(limit(Params)),
+    Cursor = qlc:cursor(Qh),
+    case Page > 1 of
+        true  ->
+            _ = qlc:next_answers(Cursor, (Page - 1) * Limit),
+            ok;
+        false -> ok
+    end,
+    Rows = qlc:next_answers(Cursor, Limit),
+    qlc:delete_cursor(Cursor),
+    #{meta  => #{page => Page, limit => Limit, count => Count},
+      data  => [RowFun(Row) || Row <- Rows]}.
+
 query_handle(Table) when is_atom(Table) ->
     qlc:q([R|| R <- ets:table(Table)]);
 query_handle([Table]) when is_atom(Table) ->
@@ -54,6 +73,16 @@ query_handle([Table]) when is_atom(Table) ->
 query_handle(Tables) ->
     qlc:append([qlc:q([E || E <- ets:table(T)]) || T <- Tables]).
 
+query_handle(Table, MatchSpec) when is_atom(Table) ->
+    Options = {traverse, {select, MatchSpec}},
+    qlc:q([R|| R <- ets:table(Table, Options)]);
+query_handle([Table], MatchSpec) when is_atom(Table) ->
+    Options = {traverse, {select, MatchSpec}},
+    qlc:q([R|| R <- ets:table(Table, Options)]);
+query_handle(Tables, MatchSpec) ->
+    Options = {traverse, {select, MatchSpec}},
+    qlc:append([qlc:q([E || E <- ets:table(T, Options)]) || T <- Tables]).
+
 count(Table) when is_atom(Table) ->
     ets:info(Table, size);
 count([Table]) when is_atom(Table) ->
@@ -61,8 +90,16 @@ count([Table]) when is_atom(Table) ->
 count(Tables) ->
     lists:sum([count(T) || T <- Tables]).
 
-count(Table, Nodes) ->
-    lists:sum([rpc_call(Node, ets, info, [Table, size], 5000) || Node <- Nodes]).
+count(Table, MatchSpec) when is_atom(Table) ->
+    [{MatchPattern, Where, _Re}] = MatchSpec,
+    NMatchSpec = [{MatchPattern, Where, [true]}],
+    ets:select_count(Table, NMatchSpec);
+count([Table], MatchSpec) when is_atom(Table) ->
+    [{MatchPattern, Where, _Re}] = MatchSpec,
+    NMatchSpec = [{MatchPattern, Where, [true]}],
+    ets:select_count(Table, NMatchSpec);
+count(Tables, MatchSpec) ->
+    lists:sum([count(T, MatchSpec) || T <- Tables]).
 
 page(Params) when is_map(Params) ->
     maps:get(<<"page">>, Params, 1);
@@ -122,7 +159,7 @@ cluster_query(Params, Tab, QsSchema, QueryFun) ->
     Rows = do_cluster_query(Nodes, Tab, Qs, QueryFun, Start, Limit+1, []),
     Meta = #{page => Page, limit => Limit},
     NMeta = case CodCnt =:= 0 of
-                true -> Meta#{count => count(Tab, Nodes)};
+                true -> Meta#{count => lists:sum([rpc_call(Node, ets, info, [Tab, size], 5000) || Node <- Nodes])};
                 _ -> Meta#{count => length(Rows)}
             end,
     #{meta => NMeta, data => lists:sublist(Rows, Limit)}.