Merge pull request #9152 from zhongwencool/trace-log-detail-api

get trace file's detail via /trace/:name/log_detail

CHANGES-5.0.md

@@ -8,6 +8,7 @@
 * No message(s) echo for the message publish APIs [#9155](https://github.com/emqx/emqx/pull/9155)
   Prior to this fix, the message publish APIs (`api/v5/publish` and `api/v5/publish/bulk`) echos the message back to the client in HTTP body.
   This change fixed it to only send back the message ID.
+* Add /trace/:name/log_detail HTTP API to return trace file's size and mtime [#9152](https://github.com/emqx/emqx/pull/9152)
 
 ## Bug fixes
 

apps/emqx/priv/bpapi.versions

@@ -20,6 +20,7 @@
 {emqx_mgmt_api_plugins,1}.
 {emqx_mgmt_cluster,1}.
 {emqx_mgmt_trace,1}.
+{emqx_mgmt_trace,2}.
 {emqx_persistent_session,1}.
 {emqx_plugin_libs,1}.
 {emqx_prometheus,1}.

apps/emqx/src/emqx_trace/emqx_trace.erl

@@ -19,6 +19,7 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("kernel/include/file.hrl").
 -include_lib("snabbkaffe/include/trace.hrl").
 
 -export([
@@ -46,6 +47,7 @@
     filename/2,
     trace_dir/0,
     trace_file/1,
+    trace_file_detail/1,
     delete_files_after_send/2
 ]).
 
@@ -193,6 +195,16 @@ trace_file(File) ->
         {error, Reason} -> {error, Node, Reason}
     end.
 
+trace_file_detail(File) ->
+    FileName = filename:join(trace_dir(), File),
+    Node = atom_to_binary(node()),
+    case file:read_file_info(FileName, [{'time', 'posix'}]) of
+        {ok, #file_info{size = Size, mtime = Mtime}} ->
+            {ok, #{size => Size, mtime => Mtime, node => Node}};
+        {error, Reason} ->
+            {error, #{reason => Reason, node => Node, file => File}}
+    end.
+
 delete_files_after_send(TraceLog, Zips) ->
     gen_server:cast(?MODULE, {delete_tag, self(), [TraceLog | Zips]}).
 

apps/emqx_management/src/emqx_mgmt_api_trace.erl

@@ -34,7 +34,8 @@
     delete_trace/2,
     update_trace/2,
     download_trace_log/2,
-    stream_log_file/2
+    stream_log_file/2,
+    log_file_detail/2
 ]).
 
 -export([validate_name/1]).
@@ -55,7 +56,14 @@ api_spec() ->
     emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true, translate_body => true}).
 
 paths() ->
-    ["/trace", "/trace/:name/stop", "/trace/:name/download", "/trace/:name/log", "/trace/:name"].
+    [
+        "/trace",
+        "/trace/:name/stop",
+        "/trace/:name/download",
+        "/trace/:name/log",
+        "/trace/:name/log_detail",
+        "/trace/:name"
+    ].
 
 schema("/trace") ->
     #{
@@ -95,7 +103,7 @@ schema("/trace/:name") ->
     #{
         'operationId' => delete_trace,
         delete => #{
-            description => "Delete trace by name",
+            description => "Delete specified trace",
             tags => ?TAGS,
             parameters => [hoconsc:ref(name)],
             responses => #{
@@ -136,6 +144,19 @@ schema("/trace/:name/download") ->
             }
         }
     };
+schema("/trace/:name/log_detail") ->
+    #{
+        'operationId' => log_file_detail,
+        get => #{
+            description => "get trace log file's metadata, such as size, last update time",
+            tags => ?TAGS,
+            parameters => [hoconsc:ref(name)],
+            responses => #{
+                200 => hoconsc:array(hoconsc:ref(log_file_detail)),
+                404 => emqx_dashboard_swagger:error_codes(['NOT_FOUND'], <<"Trace Name Not Found">>)
+            }
+        }
+    };
 schema("/trace/:name/log") ->
     #{
         'operationId' => stream_log_file,
@@ -158,6 +179,13 @@ schema("/trace/:name/log") ->
         }
     }.
 
+fields(log_file_detail) ->
+    fields(node) ++
+        [
+            {size, hoconsc:mk(integer(), #{desc => "file size"})},
+            {mtime,
+                hoconsc:mk(integer(), #{desc => "the modification and last access times of a file"})}
+        ];
 fields(trace) ->
     [
         {name,
@@ -265,7 +293,8 @@ fields(node) ->
                 #{
                     desc => "Node name",
                     in => query,
-                    required => false
+                    required => false,
+                    example => "emqx@127.0.0.1"
                 }
             )}
     ];
@@ -323,7 +352,7 @@ trace(get, _Params) ->
                 emqx_trace:format(List0)
             ),
             Nodes = mria_mnesia:running_nodes(),
-            TraceSize = wrap_rpc(emqx_mgmt_trace_proto_v1:get_trace_size(Nodes)),
+            TraceSize = wrap_rpc(emqx_mgmt_trace_proto_v2:get_trace_size(Nodes)),
             AllFileSize = lists:foldl(fun(F, Acc) -> maps:merge(Acc, F) end, #{}, TraceSize),
             Now = erlang:system_time(second),
             Traces =
@@ -471,19 +500,43 @@ group_trace_file(ZipDir, TraceLog, TraceFiles) ->
     ).
 
 collect_trace_file(Nodes, TraceLog) ->
-    wrap_rpc(emqx_mgmt_trace_proto_v1:trace_file(Nodes, TraceLog)).
+    wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file(Nodes, TraceLog)).
+
+collect_trace_file_detail(TraceLog) ->
+    Nodes = mria_mnesia:running_nodes(),
+    wrap_rpc(emqx_mgmt_trace_proto_v2:trace_file_detail(Nodes, TraceLog)).
 
 wrap_rpc({GoodRes, BadNodes}) ->
     BadNodes =/= [] andalso
         ?SLOG(error, #{msg => "rpc_call_failed", bad_nodes => BadNodes}),
     GoodRes.
 
+log_file_detail(get, #{bindings := #{name := Name}}) ->
+    case emqx_trace:get_trace_filename(Name) of
+        {ok, TraceLog} ->
+            TraceFiles = collect_trace_file_detail(TraceLog),
+            {200, group_trace_file_detail(TraceFiles)};
+        {error, not_found} ->
+            ?NOT_FOUND(Name)
+    end.
+
+group_trace_file_detail(TraceLogDetail) ->
+    GroupFun =
+        fun
+            ({ok, Info}, Acc) ->
+                [Info | Acc];
+            ({error, Error}, Acc) ->
+                ?SLOG(error, Error#{msg => "read_trace_file_failed"}),
+                Acc
+        end,
+    lists:foldl(GroupFun, [], TraceLogDetail).
+
 stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) ->
     Position = maps:get(<<"position">>, Query, 0),
     Bytes = maps:get(<<"bytes">>, Query, 1000),
     case parse_node(Query, node()) of
         {ok, Node} ->
-            case emqx_mgmt_trace_proto_v1:read_trace_file(Node, Name, Position, Bytes) of
+            case emqx_mgmt_trace_proto_v2:read_trace_file(Node, Name, Position, Bytes) of
                 {ok, Bin} ->
                     Meta = #{<<"position">> => Position + byte_size(Bin), <<"bytes">> => Bytes},
                     {200, #{meta => Meta, items => Bin}};

apps/emqx_management/src/proto/emqx_mgmt_trace_proto_v2.erl

@@ -0,0 +1,66 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_mgmt_trace_proto_v2).
+
+-behaviour(emqx_bpapi).
+
+-export([
+    introduced_in/0,
+
+    trace_file/2,
+    trace_file_detail/2,
+    get_trace_size/1,
+    read_trace_file/4
+]).
+
+-include_lib("emqx/include/bpapi.hrl").
+
+introduced_in() ->
+    "5.0.9".
+
+-spec get_trace_size([node()]) ->
+    emqx_rpc:multicall_result(#{{node(), file:name_all()} => non_neg_integer()}).
+get_trace_size(Nodes) ->
+    rpc:multicall(Nodes, emqx_mgmt_api_trace, get_trace_size, [], 30000).
+
+-spec trace_file([node()], file:name_all()) ->
+    emqx_rpc:multicall_result(
+        {ok, Node :: list(), Binary :: binary()}
+        | {error, Node :: list(), Reason :: term()}
+    ).
+trace_file(Nodes, File) ->
+    rpc:multicall(Nodes, emqx_trace, trace_file, [File], 60000).
+
+-spec trace_file_detail([node()], file:name_all()) ->
+    emqx_rpc:multicall_result(
+        {ok, #{
+            size => non_neg_integer(),
+            mtime => file:date_time() | non_neg_integer() | 'undefined',
+            node => atom()
+        }}
+        | {error, #{reason => term(), node => atom(), file => file:name_all()}}
+    ).
+trace_file_detail(Nodes, File) ->
+    rpc:multicall(Nodes, emqx_trace, trace_file_detail, [File], 25000).
+
+-spec read_trace_file(node(), binary(), non_neg_integer(), non_neg_integer()) ->
+    {ok, binary()}
+    | {error, _}
+    | {eof, non_neg_integer()}
+    | {badrpc, _}.
+read_trace_file(Node, Name, Position, Limit) ->
+    rpc:call(Node, emqx_mgmt_api_trace, read_trace_file, [Name, Position, Limit], 15000).

apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl

@@ -175,7 +175,7 @@ t_create_failed(_Config) ->
     emqx_trace:clear(),
     ok.
 
-t_download_log(_Config) ->
+t_log_file(_Config) ->
     ClientId = <<"client-test-download">>,
     Now = erlang:system_time(second),
     Name = <<"test_client_id">>,
@@ -191,6 +191,12 @@ t_download_log(_Config) ->
     ],
     ok = emqx_trace_handler_SUITE:filesync(Name, clientid),
     Header = auth_header_(),
+    ?assertMatch(
+        {error, {"HTTP/1.1", 404, "Not Found"}, _},
+        request_api(get, api_path("trace/test_client_not_found/log_detail"), Header)
+    ),
+    {ok, Detail} = request_api(get, api_path("trace/test_client_id/log_detail"), Header),
+    ?assertMatch([#{<<"mtime">> := _, <<"size">> := _, <<"node">> := _}], json(Detail)),
     {ok, Binary} = request_api(get, api_path("trace/test_client_id/download"), Header),
     {ok, [
         _Comment,