Merge pull request #14077 from HJianBo/truncate_audit_log_in_db_storing

perf(audit): support to truncate large log contents

apps/emqx_audit/src/emqx_audit.app.src

@@ -1,6 +1,6 @@
 {application, emqx_audit, [
     {description, "Audit log for EMQX"},
-    {vsn, "0.1.1"},
+    {vsn, "0.1.2"},
     {registered, []},
     {mod, {emqx_audit_app, []}},
     {applications, [kernel, stdlib, emqx]},

apps/emqx_audit/src/emqx_audit.erl

@@ -29,6 +29,8 @@
 
 -define(FILTER_REQ, [cert, host_info, has_sent_resp, pid, path_info, peer, ref, sock, streamid]).
 
+-define(CHARS_LIMIT_IN_DB, 1024).
+
 -ifdef(TEST).
 -define(INTERVAL, 100).
 -else.
@@ -38,8 +40,8 @@
 to_audit(#{from := cli, cmd := Cmd, args := Args, duration_ms := DurationMs}) ->
     #?AUDIT{
         operation_id = <<"">>,
-        operation_type = atom_to_binary(Cmd),
-        args = Args,
+        operation_type = truncate_large_term(Cmd),
+        args = truncate_large_term(Args),
         operation_result = <<"">>,
         failure = <<"">>,
         duration_ms = DurationMs,
@@ -65,7 +67,7 @@ to_audit(#{from := erlang_console, function := F, args := Args}) ->
         http_method = <<"">>,
         http_request = <<"">>,
         duration_ms = 0,
-        args = iolist_to_binary(io_lib:format("~p: ~ts", [F, Args]))
+        args = truncate_large_term({F, Args})
     };
 to_audit(#{from := From} = Log) when is_atom(From) ->
     #{
@@ -93,7 +95,7 @@ to_audit(#{from := From} = Log) when is_atom(From) ->
         %% request detail
         http_status_code = StatusCode,
         http_method = Method,
-        http_request = Request,
+        http_request = truncate_large_term(Request),
         duration_ms = DurationMs,
         args = <<"">>
     }.
@@ -243,3 +245,6 @@ log_to_file(Level, Meta, #{module := Module} = Handler) ->
                     )
             end
     end.
+
+truncate_large_term(Req) ->
+    unicode:characters_to_binary(io_lib:format("~0p", [Req], [{chars_limit, ?CHARS_LIMIT_IN_DB}])).

apps/emqx_audit/test/emqx_audit_api_SUITE.erl

@@ -94,12 +94,7 @@ t_http_api(_) ->
                     <<"operation_id">> := <<"/configs/global_zone">>,
                     <<"source_ip">> := <<"127.0.0.1">>,
                     <<"source">> := _,
-                    <<"http_request">> := #{
-                        <<"method">> := <<"put">>,
-                        <<"body">> := #{<<"mqtt">> := #{<<"max_qos_allowed">> := 1}},
-                        <<"bindings">> := _,
-                        <<"headers">> := #{<<"authorization">> := <<"******">>}
-                    },
+                    <<"http_request">> := _,
                     <<"http_status_code">> := 200,
                     <<"operation_result">> := <<"success">>,
                     <<"operation_type">> := <<"configs">>
@@ -166,7 +161,7 @@ t_cli(_Config) ->
                 <<"operation_id">> := <<"">>,
                 <<"source_ip">> := <<"">>,
                 <<"operation_type">> := <<"conf">>,
-                <<"args">> := [<<"show">>, <<"log">>],
+                <<"args">> := <<"[<<\"show\">>,<<\"log\">>]">>,
                 <<"node">> := _,
                 <<"source">> := <<"">>,
                 <<"http_request">> := <<"">>
@@ -184,7 +179,7 @@ t_cli(_Config) ->
     {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader),
     #{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]),
     ?assertMatch(
-        [ShowLogEntry, #{<<"operation_type">> := <<"emqx">>, <<"args">> := [<<"start">>]}],
+        [ShowLogEntry, #{<<"operation_type">> := <<"emqx">>, <<"args">> := <<"[<<\"start\">>]">>}],
         Data1
     ),
     {ok, Res2} = emqx_mgmt_api_test_util:request_api(

changes/ee/perf-14077.en.md

@@ -0,0 +1 @@
+Avoid storing excessively lengthy audit log content with log truncation.