Merge pull request #12172 from zhongwencool/audit-max-size-microsecond

fix: use microsecond precision as the primary key for audit logs

apps/emqx/integration_test/emqx_persistent_session_ds_SUITE.erl

@@ -516,7 +516,7 @@ do_t_session_expiration(_Config, Opts) ->
 
 t_session_gc(Config) ->
     GCInterval = ?config(gc_interval, Config),
-    [Node1, Node2, Node3] = Nodes = ?config(nodes, Config),
+    [Node1, Node2, _Node3] = Nodes = ?config(nodes, Config),
     CoreNodes = [Node1, Node2],
     [
         Port1,

apps/emqx_audit/src/emqx_audit.erl

@@ -119,7 +119,7 @@ log_to_db(Log) ->
     Audit0 = to_audit(Log),
     Audit = Audit0#?AUDIT{
         node = node(),
-        created_at = erlang:system_time(millisecond)
+        created_at = erlang:system_time(microsecond)
     },
     mria:dirty_write(?AUDIT, Audit).
 

apps/emqx_audit/src/emqx_audit_api.erl

@@ -32,7 +32,7 @@
     {<<"http_method">>, atom},
     {<<"gte_created_at">>, timestamp},
     {<<"lte_created_at">>, timestamp},
-    {<<"gte_duration_ms">>, timestamp},
+    {<<"gte_duration_ms">>, integer},
     {<<"lte_duration_ms">>, integer}
 ]).
 -define(DISABLE_MSG, <<"Audit is disabled">>).
@@ -130,14 +130,14 @@ schema("/audit") ->
                         desc => ?DESC(filter_lte_duration_ms)
                     })},
                 {gte_created_at,
-                    ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
+                    ?HOCON(emqx_utils_calendar:epoch_microsecond(), #{
                         in => query,
                         required => false,
                         example => <<"2023-10-15T00:00:00.820384+08:00">>,
                         desc => ?DESC(filter_gte_created_at)
                     })},
                 {lte_created_at,
-                    ?HOCON(emqx_utils_calendar:epoch_millisecond(), #{
+                    ?HOCON(emqx_utils_calendar:epoch_microsecond(), #{
                         in => query,
                         example => <<"2023-10-16T00:00:00.820384+08:00">>,
                         required => false,
@@ -170,7 +170,7 @@ fields(audit) ->
     [
         {created_at,
             ?HOCON(
-                emqx_utils_calendar:epoch_millisecond(),
+                emqx_utils_calendar:epoch_microsecond(),
                 #{
                     desc => "The time when the log is created"
                 }

apps/emqx_audit/test/emqx_audit_api_SUITE.erl

@@ -140,9 +140,9 @@ t_disabled(_) ->
 
 t_cli(_Config) ->
     Size = mnesia:table_info(emqx_audit, size),
-    TimeInt = erlang:system_time(millisecond) - 10,
+    TimeInt = erlang:system_time(microsecond) - 1000,
     Time = integer_to_list(TimeInt),
-    DateStr = calendar:system_time_to_rfc3339(TimeInt, [{unit, millisecond}]),
+    DateStr = calendar:system_time_to_rfc3339(TimeInt, [{unit, microsecond}]),
     Date = emqx_http_lib:uri_encode(DateStr),
     ok = emqx_ctl:run_command(["conf", "show", "log"]),
     AuditPath = emqx_mgmt_api_test_util:api_path(["audit"]),
@@ -164,7 +164,11 @@ t_cli(_Config) ->
         ],
         Data
     ),
-
+    %% check create at is valid
+    [#{<<"created_at">> := CreateAtRaw}] = Data,
+    CreateAt = calendar:rfc3339_to_system_time(binary_to_list(CreateAtRaw), [{unit, microsecond}]),
+    ?assert(CreateAt > TimeInt, CreateAtRaw),
+    ?assert(CreateAt < TimeInt + 5000000, CreateAtRaw),
     %% check cli filter
     {ok, Res1} = emqx_mgmt_api_test_util:request_api(get, AuditPath, "from=cli", AuthHeader),
     #{<<"data">> := Data1} = emqx_utils_json:decode(Res1, [return_maps]),
@@ -174,25 +178,41 @@ t_cli(_Config) ->
     ),
     ?assertMatch(#{<<"data">> := []}, emqx_utils_json:decode(Res2, [return_maps])),
 
-    %% check created_at filter
+    %% check created_at filter microsecond
     {ok, Res3} = emqx_mgmt_api_test_util:request_api(
         get, AuditPath, "gte_created_at=" ++ Time, AuthHeader
     ),
     #{<<"data">> := Data3} = emqx_utils_json:decode(Res3, [return_maps]),
     ?assertEqual(1, erlang:length(Data3)),
+    %% check created_at filter rfc3339
     {ok, Res31} = emqx_mgmt_api_test_util:request_api(
         get, AuditPath, "gte_created_at=" ++ Date, AuthHeader
     ),
     ?assertEqual(Res3, Res31),
+    %% check created_at filter millisecond
+    TimeMs = integer_to_list(TimeInt div 1000),
+    {ok, Res32} = emqx_mgmt_api_test_util:request_api(
+        get, AuditPath, "gte_created_at=" ++ TimeMs, AuthHeader
+    ),
+    ?assertEqual(Res3, Res32),
+
+    %% check created_at filter microsecond
     {ok, Res4} = emqx_mgmt_api_test_util:request_api(
         get, AuditPath, "lte_created_at=" ++ Time, AuthHeader
     ),
     #{<<"data">> := Data4} = emqx_utils_json:decode(Res4, [return_maps]),
     ?assertEqual(Size, erlang:length(Data4)),
+
+    %% check created_at filter rfc3339
     {ok, Res41} = emqx_mgmt_api_test_util:request_api(
         get, AuditPath, "lte_created_at=" ++ Date, AuthHeader
     ),
     ?assertEqual(Res4, Res41),
+    %% check created_at filter millisecond
+    {ok, Res42} = emqx_mgmt_api_test_util:request_api(
+        get, AuditPath, "lte_created_at=" ++ TimeMs, AuthHeader
+    ),
+    ?assertEqual(Res4, Res42),
 
     %% check duration_ms filter
     {ok, Res5} = emqx_mgmt_api_test_util:request_api(
@@ -224,7 +244,7 @@ t_max_size(_Config) ->
         fun(_) ->
             ok = emqx_ctl:run_command(["conf", "show", "log"])
         end,
-        lists:duplicate(110, 1)
+        lists:duplicate(100, 1)
     ),
     _ = mnesia:dump_log(),
     LogCount = wait_for_dirty_write_log_done(1500),

apps/emqx_conf/src/emqx_conf_schema_types.erl

@@ -143,6 +143,24 @@ readable("epoch_millisecond()") ->
             ]
         }
     };
+readable("epoch_microsecond()") ->
+    %% only for swagger
+    #{
+        swagger => #{
+            <<"oneOf">> => [
+                #{
+                    type => integer,
+                    example => 1640995200000000,
+                    description => <<"epoch-microsecond">>
+                },
+                #{
+                    type => string,
+                    example => <<"2022-01-01T00:00:00.000000Z">>,
+                    format => <<"date-time">>
+                }
+            ]
+        }
+    };
 readable("duration()") ->
     #{
         swagger => #{type => string, example => <<"12m">>},

apps/emqx_utils/src/emqx_utils_calendar.erl

@@ -29,6 +29,7 @@
 %% API
 -export([
     to_epoch_millisecond/1,
+    to_epoch_microsecond/1,
     to_epoch_second/1,
     human_readable_duration_string/1
 ]).
@@ -54,6 +55,7 @@
 %% so the maximum date can reach 9999-12-31 which is ample.
 -define(MAXIMUM_EPOCH, 253402214400).
 -define(MAXIMUM_EPOCH_MILLI, 253402214400_000).
+-define(MAXIMUM_EPOCH_MICROS, 253402214400_000_000).
 
 -define(DATE_PART, [
     year,
@@ -75,13 +77,16 @@
 
 -reflect_type([
     epoch_millisecond/0,
-    epoch_second/0
+    epoch_second/0,
+    epoch_microsecond/0
 ]).
 
 -type epoch_second() :: non_neg_integer().
 -type epoch_millisecond() :: non_neg_integer().
+-type epoch_microsecond() :: non_neg_integer().
 -typerefl_from_string({epoch_second/0, ?MODULE, to_epoch_second}).
 -typerefl_from_string({epoch_millisecond/0, ?MODULE, to_epoch_millisecond}).
+-typerefl_from_string({epoch_microsecond/0, ?MODULE, to_epoch_microsecond}).
 
 %%--------------------------------------------------------------------
 %% Epoch <-> RFC 3339
@@ -93,6 +98,9 @@ to_epoch_second(DateTime) ->
 to_epoch_millisecond(DateTime) ->
     to_epoch(DateTime, millisecond).
 
+to_epoch_microsecond(DateTime) ->
+    to_epoch(DateTime, microsecond).
+
 to_epoch(DateTime, Unit) ->
     try
         case string:to_integer(DateTime) of
@@ -131,6 +139,14 @@ validate_epoch(Epoch, second) when Epoch =< ?MAXIMUM_EPOCH ->
     {ok, Epoch};
 validate_epoch(Epoch, millisecond) when Epoch =< ?MAXIMUM_EPOCH_MILLI ->
     {ok, Epoch};
+%% http api use millisecond but we should transform to microsecond
+validate_epoch(Epoch, microsecond) when
+    Epoch >= ?MAXIMUM_EPOCH andalso
+        Epoch =< ?MAXIMUM_EPOCH_MILLI
+->
+    {ok, Epoch * 1000};
+validate_epoch(Epoch, microsecond) when Epoch =< ?MAXIMUM_EPOCH_MICROS ->
+    {ok, Epoch};
 validate_epoch(_Epoch, _Unit) ->
     {error, bad_epoch}.