Merge pull request #11680 from thalesmg/audit-not-on-ce-r53-20230925

fix(audit): only support audit log on enterprise edition

apps/emqx/test/emqx_cth_suite.erl

@@ -52,7 +52,7 @@
 %%    (e.g. in `init_per_suite/1` / `init_per_group/2`), providing the appspecs
 %%    and unique work dir for the testrun (e.g. `work_dir/1`). Save the result
 %%    in a context.
-%% 3. Call `emqx_cth_sutie:stop/1` to stop the applications after the testrun
+%% 3. Call `emqx_cth_suite:stop/1` to stop the applications after the testrun
 %%    finishes (e.g. in `end_per_suite/1` / `end_per_group/2`), providing the
 %%    result from step 2.
 -module(emqx_cth_suite).
@@ -245,6 +245,9 @@ spec_fmt(ffun, {_, X}) -> X.
 
 maybe_configure_app(_App, #{config := false}) ->
     ok;
+maybe_configure_app(_App, AppConfig = #{schema_mod := SchemaModule}) when is_atom(SchemaModule) ->
+    #{config := Config} = AppConfig,
+    configure_app(SchemaModule, Config);
 maybe_configure_app(App, #{config := Config}) ->
     case app_schema(App) of
         {ok, SchemaModule} ->

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -43,6 +43,9 @@
 ]).
 -export([conf_get/2, conf_get/3, keys/2, filter/1]).
 
+%% internal exports for `emqx_enterprise_schema' only.
+-export([ensure_unicode_path/2, convert_rotation/2, log_handler_common_confs/2]).
+
 %% Static apps which merge their configs into the merged emqx.conf
 %% The list can not be made a dynamic read at run-time as it is used
 %% by nodetool to generate app.<time>.config before EMQX is started
@@ -962,15 +965,6 @@ fields("log") ->
                     aliases => [file_handlers],
                     importance => ?IMPORTANCE_HIGH
                 }
-            )},
-        {"audit",
-            sc(
-                ?R_REF("log_audit_handler"),
-                #{
-                    desc => ?DESC("log_audit_handler"),
-                    importance => ?IMPORTANCE_HIGH,
-                    default => #{<<"enable">> => true, <<"level">> => <<"info">>}
-                }
             )}
     ];
 fields("console_handler") ->
@@ -1012,49 +1006,6 @@ fields("log_file_handler") ->
                 }
             )}
     ] ++ log_handler_common_confs(file, #{});
-fields("log_audit_handler") ->
-    [
-        {"path",
-            sc(
-                file(),
-                #{
-                    desc => ?DESC("audit_file_handler_path"),
-                    default => <<"${EMQX_LOG_DIR}/audit.log">>,
-                    importance => ?IMPORTANCE_HIGH,
-                    converter => fun(Path, Opts) ->
-                        emqx_schema:naive_env_interpolation(ensure_unicode_path(Path, Opts))
-                    end
-                }
-            )},
-        {"rotation_count",
-            sc(
-                range(1, 128),
-                #{
-                    default => 10,
-                    converter => fun convert_rotation/2,
-                    desc => ?DESC("log_rotation_count"),
-                    importance => ?IMPORTANCE_MEDIUM
-                }
-            )},
-        {"rotation_size",
-            sc(
-                hoconsc:union([infinity, emqx_schema:bytesize()]),
-                #{
-                    default => <<"50MB">>,
-                    desc => ?DESC("log_file_handler_max_size"),
-                    importance => ?IMPORTANCE_MEDIUM
-                }
-            )}
-    ] ++
-        %% Only support json
-        lists:keydelete(
-            "formatter",
-            1,
-            log_handler_common_confs(
-                file,
-                #{level => info, level_desc => "audit_handler_level"}
-            )
-        );
 fields("log_overload_kill") ->
     [
         {"enable",
@@ -1145,8 +1096,6 @@ desc("console_handler") ->
     ?DESC("desc_console_handler");
 desc("log_file_handler") ->
     ?DESC("desc_log_file_handler");
-desc("log_audit_handler") ->
-    ?DESC("desc_audit_log_handler");
 desc("log_rotation") ->
     ?DESC("desc_log_rotation");
 desc("log_overload_kill") ->

apps/emqx_conf/test/emqx_conf_logger_SUITE.erl

@@ -78,16 +78,7 @@ t_log_conf(_Conf) ->
                 <<"time_offset">> => <<"system">>
             },
         <<"file">> =>
-            #{<<"default">> => FileExpect},
-        <<"audit">> =>
-            #{
-                <<"enable">> => true,
-                <<"level">> => <<"info">>,
-                <<"path">> => <<"log/audit.log">>,
-                <<"rotation_count">> => 10,
-                <<"rotation_size">> => <<"50MB">>,
-                <<"time_offset">> => <<"system">>
-            }
+            #{<<"default">> => FileExpect}
     },
     ?assertEqual(ExpectLog1, emqx_conf:get_raw([<<"log">>])),
     UpdateLog0 = emqx_utils_maps:deep_remove([<<"file">>, <<"default">>], ExpectLog1),

apps/emqx_conf/test/emqx_conf_schema_tests.erl

@@ -181,23 +181,8 @@ validate_log(Conf) ->
         }},
         FileHandler
     ),
-    AuditHandler = lists:keyfind(emqx_audit, 2, FileHandlers),
-    %% default is enable and log level is info.
-    ?assertMatch(
-        {handler, emqx_audit, logger_disk_log_h, #{
-            config := #{
-                type := wrap,
-                file := "log/audit.log",
-                max_no_bytes := _,
-                max_no_files := _
-            },
-            filesync_repeat_interval := no_repeat,
-            filters := [{filter_audit, {_, stop}}],
-            formatter := _,
-            level := info
-        }},
-        AuditHandler
-    ),
+    %% audit is an EE-only feature
+    ?assertNot(lists:keyfind(emqx_audit, 2, FileHandlers)),
     ConsoleHandler = lists:keyfind(logger_std_h, 3, Loggers),
     ?assertEqual(
         {handler, console, logger_std_h, #{

apps/emqx_enterprise/src/emqx_enterprise_schema.erl

@@ -6,6 +6,9 @@
 
 -behaviour(hocon_schema).
 
+-include_lib("typerefl/include/types.hrl").
+-include_lib("hocon/include/hoconsc.hrl").
+
 -export([namespace/0, roots/0, fields/1, translations/0, translation/1, desc/1, validations/0]).
 
 -define(EE_SCHEMA_MODULES, [
@@ -22,6 +25,53 @@ roots() ->
 
 fields("node") ->
     redefine_node(emqx_conf_schema:fields("node"));
+fields("log") ->
+    redefine_log(emqx_conf_schema:fields("log"));
+fields("log_audit_handler") ->
+    [
+        {"path",
+            hoconsc:mk(
+                emqx_conf_schema:file(),
+                #{
+                    desc => ?DESC(emqx_conf_schema, "audit_file_handler_path"),
+                    default => <<"${EMQX_LOG_DIR}/audit.log">>,
+                    importance => ?IMPORTANCE_HIGH,
+                    converter => fun(Path, Opts) ->
+                        emqx_schema:naive_env_interpolation(
+                            emqx_conf_schema:ensure_unicode_path(Path, Opts)
+                        )
+                    end
+                }
+            )},
+        {"rotation_count",
+            hoconsc:mk(
+                range(1, 128),
+                #{
+                    default => 10,
+                    converter => fun emqx_conf_schema:convert_rotation/2,
+                    desc => ?DESC(emqx_conf_schema, "log_rotation_count"),
+                    importance => ?IMPORTANCE_MEDIUM
+                }
+            )},
+        {"rotation_size",
+            hoconsc:mk(
+                hoconsc:union([infinity, emqx_schema:bytesize()]),
+                #{
+                    default => <<"50MB">>,
+                    desc => ?DESC(emqx_conf_schema, "log_file_handler_max_size"),
+                    importance => ?IMPORTANCE_MEDIUM
+                }
+            )}
+    ] ++
+        %% Only support json
+        lists:keydelete(
+            "formatter",
+            1,
+            emqx_conf_schema:log_handler_common_confs(
+                file,
+                #{level => info, level_desc => "audit_handler_level"}
+            )
+        );
 fields(Name) ->
     ee_delegate(fields, ?EE_SCHEMA_MODULES, Name).
 
@@ -31,6 +81,8 @@ translations() ->
 translation(Name) ->
     emqx_conf_schema:translation(Name).
 
+desc("log_audit_handler") ->
+    ?DESC(emqx_conf_schema, "desc_audit_log_handler");
 desc(Name) ->
     ee_delegate(desc, ?EE_SCHEMA_MODULES, Name).
 
@@ -60,13 +112,20 @@ ee_delegate(Method, [], Name) ->
     apply(emqx_conf_schema, Method, [Name]).
 
 redefine_roots(Roots) ->
-    Overrides = [{"node", #{type => hoconsc:ref(?MODULE, "node")}}],
+    Overrides = [
+        {"node", #{type => hoconsc:ref(?MODULE, "node")}},
+        {"log", #{type => hoconsc:ref(?MODULE, "log")}}
+    ],
     override(Roots, Overrides).
 
 redefine_node(Fields) ->
     Overrides = [],
     override(Fields, Overrides).
 
+redefine_log(Fields) ->
+    Overrides = [],
+    override(Fields, Overrides) ++ audit_log_conf().
+
 override(Fields, []) ->
     Fields;
 override(Fields, [{Name, Override} | More]) ->
@@ -81,3 +140,19 @@ find_schema(Name, Fields) ->
 
 replace_schema(Name, Schema, Fields) ->
     lists:keyreplace(Name, 1, Fields, {Name, Schema}).
+
+audit_log_conf() ->
+    [
+        {"audit",
+            hoconsc:mk(
+                hoconsc:ref(?MODULE, "log_audit_handler"),
+                #{
+                    %% note: we need to keep the descriptions associated with
+                    %% `emqx_conf_schema' module hocon i18n file because that's what
+                    %% `emqx_conf:gen_config_md' seems to expect.
+                    desc => ?DESC(emqx_conf_schema, "log_audit_handler"),
+                    importance => ?IMPORTANCE_HIGH,
+                    default => #{<<"enable">> => true, <<"level">> => <<"info">>}
+                }
+            )}
+    ].

apps/emqx_enterprise/test/emqx_enterprise_schema_SUITE.erl

@@ -13,6 +13,25 @@
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
+init_per_testcase(t_audit_log_conf, Config) ->
+    Apps = emqx_cth_suite:start(
+        [
+            emqx_enterprise,
+            {emqx_conf, #{schema_mod => emqx_enterprise_schema}}
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{apps, Apps} | Config];
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(t_audit_log_conf, Config) ->
+    Apps = ?config(apps, Config),
+    ok = emqx_cth_suite:stop(Apps),
+    ok;
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
 %%------------------------------------------------------------------------------
 %% Tests
 %%------------------------------------------------------------------------------
@@ -50,3 +69,36 @@ t_translations(_Config) ->
         emqx_conf_schema:translation(Root),
         emqx_enterprise_schema:translation(Root)
     ).
+
+t_audit_log_conf(_Config) ->
+    FileExpect = #{
+        <<"enable">> => true,
+        <<"formatter">> => <<"text">>,
+        <<"level">> => <<"warning">>,
+        <<"rotation_count">> => 10,
+        <<"rotation_size">> => <<"50MB">>,
+        <<"time_offset">> => <<"system">>,
+        <<"path">> => <<"log/emqx.log">>
+    },
+    ExpectLog1 = #{
+        <<"console">> =>
+            #{
+                <<"enable">> => false,
+                <<"formatter">> => <<"text">>,
+                <<"level">> => <<"warning">>,
+                <<"time_offset">> => <<"system">>
+            },
+        <<"file">> =>
+            #{<<"default">> => FileExpect},
+        <<"audit">> =>
+            #{
+                <<"enable">> => true,
+                <<"level">> => <<"info">>,
+                <<"path">> => <<"log/audit.log">>,
+                <<"rotation_count">> => 10,
+                <<"rotation_size">> => <<"50MB">>,
+                <<"time_offset">> => <<"system">>
+            }
+    },
+    ?assertEqual(ExpectLog1, emqx_conf:get_raw([<<"log">>])),
+    ok.

apps/emqx_enterprise/test/emqx_enterprise_schema_tests.erl

@@ -16,3 +16,38 @@ doc_gen_test() ->
             ok = emqx_conf:dump_schema(Dir, emqx_enterprise_schema)
         end
     }.
+
+audit_log_test() ->
+    ensure_acl_conf(),
+    Conf0 = <<"node {cookie = aaa, data_dir = \"/tmp\"}">>,
+    {ok, ConfMap0} = hocon:binary(Conf0, #{format => richmap}),
+    ConfList = hocon_tconf:generate(emqx_enterprise_schema, ConfMap0),
+    Kernel = proplists:get_value(kernel, ConfList),
+    Loggers = proplists:get_value(logger, Kernel),
+    FileHandlers = lists:filter(fun(L) -> element(3, L) =:= logger_disk_log_h end, Loggers),
+    AuditHandler = lists:keyfind(emqx_audit, 2, FileHandlers),
+    %% default is enable and log level is info.
+    ?assertMatch(
+        {handler, emqx_audit, logger_disk_log_h, #{
+            config := #{
+                type := wrap,
+                file := "log/audit.log",
+                max_no_bytes := _,
+                max_no_files := _
+            },
+            filesync_repeat_interval := no_repeat,
+            filters := [{filter_audit, {_, stop}}],
+            formatter := _,
+            level := info
+        }},
+        AuditHandler
+    ),
+    ok.
+
+ensure_acl_conf() ->
+    File = emqx_schema:naive_env_interpolation(<<"${EMQX_ETC_DIR}/acl.conf">>),
+    ok = filelib:ensure_dir(filename:dirname(File)),
+    case filelib:is_regular(File) of
+        true -> ok;
+        false -> file:write_file(File, <<"">>)
+    end.

rel/i18n/emqx_conf_schema.hocon

@@ -841,7 +841,4 @@ Defaults to 100000."""
 node_channel_cleanup_batch_size.label:
 """Node Channel Cleanup Batch Size"""
 
-prevent_overlapping_partitions.desc:
-"""https://www.erlang.org/doc/man/global.html#description"""
-
 }