Merge remote-tracking branch 'origin/master' into tag-v5026

apps/emqx_authn/src/emqx_authn.erl

@@ -106,7 +106,7 @@ get_enabled_authns() ->
     AuthnTypes = lists:usort([
         Type
      || #{authenticators := As} <- Chains,
-        #{id := Type} <- As
+        #{id := Type, enable := true} <- As
     ]),
     OverriddenListeners =
         lists:foldl(

apps/emqx_authz/src/emqx_authz.erl

@@ -417,7 +417,7 @@ do_authorize(
     end.
 
 get_enabled_authzs() ->
-    lists:usort([Type || #{type := Type} <- lookup()]).
+    lists:usort([Type || #{type := Type, enable := true} <- lookup()]).
 
 %%--------------------------------------------------------------------
 %% Internal function

apps/emqx_authz/test/emqx_authz_SUITE.erl

@@ -366,7 +366,7 @@ t_get_enabled_authzs_none_enabled(_Config) ->
     ?assertEqual([], emqx_authz:get_enabled_authzs()).
 
 t_get_enabled_authzs_some_enabled(_Config) ->
-    {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE4]),
+    {ok, _} = emqx_authz:update(?CMD_REPLACE, [?SOURCE4, ?SOURCE5#{<<"enable">> := false}]),
     ?assertEqual([postgresql], emqx_authz:get_enabled_authzs()).
 
 t_subscribe_deny_disconnect_publishes_last_will_testament(_Config) ->

apps/emqx_dashboard/src/emqx_dashboard_bad_api.erl

@@ -21,11 +21,12 @@
 -export([init/2]).
 
 init(Req0, State) ->
-    ?SLOG(warning, #{msg => "unexpected_api_access", request => Req0}),
+    RedactedReq = emqx_utils:redact(Req0),
+    ?SLOG(warning, #{msg => "unexpected_api_access", request => RedactedReq}),
     Req = cowboy_req:reply(
         404,
         #{<<"content-type">> => <<"application/json">>},
         <<"{\"code\": \"API_NOT_EXIST\", \"message\": \"Request Path Not Found\"}">>,
-        Req0
+        RedactedReq
     ),
     {ok, Req, State}.

apps/emqx_ft/test/props/prop_emqx_ft_assembly.erl

@@ -58,7 +58,7 @@ prop_coverage_likely_incomplete() ->
         {filesize_t(), segsizes_t(), filesize_t()},
         ?FORALL(
             Fragments,
-            noshrink(segments_t(Filesize, Segsizes, Hole)),
+            noshrink(segments_t(Filesize, Segsizes, (Hole rem max(Filesize, 1)))),
             ?TIMEOUT(
                 ?COVERAGE_TIMEOUT,
                 begin
@@ -174,7 +174,7 @@ segment_t(Filesize, Segsizes, Hole) ->
     ?SUCHTHATMAYBE(
         {Offset, Size},
         segment_t(Filesize, Segsizes),
-        (Hole rem Filesize) =< Offset orelse (Hole rem Filesize) > (Offset + Size)
+        Hole =< Offset orelse Hole > (Offset + Size)
     ).
 
 segment_t(Filesize, Segsizes) ->

apps/emqx_machine/src/emqx_machine_boot.erl

@@ -24,6 +24,7 @@
 -export([stop_port_apps/0]).
 
 -dialyzer({no_match, [basic_reboot_apps/0]}).
+-dialyzer({no_match, [basic_reboot_apps_edition/1]}).
 
 -ifdef(TEST).
 -export([sorted_reboot_apps/1, reboot_apps/0]).
@@ -126,39 +127,40 @@ reboot_apps() ->
     BaseRebootApps ++ ConfigApps.
 
 basic_reboot_apps() ->
-    CE =
-        ?BASIC_REBOOT_APPS ++
-            [
-                emqx_prometheus,
-                emqx_modules,
-                emqx_dashboard,
-                emqx_connector,
-                emqx_gateway,
-                emqx_resource,
-                emqx_rule_engine,
-                emqx_bridge,
-                emqx_plugin_libs,
-                emqx_management,
-                emqx_retainer,
-                emqx_exhook,
-                emqx_authn,
-                emqx_authz,
-                emqx_slow_subs,
-                emqx_auto_subscribe,
-                emqx_plugins
-            ],
-    case emqx_release:edition() of
-        ce ->
-            CE ++ [emqx_telemetry];
-        ee ->
-            CE ++
-                [
-                    emqx_s3,
-                    emqx_ft,
-                    emqx_eviction_agent,
-                    emqx_node_rebalance
-                ]
-    end.
+    ?BASIC_REBOOT_APPS ++
+        [
+            emqx_prometheus,
+            emqx_modules,
+            emqx_dashboard,
+            emqx_connector,
+            emqx_gateway,
+            emqx_resource,
+            emqx_rule_engine,
+            emqx_bridge,
+            emqx_plugin_libs,
+            emqx_management,
+            emqx_retainer,
+            emqx_exhook,
+            emqx_authn,
+            emqx_authz,
+            emqx_slow_subs,
+            emqx_auto_subscribe,
+            emqx_plugins
+        ] ++ basic_reboot_apps_edition(emqx_release:edition()).
+
+basic_reboot_apps_edition(ce) ->
+    [emqx_telemetry];
+basic_reboot_apps_edition(ee) ->
+    [
+        emqx_license,
+        emqx_s3,
+        emqx_ft,
+        emqx_eviction_agent,
+        emqx_node_rebalance
+    ];
+%% unexcepted edition, should not happen
+basic_reboot_apps_edition(_) ->
+    [].
 
 sorted_reboot_apps() ->
     Apps = [{App, app_deps(App)} || App <- reboot_apps()],

apps/emqx_telemetry/src/emqx_telemetry.erl

@@ -121,7 +121,7 @@ start_reporting() ->
 
 %% @doc Stop the reporting timer.
 %% This is an async notification which never fails.
-%% This is a no-op in enterprise eidtion.
+%% This is a no-op in enterprise edition.
 stop_reporting() ->
     gen_server:cast(?MODULE, stop_reporting).
 

apps/emqx_telemetry/src/emqx_telemetry_config.erl

@@ -32,11 +32,12 @@
 -export([
     on_server_start/0,
     on_server_stop/0,
-    is_official_version/1
+    is_official_version/1,
+    is_official_version/0
 ]).
 
 is_enabled() ->
-    IsOfficial = ?MODULE:is_official_version(emqx_release:version()),
+    IsOfficial = ?MODULE:is_official_version(),
     emqx_conf:get([telemetry, enable], IsOfficial).
 
 on_server_start() ->
@@ -53,7 +54,9 @@ set_telemetry_status(Status) ->
     end.
 
 pre_config_update(_, {set_telemetry_status, Status}, RawConf) ->
-    {ok, RawConf#{<<"enable">> => Status}}.
+    {ok, RawConf#{<<"enable">> => Status}};
+pre_config_update(_, NewConf, _OldConf) ->
+    {ok, NewConf}.
 
 post_config_update(
     _,
@@ -65,6 +68,11 @@ post_config_update(
     case Status of
         true -> emqx_telemetry:start_reporting();
         false -> emqx_telemetry:stop_reporting()
+    end;
+post_config_update(_, _UpdateReq, NewConf, _OldConf, _AppEnvs) ->
+    case maps:get(enable, NewConf, ?MODULE:is_official_version()) of
+        true -> emqx_telemetry:start_reporting();
+        false -> emqx_telemetry:stop_reporting()
     end.
 
 cfg_update(Path, Action, Params) ->
@@ -74,6 +82,9 @@ cfg_update(Path, Action, Params) ->
         #{override_to => cluster}
     ).
 
+is_official_version() ->
+    is_official_version(emqx_release:version()).
+
 is_official_version(Version) ->
     Pt = "^\\d+\\.\\d+(?:\\.\\d+)?(?:(-(?:alpha|beta|rc)\\.[1-9][0-9]*))?$",
     match =:= re:run(Version, Pt, [{capture, none}]).

apps/emqx_telemetry/test/emqx_telemetry_api_SUITE.erl

@@ -30,6 +30,7 @@ all() ->
 
 init_per_suite(Config) ->
     ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF),
+    ok = emqx_common_test_helpers:load_config(emqx_telemetry_schema, ?BASE_CONF),
     ok = emqx_mgmt_api_test_util:init_suite(
         [emqx_conf, emqx_authn, emqx_authz, emqx_telemetry],
         fun set_special_configs/1
@@ -52,32 +53,26 @@ end_per_suite(_Config) ->
     ok.
 
 init_per_testcase(t_status_non_official, Config) ->
-    meck:new(emqx_telemetry, [non_strict, passthrough]),
-    meck:expect(emqx_telemetry, official_version, 1, false),
+    meck:new(emqx_telemetry_config, [non_strict, passthrough]),
+    meck:expect(emqx_telemetry_config, is_official_version, 0, false),
+    %% check non-official telemetry is disable by default
+    {ok, _} = emqx:update_config([telemetry], #{}),
     Config;
-init_per_testcase(t_status, Config) ->
-    meck:new(emqx_telemetry, [non_strict, passthrough]),
-    meck:expect(emqx_telemetry, enable, fun() -> ok end),
-    {ok, _, _} =
-        request(
-            put,
-            uri(["telemetry", "status"]),
-            #{<<"enable">> => true}
-        ),
+init_per_testcase(t_status_official, Config) ->
+    meck:new(emqx_telemetry_config, [non_strict, passthrough]),
+    meck:expect(emqx_telemetry_config, is_official_version, 0, true),
+    %% check official telemetry is enable by default
+    {ok, _} = emqx:update_config([telemetry], #{}),
     Config;
 init_per_testcase(_TestCase, Config) ->
-    {ok, _, _} =
-        request(
-            put,
-            uri(["telemetry", "status"]),
-            #{<<"enable">> => true}
-        ),
+    %% Force enable telemetry to check data.
+    {ok, _} = emqx:update_config([telemetry], #{<<"enable">> => true}),
     Config.
 
 end_per_testcase(t_status_non_official, _Config) ->
-    meck:unload(emqx_telemetry);
+    meck:unload(emqx_telemetry_config);
 end_per_testcase(t_status, _Config) ->
-    meck:unload(emqx_telemetry);
+    meck:unload(emqx_telemetry_config);
 end_per_testcase(_TestCase, _Config) ->
     ok.
 
@@ -95,39 +90,50 @@ set_special_configs(_App) ->
 %% Tests
 %%------------------------------------------------------------------------------
 
-t_status(_) ->
+%% official's telemetry is enabled by default
+t_status_official(_) ->
+    check_status(true).
+
+%% non official's telemetry is disabled by default
+t_status_non_official(_) ->
+    check_status(false).
+
+check_status(Default) ->
+    ct:pal("Check telemetry status:~p~n", [emqx_telemetry_config:is_official_version()]),
+    ?assertEqual(Default, is_telemetry_process_enabled()),
     ?assertMatch(
         {ok, 200, _},
         request(
             put,
             uri(["telemetry", "status"]),
-            #{<<"enable">> => false}
+            #{<<"enable">> => (not Default)}
         )
     ),
 
     {ok, 200, Result0} =
         request(get, uri(["telemetry", "status"])),
-
     ?assertEqual(
-        #{<<"enable">> => false},
+        #{<<"enable">> => (not Default)},
         emqx_utils_json:decode(Result0)
     ),
+    ?assertEqual((not Default), is_telemetry_process_enabled()),
 
     ?assertMatch(
         {ok, 400, _},
         request(
             put,
             uri(["telemetry", "status"]),
-            #{<<"enable">> => false}
+            #{<<"enable">> => (not Default)}
         )
     ),
+    ?assertEqual((not Default), is_telemetry_process_enabled()),
 
     ?assertMatch(
         {ok, 200, _},
         request(
             put,
             uri(["telemetry", "status"]),
-            #{<<"enable">> => true}
+            #{<<"enable">> => Default}
         )
     ),
 
@@ -135,30 +141,24 @@ t_status(_) ->
         request(get, uri(["telemetry", "status"])),
 
     ?assertEqual(
-        #{<<"enable">> => true},
+        #{<<"enable">> => Default},
         emqx_utils_json:decode(Result1)
     ),
+    ?assertEqual(Default, is_telemetry_process_enabled()),
 
     ?assertMatch(
         {ok, 400, _},
         request(
             put,
             uri(["telemetry", "status"]),
-            #{<<"enable">> => true}
+            #{<<"enable">> => Default}
         )
-    ).
-
-t_status_non_official(_) ->
-    ?assertMatch(
-        {ok, 200, _},
-        request(
-            put,
-            uri(["telemetry", "status"]),
-            #{<<"enable">> => false}
-        )
-    ).
+    ),
+    ?assertEqual(Default, is_telemetry_process_enabled()),
+    ok.
 
 t_data(_) ->
+    ?assert(is_telemetry_process_enabled()),
     {ok, 200, Result} =
         request(get, uri(["telemetry", "data"])),
 
@@ -191,3 +191,23 @@ t_data(_) ->
         request(get, uri(["telemetry", "data"])),
 
     ok.
+
+%% Support emqx:update_config([telemetry], Conf).
+t_conf_update(_) ->
+    Conf = emqx:get_raw_config([telemetry]),
+    ?assert(is_telemetry_process_enabled()),
+    {ok, 200, Result1} = request(get, uri(["telemetry", "status"])),
+    ?assertEqual(#{<<"enable">> => true}, emqx_utils_json:decode(Result1)),
+    {ok, _} = emqx:update_config([telemetry], Conf#{<<"enable">> => false}),
+    {ok, 200, Result2} = request(get, uri(["telemetry", "status"])),
+    ?assertEqual(#{<<"enable">> => false}, emqx_utils_json:decode(Result2)),
+    ?assertNot(is_telemetry_process_enabled()),
+    %% reset to true
+    {ok, _} = emqx:update_config([telemetry], Conf#{<<"enable">> => true}),
+    ?assert(is_telemetry_process_enabled()),
+    ok.
+
+is_telemetry_process_enabled() ->
+    %% timer is not undefined.
+    Timer = element(6, sys:get_state(emqx_telemetry)),
+    is_reference(Timer).

apps/emqx_utils/src/emqx_utils.app.src

@@ -2,7 +2,7 @@
 {application, emqx_utils, [
     {description, "Miscellaneous utilities for EMQX apps"},
     % strict semver, bump manually!
-    {vsn, "5.0.1"},
+    {vsn, "5.0.2"},
     {modules, [
         emqx_utils,
         emqx_utils_api,

apps/emqx_utils/src/emqx_utils.erl

@@ -575,6 +575,9 @@ try_to_existing_atom(Convert, Data, Encoding) ->
 is_sensitive_key(token) -> true;
 is_sensitive_key("token") -> true;
 is_sensitive_key(<<"token">>) -> true;
+is_sensitive_key(authorization) -> true;
+is_sensitive_key("authorization") -> true;
+is_sensitive_key(<<"authorization">>) -> true;
 is_sensitive_key(password) -> true;
 is_sensitive_key("password") -> true;
 is_sensitive_key(<<"password">>) -> true;

changes/ce/fix-10820.en.md

@@ -0,0 +1,6 @@
+In case the cluster updated license before the new node join in. The new node will not apply the updated license.
+After this change, the new joined node will use the cluster's license key.
+
+Sometimes the new node must start with a outdated license.
+e.g. use emqx-operator deployed and needed to scale up after license expired.
+At the time the cluster's license key already updated by API/CLI, but the new node won't use it.

changes/ce/fix-10833.en.md

@@ -0,0 +1 @@
+Only include enabled authenticators and authorizers in telemetry report, not all of them.

changes/ce/fix-10851.en.md

@@ -0,0 +1 @@
+Obfuscated sensitive data in the bad API logging.