Merge branch 'master' into sync-master-to-release-54

.gitignore

@@ -68,3 +68,4 @@ lux_logs/
 bom.json
 ct_run*/
 apps/emqx_conf/etc/emqx.conf.all.rendered*
+rebar-git-cache.tar

apps/emqx_auth/src/emqx_authn/emqx_authn_api.erl

@@ -147,7 +147,7 @@ schema("/authentication") ->
             description => ?DESC(authentication_get),
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_example(
-                    hoconsc:array(emqx_authn_schema:authenticator_type()),
+                    hoconsc:array(authenticator_type(config)),
                     authenticator_array_example()
                 )
             }
@@ -156,12 +156,12 @@ schema("/authentication") ->
             tags => ?API_TAGS_GLOBAL,
             description => ?DESC(authentication_post),
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
-                emqx_authn_schema:authenticator_type(),
+                authenticator_type(api_write),
                 authenticator_examples()
             ),
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_examples(
-                    emqx_authn_schema:authenticator_type(),
+                    authenticator_type(config),
                     authenticator_examples()
                 ),
                 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>),
@@ -178,7 +178,7 @@ schema("/authentication/:id") ->
             parameters => [param_auth_id()],
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_examples(
-                    emqx_authn_schema:authenticator_type(),
+                    authenticator_type(config),
                     authenticator_examples()
                 ),
                 404 => error_codes([?NOT_FOUND], <<"Not Found">>)
@@ -189,7 +189,7 @@ schema("/authentication/:id") ->
             description => ?DESC(authentication_id_put),
             parameters => [param_auth_id()],
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
-                emqx_authn_schema:authenticator_type(),
+                authenticator_type(api_write),
                 authenticator_examples()
             ),
             responses => #{
@@ -236,7 +236,7 @@ schema("/listeners/:listener_id/authentication") ->
             parameters => [param_listener_id()],
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_example(
-                    hoconsc:array(emqx_authn_schema:authenticator_type()),
+                    hoconsc:array(authenticator_type(config)),
                     authenticator_array_example()
                 )
             }
@@ -247,12 +247,12 @@ schema("/listeners/:listener_id/authentication") ->
             description => ?DESC(listeners_listener_id_authentication_post),
             parameters => [param_listener_id()],
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
-                emqx_authn_schema:authenticator_type(),
+                authenticator_type(api_write),
                 authenticator_examples()
             ),
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_examples(
-                    emqx_authn_schema:authenticator_type(),
+                    authenticator_type(config),
                     authenticator_examples()
                 ),
                 400 => error_codes([?BAD_REQUEST], <<"Bad Request">>),
@@ -270,7 +270,7 @@ schema("/listeners/:listener_id/authentication/:id") ->
             parameters => [param_listener_id(), param_auth_id()],
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_examples(
-                    emqx_authn_schema:authenticator_type(),
+                    authenticator_type(config),
                     authenticator_examples()
                 ),
                 404 => error_codes([?NOT_FOUND], <<"Not Found">>)
@@ -282,7 +282,7 @@ schema("/listeners/:listener_id/authentication/:id") ->
             description => ?DESC(listeners_listener_id_authentication_id_put),
             parameters => [param_listener_id(), param_auth_id()],
             'requestBody' => emqx_dashboard_swagger:schema_with_examples(
-                emqx_authn_schema:authenticator_type(),
+                authenticator_type(api_write),
                 authenticator_examples()
             ),
             responses => #{
@@ -1278,6 +1278,9 @@ paginated_list_type(Type) ->
         {meta, ref(emqx_dashboard_swagger, meta)}
     ].
 
+authenticator_type(Kind) ->
+    emqx_authn_schema:authenticator_type(Kind).
+
 authenticator_array_example() ->
     [Config || #{value := Config} <- maps:values(authenticator_examples())].
 

apps/emqx_auth/src/emqx_authn/emqx_authn_password_hashing.erl

@@ -53,7 +53,8 @@
 
 -export([
     type_ro/1,
-    type_rw/1
+    type_rw/1,
+    type_rw_api/1
 ]).
 
 -export([
@@ -67,21 +68,17 @@
 -define(SALT_ROUNDS_MAX, 10).
 
 namespace() -> "authn-hash".
-roots() -> [pbkdf2, bcrypt, bcrypt_rw, simple].
+roots() -> [pbkdf2, bcrypt, bcrypt_rw, bcrypt_rw_api, simple].
 
 fields(bcrypt_rw) ->
     fields(bcrypt) ++
         [
-            {salt_rounds,
-                sc(
-                    range(?SALT_ROUNDS_MIN, ?SALT_ROUNDS_MAX),
-                    #{
-                        default => ?SALT_ROUNDS_MAX,
-                        example => ?SALT_ROUNDS_MAX,
-                        desc => "Work factor for BCRYPT password generation.",
-                        converter => fun salt_rounds_converter/2
-                    }
-                )}
+            {salt_rounds, fun bcrypt_salt_rounds/1}
+        ];
+fields(bcrypt_rw_api) ->
+    fields(bcrypt) ++
+        [
+            {salt_rounds, fun bcrypt_salt_rounds_api/1}
         ];
 fields(bcrypt) ->
     [{name, sc(bcrypt, #{required => true, desc => "BCRYPT password hashing."})}];
@@ -110,6 +107,15 @@ fields(simple) ->
         {salt_position, fun salt_position/1}
     ].
 
+bcrypt_salt_rounds(converter) -> fun salt_rounds_converter/2;
+bcrypt_salt_rounds(Option) -> bcrypt_salt_rounds_api(Option).
+
+bcrypt_salt_rounds_api(type) -> range(?SALT_ROUNDS_MIN, ?SALT_ROUNDS_MAX);
+bcrypt_salt_rounds_api(default) -> ?SALT_ROUNDS_MAX;
+bcrypt_salt_rounds_api(example) -> ?SALT_ROUNDS_MAX;
+bcrypt_salt_rounds_api(desc) -> "Work factor for BCRYPT password generation.";
+bcrypt_salt_rounds_api(_) -> undefined.
+
 salt_rounds_converter(undefined, _) ->
     undefined;
 salt_rounds_converter(I, _) when is_integer(I) ->
@@ -119,6 +125,8 @@ salt_rounds_converter(X, _) ->
 
 desc(bcrypt_rw) ->
     "Settings for bcrypt password hashing algorithm (for DB backends with write capability).";
+desc(bcrypt_rw_api) ->
+    desc(bcrypt_rw);
 desc(bcrypt) ->
     "Settings for bcrypt password hashing algorithm.";
 desc(pbkdf2) ->
@@ -143,14 +151,20 @@ dk_length(desc) ->
 dk_length(_) ->
     undefined.
 
-%% for simple_authn/emqx_authn_mnesia
+%% for emqx_authn_mnesia
 type_rw(type) ->
     hoconsc:union(rw_refs());
-type_rw(default) ->
-    #{<<"name">> => sha256, <<"salt_position">> => prefix};
 type_rw(desc) ->
     "Options for password hash creation and verification.";
-type_rw(_) ->
+type_rw(Option) ->
+    type_ro(Option).
+
+%% for emqx_authn_mnesia API
+type_rw_api(type) ->
+    hoconsc:union(api_refs());
+type_rw_api(desc) ->
+    "Options for password hash creation and verification through API.";
+type_rw_api(_) ->
     undefined.
 
 %% for other authn resources
@@ -242,31 +256,41 @@ check_password(#{name := Other, salt_position := SaltPosition}, Salt, PasswordHa
 %%------------------------------------------------------------------------------
 
 rw_refs() ->
-    All = [
-        hoconsc:ref(?MODULE, bcrypt_rw),
-        hoconsc:ref(?MODULE, pbkdf2),
-        hoconsc:ref(?MODULE, simple)
-    ],
-    fun
-        (all_union_members) -> All;
-        ({value, #{<<"name">> := <<"bcrypt">>}}) -> [hoconsc:ref(?MODULE, bcrypt_rw)];
-        ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [hoconsc:ref(?MODULE, pbkdf2)];
-        ({value, #{<<"name">> := _}}) -> [hoconsc:ref(?MODULE, simple)];
-        ({value, _}) -> throw(#{reason => "algorithm_name_missing"})
-    end.
+    union_selector(rw).
 
 ro_refs() ->
-    All = [
-        hoconsc:ref(?MODULE, bcrypt),
-        hoconsc:ref(?MODULE, pbkdf2),
-        hoconsc:ref(?MODULE, simple)
-    ],
+    union_selector(ro).
+
+api_refs() ->
+    union_selector(api).
+
+sc(Type, Meta) -> hoconsc:mk(Type, Meta).
+
+union_selector(Kind) ->
     fun
-        (all_union_members) -> All;
-        ({value, #{<<"name">> := <<"bcrypt">>}}) -> [hoconsc:ref(?MODULE, bcrypt)];
-        ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [hoconsc:ref(?MODULE, pbkdf2)];
-        ({value, #{<<"name">> := _}}) -> [hoconsc:ref(?MODULE, simple)];
+        (all_union_members) -> refs(Kind);
+        ({value, #{<<"name">> := <<"bcrypt">>}}) -> [bcrypt_ref(Kind)];
+        ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [pbkdf2_ref(Kind)];
+        ({value, #{<<"name">> := _}}) -> [simple_ref(Kind)];
         ({value, _}) -> throw(#{reason => "algorithm_name_missing"})
     end.
 
-sc(Type, Meta) -> hoconsc:mk(Type, Meta).
+refs(Kind) ->
+    [
+        bcrypt_ref(Kind),
+        pbkdf2_ref(Kind),
+        simple_ref(Kind)
+    ].
+
+pbkdf2_ref(_) ->
+    hoconsc:ref(?MODULE, pbkdf2).
+
+bcrypt_ref(rw) ->
+    hoconsc:ref(?MODULE, bcrypt_rw);
+bcrypt_ref(api) ->
+    hoconsc:ref(?MODULE, bcrypt_rw_api);
+bcrypt_ref(_) ->
+    hoconsc:ref(?MODULE, bcrypt).
+
+simple_ref(_) ->
+    hoconsc:ref(?MODULE, simple).

apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl

@@ -34,7 +34,9 @@
     tags/0,
     fields/1,
     authenticator_type/0,
+    authenticator_type/1,
     authenticator_type_without/1,
+    authenticator_type_without/2,
     mechanism/1,
     backend/1
 ]).
@@ -43,17 +45,35 @@
     global_auth_fields/0
 ]).
 
+-export_type([shema_kind/0]).
+
 -define(AUTHN_MODS_PT_KEY, {?MODULE, authn_schema_mods}).
+-define(DEFAULT_SCHEMA_KIND, config).
 
 %%--------------------------------------------------------------------
 %% Authn Source Schema Behaviour
 %%--------------------------------------------------------------------
 
 -type schema_ref() :: ?R_REF(module(), hocon_schema:name()).
+-type shema_kind() ::
+    %% api_write: schema for mutating API request validation
+    api_write
+    %% config: schema for config validation
+    | config.
 -callback refs() -> [schema_ref()].
--callback select_union_member(emqx_config:raw_config()) -> schema_ref() | undefined | no_return().
+-callback refs(shema_kind()) -> [schema_ref()].
+-callback select_union_member(emqx_config:raw_config()) -> [schema_ref()] | undefined | no_return().
+-callback select_union_member(shema_kind(), emqx_config:raw_config()) ->
+    [schema_ref()] | undefined | no_return().
 -callback fields(hocon_schema:name()) -> [hocon_schema:field()].
 
+-optional_callbacks([
+    select_union_member/1,
+    select_union_member/2,
+    refs/0,
+    refs/1
+]).
+
 roots() -> [].
 
 injected_fields(AuthnSchemaMods) ->
@@ -67,45 +87,63 @@ tags() ->
     [<<"Authentication">>].
 
 authenticator_type() ->
-    hoconsc:union(union_member_selector(provider_schema_mods())).
+    authenticator_type(?DEFAULT_SCHEMA_KIND).
+
+authenticator_type(Kind) ->
+    hoconsc:union(union_member_selector(Kind, provider_schema_mods())).
 
 authenticator_type_without(ProviderSchemaMods) ->
+    authenticator_type_without(?DEFAULT_SCHEMA_KIND, ProviderSchemaMods).
+
+authenticator_type_without(Kind, ProviderSchemaMods) ->
     hoconsc:union(
-        union_member_selector(provider_schema_mods() -- ProviderSchemaMods)
+        union_member_selector(Kind, provider_schema_mods() -- ProviderSchemaMods)
     ).
 
-union_member_selector(Mods) ->
-    AllTypes = config_refs(Mods),
+union_member_selector(Kind, Mods) ->
+    AllTypes = config_refs(Kind, Mods),
     fun
         (all_union_members) -> AllTypes;
-        ({value, Value}) -> select_union_member(Value, Mods)
+        ({value, Value}) -> select_union_member(Kind, Value, Mods)
     end.
 
-select_union_member(#{<<"mechanism">> := Mechanism, <<"backend">> := Backend}, []) ->
+select_union_member(_Kind, #{<<"mechanism">> := Mechanism, <<"backend">> := Backend}, []) ->
     throw(#{
         reason => "unsupported_mechanism",
         mechanism => Mechanism,
         backend => Backend
     });
-select_union_member(#{<<"mechanism">> := Mechanism}, []) ->
+select_union_member(_Kind, #{<<"mechanism">> := Mechanism}, []) ->
     throw(#{
         reason => "unsupported_mechanism",
         mechanism => Mechanism
     });
-select_union_member(#{<<"mechanism">> := _} = Value, [Mod | Mods]) ->
-    case Mod:select_union_member(Value) of
+select_union_member(Kind, #{<<"mechanism">> := _} = Value, [Mod | Mods]) ->
+    case mod_select_union_member(Kind, Value, Mod) of
         undefined ->
-            select_union_member(Value, Mods);
+            select_union_member(Kind, Value, Mods);
         Member ->
             Member
     end;
-select_union_member(#{} = _Value, _Mods) ->
+select_union_member(_Kind, #{} = _Value, _Mods) ->
     throw(#{reason => "missing_mechanism_field"});
-select_union_member(Value, _Mods) ->
+select_union_member(_Kind, Value, _Mods) ->
     throw(#{reason => "not_a_struct", value => Value}).
 
-config_refs(Mods) ->
-    lists:append([Mod:refs() || Mod <- Mods]).
+mod_select_union_member(Kind, Value, Mod) ->
+    emqx_utils:call_first_defined([
+        {Mod, select_union_member, [Kind, Value]},
+        {Mod, select_union_member, [Value]}
+    ]).
+
+config_refs(Kind, Mods) ->
+    lists:append([mod_refs(Kind, Mod) || Mod <- Mods]).
+
+mod_refs(Kind, Mod) ->
+    emqx_utils:call_first_defined([
+        {Mod, refs, [Kind]},
+        {Mod, refs, []}
+    ]).
 
 root_type() ->
     hoconsc:array(authenticator_type()).

apps/emqx_auth/test/emqx_authn/emqx_authn_api_SUITE.erl

@@ -63,14 +63,16 @@ end_per_testcase(_, Config) ->
 init_per_suite(Config) ->
     Apps = emqx_cth_suite:start(
         [
-            emqx,
             emqx_conf,
+            emqx,
             emqx_auth,
+            %% to load schema
+            {emqx_auth_mnesia, #{start => false}},
             emqx_management,
             {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}
         ],
         #{
-            work_dir => ?config(priv_dir, Config)
+            work_dir => filename:join(?config(priv_dir, Config), ?MODULE)
         }
     ),
     _ = emqx_common_test_http:create_default_app(),
@@ -535,6 +537,36 @@ ignore_switch_to_global_chain(_) ->
     ),
     ok = emqtt:disconnect(Client4).
 
+t_bcrypt_validation(_Config) ->
+    BaseConf = #{
+        mechanism => <<"password_based">>,
+        backend => <<"built_in_database">>,
+        user_id_type => <<"username">>
+    },
+    BcryptValid = #{
+        name => <<"bcrypt">>,
+        salt_rounds => 10
+    },
+    BcryptInvalid = #{
+        name => <<"bcrypt">>,
+        salt_rounds => 15
+    },
+
+    ConfValid = BaseConf#{password_hash_algorithm => BcryptValid},
+    ConfInvalid = BaseConf#{password_hash_algorithm => BcryptInvalid},
+
+    {ok, 400, _} = request(
+        post,
+        uri([?CONF_NS]),
+        ConfInvalid
+    ),
+
+    {ok, 200, _} = request(
+        post,
+        uri([?CONF_NS]),
+        ConfValid
+    ).
+
 %%------------------------------------------------------------------------------
 %% Helpers
 %%------------------------------------------------------------------------------

apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl

@@ -70,6 +70,7 @@ init_per_testcase(TestCase, Config) when
     {ok, _} = emqx:update_config([authorization, deny_action], disconnect),
     Config;
 init_per_testcase(_TestCase, Config) ->
+    _ = file:delete(emqx_authz_file:acl_conf_file()),
     {ok, _} = emqx_authz:update(?CMD_REPLACE, []),
     Config.
 

apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl

@@ -24,27 +24,30 @@
 -export([
     fields/1,
     desc/1,
-    refs/0,
-    select_union_member/1
+    refs/1,
+    select_union_member/2
 ]).
 
-refs() ->
+refs(api_write) ->
+    [?R_REF(builtin_db_api)];
+refs(_) ->
     [?R_REF(builtin_db)].
 
-select_union_member(#{
+select_union_member(Kind, #{
     <<"mechanism">> := ?AUTHN_MECHANISM_SIMPLE_BIN, <<"backend">> := ?AUTHN_BACKEND_BIN
 }) ->
-    refs();
-select_union_member(_) ->
+    refs(Kind);
+select_union_member(_Kind, _Value) ->
     undefined.
 
 fields(builtin_db) ->
     [
-        {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SIMPLE)},
-        {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)},
-        {user_id_type, fun user_id_type/1},
         {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1}
-    ] ++ emqx_authn_schema:common_fields().
+    ] ++ common_fields();
+fields(builtin_db_api) ->
+    [
+        {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw_api/1}
+    ] ++ common_fields().
 
 desc(builtin_db) ->
     ?DESC(builtin_db);
@@ -56,3 +59,10 @@ user_id_type(desc) -> ?DESC(?FUNCTION_NAME);
 user_id_type(default) -> <<"username">>;
 user_id_type(required) -> true;
 user_id_type(_) -> undefined.
+
+common_fields() ->
+    [
+        {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM_SIMPLE)},
+        {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)},
+        {user_id_type, fun user_id_type/1}
+    ] ++ emqx_authn_schema:common_fields().

apps/emqx_auth_redis/src/emqx_authn_redis.erl

@@ -18,100 +18,16 @@
 
 -include_lib("emqx_auth/include/emqx_authn.hrl").
 -include_lib("emqx/include/logger.hrl").
--include_lib("hocon/include/hoconsc.hrl").
 
--behaviour(hocon_schema).
 -behaviour(emqx_authn_provider).
 
 -export([
-    namespace/0,
-    tags/0,
-    roots/0,
-    fields/1,
-    desc/1
-]).
-
--export([
-    refs/0,
-    union_member_selector/1,
     create/2,
     update/2,
     authenticate/2,
     destroy/1
 ]).
 
-%%------------------------------------------------------------------------------
-%% Hocon Schema
-%%------------------------------------------------------------------------------
-
-namespace() -> "authn".
-
-tags() ->
-    [<<"Authentication">>].
-
-%% used for config check when the schema module is resolved
-roots() ->
-    [
-        {?CONF_NS,
-            hoconsc:mk(
-                hoconsc:union(fun ?MODULE:union_member_selector/1),
-                #{}
-            )}
-    ].
-
-fields(redis_single) ->
-    common_fields() ++ emqx_redis:fields(single);
-fields(redis_cluster) ->
-    common_fields() ++ emqx_redis:fields(cluster);
-fields(redis_sentinel) ->
-    common_fields() ++ emqx_redis:fields(sentinel).
-
-desc(redis_single) ->
-    ?DESC(single);
-desc(redis_cluster) ->
-    ?DESC(cluster);
-desc(redis_sentinel) ->
-    ?DESC(sentinel);
-desc(_) ->
-    "".
-
-common_fields() ->
-    [
-        {mechanism, emqx_authn_schema:mechanism(password_based)},
-        {backend, emqx_authn_schema:backend(redis)},
-        {cmd, fun cmd/1},
-        {password_hash_algorithm, fun emqx_authn_password_hashing:type_ro/1}
-    ] ++ emqx_authn_schema:common_fields().
-
-cmd(type) -> string();
-cmd(desc) -> ?DESC(?FUNCTION_NAME);
-cmd(required) -> true;
-cmd(_) -> undefined.
-
-refs() ->
-    [
-        hoconsc:ref(?MODULE, redis_single),
-        hoconsc:ref(?MODULE, redis_cluster),
-        hoconsc:ref(?MODULE, redis_sentinel)
-    ].
-
-union_member_selector(all_union_members) ->
-    refs();
-union_member_selector({value, Value}) ->
-    refs(Value).
-
-refs(#{<<"redis_type">> := <<"single">>}) ->
-    [hoconsc:ref(?MODULE, redis_single)];
-refs(#{<<"redis_type">> := <<"cluster">>}) ->
-    [hoconsc:ref(?MODULE, redis_cluster)];
-refs(#{<<"redis_type">> := <<"sentinel">>}) ->
-    [hoconsc:ref(?MODULE, redis_sentinel)];
-refs(_) ->
-    throw(#{
-        field_name => redis_type,
-        expected => "single | cluster | sentinel"
-    }).
-
 %%------------------------------------------------------------------------------
 %% APIs
 %%------------------------------------------------------------------------------

apps/emqx_dashboard/src/emqx_dashboard_token.erl

@@ -248,8 +248,8 @@ clean_expired_jwt(Now) ->
 
 -if(?EMQX_RELEASE_EDITION == ee).
 check_rbac(Req, JWT) ->
-    #?ADMIN_JWT{exptime = _ExpTime, extra = Extra, username = _Username} = JWT,
-    case emqx_dashboard_rbac:check_rbac(Req, Extra) of
+    #?ADMIN_JWT{exptime = _ExpTime, extra = Extra, username = Username} = JWT,
+    case emqx_dashboard_rbac:check_rbac(Req, Username, Extra) of
         true ->
             save_new_jwt(JWT);
         _ ->

apps/emqx_dashboard_rbac/src/emqx_dashboard_rbac.erl

@@ -7,7 +7,7 @@
 -include_lib("emqx_dashboard/include/emqx_dashboard.hrl").
 
 -export([
-    check_rbac/2,
+    check_rbac/3,
     role/1,
     valid_dashboard_role/1,
     valid_api_role/1
@@ -16,13 +16,13 @@
 -dialyzer({nowarn_function, role/1}).
 %%=====================================================================
 %% API
-check_rbac(Req, Extra) ->
+check_rbac(Req, Username, Extra) ->
     Role = role(Extra),
     Method = cowboy_req:method(Req),
     AbsPath = cowboy_req:path(Req),
     case emqx_dashboard_swagger:get_relative_uri(AbsPath) of
         {ok, Path} ->
-            check_rbac(Role, Method, Path);
+            check_rbac(Role, Method, Path, Username);
         _ ->
             false
     end.
@@ -47,23 +47,6 @@ valid_api_role(Role) ->
     valid_role(api, Role).
 
 %% ===================================================================
-check_rbac(?ROLE_SUPERUSER, _, _) ->
-    true;
-check_rbac(?ROLE_API_SUPERUSER, _, _) ->
-    true;
-check_rbac(?ROLE_VIEWER, <<"GET">>, _) ->
-    true;
-check_rbac(?ROLE_API_VIEWER, <<"GET">>, _) ->
-    true;
-%% this API is a special case
-check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>) ->
-    true;
-check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish">>) ->
-    true;
-check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish/bulk">>) ->
-    true;
-check_rbac(_, _, _) ->
-    false.
 
 valid_role(Type, Role) ->
     case lists:member(Role, role_list(Type)) of
@@ -73,6 +56,32 @@ valid_role(Type, Role) ->
             {error, <<"Role does not exist">>}
     end.
 
+%% ===================================================================
+check_rbac(?ROLE_SUPERUSER, _, _, _) ->
+    true;
+check_rbac(?ROLE_API_SUPERUSER, _, _, _) ->
+    true;
+check_rbac(?ROLE_VIEWER, <<"GET">>, _, _) ->
+    true;
+check_rbac(?ROLE_API_VIEWER, <<"GET">>, _, _) ->
+    true;
+check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish">>, _) ->
+    true;
+check_rbac(?ROLE_API_PUBLISHER, <<"POST">>, <<"/publish/bulk">>, _) ->
+    true;
+%% everyone should allow to logout
+check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/logout">>, _) ->
+    true;
+%% viewer should allow to change self password,
+%% superuser should allow to change any user
+check_rbac(?ROLE_VIEWER, <<"POST">>, <<"/users/", SubPath/binary>>, Username) ->
+    case binary:split(SubPath, <<"/">>, [global]) of
+        [Username, <<"change_pwd">>] -> true;
+        _ -> false
+    end;
+check_rbac(_, _, _, _) ->
+    false.
+
 role_list(dashboard) ->
     [?ROLE_VIEWER, ?ROLE_SUPERUSER];
 role_list(api) ->

apps/emqx_dashboard_rbac/test/emqx_dashboard_rbac_SUITE.erl

@@ -160,6 +160,34 @@ t_login_out(_) ->
     {ok, Username} = emqx_dashboard_admin:verify_token(FakeReq, Token),
     ok.
 
+t_change_pwd(_) ->
+    Viewer1 = <<"viewer1">>,
+    Viewer2 = <<"viewer2">>,
+    SuperUser = <<"super_user">>,
+    Password = <<"public_www1">>,
+    Desc = <<"desc">>,
+    {ok, _} = emqx_dashboard_admin:add_user(Viewer1, Password, ?ROLE_VIEWER, Desc),
+    {ok, _} = emqx_dashboard_admin:add_user(Viewer2, Password, ?ROLE_VIEWER, Desc),
+    {ok, _} = emqx_dashboard_admin:add_user(SuperUser, Password, ?ROLE_SUPERUSER, Desc),
+    {ok, ?ROLE_VIEWER, Viewer1Token} = emqx_dashboard_admin:sign_token(Viewer1, Password),
+    {ok, ?ROLE_SUPERUSER, SuperToken} = emqx_dashboard_admin:sign_token(SuperUser, Password),
+    %% viewer can change own password
+    ?assertEqual({ok, Viewer1}, change_pwd(Viewer1Token, Viewer1)),
+    %% viewer can't change other's password
+    ?assertEqual({error, unauthorized_role}, change_pwd(Viewer1Token, Viewer2)),
+    ?assertEqual({error, unauthorized_role}, change_pwd(Viewer1Token, SuperUser)),
+    %% superuser can change other's password
+    ?assertEqual({ok, SuperUser}, change_pwd(SuperToken, Viewer1)),
+    ?assertEqual({ok, SuperUser}, change_pwd(SuperToken, Viewer2)),
+    ?assertEqual({ok, SuperUser}, change_pwd(SuperToken, SuperUser)),
+    ok.
+
+change_pwd(Token, Username) ->
+    Path = "/users/" ++ binary_to_list(Username) ++ "/change_pwd",
+    Path1 = erlang:list_to_binary(emqx_dashboard_swagger:relative_uri(Path)),
+    Req = #{method => <<"POST">>, path => Path1},
+    emqx_dashboard_admin:verify_token(Req, Token).
+
 add_default_superuser() ->
     {ok, _NewUser} = emqx_dashboard_admin:add_user(
         ?DEFAULT_SUPERUSER,

apps/emqx_gateway_coap/src/emqx_coap_channel.erl

@@ -86,7 +86,6 @@
 -define(INFO_KEYS, [conninfo, conn_state, clientinfo, session]).
 
 -define(DEF_IDLE_TIME, timer:seconds(30)).
--define(GET_IDLE_TIME(Cfg), maps:get(idle_timeout, Cfg, ?DEF_IDLE_TIME)).
 
 -import(emqx_coap_medium, [reply/2, reply/3, reply/4, iter/3, iter/4]).
 
@@ -150,8 +149,7 @@ init(
             mountpoint => Mountpoint
         }
     ),
-    %% FIXME: it should coap.hearbeat instead of idle_timeout?
-    Heartbeat = ?GET_IDLE_TIME(Config),
+    Heartbeat = maps:get(heartbeat, Config, ?DEF_IDLE_TIME),
     #channel{
         ctx = Ctx,
         conninfo = ConnInfo,
@@ -179,8 +177,8 @@ send_request(Channel, Request) ->
     | {ok, replies(), channel()}
     | {shutdown, Reason :: term(), channel()}
     | {shutdown, Reason :: term(), replies(), channel()}.
-handle_in(Msg, ChannleT) ->
-    Channel = ensure_keepalive_timer(ChannleT),
+handle_in(Msg, Channel0) ->
+    Channel = ensure_keepalive_timer(Channel0),
     case emqx_coap_message:is_request(Msg) of
         true ->
             check_auth_state(Msg, Channel);
@@ -321,6 +319,9 @@ handle_call(Req, _From, Channel) ->
 handle_cast(close, Channel) ->
     ?SLOG(info, #{msg => "close_connection"}),
     shutdown(normal, Channel);
+handle_cast(inc_recv_pkt, Channel) ->
+    _ = emqx_pd:inc_counter(recv_pkt, 1),
+    {ok, Channel};
 handle_cast(Req, Channel) ->
     ?SLOG(error, #{msg => "unexpected_cast", cast => Req}),
     {ok, Channel}.
@@ -455,6 +456,13 @@ check_token(
                     Reply = emqx_coap_message:piggyback({error, unauthorized}, Msg),
                     {shutdown, normal, Reply, Channel};
                 true ->
+                    %% hack: since each message request can spawn a new connection
+                    %% process, we can't rely on the `inc_incoming_stats' call in
+                    %% `emqx_gateway_conn:handle_incoming' to properly keep track of
+                    %% bumping incoming requests for an existing channel.  Since this
+                    %% number is used by keepalive, we have to bump it inside the
+                    %% requested channel/connection pid so heartbeats actually work.
+                    emqx_gateway_cm:cast(coap, ReqClientId, inc_recv_pkt),
                     call_session(handle_request, Msg, Channel)
             end;
         _ ->

apps/emqx_gateway_coap/test/emqx_coap_SUITE.erl

@@ -83,10 +83,26 @@ init_per_testcase(t_connection_with_authn_failed, Config) ->
         fun(_) -> {error, bad_username_or_password} end
     ),
     Config;
+init_per_testcase(t_heartbeat, Config) ->
+    NewHeartbeat = 800,
+    OldConf = emqx:get_raw_config([gateway, coap]),
+    {ok, _} = emqx_gateway_conf:update_gateway(
+        coap,
+        OldConf#{<<"heartbeat">> => <<"800ms">>}
+    ),
+    [
+        {old_conf, OldConf},
+        {new_heartbeat, NewHeartbeat}
+        | Config
+    ];
 init_per_testcase(_, Config) ->
     ok = meck:new(emqx_access_control, [passthrough]),
     Config.
 
+end_per_testcase(t_heartbeat, Config) ->
+    OldConf = ?config(old_conf, Config),
+    {ok, _} = emqx_gateway_conf:update_gateway(coap, OldConf),
+    ok;
 end_per_testcase(_, Config) ->
     ok = meck:unload(emqx_access_control),
     Config.
@@ -123,13 +139,49 @@ t_connection(_) ->
         ),
 
         %% heartbeat
-        HeartURI =
-            ?MQTT_PREFIX ++
-                "/connection?clientid=client1&token=" ++
-                Token,
+        {ok, changed, _} = send_heartbeat(Token),
 
-        ?LOGT("send heartbeat request:~ts~n", [HeartURI]),
-        {ok, changed, _} = er_coap_client:request(put, HeartURI),
+        disconnection(Channel, Token),
+
+        timer:sleep(100),
+        ?assertEqual(
+            [],
+            emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>)
+        )
+    end,
+    do(Action).
+
+t_heartbeat(Config) ->
+    Heartbeat = ?config(new_heartbeat, Config),
+    Action = fun(Channel) ->
+        Token = connection(Channel),
+
+        timer:sleep(100),
+        ?assertNotEqual(
+            [],
+            emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>)
+        ),
+
+        %% must keep client connection alive
+        Delay = Heartbeat div 2,
+        lists:foreach(
+            fun(_) ->
+                ?assertMatch({ok, changed, _}, send_heartbeat(Token)),
+                timer:sleep(Delay)
+            end,
+            lists:seq(1, 5)
+        ),
+
+        ?assertNotEqual(
+            [],
+            emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>)
+        ),
+
+        timer:sleep(Heartbeat * 2),
+        ?assertEqual(
+            [],
+            emqx_gateway_cm_registry:lookup_channels(coap, <<"client1">>)
+        ),
 
         disconnection(Channel, Token),
 
@@ -491,6 +543,15 @@ t_connectionless_pubsub(_) ->
 %%--------------------------------------------------------------------
 %% helpers
 
+send_heartbeat(Token) ->
+    HeartURI =
+        ?MQTT_PREFIX ++
+            "/connection?clientid=client1&token=" ++
+            Token,
+
+    ?LOGT("send heartbeat request:~ts~n", [HeartURI]),
+    er_coap_client:request(put, HeartURI).
+
 connection(Channel) ->
     URI =
         ?MQTT_PREFIX ++

apps/emqx_utils/src/emqx_utils.erl

@@ -62,7 +62,8 @@
     merge_lists/3,
     tcp_keepalive_opts/4,
     format/1,
-    format_mfal/1
+    format_mfal/1,
+    call_first_defined/1
 ]).
 
 -export([
@@ -554,6 +555,22 @@ format_mfal(Data) ->
             undefined
     end.
 
+-spec call_first_defined(list({module(), atom(), list()})) -> term() | no_return().
+call_first_defined([{Module, Function, Args} | Rest]) ->
+    try
+        apply(Module, Function, Args)
+    catch
+        error:undef:Stacktrace ->
+            case Stacktrace of
+                [{Module, Function, _, _} | _] ->
+                    call_first_defined(Rest);
+                _ ->
+                    erlang:raise(error, undef, Stacktrace)
+            end
+    end;
+call_first_defined([]) ->
+    error(none_fun_is_defined).
+
 %%------------------------------------------------------------------------------
 %% Internal Functions
 %%------------------------------------------------------------------------------

bin/nodetool

@@ -21,12 +21,13 @@ main(Args) ->
                     ok
             end
     end,
-    ok = add_libs_dir(),
     case Args of
         ["hocon" | Rest] ->
+            ok = add_libs_dir(),
             %% forward the call to hocon_cli
             hocon_cli:main(Rest);
         ["check_license_key", Key0] ->
+            ok = add_libs_dir(),
             Key = cleanup_key(Key0),
             check_license(#{key => Key});
         _ ->

build

@@ -6,7 +6,11 @@
 
 set -euo pipefail
 
-[ "${DEBUG:-0}" -eq 1 ] && set -x
+if [ "${DEBUG:-0}" -eq 1 ]; then
+    set -x
+    # set this for rebar3
+    export DIAGNOSTIC=1
+fi
 
 PROFILE_ARG="$1"
 ARTIFACT="$2"
@@ -449,17 +453,33 @@ make_docker() {
     if [ "${DOCKER_PUSH:-false}" = true ]; then
         DOCKER_BUILDX_ARGS+=(--push)
     fi
+    if [ -d "${REBAR_GIT_CACHE_DIR:-}" ]; then
+        cache_tar="$(pwd)/rebar-git-cache.tar"
+        if [ ! -f "${cache_tar}" ]; then
+            pushd "${REBAR_GIT_CACHE_DIR}" >/dev/null
+            tar -cf "${cache_tar}" .
+            popd >/dev/null
+        fi
+    fi
+    if [ -n "${DEBUG:-}" ]; then
+        DOCKER_BUILDX_ARGS+=(--build-arg DEBUG="${DEBUG}" --progress=plain)
+    fi
+
     # shellcheck disable=SC2015
     [ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true
     trap docker_cleanup EXIT
     {
-        echo '/_build'
-        echo '/deps'
-        echo '/*.lock'
+        echo '_build/'
+        echo 'deps/'
+        echo '*.lock'
+        echo '_packages/'
+        echo '.vs/'
+        echo '.vscode/'
+        echo 'lux_logs/'
+        echo '_upgrade_base/'
     } >> ./.dockerignore
-    set -x
+    echo "Docker build args: ${DOCKER_BUILDX_ARGS[*]}"
     docker buildx build "${DOCKER_BUILDX_ARGS[@]}" .
-    [[ "${DEBUG:-}" -eq 1 ]] || set +x
     echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag
 }
 

changes/ce/feat-11785.en.md

@@ -0,0 +1 @@
+Allow viewer to change their own passwords, viewer can't change other's password.

changes/ce/feat-11787.en.md

@@ -0,0 +1,3 @@
+Improve `emqx` command performance.
+
+Avoid loading EMQX application code in `nodetool` script unless necessary.

changes/ce/fix-11771.en.md

@@ -0,0 +1 @@
+Fixed validation of Bcrypt salt rounds in authentification management through the API/Dashboard.

changes/ce/fix-11791.en.md

@@ -0,0 +1 @@
+Fixed an issue that prevented heartbeats from correctly keeping the CoAP Gateway connections alive.

deploy/docker/Dockerfile

@@ -1,23 +1,35 @@
 ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.1-4:1.14.5-25.3.2-2-debian11
 ARG RUN_FROM=public.ecr.aws/debian/debian:11-slim
 FROM ${BUILD_FROM} AS builder
+ARG DEBUG=0
 
 COPY . /emqx
 
 ARG EMQX_NAME=emqx
 ARG PKG_VSN
+
 ENV EMQX_RELUP=false
+ENV DEBUG=${DEBUG}
+ENV EMQX_REL_FORM='docker'
+
+WORKDIR /emqx/
+
+RUN git config --global --add safe.directory '*'
 
-RUN export PROFILE=${EMQX_NAME%%-elixir} \
-    && export EMQX_NAME1=$EMQX_NAME \
-    && export EMQX_NAME=$PROFILE \
-    && export EMQX_REL_PATH="/emqx/_build/$EMQX_NAME/rel/emqx" \
-    && export EMQX_REL_FORM='docker' \
-    && cd /emqx \
-    && make $EMQX_NAME1 \
-    && rm -f $EMQX_REL_PATH/*.tar.gz \
+RUN if [ -f rebar-git-cache.tar ]; then \
+        mkdir .cache && \
+        tar -xf rebar-git-cache.tar -C .cache && \
+        export REBAR_GIT_CACHE_DIR='/emqx/.cache' && \
+        export REBAR_GIT_CACHE_REF_AUTOFILL=0 ;\
+    fi \
+    && export PROFILE=${EMQX_NAME%%-elixir} \
+    && export EMQX_NAME1="${EMQX_NAME}" \
+    && export EMQX_NAME=${PROFILE} \
+    && export EMQX_REL_PATH="/emqx/_build/${EMQX_NAME}/rel/emqx" \
+    && make ${EMQX_NAME1} \
+    && rm -f ${EMQX_REL_PATH}/*.tar.gz \
     && mkdir -p /emqx-rel \
-    && mv $EMQX_REL_PATH /emqx-rel
+    && mv ${EMQX_REL_PATH} /emqx-rel
 
 FROM $RUN_FROM
 ARG EXTRA_DEPS=''

scripts/ensure-rebar3.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+[ "${DEBUG:-0}" -eq 1 ] && set -x
+
 ## rebar3 tag 3.19.0-emqx-1 is compiled using latest official OTP-24 image.
 ## we have to use an otp24-compiled rebar3 because the defination of record #application{}
 ## in systools.hrl is changed in otp24.
@@ -14,7 +16,7 @@ case ${OTP_VSN} in
         VERSION="3.18.0-emqx-1"
         ;;
     25*)
-        VERSION="3.19.0-emqx-8"
+        VERSION="3.19.0-emqx-9"
         ;;
     *)
         echo "Unsupporetd Erlang/OTP version $OTP_VSN"

scripts/pre-compile.sh

@@ -2,6 +2,8 @@
 
 set -euo pipefail
 
+[ "${DEBUG:-0}" -eq 1 ] && set -x
+
 # NOTE: PROFILE_STR may not be exactly PROFILE (emqx or emqx-enterprise)
 # it might be with suffix such as -pkg etc.
 PROFILE_STR="${1}"
@@ -28,5 +30,6 @@ curl -L --silent --show-error \
      --output "apps/emqx_dashboard/priv/desc.zh.hocon" \
     'https://raw.githubusercontent.com/emqx/emqx-i18n/main/desc.zh.hocon'
 
-# generate sbom
-./scripts/update-bom.sh "$PROFILE_STR" ./rel
+# TODO
+# make sbom a build artifcat
+# ./scripts/update-bom.sh "$PROFILE_STR" ./rel

scripts/update-bom.sh

@@ -8,4 +8,3 @@ PROFILE="$1"
 REL_DIR="$2"
 
 ./rebar3 as "$PROFILE" sbom -f -o "$REL_DIR/bom.json"
-