Merge pull request #9718 from zmstone/0108-refactor-authn-schema-union-member-selector

0108 refactor authn schema union member selector

apps/emqx/src/emqx_authentication.erl

@@ -759,9 +759,10 @@ maybe_unhook(State) ->
     State.
 
 do_create_authenticator(AuthenticatorID, #{enable := Enable} = Config, Providers) ->
-    case maps:get(authn_type(Config), Providers, undefined) of
+    Type = authn_type(Config),
+    case maps:get(Type, Providers, undefined) of
         undefined ->
-            {error, no_available_provider};
+            {error, {no_available_provider_for, Type}};
         Provider ->
             case Provider:create(AuthenticatorID, Config) of
                 {ok, State} ->

apps/emqx/src/emqx_authentication_config.erl

@@ -136,7 +136,7 @@ do_pre_config_update({move_authenticator, _ChainName, AuthenticatorID, Position}
 ) ->
     ok | {ok, map()} | {error, term()}.
 post_config_update(_, UpdateReq, NewConfig, OldConfig, AppEnvs) ->
-    do_post_config_update(UpdateReq, check_configs(to_list(NewConfig)), OldConfig, AppEnvs).
+    do_post_config_update(UpdateReq, to_list(NewConfig), OldConfig, AppEnvs).
 
 do_post_config_update({create_authenticator, ChainName, Config}, NewConfig, _OldConfig, _AppEnvs) ->
     NConfig = get_authenticator_config(authenticator_id(Config), NewConfig),
@@ -175,56 +175,6 @@ do_post_config_update(
 ) ->
     emqx_authentication:move_authenticator(ChainName, AuthenticatorID, Position).
 
-check_configs(Configs) ->
-    Providers = emqx_authentication:get_providers(),
-    lists:map(fun(C) -> do_check_config(C, Providers) end, Configs).
-
-do_check_config(Config, Providers) ->
-    Type = authn_type(Config),
-    case maps:get(Type, Providers, false) of
-        false ->
-            ?SLOG(warning, #{
-                msg => "unknown_authn_type",
-                type => Type,
-                providers => Providers
-            }),
-            throw({unknown_authn_type, Type});
-        Module ->
-            do_check_config(Type, Config, Module)
-    end.
-
-do_check_config(Type, Config, Module) ->
-    F =
-        case erlang:function_exported(Module, check_config, 1) of
-            true ->
-                fun Module:check_config/1;
-            false ->
-                fun(C) ->
-                    Key = list_to_binary(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME),
-                    AtomKey = list_to_atom(?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME),
-                    R = hocon_tconf:check_plain(
-                        Module,
-                        #{Key => C},
-                        #{atom_key => true}
-                    ),
-                    maps:get(AtomKey, R)
-                end
-        end,
-    try
-        F(Config)
-    catch
-        C:E:S ->
-            ?SLOG(warning, #{
-                msg => "failed_to_check_config",
-                config => Config,
-                type => Type,
-                exception => C,
-                reason => E,
-                stacktrace => S
-            }),
-            throw({bad_authenticator_config, #{type => Type, reason => E}})
-    end.
-
 to_list(undefined) -> [];
 to_list(M) when M =:= #{} -> [];
 to_list(M) when is_map(M) -> [M];

apps/emqx/src/emqx_config.erl

@@ -366,13 +366,6 @@ schema_default(Schema) ->
     case hocon_schema:field_schema(Schema, type) of
         ?ARRAY(_) ->
             [];
-        ?LAZY(?ARRAY(_)) ->
-            [];
-        ?LAZY(?UNION(Members)) ->
-            case [A || ?ARRAY(A) <- hoconsc:union_members(Members)] of
-                [_ | _] -> [];
-                _ -> #{}
-            end;
         _ ->
             #{}
     end.
@@ -407,8 +400,7 @@ merge_envs(SchemaMod, RawConf) ->
     Opts = #{
         required => false,
         format => map,
-        apply_override_envs => true,
-        check_lazy => true
+        apply_override_envs => true
     },
     hocon_tconf:merge_env_overrides(SchemaMod, RawConf, all, Opts).
 
@@ -421,39 +413,15 @@ check_config(SchemaMod, RawConf, Opts0) ->
     try
         do_check_config(SchemaMod, RawConf, Opts0)
     catch
-        throw:{Schema, Errors} ->
-            compact_errors(Schema, Errors)
+        throw:Errors:Stacktrace ->
+            {error, Reason} = emqx_hocon:compact_errors(Errors, Stacktrace),
+            erlang:raise(throw, Reason, Stacktrace)
     end.
 
-%% HOCON tries to be very informative about all the detailed errors
-%% it's maybe too much when reporting to the user
--spec compact_errors(any(), any()) -> no_return().
-compact_errors(Schema, [Error0 | More]) when is_map(Error0) ->
-    Error1 =
-        case length(More) of
-            0 ->
-                Error0;
-            _ ->
-                Error0#{unshown_errors => length(More)}
-        end,
-    Error =
-        case is_atom(Schema) of
-            true ->
-                Error1#{schema_module => Schema};
-            false ->
-                Error1
-        end,
-    throw(Error);
-compact_errors(Schema, Errors) ->
-    %% unexpected, we need the stacktrace reported, hence error
-    error({Schema, Errors}).
-
 do_check_config(SchemaMod, RawConf, Opts0) ->
     Opts1 = #{
         return_plain => true,
-        format => map,
-        %% Don't check lazy types, such as authenticate
-        check_lazy => false
+        format => map
     },
     Opts = maps:merge(Opts0, Opts1),
     {AppEnvs, CheckedConf} =

apps/emqx/src/emqx_hocon.erl

@@ -20,6 +20,8 @@
 -export([
     format_path/1,
     check/2,
+    check/3,
+    compact_errors/2,
     format_error/1,
     format_error/2,
     make_schema/1
@@ -36,20 +38,23 @@ format_path([Name | Rest]) -> [iol(Name), "." | format_path(Rest)].
 %% Always return plain map with atom keys.
 -spec check(module(), hocon:config() | iodata()) ->
     {ok, hocon:config()} | {error, any()}.
-check(SchemaModule, Conf) when is_map(Conf) ->
+check(SchemaModule, Conf) ->
     %% TODO: remove required
     %% fields should state required or not in their schema
     Opts = #{atom_key => true, required => false},
+    check(SchemaModule, Conf, Opts).
+
+check(SchemaModule, Conf, Opts) when is_map(Conf) ->
     try
         {ok, hocon_tconf:check_plain(SchemaModule, Conf, Opts)}
     catch
-        throw:Reason ->
-            {error, Reason}
+        throw:Errors:Stacktrace ->
+            compact_errors(Errors, Stacktrace)
     end;
-check(SchemaModule, HoconText) ->
+check(SchemaModule, HoconText, Opts) ->
     case hocon:binary(HoconText, #{format => map}) of
         {ok, MapConfig} ->
-            check(SchemaModule, MapConfig);
+            check(SchemaModule, MapConfig, Opts);
         {error, Reason} ->
             {error, Reason}
     end.
@@ -90,3 +95,34 @@ iol(L) when is_list(L) -> L.
 
 no_stacktrace(Map) ->
     maps:without([stacktrace], Map).
+
+%% @doc HOCON tries to be very informative about all the detailed errors
+%% it's maybe too much when reporting to the user
+-spec compact_errors(any(), Stacktrace :: list()) -> {error, any()}.
+compact_errors({SchemaModule, Errors}, Stacktrace) ->
+    compact_errors(SchemaModule, Errors, Stacktrace).
+
+compact_errors(SchemaModule, [Error0 | More], _Stacktrace) when is_map(Error0) ->
+    Error1 =
+        case length(More) of
+            0 ->
+                Error0;
+            N ->
+                Error0#{unshown_errors_count => N}
+        end,
+    Error =
+        case is_atom(SchemaModule) of
+            true ->
+                Error1#{schema_module => SchemaModule};
+            false ->
+                Error1
+        end,
+    {error, Error};
+compact_errors(SchemaModule, Error, Stacktrace) ->
+    %% unexpected, we need the stacktrace reported
+    %% if this happens it's a bug in hocon_tconf
+    {error, #{
+        schema_module => SchemaModule,
+        exception => Error,
+        stacktrace => Stacktrace
+    }}.

apps/emqx/src/emqx_schema.erl

@@ -2352,25 +2352,23 @@ authentication(Which) ->
             global -> ?DESC(global_authentication);
             listener -> ?DESC(listener_authentication)
         end,
-    %% The runtime module injection
-    %% from EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY
-    %% is for now only affecting document generation.
-    %% maybe in the future, we can find a more straightforward way to support
-    %% * document generation (at compile time)
-    %% * type checks before boot (in bin/emqx config generation)
-    %% * type checks at runtime (when changing configs via management API)
-    Type0 =
+    %% poor man's dependency injection
+    %% this is due to the fact that authn is implemented outside of 'emqx' app.
+    %% so it can not be a part of emqx_schema since 'emqx' app is supposed to
+    %% work standalone.
+    Type =
         case persistent_term:get(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY, undefined) of
-            undefined -> hoconsc:array(typerefl:map());
-            Module -> Module:root_type()
+            undefined ->
+                hoconsc:array(typerefl:map());
+            Module ->
+                Module:root_type()
         end,
-    %% It is a lazy type because when handling runtime update requests
-    %% the config is not checked by emqx_schema, but by the injected schema
-    Type = hoconsc:lazy(Type0),
-    #{
-        type => Type,
-        desc => Desc
-    }.
+    hoconsc:mk(Type, #{desc => Desc, converter => fun ensure_array/2}).
+
+%% the older version schema allows individual element (instead of a chain) in config
+ensure_array(undefined, _) -> undefined;
+ensure_array(L, _) when is_list(L) -> L;
+ensure_array(M, _) -> [M].
 
 -spec qos() -> typerefl:type().
 qos() ->

apps/emqx/test/emqx_authentication_SUITE.erl

@@ -52,50 +52,10 @@
     )
 ).
 
-%%------------------------------------------------------------------------------
-%% Hocon Schema
-%%------------------------------------------------------------------------------
-
-roots() ->
-    [
-        {config, #{
-            type => hoconsc:union([
-                hoconsc:ref(?MODULE, type1),
-                hoconsc:ref(?MODULE, type2)
-            ])
-        }}
-    ].
-
-fields(type1) ->
-    [
-        {mechanism, {enum, [password_based]}},
-        {backend, {enum, [built_in_database]}},
-        {enable, fun enable/1}
-    ];
-fields(type2) ->
-    [
-        {mechanism, {enum, [password_based]}},
-        {backend, {enum, [mysql]}},
-        {enable, fun enable/1}
-    ].
-
-enable(type) -> boolean();
-enable(default) -> true;
-enable(_) -> undefined.
-
 %%------------------------------------------------------------------------------
 %% Callbacks
 %%------------------------------------------------------------------------------
 
-check_config(C) ->
-    #{config := R} =
-        hocon_tconf:check_plain(
-            ?MODULE,
-            #{<<"config">> => C},
-            #{atom_key => true}
-        ),
-    R.
-
 create(_AuthenticatorID, _Config) ->
     {ok, #{mark => 1}}.
 
@@ -200,7 +160,7 @@ t_authenticator(Config) when is_list(Config) ->
     % Create an authenticator when the provider does not exist
 
     ?assertEqual(
-        {error, no_available_provider},
+        {error, {no_available_provider_for, {password_based, built_in_database}}},
         ?AUTHN:create_authenticator(ChainName, AuthenticatorConfig1)
     ),
 
@@ -335,14 +295,14 @@ t_update_config(Config) when is_list(Config) ->
     ok = register_provider(?config("auth2"), ?MODULE),
     Global = ?config(global),
     AuthenticatorConfig1 = #{
-        <<"mechanism">> => <<"password_based">>,
-        <<"backend">> => <<"built_in_database">>,
-        <<"enable">> => true
+        mechanism => password_based,
+        backend => built_in_database,
+        enable => true
     },
     AuthenticatorConfig2 = #{
-        <<"mechanism">> => <<"password_based">>,
-        <<"backend">> => <<"mysql">>,
-        <<"enable">> => true
+        mechanism => password_based,
+        backend => mysql,
+        enable => true
     },
     ID1 = <<"password_based:built_in_database">>,
     ID2 = <<"password_based:mysql">>,

apps/emqx_authn/src/emqx_authn.erl

@@ -20,7 +20,6 @@
     providers/0,
     check_config/1,
     check_config/2,
-    check_configs/1,
     %% for telemetry information
     get_enabled_authns/0
 ]).
@@ -39,16 +38,6 @@ providers() ->
         {{scram, built_in_database}, emqx_enhanced_authn_scram_mnesia}
     ].
 
-check_configs(CM) when is_map(CM) ->
-    check_configs([CM]);
-check_configs(CL) ->
-    check_configs(CL, 1).
-
-check_configs([], _Nth) ->
-    [];
-check_configs([Config | Configs], Nth) ->
-    [check_config(Config, #{id_for_log => Nth}) | check_configs(Configs, Nth + 1)].
-
 check_config(Config) ->
     check_config(Config, #{}).
 
@@ -67,21 +56,32 @@ do_check_config(#{<<"mechanism">> := Mec0} = Config, Opts) ->
         end,
     case lists:keyfind(Key, 1, providers()) of
         false ->
-            throw(#{error => unknown_authn_provider, which => Key});
+            Reason =
+                case Key of
+                    {M, B} ->
+                        #{mechanism => M, backend => B};
+                    M ->
+                        #{mechanism => M}
+                end,
+            throw(Reason#{error => unknown_authn_provider});
         {_, ProviderModule} ->
-            hocon_tconf:check_plain(
-                ProviderModule,
-                #{?CONF_NS_BINARY => Config},
-                Opts#{atom_key => true}
-            )
+            do_check_config_maybe_throw(ProviderModule, Config, Opts)
     end;
-do_check_config(Config, Opts) when is_map(Config) ->
+do_check_config(Config, _Opts) when is_map(Config) ->
     throw(#{
         error => invalid_config,
-        which => maps:get(id_for_log, Opts, unknown),
         reason => "mechanism_field_required"
     }).
 
+do_check_config_maybe_throw(ProviderModule, Config0, Opts) ->
+    Config = #{?CONF_NS_BINARY => Config0},
+    case emqx_hocon:check(ProviderModule, Config, Opts#{atom_key => true}) of
+        {ok, Checked} ->
+            Checked;
+        {error, Reason} ->
+            throw(Reason)
+    end.
+
 %% The atoms have to be loaded already,
 %% which might be an issue for plugins which are loaded after node boot
 %% but they should really manage their own configs in that case.

apps/emqx_authn/src/emqx_authn_api.erl

@@ -1232,15 +1232,10 @@ serialize_error({unknown_authn_type, Type}) ->
         code => <<"BAD_REQUEST">>,
         message => binfmt("Unknown type '~p'", [Type])
     }};
-serialize_error({bad_authenticator_config, Reason}) ->
-    {400, #{
-        code => <<"BAD_REQUEST">>,
-        message => binfmt("Bad authenticator config ~p", [Reason])
-    }};
 serialize_error(Reason) ->
     {400, #{
         code => <<"BAD_REQUEST">>,
-        message => binfmt("~p", [Reason])
+        message => binfmt("~0p", [Reason])
     }}.
 
 parse_position(<<"front">>) ->

apps/emqx_authn/src/emqx_authn_app.erl

@@ -35,6 +35,9 @@
 %%------------------------------------------------------------------------------
 
 start(_StartType, _StartArgs) ->
+    %% required by test cases, ensure the injection of
+    %% EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY
+    _ = emqx_conf_schema:roots(),
     ok = mria_rlog:wait_for_shards([?AUTH_SHARD], infinity),
     {ok, Sup} = emqx_authn_sup:start_link(),
     case initialize() of
@@ -43,34 +46,23 @@ start(_StartType, _StartArgs) ->
     end.
 
 stop(_State) ->
-    ok = deinitialize(),
-    ok.
+    ok = deinitialize().
 
 %%------------------------------------------------------------------------------
 %% Internal functions
 %%------------------------------------------------------------------------------
 
 initialize() ->
-    try
-        ok = ?AUTHN:register_providers(emqx_authn:providers()),
-
-        lists:foreach(
-            fun({ChainName, RawAuthConfigs}) ->
-                AuthConfig = emqx_authn:check_configs(RawAuthConfigs),
-                ?AUTHN:initialize_authentication(
-                    ChainName,
-                    AuthConfig
-                )
-            end,
-            chain_configs()
-        )
-    of
-        ok -> ok
-    catch
-        throw:Reason ->
-            ?SLOG(error, #{msg => "failed_to_initialize_authentication", reason => Reason}),
-            {error, {failed_to_initialize_authentication, Reason}}
-    end.
+    ok = ?AUTHN:register_providers(emqx_authn:providers()),
+    lists:foreach(
+        fun({ChainName, AuthConfig}) ->
+            ?AUTHN:initialize_authentication(
+                ChainName,
+                AuthConfig
+            )
+        end,
+        chain_configs()
+    ).
 
 deinitialize() ->
     ok = ?AUTHN:deregister_providers(provider_types()),
@@ -80,12 +72,12 @@ chain_configs() ->
     [global_chain_config() | listener_chain_configs()].
 
 global_chain_config() ->
-    {?GLOBAL, emqx:get_raw_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY], [])}.
+    {?GLOBAL, emqx:get_config([?EMQX_AUTHENTICATION_CONFIG_ROOT_NAME_BINARY], [])}.
 
 listener_chain_configs() ->
     lists:map(
         fun({ListenerID, _}) ->
-            {ListenerID, emqx:get_raw_config(auth_config_path(ListenerID), [])}
+            {ListenerID, emqx:get_config(auth_config_path(ListenerID), [])}
         end,
         emqx_listeners:list()
     ).

apps/emqx_authn/src/emqx_authn_password_hashing.erl

@@ -64,7 +64,7 @@
 ]).
 
 namespace() -> "authn-hash".
-roots() -> [pbkdf2, bcrypt, bcrypt_rw, other_algorithms].
+roots() -> [pbkdf2, bcrypt, bcrypt_rw, simple].
 
 fields(bcrypt_rw) ->
     fields(bcrypt) ++
@@ -96,7 +96,7 @@ fields(pbkdf2) ->
             )},
         {dk_length, fun dk_length/1}
     ];
-fields(other_algorithms) ->
+fields(simple) ->
     [
         {name,
             sc(
@@ -112,8 +112,8 @@ desc(bcrypt) ->
     "Settings for bcrypt password hashing algorithm.";
 desc(pbkdf2) ->
     "Settings for PBKDF2 password hashing algorithm.";
-desc(other_algorithms) ->
-    "Settings for other password hashing algorithms.";
+desc(simple) ->
+    "Settings for simple algorithms.";
 desc(_) ->
     undefined.
 
@@ -231,17 +231,31 @@ check_password(#{name := Other, salt_position := SaltPosition}, Salt, PasswordHa
 %%------------------------------------------------------------------------------
 
 rw_refs() ->
-    [
+    All = [
         hoconsc:ref(?MODULE, bcrypt_rw),
         hoconsc:ref(?MODULE, pbkdf2),
-        hoconsc:ref(?MODULE, other_algorithms)
-    ].
+        hoconsc:ref(?MODULE, simple)
+    ],
+    fun
+        (all_union_members) -> All;
+        ({value, #{<<"name">> := <<"bcrypt">>}}) -> [hoconsc:ref(?MODULE, bcrypt_rw)];
+        ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [hoconsc:ref(?MODULE, pbkdf2)];
+        ({value, #{<<"name">> := _}}) -> [hoconsc:ref(?MODULE, simple)];
+        ({value, _}) -> throw(#{reason => "algorithm_name_missing"})
+    end.
 
 ro_refs() ->
-    [
+    All = [
         hoconsc:ref(?MODULE, bcrypt),
         hoconsc:ref(?MODULE, pbkdf2),
-        hoconsc:ref(?MODULE, other_algorithms)
-    ].
+        hoconsc:ref(?MODULE, simple)
+    ],
+    fun
+        (all_union_members) -> All;
+        ({value, #{<<"name">> := <<"bcrypt">>}}) -> [hoconsc:ref(?MODULE, bcrypt)];
+        ({value, #{<<"name">> := <<"pbkdf2">>}}) -> [hoconsc:ref(?MODULE, pbkdf2)];
+        ({value, #{<<"name">> := _}}) -> [hoconsc:ref(?MODULE, simple)];
+        ({value, _}) -> throw(#{reason => "algorithm_name_missing"})
+    end.
 
 sc(Type, Meta) -> hoconsc:mk(Type, Meta).

apps/emqx_authn/src/emqx_authn_schema.erl

@@ -45,24 +45,79 @@ enable(desc) -> ?DESC(?FUNCTION_NAME);
 enable(_) -> undefined.
 
 authenticator_type() ->
-    hoconsc:union(config_refs([Module || {_AuthnType, Module} <- emqx_authn:providers()])).
+    hoconsc:union(union_member_selector(emqx_authn:providers())).
 
 authenticator_type_without_scram() ->
     Providers = lists:filtermap(
         fun
-            ({{password_based, _Backend}, Mod}) ->
-                {true, Mod};
-            ({jwt, Mod}) ->
-                {true, Mod};
             ({{scram, _Backend}, _Mod}) ->
-                false
+                false;
+            (_) ->
+                true
         end,
         emqx_authn:providers()
     ),
-    hoconsc:union(config_refs(Providers)).
-
-config_refs(Modules) ->
-    lists:append([Module:refs() || Module <- Modules]).
+    hoconsc:union(union_member_selector(Providers)).
+
+config_refs(Providers) ->
+    lists:append([Module:refs() || {_, Module} <- Providers]).
+
+union_member_selector(Providers) ->
+    Types = config_refs(Providers),
+    fun
+        (all_union_members) -> Types;
+        ({value, Value}) -> select_union_member(Value, Providers)
+    end.
+
+select_union_member(#{<<"mechanism">> := _} = Value, Providers0) ->
+    BackendVal = maps:get(<<"backend">>, Value, undefined),
+    MechanismVal = maps:get(<<"mechanism">>, Value),
+    BackendFilterFn = fun
+        ({{_Mec, Backend}, _Mod}) ->
+            BackendVal =:= atom_to_binary(Backend);
+        (_) ->
+            BackendVal =:= undefined
+    end,
+    MechanismFilterFn = fun
+        ({{Mechanism, _Backend}, _Mod}) ->
+            MechanismVal =:= atom_to_binary(Mechanism);
+        ({Mechanism, _Mod}) ->
+            MechanismVal =:= atom_to_binary(Mechanism)
+    end,
+    case lists:filter(BackendFilterFn, Providers0) of
+        [] ->
+            throw(#{reason => "unknown_backend", backend => BackendVal});
+        Providers1 ->
+            case lists:filter(MechanismFilterFn, Providers1) of
+                [] ->
+                    throw(#{
+                        reason => "unsupported_mechanism",
+                        mechanism => MechanismVal,
+                        backend => BackendVal
+                    });
+                [{_, Module}] ->
+                    try_select_union_member(Module, Value)
+            end
+    end;
+select_union_member(Value, _Providers) when is_map(Value) ->
+    throw(#{reason => "missing_mechanism_field"});
+select_union_member(Value, _Providers) ->
+    throw(#{reason => "not_a_struct", value => Value}).
+
+try_select_union_member(Module, Value) ->
+    %% some modules have `union_member_selector/1' exported to help selecting
+    %% the sub-types, they are:
+    %%   emqx_authn_http
+    %%   emqx_authn_jwt
+    %%   emqx_authn_mongodb
+    %%   emqx_authn_redis
+    try
+        Module:union_member_selector({value, Value})
+    catch
+        error:undef ->
+            %% otherwise expect only one member from this module
+            Module:refs()
+    end.
 
 %% authn is a core functionality however implemented outside of emqx app
 %% in emqx_schema, 'authentication' is a map() type which is to allow

apps/emqx_authn/src/simple_authn/emqx_authn_http.erl

@@ -40,6 +40,7 @@
 
 -export([
     refs/0,
+    union_member_selector/1,
     create/2,
     update/2,
     authenticate/2,
@@ -59,19 +60,19 @@ roots() ->
     [
         {?CONF_NS,
             hoconsc:mk(
-                hoconsc:union(refs()),
+                hoconsc:union(fun union_member_selector/1),
                 #{}
             )}
     ].
 
 fields(get) ->
     [
-        {method, #{type => get, required => true, default => get, desc => ?DESC(method)}},
+        {method, #{type => get, required => true, desc => ?DESC(method)}},
         {headers, fun headers_no_content_type/1}
     ] ++ common_fields();
 fields(post) ->
     [
-        {method, #{type => post, required => true, default => post, desc => ?DESC(method)}},
+        {method, #{type => post, required => true, desc => ?DESC(method)}},
         {headers, fun headers/1}
     ] ++ common_fields().
 
@@ -159,6 +160,21 @@ refs() ->
         hoconsc:ref(?MODULE, post)
     ].
 
+union_member_selector(all_union_members) ->
+    refs();
+union_member_selector({value, Value}) ->
+    refs(Value).
+
+refs(#{<<"method">> := <<"get">>}) ->
+    [hoconsc:ref(?MODULE, get)];
+refs(#{<<"method">> := <<"post">>}) ->
+    [hoconsc:ref(?MODULE, post)];
+refs(_) ->
+    throw(#{
+        field_name => method,
+        expected => "get | post"
+    }).
+
 create(_AuthenticatorID, Config) ->
     create(Config).
 

apps/emqx_authn/src/simple_authn/emqx_authn_jwt.erl

@@ -21,7 +21,6 @@
 -include_lib("hocon/include/hoconsc.hrl").
 
 -behaviour(hocon_schema).
--behaviour(emqx_authentication).
 
 -export([
     namespace/0,
@@ -33,6 +32,7 @@
 
 -export([
     refs/0,
+    union_member_selector/1,
     create/2,
     update/2,
     authenticate/2,
@@ -52,7 +52,7 @@ roots() ->
     [
         {?CONF_NS,
             hoconsc:mk(
-                hoconsc:union(refs()),
+                hoconsc:union(fun union_member_selector/1),
                 #{}
             )}
     ].
@@ -165,6 +165,31 @@ refs() ->
         hoconsc:ref(?MODULE, 'jwks')
     ].
 
+union_member_selector(all_union_members) ->
+    refs();
+union_member_selector({value, V}) ->
+    UseJWKS = maps:get(<<"use_jwks">>, V, undefined),
+    select_ref(boolean(UseJWKS), V).
+
+%% this field is technically a boolean type,
+%% but union member selection is done before type casting (by typrefl),
+%% so we have to allow strings too
+boolean(<<"true">>) -> true;
+boolean(<<"false">>) -> false;
+boolean(Other) -> Other.
+
+select_ref(true, _) ->
+    [hoconsc:ref(?MODULE, 'jwks')];
+select_ref(false, #{<<"public_key">> := _}) ->
+    [hoconsc:ref(?MODULE, 'public-key')];
+select_ref(false, _) ->
+    [hoconsc:ref(?MODULE, 'hmac-based')];
+select_ref(_, _) ->
+    throw(#{
+        field_name => use_jwks,
+        expected => "true | false"
+    }).
+
 create(_AuthenticatorID, Config) ->
     create(Config).
 

apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl

@@ -33,6 +33,7 @@
 
 -export([
     refs/0,
+    union_member_selector/1,
     create/2,
     update/2,
     authenticate/2,
@@ -52,7 +53,7 @@ roots() ->
     [
         {?CONF_NS,
             hoconsc:mk(
-                hoconsc:union(refs()),
+                hoconsc:union(fun union_member_selector/1),
                 #{}
             )}
     ].
@@ -246,3 +247,20 @@ is_superuser(Doc, #{is_superuser_field := IsSuperuserField}) ->
     emqx_authn_utils:is_superuser(#{<<"is_superuser">> => IsSuperuser});
 is_superuser(_, _) ->
     emqx_authn_utils:is_superuser(#{<<"is_superuser">> => false}).
+
+union_member_selector(all_union_members) ->
+    refs();
+union_member_selector({value, Value}) ->
+    refs(Value).
+
+refs(#{<<"mongo_type">> := <<"single">>}) ->
+    [hoconsc:ref(?MODULE, standalone)];
+refs(#{<<"mongo_type">> := <<"rs">>}) ->
+    [hoconsc:ref(?MODULE, 'replica-set')];
+refs(#{<<"mongo_type">> := <<"sharded">>}) ->
+    [hoconsc:ref(?MODULE, 'sharded-cluster')];
+refs(_) ->
+    throw(#{
+        field_name => mongo_type,
+        expected => "single | rs | sharded"
+    }).

apps/emqx_authn/src/simple_authn/emqx_authn_redis.erl

@@ -33,6 +33,7 @@
 
 -export([
     refs/0,
+    union_member_selector/1,
     create/2,
     update/2,
     authenticate/2,
@@ -52,7 +53,7 @@ roots() ->
     [
         {?CONF_NS,
             hoconsc:mk(
-                hoconsc:union(refs()),
+                hoconsc:union(fun union_member_selector/1),
                 #{}
             )}
     ].
@@ -97,6 +98,23 @@ refs() ->
         hoconsc:ref(?MODULE, sentinel)
     ].
 
+union_member_selector(all_union_members) ->
+    refs();
+union_member_selector({value, Value}) ->
+    refs(Value).
+
+refs(#{<<"redis_type">> := <<"single">>}) ->
+    [hoconsc:ref(?MODULE, standalone)];
+refs(#{<<"redis_type">> := <<"cluster">>}) ->
+    [hoconsc:ref(?MODULE, cluster)];
+refs(#{<<"redis_type">> := <<"sentinel">>}) ->
+    [hoconsc:ref(?MODULE, sentinel)];
+refs(_) ->
+    throw(#{
+        field_name => redis_type,
+        expected => "single | cluster | sentinel"
+    }).
+
 create(_AuthenticatorID, Config) ->
     create(Config).
 

apps/emqx_authn/test/emqx_authn_SUITE.erl

@@ -168,6 +168,38 @@ t_password_undefined(Config) when is_list(Config) ->
     end,
     ok.
 
+t_union_selector_errors({init, Config}) ->
+    Config;
+t_union_selector_errors({'end', _Config}) ->
+    ok;
+t_union_selector_errors(Config) when is_list(Config) ->
+    Conf0 = #{
+        <<"mechanism">> => <<"password_based">>,
+        <<"backend">> => <<"mysql">>
+    },
+    Conf1 = Conf0#{<<"mechanism">> => <<"unknown-atom-xx">>},
+    ?assertThrow(#{error := unknown_mechanism}, emqx_authn:check_config(Conf1)),
+    Conf2 = Conf0#{<<"backend">> => <<"unknown-atom-xx">>},
+    ?assertThrow(#{error := unknown_backend}, emqx_authn:check_config(Conf2)),
+    Conf3 = Conf0#{<<"backend">> => <<"unknown">>, <<"mechanism">> => <<"unknown">>},
+    ?assertThrow(
+        #{
+            error := unknown_authn_provider,
+            backend := unknown,
+            mechanism := unknown
+        },
+        emqx_authn:check_config(Conf3)
+    ),
+    Res = catch (emqx_authn:check_config(#{<<"mechanism">> => <<"unknown">>})),
+    ?assertEqual(
+        #{
+            error => unknown_authn_provider,
+            mechanism => unknown
+        },
+        Res
+    ),
+    ok.
+
 parse(Bytes) ->
     {ok, Frame, <<>>, {none, _}} = emqx_frame:parse(Bytes),
     Frame.

apps/emqx_authn/test/emqx_authn_mnesia_SUITE.erl

@@ -49,36 +49,6 @@ end_per_testcase(_Case, Config) ->
 %% Tests
 %%------------------------------------------------------------------------------
 
--define(CONF(Conf), #{?CONF_NS_BINARY => Conf}).
-
-t_check_schema(_Config) ->
-    ConfigOk = #{
-        <<"mechanism">> => <<"password_based">>,
-        <<"backend">> => <<"built_in_database">>,
-        <<"user_id_type">> => <<"username">>,
-        <<"password_hash_algorithm">> => #{
-            <<"name">> => <<"bcrypt">>,
-            <<"salt_rounds">> => <<"6">>
-        }
-    },
-
-    hocon_tconf:check_plain(emqx_authn_mnesia, ?CONF(ConfigOk)),
-
-    ConfigNotOk = #{
-        <<"mechanism">> => <<"password_based">>,
-        <<"backend">> => <<"built_in_database">>,
-        <<"user_id_type">> => <<"username">>,
-        <<"password_hash_algorithm">> => #{
-            <<"name">> => <<"md6">>
-        }
-    },
-
-    ?assertException(
-        throw,
-        {emqx_authn_mnesia, _},
-        hocon_tconf:check_plain(emqx_authn_mnesia, ?CONF(ConfigNotOk))
-    ).
-
 t_create(_) ->
     Config0 = config(),
 

apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl

@@ -105,17 +105,12 @@ t_update_with_invalid_config(_Config) ->
     AuthConfig = raw_pgsql_auth_config(),
     BadConfig = maps:without([<<"server">>], AuthConfig),
     ?assertMatch(
-        {error,
-            {bad_authenticator_config, #{
-                reason :=
-                    {emqx_authn_pgsql, [
-                        #{
-                            kind := validation_error,
-                            path := "authentication.server",
-                            reason := required_field
-                        }
-                    ]}
-            }}},
+        {error, #{
+            kind := validation_error,
+            matched_type := "authn-postgresql:authentication",
+            path := "authentication.1.server",
+            reason := required_field
+        }},
         emqx:update_config(
             ?PATH,
             {create_authenticator, ?GLOBAL, BadConfig}

apps/emqx_authn/test/emqx_authn_redis_SUITE.erl

@@ -160,10 +160,12 @@ t_create_invalid_config(_Config) ->
     Config0 = raw_redis_auth_config(),
     Config = maps:without([<<"server">>], Config0),
     ?assertMatch(
-        {error,
-            {bad_authenticator_config, #{
-                reason := {emqx_authn_redis, [#{kind := validation_error}]}
-            }}},
+        {error, #{
+            kind := validation_error,
+            matched_type := "authn-redis:standalone",
+            path := "authentication.1.server",
+            reason := required_field
+        }},
         emqx:update_config(?PATH, {create_authenticator, ?GLOBAL, Config})
     ),
     ?assertMatch([], emqx_config:get_raw([authentication])),

apps/emqx_authn/test/emqx_authn_schema_SUITE.erl

@@ -0,0 +1,190 @@
+-module(emqx_authn_schema_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-include("emqx_authn.hrl").
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    _ = application:load(emqx_conf),
+    emqx_common_test_helpers:start_apps([emqx_authn]),
+    Config.
+
+end_per_suite(_) ->
+    emqx_common_test_helpers:stop_apps([emqx_authn]),
+    ok.
+
+init_per_testcase(_Case, Config) ->
+    {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
+    mria:clear_table(emqx_authn_mnesia),
+    Config.
+
+end_per_testcase(_Case, Config) ->
+    Config.
+
+-define(CONF(Conf), #{?CONF_NS_BINARY => Conf}).
+
+t_check_schema(_Config) ->
+    Check = fun(C) -> emqx_config:check_config(emqx_schema, ?CONF(C)) end,
+    ConfigOk = #{
+        <<"mechanism">> => <<"password_based">>,
+        <<"backend">> => <<"built_in_database">>,
+        <<"user_id_type">> => <<"username">>,
+        <<"password_hash_algorithm">> => #{
+            <<"name">> => <<"bcrypt">>,
+            <<"salt_rounds">> => <<"6">>
+        }
+    },
+    _ = Check(ConfigOk),
+
+    ConfigNotOk = #{
+        <<"mechanism">> => <<"password_based">>,
+        <<"backend">> => <<"built_in_database">>,
+        <<"user_id_type">> => <<"username">>,
+        <<"password_hash_algorithm">> => #{
+            <<"name">> => <<"md6">>
+        }
+    },
+    ?assertThrow(
+        #{
+            path := "authentication.1.password_hash_algorithm.name",
+            matched_type := "authn-builtin_db:authentication/authn-hash:simple",
+            reason := unable_to_convert_to_enum_symbol
+        },
+        Check(ConfigNotOk)
+    ),
+
+    ConfigMissingAlgoName = #{
+        <<"mechanism">> => <<"password_based">>,
+        <<"backend">> => <<"built_in_database">>,
+        <<"user_id_type">> => <<"username">>,
+        <<"password_hash_algorithm">> => #{
+            <<"foo">> => <<"bar">>
+        }
+    },
+
+    ?assertThrow(
+        #{
+            path := "authentication.1.password_hash_algorithm",
+            reason := "algorithm_name_missing",
+            matched_type := "authn-builtin_db:authentication"
+        },
+        Check(ConfigMissingAlgoName)
+    ).
+
+t_union_member_selector(_) ->
+    ?assertMatch(#{authentication := undefined}, check(undefined)),
+    C1 = #{<<"backend">> => <<"built_in_database">>},
+    ?assertThrow(
+        #{
+            path := "authentication.1",
+            reason := "missing_mechanism_field"
+        },
+        check(C1)
+    ),
+    C2 = <<"foobar">>,
+    ?assertThrow(
+        #{
+            path := "authentication.1",
+            reason := "not_a_struct",
+            value := <<"foobar">>
+        },
+        check(C2)
+    ),
+    Base = #{
+        <<"user_id_type">> => <<"username">>,
+        <<"password_hash_algorithm">> => #{
+            <<"name">> => <<"plain">>
+        }
+    },
+    BadBackend = Base#{<<"mechanism">> => <<"password_based">>, <<"backend">> => <<"bar">>},
+    ?assertThrow(
+        #{
+            reason := "unknown_backend",
+            backend := <<"bar">>
+        },
+        check(BadBackend)
+    ),
+    BadMechanism = Base#{<<"mechanism">> => <<"foo">>, <<"backend">> => <<"built_in_database">>},
+    ?assertThrow(
+        #{
+            reason := "unsupported_mechanism",
+            mechanism := <<"foo">>,
+            backend := <<"built_in_database">>
+        },
+        check(BadMechanism)
+    ),
+    BadCombination = Base#{<<"mechanism">> => <<"scram">>, <<"backend">> => <<"http">>},
+    ?assertThrow(
+        #{
+            reason := "unsupported_mechanism",
+            mechanism := <<"scram">>,
+            backend := <<"http">>
+        },
+        check(BadCombination)
+    ),
+    ok.
+
+t_http_auth_selector(_) ->
+    C1 = #{
+        <<"mechanism">> => <<"password_based">>,
+        <<"backend">> => <<"http">>
+    },
+    ?assertThrow(
+        #{
+            field_name := method,
+            expected := "get | post"
+        },
+        check(C1)
+    ),
+    ok.
+
+t_mongo_auth_selector(_) ->
+    C1 = #{
+        <<"mechanism">> => <<"password_based">>,
+        <<"backend">> => <<"mongodb">>
+    },
+    ?assertThrow(
+        #{
+            field_name := mongo_type,
+            expected := "single | rs | sharded"
+        },
+        check(C1)
+    ),
+    ok.
+
+t_redis_auth_selector(_) ->
+    C1 = #{
+        <<"mechanism">> => <<"password_based">>,
+        <<"backend">> => <<"redis">>
+    },
+    ?assertThrow(
+        #{
+            field_name := redis_type,
+            expected := "single | cluster | sentinel"
+        },
+        check(C1)
+    ),
+    ok.
+
+t_redis_jwt_selector(_) ->
+    C1 = #{
+        <<"mechanism">> => <<"jwt">>
+    },
+    ?assertThrow(
+        #{
+            field_name := use_jwks,
+            expected := "true | false"
+        },
+        check(C1)
+    ),
+    ok.
+
+check(C) ->
+    {_Mappings, Checked} = emqx_config:check_config(emqx_schema, ?CONF(C)),
+    Checked.

apps/emqx_authn/test/emqx_authn_schema_tests.erl

@@ -0,0 +1,135 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_authn_schema_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+%% schema error
+-define(ERR(Reason), {error, Reason}).
+
+union_member_selector_mongo_test_() ->
+    Check = fun(Txt) -> check(emqx_authn_mongodb, Txt) end,
+    [
+        {"unknown", fun() ->
+            ?assertMatch(
+                ?ERR(#{field_name := mongo_type, expected := _}),
+                Check("{mongo_type: foobar}")
+            )
+        end},
+        {"single", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-mongodb:standalone"}),
+                Check("{mongo_type: single}")
+            )
+        end},
+        {"replica-set", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-mongodb:replica-set"}),
+                Check("{mongo_type: rs}")
+            )
+        end},
+        {"sharded", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-mongodb:sharded-cluster"}),
+                Check("{mongo_type: sharded}")
+            )
+        end}
+    ].
+
+union_member_selector_jwt_test_() ->
+    Check = fun(Txt) -> check(emqx_authn_jwt, Txt) end,
+    [
+        {"unknown", fun() ->
+            ?assertMatch(
+                ?ERR(#{field_name := use_jwks, expected := "true | false"}),
+                Check("{use_jwks = 1}")
+            )
+        end},
+        {"jwks", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-jwt:jwks"}),
+                Check("{use_jwks = true}")
+            )
+        end},
+        {"publick-key", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-jwt:public-key"}),
+                Check("{use_jwks = false, public_key = 1}")
+            )
+        end},
+        {"hmac-based", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-jwt:hmac-based"}),
+                Check("{use_jwks = false}")
+            )
+        end}
+    ].
+
+union_member_selector_redis_test_() ->
+    Check = fun(Txt) -> check(emqx_authn_redis, Txt) end,
+    [
+        {"unknown", fun() ->
+            ?assertMatch(
+                ?ERR(#{field_name := redis_type, expected := _}),
+                Check("{redis_type = 1}")
+            )
+        end},
+        {"single", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-redis:standalone"}),
+                Check("{redis_type = single}")
+            )
+        end},
+        {"cluster", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-redis:cluster"}),
+                Check("{redis_type = cluster}")
+            )
+        end},
+        {"sentinel", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-redis:sentinel"}),
+                Check("{redis_type = sentinel}")
+            )
+        end}
+    ].
+
+union_member_selector_http_test_() ->
+    Check = fun(Txt) -> check(emqx_authn_http, Txt) end,
+    [
+        {"unknown", fun() ->
+            ?assertMatch(
+                ?ERR(#{field_name := method, expected := _}),
+                Check("{method = 1}")
+            )
+        end},
+        {"get", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-http:get"}),
+                Check("{method = get}")
+            )
+        end},
+        {"post", fun() ->
+            ?assertMatch(
+                ?ERR(#{matched_type := "authn-http:post"}),
+                Check("{method = post}")
+            )
+        end}
+    ].
+
+check(Module, HoconConf) ->
+    emqx_hocon:check(Module, ["authentication= ", HoconConf]).

apps/emqx_conf/src/emqx_conf.app.src

@@ -1,6 +1,6 @@
 {application, emqx_conf, [
     {description, "EMQX configuration management"},
-    {vsn, "0.1.11"},
+    {vsn, "0.1.12"},
     {registered, []},
     {mod, {emqx_conf_app, []}},
     {applications, [kernel, stdlib]},

apps/emqx_conf/src/emqx_conf.erl

@@ -296,8 +296,6 @@ hocon_schema_to_spec(Type, LocalModule) when ?IS_TYPEREFL(Type) ->
 hocon_schema_to_spec(?ARRAY(Item), LocalModule) ->
     {Schema, Refs} = hocon_schema_to_spec(Item, LocalModule),
     {#{type => array, items => Schema}, Refs};
-hocon_schema_to_spec(?LAZY(Item), LocalModule) ->
-    hocon_schema_to_spec(Item, LocalModule);
 hocon_schema_to_spec(?ENUM(Items), _LocalModule) ->
     {#{type => enum, symbols => Items}, []};
 hocon_schema_to_spec(?MAP(Name, Type), LocalModule) ->

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -609,8 +609,6 @@ hocon_schema_to_spec(Type, LocalModule) when ?IS_TYPEREFL(Type) ->
 hocon_schema_to_spec(?ARRAY(Item), LocalModule) ->
     {Schema, Refs} = hocon_schema_to_spec(Item, LocalModule),
     {#{type => array, items => Schema}, Refs};
-hocon_schema_to_spec(?LAZY(Item), LocalModule) ->
-    hocon_schema_to_spec(Item, LocalModule);
 hocon_schema_to_spec(?ENUM(Items), _LocalModule) ->
     {#{type => string, enum => Items}, []};
 hocon_schema_to_spec(?MAP(Name, Type), LocalModule) ->

bin/emqx

@@ -911,7 +911,7 @@ fi
 if [ $IS_BOOT_COMMAND = 'yes' ] && [ "$COOKIE" = "$EMQX_DEFAULT_ERLANG_COOKIE" ]; then
     logwarn "Default (insecure) Erlang cookie is in use."
     logwarn "Configure node.cookie in $EMQX_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE"
-    logwarn "Use the same config value for all nodes in the cluster."
+    logwarn "NOTE: Use the same cookie for all nodes in the cluster."
 fi
 
 ## check if OTP version has mnesia_hook feature; if not, fallback to