chore(auth): update tests

apps/emqx/test/emqx_common_test_helpers.erl

@@ -39,7 +39,8 @@
     flush/1,
     load/1,
     render_and_load_app_config/1,
-    render_and_load_app_config/2
+    render_and_load_app_config/2,
+    copy_acl_conf/0
 ]).
 
 -export([
@@ -527,11 +528,11 @@ copy_certs(_, _) ->
 
 copy_acl_conf() ->
     Dest = filename:join([code:lib_dir(emqx), "etc/acl.conf"]),
-    case code:lib_dir(emqx_auth_file) of
+    case code:lib_dir(emqx_auth) of
         {error, bad_name} ->
             (not filelib:is_regular(Dest)) andalso file:write_file(Dest, <<"">>);
         _ ->
-            {ok, _} = file:copy(deps_path(emqx_auth_file, "etc/acl.conf"), Dest)
+            {ok, _} = file:copy(deps_path(emqx_auth, "etc/acl.conf"), Dest)
     end,
     ok.
 

apps/emqx/test/emqx_cth_suite.erl

@@ -361,10 +361,16 @@ default_appspec(emqx_conf, SuiteOpts) ->
     ),
     #{
         config => SharedConfig,
-        % NOTE
-        % We inform `emqx` of our config loader before starting `emqx_conf` so that it won't
-        % overwrite everything with a default configuration.
-        before_start => fun inhibit_config_loader/2
+        before_start => fun(App, Conf) ->
+            % NOTE
+            % We inform `emqx` of our config loader before starting `emqx_conf` so that it won't
+            % overwrite everything with a default configuration.
+            ok = inhibit_config_loader(App, Conf),
+            % NOTE
+            % This should be done to pass authz schema validations.
+            % In production, acl.conf file is created by the release process.
+            ok = emqx_common_test_helpers:copy_acl_conf()
+        end
     };
 default_appspec(emqx_dashboard, _SuiteOpts) ->
     #{

apps/emqx_auth/src/emqx_auth_app.erl

@@ -37,7 +37,7 @@
 start(_StartType, _StartArgs) ->
     %% required by test cases, ensure the injection of schema
     _ = emqx_conf_schema:roots(),
-    {ok, Sup} = emqx_authn_sup:start_link(),
+    {ok, Sup} = emqx_auth_sup:start_link(),
     ok = emqx_authz:init(),
     {ok, Sup}.
 

apps/emqx_auth/src/emqx_authn/emqx_authn_chains.erl

@@ -372,6 +372,17 @@ handle_call(
     end;
 handle_call({deregister_providers, AuthNTypes}, _From, #{providers := Providers} = State) ->
     reply(ok, State#{providers := maps:without(AuthNTypes, Providers)});
+%% Do not handle anything else before initialization is done.
+%% TODO convert gen_server to gen_statem
+handle_call(_, _From, #{init_done := false, providers := Providers} = State) ->
+    ProviderTypes = maps:keys(Providers),
+    Chains = chain_configs(),
+    ?SLOG(error, #{
+        msg => "authentication_not_initialized",
+        configured_provider_types => configured_provider_types(Chains),
+        registered_provider_types => ProviderTypes
+    }),
+    reply({error, not_initialized}, State);
 handle_call({delete_chain, ChainName}, _From, State) ->
     UpdateFun = fun(Chain) ->
         {_MatchedIDs, NewChain} = do_delete_authenticators(fun(_) -> true end, Chain),
@@ -469,14 +480,9 @@ code_change(_OldVsn, State, _Extra) ->
 %%------------------------------------------------------------------------------
 
 initialize_authentication(Providers) ->
-    Chains = chain_configs(),
     ProviderTypes = maps:keys(Providers),
-    HasProviders = lists:all(
-        fun({_, ChainConfigs}) ->
-            has_providers_for_configs(ChainConfigs, ProviderTypes)
-        end,
-        Chains
-    ),
+    Chains = chain_configs(),
+    HasProviders = has_providers_for_configs(Chains, ProviderTypes),
     do_initialize_authentication(Providers, Chains, HasProviders).
 
 do_initialize_authentication(_Providers, _Chains, _HasProviders = false) ->
@@ -513,25 +519,30 @@ initialize_chain_authentication(Providers, ChainName, AuthenticatorsConfig) ->
         to_list(AuthenticatorsConfig)
     ).
 
-has_providers_for_configs(AuthConfig, ProviderTypes) ->
+has_providers_for_configs(Chains, ProviderTypes) ->
+    (configured_provider_types(Chains) -- ProviderTypes) =:= [].
+
+configured_provider_types(Chains) ->
+    {_, ChainConfs} = lists:unzip(Chains),
+    ProviderTypes = lists:flatmap(
+        fun provider_types_for_chain/1,
+        ChainConfs
+    ),
+    lists:usort(ProviderTypes).
+
+provider_types_for_chain(AuthConfig) ->
     Configs = to_list(AuthConfig),
-    lists:all(
+    lists:map(
         fun(Config) ->
-            has_providers_for_config(Config, ProviderTypes)
+            provider_type_for_config(Config)
         end,
         Configs
     ).
 
-has_providers_for_config(_Config, []) ->
-    false;
-has_providers_for_config(#{mechanism := Mechanism, backend := Backend}, [
-    {Mechanism, Backend} | _ProviderTypes
-]) ->
-    true;
-has_providers_for_config(#{mechanism := Mechanism}, [Mechanism | _ProviderTypes]) ->
-    true;
-has_providers_for_config(Config, [_ProviderType | ProviderTypes]) ->
-    has_providers_for_config(Config, ProviderTypes).
+provider_type_for_config(#{mechanism := Mechanism, backend := Backend}) ->
+    {Mechanism, Backend};
+provider_type_for_config(#{mechanism := Mechanism}) ->
+    Mechanism.
 
 handle_update_authenticator(Chain, AuthenticatorID, Config) ->
     #chain{authenticators = Authenticators} = Chain,

apps/emqx_auth/src/emqx_authn/emqx_authn_config.erl

@@ -228,9 +228,9 @@ create_or_update_authenticators(OldIds, ChainName, NewConfig) ->
             Id = authenticator_id(Conf),
             case lists:member(Id, OldIds) of
                 true ->
-                    emqx_authn_chains:update_authenticator(ChainName, Id, Conf);
+                    {ok, _} = emqx_authn_chains:update_authenticator(ChainName, Id, Conf);
                 false ->
-                    emqx_authn_chains:create_authenticator(ChainName, Conf)
+                    {ok, _} = emqx_authn_chains:create_authenticator(ChainName, Conf)
             end
         end,
         NewConfig
@@ -245,7 +245,7 @@ delete_authenticators(NewIds, ChainName, OldConfig) ->
                 true ->
                     ok;
                 false ->
-                    emqx_authn_chains:delete_authenticator(ChainName, Id)
+                    ok = emqx_authn_chains:delete_authenticator(ChainName, Id)
             end
         end,
         OldConfig

apps/emqx_auth/src/emqx_authn/emqx_authn_enterprise.erl

@@ -10,16 +10,6 @@
 
 -if(?EMQX_RELEASE_EDITION == ee).
 
-% providers() ->
-%     [
-%         {{password_based, ldap}, emqx_authn_ldap},
-%         {{password_based, ldap_bind}, emqx_ldap_authn_bind},
-%         {gcp_device, emqx_gcp_device_authn}
-%     ].
-
-% resource_provider() ->
-%     [emqx_authn_ldap, emqx_ldap_authn_bind].
-
 provider_schema_mods() ->
     ?EE_PROVIDER_SCHEMA_MODS.
 
@@ -28,9 +18,4 @@ provider_schema_mods() ->
 provider_schema_mods() ->
     [].
 
-% providers() ->
-%     [].
-
-% resource_provider() ->
-%      [].
 -endif.

apps/emqx_auth/src/emqx_authn/emqx_authn_schema.erl

@@ -22,6 +22,8 @@
 -include("emqx_authn_schema.hrl").
 -include("emqx_authn_chains.hrl").
 
+-include_lib("eunit/include/eunit.hrl").
+
 -behaviour(emqx_schema_hooks).
 -export([
     injected_fields/0
@@ -30,7 +32,6 @@
 -export([
     common_fields/0,
     roots/0,
-    % validations/0,
     tags/0,
     fields/1,
     authenticator_type/0,
@@ -39,6 +40,10 @@
     backend/1
 ]).
 
+-export([
+    global_auth_fields/0
+]).
+
 %%--------------------------------------------------------------------
 %% Authn Source Schema Behaviour
 %%--------------------------------------------------------------------
@@ -110,6 +115,7 @@ global_auth_fields() ->
                 desc => ?DESC(global_authentication),
                 converter => fun ensure_array/2,
                 default => [],
+                validator => validator(),
                 importance => ?IMPORTANCE_LOW
             })}
     ].
@@ -121,6 +127,7 @@ mqtt_listener_auth_fields() ->
                 desc => ?DESC(listener_authentication),
                 converter => fun ensure_array/2,
                 default => [],
+                validator => validator(),
                 importance => ?IMPORTANCE_HIDDEN
             })}
     ].
@@ -206,6 +213,33 @@ common_field() ->
         {"rate_last5m", ?HOCON(float(), #{desc => ?DESC("rate_last5m")})}
     ].
 
+validator() ->
+    Validations = lists:flatmap(
+        fun validations/1,
+        provider_schema_mods()
+    ),
+    fun(AuthConf) ->
+        lists:foreach(
+            fun(Conf) ->
+                lists:foreach(
+                    fun({_Name, Validation}) ->
+                        Validation(Conf)
+                    end,
+                    Validations
+                )
+            end,
+            wrap_list(AuthConf)
+        )
+    end.
+
+validations(Mod) ->
+    case erlang:function_exported(Mod, validations, 0) of
+        true ->
+            Mod:validations();
+        false ->
+            []
+    end.
+
 provider_schema_mods() ->
     ?PROVIDER_SCHEMA_MODS ++ emqx_authn_enterprise:provider_schema_mods().
 
@@ -223,3 +257,8 @@ array(Name) ->
 
 array(Name, DescId) ->
     {Name, ?HOCON(?R_REF(Name), #{desc => ?DESC(DescId)})}.
+
+wrap_list(Map) when is_map(Map) ->
+    [Map];
+wrap_list(L) when is_list(L) ->
+    L.

apps/emqx_auth/src/emqx_authz/emqx_authz.erl

@@ -57,8 +57,6 @@
     maybe_read_source_files_safe/1
 ]).
 
-% -export([acl_conf_file/0]).
-
 %% Data backup
 -export([
     import_config/1,
@@ -89,9 +87,9 @@ init() ->
     ok = register_metrics(),
     emqx_conf:add_handler(?CONF_KEY_PATH, ?MODULE),
     emqx_conf:add_handler(?ROOT_KEY, ?MODULE),
-    emqx_authz_source_registry:create(),
     ok = emqx_hooks:put('client.authorize', {?MODULE, authorize_deny, []}, ?HP_AUTHZ),
     ok = register_source(client_info, emqx_authz_client_info),
+    ok = register_source(file, emqx_authz_file),
     ok.
 
 register_source(Type, Module) ->
@@ -748,6 +746,13 @@ type_take(Type, Sources) ->
 -compile(export_all).
 
 merge_sources_test() ->
+    ok = emqx_authz_source_registry:create(),
+    ok = lists:foreach(
+        fun(Type) ->
+            ok = emqx_authz_source_registry:register(Type, ?MODULE)
+        end,
+        [file, http, mysql, mongodb, redis, postgresql]
+    ),
     Default = [emqx_authz_schema:default_authz()],
     Http = #{<<"type">> => <<"http">>, <<"enable">> => true},
     Mysql = #{<<"type">> => <<"mysql">>, <<"enable">> => true},

apps/emqx_auth/src/emqx_authz/emqx_authz_schema.erl

@@ -118,7 +118,7 @@ desc(_) ->
 injected_fields() ->
     #{
         'roots.high' => [
-            {?CONF_NS, ?HOCON(?R_REF("authorization"), #{desc => ?DESC(?CONF_NS)})}
+            {?CONF_NS, ?HOCON(?R_REF(?CONF_NS), #{desc => ?DESC(?CONF_NS)})}
         ]
     }.
 

apps/emqx_auth/src/emqx_authz/emqx_authz_sup.erl

@@ -33,6 +33,7 @@ start_link() ->
     supervisor:start_link({local, ?SERVER}, ?MODULE, []).
 
 init([]) ->
+    emqx_authz_source_registry:create(),
     SupFlags = #{
         strategy => one_for_all,
         intensity => 0,

apps/emqx_auth_file/src/emqx_authz_file.erl

@@ -66,7 +66,9 @@ authorize(Client, PubSub, Topic, #{annotations := #{rules := Rules}}) ->
 
 read_files(#{<<"path">> := Path} = Source) ->
     {ok, Rules} = read_file(Path),
-    maps:remove(<<"path">>, Source#{<<"rules">> => Rules}).
+    maps:remove(<<"path">>, Source#{<<"rules">> => Rules});
+read_files(#{<<"rules">> := _} = Source) ->
+    Source.
 
 write_files(#{<<"rules">> := Rules} = Source0) ->
     AclPath = ?MODULE:acl_conf_file(),
@@ -75,7 +77,9 @@ write_files(#{<<"rules">> := Rules} = Source0) ->
     ok = check_acl_file_rules(AclPath, Rules),
     ok = write_file(AclPath, Rules),
     Source1 = maps:remove(<<"rules">>, Source0),
-    maps:put(<<"path">>, AclPath, Source1).
+    maps:put(<<"path">>, AclPath, Source1);
+write_files(#{<<"path">> := _} = Source) ->
+    Source.
 
 %%--------------------------------------------------------------------
 %% Internal functions

apps/emqx_auth/test/emqx_authn/emqx_authn_SUITE.erl

@@ -81,7 +81,7 @@ t_fill_defaults(Config) when is_list(Config) ->
     ).
 
 t_will_message_connection_denied({init, Config}) ->
-    emqx_common_test_helpers:start_apps([emqx_conf, emqx_auth, emqx_auth_file]),
+    emqx_common_test_helpers:start_apps([emqx_conf, emqx_auth]),
     emqx_authn_test_lib:register_fake_providers([{password_based, built_in_database}]),
     AuthnConfig = #{
         <<"mechanism">> => <<"password_based">>,
@@ -106,7 +106,7 @@ t_will_message_connection_denied({'end', _Config}) ->
         [authentication],
         {delete_authenticator, 'mqtt:global', <<"password_based:built_in_database">>}
     ),
-    emqx_common_test_helpers:stop_apps([emqx_auth_file, emqx_auth, emqx_conf]),
+    emqx_common_test_helpers:stop_apps([emqx_auth, emqx_conf]),
     ok;
 t_will_message_connection_denied(Config) when is_list(Config) ->
     process_flag(trap_exit, true),

apps/emqx_auth/test/emqx_authn/emqx_authn_schema_tests.erl

@@ -22,114 +22,113 @@
 -define(ERR(Reason), {error, Reason}).
 
 union_member_selector_mongo_test_() ->
-    Check = fun(Txt) -> check(emqx_authn_mongodb, Txt) end,
     [
         {"unknown", fun() ->
             ?assertMatch(
                 ?ERR(#{field_name := mongo_type, expected := _}),
-                Check("{mongo_type: foobar}")
+                check("{mechanism = password_based, backend = mongodb, mongo_type = foobar}")
             )
         end},
         {"single", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:mongo_single"}),
-                Check("{mongo_type: single}")
+                ?ERR(#{matched_type := "mongo_single"}),
+                check("{mechanism = password_based, backend = mongodb, mongo_type = single}")
             )
         end},
         {"replica-set", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:mongo_rs"}),
-                Check("{mongo_type: rs}")
+                ?ERR(#{matched_type := "mongo_rs"}),
+                check("{mechanism = password_based, backend = mongodb, mongo_type = rs}")
             )
         end},
         {"sharded", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:mongo_sharded"}),
-                Check("{mongo_type: sharded}")
+                ?ERR(#{matched_type := "mongo_sharded"}),
+                check("{mechanism = password_based, backend = mongodb, mongo_type = sharded}")
             )
         end}
     ].
 
 union_member_selector_jwt_test_() ->
-    Check = fun(Txt) -> check(emqx_authn_jwt, Txt) end,
     [
         {"unknown", fun() ->
             ?assertMatch(
                 ?ERR(#{field_name := use_jwks, expected := "true | false"}),
-                Check("{use_jwks = 1}")
+                check("{mechanism = jwt, use_jwks = 1}")
             )
         end},
         {"jwks", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:jwt_jwks"}),
-                Check("{use_jwks = true}")
+                ?ERR(#{matched_type := "jwt_jwks"}),
+                check("{mechanism = jwt, use_jwks = true}")
             )
         end},
         {"publick-key", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:jwt_public_key"}),
-                Check("{use_jwks = false, public_key = 1}")
+                ?ERR(#{matched_type := "jwt_public_key"}),
+                check("{mechanism = jwt, use_jwks = false, public_key = 1}")
             )
         end},
         {"hmac-based", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:jwt_hmac"}),
-                Check("{use_jwks = false}")
+                ?ERR(#{matched_type := "jwt_hmac"}),
+                check("{mechanism = jwt, use_jwks = false}")
             )
         end}
     ].
 
 union_member_selector_redis_test_() ->
-    Check = fun(Txt) -> check(emqx_authn_redis, Txt) end,
     [
         {"unknown", fun() ->
             ?assertMatch(
                 ?ERR(#{field_name := redis_type, expected := _}),
-                Check("{redis_type = 1}")
+                check("{mechanism = password_based, backend = redis, redis_type = 1}")
             )
         end},
         {"single", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:redis_single"}),
-                Check("{redis_type = single}")
+                ?ERR(#{matched_type := "redis_single"}),
+                check("{mechanism = password_based, backend = redis, redis_type = single}")
             )
         end},
         {"cluster", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:redis_cluster"}),
-                Check("{redis_type = cluster}")
+                ?ERR(#{matched_type := "redis_cluster"}),
+                check("{mechanism = password_based, backend = redis, redis_type = cluster}")
             )
         end},
         {"sentinel", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:redis_sentinel"}),
-                Check("{redis_type = sentinel}")
+                ?ERR(#{matched_type := "redis_sentinel"}),
+                check("{mechanism = password_based, backend = redis, redis_type = sentinel}")
             )
         end}
     ].
 
 union_member_selector_http_test_() ->
-    Check = fun(Txt) -> check(emqx_authn_http, Txt) end,
     [
         {"unknown", fun() ->
             ?assertMatch(
                 ?ERR(#{field_name := method, expected := _}),
-                Check("{method = 1}")
+                check("{mechanism = password_based, backend = http, method = 1}")
             )
         end},
         {"get", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:http_get"}),
-                Check("{method = get}")
+                ?ERR(#{matched_type := "http_get"}),
+                check("{mechanism = password_based, backend = http, method = get}")
             )
         end},
         {"post", fun() ->
             ?assertMatch(
-                ?ERR(#{matched_type := "authn:http_post"}),
-                Check("{method = post}")
+                ?ERR(#{matched_type := "http_post"}),
+                check("{mechanism = password_based, backend = http, method = post}")
             )
         end}
     ].
 
-check(Module, HoconConf) ->
-    emqx_hocon:check(Module, ["authentication= ", HoconConf]).
+check(HoconConf) ->
+    emqx_hocon:check(
+        #{roots => emqx_authn_schema:global_auth_fields()},
+        ["authentication= ", HoconConf]
+    ).

apps/emqx_auth/test/emqx_authz/emqx_authz_SUITE.erl

@@ -42,7 +42,7 @@ init_per_suite(Config) ->
         emqx_authz_file,
         acl_conf_file,
         fun() ->
-            emqx_common_test_helpers:deps_path(emqx_auth_file, "etc/acl.conf")
+            emqx_common_test_helpers:deps_path(emqx_auth, "etc/acl.conf")
         end
     ),
     Apps = emqx_cth_suite:start(
@@ -51,7 +51,6 @@ init_per_suite(Config) ->
             {emqx_conf,
                 "authorization { cache { enable = false }, no_match = deny, sources = [] }"},
             emqx_auth,
-            emqx_auth_file,
             emqx_auth_http,
             emqx_auth_mnesia,
             emqx_auth_redis,

apps/emqx_auth/test/emqx_authz/emqx_authz_api_sources_SUITE.erl

@@ -110,7 +110,7 @@ init_per_suite(Config) ->
         emqx_authz_file,
         acl_conf_file,
         fun() ->
-            emqx_common_test_helpers:deps_path(emqx_auth_file, "etc/acl.conf")
+            emqx_common_test_helpers:deps_path(emqx_auth, "etc/acl.conf")
         end
     ),
 
@@ -120,7 +120,6 @@ init_per_suite(Config) ->
             {emqx_conf,
                 "authorization { cache { enable = false }, no_match = deny, sources = [] }"},
             emqx_auth,
-            emqx_auth_file,
             emqx_auth_http,
             emqx_auth_mnesia,
             emqx_auth_redis,

apps/emqx_auth_file/test/emqx_authz_file_SUITE.erl

@@ -43,8 +43,7 @@ init_per_testcase(TestCase, Config) ->
         [
             {emqx_conf, "authorization.no_match = deny, authorization.cache.enable = false"},
             emqx,
-            emqx_auth,
-            emqx_auth_file
+            emqx_auth
         ],
         #{work_dir => filename:join(?config(priv_dir, Config), TestCase)}
     ),

apps/emqx_auth/test/emqx_authz/emqx_authz_rich_actions_SUITE.erl

@@ -36,8 +36,7 @@ init_per_testcase(TestCase, Config) ->
         [
             emqx,
             {emqx_conf, "authorization.no_match = deny, authorization.cache.enable = false"},
-            emqx_auth,
-            emqx_auth_file
+            emqx_auth
         ],
         #{work_dir => filename:join(?config(priv_dir, Config), TestCase)}
     ),

apps/emqx_auth/test/emqx_authz/emqx_authz_schema_tests.erl

@@ -113,4 +113,4 @@ check(Txt0) ->
     end.
 
 schema() ->
-    #{roots => emqx_authz_schema:fields("authorization")}.
+    #{roots => emqx_authz_schema:authz_fields()}.

apps/emqx_auth_file/rebar.config

@@ -1,7 +0,0 @@
-%% -*- mode: erlang -*-
-{deps, [
-    {emqx, {path, "../emqx"}},
-    {emqx_utils, {path, "../emqx_utils"}},
-    {emqx_auth, {path, "../emqx_auth"}}
-
-]}.

apps/emqx_auth_file/src/emqx_auth_file.app.src

@@ -1,18 +0,0 @@
-%% -*- mode: erlang -*-
-{application, emqx_auth_file, [
-    {description, "EMQX File-based Authentication and Authorization"},
-    {vsn, "0.1.0"},
-    {registered, []},
-    {mod, {emqx_auth_file_app, []}},
-    {applications, [
-        kernel,
-        stdlib,
-        emqx,
-        emqx_auth
-    ]},
-    {env, []},
-    {modules, []},
-
-    {licenses, ["Apache 2.0"]},
-    {links, []}
-]}.

apps/emqx_auth_file/src/emqx_auth_file_app.erl

@@ -1,32 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_auth_file_app).
-
--include("emqx_auth_file.hrl").
-
--behaviour(application).
-
--export([start/2, stop/1]).
-
-start(_StartType, _StartArgs) ->
-    ok = emqx_authz:register_source(?AUTHZ_TYPE, emqx_authz_file),
-    {ok, Sup} = emqx_auth_file_sup:start_link(),
-    {ok, Sup}.
-
-stop(_State) ->
-    ok = emqx_authz:unregister_source(?AUTHZ_TYPE),
-    ok.

apps/emqx_auth_file/src/emqx_auth_file_sup.erl

@@ -1,37 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_auth_file_sup).
-
--behaviour(supervisor).
-
--export([start_link/0]).
-
--export([init/1]).
-
--define(SERVER, ?MODULE).
-
-start_link() ->
-    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
-
-init([]) ->
-    SupFlags = #{
-        strategy => one_for_all,
-        intensity => 0,
-        period => 1
-    },
-    ChildSpecs = [],
-    {ok, {SupFlags, ChildSpecs}}.

apps/emqx_auth_http/src/emqx_authn_http_schema.erl

@@ -24,12 +24,19 @@
 
 -export([
     fields/1,
+    validations/0,
     desc/1,
     refs/0,
     select_union_member/1
 ]).
 
 -define(NOT_EMPTY(MSG), emqx_resource_validator:not_empty(MSG)).
+-define(THROW_VALIDATION_ERROR(ERROR, MESSAGE),
+    throw(#{
+        error => ERROR,
+        message => MESSAGE
+    })
+).
 
 refs() ->
     [?R_REF(http_get), ?R_REF(http_post)].
@@ -78,6 +85,12 @@ desc(http_post) ->
 desc(_) ->
     undefined.
 
+validations() ->
+    [
+        {check_ssl_opts, fun check_ssl_opts/1},
+        {check_headers, fun check_headers/1}
+    ].
+
 common_fields() ->
     [
         {mechanism, emqx_authn_schema:mechanism(?AUTHN_MECHANISM)},
@@ -130,3 +143,39 @@ request_timeout(type) -> emqx_schema:duration_ms();
 request_timeout(desc) -> ?DESC(?FUNCTION_NAME);
 request_timeout(default) -> <<"5s">>;
 request_timeout(_) -> undefined.
+
+check_ssl_opts(#{
+    backend := ?AUTHN_BACKEND, url := <<"https://", _/binary>>, ssl := #{enable := false}
+}) ->
+    ?THROW_VALIDATION_ERROR(
+        invalid_ssl_opts,
+        <<"it's required to enable the TLS option to establish a https connection">>
+    );
+check_ssl_opts(#{
+    <<"backend">> := ?AUTHN_BACKEND,
+    <<"url">> := <<"https://", _/binary>>,
+    <<"ssl">> := #{<<"enable">> := false}
+}) ->
+    ?THROW_VALIDATION_ERROR(
+        invalid_ssl_opts,
+        <<"it's required to enable the TLS option to establish a https connection">>
+    );
+check_ssl_opts(_) ->
+    ok.
+
+check_headers(#{backend := ?AUTHN_BACKEND, headers := Headers, method := get}) ->
+    do_check_get_headers(Headers);
+check_headers(#{<<"backend">> := ?AUTHN_BACKEND, <<"headers">> := Headers, <<"method">> := get}) ->
+    do_check_get_headers(Headers);
+check_headers(_) ->
+    ok.
+
+do_check_get_headers(Headers) ->
+    case maps:is_key(<<"content-type">>, Headers) of
+        false ->
+            ok;
+        true ->
+            ?THROW_VALIDATION_ERROR(
+                invalid_headers, <<"HTTP GET requests cannot include content-type header.">>
+            )
+    end.

apps/emqx_auth_ldap/src/emqx_auth_ldap.app.src

@@ -7,7 +7,9 @@
     {applications, [
         kernel,
         stdlib,
-        emqx_auth
+        emqx_auth,
+        emqx_resource,
+        emqx_connector
     ]},
     {env, []},
     {modules, []},

apps/emqx_auth_ldap/src/emqx_authn_ldap_bind_schema.erl

@@ -23,6 +23,7 @@
 
 -export([
     fields/1,
+    desc/1,
     refs/0,
     select_union_member/1
 ]).
@@ -51,10 +52,10 @@ fields(ldap_bind) ->
         emqx_authn_schema:common_fields() ++
         emqx_ldap:fields(config) ++ emqx_ldap:fields(bind_opts).
 
-% desc(ldap_bind) ->
-%     ?DESC(ldap_bind);
-% desc(_) ->
-%     undefined.
+desc(ldap_bind) ->
+    ?DESC(ldap_bind);
+desc(_) ->
+    undefined.
 
 query_timeout(type) -> emqx_schema:timeout_duration_ms();
 query_timeout(desc) -> ?DESC(?FUNCTION_NAME);

apps/emqx_auth_ldap/src/emqx_authn_ldap_schema.erl

@@ -23,6 +23,7 @@
 
 -export([
     fields/1,
+    desc/1,
     refs/0,
     select_union_member/1
 ]).
@@ -51,10 +52,10 @@ fields(ldap) ->
         emqx_authn_schema:common_fields() ++
         emqx_ldap:fields(config).
 
-% desc(ldap) ->
-%     ?DESC(ldap);
-% desc(_) ->
-%     undefined.
+desc(ldap) ->
+    ?DESC(ldap);
+desc(_) ->
+    undefined.
 
 password_attribute(type) -> string();
 password_attribute(desc) -> ?DESC(?FUNCTION_NAME);

apps/emqx_auth_ldap/test/emqx_authn_ldap_SUITE.erl

@@ -31,7 +31,7 @@ init_per_suite(Config) ->
     _ = application:load(emqx_conf),
     case emqx_common_test_helpers:is_tcp_server_available(?LDAP_HOST, ?LDAP_DEFAULT_PORT) of
         true ->
-            Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_auth], #{
+            Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_auth, emqx_auth_ldap], #{
                 work_dir => ?config(priv_dir, Config)
             }),
             {ok, _} = emqx_resource:create_local(

apps/emqx_auth_ldap/test/emqx_authn_ldap_bind_SUITE.erl

@@ -21,7 +21,6 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_testcase(_, Config) ->
-    emqx_authentication:initialize_authentication(?GLOBAL, []),
     emqx_authn_test_lib:delete_authenticators(
         [authentication],
         ?GLOBAL
@@ -32,7 +31,7 @@ init_per_suite(Config) ->
     _ = application:load(emqx_conf),
     case emqx_common_test_helpers:is_tcp_server_available(?LDAP_HOST, ?LDAP_DEFAULT_PORT) of
         true ->
-            Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_authn], #{
+            Apps = emqx_cth_suite:start([emqx, emqx_conf, emqx_auth, emqx_auth_ldap], #{
                 work_dir => ?config(priv_dir, Config)
             }),
             {ok, _} = emqx_resource:create_local(
@@ -67,7 +66,7 @@ t_create(_Config) ->
         {create_authenticator, ?GLOBAL, AuthConfig}
     ),
 
-    {ok, [#{provider := emqx_ldap_authn_bind}]} = emqx_authentication:list_authenticators(?GLOBAL),
+    {ok, [#{provider := emqx_authn_ldap_bind}]} = emqx_authn_chains:list_authenticators(?GLOBAL),
     emqx_authn_test_lib:delete_config(?ResourceID).
 
 t_create_invalid(_Config) ->
@@ -88,7 +87,7 @@ t_create_invalid(_Config) ->
             emqx_authn_test_lib:delete_config(?ResourceID),
             ?assertEqual(
                 {error, {not_found, {chain, ?GLOBAL}}},
-                emqx_authentication:list_authenticators(?GLOBAL)
+                emqx_authn_chains:list_authenticators(?GLOBAL)
             )
         end,
         InvalidConfigs
@@ -135,10 +134,10 @@ t_destroy(_Config) ->
         {create_authenticator, ?GLOBAL, AuthConfig}
     ),
 
-    {ok, [#{provider := emqx_ldap_authn_bind, state := State}]} =
-        emqx_authentication:list_authenticators(?GLOBAL),
+    {ok, [#{provider := emqx_authn_ldap_bind, state := State}]} =
+        emqx_authn_chains:list_authenticators(?GLOBAL),
 
-    {ok, _} = emqx_ldap_authn_bind:authenticate(
+    {ok, _} = emqx_authn_ldap_bind:authenticate(
         #{
             username => <<"mqttuser0001">>,
             password => <<"mqttuser0001">>
@@ -154,7 +153,7 @@ t_destroy(_Config) ->
     % Authenticator should not be usable anymore
     ?assertMatch(
         ignore,
-        emqx_ldap_authn_bind:authenticate(
+        emqx_authn_ldap_bind:authenticate(
             #{
                 username => <<"mqttuser0001">>,
                 password => <<"mqttuser0001">>

apps/emqx_auth_ldap/test/emqx_authz_ldap_SUITE.erl

@@ -25,7 +25,7 @@ init_per_suite(Config) ->
     case emqx_common_test_helpers:is_tcp_server_available(?LDAP_HOST, ?LDAP_DEFAULT_PORT) of
         true ->
             ok = emqx_common_test_helpers:start_apps(
-                [emqx_conf, emqx_authz],
+                [emqx_conf, emqx_auth, emqx_auth_ldap],
                 fun set_special_configs/1
             ),
             ok = start_apps([emqx_resource]),
@@ -39,7 +39,7 @@ end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = emqx_resource:remove_local(?LDAP_RESOURCE),
     ok = stop_apps([emqx_resource]),
-    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_auth, emqx_auth_ldap]).
 
 init_per_group(Group, Config) ->
     [{test_case, emqx_authz_test_lib:get_case(Group, cases())} | Config].

apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl

@@ -76,6 +76,7 @@
     emqx_conf,
     emqx,
     emqx_auth,
+    emqx_auth_mnesia,
     emqx_management,
     {emqx_rule_engine, "rule_engine { rules {} }"},
     {emqx_bridge, "bridges {}"}

apps/emqx_conf/test/emqx_conf_cli_SUITE.erl

@@ -27,11 +27,11 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_auth]),
+    emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_auth, emqx_auth_redis]),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_auth]).
+    emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_auth, emqx_auth_redis]).
 
 t_load_config(Config) ->
     Authz = authorization,

apps/emqx_conf/test/emqx_conf_schema_tests.erl

@@ -338,13 +338,11 @@ log_rotation_count_limit_test() ->
     """
 ).
 
--define(ERROR(Reason),
+-define(ERROR(Error),
     {emqx_conf_schema, [
         #{
             kind := validation_error,
-            reason := integrity_validation_failure,
-            result := _,
-            validation_name := Reason
+            reason := #{error := Error}
         }
     ]}
 ).
@@ -374,7 +372,7 @@ authn_validations_test() ->
     Conf2 = <<BaseConf/binary, DisableSSLWithHttps/binary>>,
     {ok, ConfMap2} = hocon:binary(Conf2, #{format => richmap}),
     ?assertThrow(
-        ?ERROR(check_http_ssl_opts),
+        ?ERROR(invalid_ssl_opts),
         hocon_tconf:map_translate(emqx_conf_schema, ConfMap2, #{format => richmap})
     ),
 

apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl

@@ -68,11 +68,11 @@ init_per_suite(Config) ->
     emqx_config:erase(gateway),
     emqx_gateway_test_utils:load_all_gateway_apps(),
     init_gateway_conf(),
-    meck:new(emqx_authz_file, [non_strict, passthrough, no_history, no_link]),
-    meck:expect(emqx_authz_file, create, fun(S) -> S end),
     emqx_mgmt_api_test_util:init_suite([
-        emqx_conf, emqx_auth, emqx_auth_file, emqx_auth_http, emqx_gateway
+        emqx_conf, emqx_auth, emqx_auth_http, emqx_gateway
     ]),
+    meck:new(emqx_authz_file, [non_strict, passthrough, no_history, no_link]),
+    meck:expect(emqx_authz_file, create, fun(S) -> S end),
     application:ensure_all_started(cowboy),
     emqx_gateway_auth_ct:start(),
     Config.
@@ -83,7 +83,7 @@ end_per_suite(Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     emqx_config:erase(gateway),
     emqx_mgmt_api_test_util:end_suite([
-        emqx_gateway, emqx_auth_http, emqx_auth_file, emqx_auth, emqx_conf
+        emqx_gateway, emqx_auth_http, emqx_auth, emqx_conf
     ]),
     Config.
 

apps/emqx_gateway_lwm2m/test/emqx_lwm2m_SUITE.erl

@@ -135,13 +135,13 @@ init_per_suite(Config) ->
     %% load application first for minirest api searching
     application:load(emqx_gateway),
     application:load(emqx_gateway_lwm2m),
-    emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn]),
+    emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_auth]),
     Config.
 
 end_per_suite(Config) ->
     timer:sleep(300),
     {ok, _} = emqx_conf:remove([<<"gateway">>, <<"lwm2m">>], #{}),
-    emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_authn]),
+    emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_auth]),
     Config.
 
 init_per_testcase(TestCase, Config) ->

apps/emqx_gcp_device/src/emqx_gcp_device_authn_schema.erl

@@ -23,6 +23,7 @@
 
 -export([
     fields/1,
+    desc/1,
     refs/0,
     select_union_member/1
 ]).
@@ -30,16 +31,16 @@
 refs() -> [?R_REF(gcp_device)].
 
 select_union_member(#{<<"mechanism">> := ?AUTHN_MECHANISM_BIN}) ->
-    [refs()];
+    refs();
 select_union_member(_Value) ->
     undefined.
 
 fields(gcp_device) ->
     [
-        {mechanism, emqx_authn_schema:mechanism('gcp_device')}
+        {mechanism, emqx_authn_schema:mechanism(gcp_device)}
     ] ++ emqx_authn_schema:common_fields().
 
-% desc(gcp_device) ->
-%     ?DESC(emqx_gcp_device_api, gcp_device);
-% desc(_) ->
-%     undefined.
+desc(gcp_device) ->
+    ?DESC(emqx_gcp_device_api, gcp_device);
+desc(_) ->
+    undefined.

apps/emqx_machine/priv/reboot_lists.eterm

@@ -49,7 +49,6 @@
             emqx_resource,
             emqx_connector,
             emqx_auth,
-            emqx_auth_file,
             emqx_auth_http,
             emqx_auth_jwt,
             emqx_auth_mnesia,

apps/emqx_machine/src/emqx_machine_boot.erl

@@ -170,14 +170,15 @@ is_app(Name) ->
     end.
 
 sorted_reboot_apps() ->
-    Apps0 = [{App, app_deps(App)} || App <- reboot_apps()],
+    RebootApps = reboot_apps(),
+    Apps0 = [{App, app_deps(App, RebootApps)} || App <- RebootApps],
     Apps = inject_bridge_deps(Apps0),
     sorted_reboot_apps(Apps).
 
-app_deps(App) ->
+app_deps(App, RebootApps) ->
     case application:get_key(App, applications) of
         undefined -> undefined;
-        {ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List)
+        {ok, List} -> lists:filter(fun(A) -> lists:member(A, RebootApps) end, List)
     end.
 
 %% `emqx_bridge' is special in that it needs all the bridges apps to

apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl

@@ -439,7 +439,6 @@ apps_to_start() ->
         emqx_management,
         emqx_dashboard,
         emqx_auth,
-        emqx_auth_file,
         emqx_auth_http,
         emqx_auth_jwt,
         emqx_auth_mnesia,

apps/emqx_slow_subs/test/emqx_slow_subs_api_SUITE.erl

@@ -60,7 +60,7 @@ init_per_suite(Config) ->
 
     ok = emqx_common_test_helpers:load_config(emqx_slow_subs_schema, ?CONF_DEFAULT),
     emqx_mgmt_api_test_util:init_suite([emqx_slow_subs]),
-    {ok, _} = application:ensure_all_started(emqx_authn),
+    {ok, _} = application:ensure_all_started(emqx_auth),
     Config.
 
 end_per_suite(Config) ->
@@ -69,7 +69,7 @@ end_per_suite(Config) ->
     mria_mnesia:delete_schema(),
     meck:unload(emqx_alarm),
 
-    application:stop(emqx_authn),
+    application:stop(emqx_auth),
     emqx_mgmt_api_test_util:end_suite([emqx_slow_subs]),
     Config.
 

apps/emqx_telemetry/test/emqx_telemetry_SUITE.erl

@@ -58,7 +58,7 @@ init_per_suite(Config) ->
         emqx_authz_file,
         acl_conf_file,
         fun() ->
-            emqx_common_test_helpers:deps_path(emqx_auth_file, "etc/acl.conf")
+            emqx_common_test_helpers:deps_path(emqx_auth, "etc/acl.conf")
         end
     ),
     ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?MODULES_CONF),

apps/emqx_telemetry/test/emqx_telemetry_api_SUITE.erl

@@ -32,7 +32,7 @@ init_per_suite(Config) ->
     ok = emqx_common_test_helpers:load_config(emqx_modules_schema, ?BASE_CONF),
     ok = emqx_common_test_helpers:load_config(emqx_telemetry_schema, ?BASE_CONF),
     ok = emqx_mgmt_api_test_util:init_suite(
-        [emqx_conf, emqx_auth, emqx_auth_file, emqx_management, emqx_telemetry],
+        [emqx_conf, emqx_auth, emqx_management, emqx_telemetry],
         fun set_special_configs/1
     ),
 
@@ -48,7 +48,7 @@ end_per_suite(_Config) ->
         }
     ),
     emqx_mgmt_api_test_util:end_suite([
-        emqx_conf, emqx_auth, emqx_auth_file, emqx_management, emqx_telemetry
+        emqx_conf, emqx_auth, emqx_management, emqx_telemetry
     ]),
     ok.
 

dev

@@ -330,7 +330,7 @@ EOF
 # copy cert files and acl.conf to etc
 copy_other_conf_files() {
     cp -r apps/emqx/etc/certs "$EMQX_ETC_DIR"/
-    cp apps/emqx_auth_file/etc/acl.conf "$EMQX_ETC_DIR"/
+    cp apps/emqx_auth/etc/acl.conf "$EMQX_ETC_DIR"/
 }
 
 is_current_profile_app() {

mix.exs

@@ -535,7 +535,7 @@ defmodule EMQXUmbrella.MixProject do
     )
 
     Mix.Generator.copy_file(
-      "apps/emqx_auth_file/etc/acl.conf",
+      "apps/emqx_auth/etc/acl.conf",
       Path.join(etc, "acl.conf"),
       force: overwrite?
     )

rebar.config.erl

@@ -108,6 +108,7 @@ is_community_umbrella_app("apps/emqx_enterprise") -> false;
 is_community_umbrella_app("apps/emqx_bridge_kinesis") -> false;
 is_community_umbrella_app("apps/emqx_bridge_azure_event_hub") -> false;
 is_community_umbrella_app("apps/emqx_ldap") -> false;
+is_community_umbrella_app("apps/emqx_auth_ldap") -> false;
 is_community_umbrella_app("apps/emqx_gcp_device") -> false;
 is_community_umbrella_app("apps/emqx_dashboard_rbac") -> false;
 is_community_umbrella_app("apps/emqx_dashboard_sso") -> false;
@@ -455,7 +456,7 @@ relx_overlay(ReleaseType, Edition) ->
         {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"},
         {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"},
         {copy, "apps/emqx_gateway_lwm2m/lwm2m_xml", "etc/lwm2m_xml"},
-        {copy, "apps/emqx_auth_file/etc/acl.conf", "etc/acl.conf"},
+        {copy, "apps/emqx_auth/etc/acl.conf", "etc/acl.conf"},
         {template, "bin/emqx.cmd", "bin/emqx.cmd"},
         {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"},
         {copy, "bin/nodetool", "bin/nodetool"},