test: fix inter-suite test teardowns

apps/emqx/src/emqx_config.erl

@@ -87,6 +87,10 @@
     remove_handlers/0
 ]).
 
+-ifdef(TEST).
+-export([erase_schema_mod_and_names/0]).
+-endif.
+
 -include("logger.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
@@ -501,6 +505,11 @@ save_schema_mod_and_names(SchemaMod) ->
         names => lists:usort(OldNames ++ RootNames)
     }).
 
+-ifdef(TEST).
+erase_schema_mod_and_names() ->
+    persistent_term:erase(?PERSIS_SCHEMA_MODS).
+-endif.
+
 -spec get_schema_mod() -> #{binary() => atom()}.
 get_schema_mod() ->
     maps:get(mods, persistent_term:get(?PERSIS_SCHEMA_MODS, #{mods => #{}})).

apps/emqx/test/emqx_SUITE.erl

@@ -26,6 +26,7 @@
 all() -> emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
+    emqx_common_test_helpers:boot_modules(all),
     emqx_common_test_helpers:start_apps([]),
     Config.
 

apps/emqx/test/emqx_common_test_helpers.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_common_test_helpers).
 
+-include("emqx_authentication.hrl").
+
 -type special_config_handler() :: fun().
 
 -type apps() :: list(atom()).
@@ -283,6 +285,14 @@ generate_config(SchemaModule, ConfigFile) when is_atom(SchemaModule) ->
 -spec stop_apps(list()) -> ok.
 stop_apps(Apps) ->
     [application:stop(App) || App <- Apps ++ [emqx, ekka, mria, mnesia]],
+    %% to avoid inter-suite flakiness
+    application:unset_env(emqx, init_config_load_done),
+    persistent_term:erase(?EMQX_AUTHENTICATION_SCHEMA_MODULE_PT_KEY),
+    emqx_config:erase_schema_mod_and_names(),
+    ok = emqx_config:delete_override_conf_files(),
+    application:unset_env(emqx, local_override_conf_file),
+    application:unset_env(emqx, cluster_override_conf_file),
+    application:unset_env(gen_rpc, port_discovery),
     ok.
 
 proj_root() ->

apps/emqx_authz/test/emqx_authz_file_SUITE.erl

@@ -55,7 +55,7 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
-    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),

apps/emqx_authz/test/emqx_authz_http_SUITE.erl

@@ -52,7 +52,7 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = stop_apps([emqx_resource, cowboy]),
-    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
 
 set_special_configs(emqx_authz) ->
     ok = emqx_authz_test_lib:reset_authorizers();

apps/emqx_authz/test/emqx_authz_mnesia_SUITE.erl

@@ -36,7 +36,7 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
-    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),

apps/emqx_authz/test/emqx_authz_mongodb_SUITE.erl

@@ -50,7 +50,7 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = stop_apps([emqx_resource]),
-    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
 
 set_special_configs(emqx_authz) ->
     ok = emqx_authz_test_lib:reset_authorizers();

apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl

@@ -57,7 +57,7 @@ end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = emqx_resource:remove_local(?MYSQL_RESOURCE),
     ok = stop_apps([emqx_resource]),
-    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),

apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl

@@ -57,7 +57,7 @@ end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = emqx_resource:remove_local(?PGSQL_RESOURCE),
     ok = stop_apps([emqx_resource]),
-    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),

apps/emqx_authz/test/emqx_authz_redis_SUITE.erl

@@ -58,7 +58,7 @@ end_per_suite(_Config) ->
     ok = emqx_authz_test_lib:restore_authorizers(),
     ok = emqx_resource:remove_local(?REDIS_RESOURCE),
     ok = stop_apps([emqx_resource]),
-    ok = emqx_common_test_helpers:stop_apps([emqx_authz]).
+    ok = emqx_common_test_helpers:stop_apps([emqx_conf, emqx_authz]).
 
 init_per_testcase(_TestCase, Config) ->
     ok = emqx_authz_test_lib:reset_authorizers(),

apps/emqx_exhook/test/emqx_exhook_SUITE.erl

@@ -24,13 +24,13 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("emqx/include/emqx_hooks.hrl").
+-include_lib("emqx_conf/include/emqx_conf.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 
 -define(DEFAULT_CLUSTER_NAME_ATOM, emqxcl).
 
 -define(OTHER_CLUSTER_NAME_ATOM, test_emqx_cluster).
 -define(OTHER_CLUSTER_NAME_STRING, "test_emqx_cluster").
--define(CLUSTER_RPC_SHARD, emqx_cluster_rpc_shard).
 
 -define(CONF_DEFAULT, <<
     "\n"
@@ -54,6 +54,8 @@
     "}\n"
 >>).
 
+-import(emqx_common_test_helpers, [on_exit/1]).
+
 %%--------------------------------------------------------------------
 %% Setups
 %%--------------------------------------------------------------------
@@ -89,7 +91,7 @@ init_per_testcase(_, Config) ->
     timer:sleep(200),
     Config.
 
-end_per_testcase(_, Config) ->
+end_per_testcase(_, _Config) ->
     case erlang:whereis(node()) of
         undefined ->
             ok;
@@ -97,7 +99,8 @@ end_per_testcase(_, Config) ->
             erlang:unlink(P),
             erlang:exit(P, kill)
     end,
-    Config.
+    emqx_common_test_helpers:call_janitor(),
+    ok.
 
 load_cfg(Cfg) ->
     ok = emqx_common_test_helpers:load_config(emqx_exhook_schema, Cfg).
@@ -300,6 +303,12 @@ t_cluster_name(_) ->
 
     emqx_common_test_helpers:stop_apps([emqx, emqx_exhook]),
     emqx_common_test_helpers:start_apps([emqx, emqx_exhook], SetEnvFun),
+    on_exit(fun() ->
+        emqx_common_test_helpers:stop_apps([emqx, emqx_exhook]),
+        load_cfg(?CONF_DEFAULT),
+        emqx_common_test_helpers:start_apps([emqx_exhook]),
+        mria:wait_for_tables([?CLUSTER_MFA, ?CLUSTER_COMMIT])
+    end),
 
     ?assertEqual(?OTHER_CLUSTER_NAME_STRING, emqx_sys:cluster_name()),
 

apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl

@@ -77,7 +77,7 @@ init_per_suite(Config) ->
 end_per_suite(Config) ->
     emqx_gateway_auth_ct:stop(),
     emqx_config:erase(gateway),
-    emqx_mgmt_api_test_util:end_suite([cowboy, emqx_authn, emqx_gateway]),
+    emqx_mgmt_api_test_util:end_suite([cowboy, emqx_conf, emqx_authn, emqx_gateway]),
     Config.
 
 init_per_testcase(_Case, Config) ->