Merge pull request #13108 from keynslug/fix/ct-flaky/mgmt

test(mgmt): avoid inter-suite contamination by switching to cth tooling

apps/emqx_management/test/emqx_mgmt_api_alarms_SUITE.erl

@@ -19,6 +19,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
 
 -define(ACT_ALARM, test_act_alarm).
 -define(DE_ACT_ALARM, test_de_act_alarm).
@@ -27,11 +28,18 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_mgmt_api_test_util:init_suite(),
-    Config.
+    Apps = emqx_cth_suite:start(
+        [
+            emqx,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{suite_apps, Apps} | Config].
 
-end_per_suite(_) ->
-    emqx_mgmt_api_test_util:end_suite().
+end_per_suite(Config) ->
+    ok = emqx_cth_suite:stop(?config(suite_apps, Config)).
 
 t_alarms_api(_) ->
     ok = emqx_alarm:activate(?ACT_ALARM),

apps/emqx_management/test/emqx_mgmt_api_api_keys_SUITE.erl

@@ -56,11 +56,18 @@ groups() ->
     ].
 
 init_per_suite(Config) ->
-    emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_management]),
-    Config.
+    Apps = emqx_cth_suite:start(
+        [
+            emqx_conf,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{suite_apps, Apps} | Config].
 
-end_per_suite(_) ->
-    emqx_mgmt_api_test_util:end_suite([emqx_conf, emqx_management]).
+end_per_suite(Config) ->
+    ok = emqx_cth_suite:stop(?config(suite_apps, Config)).
 
 t_bootstrap_file(_) ->
     TestPath = <<"/api/v5/status">>,

apps/emqx_management/test/emqx_mgmt_api_banned_SUITE.erl

@@ -19,6 +19,7 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
 
 -define(EXPIRATION_TIME, 31536000).
 
@@ -26,11 +27,18 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_mgmt_api_test_util:init_suite(),
-    Config.
+    Apps = emqx_cth_suite:start(
+        [
+            emqx,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{suite_apps, Apps} | Config].
 
-end_per_suite(_) ->
-    emqx_mgmt_api_test_util:end_suite().
+end_per_suite(Config) ->
+    ok = emqx_cth_suite:stop(?config(suite_apps, Config)).
 
 t_create(_Config) ->
     Now = erlang:system_time(second),

apps/emqx_management/test/emqx_mgmt_api_cluster_SUITE.erl

@@ -41,9 +41,12 @@ init_per_testcase(TC = t_cluster_invite_api_timeout, Config0) ->
 init_per_testcase(TC = t_cluster_invite_async, Config0) ->
     Config = [{tc_name, TC} | Config0],
     [{cluster, cluster(Config)} | setup(Config)];
-init_per_testcase(_TC, Config) ->
-    emqx_mgmt_api_test_util:init_suite(?APPS),
-    Config.
+init_per_testcase(TC, Config) ->
+    Apps = emqx_cth_suite:start(
+        ?APPS ++ [emqx_mgmt_api_test_util:emqx_dashboard()],
+        #{work_dir => emqx_cth_suite:work_dir(TC, Config)}
+    ),
+    [{tc_apps, Apps} | Config].
 
 end_per_testcase(t_cluster_topology_api_replicants, Config) ->
     emqx_cth_cluster:stop(?config(cluster, Config)),
@@ -54,8 +57,8 @@ end_per_testcase(t_cluster_invite_api_timeout, Config) ->
 end_per_testcase(t_cluster_invite_async, Config) ->
     emqx_cth_cluster:stop(?config(cluster, Config)),
     cleanup(Config);
-end_per_testcase(_TC, _Config) ->
-    emqx_mgmt_api_test_util:end_suite(?APPS).
+end_per_testcase(_TC, Config) ->
+    ok = emqx_cth_suite:stop(?config(tc_apps, Config)).
 
 t_cluster_topology_api_empty_resp(_) ->
     ClusterTopologyPath = emqx_mgmt_api_test_util:api_path(["cluster", "topology"]),

apps/emqx_management/test/emqx_mgmt_api_configs_SUITE.erl

@@ -25,11 +25,18 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_mgmt_api_test_util:init_suite([emqx_conf]),
-    Config.
+    Apps = emqx_cth_suite:start(
+        [
+            emqx_conf,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{suite_apps, Apps} | Config].
 
-end_per_suite(_) ->
-    emqx_mgmt_api_test_util:end_suite([emqx_conf]).
+end_per_suite(Config) ->
+    ok = emqx_cth_suite:stop(?config(suite_apps, Config)).
 
 init_per_testcase(TestCase = t_configs_node, Config) ->
     ?MODULE:TestCase({'init', Config});

apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl

@@ -74,13 +74,10 @@ init_group_apps(Config, CTConfig) ->
         [
             {emqx_conf, Config},
             emqx_management,
-            {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}
+            emqx_mgmt_api_test_util:emqx_dashboard()
         ],
-        #{
-            work_dir => emqx_cth_suite:work_dir(CTConfig)
-        }
+        #{work_dir => emqx_cth_suite:work_dir(CTConfig)}
     ),
-    {ok, _} = emqx_common_test_http:create_default_app(),
     [{suite_apps, Apps} | CTConfig].
 
 end_per_group(_Group, Config) ->

apps/emqx_management/test/emqx_mgmt_api_metrics_SUITE.erl

@@ -19,16 +19,24 @@
 -compile(nowarn_export_all).
 
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_mgmt_api_test_util:init_suite(),
-    Config.
+    Apps = emqx_cth_suite:start(
+        [
+            emqx,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{suite_apps, Apps} | Config].
 
-end_per_suite(_) ->
-    emqx_mgmt_api_test_util:end_suite().
+end_per_suite(Config) ->
+    ok = emqx_cth_suite:stop(?config(suite_apps, Config)).
 
 t_metrics_api(_) ->
     {ok, MetricsResponse} = request_helper("metrics?aggregate=true"),

apps/emqx_management/test/emqx_mgmt_api_nodes_SUITE.erl

@@ -29,11 +29,10 @@ init_per_suite(Config) ->
         [
             emqx_conf,
             emqx_management,
-            {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}
+            emqx_mgmt_api_test_util:emqx_dashboard()
         ],
         #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
-    {ok, _} = emqx_common_test_http:create_default_app(),
     [{suite_apps, Apps} | Config].
 
 end_per_suite(Config) ->

apps/emqx_management/test/emqx_mgmt_api_plugins_SUITE.erl

@@ -34,24 +34,23 @@ all() ->
 
 init_per_suite(Config) ->
     WorkDir = proplists:get_value(data_dir, Config),
-    ok = filelib:ensure_dir(WorkDir),
     DemoShDir1 = string:replace(WorkDir, "emqx_mgmt_api_plugins", "emqx_plugins"),
     DemoShDir = lists:flatten(string:replace(DemoShDir1, "emqx_management", "emqx_plugins")),
-    OrigInstallDir = emqx_plugins:get_config_interal(install_dir, undefined),
+    Apps = emqx_cth_suite:start(
+        [
+            emqx_conf,
+            emqx_plugins,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
     ok = filelib:ensure_dir(DemoShDir),
-    emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_plugins]),
     emqx_plugins:put_config_internal(install_dir, DemoShDir),
-    [{demo_sh_dir, DemoShDir}, {orig_install_dir, OrigInstallDir} | Config].
+    [{apps, Apps}, {demo_sh_dir, DemoShDir} | Config].
 
 end_per_suite(Config) ->
-    emqx_common_test_helpers:boot_modules(all),
-    %% restore config
-    case proplists:get_value(orig_install_dir, Config) of
-        undefined -> ok;
-        OrigInstallDir -> emqx_plugins:put_config_internal(install_dir, OrigInstallDir)
-    end,
-    emqx_mgmt_api_test_util:end_suite([emqx_plugins, emqx_conf]),
-    ok.
+    ok = emqx_cth_suite:stop(?config(apps, Config)).
 
 init_per_testcase(t_cluster_update_order = TestCase, Config0) ->
     Config = [{api_port, 18085} | Config0],

apps/emqx_management/test/emqx_mgmt_api_trace_SUITE.erl

@@ -24,6 +24,7 @@
 -include_lib("stdlib/include/zip.hrl").
 -include_lib("snabbkaffe/include/snabbkaffe.hrl").
 -include_lib("emqx/include/logger.hrl").
+-include_lib("common_test/include/ct.hrl").
 
 %%--------------------------------------------------------------------
 %% Setups
@@ -33,11 +34,18 @@ all() ->
     emqx_common_test_helpers:all(?MODULE).
 
 init_per_suite(Config) ->
-    emqx_mgmt_api_test_util:init_suite(),
-    Config.
-
-end_per_suite(_) ->
-    emqx_mgmt_api_test_util:end_suite().
+    Apps = emqx_cth_suite:start(
+        [
+            emqx,
+            emqx_management,
+            emqx_mgmt_api_test_util:emqx_dashboard()
+        ],
+        #{work_dir => emqx_cth_suite:work_dir(Config)}
+    ),
+    [{apps, Apps} | Config].
+
+end_per_suite(Config) ->
+    ok = emqx_cth_suite:stop(?config(apps, Config)).
 
 t_http_test(_Config) ->
     emqx_trace:clear(),