feat: make `/data/export` parameterizable, remove `/data/export_cloud`

Fixes https://emqx.atlassian.net/browse/EMQX-13381

apps/emqx_management/src/emqx_mgmt_api_data_backup.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_mgmt_api_data_backup).
 
+-feature(maybe_expr, enable).
+
 -behaviour(minirest_api).
 
 -include_lib("emqx/include/logger.hrl").
@@ -25,7 +27,6 @@
 
 -export([
     data_export/2,
-    data_export_cloud/2,
     data_import/2,
     data_files/2,
     data_file_by_name/2
@@ -57,7 +58,6 @@ api_spec() ->
 paths() ->
     [
         "/data/export",
-        "/data/export_cloud",
         "/data/import",
         "/data/files",
         "/data/files/:filename"
@@ -69,28 +69,22 @@ schema("/data/export") ->
         post => #{
             tags => ?TAGS,
             desc => <<"Export a data backup file">>,
+            'requestBody' => emqx_dashboard_swagger:schema_with_example(
+                ?R_REF(export_request_body),
+                export_request_example()
+            ),
             responses => #{
                 200 =>
                     emqx_dashboard_swagger:schema_with_example(
                         ?R_REF(backup_file_info),
                         backup_file_info_example()
-                    )
-            }
-        }
-    };
-schema("/data/export_cloud") ->
-    #{
-        'operationId' => data_export_cloud,
-        post => #{
-            tags => ?TAGS,
-            hidden => true,
-            desc => <<"Export a data backup file with limited scope (cloud edition)">>,
-            responses => #{
-                200 =>
-                    emqx_dashboard_swagger:schema_with_example(
-                        ?R_REF(backup_file_info),
-                        backup_file_info_example()
-                    )
+                    ),
+                400 => emqx_dashboard_swagger:error_codes(
+                    [?BAD_REQUEST], <<"Invalid table sets: bar, foo">>
+                ),
+                500 => emqx_dashboard_swagger:error_codes(
+                    [?BAD_REQUEST], <<"Error processing export: ...">>
+                )
             }
         }
     };
@@ -197,6 +191,33 @@ fields(backup_file_info) ->
                 required => true
             })}
     ];
+fields(export_request_body) ->
+    AllTableSetNames = emqx_mgmt_data_backup:all_table_set_names(),
+    TableSetsDesc = iolist_to_binary([
+        [
+            <<"Sets of tables to export. Exports all if omitted.">>,
+            <<" Valid values:\n\n">>
+        ]
+        | lists:map(fun(Name) -> ["- ", Name, $\n] end, AllTableSetNames)
+    ]),
+    [
+        {table_sets,
+            hoconsc:mk(
+                hoconsc:array(binary()),
+                #{
+                    required => false,
+                    desc => TableSetsDesc
+                }
+            )},
+        {root_keys,
+            hoconsc:mk(
+                hoconsc:array(binary()),
+                #{
+                    required => false,
+                    desc => <<"Sets of root configuration keys to export. Exports all if omitted.">>
+                }
+            )}
+    ];
 fields(import_request_body) ->
     [?node_field(false), ?filename_field(true)];
 fields(data_backup_file) ->
@@ -213,20 +234,24 @@ fields(data_backup_file) ->
 %% HTTP API Callbacks
 %%------------------------------------------------------------------------------
 
-data_export(post, _Request) ->
-    case emqx_mgmt_data_backup:export() of
-        {ok, #{filename := FileName} = File} ->
-            {200, File#{filename => filename:basename(FileName)}};
-        Error ->
-            Error
-    end.
-
-data_export_cloud(post, _Request) ->
-    case emqx_mgmt_data_backup:export_for_cloud() of
-        {ok, #{filename := FileName} = File} ->
-            {200, File#{filename => filename:basename(FileName)}};
-        Error ->
-            Error
+data_export(post, #{body := Params}) ->
+    maybe
+        {ok, Opts} ?= parse_export_request(Params),
+        {ok, #{filename := FileName} = File} ?= emqx_mgmt_data_backup:export(Opts),
+        {200, File#{filename => filename:basename(FileName)}}
+    else
+        {error, {bad_table_sets, InvalidSetNames}} ->
+            Msg = iolist_to_binary([
+                <<"Invalid table sets: ">>,
+                lists:join(<<", ">>, InvalidSetNames)
+            ]),
+            {400, #{code => ?BAD_REQUEST, message => Msg}};
+        {error, Reason} ->
+            Msg = iolist_to_binary([
+                <<"Error processing export: ">>,
+                emqx_utils_conv:bin(Reason)
+            ]),
+            {500, #{code => 'INTERNAL_ERROR', message => Msg}}
     end.
 
 data_import(post, #{body := #{<<"filename">> := FileName} = Body}) ->
@@ -291,6 +316,32 @@ data_file_by_name(Method, #{bindings := #{filename := Filename}, query_string :=
 %% Internal functions
 %%------------------------------------------------------------------------------
 
+parse_export_request(Params) ->
+    Opts0 = #{},
+    Opts1 =
+        maybe
+            {ok, Keys0} ?= maps:find(<<"root_keys">>, Params),
+            Keys = lists:usort(Keys0),
+            Transform = fun(RawConf) ->
+                maps:with(Keys, RawConf)
+            end,
+            Opts0#{raw_conf_transform => Transform}
+        else
+            error -> Opts0
+        end,
+    maybe
+        {ok, TableSetNames0} ?= maps:find(<<"table_sets">>, Params),
+        TableSetNames = lists:usort(TableSetNames0),
+        {ok, Filter} ?= emqx_mgmt_data_backup:compile_mnesia_table_filter(TableSetNames),
+        {ok, Opts1#{mnesia_table_filter => Filter}}
+    else
+        error ->
+            {ok, Opts1};
+        {error, InvalidSetNames0} ->
+            InvalidSetNames = lists:sort(InvalidSetNames0),
+            {error, {bad_table_sets, InvalidSetNames}}
+    end.
+
 get_or_delete_file(get, Filename, Node) ->
     emqx_mgmt_data_backup_proto_v1:read_file(Node, Filename, infinity);
 get_or_delete_file(delete, Filename, Node) ->
@@ -369,6 +420,23 @@ backup_file_info_example() ->
         size => 22740
     }.
 
+export_request_example() ->
+    #{
+        table_sets => [
+            <<"banned">>,
+            <<"builtin_authn">>,
+            <<"builtin_authn_scram">>,
+            <<"builtin_authz">>
+        ],
+        root_keys => [
+            <<"connectors">>,
+            <<"actions">>,
+            <<"sources">>,
+            <<"rule_engine">>,
+            <<"schema_registry">>
+        ]
+    }.
+
 files_response_example() ->
     #{
         data => [

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -16,9 +16,12 @@
 
 -module(emqx_mgmt_data_backup).
 
+-feature(maybe_expr, enable).
+
 -export([
     export/0,
-    export_for_cloud/0,
+    all_table_set_names/0,
+    compile_mnesia_table_filter/1,
     export/1,
     import/1,
     import/2,
@@ -111,6 +114,14 @@
 -type import_res() ::
     {ok, #{db_errors => db_error_details(), config_errors => config_error_details()}} | {error, _}.
 
+-type export_opts() :: #{
+    mnesia_table_filter => mnesia_table_filter(),
+    print_fun => fun((io:format(), [term()]) -> ok),
+    raw_conf_transform => fun((raw_config()) -> raw_config())
+}.
+-type raw_config() :: #{binary() => any()}.
+-type mnesia_table_filter() :: fun((atom()) -> boolean()).
+
 %%------------------------------------------------------------------------------
 %% APIs
 %%------------------------------------------------------------------------------
@@ -119,16 +130,33 @@
 export() ->
     export(?DEFAULT_OPTS).
 
--spec export_for_cloud() -> {ok, backup_file_info()} | {error, _}.
-export_for_cloud() ->
-    Default = ?DEFAULT_OPTS,
-    Opts = Default#{
-        raw_conf_transform => fun cloud_export_raw_conf_transform/1,
-        mnesia_table_filter => fun cloud_export_mnesia_table_filter/1
-    },
-    export(Opts).
+-spec compile_mnesia_table_filter([binary()]) -> {ok, mnesia_table_filter()} | {error, any()}.
+compile_mnesia_table_filter(TableSets) ->
+    Mapping = table_set_to_module_mapping(),
+    {TableNames, Errors} =
+        lists:foldl(
+            fun(TableSetName, {TableAcc, ErrorAcc}) ->
+                maybe
+                    {ok, Mod} ?= maps:find(TableSetName, Mapping),
+                    Tables = Mod:backup_tables(),
+                    {lists:usort(Tables ++ TableAcc), ErrorAcc}
+                else
+                    error ->
+                        {TableAcc, [TableSetName | ErrorAcc]}
+                end
+            end,
+            {[], []},
+            TableSets
+        ),
+    case Errors of
+        [] ->
+            Filter = fun(Table) -> lists:member(Table, TableNames) end,
+            {ok, Filter};
+        _ ->
+            {error, Errors}
+    end.
 
--spec export(map()) -> {ok, backup_file_info()} | {error, _}.
+-spec export(export_opts()) -> {ok, backup_file_info()} | {error, _}.
 export(Opts) ->
     {BackupName, TarDescriptor} = prepare_new_backup(Opts),
     try
@@ -148,6 +176,27 @@ export(Opts) ->
         file:del_dir_r(BackupName)
     end.
 
+-spec all_table_set_names() -> [binary()].
+all_table_set_names() ->
+    Key = {?MODULE, all_table_set_names},
+    case persistent_term:get(Key, undefined) of
+        undefined ->
+            Names = build_all_table_set_names(),
+            persistent_term:put(Key, Names),
+            Names;
+        Names ->
+            Names
+    end.
+
+build_all_table_set_names() ->
+    Mods = modules_with_mnesia_tabs_to_backup(),
+    lists:sort(
+        lists:map(
+            fun emqx_db_backup:table_set_name/1,
+            Mods
+        )
+    ).
+
 -spec import(file:filename_all()) -> import_res().
 import(BackupFileName) ->
     import(BackupFileName, ?DEFAULT_OPTS).
@@ -440,13 +489,13 @@ do_export_mnesia_tab(TabName, BackupName) ->
 -ifdef(TEST).
 tabs_to_backup() ->
     %% Allow mocking in tests
-    ?MODULE:mnesia_tabs_to_backup().
+    ?MODULE:modules_with_mnesia_tabs_to_backup().
 -else.
 tabs_to_backup() ->
-    mnesia_tabs_to_backup().
+    modules_with_mnesia_tabs_to_backup().
 -endif.
 
-mnesia_tabs_to_backup() ->
+modules_with_mnesia_tabs_to_backup() ->
     lists:flatten([M || M <- find_behaviours(emqx_db_backup)]).
 
 mnesia_backup_name(Path, TabName) ->
@@ -1008,31 +1057,27 @@ apps() ->
         end
     ].
 
-cloud_export_raw_conf_transform(RawConf) ->
-    maps:with(
-        [
-            <<"connectors">>,
-            <<"actions">>,
-            <<"sources">>,
-            <<"rule_engine">>,
-            <<"schema_registry">>
-        ],
-        RawConf
-    ).
+-spec table_set_to_module_mapping() -> #{binary() => module()}.
+table_set_to_module_mapping() ->
+    Key = {?MODULE, table_set_to_module_mapping},
+    case persistent_term:get(Key, undefined) of
+        undefined ->
+            Mapping = build_table_set_to_module_mapping(),
+            persistent_term:put(Key, Mapping),
+            Mapping;
+        Mapping ->
+            Mapping
+    end.
 
-cloud_export_mnesia_table_filter(TableName) ->
-    lists:member(
-        TableName,
-        [
-            %% mnesia builtin authn
-            emqx_authn_mnesia,
-            emqx_authn_scram_mnesia,
-            %% mnesia builtin authz
-            emqx_acl,
-            %% banned
-            emqx_banned,
-            emqx_banned_rules
-        ]
+build_table_set_to_module_mapping() ->
+    Mods = modules_with_mnesia_tabs_to_backup(),
+    lists:foldl(
+        fun(Mod, Acc) ->
+            Name = emqx_db_backup:table_set_name(Mod),
+            Acc#{Name => Mod}
+        end,
+        #{},
+        Mods
     ).
 
 -ifdef(TEST).

apps/emqx_management/test/emqx_mgmt_api_data_backup_SUITE.erl

@@ -133,14 +133,26 @@ t_import_ee_backup(Config) ->
         ce -> ok
     end.
 
-%% Simple smoke test for cloud export API.
+%% Simple smoke test for cloud export API (export with scoped table set names and root
+%% keys).
 t_export_cloud(Config) ->
     Auth = ?config(auth, Config),
-    {ok, RawResp} = export_cloud_backup(?NODE1_PORT, Auth),
-    #{<<"filename">> := Filepath} = emqx_utils_json:decode(RawResp),
+    Resp = export_cloud_backup(?NODE1_PORT, Auth),
+    {200, #{<<"filename">> := Filepath}} = Resp,
     {ok, _} = import_backup(?NODE1_PORT, Auth, Filepath),
     ok.
 
+%% Checks returned error when one or more invalid table set names are given to the export
+%% request.
+t_export_bad_table_sets(Config) ->
+    Auth = ?config(auth, Config),
+    Body = #{<<"table_sets">> => [<<"foo">>, <<"bar">>, <<"foo">>]},
+    ?assertMatch(
+        {400, #{<<"message">> := <<"Invalid table sets: bar, foo">>}},
+        export_backup2(?NODE1_PORT, Auth, Body)
+    ),
+    ok.
+
 do_init_per_testcase(TC, Config) ->
     Cluster = [Core1, _Core2, Repl] = cluster(TC, Config),
     Auth = auth_header(Core1),
@@ -247,12 +259,30 @@ assert_second_call(delete, Res) ->
     ?assertMatch({error, {_, 404, _}}, Res).
 
 export_cloud_backup(NodeApiPort, Auth) ->
-    Path = ["data", "export_cloud"],
-    request(post, NodeApiPort, Path, Auth).
+    Body = #{
+        <<"table_sets">> => [
+            <<"banned">>,
+            <<"builtin_authn">>,
+            <<"builtin_authn_scram">>,
+            <<"builtin_authz">>
+        ],
+        <<"root_keys">> => [
+            <<"connectors">>,
+            <<"actions">>,
+            <<"sources">>,
+            <<"rule_engine">>,
+            <<"schema_registry">>
+        ]
+    },
+    export_backup2(NodeApiPort, Auth, Body).
 
 export_backup(NodeApiPort, Auth) ->
     Path = ["data", "export"],
-    request(post, NodeApiPort, Path, Auth).
+    request(post, NodeApiPort, Path, _Body = #{}, Auth).
+
+export_backup2(NodeApiPort, Auth, Body) ->
+    Path = emqx_mgmt_api_test_util:api_path(?api_base_url(NodeApiPort), ["data", "export"]),
+    emqx_mgmt_api_test_util:simple_request(post, Path, Body, Auth).
 
 import_backup(NodeApiPort, Auth, BackupName) ->
     import_backup(NodeApiPort, Auth, BackupName, undefined).
@@ -339,7 +369,7 @@ wait_for_auth_replication(ReplNode, Retries) ->
 apps_spec(APIPort, TC) ->
     common_apps_spec() ++
         app_spec_dashboard(APIPort) ++
-        upload_import_apps_spec(TC).
+        test_case_specific_apps_spec(TC).
 
 common_apps_spec() ->
     [
@@ -367,7 +397,7 @@ app_spec_dashboard(APIPort) ->
         }}
     ].
 
-upload_import_apps_spec(TC) when
+test_case_specific_apps_spec(TC) when
     TC =:= t_upload_ee_backup;
     TC =:= t_import_ee_backup;
     TC =:= t_upload_ce_backup;
@@ -382,5 +412,10 @@ upload_import_apps_spec(TC) when
         emqx_modules,
         emqx_bridge
     ];
-upload_import_apps_spec(_TC) ->
+test_case_specific_apps_spec(t_export_cloud) ->
+    [
+        emqx_auth,
+        emqx_auth_mnesia
+    ];
+test_case_specific_apps_spec(_TC) ->
     [].

apps/emqx_management/test/emqx_mgmt_api_test_util.erl

@@ -112,7 +112,7 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, [], Opts) when
             _ -> Url ++ "?" ++ build_query_string(QueryParams)
         end,
     do_request_api(Method, {NewUrl, build_http_header(AuthOrHeaders)}, Opts);
-request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when
+request_api(Method, Url, QueryParams, AuthOrHeaders, Body0, Opts) when
     (Method =:= post) orelse
         (Method =:= patch) orelse
         (Method =:= put) orelse
@@ -124,9 +124,14 @@ request_api(Method, Url, QueryParams, AuthOrHeaders, Body, Opts) when
             "" -> Url;
             _ -> Url ++ "?" ++ QueryParams
         end,
+    Body =
+        case Body0 of
+            {raw, B} -> B;
+            _ -> emqx_utils_json:encode(Body0)
+        end,
     do_request_api(
         Method,
-        {NewUrl, build_http_header(AuthOrHeaders), ContentType, emqx_utils_json:encode(Body)},
+        {NewUrl, build_http_header(AuthOrHeaders), ContentType, Body},
         maps:remove('content-type', Opts)
     ).
 

apps/emqx_management/test/emqx_mgmt_data_backup_SUITE.erl

@@ -91,7 +91,7 @@ init_per_testcase(TC = t_verify_imported_mnesia_tab_on_cluster, Config) ->
     [{cluster, cluster(TC, Config)} | setup(TC, Config)];
 init_per_testcase(t_mnesia_bad_tab_schema, Config) ->
     meck:new(emqx_mgmt_data_backup, [passthrough]),
-    meck:expect(TC = emqx_mgmt_data_backup, mnesia_tabs_to_backup, 0, [?MODULE]),
+    meck:expect(TC = emqx_mgmt_data_backup, modules_with_mnesia_tabs_to_backup, 0, [?MODULE]),
     setup(TC, Config);
 init_per_testcase(TC, Config) ->
     setup(TC, Config).
@@ -207,7 +207,36 @@ t_export_ram_retained_messages(_Config) ->
 
 t_export_cloud_subset(Config) ->
     setup_t_export_cloud_subset_scenario(),
-    {ok, #{filename := BackupFileName}} = emqx_mgmt_data_backup:export_for_cloud(),
+    Opts = #{
+        raw_conf_transform => fun(RawConf) ->
+            maps:with(
+                [
+                    <<"connectors">>,
+                    <<"actions">>,
+                    <<"sources">>,
+                    <<"rule_engine">>,
+                    <<"schema_registry">>
+                ],
+                RawConf
+            )
+        end,
+        mnesia_table_filter => fun(TableName) ->
+            lists:member(
+                TableName,
+                [
+                    %% mnesia builtin authn
+                    emqx_authn_mnesia,
+                    emqx_authn_scram_mnesia,
+                    %% mnesia builtin authz
+                    emqx_acl,
+                    %% banned
+                    emqx_banned,
+                    emqx_banned_rules
+                ]
+            )
+        end
+    },
+    {ok, #{filename := BackupFileName}} = emqx_mgmt_data_backup:export(Opts),
     #{
         cluster_hocon := RawHocon,
         mnesia_tables := Tables

apps/emqx_message_transformation/test/emqx_message_transformation_http_api_SUITE.erl

@@ -262,7 +262,7 @@ upload_backup(BackupFilePath) ->
 
 export_backup() ->
     Path = emqx_mgmt_api_test_util:api_path(["data", "export"]),
-    Res = request(post, Path, []),
+    Res = request(post, Path, {raw, <<>>}),
     simplify_result(Res).
 
 import_backup(BackupName) ->

apps/emqx_schema_validation/test/emqx_schema_validation_http_api_SUITE.erl

@@ -243,7 +243,7 @@ upload_backup(BackupFilePath) ->
 
 export_backup() ->
     Path = emqx_mgmt_api_test_util:api_path(["data", "export"]),
-    Res = request(post, Path, []),
+    Res = request(post, Path, {raw, <<>>}),
     simplify_result(Res).
 
 import_backup(BackupName) ->