chore: delete import and export feature

apps/emqx_authentication/rebar.config

@@ -1,5 +1,4 @@
-{deps, [{jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}}
-       ]}.
+{deps, []}.
 
 {edoc_opts, [{preprocess, true}]}.
 {erl_opts, [warn_unused_vars,

apps/emqx_connector/rebar.config

@@ -4,7 +4,21 @@
 ]}.
 
 {deps, [
-  {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}
+  {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.1"}}},
+  {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
+  {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}},
+  {epgsql, {git, "https://github.com/epgsql/epgsql", {tag, "4.4.0"}}},
+  %% NOTE: mind poolboy version when updating mongodb-erlang version
+  {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.7"}}},
+  %% NOTE: mind poolboy version when updating eredis_cluster version
+  {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.6"}}},
+  %% mongodb-erlang uses a special fork https://github.com/comtihon/poolboy.git
+  %% (which has overflow_ttl feature added).
+  %% However, it references `{branch, "master}` (commit 9c06a9a on 2021-04-07).
+  %% By accident, We have always been using the upstream fork due to
+  %% eredis_cluster's dependency getting resolved earlier.
+  %% Here we pin 1.5.2 to avoid surprises in the future.
+  {poolboy, {git, "https://github.com/emqx/poolboy.git", {tag, "1.5.2"}}}
 ]}.
 
 {shell, [

apps/emqx_management/src/emqx_mgmt_api_data.erl

@@ -1,186 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_mgmt_api_data).
-
--include_lib("emqx/include/emqx.hrl").
-
--include_lib("kernel/include/file.hrl").
-
--include("emqx_mgmt.hrl").
-
--rest_api(#{name   => export,
-            method => 'POST',
-            path   => "/data/export",
-            func   => export,
-            descr  => "Export data"}).
-
--rest_api(#{name   => list_exported,
-            method => 'GET',
-            path   => "/data/export",
-            func   => list_exported,
-            descr  => "List exported file"}).
-
--rest_api(#{name   => import,
-            method => 'POST',
-            path   => "/data/import",
-            func   => import,
-            descr  => "Import data"}).
-
--rest_api(#{name   => download,
-            method => 'GET',
-            path   => "/data/file/:filename",
-            func   => download,
-            descr  => "Download data file to local"}).
-
--rest_api(#{name   => upload,
-            method => 'POST',
-            path   => "/data/file",
-            func   => upload,
-            descr  => "Upload data file from local"}).
-
--rest_api(#{name   => delete,
-            method => 'DELETE',
-            path   => "/data/file/:filename",
-            func   => delete,
-            descr  => "Delete data file"}).
-
--export([ export/2
-        , list_exported/2
-        , import/2
-        , download/2
-        , upload/2
-        , delete/2
-        ]).
-
--export([ get_list_exported/0
-        , do_import/1
-        ]).
-
-export(_Bindings, _Params) ->
-    case emqx_mgmt_data_backup:export() of
-        {ok, File = #{filename := Filename}} ->
-            minirest:return({ok, File#{filename => filename:basename(Filename)}});
-        Return -> minirest:return(Return)
-    end.
-
-list_exported(_Bindings, _Params) ->
-    List = [ rpc:call(Node, ?MODULE, get_list_exported, []) || Node <- ekka_mnesia:running_nodes() ],
-    NList = lists:map(fun({_, FileInfo}) -> FileInfo end, lists:keysort(1, lists:append(List))),
-    minirest:return({ok, NList}).
-
-get_list_exported() ->
-    Dir = emqx:get_env(data_dir),
-    {ok, Files} = file:list_dir_all(Dir),
-    lists:foldl(
-        fun(File, Acc) ->
-            case filename:extension(File) =:= ".json" of
-                true ->
-                    FullFile = filename:join([Dir, File]),
-                    case file:read_file_info(FullFile) of
-                        {ok, #file_info{size = Size, ctime = CTime = {{Y, M, D}, {H, MM, S}}}} ->
-                            CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y, M, D, H, MM, S]),
-                            Seconds = calendar:datetime_to_gregorian_seconds(CTime),
-                            [{Seconds, [{filename, list_to_binary(File)},
-                                        {size, Size},
-                                        {created_at, list_to_binary(CreatedAt)},
-                                        {node, node()}
-                                       ]} | Acc];
-                        {error, Reason} ->
-                            logger:error("Read file info of ~s failed with: ~p", [File, Reason]),
-                            Acc
-                    end;
-                false -> Acc
-            end
-        end, [], Files).
-
-import(_Bindings, Params) ->
-    case proplists:get_value(<<"filename">>, Params) of
-        undefined ->
-            Result = import_content(Params),
-            minirest:return(Result);
-        Filename ->
-            case proplists:get_value(<<"node">>, Params) of
-                undefined ->
-                    Result = do_import(Filename),
-                    minirest:return(Result);
-                Node ->
-                    case lists:member(Node,
-                          [ erlang:atom_to_binary(N, utf8) || N <- ekka_mnesia:running_nodes() ]
-                         ) of
-                        true -> minirest:return(rpc:call(erlang:binary_to_atom(Node, utf8), ?MODULE, do_import, [Filename]));
-                        false -> minirest:return({error, no_existent_node})
-                    end
-            end
-    end.
-
-do_import(Filename) ->
-    FullFilename = fullname(Filename),
-    emqx_mgmt_data_backup:import(FullFilename, "{}").
-
-download(#{filename := Filename}, _Params) ->
-    FullFilename = fullname(Filename),
-    case file:read_file(FullFilename) of
-        {ok, Bin} ->
-            {ok, #{filename => list_to_binary(Filename),
-                   file => Bin}};
-        {error, Reason} ->
-            minirest:return({error, Reason})
-    end.
-
-upload(Bindings, Params) ->
-    do_upload(Bindings, maps:from_list(Params)).
-
-do_upload(_Bindings, #{<<"filename">> := Filename,
-                       <<"file">> := Bin}) ->
-    FullFilename = fullname(Filename),
-    case file:write_file(FullFilename, Bin) of
-        ok ->
-            minirest:return({ok, [{node, node()}]});
-        {error, Reason} ->
-            minirest:return({error, Reason})
-    end;
-do_upload(Bindings, Params = #{<<"file">> := _}) ->
-    do_upload(Bindings, Params#{<<"filename">> => tmp_filename()});
-do_upload(_Bindings, _Params) ->
-    minirest:return({error, missing_required_params}).
-
-delete(#{filename := Filename}, _Params) ->
-    FullFilename = fullname(Filename),
-    case file:delete(FullFilename) of
-        ok ->
-            minirest:return();
-        {error, Reason} ->
-            minirest:return({error, Reason})
-    end.
-
-import_content(Content) ->
-    File = dump_to_tmp_file(Content),
-    do_import(File).
-
-dump_to_tmp_file(Content) ->
-    Bin = emqx_json:encode(Content),
-    Filename = tmp_filename(),
-    ok = file:write_file(fullname(Filename), Bin),
-    Filename.
-
-fullname(Name) ->
-    filename:join(emqx:get_env(data_dir), Name).
-
-tmp_filename() ->
-    Seconds = erlang:system_time(second),
-    {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds),
-    io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]).

apps/emqx_management/src/emqx_mgmt_cli.erl

@@ -38,7 +38,6 @@
         , trace/1
         , log/1
         , mgmt/1
-        , data/1
         , acl/1
         ]).
 
@@ -544,36 +543,6 @@ stop_listener(#{listen_on := ListenOn} = Listener, _Input) ->
                            [ID, ListenOnStr, Reason])
     end.
 
-%%--------------------------------------------------------------------
-%% @doc data Command
-
-data(["export"]) ->
-    case emqx_mgmt_data_backup:export() of
-        {ok, #{filename := Filename}} ->
-            emqx_ctl:print("The emqx data has been successfully exported to ~s.~n", [Filename]);
-        {error, Reason} ->
-            emqx_ctl:print("The emqx data export failed due to ~p.~n", [Reason])
-    end;
-
-data(["import", Filename]) ->
-    data(["import", Filename, "--env", "{}"]);
-data(["import", Filename, "--env", Env]) ->
-    case emqx_mgmt_data_backup:import(Filename, Env) of
-        ok ->
-            emqx_ctl:print("The emqx data has been imported successfully.~n");
-        {error, import_failed} ->
-            emqx_ctl:print("The emqx data import failed.~n");
-        {error, unsupported_version} ->
-            emqx_ctl:print("The emqx data import failed: Unsupported version.~n");
-        {error, Reason} ->
-            emqx_ctl:print("The emqx data import failed: ~0p while reading ~s.~n", [Reason, Filename])
-    end;
-
-data(_) ->
-    emqx_ctl:usage([{"data import <File> [--env '<json>']",
-                     "Import data from the specified file, possibly with overrides"},
-                    {"data export", "Export data"}]).
-
 %%--------------------------------------------------------------------
 %% @doc acl Command
 

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -1,739 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_mgmt_data_backup).
-
--include("emqx_mgmt.hrl").
--include_lib("emqx_rule_engine/include/rule_engine.hrl").
--include_lib("emqx/include/emqx.hrl").
--include_lib("kernel/include/file.hrl").
-
--ifdef(EMQX_ENTERPRISE).
--export([ export_modules/0
-        , export_schemas/0
-        , export_confs/0
-        , import_modules/1
-        , import_schemas/1
-        , import_confs/2
-        ]).
--endif.
-
--export([ export_rules/0
-        , export_resources/0
-        , export_blacklist/0
-        , export_applications/0
-        , export_users/0
-        , export_auth_mnesia/0
-        , export_acl_mnesia/0
-        , import_resources_and_rules/3
-        , import_rules/1
-        , import_resources/1
-        , import_blacklist/1
-        , import_applications/1
-        , import_users/1
-        , import_auth_clientid/1 %% BACKW: 4.1.x
-        , import_auth_username/1 %% BACKW: 4.1.x
-        , import_auth_mnesia/2
-        , import_acl_mnesia/2
-        , to_version/1
-        ]).
-
--export([ export/0
-        , import/2
-        ]).
-
-%%--------------------------------------------------------------------
-%% Data Export and Import
-%%--------------------------------------------------------------------
-
-export_rules() ->
-    lists:map(fun(#rule{id = RuleId,
-                        rawsql = RawSQL,
-                        actions = Actions,
-                        enabled = Enabled,
-                        description = Desc}) ->
-                   [{id, RuleId},
-                    {rawsql, RawSQL},
-                    {actions, actions_to_prop_list(Actions)},
-                    {enabled, Enabled},
-                    {description, Desc}]
-               end, emqx_rule_registry:get_rules()).
-
-export_resources() ->
-    lists:map(fun(#resource{id = Id,
-                            type = Type,
-                            config = Config,
-                            created_at = CreatedAt,
-                            description = Desc}) ->
-                   NCreatedAt = case CreatedAt of
-                                    undefined -> null;
-                                    _ -> CreatedAt
-                                end,
-                   [{id, Id},
-                    {type, Type},
-                    {config, maps:to_list(Config)},
-                    {created_at, NCreatedAt},
-                    {description, Desc}]
-               end, emqx_rule_registry:get_resources()).
-
-export_blacklist() ->
-    lists:map(fun(#banned{who = Who, by = By, reason = Reason, at = At, until = Until}) ->
-                  NWho = case Who of
-                             {peerhost, Peerhost} -> {peerhost, inet:ntoa(Peerhost)};
-                             _ -> Who
-                         end,
-                  [{who, [NWho]}, {by, By}, {reason, Reason}, {at, At}, {until, Until}]
-              end, ets:tab2list(emqx_banned)).
-
-export_applications() ->
-    lists:map(fun({_, AppID, AppSecret, Name, Desc, Status, Expired}) ->
-                  [{id, AppID}, {secret, AppSecret}, {name, Name}, {desc, Desc}, {status, Status}, {expired, Expired}]
-              end, ets:tab2list(mqtt_app)).
-
-export_users() ->
-    lists:map(fun({_, Username, Password, Tags}) ->
-                  [{username, Username}, {password, base64:encode(Password)}, {tags, Tags}]
-              end, ets:tab2list(mqtt_admin)).
-
-export_auth_mnesia() ->
-    case ets:info(emqx_user) of
-        undefined -> [];
-        _ ->
-            lists:map(fun({_, {Type, Login}, Password, CreatedAt}) ->
-                          [{login, Login}, {type, Type}, {password, base64:encode(Password)}, {created_at, CreatedAt}]
-                      end, ets:tab2list(emqx_user))
-    end.
-
-export_acl_mnesia() ->
-    case ets:info(emqx_acl) of
-        undefined -> [];
-        _ ->
-            lists:map(fun({_, Filter, Action, Access, CreatedAt}) ->
-                          Filter1 = case Filter of
-                              {{Type, TypeValue}, Topic} ->
-                                  [{type, Type}, {type_value, TypeValue}, {topic, Topic}];
-                              {Type, Topic} ->
-                                  [{type, Type}, {topic, Topic}]
-                          end,
-                          Filter1 ++ [{action, Action}, {access, Access}, {created_at, CreatedAt}]
-                      end, ets:tab2list(emqx_acl))
-    end.
-
--ifdef(EMQX_ENTERPRISE).
-export_modules() ->
-    case ets:info(emqx_modules) of
-        undefined -> [];
-        _ ->
-           lists:map(fun({_, Id, Type, Config, Enabled, CreatedAt, Description}) ->
-                          [{id, Id},
-                           {type, Type},
-                           {config, Config},
-                           {enabled, Enabled},
-                           {created_at, CreatedAt},
-                           {description, Description}
-                          ]
-                      end, ets:tab2list(emqx_modules))
-    end.
-
-export_schemas() ->
-    case ets:info(emqx_schema) of
-        undefined -> [];
-        _ ->
-            [emqx_schema_api:format_schema(Schema) || Schema <- emqx_schema_registry:get_all_schemas()]
-    end.
-
-export_confs() ->
-    case ets:info(emqx_conf_info) of
-        undefined -> {[], []};
-        _ ->
-            {lists:map(fun({_, Key, Confs}) ->
-                case Key of
-                    {_Zone, Name} ->
-                        [{zone, list_to_binary(Name)},
-                         {confs, confs_to_binary(Confs)}];
-                    {_Listener, Type, Name} ->
-                        [{type, list_to_binary(Type)},
-                         {name, list_to_binary(Name)},
-                         {confs, confs_to_binary(Confs)}];
-                    Name ->
-                        [{name, list_to_binary(Name)},
-                         {confs, confs_to_binary(Confs)}]
-                end
-            end, ets:tab2list(emqx_conf_b)),
-            lists:map(fun({_, {_Listener, Type, Name}, Status}) ->
-                [{type, list_to_binary(Type)},
-                 {name, list_to_binary(Name)},
-                 {status, Status}]
-            end, ets:tab2list(emqx_listeners_state))}
-    end.
-
-confs_to_binary(Confs) ->
-    [{list_to_binary(Key), list_to_binary(Val)} || {Key, Val} <-Confs].
-
--endif.
-
-import_rule(#{<<"id">> := RuleId,
-              <<"rawsql">> := RawSQL,
-              <<"actions">> := Actions,
-              <<"enabled">> := Enabled,
-              <<"description">> := Desc}) ->
-    Rule = #{id => RuleId,
-             rawsql => RawSQL,
-             actions => map_to_actions(Actions),
-             enabled => Enabled,
-             description => Desc},
-    try emqx_rule_engine:create_rule(Rule)
-    catch throw:{resource_not_initialized, _ResId} ->
-        emqx_rule_engine:create_rule(Rule#{enabled => false})
-    end.
-
-map_to_actions(Maps) ->
-    [map_to_action(M) || M <- Maps].
-
-map_to_action(Map = #{<<"id">> := ActionInstId, <<"name">> := Name, <<"args">> := Args}) ->
-    #{id => ActionInstId,
-      name => any_to_atom(Name),
-      args => Args,
-      fallbacks => map_to_actions(maps:get(<<"fallbacks">>, Map, []))}.
-
-
-import_rules(Rules) ->
-    lists:foreach(fun(Rule) ->
-                      import_rule(Rule)
-                  end, Rules).
-
-import_resources(Reources) ->
-    lists:foreach(fun(Resource) ->
-                      import_resource(Resource)
-                  end, Reources).
-
-import_resource(#{<<"id">> := Id,
-                  <<"type">> := Type,
-                  <<"config">> := Config,
-                  <<"created_at">> := CreatedAt,
-                  <<"description">> := Desc}) ->
-    NCreatedAt = case CreatedAt of
-                     null -> undefined;
-                     _ -> CreatedAt
-                 end,
-    emqx_rule_engine:create_resource(#{id => Id,
-                                       type => any_to_atom(Type),
-                                       config => Config,
-                                       created_at => NCreatedAt,
-                                       description => Desc}).
-import_resources_and_rules(Resources, Rules, FromVersion)
-  when FromVersion =:= "4.0" orelse
-       FromVersion =:= "4.1" orelse
-       FromVersion =:= "4.2" ->
-    Configs = lists:foldl(fun compatible_version/2 , [], Resources),
-    lists:foreach(fun(#{<<"actions">> := Actions} = Rule) ->
-                      NActions = apply_new_config(Actions, Configs),
-                      import_rule(Rule#{<<"actions">> := NActions})
-                  end, Rules);
-import_resources_and_rules(Resources, Rules, _FromVersion) ->
-    import_resources(Resources),
-    import_rules(Rules).
-
-%% 4.2.5 +
-compatible_version(#{<<"id">> := ID,
-                     <<"type">> := <<"web_hook">>,
-                     <<"config">> := #{<<"connect_timeout">> := ConnectTimeout,
-                                       <<"content_type">> := ContentType,
-                                       <<"headers">> := Headers,
-                                       <<"method">> := Method,
-                                       <<"pool_size">> := PoolSize,
-                                       <<"request_timeout">> := RequestTimeout,
-                                       <<"url">> := URL}} = Resource, Acc) ->
-    CovertFun = fun(Int) ->
-        list_to_binary(integer_to_list(Int) ++ "s")
-    end,
-    Cfg = make_new_config(#{<<"pool_size">> => PoolSize,
-                            <<"connect_timeout">> => CovertFun(ConnectTimeout),
-                            <<"request_timeout">> => CovertFun(RequestTimeout),
-                            <<"url">> => URL}),
-    {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
-    NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)),
-    [{ID, #{headers => NHeaders, method => Method}} | Acc];
-% 4.2.0
-compatible_version(#{<<"id">> := ID,
-                    <<"type">> := <<"web_hook">>,
-                    <<"config">> := #{<<"headers">> := Headers,
-                                    <<"method">> := Method,%% 4.2.0 Different here
-                                    <<"url">> := URL}} = Resource, Acc) ->
-    Cfg = make_new_config(#{<<"url">> => URL}),
-    {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
-    NHeaders = maps:put(<<"content-type">>, <<"application/json">> , covert_empty_headers(Headers)),
-    [{ID, #{headers => NHeaders, method => Method}} | Acc];
-
-%% bridge mqtt
-%% 4.2.0 - 4.2.5 bridge_mqtt, ssl enabled from on/off to true/false
-compatible_version(#{<<"type">> := <<"bridge_mqtt">>,
-                     <<"id">> := ID, %% begin 4.2.0.
-                     <<"config">> := #{<<"ssl">> := Ssl} = Config} = Resource, Acc) ->
-    NewConfig = Config#{<<"ssl">> := flag_to_boolean(Ssl),
-                        <<"pool_size">> => case maps:get(<<"pool_size">>, Config, undefined) of %% 4.0.x, compatible `pool_size`
-                                                undefined -> 8;
-                                                PoolSize -> PoolSize
-                                            end},
-    {ok, _Resource} = import_resource(Resource#{<<"config">> := NewConfig}),
-    [{ID, NewConfig} | Acc];
-
-% 4.2.3, add :content_type
-compatible_version(#{<<"id">> := ID,
-                    <<"type">> := <<"web_hook">>,
-                    <<"config">> := #{<<"headers">> := Headers,
-                                      <<"content_type">> := ContentType,%% 4.2.3 Different here
-                                      <<"method">> := Method,
-                                      <<"url">> := URL}} = Resource, Acc) ->
-    Cfg = make_new_config(#{<<"url">> => URL}),
-    {ok, _Resource} = import_resource(Resource#{<<"config">> := Cfg}),
-    NHeaders = maps:put(<<"content-type">>, ContentType, covert_empty_headers(Headers)),
-    [{ID, #{headers => NHeaders, method => Method}} | Acc];
-% normal version
-compatible_version(Resource, Acc) ->
-    {ok, _Resource} = import_resource(Resource),
-    Acc.
-
-make_new_config(Cfg) ->
-    Config = #{<<"pool_size">> => 8,
-               <<"connect_timeout">> => <<"5s">>,
-               <<"request_timeout">> => <<"5s">>,
-               <<"cacertfile">> => <<>>,
-               <<"certfile">> => <<>>,
-               <<"keyfile">> => <<>>,
-               <<"verify">> => false},
-    maps:merge(Cfg, Config).
-
-apply_new_config(Actions, Configs) ->
-    apply_new_config(Actions, Configs, []).
-
-apply_new_config([], _Configs, Acc) ->
-    Acc;
-apply_new_config(Actions, [], []) ->
-    Actions;
-apply_new_config([Action = #{<<"name">> := <<"data_to_webserver">>,
-                             <<"args">> := #{<<"$resource">> := ID,
-                                            <<"path">> := Path,
-                                            <<"payload_tmpl">> := PayloadTmpl}} | More], Configs, Acc) ->
-    case proplists:get_value(ID, Configs, undefined) of
-        undefined ->
-            apply_new_config(More, Configs, [Action | Acc]);
-        #{headers := Headers, method := Method} ->
-            Args = #{<<"$resource">> => ID,
-                     <<"body">> => PayloadTmpl,
-                     <<"headers">> => Headers,
-                     <<"method">> => Method,
-                     <<"path">> => Path},
-            apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc])
-    end;
-
-apply_new_config([Action = #{<<"args">> := #{<<"$resource">> := ResourceId,
-                                             <<"forward_topic">> := ForwardTopic,
-                                             <<"payload_tmpl">> := PayloadTmpl},
-                           <<"fallbacks">> := _Fallbacks,
-                           <<"id">> := _Id,
-                           <<"name">> := <<"data_to_mqtt_broker">>} | More], Configs, Acc) ->
-            Args = #{<<"$resource">> => ResourceId,
-                     <<"payload_tmpl">> => PayloadTmpl,
-                     <<"forward_topic">> => ForwardTopic},
-            apply_new_config(More, Configs, [Action#{<<"args">> := Args} | Acc]).
-
-
-actions_to_prop_list(Actions) ->
-    [action_to_prop_list(Act) || Act <- Actions].
-
-action_to_prop_list({action_instance, ActionInstId, Name, FallbackActions, Args}) ->
-    [{id, ActionInstId},
-     {name, Name},
-     {fallbacks, actions_to_prop_list(FallbackActions)},
-     {args, Args}].
-
-import_blacklist(Blacklist) ->
-    lists:foreach(fun(#{<<"who">> := Who,
-                        <<"by">> := By,
-                        <<"reason">> := Reason,
-                        <<"at">> := At,
-                        <<"until">> := Until}) ->
-                      NWho = case Who of
-                                 #{<<"peerhost">> := Peerhost} ->
-                                     {ok, NPeerhost} = inet:parse_address(Peerhost),
-                                     {peerhost, NPeerhost};
-                                 #{<<"clientid">> := ClientId} -> {clientid, ClientId};
-                                 #{<<"username">> := Username} -> {username, Username}
-                             end,
-                     emqx_banned:create(#banned{who = NWho, by = By, reason = Reason, at = At, until = Until})
-                  end, Blacklist).
-
-import_applications(Apps) ->
-    lists:foreach(fun(#{<<"id">> := AppID,
-                        <<"secret">> := AppSecret,
-                        <<"name">> := Name,
-                        <<"desc">> := Desc,
-                        <<"status">> := Status,
-                        <<"expired">> := Expired}) ->
-                      NExpired = case is_integer(Expired) of
-                                     true -> Expired;
-                                     false -> undefined
-                                 end,
-                      emqx_mgmt_auth:force_add_app(AppID, Name, AppSecret, Desc, Status, NExpired)
-                  end, Apps).
-
-import_users(Users) ->
-    lists:foreach(fun(#{<<"username">> := Username,
-                        <<"password">> := Password,
-                        <<"tags">> := Tags}) ->
-                      NPassword = base64:decode(Password),
-                      emqx_dashboard_admin:force_add_user(Username, NPassword, Tags)
-                  end, Users).
-
-import_auth_clientid(Lists) ->
-    case ets:info(emqx_user) of
-        undefined -> ok;
-        _ ->
-            lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password}) ->
-                                  mnesia:dirty_write({emqx_user, {clientid, Clientid}
-                                                               , base64:decode(Password)
-                                                               , erlang:system_time(millisecond)})
-                          end, Lists)
-    end.
-
-import_auth_username(Lists) ->
-    case ets:info(emqx_user) of
-        undefined -> ok;
-        _ ->
-            lists:foreach(fun(#{<<"username">> := Username, <<"password">> := Password}) ->
-                            mnesia:dirty_write({emqx_user, {username, Username}, base64:decode(Password), erlang:system_time(millisecond)})
-                          end, Lists)
-    end.
-
--ifdef(EMQX_ENTERPRISE).
-import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse
-                                            FromVersion =:= "4.1" ->
-    do_import_auth_mnesia_by_old_data(Auths);
-import_auth_mnesia(Auths, _) ->
-    do_import_auth_mnesia(Auths).
-
-import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse
-                                          FromVersion =:= "4.1" ->
-    do_import_acl_mnesia_by_old_data(Acls);
-
-import_acl_mnesia(Acls, _) ->
-    do_import_acl_mnesia(Acls).
--else.
-import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.3" ->
-    do_import_auth_mnesia(Auths);
-import_auth_mnesia(Auths, _FromVersion) ->
-    do_import_auth_mnesia_by_old_data(Auths).
-
-import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.3" ->
-    do_import_acl_mnesia(Acls);
-import_acl_mnesia(Acls, _FromVersion) ->
-    do_import_acl_mnesia_by_old_data(Acls).
-
--endif.
-
-do_import_auth_mnesia_by_old_data(Auths) ->
-    case ets:info(emqx_user) of
-        undefined -> ok;
-        _ ->
-            CreatedAt = erlang:system_time(millisecond),
-            lists:foreach(fun(#{<<"login">> := Login,
-                                <<"password">> := Password}) ->
-                            mnesia:dirty_write({emqx_user, {get_old_type(), Login}, base64:decode(Password), CreatedAt})
-                          end, Auths)
-    end.
-
-
-do_import_auth_mnesia(Auths) ->
-    case ets:info(emqx_user) of
-        undefined -> ok;
-        _ ->
-            lists:foreach(fun(#{<<"login">> := Login,
-                                <<"type">> := Type,
-                                <<"password">> := Password } = Map) ->
-                            CreatedAt = maps:get(<<"created_at">>, Map, erlang:system_time(millisecond)),
-                            mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt})
-                          end, Auths)
-    end.
-
-do_import_acl_mnesia_by_old_data(Acls) ->
-    case ets:info(emqx_acl) of
-        undefined -> ok;
-        _ ->
-            CreatedAt = erlang:system_time(millisecond),
-            lists:foreach(fun(#{<<"login">> := Login,
-                                <<"topic">> := Topic,
-                                <<"allow">> := Allow,
-                                <<"action">> := Action}) ->
-                            Allow1 = case any_to_atom(Allow) of
-                                         true -> allow;
-                                         false -> deny
-                                     end,
-                            mnesia:dirty_write({emqx_acl, {{get_old_type(), Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt})
-                          end, Acls)
-    end.
-do_import_acl_mnesia(Acls) ->
-    case ets:info(emqx_acl) of
-        undefined -> ok;
-        _ ->
-            lists:foreach(fun(Map = #{<<"action">> := Action,
-                                      <<"access">> := Access}) ->
-                            Topic = maps:get(<<"topic">>, Map),
-                            Login = case maps:get(<<"type_value">>, Map, undefined) of
-                                undefined ->
-                                    all;
-                                Value ->
-                                    {any_to_atom(maps:get(<<"type">>, Map)), Value}
-                            end,
-                            emqx_acl_mnesia_cli:add_acl(Login, Topic, any_to_atom(Action), any_to_atom(Access))
-                          end, Acls)
-    end.
-
--ifdef(EMQX_ENTERPRISE).
--dialyzer({nowarn_function, [import_modules/1]}).
-import_modules(Modules) ->
-    case ets:info(emqx_modules) of
-        undefined ->
-            ok;
-        _ ->
-           lists:foreach(fun(#{<<"id">> := Id,
-                               <<"type">> := Type,
-                               <<"config">> := Config,
-                               <<"enabled">> := Enabled,
-                               <<"created_at">> := CreatedAt,
-                               <<"description">> := Description}) ->
-                            _ = emqx_modules:import_module({Id, any_to_atom(Type), Config, Enabled, CreatedAt, Description})
-                         end, Modules)
-    end.
-
-
-import_schemas(Schemas) ->
-    case ets:info(emqx_schema) of
-        undefined -> ok;
-        _ -> [emqx_schema_registry:add_schema(emqx_schema_api:make_schema_params(Schema)) || Schema <- Schemas]
-    end.
-
-import_confs(Configs, ListenersState) ->
-    case ets:info(emqx_conf_info) of
-        undefined -> ok;
-        _ ->
-            emqx_conf:import_confs(Configs, ListenersState)
-    end.
-
--endif.
-
-any_to_atom(L) when is_list(L) -> list_to_atom(L);
-any_to_atom(B) when is_binary(B) -> binary_to_atom(B, utf8);
-any_to_atom(A) when is_atom(A) -> A.
-
-to_version(Version) when is_integer(Version) ->
-    integer_to_list(Version);
-to_version(Version) when is_binary(Version) ->
-    binary_to_list(Version);
-to_version(Version) when is_list(Version) ->
-    Version.
-
-export() ->
-    Seconds = erlang:system_time(second),
-    Data = do_export_data() ++ [{date, erlang:list_to_binary(emqx_mgmt_util:strftime(Seconds))}],
-    {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds),
-    Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]),
-    NFilename = filename:join([emqx:get_env(data_dir), Filename]),
-    ok = filelib:ensure_dir(NFilename),
-    case file:write_file(NFilename, emqx_json:encode(Data)) of
-        ok ->
-            case file:read_file_info(NFilename) of
-                {ok, #file_info{size = Size, ctime = {{Y1, M1, D1}, {H1, MM1, S1}}}} ->
-                    CreatedAt = io_lib:format("~p-~p-~p ~p:~p:~p", [Y1, M1, D1, H1, MM1, S1]),
-                    {ok, #{filename => list_to_binary(NFilename),
-                           size => Size,
-                           created_at => list_to_binary(CreatedAt),
-                           node => node()
-                          }};
-                Error -> Error
-            end;
-        Error -> Error
-    end.
-
-do_export_data() ->
-    Version = string:sub_string(emqx_sys:version(), 1, 3),
-    [{version, erlang:list_to_binary(Version)},
-     {rules, export_rules()},
-     {resources, export_resources()},
-     {blacklist, export_blacklist()},
-     {apps, export_applications()},
-     {users, export_users()},
-     {auth_mnesia, export_auth_mnesia()},
-     {acl_mnesia, export_acl_mnesia()}
-     ] ++ do_export_extra_data().
-
--ifdef(EMQX_ENTERPRISE).
-do_export_extra_data() ->
-    {Configs, State} = export_confs(),
-    [{modules, export_modules()},
-     {schemas, export_schemas()},
-     {configs, Configs},
-     {listeners_state, State}
-    ].
--else.
-do_export_extra_data() -> [].
--endif.
-
--ifdef(EMQX_ENTERPRISE).
-import(Filename, OverridesJson) ->
-    case file:read_file(Filename) of
-        {ok, Json} ->
-            Imported = emqx_json:decode(Json, [return_maps]),
-            Overrides = emqx_json:decode(OverridesJson, [return_maps]),
-            Data = maps:merge(Imported, Overrides),
-            Version = to_version(maps:get(<<"version">>, Data)),
-            read_global_auth_type(Data),
-            try
-                do_import_data(Data, Version),
-                logger:debug("The emqx data has been imported successfully"),
-                ok
-            catch Class:Reason:Stack ->
-                logger:error("The emqx data import failed: ~0p", [{Class, Reason, Stack}]),
-                {error, import_failed}
-            end;
-        Error -> Error
-    end.
--else.
-import(Filename, OverridesJson) ->
-    case file:read_file(Filename) of
-        {ok, Json} ->
-            Imported = emqx_json:decode(Json, [return_maps]),
-            Overrides = emqx_json:decode(OverridesJson, [return_maps]),
-            Data = maps:merge(Imported, Overrides),
-            Version = to_version(maps:get(<<"version">>, Data)),
-            read_global_auth_type(Data),
-            case is_version_supported(Data, Version) of
-                true  ->
-                    try
-                        do_import_data(Data, Version),
-                        logger:debug("The emqx data has been imported successfully"),
-                        ok
-                    catch Class:Reason:Stack ->
-                        logger:error("The emqx data import failed: ~0p", [{Class, Reason, Stack}]),
-                        {error, import_failed}
-                    end;
-                false ->
-                    logger:error("Unsupported version: ~p", [Version]),
-                    {error, unsupported_version, Version}
-            end;
-        Error -> Error
-    end.
--endif.
-
-do_import_data(Data, Version) ->
-    do_import_extra_data(Data, Version),
-    import_resources_and_rules(maps:get(<<"resources">>, Data, []), maps:get(<<"rules">>, Data, []), Version),
-    import_blacklist(maps:get(<<"blacklist">>, Data, [])),
-    import_applications(maps:get(<<"apps">>, Data, [])),
-    import_users(maps:get(<<"users">>, Data, [])),
-    import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])),
-    import_auth_username(maps:get(<<"auth_username">>, Data, [])),
-    import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version),
-    import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version).
-
--ifdef(EMQX_ENTERPRISE).
-do_import_extra_data(Data, _Version) ->
-    _ = import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])),
-    _ = import_modules(maps:get(<<"modules">>, Data, [])),
-    _ = import_schemas(maps:get(<<"schemas">>, Data, [])),
-    ok.
--else.
-do_import_extra_data(_Data, _Version) -> ok.
--endif.
-
-covert_empty_headers([]) -> #{};
-covert_empty_headers(Other) -> Other.
-
-flag_to_boolean(<<"on">>) -> true;
-flag_to_boolean(<<"off">>) -> false;
-flag_to_boolean(Other) -> Other.
-
--ifndef(EMQX_ENTERPRISE).
-is_version_supported(Data, Version) ->
-    case { maps:get(<<"auth_clientid">>, Data, [])
-         , maps:get(<<"auth_username">>, Data, [])
-         , maps:get(<<"auth_mnesia">>, Data, [])} of
-        {[], [], []} -> lists:member(Version, ?VERSIONS);
-        _ -> is_version_supported2(Version)
-    end.
-
-is_version_supported2("4.1") ->
-    true;
-is_version_supported2("4.3") ->
-    true;
-is_version_supported2(Version) ->
-    case re:run(Version, "^4.[02].\\d+$", [{capture, none}]) of
-        match ->
-            try lists:map(fun erlang:list_to_integer/1, string:tokens(Version, ".")) of
-                [4, 2, N] -> N >= 11;
-                [4, 0, N] -> N >= 13;
-                _ -> false
-            catch
-                _ : _ -> false
-            end;
-        nomatch ->
-            false
-    end.
--endif.
-
-read_global_auth_type(Data) ->
-    case {maps:get(<<"auth_mnesia">>, Data, []), maps:get(<<"acl_mnesia">>, Data, [])} of
-        {[], []} ->
-            %% Auth mnesia plugin is not used:
-            ok;
-        _ ->
-            do_read_global_auth_type(Data)
-    end.
-
--ifdef(EMQX_ENTERPRISE).
-do_read_global_auth_type(Data) ->
-    case Data of
-        #{<<"auth.mnesia.as">> := <<"username">>} ->
-            application:set_env(emqx_auth_mnesia, as, username);
-        #{<<"auth.mnesia.as">> := <<"clientid">>} ->
-            application:set_env(emqx_auth_mnesia, as, clientid);
-        _ ->
-            ok
-    end.
-
--else.
-do_read_global_auth_type(Data) ->
-    case Data of
-        #{<<"auth.mnesia.as">> := <<"username">>} ->
-            application:set_env(emqx_auth_mnesia, as, username);
-        #{<<"auth.mnesia.as">> := <<"clientid">>} ->
-            application:set_env(emqx_auth_mnesia, as, clientid);
-        _ ->
-            logger:error("While importing data from EMQX versions prior to 4.3 "
-                         "it is necessary to specify the value of \"auth.mnesia.as\" parameter "
-                         "as it was configured in etc/plugins/emqx_auth_mnesia.conf.\n"
-                         "Use the following command to import data:\n"
-                         "  $ emqx_ctl data import <filename> --env '{\"auth.mnesia.as\":\"username\"}'\n"
-                         "or\n"
-                         "  $ emqx_ctl data import <filename> --env '{\"auth.mnesia.as\":\"clientid\"}'",
-                         []),
-            error(import_failed)
-    end.
--endif.
-
-get_old_type() ->
-    {ok, Type} = application:get_env(emqx_auth_mnesia, as),
-    Type.

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl

@@ -1,176 +0,0 @@
-%%--------------------------------------------------------------------
-%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
-%%
-%% Licensed under the Apache License, Version 2.0 (the "License");
-%% you may not use this file except in compliance with the License.
-%% You may obtain a copy of the License at
-%%
-%%     http://www.apache.org/licenses/LICENSE-2.0
-%%
-%% Unless required by applicable law or agreed to in writing, software
-%% distributed under the License is distributed on an "AS IS" BASIS,
-%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-%% See the License for the specific language governing permissions and
-%% limitations under the License.
-%%--------------------------------------------------------------------
-
--module(emqx_auth_mnesia_migration_SUITE).
-
--compile(export_all).
--compile(nowarn_export_all).
-
--include_lib("eunit/include/eunit.hrl").
-
--include_lib("emqx/include/emqx.hrl").
--include_lib("emqx/include/emqx_mqtt.hrl").
--include_lib("emqx_auth_mnesia/include/emqx_auth_mnesia.hrl").
-
-matrix() ->
-    [{ImportAs, Version} || ImportAs <- [clientid, username]
-                          , Version <- ["v4.2.10", "v4.1.5"]].
-
-all() ->
-    [t_import_4_0, t_import_4_1, t_import_4_2].
-
-groups() ->
-    [{username, [], cases()}, {clientid, [], cases()}].
-
-cases() ->
-    [t_import].
-
-init_per_suite(Config) ->
-    emqx_ct_helpers:start_apps([emqx_management, emqx_dashboard, emqx_auth_mnesia]),
-    application:set_env(ekka, strict_mode, true),
-    ekka_mnesia:start(),
-    emqx_mgmt_auth:mnesia(boot),
-    Config.
-
-end_per_suite(_Config) ->
-    emqx_ct_helpers:stop_apps([emqx_modules, emqx_management, emqx_dashboard, emqx_auth_mnesia]),
-    ekka_mnesia:ensure_stopped().
-
-init_per_testcase(_, Config) ->
-    Config.
-
-end_per_testcase(_, _Config) ->
-    {atomic,ok} = mnesia:clear_table(emqx_acl),
-    {atomic,ok} = mnesia:clear_table(emqx_user),
-    ok.
--ifdef(EMQX_ENTERPRISE).
-t_import_4_0(Config) ->
-    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
-    ?assertMatch(ok, do_import("e4.0.10.json", Config, Overrides)),
-    timer:sleep(100),
-    ct:pal("---~p~n", [ets:tab2list(emqx_user)]),
-    test_import(username, {<<"emqx_username">>, <<"public">>}),
-    test_import(clientid, {<<"emqx_c">>, <<"public">>}),
-
-    Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}),
-    ?assertMatch(ok, do_import("e4.0.10.json", Config, Overrides1)),
-    timer:sleep(100),
-    test_import(username, {<<"emqx_c">>, <<"public">>}),
-    test_import(username, {<<"emqx_username">>, <<"public">>}).
-t_import_4_1(Config) ->
-    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
-    ?assertMatch(ok, do_import("e4.1.1.json", Config, Overrides)),
-    timer:sleep(100),
-    test_import(clientid, {<<"emqx_c">>, <<"public">>}),
-    test_import(clientid, {<<"emqx_c">>, <<"public">>}),
-
-    Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}),
-    ?assertMatch(ok, do_import("e4.1.1.json", Config, Overrides1)),
-    timer:sleep(100),
-    test_import(username, {<<"emqx_c">>, <<"public">>}),
-    test_import(clientid, {<<"emqx_clientid">>, <<"public">>}).
-
-t_import_4_2(Config) ->
-    ?assertMatch(ok, do_import("e4.2.9.json", Config, "{}")),
-    timer:sleep(100),
-    test_import(username, {<<"emqx_c">>, <<"public">>}),
-    test_import(clientid, {<<"emqx_clientid">>, <<"public">>}).
-
--else.
-t_import_4_0(Config) ->
-    ?assertMatch(ok, do_import("v4.0.11-no-auth.json", Config)),
-    timer:sleep(100),
-    ?assertMatch(0, ets:info(emqx_user, size)),
-
-    ?assertMatch({error, unsupported_version, "4.0"}, do_import("v4.0.11.json", Config)),
-
-    ?assertMatch(ok, do_import("v4.0.13.json", Config)),
-    timer:sleep(100),
-    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
-    test_import(username, {<<"user_for_test">>, <<"public">>}).
-
-t_import_4_1(Config) ->
-    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
-    ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides)),
-    timer:sleep(100),
-    test_import(clientid, {<<"user_mnesia">>, <<"public">>}),
-    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
-    test_import(username, {<<"user_for_test">>, <<"public">>}),
-
-    Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}),
-    ?assertMatch(ok, do_import("v4.1.5.json", Config, Overrides1)),
-    timer:sleep(100),
-    test_import(username, {<<"user_mnesia">>, <<"public">>}),
-    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
-    test_import(username, {<<"user_for_test">>, <<"public">>}).
-
-t_import_4_2(Config) ->
-    ?assertMatch(ok, do_import("v4.2.10-no-auth.json", Config)),
-    timer:sleep(100),
-    ?assertMatch(0, ets:info(emqx_user, size)),
-
-    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
-    ?assertMatch({error, unsupported_version, "4.2"}, do_import("v4.2.10.json", Config, Overrides)),
-
-    Overrides1 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
-    ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides1)),
-    timer:sleep(100),
-    test_import(clientid, {<<"user_mnesia">>, <<"public">>}),
-    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
-    test_import(username, {<<"user_for_test">>, <<"public">>}),
-
-    Overrides2 = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(username)}),
-    ?assertMatch(ok, do_import("v4.2.11.json", Config, Overrides2)),
-    timer:sleep(100),
-    test_import(username, {<<"user_mnesia">>, <<"public">>}),
-    test_import(clientid, {<<"client_for_test">>, <<"public">>}),
-    test_import(username, {<<"user_for_test">>, <<"public">>}),
-
-    ?assertMatch([#emqx_acl{
-                     filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>},
-                     action = pub,
-                     access = allow
-                    },
-                  #emqx_acl{
-                     filter = {{Type,<<"emqx_c">>}, <<"Topic/A">>},
-                     action = sub,
-                     access = allow
-                    }],
-                 lists:sort(ets:tab2list(emqx_acl))).
--endif.
-
-do_import(File, Config) ->
-    do_import(File, Config, "{}").
-
-do_import(File, Config, Overrides) ->
-    mnesia:clear_table(emqx_acl),
-    mnesia:clear_table(emqx_user),
-    Filename = filename:join(proplists:get_value(data_dir, Config), File),
-    emqx_mgmt_data_backup:import(Filename, Overrides).
-
-test_import(username, {Username, Password}) ->
-    [#emqx_user{password = _}] = ets:lookup(emqx_user, {username, Username}),
-    Req = #{clientid => <<"anyname">>,
-            username => Username,
-            password => Password},
-    ?assertMatch({stop, #{auth_result := success}},
-                 emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256}));
-test_import(clientid, {ClientID, Password}) ->
-    [#emqx_user{password = _}] = ets:lookup(emqx_user, {clientid, ClientID}),
-    Req = #{clientid => ClientID,
-            password => Password},
-    ?assertMatch({stop, #{auth_result := success}},
-                 emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})).

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh

@@ -1,134 +0,0 @@
-#!/bin/bash
-set -eux pipefail
-# Helper script for creating data export files
-
-container() {
-    version="${1}"
-    if [ -z ${2+x} ]; then
-        ee=""
-    else
-        ee="-ee"
-    fi
-    container="emqx/emqx${ee}:${version}"
-    docker rm -f emqx || true
-    docker run "$container" true # Make sure the image is cached locally
-    docker run --rm -e EMQX_LOADED_PLUGINS="emqx_auth_mnesia emqx_auth_clientid emqx_management" \
-           --name emqx -p 8081:8081 "$container" emqx foreground &
-    sleep 7
-}
-
-create_acls () {
-    url="${1}"
-    curl -f -v "http://localhost:8081/$url" -u admin:public -d@- <<EOF
-[
-  {
-    "login":"emqx_c",
-    "topic":"Topic/A",
-    "action":"pub",
-    "allow": true
-  },
-  {
-    "login":"emqx_c",
-    "topic":"Topic/A",
-    "action":"sub",
-    "allow": true
-  }
-]
-EOF
-}
-
-create_user () {
-    url="${1}"
-    curl -f -v "http://localhost:8081/api/v4/$url" -u admin:public -d@- <<EOF
-{
-    "login": "emqx_c",
-    "password": "emqx_p",
-    "is_superuser": true
-}
-EOF
-}
-
-export_data() {
-    filename="${1}"
-
-    docker exec emqx emqx_ctl data export
-    docker exec emqx sh -c 'cat data/*.json' | jq > "$filename.json"
-    cat "${filename}.json"
-}
-
-
-collect_4_2_no_mnesia_auth () {
-    container "4.2.10"
-
-    # Add clientid
-    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
-
-    export_data "v4.2.10-no-auth"
-}
-
-collect_4_2 () {
-    container "4.2.10"
-    create_acls "api/v4/mqtt_acl"
-    create_user mqtt_user
-
-    # Add clientid
-    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
-
-    export_data "v4.2.10"
-}
-
-
-collect_e4_2 () {
-    container "4.2.5" "ee"
-    # Add ACLs:
-    docker exec emqx emqx_ctl acl add username emqx_c Topic/A pubsub allow
-    # Create users
-    docker exec emqx emqx_ctl user add emqx_c emqx_p
-
-    # Add clientid
-    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
-
-    export_data "e4.2.5"
-}
-
-collect_e4_1 () {
-    container "4.1.1" "ee"
-    # Add ACLs:
-    create_acls "api/v4/emqx_acl"
-    # Create users
-    create_user "auth_user"
-
-    # Add clientid
-    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
-
-    export_data "e4.1.1"
-}
-
-collect_4_1 () {
-    container "v4.1.5"
-    create_acls "api/v4/emqx_acl"
-    create_user auth_user
-
-    # Add clientid
-    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
-
-    export_data "v4.1.5"
-}
-
-collect_4_0 () {
-    container "v4.0.11"
-
-    # Add clientid
-    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
-
-    export_data "v4.0.11"
-}
-
-collect_4_0
-collect_4_1
-collect_4_2
-collect_4_2_no_mnesia_auth
-collect_e4_2
-collect_e4_1
-
-docker kill emqx

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11-no-auth.json

@@ -1,23 +0,0 @@
-{
-  "version": "4.0",
-  "users": [],
-  "schemas": [],
-  "rules": [],
-  "resources": [],
-  "date": "2021-04-10 11:45:26",
-  "blacklist": [],
-  "auth_username": [],
-  "auth_mnesia": [],
-  "auth_clientid": [],
-  "apps": [
-    {
-      "status": true,
-      "secret": "public",
-      "name": "Default",
-      "id": "admin",
-      "expired": "undefined",
-      "desc": "Application user"
-    }
-  ],
-  "acl_mnesia": []
-}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.11.json

@@ -1,28 +0,0 @@
-{
-  "version": "4.0",
-  "users": [],
-  "schemas": [],
-  "rules": [],
-  "resources": [],
-  "date": "2021-04-10 11:45:26",
-  "blacklist": [],
-  "auth_username": [],
-  "auth_mnesia": [],
-  "auth_clientid": [
-    {
-      "password": "9Sv2tzJlNDlmNWZhYWQ5Yzc4MWUwNmFhZWI4NjFlMDM2OWEzYmE1OTkxOTBhOGQ4N2Y3MzExY2ZiZmIxNTFkMTdkZmY=",
-      "clientid": "emqx_clientid"
-    }
-  ],
-  "apps": [
-    {
-      "status": true,
-      "secret": "public",
-      "name": "Default",
-      "id": "admin",
-      "expired": "undefined",
-      "desc": "Application user"
-    }
-  ],
-  "acl_mnesia": []
-}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.13.json

@@ -1,37 +0,0 @@
-{
-    "version":"4.0.13",
-    "users":[
-        {
-            "username":"admin",
-            "tags":"administrator",
-            "password":"p6C65OF0BQhvPmCziM2yRa8JN5o="
-        }
-    ],
-    "schemas":[],
-    "rules":[],
-    "resources":[],
-    "date":"2021-04-16 11:20:00",
-    "blacklist":[],
-    "auth_username":[
-        {
-            "username":"user_for_test",
-            "password":"ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
-        }
-    ],
-    "auth_clientid":[
-        {
-            "password":"JBgSnzIxOWNiMDU1ZWFiNDAwMjVhOTQzZThlZjkxN2JlZWE4MGE4YzlmM2I5MjQ4OGI1NjllY2Q4NGQ4NjhjYzQ1NDM=",
-            "clientid":"client_for_test"
-        }
-    ],
-    "apps":[
-        {
-            "status":true,
-            "secret":"public",
-            "name":"Default",
-            "id":"admin",
-            "expired":"undefined",
-            "desc":"Application user"
-        }
-    ]
-}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json

@@ -1,58 +0,0 @@
-{
-  "version": "4.1",
-  "users": [
-    {
-      "username": "admin",
-      "tags": "administrator",
-      "password": "R0TpDmJtE/d5rIXAm6YY61RI0mg="
-    }
-  ],
-  "schemas": [],
-  "rules": [],
-  "resources": [],
-  "date": "2021-04-07 14:28:58",
-  "blacklist": [],
-  "auth_username": [
-    {
-      "username":"user_for_test",
-      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
-    }
-  ],
-  "auth_mnesia": [
-    {
-      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=",
-      "login": "user_mnesia",
-      "is_superuser": true
-    }
-  ],
-  "auth_clientid": [
-    {
-      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=",
-      "clientid": "client_for_test"
-    }
-  ],
-  "apps": [
-    {
-      "status": true,
-      "secret": "public",
-      "name": "Default",
-      "id": "admin",
-      "expired": "undefined",
-      "desc": "Application user"
-    }
-  ],
-  "acl_mnesia": [
-    {
-      "topic": "Topic/A",
-      "login": "emqx_c",
-      "allow": true,
-      "action": "sub"
-    },
-    {
-      "topic": "Topic/A",
-      "login": "emqx_c",
-      "allow": true,
-      "action": "pub"
-    }
-  ]
-}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10-no-auth.json

@@ -1,29 +0,0 @@
-{
-  "version": "4.2",
-  "date": "2021-04-12 10:41:10",
-  "rules": [],
-  "resources": [],
-  "blacklist": [],
-  "apps": [
-    {
-      "id": "admin",
-      "secret": "public",
-      "name": "Default",
-      "desc": "Application user",
-      "status": true,
-      "expired": "undefined"
-    }
-  ],
-  "users": [
-    {
-      "username": "admin",
-      "password": "e5M8oWEwQVqjdqceQIthC+3cPoY=",
-      "tags": "administrator"
-    }
-  ],
-  "auth_clientid": [],
-  "auth_username": [],
-  "auth_mnesia": [],
-  "acl_mnesia": [],
-  "schemas": []
-}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.10.json

@@ -1,53 +0,0 @@
-{
-  "version": "4.2",
-  "date": "2021-04-12 10:40:58",
-  "rules": [],
-  "resources": [],
-  "blacklist": [],
-  "apps": [
-    {
-      "id": "admin",
-      "secret": "public",
-      "name": "Default",
-      "desc": "Application user",
-      "status": true,
-      "expired": "undefined"
-    }
-  ],
-  "users": [
-    {
-      "username": "admin",
-      "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=",
-      "tags": "administrator"
-    }
-  ],
-  "auth_clientid": [
-    {
-      "clientid": "emqx_clientid",
-      "password": "UNb0e2RhNDc3NWIyNjg5Yjg4ZDExOTVhNWFkY2MzNGFmNzY2OTNmNmRlYzE4Y2ZiZjRjNzIyMWZlZTljZmEyZDE5Yzc="
-    }
-  ],
-  "auth_username": [],
-  "auth_mnesia": [
-    {
-      "login": "emqx_c",
-      "password": "ceb5e917f7930ae8f0dc3ceb496a428f7e644736eebca36a2b8f6bbac756171a",
-      "is_superuser": true
-    }
-  ],
-  "acl_mnesia": [
-    {
-      "login": "emqx_c",
-      "topic": "Topic/A",
-      "action": "sub",
-      "allow": true
-    },
-    {
-      "login": "emqx_c",
-      "topic": "Topic/A",
-      "action": "pub",
-      "allow": true
-    }
-  ],
-  "schemas": []
-}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.11.json

@@ -1,58 +0,0 @@
-{
-  "version": "4.2.11",
-  "date": "2021-04-12 10:40:58",
-  "rules": [],
-  "resources": [],
-  "blacklist": [],
-  "apps": [
-    {
-      "id": "admin",
-      "secret": "public",
-      "name": "Default",
-      "desc": "Application user",
-      "status": true,
-      "expired": "undefined"
-    }
-  ],
-  "users": [
-    {
-      "username": "test",
-      "password": "8Vd7+gVg2J3nE1Xjyxqd59sA5mo=",
-      "tags": "administrator"
-    }
-  ],
-  "auth_clientid": [
-    {
-      "clientid": "client_for_test",
-      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
-    }
-  ],
-  "auth_username": [
-    {
-      "username": "user_for_test",
-      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU="
-    }
-  ],
-  "auth_mnesia": [
-    {
-      "login": "user_mnesia",
-      "password": "ARLrzTRhZTI1MzgxNjdjMDU5ODFhZDU3ZTdmNzJiOWM5MWUwMTFkNDk4OGUyZWUyYmU0ZTE2ZTg2OWNhMGQyYWQ5ZmU=",
-      "is_superuser": true
-    }
-  ],
-  "acl_mnesia": [
-    {
-      "login": "emqx_c",
-      "topic": "Topic/A",
-      "action": "sub",
-      "allow": true
-    },
-    {
-      "login": "emqx_c",
-      "topic": "Topic/A",
-      "action": "pub",
-      "allow": true
-    }
-  ],
-  "schemas": []
-}

rebar.config

@@ -35,7 +35,6 @@
 {deps,
     [ {gpb, "4.11.2"} %% gpb only used to build, but not for release, pin it here to avoid fetching a wrong version due to rebar plugins scattered in all the deps
     , {ehttpc, {git, "https://github.com/emqx/ehttpc", {tag, "0.1.6"}}}
-    , {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.6.7"}}}
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}