feat(management): implement the auth-mnesia import/export

All codes picks from emqx-management:dev/e4.2.3

apps/emqx_management/src/emqx_mgmt.erl

@@ -124,21 +124,17 @@
         , export_blacklist/0
         , export_applications/0
         , export_users/0
-        , export_auth_clientid/0
-        , export_auth_username/0
         , export_auth_mnesia/0
         , export_acl_mnesia/0
-        , export_schemas/0
         , import_rules/1
         , import_resources/1
         , import_blacklist/1
         , import_applications/1
         , import_users/1
-        , import_auth_clientid/1
-        , import_auth_username/1
-        , import_auth_mnesia/1
-        , import_acl_mnesia/1
-        , import_schemas/1
+        , import_auth_clientid/1 %% BACKW: 4.1.x
+        , import_auth_username/1 %% BACKW: 4.1.x
+        , import_auth_mnesia/2
+        , import_acl_mnesia/2
         , to_version/1
         ]).
 
@@ -612,78 +608,59 @@ export_rules() ->
                end, emqx_rule_registry:get_rules()).
 
 export_resources() ->
-    lists:foldl(fun({_, Id, Type, Config, CreatedAt, Desc}, Acc) ->
+    lists:map(fun({_, Id, Type, Config, CreatedAt, Desc}) ->
                     NCreatedAt = case CreatedAt of
                                      undefined -> null;
                                      _ -> CreatedAt
                                  end,
-                    [[{id, Id},
-                      {type, Type},
-                      {config, maps:to_list(Config)},
-                      {created_at, NCreatedAt},
-                      {description, Desc}] | Acc]
-               end, [], emqx_rule_registry:get_resources()).
+                    [{id, Id},
+                     {type, Type},
+                     {config, maps:to_list(Config)},
+                     {created_at, NCreatedAt},
+                     {description, Desc}]
+               end, emqx_rule_registry:get_resources()).
 
 export_blacklist() ->
-    lists:foldl(fun(#banned{who = Who, by = By, reason = Reason, at = At, until = Until}, Acc) ->
+    lists:map(fun(#banned{who = Who, by = By, reason = Reason, at = At, until = Until}) ->
                     NWho = case Who of
                                {peerhost, Peerhost} -> {peerhost, inet:ntoa(Peerhost)};
                                _ -> Who
                            end,
-                    [[{who, [NWho]}, {by, By}, {reason, Reason}, {at, At}, {until, Until}] | Acc]
-                end, [], ets:tab2list(emqx_banned)).
+                    [{who, [NWho]}, {by, By}, {reason, Reason}, {at, At}, {until, Until}]
+                end, ets:tab2list(emqx_banned)).
 
 export_applications() ->
-    lists:foldl(fun({_, AppID, AppSecret, Name, Desc, Status, Expired}, Acc) ->
-                    [[{id, AppID}, {secret, AppSecret}, {name, Name}, {desc, Desc}, {status, Status}, {expired, Expired}] | Acc]
-                end, [], ets:tab2list(mqtt_app)).
+    lists:map(fun({_, AppID, AppSecret, Name, Desc, Status, Expired}) ->
+                    [{id, AppID}, {secret, AppSecret}, {name, Name}, {desc, Desc}, {status, Status}, {expired, Expired}]
+                end, ets:tab2list(mqtt_app)).
 
 export_users() ->
-    lists:foldl(fun({_, Username, Password, Tags}, Acc) ->
-                    [[{username, Username}, {password, base64:encode(Password)}, {tags, Tags}] | Acc]
-                end, [], ets:tab2list(mqtt_admin)).
-
-export_auth_clientid() ->
-    case ets:info(emqx_auth_clientid) of
-        undefined -> [];
-        _ ->
-            lists:foldl(fun({_, ClientId, Password}, Acc) ->
-                            [[{clientid, ClientId}, {password, Password}] | Acc]
-                        end, [], ets:tab2list(emqx_auth_clientid))
-    end.
-
-export_auth_username() ->
-    case ets:info(emqx_auth_username) of
-        undefined -> [];
-        _ ->
-            lists:foldl(fun({_, Username, Password}, Acc) ->
-                            [[{username, Username}, {password, Password}] | Acc]
-                        end, [], ets:tab2list(emqx_auth_username))
-    end.
+    lists:map(fun({_, Username, Password, Tags}) ->
+                    [{username, Username}, {password, base64:encode(Password)}, {tags, Tags}]
+                end, ets:tab2list(mqtt_admin)).
 
 export_auth_mnesia() ->
     case ets:info(emqx_user) of
         undefined -> [];
         _ ->
-            lists:foldl(fun({_, Login, Password, IsSuperuser}, Acc) ->
-                            [[{login, Login}, {password, Password}, {is_superuser, IsSuperuser}] | Acc]
-                        end, [], ets:tab2list(emqx_user))
+            lists:map(fun({_, {Type, Login}, Password, CreatedAt}) ->
+                            [{login, Login}, {type, Type}, {password, base64:encode(Password)}, {created_at, CreatedAt}]
+                        end, ets:tab2list(emqx_user))
     end.
 
 export_acl_mnesia() ->
     case ets:info(emqx_acl) of
         undefined -> [];
         _ ->
-            lists:foldl(fun({_, Login, Topic, Action, Allow}, Acc) ->
-                            [[{login, Login}, {topic, Topic}, {action, Action}, {allow, Allow}] | Acc]
-                        end, [], ets:tab2list(emqx_acl))
-    end.
-
-export_schemas() ->
-    case ets:info(emqx_schema) of
-        undefined -> [];
-        _ ->
-            [emqx_schema_api:format_schema(Schema) || Schema <- emqx_schema_registry:get_all_schemas()]
+            lists:map(fun({_, Filter, Action, Access, CreatedAt}) ->
+                            Filter1 = case Filter of
+                                {{Type, TypeValue}, Topic} ->
+                                    [{type, Type}, {type_value, TypeValue}, {topic, Topic}];
+                                {Type, Topic} ->
+                                    [{type, Type}, {topic, Topic}]
+                            end,
+                            Filter1 ++ [{action, Action}, {access, Access}, {created_at, CreatedAt}]
+                        end, ets:tab2list(emqx_acl))
     end.
 
 import_rules(Rules) ->
@@ -761,44 +738,80 @@ import_users(Users) ->
                   end, Users).
 
 import_auth_clientid(Lists) ->
-    case ets:info(emqx_auth_clientid) of
+    case ets:info(emqx_user) of
         undefined -> ok;
         _ ->
-            [ mnesia:dirty_write({emqx_auth_clientid, ClientId, Password}) || #{<<"clientid">> := ClientId,
-                                                                               <<"password">> := Password} <- Lists ]
+            [ mnesia:dirty_write({emqx_user, {clientid, Clientid}, base64:decode(Password), erlang:system_time(millisecond)})
+              || #{<<"clientid">> := Clientid, <<"password">> := Password} <- Lists ]
     end.
 
 import_auth_username(Lists) ->
-    case ets:info(emqx_auth_username) of
+    case ets:info(emqx_user) of
         undefined -> ok;
         _ ->
-            [ mnesia:dirty_write({emqx_auth_username, Username, Password}) || #{<<"username">> := Username,
-                                                                               <<"password">> := Password} <- Lists ]
+            [ mnesia:dirty_write({emqx_user, {username, Username}, base64:decode(Password), erlang:system_time(millisecond)})
+              || #{<<"username">> := Username, <<"password">> := Password} <- Lists ]
     end.
 
-import_auth_mnesia(Auths) ->
+import_auth_mnesia(Auths, FromVersion) when FromVersion =:= "4.0" orelse
+                                            FromVersion =:= "4.1" ->
+    case ets:info(emqx_user) of
+        undefined -> ok;
+        _ ->
+            CreatedAt = erlang:system_time(millisecond),
+            [ begin
+                mnesia:dirty_write({emqx_user, {username, Login}, base64:decode(Password), CreatedAt})
+              end
+              || #{<<"login">> := Login,
+                   <<"password">> := Password} <- Auths ]
+
+    end;
+
+import_auth_mnesia(Auths, _) ->
     case ets:info(emqx_user) of
         undefined -> ok;
         _ ->
-            [ mnesia:dirty_write({emqx_user, Login, Password, IsSuperuser}) || #{<<"login">> := Login,
-                                                                                 <<"password">> := Password,
-                                                                                 <<"is_superuser">> := IsSuperuser} <- Auths ]
+            [ mnesia:dirty_write({emqx_user, {any_to_atom(Type), Login}, base64:decode(Password), CreatedAt})
+              || #{<<"login">> := Login,
+                   <<"type">> := Type,
+                   <<"password">> := Password,
+                   <<"created_at">> := CreatedAt } <- Auths ]
     end.
 
-import_acl_mnesia(Acls) ->
+import_acl_mnesia(Acls, FromVersion) when FromVersion =:= "4.0" orelse
+                                          FromVersion =:= "4.1" ->
     case ets:info(emqx_acl) of
         undefined -> ok;
         _ ->
-            [ mnesia:dirty_write({emqx_acl ,Login, Topic, Action, Allow}) || #{<<"login">> := Login,
-                                                                               <<"topic">> := Topic,
-                                                                               <<"action">> := Action,
-                                                                               <<"allow">> := Allow} <- Acls ]
-    end.
+            CreatedAt = erlang:system_time(millisecond),
+            [begin
+                 Allow1 = case any_to_atom(Allow) of
+                              true -> allow;
+                              false -> deny
+                          end,
+                 mnesia:dirty_write({emqx_acl, {{username, Login}, Topic}, any_to_atom(Action), Allow1, CreatedAt})
+             end || #{<<"login">> := Login,
+                      <<"topic">> := Topic,
+                      <<"allow">> := Allow,
+                      <<"action">> := Action} <- Acls]
+    end;
 
-import_schemas(Schemas) ->
-    case ets:info(emqx_schema) of
+import_acl_mnesia(Acls, _) ->
+    case ets:info(emqx_acl) of
         undefined -> ok;
-        _ -> [emqx_schema_registry:add_schema(emqx_schema_api:make_schema_params(Schema)) || Schema <- Schemas]
+        _ ->
+            [ begin
+              Filter = case maps:get(<<"type_value">>, Map, undefined) of
+                  undefined ->
+                      {any_to_atom(maps:get(<<"type">>, Map)), maps:get(<<"topic">>, Map)};
+                  Value ->
+                      {{any_to_atom(maps:get(<<"type">>, Map)), Value}, maps:get(<<"topic">>, Map)}
+              end,
+              mnesia:dirty_write({emqx_acl ,Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt})
+              end
+              || Map = #{<<"action">> := Action,
+                         <<"access">> := Access,
+                         <<"created_at">> := CreatedAt} <- Acls ]
     end.
 
 any_to_atom(L) when is_list(L) -> list_to_atom(L);

apps/emqx_management/src/emqx_mgmt_api_apps.erl

@@ -70,13 +70,13 @@ add_app(_Bindings, Params) ->
     Expired = get_value(<<"expired">>, Params),
     case emqx_mgmt_auth:add_app(AppId, Name, Secret, Desc, Status, Expired) of
         {ok, AppSecret} -> return({ok, #{secret => AppSecret}});
-        {error, Reason} -> return({error, ?ERROR2, Reason})
+        {error, Reason} -> return({error, Reason})
     end.
 
 del_app(#{appid := AppId}, _Params) ->
     case emqx_mgmt_auth:del_app(AppId) of
         ok -> return();
-        {error, Reason} -> return({error, ?ERROR2, Reason})
+        {error, Reason} -> return({error, Reason})
     end.
 
 list_apps(_Bindings, _Params) ->
@@ -102,7 +102,7 @@ update_app(#{appid := AppId}, Params) ->
     Expired = get_value(<<"expired">>, Params),
     case emqx_mgmt_auth:update_app(AppId, Name, Desc, Status, Expired) of
         ok -> return();
-        {error, Reason} -> return({error, ?ERROR2, Reason})
+        {error, Reason} -> return({error, Reason})
     end.
 
 format({AppId, _AppSecret, Name, Desc, Status, Expired}) ->

apps/emqx_management/src/emqx_mgmt_api_data.erl

@@ -80,11 +80,8 @@ export(_Bindings, _Params) ->
     Blacklist = emqx_mgmt:export_blacklist(),
     Apps = emqx_mgmt:export_applications(),
     Users = emqx_mgmt:export_users(),
-    AuthClientid = emqx_mgmt:export_auth_clientid(),
-    AuthUsername = emqx_mgmt:export_auth_username(),
     AuthMnesia = emqx_mgmt:export_auth_mnesia(),
     AclMnesia = emqx_mgmt:export_acl_mnesia(),
-    Schemas = emqx_mgmt:export_schemas(),
     Seconds = erlang:system_time(second),
     {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds),
     Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]),
@@ -97,11 +94,8 @@ export(_Bindings, _Params) ->
             {blacklist, Blacklist},
             {apps, Apps},
             {users, Users},
-            {auth_clientid, AuthClientid},
-            {auth_username, AuthUsername},
             {auth_mnesia, AuthMnesia},
-            {acl_mnesia, AclMnesia},
-            {schemas, Schemas}
+            {acl_mnesia, AclMnesia}
            ],
 
     Bin = emqx_json:encode(Data),
@@ -180,18 +174,18 @@ do_import(Filename) ->
             case lists:member(Version, ?VERSIONS) of
                 true  ->
                     try
-                        emqx_mgmt:import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])),
+                        %emqx_mgmt:import_confs(maps:get(<<"configs">>, Data, []), maps:get(<<"listeners_state">>, Data, [])),
                         emqx_mgmt:import_resources(maps:get(<<"resources">>, Data, [])),
                         emqx_mgmt:import_rules(maps:get(<<"rules">>, Data, [])),
                         emqx_mgmt:import_blacklist(maps:get(<<"blacklist">>, Data, [])),
                         emqx_mgmt:import_applications(maps:get(<<"apps">>, Data, [])),
                         emqx_mgmt:import_users(maps:get(<<"users">>, Data, [])),
-                        emqx_mgmt:import_modules(maps:get(<<"modules">>, Data, [])),
+                        %emqx_mgmt:import_modules(maps:get(<<"modules">>, Data, [])),
                         _ = emqx_mgmt:import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])),
                         _ = emqx_mgmt:import_auth_username(maps:get(<<"auth_username">>, Data, [])),
                         _ = emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version),
                         _ = emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version),
-                        _ = emqx_mgmt:import_schemas(maps:get(<<"schemas">>, Data, [])),
+                        %_ = emqx_mgmt:import_schemas(maps:get(<<"schemas">>, Data, [])),
                         logger:debug("The emqx data has been imported successfully"),
                         ok
                     catch Class:Reason:Stack ->

apps/emqx_management/src/emqx_mgmt_cli.erl

@@ -563,11 +563,8 @@ data(["export"]) ->
     Blacklist = emqx_mgmt:export_blacklist(),
     Apps = emqx_mgmt:export_applications(),
     Users = emqx_mgmt:export_users(),
-    AuthClientID = emqx_mgmt:export_auth_clientid(),
-    AuthUsername = emqx_mgmt:export_auth_username(),
     AuthMnesia = emqx_mgmt:export_auth_mnesia(),
     AclMnesia = emqx_mgmt:export_acl_mnesia(),
-    Schemas = emqx_mgmt:export_schemas(),
     Seconds = erlang:system_time(second),
     {{Y, M, D}, {H, MM, S}} = emqx_mgmt_util:datetime(Seconds),
     Filename = io_lib:format("emqx-export-~p-~p-~p-~p-~p-~p.json", [Y, M, D, H, MM, S]),
@@ -580,11 +577,8 @@ data(["export"]) ->
             {blacklist, Blacklist},
             {apps, Apps},
             {users, Users},
-            {auth_clientid, AuthClientID},
-            {auth_username, AuthUsername},
             {auth_mnesia, AuthMnesia},
-            {acl_mnesia, AclMnesia},
-            {schemas, Schemas}],
+            {acl_mnesia, AclMnesia}],
     ok = filelib:ensure_dir(NFilename),
     case file:write_file(NFilename, emqx_json:encode(Data)) of
         ok ->
@@ -608,9 +602,8 @@ data(["import", Filename]) ->
                         emqx_mgmt:import_users(maps:get(<<"users">>, Data, [])),
                         _ = emqx_mgmt:import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])),
                         _ = emqx_mgmt:import_auth_username(maps:get(<<"auth_username">>, Data, [])),
-                        _ = emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, [])),
-                        _ = emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, [])),
-                        _ = emqx_mgmt:import_schemas(maps:get(<<"schemas">>, Data, [])),
+                        _ = emqx_mgmt:import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version),
+                        _ = emqx_mgmt:import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version),
                         emqx_ctl:print("The emqx data has been imported successfully.~n")
                     catch Class:Reason:Stack ->
                         emqx_ctl:print("The emqx data import failed due: ~0p~n", [{Class,Reason,Stack}])

apps/emqx_management/src/emqx_mgmt_http.erl

@@ -73,7 +73,8 @@ ranch_opts(Port, Options0) ->
             socket_opts => [{port, Port} | Options]},
     Res.
 
-stop_listener({Proto, _Port, _}) ->
+stop_listener({Proto, Port, _}) ->
+    io:format("Stop http:management listener on ~s successfully.~n",[format(Port)]),
     minirest:stop_http(listener_name(Proto)).
 
 listeners() ->
@@ -123,3 +124,10 @@ filter(#{app := App}) ->
         false -> false;
         Plugin -> Plugin#plugin.active
     end.
+
+format(Port) when is_integer(Port) ->
+    io_lib:format("0.0.0.0:~w", [Port]);
+format({Addr, Port}) when is_list(Addr) ->
+    io_lib:format("~s:~w", [Addr, Port]);
+format({Addr, Port}) when is_tuple(Addr) ->
+    io_lib:format("~s:~w", [inet:ntoa(Addr), Port]).