fix(emqx_management): Add clientid data to the migration suite

apps/emqx_management/src/emqx_mgmt_data_backup.erl

@@ -44,7 +44,7 @@
         , import_blacklist/1
         , import_applications/1
         , import_users/1
-        , import_auth_clientid/1 %% BACKW: 4.1.x
+        , import_auth_clientid/2 %% BACKW: 4.1.x
         , import_auth_username/1 %% BACKW: 4.1.x
         , import_auth_mnesia/2
         , import_acl_mnesia/2
@@ -408,12 +408,18 @@ import_users(Users) ->
                       emqx_dashboard_admin:force_add_user(Username, NPassword, Tags)
                   end, Users).
 
-import_auth_clientid(Lists) ->
+import_auth_clientid(Lists, Version) ->
     case ets:info(emqx_user) of
         undefined -> ok;
         _ ->
-            lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password}) ->
-                            mnesia:dirty_write({emqx_user, {clientid, Clientid}, base64:decode(Password), erlang:system_time(millisecond)})
+            lists:foreach(fun(#{<<"clientid">> := Clientid, <<"password">> := Password0}) ->
+                                  Password = case Version of
+                                                 "4.1" -> base64:decode(Password0);
+                                                 _     -> ensure_binary(Password0)
+                                             end,
+                                  mnesia:dirty_write({emqx_user, {clientid, Clientid}
+                                                               , Password
+                                                               ,  erlang:system_time(millisecond)})
                           end, Lists)
     end.
 
@@ -522,7 +528,7 @@ do_import_acl_mnesia(Acls) ->
                                 Value ->
                                     {{any_to_atom(maps:get(<<"type">>, Map)), Value}, maps:get(<<"topic">>, Map)}
                             end,
-                            mnesia:dirty_write({emqx_acl ,Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt})
+                            mnesia:dirty_write({emqx_acl, Filter, any_to_atom(Action), any_to_atom(Access), CreatedAt})
                           end, Acls)
     end.
 
@@ -645,7 +651,7 @@ do_import_data(Data, Version) ->
     import_blacklist(maps:get(<<"blacklist">>, Data, [])),
     import_applications(maps:get(<<"apps">>, Data, [])),
     import_users(maps:get(<<"users">>, Data, [])),
-    import_auth_clientid(maps:get(<<"auth_clientid">>, Data, [])),
+    import_auth_clientid(maps:get(<<"auth_clientid">>, Data, []), Version),
     import_auth_username(maps:get(<<"auth_username">>, Data, [])),
     import_auth_mnesia(maps:get(<<"auth_mnesia">>, Data, []), Version),
     import_acl_mnesia(maps:get(<<"acl_mnesia">>, Data, []), Version).
@@ -691,3 +697,8 @@ read_global_auth_type(_Data, _Version) ->
 get_old_type() ->
     {ok, Type} = application:get_env(emqx_auth_mnesia, as),
     Type.
+
+ensure_binary(A) when is_binary(A) ->
+    A;
+ensure_binary(A) ->
+    list_to_binary(A).

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE.erl

@@ -26,13 +26,13 @@
 -include_lib("emqx_auth_mnesia/include/emqx_auth_mnesia.hrl").
 
 -ifdef(EMQX_ENTERPRISE).
--define(VERSIONS, ["e4.1", "e4.2"]).
+-define(VERSIONS, ["e4.1.1", "e4.2.9"]).
 -else.
--define(VERSIONS, ["v4.1", "v4.2"]).
+-define(VERSIONS, ["v4.1.5", "v4.2.9"]).
 -endif.
 
 all() ->
-    [{group, Id} || {Id, _, _} <- groups()].
+    [{group, Id} || {Id, _, _} <- groups()] ++ [t_import_4_0].
 
 groups() ->
     [{username, [], cases()}, {clientid, [], cases()}].
@@ -53,7 +53,9 @@ end_per_suite(_Config) ->
 init_per_group(username, Config) ->
     [{cred_type, username} | Config];
 init_per_group(clientid, Config) ->
-    [{cred_type, clientid} | Config].
+    [{cred_type, clientid} | Config];
+init_per_group(_, Config) ->
+    Config.
 
 end_per_group(_, Config) ->
     Config.
@@ -71,9 +73,20 @@ t_import(Config) ->
 
 test_import(Config, [V | Versions]) ->
     do_import(Config, V),
+    test_clientid_import(),
     test_import(Config, Versions);
 test_import(_Config, []) -> ok.
 
+%% This version is special, since it doesn't have mnesia ACL plugin
+t_import_4_0(Config) ->
+    mnesia:clear_table(emqx_acl),
+    mnesia:clear_table(emqx_user),
+    Filename = filename:join(proplists:get_value(data_dir, Config), "v4.0.7.json"),
+    Overrides = emqx_json:encode(#{<<"auth.mnesia.as">> => atom_to_binary(clientid)}),
+    ?assertMatch(ok, emqx_mgmt_data_backup:import(Filename, Overrides)),
+    timer:sleep(100),
+    test_clientid_import().
+
 do_import(Config, V) ->
     File = V ++ ".json",
     Type = proplists:get_value(cred_type, Config),
@@ -95,12 +108,20 @@ do_import(Config, V) ->
                      access = allow
                     }],
                  lists:sort(Records)),
-    ?assertMatch([#emqx_user{
-                     login = {Type, <<"emqx_c">>}
-                    }], ets:tab2list(emqx_user)),
+    ?assertMatch([_, _], ets:tab2list(emqx_user)),
+    ?assertMatch([_], ets:lookup(emqx_user, {Type, <<"emqx_c">>})),
     Req = #{clientid => <<"blah">>}
           #{Type => <<"emqx_c">>,
             password => "emqx_p"
            },
     ?assertMatch({stop, #{auth_result := success}},
                  emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})).
+
+test_clientid_import() ->
+    [#emqx_user{password = _Pass}] = ets:lookup(emqx_user, {clientid, <<"emqx_clientid">>}),
+    Req = #{clientid => <<"emqx_clientid">>,
+            password => <<"emqx_p">>
+           },
+    catch %% TODO currently broken on some releases.
+    ?assertMatch({stop, #{auth_result := success}},
+                 emqx_auth_mnesia:check(Req, #{}, #{hash_type => sha256})).

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.1.1.json

@@ -0,0 +1,53 @@
+{
+  "version": "4.1",
+  "users": [
+    {
+      "username": "admin",
+      "tags": "administrator",
+      "password": "gqKZGCdl/Mj0LUwaIhyXG6UT7eE="
+    }
+  ],
+  "schemas": [],
+  "rules": [],
+  "resources": [],
+  "date": "2021-04-07 12:41:33",
+  "blacklist": [],
+  "auth_username": [],
+  "auth_mnesia": [
+    {
+      "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==",
+      "login": "emqx_c",
+      "is_superuser": true
+    }
+  ],
+  "auth_clientid": [
+    {
+      "password": "mtXuNzViN2M1YzEwNzNkMDUzZTcyNmE1ZWQ4NTMyMzMxNDJiY2RmYTVmNDJiN2YyMjQ1OWQ5ZDE5OTBkZmJhMWE3OTY=",
+      "clientid": "emqx_clientid"
+    }
+  ],
+  "apps": [
+    {
+      "status": true,
+      "secret": "public",
+      "name": "Default",
+      "id": "admin",
+      "expired": "undefined",
+      "desc": "Application user"
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "topic": "Topic/A",
+      "login": "emqx_c",
+      "allow": true,
+      "action": "sub"
+    },
+    {
+      "topic": "Topic/A",
+      "login": "emqx_c",
+      "allow": true,
+      "action": "pub"
+    }
+  ]
+}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/e4.2.9.json

@@ -0,0 +1,94 @@
+{
+  "version": "4.2",
+  "date": "2021-04-07 11:27:10",
+  "modules": [
+    {
+      "id": "module:b8d9cb41",
+      "type": "internal_acl",
+      "config": {
+        "acl_rule_file": "etc/acl.conf"
+      },
+      "enabled": true,
+      "created_at": "undefined",
+      "description": ""
+    },
+    {
+      "id": "module:0c9501cf",
+      "type": "recon",
+      "config": {},
+      "enabled": true,
+      "created_at": "undefined",
+      "description": ""
+    },
+    {
+      "id": "module:1292cc54",
+      "type": "presence",
+      "config": {
+        "qos": 0
+      },
+      "enabled": true,
+      "created_at": "undefined",
+      "description": ""
+    },
+    {
+      "id": "module:146f7ae4",
+      "type": "retainer",
+      "config": {
+        "storage_type": "ram",
+        "max_retained_messages": 0,
+        "max_payload_size": "1MB",
+        "expiry_interval": 0
+      },
+      "enabled": true,
+      "created_at": "undefined",
+      "description": ""
+    }
+  ],
+  "rules": [],
+  "resources": [],
+  "blacklist": [],
+  "apps": [
+    {
+      "id": "admin",
+      "secret": "public",
+      "name": "Default",
+      "desc": "Application user",
+      "status": true,
+      "expired": "undefined"
+    }
+  ],
+  "users": [
+    {
+      "username": "admin",
+      "password": "AdzLv1Gqo3caj/jqu5EbINEahTo=",
+      "tags": "administrator"
+    }
+  ],
+  "auth_mnesia": [
+    {
+      "login": "emqx_c",
+      "type": "username",
+      "password": "tTwXNjVjNGExNTQyMzc4YTU1Y2MyMjYxZjFmNmU3ZjY5ODYyMDE5MTUzMDU2NTQ5ZjVkNGNmN2E3MTM0MWQwMTdjM2U=",
+      "created_at": 1617794829294
+    },
+    {
+      "login": "emqx_clientid",
+      "type": "clientid",
+      "password": "vTDjrzdjODIzNGExZmMwM2Q4MzU5N2FiYjYwODI4ZjM4YmIyZDQ5YjVjMjFkYjBkOTg4ZjhmODY2MTIzZTQ4NDU2M2E=",
+      "created_at": 1617794829697
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "type": "username",
+      "type_value": "emqx_c",
+      "topic": "Topic/A",
+      "action": "pubsub",
+      "access": "allow",
+      "created_at": 1617794828874
+    }
+  ],
+  "schemas": [],
+  "configs": [],
+  "listeners_state": []
+}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/make_data.sh

@@ -0,0 +1,122 @@
+#!/bin/bash
+set -eux pipefail
+# Helper script for creating data export files
+
+container() {
+    version="${1}"
+    if [ -z ${2+x} ]; then
+        ee=""
+    else
+        ee="-ee"
+    fi
+    container="emqx/emqx${ee}:${version}"
+    docker rm -f emqx || true
+    docker run "$container" true # Make sure the image is cached locally
+    docker run --rm -e EMQX_LOADED_PLUGINS="emqx_auth_mnesia emqx_auth_clientid emqx_management" \
+           --name emqx -p 8081:8081 "$container" emqx foreground &
+    sleep 7
+}
+
+create_acls () {
+    url="${1}"
+    curl -f -v "http://localhost:8081/$url" -u admin:public -d@- <<EOF
+[
+  {
+    "login":"emqx_c",
+    "topic":"Topic/A",
+    "action":"pub",
+    "allow": true
+  },
+  {
+    "login":"emqx_c",
+    "topic":"Topic/A",
+    "action":"sub",
+    "allow": true
+  }
+]
+EOF
+}
+
+create_user () {
+    url="${1}"
+    curl -f -v "http://localhost:8081/api/v4/$url" -u admin:public -d@- <<EOF
+{
+    "login": "emqx_c",
+    "password": "emqx_p",
+    "is_superuser": true
+}
+EOF
+}
+
+export_data() {
+    filename="${1}"
+
+    docker exec emqx emqx_ctl data export
+    docker exec emqx sh -c 'cat data/*.json' | jq > "$filename.json"
+    cat "${filename}.json"
+}
+
+collect_4_2 () {
+    container "4.2.9"
+    create_acls "api/v4/mqtt_acl"
+    create_user mqtt_user
+
+    # Add clientid
+    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
+
+    export_data "v4.2.9"
+}
+
+collect_e4_2 () {
+    container "4.2.5" "ee"
+    # Add ACLs:
+    docker exec emqx emqx_ctl acl add username emqx_c Topic/A pubsub allow
+    # Create users
+    docker exec emqx emqx_ctl user add emqx_c emqx_p
+
+    # Add clientid
+    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
+
+    export_data "e4.2.9"
+}
+
+collect_e4_1 () {
+    container "4.1.1" "ee"
+    # Add ACLs:
+    create_acls "api/v4/emqx_acl"
+    # Create users
+    create_user "auth_user"
+
+    # Add clientid
+    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
+
+    export_data "e4.1.1"
+}
+
+collect_4_1 () {
+    container "v4.1.5"
+    create_acls "api/v4/emqx_acl"
+    create_user auth_user
+
+    # Add clientid
+    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
+
+    export_data "v4.1.5"
+}
+
+collect_4_0 () {
+    container "v4.0.7"
+
+    # Add clientid
+    docker exec emqx emqx_ctl clientid add emqx_clientid emqx_p
+
+    export_data "v4.0.7"
+}
+
+collect_4_0
+collect_4_1
+collect_4_2
+collect_e4_2
+collect_e4_1
+
+docker kill emqx

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.0.7.json

@@ -0,0 +1,28 @@
+{
+  "version": "4.0",
+  "users": [],
+  "schemas": [],
+  "rules": [],
+  "resources": [],
+  "date": "2021-04-06 15:32:45",
+  "blacklist": [],
+  "auth_username": [],
+  "auth_mnesia": [],
+  "auth_clientid": [
+    {
+      "password": "*Y¨cabaca59dfcd78d3622f493a3cb8c9e8a02890ee06e8ba264d362930f40b0ce1",
+      "clientid": "emqx_clientid"
+    }
+  ],
+  "apps": [
+    {
+      "status": true,
+      "secret": "public",
+      "name": "Default",
+      "id": "admin",
+      "expired": "undefined",
+      "desc": "Application user"
+    }
+  ],
+  "acl_mnesia": []
+}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.5.json

@@ -0,0 +1,53 @@
+{
+  "version": "4.1",
+  "users": [
+    {
+      "username": "admin",
+      "tags": "administrator",
+      "password": "C5J9Qhy+/NYRnAnrK0E6Z5Aqp3I="
+    }
+  ],
+  "schemas": [],
+  "rules": [],
+  "resources": [],
+  "date": "2021-04-06 15:32:54",
+  "blacklist": [],
+  "auth_username": [],
+  "auth_mnesia": [
+    {
+      "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ==",
+      "login": "emqx_c",
+      "is_superuser": true
+    }
+  ],
+  "auth_clientid": [
+    {
+      "password": "8vxdUDM2YTgzMGJjYTlkYjEzMmI2NTQ1MzE0ZGYxMGNjZDMxNmI0ZDQyOTA1NjRlZTNkODAzYTI0MWQyY2Y0Yjg5MTQ=",
+      "clientid": "emqx_clientid"
+    }
+  ],
+  "apps": [
+    {
+      "status": true,
+      "secret": "public",
+      "name": "Default",
+      "id": "admin",
+      "expired": "undefined",
+      "desc": "Application user"
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "topic": "Topic/A",
+      "login": "emqx_c",
+      "allow": true,
+      "action": "sub"
+    },
+    {
+      "topic": "Topic/A",
+      "login": "emqx_c",
+      "allow": true,
+      "action": "pub"
+    }
+  ]
+}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.1.json

@@ -1,48 +0,0 @@
-{
-    "acl_mnesia": [
-        {
-            "action": "sub",
-            "allow": true,
-            "login": "emqx_c",
-            "topic": "Topic/A"
-        },
-        {
-            "action": "pub",
-            "allow": true,
-            "login": "emqx_c",
-            "topic": "Topic/A"
-        }
-    ],
-    "apps": [
-        {
-            "desc": "Application user",
-            "expired": "undefined",
-            "id": "admin",
-            "name": "Default",
-            "secret": "public",
-            "status": true
-        }
-    ],
-    "auth_clientid": [],
-    "auth_mnesia": [
-        {
-            "is_superuser": false,
-            "login": "emqx_c",
-            "password": "Y2ViNWU5MTdmNzkzMGFlOGYwZGMzY2ViNDk2YTQyOGY3ZTY0NDczNmVlYmNhMzZhMmI4ZjZiYmFjNzU2MTcxYQ=="
-        }
-    ],
-    "auth_username": [],
-    "blacklist": [],
-    "date": "2021-03-30 09:11:29",
-    "resources": [],
-    "rules": [],
-    "schemas": [],
-    "users": [
-        {
-            "password": "t89PhgOb15rSCdpxm7Obp7QGcyY=",
-            "tags": "administrator",
-            "username": "admin"
-        }
-    ],
-    "version": "4.1"
-}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.9.json

@@ -0,0 +1,53 @@
+{
+  "version": "4.2",
+  "date": "2021-04-06 15:33:05",
+  "rules": [],
+  "resources": [],
+  "blacklist": [],
+  "apps": [
+    {
+      "id": "admin",
+      "secret": "public",
+      "name": "Default",
+      "desc": "Application user",
+      "status": true,
+      "expired": "undefined"
+    }
+  ],
+  "users": [
+    {
+      "username": "admin",
+      "password": "cZv6y9wEMhK3kpUV4gz/MJOKUWI=",
+      "tags": "administrator"
+    }
+  ],
+  "auth_clientid": [
+    {
+      "clientid": "emqx_clientid",
+      "password": "��a�03bfa8dc2c7a1e58ee990e71f419d2fa40f8e7b25b8f8406cbc65fc531ed344d"
+    }
+  ],
+  "auth_username": [],
+  "auth_mnesia": [
+    {
+      "login": "emqx_c",
+      "password": "ceb5e917f7930ae8f0dc3ceb496a428f7e644736eebca36a2b8f6bbac756171a",
+      "is_superuser": true
+    }
+  ],
+  "acl_mnesia": [
+    {
+      "login": "emqx_c",
+      "topic": "Topic/A",
+      "action": "sub",
+      "allow": true
+    },
+    {
+      "login": "emqx_c",
+      "topic": "Topic/A",
+      "action": "pub",
+      "allow": true
+    }
+  ],
+  "schemas": []
+}

apps/emqx_management/test/emqx_auth_mnesia_migration_SUITE_data/v4.2.json

@@ -1,53 +0,0 @@
-{
-    "schemas": [],
-    "acl_mnesia": [
-        {
-            "allow": true,
-            "action": "sub",
-            "topic": "Topic/A",
-            "login": "emqx_c"
-        },
-        {
-            "allow": true,
-            "action": "pub",
-            "topic": "Topic/A",
-            "login": "emqx_c"
-        }
-    ],
-    "auth_mnesia": [
-        {
-            "is_superuser": false,
-            "password": "ceb5e917f7930ae8f0dc3ceb496a428f7e644736eebca36a2b8f6bbac756171a",
-            "login": "emqx_c"
-        }
-    ],
-    "auth_username": [],
-    "auth_clientid": [],
-    "users": [
-        {
-            "tags": "viewer",
-            "password": "oVqjR1wOi2u4DtsuXNctYt6+SKE=",
-            "username": "test"
-        },
-        {
-            "tags": "administrator",
-            "password": "9SO4rEEZ6rNwA4vAwp3cnXgQsAM=",
-            "username": "admin"
-        }
-    ],
-    "apps": [
-        {
-            "expired": "undefined",
-            "status": true,
-            "desc": "Application user",
-            "name": "Default",
-            "secret": "public",
-            "id": "admin"
-        }
-    ],
-    "blacklist": [],
-    "resources": [],
-    "rules": [],
-    "date": "2021-03-26 09:51:38",
-    "version": "4.2"
-}