Merge pull request #13336 from zhongwencool/authn-boostrap-file

feat: support bootstrap_file on authentication for build-in-database

apps/emqx_auth_mnesia/src/emqx_auth_mnesia.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_auth_mnesia, [
     {description, "EMQX Buitl-in Database Authentication and Authorization"},
-    {vsn, "0.1.5"},
+    {vsn, "0.1.6"},
     {registered, []},
     {mod, {emqx_auth_mnesia_app, []}},
     {applications, [

apps/emqx_auth_mnesia/src/emqx_authn_mnesia.erl

@@ -116,7 +116,7 @@ create(
         user_id_type := Type,
         password_hash_algorithm := Algorithm,
         user_group := UserGroup
-    }
+    } = Config
 ) ->
     ok = emqx_authn_password_hashing:init(Algorithm),
     State = #{
@@ -124,6 +124,7 @@ create(
         user_id_type => Type,
         password_hash_algorithm => Algorithm
     },
+    ok = boostrap_user_from_file(Config, State),
     {ok, State}.
 
 update(Config, _State) ->
@@ -338,8 +339,24 @@ run_fuzzy_filter(
 %%------------------------------------------------------------------------------
 
 insert_user(UserGroup, UserID, PasswordHash, Salt, IsSuperuser) ->
-    UserInfoRecord = user_info_record(UserGroup, UserID, PasswordHash, Salt, IsSuperuser),
-    insert_user(UserInfoRecord).
+    UserInfoRecord =
+        #user_info{user_id = DBUserID} =
+        user_info_record(UserGroup, UserID, PasswordHash, Salt, IsSuperuser),
+    case mnesia:read(?TAB, DBUserID, write) of
+        [] ->
+            insert_user(UserInfoRecord);
+        [UserInfoRecord] ->
+            ok;
+        [_] ->
+            ?SLOG(warning, #{
+                msg => "bootstrap_authentication_overridden_in_the_built_in_database",
+                user_id => UserID,
+                group_id => UserGroup,
+                suggestion =>
+                    "If you have made changes in other way, remove the user_id from the bootstrap file."
+            }),
+            insert_user(UserInfoRecord)
+    end.
 
 insert_user(#user_info{} = UserInfoRecord) ->
     mnesia:write(?TAB, UserInfoRecord, write).
@@ -537,3 +554,25 @@ find_password_hash(_, _, _) ->
 is_superuser(#{<<"is_superuser">> := <<"true">>}) -> true;
 is_superuser(#{<<"is_superuser">> := true}) -> true;
 is_superuser(_) -> false.
+
+boostrap_user_from_file(Config, State) ->
+    case maps:get(boostrap_file, Config, <<>>) of
+        <<>> ->
+            ok;
+        FileName0 ->
+            #{boostrap_type := Type} = Config,
+            FileName = emqx_schema:naive_env_interpolation(FileName0),
+            case file:read_file(FileName) of
+                {ok, FileData} ->
+                    %% if there is a key conflict, override with the key which from the bootstrap file
+                    _ = import_users({Type, FileName, FileData}, State),
+                    ok;
+                {error, Reason} ->
+                    ?SLOG(warning, #{
+                        msg => "boostrap_authn_built_in_database_failed",
+                        boostrap_file => FileName,
+                        boostrap_type => Type,
+                        reason => emqx_utils:explain_posix(Reason)
+                    })
+            end
+    end.

apps/emqx_auth_mnesia/src/emqx_authn_mnesia_schema.erl

@@ -46,7 +46,7 @@ select_union_member(_Kind, _Value) ->
 fields(builtin_db) ->
     [
         {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw/1}
-    ] ++ common_fields();
+    ] ++ common_fields() ++ bootstrap_fields();
 fields(builtin_db_api) ->
     [
         {password_hash_algorithm, fun emqx_authn_password_hashing:type_rw_api/1}
@@ -69,3 +69,24 @@ common_fields() ->
         {backend, emqx_authn_schema:backend(?AUTHN_BACKEND)},
         {user_id_type, fun user_id_type/1}
     ] ++ emqx_authn_schema:common_fields().
+
+bootstrap_fields() ->
+    [
+        {bootstrap_file,
+            ?HOCON(
+                binary(),
+                #{
+                    desc => ?DESC(bootstrap_file),
+                    required => false,
+                    default => <<>>
+                }
+            )},
+        {bootstrap_type,
+            ?HOCON(
+                ?ENUM([hash, plain]), #{
+                    desc => ?DESC(bootstrap_type),
+                    required => false,
+                    default => <<"plain">>
+                }
+            )}
+    ].

apps/emqx_auth_mnesia/test/emqx_authn_mnesia_SUITE.erl

@@ -54,7 +54,74 @@ t_create(_) ->
     {ok, _} = emqx_authn_mnesia:create(?AUTHN_ID, Config0),
 
     Config1 = Config0#{password_hash_algorithm => #{name => sha256}},
-    {ok, _} = emqx_authn_mnesia:create(?AUTHN_ID, Config1).
+    {ok, _} = emqx_authn_mnesia:create(?AUTHN_ID, Config1),
+    ok.
+t_bootstrap_file(_) ->
+    Config = config(),
+    %% hash to hash
+    HashConfig = Config#{password_hash_algorithm => #{name => sha256, salt_position => suffix}},
+    ?assertMatch(
+        [
+            {user_info, {_, <<"myuser1">>}, _, _, true},
+            {user_info, {_, <<"myuser2">>}, _, _, false}
+        ],
+        test_bootstrap_file(HashConfig, hash, <<"user-credentials.json">>)
+    ),
+    ?assertMatch(
+        [
+            {user_info, {_, <<"myuser3">>}, _, _, true},
+            {user_info, {_, <<"myuser4">>}, _, _, false}
+        ],
+        test_bootstrap_file(HashConfig, hash, <<"user-credentials.csv">>)
+    ),
+
+    %% plain to plain
+    PlainConfig = Config#{
+        password_hash_algorithm =>
+            #{name => plain, salt_position => disable}
+    },
+    ?assertMatch(
+        [
+            {user_info, {_, <<"myuser1">>}, <<"password1">>, _, true},
+            {user_info, {_, <<"myuser2">>}, <<"password2">>, _, false}
+        ],
+        test_bootstrap_file(PlainConfig, plain, <<"user-credentials-plain.json">>)
+    ),
+    ?assertMatch(
+        [
+            {user_info, {_, <<"myuser3">>}, <<"password3">>, _, true},
+            {user_info, {_, <<"myuser4">>}, <<"password4">>, _, false}
+        ],
+        test_bootstrap_file(PlainConfig, plain, <<"user-credentials-plain.csv">>)
+    ),
+    %% plain to hash
+    ?assertMatch(
+        [
+            {user_info, {_, <<"myuser1">>}, _, _, true},
+            {user_info, {_, <<"myuser2">>}, _, _, false}
+        ],
+        test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.json">>)
+    ),
+    ?assertMatch(
+        [
+            {user_info, {_, <<"myuser3">>}, _, _, true},
+            {user_info, {_, <<"myuser4">>}, _, _, false}
+        ],
+        test_bootstrap_file(HashConfig, plain, <<"user-credentials-plain.csv">>)
+    ),
+    ok.
+
+test_bootstrap_file(Config0, Type, File) ->
+    {Type, Filename, _FileData} = sample_filename_and_data(Type, File),
+    Config2 = Config0#{
+        boostrap_file => Filename,
+        boostrap_type => Type
+    },
+    {ok, State0} = emqx_authn_mnesia:create(?AUTHN_ID, Config2),
+    Result = ets:tab2list(emqx_authn_mnesia),
+    ok = emqx_authn_mnesia:destroy(State0),
+    ?assertMatch([], ets:tab2list(emqx_authn_mnesia)),
+    Result.
 
 t_update(_) ->
     Config0 = config(),

changes/ce/feat-13336.en.md

@@ -0,0 +1 @@
+Added new configs `bootstrap_file` and `bootstrap_type` for built-in database for authentication to support bootstrapping the table with csv and json file.

rel/i18n/emqx_authn_mnesia_schema.hocon

@@ -9,4 +9,34 @@ user_id_type.desc:
 user_id_type.label:
 """Authentication ID Type"""
 
+bootstrap_file.desc:
+"""The bootstrap file imports users into the built-in database.
+The file content format is determined by `bootstrap_type`.
+Remove the item from the bootstrap file when you have made changes in other way,
+otherwise, after restarting, the bootstrap item will be overridden again."""
+
+bootstrap_file.label:
+"""Bootstrap File Path"""
+
+bootstrap_type.desc:
+"""Specify which type of content the bootstrap file has.
+
+- **`plain`**:
+  - Expected data fields: `user_id`, `password`, `is_superuser`
+  - `user_id`: Can be Client ID or username, depending on built-in database authentication's `user_id_type` config.
+  - `password`: User's plaintext password.
+  - `is_superuser`: Boolean, user's administrative status.
+
+- **`hash`**:
+  - Expected data fields: `user_id`,`password_hash`,`salt`,`is_superuser`
+  - Definitions similar to `plain` type, with `password_hash` and `salt` added for security.
+
+The content can be either in CSV, or JSON format.
+
+Here is a CSV example: `user_id,password_hash,salt,is_superuser\nmy_user,b6c743545a7817ae8c8f624371d5f5f0373234bb0ff36b8ffbf19bce0e06ab75,de1024f462fb83910fd13151bd4bd235,true`
+
+And JSON content should be decoded into an array of objects, for example: `[{"user_id": "my_user","password": "s3cr3tp@ssw0rd","is_superuser": true}]`.
+
+The hash string for `password_hash` depends on how `password_hash_algorithm` is configured for the built-in database authentication mechanism. For example, if it's configured as `password_hash_algorithm {name = sha256, salt_position = suffix}`, then the salt is appended to the password before hashed. Here is the equivalent Python expression: `hashlib.sha256(password + salt).hexdigest()`."""
+
 }