chore: support api_key.bootstrap_file config

apps/emqx/src/emqx_packet.erl

@@ -492,7 +492,7 @@ format_variable(undefined, _, _) ->
 format_variable(Variable, undefined, PayloadEncode) ->
     format_variable(Variable, PayloadEncode);
 format_variable(Variable, Payload, PayloadEncode) ->
-    [format_variable(Variable, PayloadEncode), format_payload(Payload, PayloadEncode)].
+    [format_variable(Variable, PayloadEncode), ",", format_payload(Payload, PayloadEncode)].
 
 format_variable(
     #mqtt_packet_connect{

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -60,7 +60,8 @@
     emqx_exhook_schema,
     emqx_psk_schema,
     emqx_limiter_schema,
-    emqx_slow_subs_schema
+    emqx_slow_subs_schema,
+    emqx_mgmt_api_key_schema
 ]).
 
 %% root config should not have a namespace

apps/emqx_dashboard/i18n/emqx_dashboard_i18n.conf

@@ -199,23 +199,12 @@ its own from which a browser should permit loading resources."""
   }
   bootstrap_users_file {
     desc {
-      en: "Initialize users file."
-      zh: "初始化用户文件"
-    }
-    label {
-      en: """Is used to add an administrative user to Dashboard when emqx is first launched,
-      the format is:
-       ```
-       username1:password1
-       username2:password2
-       ```
-"""
-      zh: """用于在首次启动 emqx 时，为 Dashboard 添加管理用户，其格式为：
-      ```
-      username1:password1
-      username2:password2
-      ```
-"""
+      en: "Deprecated, use api_key.bootstrap_file"
+      zh: "已废弃，请使用 api_key.bootstrap_file"
+    }
+    label {
+      en: """Deprecated"""
+      zh: """已废弃"""
     }
   }
 }

apps/emqx_dashboard/src/emqx_dashboard_admin.erl

@@ -51,8 +51,7 @@
 
 -export([
     add_default_user/0,
-    default_username/0,
-    add_bootstrap_users/0
+    default_username/0
 ]).
 
 -type emqx_admin() :: #?ADMIN{}.
@@ -85,21 +84,6 @@ mnesia(boot) ->
 add_default_user() ->
     add_default_user(binenv(default_username), binenv(default_password)).
 
--spec add_bootstrap_users() -> ok | {error, _}.
-add_bootstrap_users() ->
-    case emqx:get_config([dashboard, bootstrap_users_file], undefined) of
-        undefined ->
-            ok;
-        File ->
-            case mnesia:table_info(?ADMIN, size) of
-                0 ->
-                    ?SLOG(debug, #{msg => "Add dashboard bootstrap users", file => File}),
-                    add_bootstrap_users(File);
-                _ ->
-                    ok
-            end
-    end.
-
 %%--------------------------------------------------------------------
 %% API
 %%--------------------------------------------------------------------
@@ -311,44 +295,3 @@ add_default_user(Username, Password) ->
         [] -> add_user(Username, Password, <<"administrator">>);
         _ -> {ok, default_user_exists}
     end.
-
-add_bootstrap_users(File) ->
-    case file:open(File, [read]) of
-        {ok, Dev} ->
-            {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]),
-            try
-                load_bootstrap_user(Dev, MP)
-            catch
-                Type:Reason ->
-                    {error, {Type, Reason}}
-            after
-                file:close(Dev)
-            end;
-        {error, Reason} = Error ->
-            ?SLOG(error, #{
-                msg => "failed to open the dashboard bootstrap users file",
-                file => File,
-                reason => Reason
-            }),
-            Error
-    end.
-
-load_bootstrap_user(Dev, MP) ->
-    case file:read_line(Dev) of
-        {ok, Line} ->
-            case re:run(Line, MP, [global, {capture, all_but_first, binary}]) of
-                {match, [[Username, Password]]} ->
-                    case add_user(Username, Password, ?BOOTSTRAP_USER_TAG) of
-                        {ok, _} ->
-                            load_bootstrap_user(Dev, MP);
-                        Error ->
-                            Error
-                    end;
-                _ ->
-                    load_bootstrap_user(Dev, MP)
-            end;
-        eof ->
-            ok;
-        Error ->
-            Error
-    end.

apps/emqx_dashboard/src/emqx_dashboard_app.erl

@@ -31,13 +31,8 @@ start(_StartType, _StartArgs) ->
     case emqx_dashboard:start_listeners() of
         ok ->
             emqx_dashboard_cli:load(),
-            case emqx_dashboard_admin:add_bootstrap_users() of
-                ok ->
-                    {ok, _} = emqx_dashboard_admin:add_default_user(),
-                    {ok, Sup};
-                Error ->
-                    Error
-            end;
+            {ok, _} = emqx_dashboard_admin:add_default_user(),
+            {ok, Sup};
         {error, Reason} ->
             {error, Reason}
     end.

apps/emqx_dashboard/src/emqx_dashboard_schema.erl

@@ -56,7 +56,15 @@ fields("dashboard") ->
         {cors, fun cors/1},
         {i18n_lang, fun i18n_lang/1},
         {bootstrap_users_file,
-            ?HOCON(binary(), #{desc => ?DESC(bootstrap_users_file), required => false})}
+            ?HOCON(
+                binary(),
+                #{
+                    desc => ?DESC(bootstrap_users_file),
+                    required => false,
+                    default => <<>>
+                    %% deprecated => {since, "5.1.0"}
+                }
+            )}
     ];
 fields("listeners") ->
     [

apps/emqx_management/i18n/emqx_mgmt_api_key.conf

@@ -0,0 +1,221 @@
+emqx_dashboard_schema {
+  listeners {
+    desc {
+      en: """HTTP(s) listeners are identified by their protocol type and are
+used to serve dashboard UI and restful HTTP API.
+Listeners must have a unique combination of port number and IP address.
+For example, an HTTP listener can listen on all configured IP addresses
+on a given port for a machine by specifying the IP address 0.0.0.0.
+Alternatively, the HTTP listener can specify a unique IP address for each listener,
+but use the same port."""
+      zh: """仪表盘监听器设置。"""
+    }
+    label {
+      en: "Listeners"
+      zh: "监听器"
+    }
+  }
+  sample_interval {
+    desc {
+      en: """How often to update metrics displayed in the dashboard.
+Note: `sample_interval` should be a divisor of 60."""
+      zh: """更新仪表板中显示的指标的时间间隔。必须小于60，且被60的整除。"""
+    }
+  }
+  token_expired_time {
+    desc {
+      en: "JWT token expiration time."
+      zh: "JWT token 过期时间"
+    }
+    label {
+      en: "Token expired time"
+      zh: "JWT 过期时间"
+    }
+  }
+  num_acceptors {
+    desc {
+      en: "Socket acceptor pool size for TCP protocols."
+      zh: "TCP协议的Socket acceptor池大小"
+    }
+    label {
+      en: "Number of acceptors"
+      zh: "Acceptor 数量"
+    }
+  }
+  max_connections {
+    desc {
+      en: "Maximum number of simultaneous connections."
+      zh: "同时处理的最大连接数"
+    }
+    label {
+      en: "Maximum connections"
+      zh: "最大连接数"
+    }
+  }
+  backlog {
+    desc {
+      en: "Defines the maximum length that the queue of pending connections can grow to."
+      zh: "排队等待连接的队列的最大长度"
+    }
+    label {
+      en: "Backlog"
+      zh: "排队长度"
+    }
+  }
+  send_timeout {
+    desc {
+      en: "Send timeout for the socket."
+      zh: "Socket发送超时时间"
+    }
+    label {
+      en: "Send timeout"
+      zh: "发送超时时间"
+    }
+  }
+  inet6 {
+    desc {
+      en: "Enable IPv6 support, default is false, which means IPv4 only."
+      zh: "启用IPv6， 如果机器不支持IPv6，请关闭此选项，否则会导致仪表盘无法使用。"
+    }
+    label {
+      en: "IPv6"
+      zh: "IPv6"
+    }
+  }
+  ipv6_v6only {
+    desc {
+      en: "Disable IPv4-to-IPv6 mapping for the listener."
+      zh: "当开启 inet6 功能的同时禁用 IPv4-to-IPv6 映射。该配置仅在 inet6 功能开启时有效。"
+    }
+    label {
+      en: "IPv6 only"
+      zh: "IPv6 only"
+    }
+  }
+  desc_dashboard {
+    desc {
+      en: "Configuration for EMQX dashboard."
+      zh: "EMQX仪表板配置"
+    }
+    label {
+      en: "Dashboard"
+      zh: "仪表板"
+    }
+  }
+  desc_listeners {
+    desc {
+      en: "Configuration for the dashboard listener."
+      zh: "仪表板监听器配置"
+    }
+    label {
+      en: "Listeners"
+      zh: "监听器"
+    }
+  }
+  desc_http {
+    desc {
+      en: "Configuration for the dashboard listener (plaintext)."
+      zh: "仪表板监听器(HTTP)配置"
+    }
+    label {
+      en: "HTTP"
+      zh: "HTTP"
+    }
+  }
+  desc_https {
+    desc {
+      en: "Configuration for the dashboard listener (TLS)."
+      zh: "仪表板监听器(HTTPS)配置"
+    }
+    label {
+      en: "HTTPS"
+      zh: "HTTPS"
+    }
+  }
+  listener_enable {
+    desc {
+        en: "Ignore or enable this listener"
+        zh: "忽略或启用该监听器配置"
+    }
+    label {
+        en: "Enable"
+        zh: "启用"
+    }
+  }
+  bind {
+    desc {
+      en: "Port without IP(18083) or port with specified IP(127.0.0.1:18083)."
+      zh: "监听的地址与端口，在dashboard更新此配置时，会重启dashboard服务。"
+    }
+    label {
+      en: "Bind"
+      zh: "绑定端口"
+    }
+  }
+  default_username {
+    desc {
+      en: "The default username of the automatically created dashboard user."
+      zh: "默认的仪表板用户名"
+    }
+    label {
+      en: "Default username"
+      zh: "默认用户名"
+    }
+  }
+  default_password {
+    desc {
+      en: """The initial default password for dashboard 'admin' user.
+For safety, it should be changed as soon as possible."""
+      zh: """默认的仪表板用户密码
+为了安全，应该尽快修改密码。"""
+    }
+    label {
+      en: "Default password"
+      zh: "默认密码"
+    }
+  }
+  cors {
+    desc {
+      en: """Support Cross-Origin Resource Sharing (CORS).
+Allows a server to indicate any origins (domain, scheme, or port) other than
+its own from which a browser should permit loading resources."""
+      zh: """支持跨域资源共享(CORS)
+允许服务器指示任何来源(域名、协议或端口)，除了本服务器之外的任何浏览器应允许加载资源。"""
+    }
+    label {
+      en: "CORS"
+      zh: "跨域资源共享"
+    }
+  }
+  i18n_lang {
+    desc {
+      en: "Internationalization language support."
+      zh: "swagger多语言支持"
+    }
+    label {
+      en: "I18n language"
+      zh: "多语言支持"
+    }
+  }
+  bootstrap_users_file {
+    desc {
+      en: "Initialize users file."
+      zh: "初始化用户文件"
+    }
+    label {
+      en: """Is used to add an administrative user to Dashboard when emqx is first launched,
+      the format is:
+       ```
+       username1:password1
+       username2:password2
+       ```
+"""
+      zh: """用于在首次启动 emqx 时，为 Dashboard 添加管理用户，其格式为：
+      ```
+      username1:password1
+      username2:password2
+      ```
+"""
+    }
+  }
+}

apps/emqx_management/i18n/emqx_mgmt_api_key_i18n.conf

@@ -0,0 +1,33 @@
+emqx_mgmt_api_key_schema {
+  api_key {
+    desc {
+      en: """API Key, can be used to request API other than the management API key and the Dashboard user management API"""
+      zh: """API 密钥， 可用于请求除管理 API 密钥及 Dashboard 用户管理 API 的其它接口"""
+    }
+    label {
+      en: "API Key"
+      zh: "API 密钥"
+    }
+  }
+  bootstrap_file {
+    desc {
+      en: """Is used to add an api_key when emqx is launched,
+      the format is:
+       ```
+       7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK
+       ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL
+       ```
+"""
+      zh: """用于在启动 emqx 时，添加 API 密钥，其格式为：
+      ```
+      7e729ae70d23144b:2QILI9AcQ9BYlVqLDHQNWN2saIjBV4egr1CZneTNKr9CpK
+      ec3907f865805db0:Ee3taYltUKtoBVD9C3XjQl9C6NXheip8Z9B69BpUv5JxVHL
+      ```
+"""
+    }
+    label {
+      en: "Initialize api_key file."
+      zh: "API 密钥初始化文件"
+    }
+  }
+}

apps/emqx_management/src/emqx_mgmt_api_app.erl

@@ -13,7 +13,7 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--module(emqx_mgmt_api_app).
+-module(emqx_mgmt_api_api_keys).
 
 -behaviour(minirest_api).
 

apps/emqx_management/src/emqx_mgmt_api_configs.erl

@@ -63,7 +63,8 @@
         <<"prometheus">>,
         <<"telemetry">>,
         <<"listeners">>,
-        <<"license">>
+        <<"license">>,
+        <<"api_key">>
     ] ++ global_zone_roots()
 ).
 

apps/emqx_management/src/emqx_mgmt_api_key_schema.erl

@@ -0,0 +1,44 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_mgmt_api_key_schema).
+
+-include_lib("hocon/include/hoconsc.hrl").
+
+-export([
+    roots/0,
+    fields/1,
+    namespace/0,
+    desc/1
+]).
+
+namespace() -> api_key.
+roots() -> ["api_key"].
+
+fields("api_key") ->
+    [
+        {bootstrap_file,
+            ?HOCON(
+                binary(),
+                #{
+                    desc => ?DESC(bootstrap_file),
+                    required => false,
+                    default => <<>>
+                }
+            )}
+    ].
+
+desc("api_key") ->
+    ?DESC(api_key).

apps/emqx_management/src/emqx_mgmt_app.erl

@@ -28,10 +28,15 @@
 -include("emqx_mgmt.hrl").
 
 start(_Type, _Args) ->
-    {ok, Sup} = emqx_mgmt_sup:start_link(),
     ok = mria_rlog:wait_for_shards([?MANAGEMENT_SHARD], infinity),
-    emqx_mgmt_cli:load(),
-    {ok, Sup}.
+    case emqx_mgmt_auth:init_bootstrap_file() of
+        ok ->
+            {ok, Sup} = emqx_mgmt_sup:start_link(),
+            ok = emqx_mgmt_cli:load(),
+            {ok, Sup};
+        {error, Reason} ->
+            {error, Reason}
+    end.
 
 stop(_State) ->
     ok.

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -15,6 +15,7 @@
 %%--------------------------------------------------------------------
 -module(emqx_mgmt_auth).
 -include_lib("emqx/include/emqx.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 %% API
 -export([mnesia/1]).
@@ -25,7 +26,8 @@
     read/1,
     update/4,
     delete/1,
-    list/0
+    list/0,
+    init_bootstrap_file/0
 ]).
 
 -export([authorize/3]).
@@ -34,7 +36,8 @@
 -export([
     do_update/4,
     do_delete/1,
-    do_create_app/3
+    do_create_app/3,
+    do_force_create_app/3
 ]).
 
 -define(APP, emqx_app).
@@ -58,6 +61,12 @@ mnesia(boot) ->
         {attributes, record_info(fields, ?APP)}
     ]).
 
+-spec init_bootstrap_file() -> ok | {error, _}.
+init_bootstrap_file() ->
+    File = bootstrap_file(),
+    ?SLOG(debug, #{msg => "init_bootstrap_api_keys_from_file", file => File}),
+    init_bootstrap_file(File).
+
 create(Name, Enable, ExpiredAt, Desc) ->
     case mnesia:table_info(?APP, size) < 1024 of
         true -> create_app(Name, Enable, ExpiredAt, Desc);
@@ -169,6 +178,9 @@ create_app(Name, Enable, ExpiredAt, Desc) ->
 create_app(App = #?APP{api_key = ApiKey, name = Name}) ->
     trans(fun ?MODULE:do_create_app/3, [App, ApiKey, Name]).
 
+force_create_app(NamePrefix, App = #?APP{api_key = ApiKey}) ->
+    trans(fun ?MODULE:do_force_create_app/3, [App, ApiKey, NamePrefix]).
+
 do_create_app(App, ApiKey, Name) ->
     case mnesia:read(?APP, Name) of
         [_] ->
@@ -183,6 +195,22 @@ do_create_app(App, ApiKey, Name) ->
             end
     end.
 
+do_force_create_app(App, ApiKey, NamePrefix) ->
+    case mnesia:match_object(?APP, #?APP{api_key = ApiKey, _ = '_'}, read) of
+        [] ->
+            NewName = generate_unique_name(NamePrefix),
+            ok = mnesia:write(App#?APP{name = NewName});
+        [#?APP{name = Name}] ->
+            ok = mnesia:write(App#?APP{name = Name})
+    end.
+
+generate_unique_name(NamePrefix) ->
+    New = list_to_binary(NamePrefix ++ emqx_misc:gen_id(16)),
+    case mnesia:read(?APP, New) of
+        [] -> New;
+        _ -> generate_unique_name(NamePrefix)
+    end.
+
 trans(Fun, Args) ->
     case mria:transaction(?COMMON_SHARD, Fun, Args) of
         {atomic, Res} -> {ok, Res};
@@ -192,3 +220,83 @@ trans(Fun, Args) ->
 generate_api_secret() ->
     Random = crypto:strong_rand_bytes(32),
     emqx_base62:encode(Random).
+
+bootstrap_file() ->
+    case emqx:get_config([api_key, bootstrap_file], <<>>) of
+        %% For compatible remove until 5.1.0
+        <<>> ->
+            emqx:get_config([dashboard, bootstrap_users_file], <<>>);
+        File ->
+            File
+    end.
+
+init_bootstrap_file(<<>>) ->
+    ok;
+init_bootstrap_file(File) ->
+    case file:open(File, [read, binary]) of
+        {ok, Dev} ->
+            {ok, MP} = re:compile(<<"(\.+):(\.+$)">>, [ungreedy]),
+            init_bootstrap_file(File, Dev, MP);
+        {error, Reason} = Error ->
+            ?SLOG(
+                error,
+                #{
+                    msg => "failed_to_open_the_bootstrap_file",
+                    file => File,
+                    reason => emqx_misc:explain_posix(Reason)
+                }
+            ),
+            Error
+    end.
+
+init_bootstrap_file(File, Dev, MP) ->
+    try
+        add_bootstrap_file(File, Dev, MP, 1)
+    catch
+        throw:Error -> {error, Error};
+        Type:Reason:Stacktrace -> {error, {Type, Reason, Stacktrace}}
+    after
+        file:close(Dev)
+    end.
+
+-define(BOOTSTRAP_TAG, <<"Bootstrapped From File">>).
+
+add_bootstrap_file(File, Dev, MP, Line) ->
+    case file:read_line(Dev) of
+        {ok, Bin} ->
+            case re:run(Bin, MP, [global, {capture, all_but_first, binary}]) of
+                {match, [[AppKey, ApiSecret]]} ->
+                    App =
+                        #?APP{
+                            enable = true,
+                            expired_at = infinity,
+                            desc = ?BOOTSTRAP_TAG,
+                            created_at = erlang:system_time(second),
+                            api_secret_hash = emqx_dashboard_admin:hash(ApiSecret),
+                            api_key = AppKey
+                        },
+                    case force_create_app("from_bootstrap_file_", App) of
+                        {ok, ok} ->
+                            add_bootstrap_file(File, Dev, MP, Line + 1);
+                        {error, Reason} ->
+                            throw(#{file => File, line => Line, content => Bin, reason => Reason})
+                    end;
+                _ ->
+                    Reason = "invalid_format",
+                    ?SLOG(
+                        error,
+                        #{
+                            msg => "failed_to_load_bootstrap_file",
+                            file => File,
+                            line => Line,
+                            content => Bin,
+                            reason => Reason
+                        }
+                    ),
+                    throw(#{file => File, line => Line, content => Bin, reason => Reason})
+            end;
+        eof ->
+            ok;
+        {error, Reason} ->
+            throw(#{file => File, line => Line, reason => Reason})
+    end.