chore: add refactor prometheus refactor changelog

apps/emqx_prometheus/src/emqx_prometheus.erl

@@ -59,6 +59,12 @@
 
 -export([collect/1]).
 
+-export([
+    %% For bpapi, deprecated_since 5.0.10, remove this when 5.1.x
+    do_start/0,
+    do_stop/0
+]).
+
 -define(C(K, L), proplists:get_value(K, L, 0)).
 
 -define(TIMER_MSG, '#interval').
@@ -112,7 +118,6 @@ handle_info({update, Conf}, State = #{timer := Timer}) ->
 handle_info(_Msg, State) ->
     {noreply, State}.
 
-
 push_to_push_gateway(Url, Headers) when is_list(Headers) ->
     Data = prometheus_text_format:format(),
     case httpc:request(post, {Url, Headers, "text/plain", Data}, ?HTTP_OPTIONS, []) of
@@ -678,3 +683,11 @@ emqx_cluster_data() ->
         {nodes_running, length(Running)},
         {nodes_stopped, length(Stopped)}
     ].
+
+%% deprecated_since 5.0.10, remove this when 5.1.x
+do_start() ->
+    emqx_prometheus_sup:start_child(?APP).
+
+%% deprecated_since 5.0.10, remove this when 5.1.x
+do_stop() ->
+    emqx_prometheus_sup:stop_child(?APP).

apps/emqx_prometheus/src/emqx_prometheus_config.erl

@@ -20,7 +20,7 @@
 -include("emqx_prometheus.hrl").
 
 -export([add_handler/0, remove_handler/0]).
--export([post_config_update/5]).
+-export([pre_config_update/3, post_config_update/5]).
 -export([update/1]).
 -export([conf/0, is_push_gateway_server_enabled/1]).
 
@@ -46,6 +46,54 @@ remove_handler() ->
     ok = emqx_config_handler:remove_handler(?PROMETHEUS),
     ok.
 
+%% when we import the config with the old version
+%% we need to respect it, and convert to new schema.
+pre_config_update(?PROMETHEUS, MergeConf, OriginConf) ->
+    OriginType = emqx_prometheus_schema:is_recommend_type(OriginConf),
+    MergeType = emqx_prometheus_schema:is_recommend_type(MergeConf),
+    {ok,
+        case {OriginType, MergeType} of
+            {true, false} -> to_recommend_type(MergeConf);
+            _ -> MergeConf
+        end}.
+
+to_recommend_type(Conf) ->
+    #{
+        <<"push_gateway">> => to_push_gateway(Conf),
+        <<"collectors">> => to_collectors(Conf)
+    }.
+
+to_push_gateway(Conf) ->
+    Init = maps:with([<<"interval">>, <<"headers">>, <<"job_name">>], Conf),
+    case maps:get(<<"push_gateway_server">>, Conf, "") of
+        "" ->
+            Init#{<<"url">> => <<"">>};
+        Url ->
+            case maps:get(<<"enable">>, Conf, false) of
+                false -> Init#{<<"url">> => <<"">>};
+                true -> Init#{<<"url">> => Url}
+            end
+    end.
+
+to_collectors(Conf) ->
+    lists:foldl(
+        fun({From, To}, Acc) ->
+            case maps:find(From, Conf) of
+                {ok, Value} -> Acc#{To => Value};
+                error -> Acc
+            end
+        end,
+        #{},
+        [
+            {<<"vm_dist_collector">>, <<"vm_dist">>},
+            {<<"mnesia_collector">>, <<"mnesia">>},
+            {<<"vm_statistics_collector">>, <<"vm_statistics">>},
+            {<<"vm_system_info_collector">>, <<"vm_system_info">>},
+            {<<"vm_memory_collector">>, <<"vm_memory">>},
+            {<<"vm_msacc_collector">>, <<"vm_msacc">>}
+        ]
+    ).
+
 post_config_update(?PROMETHEUS, _Req, New, Old, AppEnvs) ->
     update_prometheus(AppEnvs),
     _ = update_push_gateway(New),

apps/emqx_prometheus/src/emqx_prometheus_schema.erl

@@ -27,7 +27,8 @@
     desc/1,
     translation/1,
     convert_headers/2,
-    validate_url/1
+    validate_url/1,
+    is_recommend_type/1
 ]).
 
 namespace() -> prometheus.
@@ -63,7 +64,7 @@ fields(recommend_setting) ->
                 }
             )},
         {collectors,
-            ?HOCON(?R_REF(collector), #{
+            ?HOCON(?R_REF(collectors), #{
                 required => false,
                 importance => ?IMPORTANCE_LOW,
                 desc => ?DESC(collectors)
@@ -110,7 +111,7 @@ fields(push_gateway) ->
                 }
             )}
     ];
-fields(collector) ->
+fields(collectors) ->
     [
         {vm_dist,
             ?HOCON(
@@ -295,13 +296,35 @@ setting_union_schema() ->
     RecommendSetting = ?R_REF(recommend_setting),
     LegacySetting = ?R_REF(legacy_deprecated_setting),
     fun
-        (all_union_members) -> [RecommendSetting, LegacySetting];
-        ({value, #{<<"enable">> := _}}) -> [LegacySetting];
-        %% all other cases treat as new config, include init empty config.
-        ({value, _}) -> [RecommendSetting]
+        (all_union_members) ->
+            [RecommendSetting, LegacySetting];
+        ({value, Setting}) ->
+            case is_recommend_type(Setting) of
+                true -> [RecommendSetting];
+                false -> [LegacySetting]
+            end
+    end.
+
+%% For it to be considered as new schema,
+%% all keys must be included in the new configuration.
+is_recommend_type(Setting) ->
+    case maps:keys(Setting) of
+        [] ->
+            true;
+        Keys ->
+            NewKeys = fields(recommend_setting),
+            Fun = fun(Key0) ->
+                Key = binary_to_existing_atom(Key0),
+                lists:keymember(Key, 1, NewKeys)
+            end,
+            lists:all(Fun, Keys)
     end.
 
 desc(prometheus) -> ?DESC(prometheus);
+desc(collectors) -> ?DESC(collectors);
+desc(legacy_deprecated_setting) -> ?DESC(legacy_deprecated_setting);
+desc(recommend_setting) -> ?DESC(recommend_setting);
+desc(push_gateway) -> ?DESC(push_gateway);
 desc(_) -> undefined.
 
 convert_headers(undefined, _) ->

apps/emqx_prometheus/src/emqx_prometheus_sup.erl

@@ -20,6 +20,7 @@
 
 -export([
     start_link/0,
+    start_child/1,
     start_child/2,
     update_child/2,
     stop_child/1
@@ -40,6 +41,10 @@
 start_link() ->
     supervisor:start_link({local, ?MODULE}, ?MODULE, []).
 
+-spec start_child(atom()) -> ok.
+start_child(Mod) when is_atom(Mod) ->
+    start_child(Mod, emqx_prometheus_config:conf()).
+
 -spec start_child(atom(), map()) -> ok.
 start_child(Mod, Conf) when is_atom(Mod) ->
     assert_started(supervisor:start_child(?MODULE, ?CHILD(Mod, Conf))).

apps/emqx_prometheus/src/proto/emqx_prometheus_proto_v1.erl

@@ -0,0 +1,41 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_prometheus_proto_v1).
+
+-behaviour(emqx_bpapi).
+
+-export([
+    introduced_in/0,
+    deprecated_since/0,
+    start/1,
+    stop/1
+]).
+
+-include_lib("emqx/include/bpapi.hrl").
+
+deprecated_since() -> "5.0.10".
+
+introduced_in() ->
+    "5.0.0".
+
+-spec start([node()]) -> emqx_rpc:multicall_result().
+start(Nodes) ->
+    rpc:multicall(Nodes, emqx_prometheus, do_start, [], 5000).
+
+-spec stop([node()]) -> emqx_rpc:multicall_result().
+stop(Nodes) ->
+    rpc:multicall(Nodes, emqx_prometheus, do_stop, [], 5000).

apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl

@@ -219,5 +219,5 @@ init(Req0, Opts) ->
         Headers
     ),
     RespHeader = #{<<"content-type">> => <<"text/plain; charset=utf-8">>},
-    Req = -cowboy_req:reply(200, RespHeader, <<"OK">>, Req0),
+    Req = cowboy_req:reply(200, RespHeader, <<"OK">>, Req0),
     {ok, Req, Opts}.

changes/ce/feat-11884.en.md

@@ -0,0 +1,4 @@
+Modified the Prometheus API and configuration to:
+- Restructure configuration sections to group related settings, improving readability and maintainability
+- Introduced `enable_basic_auth` configuration for basic authentication on the scrape API endpoint, enhancing security
+- Maintained backwards compatibility while refactoring code, avoiding breaking changes

rel/i18n/emqx_prometheus_schema.hocon

@@ -17,7 +17,7 @@ Default value is: <code>${name}/instance/${name}~${host}</code>"""
 
 prometheus.desc:
 """EMQX's Prometheus scraping endpoint is enabled by default without authentication.
-You can inspect it with a `curl` command like this: `curl -f "127.0.0.1:18083/api/v5/prometheus/stats"`<br/>"""
+You can inspect it with a `curl` command like this: `curl -f "127.0.0.1:18083/api/v5/prometheus/stats"`"""
 
 prometheus.label:
 """Prometheus"""
@@ -25,11 +25,17 @@ prometheus.label:
 push_gateway.desc:
 """Push Gateway is optional, should not be configured if prometheus is to scrape EMQX."""
 
+enable_basic_auth.desc:
+"""Enable or disable basic authentication for prometheus scrape api, not for Push Gateway"""
+
 collectors.desc:
 """The internal advanced metrics of the virtual machine are initially disabled
 and are usually only enabled during performance testing.
 Enabling them will increase the CPU load."""
 
+recommend_setting.desc:
+"""Recommended setting"""
+
 push_gateway_url.desc:
 """URL of Pushgateway server. Pushgateway is optional, should not be configured if prometheus is to scrape EMQX.
 Set url to "" to disable push gateway"""
@@ -54,37 +60,43 @@ vm_statistics_collector.desc:
 vm_system_info_collector.desc:
 """Enable or disable VM system info collector."""
 
+legacy_deprecated_setting.desc:
+"""Deprecated"""
+
 legacy_enable.desc:
-"""Deprecated, use prometheus.push_gateway.url instead"""
+"""Deprecated, use `prometheus.push_gateway.url` instead"""
 
 legacy_headers.desc:
-"""Deprecated, use prometheus.push_gateway.headers instead"""
+"""Deprecated, use `prometheus.push_gateway.headers` instead"""
 
 legacy_interval.desc:
-"""Deprecated, use prometheus.push_gateway.interval instead"""
+"""Deprecated, use `prometheus.push_gateway.interval` instead"""
 
 legacy_job_name.desc:
-"""Deprecated, use prometheus.push_gateway.job_name instead"""
+"""Deprecated, use `prometheus.push_gateway.job_name` instead"""
 
 legacy_push_gateway_server.desc:
-"""Deprecated, use prometheus.push_gateway.url instead"""
+"""Deprecated, use `prometheus.push_gateway.url` instead"""
 
 legacy_mnesia_collector.desc:
-"""Deprecated, use prometheus.collectors.mnesia instead"""
+"""Deprecated, use `prometheus.collectors.mnesia` instead"""
 
 legacy_vm_dist_collector.desc:
-"""Deprecated, use prometheus.collectors.vm_dist instead"""
+"""Deprecated, use `prometheus.collectors.vm_dist` instead"""
 
 legacy_vm_memory_collector.desc:
-"""Deprecated, use prometheus.collectors.vm_memory instead"""
+"""Deprecated, use `prometheus.collectors.vm_memory` instead"""
 
 legacy_vm_msacc_collector.desc:
-"""Deprecated, use prometheus.collectors.vm_msacc instead"""
+"""Deprecated, use `prometheus.collectors.vm_msacc` instead"""
 
 legacy_vm_statistics_collector.desc:
-"""Deprecated, use prometheus.collectors.vm_statistics instead"""
+"""Deprecated, use `prometheus.collectors.vm_statistics` instead"""
 
 legacy_vm_system_info_collector.desc:
-"""Deprecated, use prometheus.collectors.vm_system_info instead"""
+"""Deprecated, use `prometheus.collectors.vm_system_info` instead"""
+
+legacy_deprecated_setting.desc:
+"""Deprecated"""
 
 }

scripts/spellcheck/dicts/emqx.txt

@@ -295,3 +295,4 @@ dnstream
 upstream
 priv
 Syskeeper
+msacc