Merge pull request #11496 from zhongwencool/erlang-prometheus-collector-disabled-default

feat: disabled the Erlang VM Prometheus exporter by default

apps/emqx/src/emqx_metrics.erl

@@ -267,7 +267,7 @@
     {counter, 'authentication.failure'}
 ]).
 
-%% Overload protetion counters
+%% Overload protection counters
 -define(OLP_METRICS, [
     {counter, 'olp.delay.ok'},
     {counter, 'olp.delay.timeout'},

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -1105,12 +1105,7 @@ translation("gen_rpc") ->
     [{"default_client_driver", fun tr_default_config_driver/1}];
 translation("prometheus") ->
     [
-        {"vm_dist_collector_metrics", fun tr_vm_dist_collector/1},
-        {"mnesia_collector_metrics", fun tr_mnesia_collector/1},
-        {"vm_statistics_collector_metrics", fun tr_vm_statistics_collector/1},
-        {"vm_system_info_collector_metrics", fun tr_vm_system_info_collector/1},
-        {"vm_memory_collector_metrics", fun tr_vm_memory_collector/1},
-        {"vm_msacc_collector_metrics", fun tr_vm_msacc_collector/1}
+        {"collectors", fun tr_prometheus_collectors/1}
     ];
 translation("vm_args") ->
     [
@@ -1120,26 +1115,53 @@ translation("vm_args") ->
 tr_vm_args_process_limit(Conf) ->
     2 * conf_get("node.max_ports", Conf, ?DEFAULT_MAX_PORTS).
 
+tr_prometheus_collectors(Conf) ->
+    [
+        %% builtin collectors
+        prometheus_boolean,
+        prometheus_counter,
+        prometheus_gauge,
+        prometheus_histogram,
+        prometheus_quantile_summary,
+        prometheus_summary,
+        %% emqx collectors
+        emqx_prometheus,
+        emqx_prometheus_mria
+        %% builtin vm collectors
+        | tr_vm_dist_collector(Conf) ++
+            tr_mnesia_collector(Conf) ++
+            tr_vm_statistics_collector(Conf) ++
+            tr_vm_system_info_collector(Conf) ++
+            tr_vm_memory_collector(Conf) ++
+            tr_vm_msacc_collector(Conf)
+    ].
+
 tr_vm_dist_collector(Conf) ->
-    metrics_enabled(conf_get("prometheus.vm_dist_collector", Conf, enabled)).
+    Enabled = conf_get("prometheus.vm_dist_collector", Conf, disabled),
+    collector_enabled(Enabled, prometheus_vm_dist_collector).
 
 tr_mnesia_collector(Conf) ->
-    metrics_enabled(conf_get("prometheus.mnesia_collector", Conf, enabled)).
+    Enabled = conf_get("prometheus.mnesia_collector", Conf, disabled),
+    collector_enabled(Enabled, prometheus_mnesia_collector).
 
 tr_vm_statistics_collector(Conf) ->
-    metrics_enabled(conf_get("prometheus.vm_statistics_collector", Conf, enabled)).
+    Enabled = conf_get("prometheus.vm_statistics_collector", Conf, disabled),
+    collector_enabled(Enabled, prometheus_vm_statistics_collector).
 
 tr_vm_system_info_collector(Conf) ->
-    metrics_enabled(conf_get("prometheus.vm_system_info_collector", Conf, enabled)).
+    Enabled = conf_get("prometheus.vm_system_info_collector", Conf, disabled),
+    collector_enabled(Enabled, prometheus_vm_system_info_collector).
 
 tr_vm_memory_collector(Conf) ->
-    metrics_enabled(conf_get("prometheus.vm_memory_collector", Conf, enabled)).
+    Enabled = conf_get("prometheus.vm_memory_collector", Conf, disabled),
+    collector_enabled(Enabled, prometheus_vm_memory_collector).
 
 tr_vm_msacc_collector(Conf) ->
-    metrics_enabled(conf_get("prometheus.vm_msacc_collector", Conf, enabled)).
+    Enabled = conf_get("prometheus.vm_msacc_collector", Conf, disabled),
+    collector_enabled(Enabled, prometheus_vm_msacc_collector).
 
-metrics_enabled(enabled) -> all;
-metrics_enabled(disabled) -> [].
+collector_enabled(enabled, Collector) -> [Collector];
+collector_enabled(disabled, _) -> [].
 
 tr_default_config_driver(Conf) ->
     conf_get("rpc.driver", Conf).

apps/emqx_prometheus/src/emqx_prometheus.app.src

@@ -2,7 +2,7 @@
 {application, emqx_prometheus, [
     {description, "Prometheus for EMQX"},
     % strict semver, bump manually!
-    {vsn, "5.0.15"},
+    {vsn, "5.0.16"},
     {modules, []},
     {registered, [emqx_prometheus_sup]},
     {applications, [kernel, stdlib, prometheus, emqx, emqx_management]},

apps/emqx_prometheus/src/emqx_prometheus_config.erl

@@ -46,12 +46,25 @@ remove_handler() ->
     ok.
 
 post_config_update(?PROMETHEUS, _Req, New, _Old, AppEnvs) ->
-    application:set_env(AppEnvs),
-    update_prometheus(New);
+    update_prometheus(AppEnvs),
+    update_push_gateway(New);
 post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) ->
     ok.
 
-update_prometheus(#{enable := true}) ->
+update_prometheus(AppEnvs) ->
+    PrevCollectors = prometheus_registry:collectors(default),
+    CurCollectors = proplists:get_value(collectors, proplists:get_value(prometheus, AppEnvs)),
+    lists:foreach(
+        fun prometheus_registry:deregister_collector/1,
+        PrevCollectors -- CurCollectors
+    ),
+    lists:foreach(
+        fun prometheus_registry:register_collector/1,
+        CurCollectors -- PrevCollectors
+    ),
+    application:set_env(AppEnvs).
+
+update_push_gateway(#{enable := true}) ->
     emqx_prometheus_sup:start_child(?APP);
-update_prometheus(#{enable := false}) ->
+update_push_gateway(#{enable := false}) ->
     emqx_prometheus_sup:stop_child(?APP).

apps/emqx_prometheus/src/emqx_prometheus_schema.erl

@@ -99,7 +99,7 @@ fields("prometheus") ->
             ?HOCON(
                 hoconsc:enum([enabled, disabled]),
                 #{
-                    default => enabled,
+                    default => disabled,
                     required => true,
                     importance => ?IMPORTANCE_LOW,
                     desc => ?DESC(mnesia_collector)
@@ -110,7 +110,7 @@ fields("prometheus") ->
             ?HOCON(
                 hoconsc:enum([enabled, disabled]),
                 #{
-                    default => enabled,
+                    default => disabled,
                     required => true,
                     importance => ?IMPORTANCE_LOW,
                     desc => ?DESC(vm_statistics_collector)
@@ -121,7 +121,7 @@ fields("prometheus") ->
             ?HOCON(
                 hoconsc:enum([enabled, disabled]),
                 #{
-                    default => enabled,
+                    default => disabled,
                     required => true,
                     importance => ?IMPORTANCE_LOW,
                     desc => ?DESC(vm_system_info_collector)
@@ -133,7 +133,7 @@ fields("prometheus") ->
             ?HOCON(
                 hoconsc:enum([enabled, disabled]),
                 #{
-                    default => enabled,
+                    default => disabled,
                     required => true,
                     importance => ?IMPORTANCE_LOW,
                     desc => ?DESC(vm_memory_collector)
@@ -144,7 +144,7 @@ fields("prometheus") ->
             ?HOCON(
                 hoconsc:enum([enabled, disabled]),
                 #{
-                    default => enabled,
+                    default => disabled,
                     required => true,
                     importance => ?IMPORTANCE_LOW,
                     desc => ?DESC(vm_msacc_collector)
@@ -178,5 +178,5 @@ validate_push_gateway_server(Url) ->
 translation(Name) ->
     %% translate 'vm_dist_collector', 'mnesia_collector', 'vm_statistics_collector',
     %% 'vm_system_info_collector', 'vm_memory_collector', 'vm_msacc_collector'
-    %% to prometheus envrionments
+    %% to prometheus environments
     emqx_conf_schema:translation(Name).

apps/emqx_prometheus/test/emqx_prometheus_SUITE.erl

@@ -30,12 +30,12 @@
     "  headers = { Authorization = \"some-authz-tokens\"}\n"
     "  job_name = \"${name}~${host}\"\n"
     "  enable = true\n"
-    "  vm_dist_collector = enabled\n"
-    "  mnesia_collector = enabled\n"
+    "  vm_dist_collector = disabled\n"
+    "  mnesia_collector = disabled\n"
     "  vm_statistics_collector = disabled\n"
     "  vm_system_info_collector = disabled\n"
-    "  vm_memory_collector = enabled\n"
-    "  vm_msacc_collector = enabled\n"
+    "  vm_memory_collector = disabled\n"
+    "  vm_msacc_collector = disabled\n"
     "}\n"
 >>).
 

apps/emqx_prometheus/test/emqx_prometheus_api_SUITE.erl

@@ -81,13 +81,30 @@ t_prometheus_api(_) ->
     ),
     #{<<"enable">> := Enable} = Conf,
     ?assertEqual(Enable, undefined =/= erlang:whereis(emqx_prometheus)),
-    NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"disabled">>},
+    NewConf = Conf#{<<"interval">> => <<"2s">>, <<"vm_statistics_collector">> => <<"enabled">>},
     {ok, Response2} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf),
 
     Conf2 = emqx_utils_json:decode(Response2, [return_maps]),
     ?assertMatch(NewConf, Conf2),
-    ?assertEqual({ok, []}, application:get_env(prometheus, vm_statistics_collector_metrics)),
-    ?assertEqual({ok, all}, application:get_env(prometheus, vm_memory_collector_metrics)),
+
+    EnvCollectors = application:get_env(prometheus, collectors, []),
+    PromCollectors = prometheus_registry:collectors(default),
+    ?assertEqual(lists:sort(EnvCollectors), lists:sort(PromCollectors)),
+    ?assert(lists:member(prometheus_vm_statistics_collector, EnvCollectors), EnvCollectors),
+
+    lists:foreach(
+        fun({C, Enabled}) ->
+            ?assertEqual(Enabled, lists:member(C, EnvCollectors), EnvCollectors)
+        end,
+        [
+            {prometheus_vm_dist_collector, false},
+            {prometheus_vm_system_info_collector, false},
+            {prometheus_vm_memory_collector, false},
+            {prometheus_mnesia_collector, false},
+            {prometheus_vm_msacc_collector, false},
+            {prometheus_vm_statistics_collector, true}
+        ]
+    ),
 
     NewConf1 = Conf#{<<"enable">> => (not Enable)},
     {ok, _Response3} = emqx_mgmt_api_test_util:request_api(put, Path, "", Auth, NewConf1),

changes/ce/feat-11496.en.md

@@ -0,0 +1 @@
+Disabled the Erlang VM Prometheus exporter by default to improve performance and security.