Merge pull request #11409 from zhongwencool/opentelemetry-metrics

feat: support opentelemetry metrics

apps/emqx/src/emqx_stats.erl

@@ -37,7 +37,8 @@
     setstat/2,
     setstat/3,
     statsfun/1,
-    statsfun/2
+    statsfun/2,
+    names/0
 ]).
 
 -export([
@@ -157,6 +158,28 @@ getstats() ->
         _ -> ets:tab2list(?TAB)
     end.
 
+names() ->
+    [
+        emqx_connections_count,
+        emqx_connections_max,
+        emqx_live_connections_count,
+        emqx_live_connections_max,
+        emqx_sessions_count,
+        emqx_sessions_max,
+        emqx_topics_count,
+        emqx_topics_max,
+        emqx_suboptions_count,
+        emqx_suboptions_max,
+        emqx_subscribers_count,
+        emqx_subscribers_max,
+        emqx_subscriptions_count,
+        emqx_subscriptions_max,
+        emqx_subscriptions_shared_count,
+        emqx_subscriptions_shared_max,
+        emqx_retained_count,
+        emqx_retained_max
+    ].
+
 %% @doc Get stats by name.
 -spec getstat(atom()) -> non_neg_integer().
 getstat(Name) ->

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -63,6 +63,7 @@
     emqx_psk_schema,
     emqx_limiter_schema,
     emqx_slow_subs_schema,
+    emqx_otel_schema,
     emqx_mgmt_api_key_schema
 ]).
 %% 1 million default ports counter

apps/emqx_machine/priv/reboot_lists.eterm

@@ -24,7 +24,12 @@
             redbug,
             xmerl,
             {hocon, load},
-            telemetry
+            telemetry,
+            {opentelemetry, load},
+            {opentelemetry_api, load},
+            {opentelemetry_experimental, load},
+            {opentelemetry_api_experimental, load},
+            {opentelemetry_exporter, load}
         ],
     %% must always be of type `load'
     common_business_apps =>
@@ -68,6 +73,7 @@
             emqx_redis,
             emqx_mysql,
             emqx_plugins,
+            emqx_opentelemetry,
             quicer,
             bcrypt,
             jq,

apps/emqx_management/src/emqx_management.app.src

@@ -2,7 +2,7 @@
 {application, emqx_management, [
     {description, "EMQX Management API and CLI"},
     % strict semver, bump manually!
-    {vsn, "5.0.26"},
+    {vsn, "5.0.27"},
     {modules, []},
     {registered, [emqx_management_sup]},
     {applications, [kernel, stdlib, emqx_plugins, minirest, emqx, emqx_ctl, emqx_bridge_http]},

apps/emqx_management/src/emqx_mgmt.erl

@@ -107,7 +107,8 @@
 %% Common Table API
 -export([
     default_row_limit/0,
-    vm_stats/0
+    vm_stats/0,
+    vm_stats/1
 ]).
 
 -elvis([{elvis_style, god_modules, disable}]).
@@ -185,22 +186,33 @@ stopped_node_info(Node) ->
     {Node, #{node => Node, node_status => 'stopped', role => core}}.
 
 vm_stats() ->
-    Idle =
-        case cpu_sup:util([detailed]) of
-            %% Not support for Windows
-            {_, 0, 0, _} -> 0;
-            {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0)
-        end,
-    RunQueue = erlang:statistics(run_queue),
+    Idle = vm_stats('cpu.idle'),
     {MemUsedRatio, MemTotal} = get_sys_memory(),
     [
-        {run_queue, RunQueue},
+        {run_queue, vm_stats('run.queue')},
         {cpu_idle, Idle},
         {cpu_use, 100 - Idle},
         {total_memory, MemTotal},
         {used_memory, erlang:round(MemTotal * MemUsedRatio)}
     ].
 
+vm_stats('cpu.idle') ->
+    case cpu_sup:util([detailed]) of
+        %% Not support for Windows
+        {_, 0, 0, _} -> 0;
+        {_Num, _Use, IdleList, _} -> proplists:get_value(idle, IdleList, 0)
+    end;
+vm_stats('cpu.use') ->
+    100 - vm_stats('cpu.idle');
+vm_stats('total.memory') ->
+    {_, MemTotal} = get_sys_memory(),
+    MemTotal;
+vm_stats('used.memory') ->
+    {MemUsedRatio, MemTotal} = get_sys_memory(),
+    erlang:round(MemTotal * MemUsedRatio);
+vm_stats('run.queue') ->
+    erlang:statistics(run_queue).
+
 %%--------------------------------------------------------------------
 %% Brokers
 %%--------------------------------------------------------------------

apps/emqx_opentelemetry/README.md

@@ -0,0 +1,4 @@
+emqx_opentelemetry
+=====
+
+OpenTelemetry metric log trace framework for EMQX.

apps/emqx_opentelemetry/rebar.config

@@ -0,0 +1,29 @@
+%% -*- mode: erlang -*-
+
+{deps, [
+    {emqx, {path, "../emqx"}}
+]}.
+
+{edoc_opts, [{preprocess, true}]}.
+{erl_opts, [
+    warn_unused_vars,
+    warn_shadow_vars,
+    warn_unused_import,
+    warn_obsolete_guard,
+    debug_info,
+    {parse_transform}
+]}.
+
+{xref_checks, [
+    undefined_function_calls,
+    undefined_functions,
+    locals_not_used,
+    deprecated_function_calls,
+    warnings_as_errors,
+    deprecated_functions
+]}.
+{cover_enabled, true}.
+{cover_opts, [verbose]}.
+{cover_export_enabled, true}.
+
+{project_plugins, [erlfmt]}.

apps/emqx_opentelemetry/src/emqx_opentelemetry.app.src

@@ -0,0 +1,15 @@
+{application, emqx_opentelemetry, [
+    {description, "OpenTelemetry for EMQX Broker"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {mod, {emqx_otel_app, []}},
+    {applications, [kernel, stdlib, emqx]},
+    {env, []},
+    {modules, []},
+    {licenses, ["Apache 2.0"]},
+    {maintainers, ["EMQX Team <contact@emqx.io>"]},
+    {links, [
+        {"Homepage", "https://emqx.io/"},
+        {"Github", "https://github.com/emqx/emqx"}
+    ]}
+]}.

apps/emqx_opentelemetry/src/emqx_otel.erl

@@ -0,0 +1,207 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_otel).
+-include_lib("emqx/include/logger.hrl").
+
+-export([start_link/1]).
+-export([get_cluster_gauge/1, get_stats_gauge/1, get_vm_gauge/1, get_metric_counter/1]).
+-export([init/1, handle_continue/2, handle_call/3, handle_cast/2, handle_info/2, terminate/2]).
+
+start_link(Conf) ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, Conf, []).
+
+init(Conf) ->
+    erlang:process_flag(trap_exit, true),
+    {ok, #{}, {continue, {setup, Conf}}}.
+
+handle_continue({setup, Conf}, State) ->
+    setup(Conf),
+    {noreply, State, hibernate}.
+
+handle_call(_Msg, _From, State) ->
+    {reply, ok, State}.
+
+handle_cast(_Msg, State) ->
+    {noreply, State}.
+
+handle_info(_Msg, State) ->
+    {noreply, State}.
+
+terminate(_Reason, _State) ->
+    cleanup(),
+    ok.
+
+setup(Conf = #{enable := true}) ->
+    ensure_apps(Conf),
+    create_metric_views();
+setup(_Conf) ->
+    cleanup(),
+    ok.
+
+ensure_apps(Conf) ->
+    #{exporter := #{interval := ExporterInterval}} = Conf,
+    {ok, _} = application:ensure_all_started(opentelemetry_exporter),
+    _ = application:stop(opentelemetry_experimental),
+    ok = application:set_env(
+        opentelemetry_experimental,
+        readers,
+        [
+            #{
+                module => otel_metric_reader,
+                config => #{
+                    exporter => {opentelemetry_exporter, #{}},
+                    export_interval_ms => ExporterInterval
+                }
+            }
+        ]
+    ),
+    {ok, _} = application:ensure_all_started(opentelemetry_experimental),
+    {ok, _} = application:ensure_all_started(opentelemetry_api_experimental),
+    ok.
+
+cleanup() ->
+    _ = application:stop(opentelemetry_experimental),
+    _ = application:stop(opentelemetry_experimental_api),
+    _ = application:stop(opentelemetry_exporter),
+    ok.
+
+create_metric_views() ->
+    Meter = opentelemetry_experimental:get_meter(),
+    StatsGauge = emqx_stats:getstats(),
+    create_gauge(Meter, StatsGauge, fun ?MODULE:get_stats_gauge/1),
+    VmGauge = lists:map(fun({K, V}) -> {normalize_name(K), V} end, emqx_mgmt:vm_stats()),
+    create_gauge(Meter, VmGauge, fun ?MODULE:get_vm_gauge/1),
+    ClusterGauge = [{'node.running', 0}, {'node.stopped', 0}],
+    create_gauge(Meter, ClusterGauge, fun ?MODULE:get_cluster_gauge/1),
+    Metrics = lists:map(fun({K, V}) -> {K, V, unit(K)} end, emqx_metrics:all()),
+    create_counter(Meter, Metrics, fun ?MODULE:get_metric_counter/1),
+    ok.
+
+unit(K) ->
+    case lists:member(K, bytes_metrics()) of
+        true -> kb;
+        false -> '1'
+    end.
+
+bytes_metrics() ->
+    [
+        'bytes.received',
+        'bytes.sent',
+        'packets.received',
+        'packets.sent',
+        'packets.connect',
+        'packets.connack.sent',
+        'packets.connack.error',
+        'packets.connack.auth_error',
+        'packets.publish.received',
+        'packets.publish.sent',
+        'packets.publish.inuse',
+        'packets.publish.error',
+        'packets.publish.auth_error',
+        'packets.publish.dropped',
+        'packets.puback.received',
+        'packets.puback.sent',
+        'packets.puback.inuse',
+        'packets.puback.missed',
+        'packets.pubrec.received',
+        'packets.pubrec.sent',
+        'packets.pubrec.inuse',
+        'packets.pubrec.missed',
+        'packets.pubrel.received',
+        'packets.pubrel.sent',
+        'packets.pubrel.missed',
+        'packets.pubcomp.received',
+        'packets.pubcomp.sent',
+        'packets.pubcomp.inuse',
+        'packets.pubcomp.missed',
+        'packets.subscribe.received',
+        'packets.subscribe.error',
+        'packets.subscribe.auth_error',
+        'packets.suback.sent',
+        'packets.unsubscribe.received',
+        'packets.unsubscribe.error',
+        'packets.unsuback.sent',
+        'packets.pingreq.received',
+        'packets.pingresp.sent',
+        'packets.disconnect.received',
+        'packets.disconnect.sent',
+        'packets.auth.received',
+        'packets.auth.sent'
+    ].
+
+get_stats_gauge(Name) ->
+    [{emqx_stats:getstat(Name), #{}}].
+
+get_vm_gauge(Name) ->
+    [{emqx_mgmt:vm_stats(Name), #{}}].
+
+get_cluster_gauge('node.running') ->
+    length(emqx:cluster_nodes(running));
+get_cluster_gauge('node.stopped') ->
+    length(emqx:cluster_nodes(stopped)).
+
+get_metric_counter(Name) ->
+    [{emqx_metrics:val(Name), #{}}].
+
+create_gauge(Meter, Names, CallBack) ->
+    lists:foreach(
+        fun({Name, _}) ->
+            true = otel_meter_server:add_view(
+                #{instrument_name => Name},
+                #{aggregation_module => otel_aggregation_last_value}
+            ),
+            otel_meter:create_observable_gauge(
+                Meter,
+                Name,
+                CallBack,
+                Name,
+                #{
+                    description => iolist_to_binary([
+                        <<"observable ">>, atom_to_binary(Name), <<" gauge">>
+                    ]),
+                    unit => '1'
+                }
+            )
+        end,
+        Names
+    ).
+
+create_counter(Meter, Counters, CallBack) ->
+    lists:foreach(
+        fun({Name, _, Unit}) ->
+            true = otel_meter_server:add_view(
+                #{instrument_name => Name},
+                #{aggregation_module => otel_aggregation_sum}
+            ),
+            otel_meter:create_observable_counter(
+                Meter,
+                Name,
+                CallBack,
+                Name,
+                #{
+                    description => iolist_to_binary([
+                        <<"observable ">>, atom_to_binary(Name), <<" counter">>
+                    ]),
+                    unit => Unit
+                }
+            )
+        end,
+        Counters
+    ).
+
+normalize_name(Name) ->
+    list_to_existing_atom(lists:flatten(string:replace(atom_to_list(Name), "_", ".", all))).

apps/emqx_opentelemetry/src/emqx_otel_api.erl

@@ -0,0 +1,112 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_otel_api).
+
+-behaviour(minirest_api).
+
+-include_lib("hocon/include/hoconsc.hrl").
+-include_lib("emqx/include/http_api.hrl").
+
+-import(hoconsc, [ref/2]).
+
+-export([
+    api_spec/0,
+    paths/0,
+    schema/1
+]).
+
+-export([config/2]).
+
+-define(TAGS, [<<"Monitor">>]).
+
+api_spec() ->
+    emqx_dashboard_swagger:spec(?MODULE, #{check_schema => true}).
+
+paths() ->
+    [
+        "/opentelemetry"
+    ].
+
+schema("/opentelemetry") ->
+    #{
+        'operationId' => config,
+        get =>
+            #{
+                description => "Get opentelmetry configuration",
+                tags => ?TAGS,
+                responses =>
+                    #{200 => otel_config_schema()}
+            },
+        put =>
+            #{
+                description => "Update opentelmetry configuration",
+                tags => ?TAGS,
+                'requestBody' => otel_config_schema(),
+                responses =>
+                    #{
+                        200 => otel_config_schema(),
+                        400 =>
+                            emqx_dashboard_swagger:error_codes(
+                                [?BAD_REQUEST], <<"Update Config Failed">>
+                            )
+                    }
+            }
+    }.
+
+%%--------------------------------------------------------------------
+%% API Handler funcs
+%%--------------------------------------------------------------------
+
+config(get, _Params) ->
+    {200, get_raw()};
+config(put, #{body := Body}) ->
+    case emqx_otel_config:update(Body) of
+        {ok, NewConfig} ->
+            {200, NewConfig};
+        {error, Reason} ->
+            Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])),
+            {400, ?BAD_REQUEST, Message}
+    end.
+
+%%--------------------------------------------------------------------
+%% Internal funcs
+%%--------------------------------------------------------------------
+
+get_raw() ->
+    Path = <<"opentelemetry">>,
+    #{Path := Conf} =
+        emqx_config:fill_defaults(
+            #{Path => emqx_conf:get_raw([Path])},
+            #{obfuscate_sensitive_values => true}
+        ),
+    Conf.
+
+otel_config_schema() ->
+    emqx_dashboard_swagger:schema_with_example(
+        ref(emqx_otel_schema, "opentelemetry"),
+        otel_config_example()
+    ).
+
+otel_config_example() ->
+    #{
+        enable => true,
+        exporter =>
+            #{
+                endpoint => "http://localhost:4317",
+                interval => "10s"
+            }
+    }.

apps/emqx_opentelemetry/src/emqx_otel_app.erl

@@ -0,0 +1,29 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_otel_app).
+
+-behaviour(application).
+
+-export([start/2, stop/1]).
+
+start(_StartType, _StartArgs) ->
+    emqx_otel_config:add_handler(),
+    emqx_otel_sup:start_link().
+
+stop(_State) ->
+    emqx_otel_config:remove_handler(),
+    ok.

apps/emqx_opentelemetry/src/emqx_otel_config.erl

@@ -0,0 +1,58 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_otel_config).
+
+-behaviour(emqx_config_handler).
+
+-define(OPTL, [opentelemetry]).
+
+-export([add_handler/0, remove_handler/0]).
+-export([post_config_update/5]).
+-export([update/1]).
+
+update(Config) ->
+    case
+        emqx_conf:update(
+            ?OPTL,
+            Config,
+            #{rawconf_with_defaults => true, override_to => cluster}
+        )
+    of
+        {ok, #{raw_config := NewConfigRows}} ->
+            {ok, NewConfigRows};
+        {error, Reason} ->
+            {error, Reason}
+    end.
+
+add_handler() ->
+    ok = emqx_config_handler:add_handler(?OPTL, ?MODULE),
+    ok.
+
+remove_handler() ->
+    ok = emqx_config_handler:remove_handler(?OPTL),
+    ok.
+
+post_config_update(?OPTL, _Req, New, _Old, AppEnvs) ->
+    application:set_env(AppEnvs),
+    ensure_otel(New);
+post_config_update(_ConfPath, _Req, _NewConf, _OldConf, _AppEnvs) ->
+    ok.
+
+ensure_otel(#{enable := true} = Conf) ->
+    _ = emqx_otel_sup:stop_otel(),
+    emqx_otel_sup:start_otel(Conf);
+ensure_otel(#{enable := false}) ->
+    emqx_otel_sup:stop_otel().

apps/emqx_opentelemetry/src/emqx_otel_schema.erl

@@ -0,0 +1,82 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_otel_schema).
+
+-include_lib("hocon/include/hoconsc.hrl").
+
+-export([
+    roots/0,
+    fields/1,
+    namespace/0,
+    desc/1
+]).
+
+namespace() -> opentelemetry.
+roots() -> ["opentelemetry"].
+
+fields("opentelemetry") ->
+    [
+        {exporter,
+            ?HOCON(
+                ?R_REF("exporter"),
+                #{desc => ?DESC(exporter)}
+            )},
+        {enable,
+            ?HOCON(
+                boolean(),
+                #{
+                    default => false,
+                    required => true,
+                    desc => ?DESC(enable)
+                }
+            )}
+    ];
+fields("exporter") ->
+    [
+        {"protocol",
+            ?HOCON(
+                %% http_protobuf is not support for metrics yet.
+                ?ENUM([grpc]),
+                #{
+                    mapping => "opentelemetry_exporter.otlp_protocol",
+                    desc => ?DESC(protocol),
+                    default => grpc,
+                    importance => ?IMPORTANCE_HIDDEN
+                }
+            )},
+        {"endpoint",
+            ?HOCON(
+                emqx_schema:url(),
+                #{
+                    mapping => "opentelemetry_exporter.otlp_endpoint",
+                    default => "http://localhost:4317",
+                    desc => ?DESC(endpoint)
+                }
+            )},
+        {"interval",
+            ?HOCON(
+                emqx_schema:timeout_duration_ms(),
+                #{
+                    default => <<"10s">>,
+                    required => true,
+                    desc => ?DESC(interval)
+                }
+            )}
+    ].
+
+desc("opentelemetry") -> ?DESC(opentelemetry);
+desc("exporter") -> ?DESC(exporter);
+desc(_) -> undefined.

apps/emqx_opentelemetry/src/emqx_otel_sup.erl

@@ -0,0 +1,67 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_otel_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/0]).
+-export([init/1]).
+-export([start_otel/1]).
+-export([stop_otel/0]).
+
+-define(CHILD(Mod, Opts), #{
+    id => Mod,
+    start => {Mod, start_link, [Opts]},
+    restart => permanent,
+    shutdown => 5000,
+    type => worker,
+    modules => [Mod]
+}).
+
+-define(WORKER, emqx_otel).
+
+start_link() ->
+    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+-spec start_otel(map()) -> ok.
+start_otel(Conf) ->
+    assert_started(supervisor:start_child(?MODULE, ?CHILD(?WORKER, Conf))).
+
+-spec stop_otel() -> ok | {error, term()}.
+stop_otel() ->
+    case supervisor:terminate_child(?MODULE, ?WORKER) of
+        ok -> supervisor:delete_child(?MODULE, ?WORKER);
+        {error, not_found} -> ok;
+        Error -> Error
+    end.
+
+init([]) ->
+    SupFlags = #{
+        strategy => one_for_one,
+        intensity => 10,
+        period => 512
+    },
+    Children =
+        case emqx_conf:get([opentelemetry]) of
+            #{enable := false} -> [];
+            #{enable := true} = Conf -> [?CHILD(?WORKER, Conf)]
+        end,
+    {ok, {SupFlags, Children}}.
+
+assert_started({ok, _Pid}) -> ok;
+assert_started({ok, _Pid, _Info}) -> ok;
+assert_started({error, {already_started, _Pid}}) -> ok;
+assert_started({error, Reason}) -> {error, Reason}.

apps/emqx_prometheus/src/emqx_prometheus.app.src

@@ -2,7 +2,7 @@
 {application, emqx_prometheus, [
     {description, "Prometheus for EMQX"},
     % strict semver, bump manually!
-    {vsn, "5.0.14"},
+    {vsn, "5.0.15"},
     {modules, []},
     {registered, [emqx_prometheus_sup]},
     {applications, [kernel, stdlib, prometheus, emqx, emqx_management]},

apps/emqx_prometheus/src/emqx_prometheus.erl

@@ -160,7 +160,7 @@ collect_mf(_Registry, Callback) ->
     Stats = emqx_stats:getstats(),
     VMData = emqx_vm_data(),
     ClusterData = emqx_cluster_data(),
-    _ = [add_collect_family(Name, Stats, Callback, gauge) || Name <- emqx_stats()],
+    _ = [add_collect_family(Name, Stats, Callback, gauge) || Name <- emqx_stats:names()],
     _ = [add_collect_family(Name, VMData, Callback, gauge) || Name <- emqx_vm()],
     _ = [add_collect_family(Name, ClusterData, Callback, gauge) || Name <- emqx_cluster()],
     _ = [add_collect_family(Name, Metrics, Callback, counter) || Name <- emqx_metrics_packets()],
@@ -176,7 +176,7 @@ collect(<<"json">>) ->
     Stats = emqx_stats:getstats(),
     VMData = emqx_vm_data(),
     #{
-        stats => maps:from_list([collect_stats(Name, Stats) || Name <- emqx_stats()]),
+        stats => maps:from_list([collect_stats(Name, Stats) || Name <- emqx_stats:names()]),
         metrics => maps:from_list([collect_stats(Name, VMData) || Name <- emqx_vm()]),
         packets => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_packets()]),
         messages => maps:from_list([collect_stats(Name, Metrics) || Name <- emqx_metrics_messages()]),
@@ -460,28 +460,6 @@ emqx_collect(emqx_cluster_nodes_stopped, ClusterData) ->
 %% Indicators
 %%--------------------------------------------------------------------
 
-emqx_stats() ->
-    [
-        emqx_connections_count,
-        emqx_connections_max,
-        emqx_live_connections_count,
-        emqx_live_connections_max,
-        emqx_sessions_count,
-        emqx_sessions_max,
-        emqx_topics_count,
-        emqx_topics_max,
-        emqx_suboptions_count,
-        emqx_suboptions_max,
-        emqx_subscribers_count,
-        emqx_subscribers_max,
-        emqx_subscriptions_count,
-        emqx_subscriptions_max,
-        emqx_subscriptions_shared_count,
-        emqx_subscriptions_shared_max,
-        emqx_retained_count,
-        emqx_retained_max
-    ].
-
 emqx_metrics_packets() ->
     [
         emqx_bytes_received,

mix.exs

@@ -98,7 +98,32 @@ defmodule EMQXUmbrella.MixProject do
       # set by hackney (dependency)
       {:ssl_verify_fun, "1.1.6", override: true},
       {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true},
-      {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true}
+      {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true},
+      {:opentelemetry_api,
+       github: "emqx/opentelemetry-erlang",
+       sparse: "apps/opentelemetry_api",
+       override: true,
+       runtime: false},
+      {:opentelemetry,
+       github: "emqx/opentelemetry-erlang",
+       sparse: "apps/opentelemetry",
+       override: true,
+       runtime: false},
+      {:opentelemetry_api_experimental,
+       github: "emqx/opentelemetry-erlang",
+       sparse: "apps/opentelemetry_api_experimental",
+       override: true,
+       runtime: false},
+      {:opentelemetry_experimental,
+       github: "emqx/opentelemetry-erlang",
+       sparse: "apps/opentelemetry_experimental",
+       override: true,
+       runtime: false},
+      {:opentelemetry_exporter,
+       github: "emqx/opentelemetry-erlang",
+       sparse: "apps/opentelemetry_exporter",
+       override: true,
+       runtime: false}
     ] ++
       emqx_apps(profile_info, version) ++
       enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
@@ -324,6 +349,7 @@ defmodule EMQXUmbrella.MixProject do
             :emqx_plugins,
             :emqx_ft,
             :emqx_s3,
+            :emqx_opentelemetry,
             :emqx_durable_storage,
             :rabbit_common
           ],

rebar.config

@@ -84,6 +84,14 @@
     %% in conflict by erlavro and rocketmq
     , {jsone, {git, "https://github.com/emqx/jsone.git", {tag, "1.7.1"}}}
     , {uuid, {git, "https://github.com/okeuday/uuid.git", {tag, "v2.0.6"}}}
+%% trace
+      , {opentelemetry_api, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_api"}}
+      , {opentelemetry, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry"}}
+      %% log metrics
+      , {opentelemetry_experimental, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_experimental"}}
+      , {opentelemetry_api_experimental, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_api_experimental"}}
+      %% export
+      , {opentelemetry_exporter, {git_subdir, "http://github.com/emqx/opentelemetry-erlang", {branch, "main"}, "apps/opentelemetry_exporter"}}
     ]}.
 
 {xref_ignores,

rel/i18n/emqx_otel_schema.hocon

@@ -0,0 +1,15 @@
+emqx_otel_schema {
+
+opentelemetry.desc: "Open Telemetry Toolkit configuration"
+
+exporter.desc: "Open Telemetry Exporter"
+
+enable.desc: "Enable or disable open telemetry metrics"
+
+protocol.desc: "Open Telemetry Exporter Protocol"
+
+endpoint.desc: "Open Telemetry Exporter Endpoint"
+
+interval.desc: "The interval of sending metrics to Open Telemetry Endpoint"
+
+}