fix(emqx_bridge): remove metrics from non-dedicated bridge API endpoints

Metrics should only be exposed via the /bridges/:id/metrics endpoint,
and not in other operations such as getting the list of all bridges, or
in the response when a bridge has been created. This commit removes all
traces of metrics for the non-dedicated API endpoints.

apps/emqx_bridge/include/emqx_bridge.hrl

@@ -99,13 +99,3 @@
         received := Rcvd
     }
 ).
-
--define(METRICS_EXAMPLE, #{
-    metrics => ?EMPTY_METRICS,
-    node_metrics => [
-        #{
-            node => node(),
-            metrics => ?EMPTY_METRICS
-        }
-    ]
-}).

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -161,22 +161,19 @@ param_path_enable() ->
             }
         )}.
 
-bridge_info_array_example(Method, WithMetrics) ->
-    [Config || #{value := Config} <- maps:values(bridge_info_examples(Method, WithMetrics))].
+bridge_info_array_example(Method) ->
+    lists:map(fun(#{value := Config}) -> Config end, maps:values(bridge_info_examples(Method))).
 
 bridge_info_examples(Method) ->
-    bridge_info_examples(Method, false).
-
-bridge_info_examples(Method, WithMetrics) ->
     maps:merge(
         #{
             <<"webhook_example">> => #{
                 summary => <<"WebHook">>,
-                value => info_example(webhook, Method, WithMetrics)
+                value => info_example(webhook, Method)
             },
             <<"mqtt_example">> => #{
                 summary => <<"MQTT Bridge">>,
-                value => info_example(mqtt, Method, WithMetrics)
+                value => info_example(mqtt, Method)
             }
         },
         ee_bridge_examples(Method)
@@ -189,35 +186,21 @@ ee_bridge_examples(Method) ->
 ee_bridge_examples(_Method) -> #{}.
 -endif.
 
-info_example(Type, Method, WithMetrics) ->
+info_example(Type, Method) ->
     maps:merge(
         info_example_basic(Type),
-        method_example(Type, Method, WithMetrics)
+        method_example(Type, Method)
     ).
 
-method_example(Type, Method, WithMetrics) when Method == get; Method == post ->
+method_example(Type, Method) when Method == get; Method == post ->
     SType = atom_to_list(Type),
     SName = SType ++ "_example",
-    TypeNameExam = #{
+    #{
         type => bin(SType),
         name => bin(SName)
-    },
-    maybe_with_metrics_example(TypeNameExam, Method, WithMetrics);
-method_example(_Type, put, _WithMetrics) ->
-    #{}.
-
-maybe_with_metrics_example(TypeNameExam, get, true) ->
-    TypeNameExam#{
-        metrics => ?EMPTY_METRICS,
-        node_metrics => [
-            #{
-                node => node(),
-                metrics => ?EMPTY_METRICS
-            }
-        ]
     };
-maybe_with_metrics_example(TypeNameExam, _, _) ->
-    TypeNameExam.
+method_example(_Type, put) ->
+    #{}.
 
 info_example_basic(webhook) ->
     #{
@@ -306,7 +289,7 @@ schema("/bridges") ->
             responses => #{
                 200 => emqx_dashboard_swagger:schema_with_example(
                     array(emqx_bridge_schema:get_response()),
-                    bridge_info_array_example(get, true)
+                    bridge_info_array_example(get)
                 )
             }
         },
@@ -587,7 +570,7 @@ maybe_deobfuscate_bridge_probe(Params) ->
     Params.
 
 lookup_from_all_nodes(BridgeType, BridgeName, SuccCode) ->
-    FormatFun = fun format_bridge_info_without_metrics/1,
+    FormatFun = fun format_bridge_info/1,
     do_lookup_from_all_nodes(BridgeType, BridgeName, SuccCode, FormatFun).
 
 lookup_from_all_nodes_metrics(BridgeType, BridgeName, SuccCode) ->
@@ -712,7 +695,7 @@ zip_bridges([BridgesFirstNode | _] = BridgesAllNodes) ->
     lists:foldl(
         fun(#{type := Type, name := Name}, Acc) ->
             Bridges = pick_bridges_by_id(Type, Name, BridgesAllNodes),
-            [format_bridge_info_with_metrics(Bridges) | Acc]
+            [format_bridge_info(Bridges) | Acc]
         end,
         [],
         BridgesFirstNode
@@ -746,24 +729,20 @@ pick_bridges_by_id(Type, Name, BridgesAllNodes) ->
         BridgesAllNodes
     ).
 
-format_bridge_info_with_metrics([FirstBridge | _] = Bridges) ->
-    Res = maps:remove(node, FirstBridge),
+format_bridge_info([FirstBridge | _] = Bridges) ->
+    Res = maps:without([node, metrics], FirstBridge),
     NodeStatus = collect_status(Bridges),
-    NodeMetrics = collect_metrics(Bridges),
     redact(Res#{
         status => aggregate_status(NodeStatus),
-        node_status => NodeStatus,
-        metrics => aggregate_metrics(NodeMetrics),
-        node_metrics => NodeMetrics
+        node_status => NodeStatus
     }).
 
-format_bridge_info_without_metrics(Bridges) ->
-    Res = format_bridge_info_with_metrics(Bridges),
-    maps:without([metrics, node_metrics], Res).
-
 format_bridge_metrics(Bridges) ->
-    Res = format_bridge_info_with_metrics(Bridges),
-    maps:with([metrics, node_metrics], Res).
+    NodeMetrics = collect_metrics(Bridges),
+    #{
+        metrics => aggregate_metrics(NodeMetrics),
+        node_metrics => NodeMetrics
+    }.
 
 collect_status(Bridges) ->
     [maps:with([node, status], B) || B <- Bridges].

apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl

@@ -267,8 +267,6 @@ t_http_crud_apis(Config) ->
                 <<"enable">> := true,
                 <<"status">> := _,
                 <<"node_status">> := [_ | _],
-                <<"metrics">> := _,
-                <<"node_metrics">> := [_ | _],
                 <<"url">> := URL2
             }
         ],
@@ -886,6 +884,7 @@ t_metrics(Config) ->
     ),
 
     %ct:pal("---bridge: ~p", [Bridge]),
+    Decoded = emqx_json:decode(Bridge, [return_maps]),
     #{
         <<"type">> := ?BRIDGE_TYPE,
         <<"name">> := Name,
@@ -893,7 +892,11 @@ t_metrics(Config) ->
         <<"status">> := _,
         <<"node_status">> := [_ | _],
         <<"url">> := URL1
-    } = emqx_json:decode(Bridge, [return_maps]),
+    } = Decoded,
+
+    %% assert that the bridge return doesn't contain metrics anymore
+    ?assertNot(maps:is_key(<<"metrics">>, Decoded)),
+    ?assertNot(maps:is_key(<<"node_metrics">>, Decoded)),
 
     BridgeID = emqx_bridge_resource:bridge_id(?BRIDGE_TYPE, Name),
 
@@ -909,9 +912,9 @@ t_metrics(Config) ->
 
     %% check that the bridge doesn't contain metrics anymore
     {ok, 200, Bridge2Str} = request(get, uri(["bridges", BridgeID]), []),
-    Decoded = emqx_json:decode(Bridge2Str, [return_maps]),
-    ?assertNot(maps:is_key(<<"metrics">>, Decoded)),
-    ?assertNot(maps:is_key(<<"node_metrics">>, Decoded)),
+    Decoded2 = emqx_json:decode(Bridge2Str, [return_maps]),
+    ?assertNot(maps:is_key(<<"metrics">>, Decoded2)),
+    ?assertNot(maps:is_key(<<"node_metrics">>, Decoded2)),
 
     %% send an message to emqx and the message should be forwarded to the HTTP server
     Body = <<"my msg">>,
@@ -942,16 +945,13 @@ t_metrics(Config) ->
         emqx_json:decode(Bridge3Str, [return_maps])
     ),
 
-    %% check for non-empty metrics when listing all bridges
+    %% check that metrics isn't returned when listing all bridges
     {ok, 200, BridgesStr} = request(get, uri(["bridges"]), []),
-    ?assertMatch(
-        [
-            #{
-                <<"metrics">> := #{<<"success">> := _},
-                <<"node_metrics">> := [_ | _]
-            }
-        ],
-        emqx_json:decode(BridgesStr, [return_maps])
+    ?assert(
+        lists:all(
+            fun(E) -> not maps:is_key(<<"metrics">>, E) end,
+            emqx_json:decode(BridgesStr, [return_maps])
+        )
     ),
     ok.
 

changes/ce/fix-10026.en.md

@@ -0,0 +1 @@
+Metrics are now only exposed via the /bridges/:id/metrics endpoint. Metrics are no longer returned in other API operations such as getting the list of all bridges, or in the response when a bridge has been created.

changes/ce/fix-10026.zh.md

@@ -0,0 +1 @@
+现在只有显式调用 `/bridges/:id/metrics` 接口时才可以获得指标数据，而其他 API 接口将不再返回相关数据。

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl

@@ -41,9 +41,7 @@ conn_bridge_examples(Method) ->
         }
     ].
 
-values(get, Type) ->
-    maps:merge(values(post, Type), ?METRICS_EXAMPLE);
-values(post, Type) ->
+values(_Method, Type) ->
     #{
         enable => true,
         type => Type,
@@ -65,9 +63,7 @@ values(post, Type) ->
             query_mode => async,
             max_queue_bytes => ?DEFAULT_QUEUE_SIZE
         }
-    };
-values(put, Type) ->
-    values(post, Type).
+    }.
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl

@@ -37,9 +37,7 @@ conn_bridge_examples(Method) ->
         }
     ].
 
-values(get) ->
-    maps:merge(values(post), ?METRICS_EXAMPLE);
-values(post) ->
+values(_Method) ->
     #{
         enable => true,
         type => dynamo,
@@ -60,9 +58,7 @@ values(post) ->
             query_mode => sync,
             max_queue_bytes => ?DEFAULT_QUEUE_SIZE
         }
-    };
-values(put) ->
-    values(post).
+    }.
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_gcp_pubsub.erl

@@ -4,7 +4,6 @@
 
 -module(emqx_ee_bridge_gcp_pubsub).
 
--include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
@@ -146,9 +145,7 @@ conn_bridge_examples(Method) ->
         }
     ].
 
-values(get) ->
-    maps:merge(values(post), ?METRICS_EXAMPLE);
-values(post) ->
+values(_Method) ->
     #{
         pubsub_topic => <<"mytopic">>,
         service_account_json =>
@@ -176,9 +173,7 @@ values(post) ->
                     <<"https://oauth2.googleapis.com/token">>,
                 type => <<"service_account">>
             }
-    };
-values(put) ->
-    values(post).
+    }.
 
 %%-------------------------------------------------------------------------------------------------
 %% Helper fns

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_hstreamdb.erl

@@ -5,7 +5,6 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include_lib("emqx_bridge/include/emqx_bridge.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
@@ -33,9 +32,7 @@ conn_bridge_examples(Method) ->
         }
     ].
 
-values(get) ->
-    maps:merge(values(post), ?METRICS_EXAMPLE);
-values(post) ->
+values(_Method) ->
     #{
         type => hstreamdb,
         name => <<"demo">>,
@@ -44,9 +41,7 @@ values(post) ->
         direction => egress,
         local_topic => <<"local/topic/#">>,
         payload => <<"${payload}">>
-    };
-values(put) ->
-    values(post).
+    }.
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_influxdb.erl

@@ -3,7 +3,6 @@
 %%--------------------------------------------------------------------
 -module(emqx_ee_bridge_influxdb).
 
--include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
@@ -46,7 +45,7 @@ conn_bridge_examples(Method) ->
     ].
 
 values(Protocol, get) ->
-    maps:merge(values(Protocol, post), ?METRICS_EXAMPLE);
+    values(Protocol, post);
 values("influxdb_api_v2", post) ->
     SupportUint = <<"uint_value=${payload.uint_key}u,">>,
     TypeOpts = #{

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_kafka.erl

@@ -3,7 +3,6 @@
 %%--------------------------------------------------------------------
 -module(emqx_ee_bridge_kafka).
 
--include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
@@ -55,7 +54,7 @@ conn_bridge_examples(Method) ->
     ].
 
 values({get, KafkaType}) ->
-    maps:merge(values({post, KafkaType}), ?METRICS_EXAMPLE);
+    values({post, KafkaType});
 values({post, KafkaType}) ->
     maps:merge(values(common_config), values(KafkaType));
 values({put, KafkaType}) ->

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl

@@ -5,7 +5,6 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include_lib("emqx_bridge/include/emqx_bridge.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
@@ -156,9 +155,6 @@ values(common, MongoType, Method, TypeOpts) ->
     Vals0 = maps:merge(MethodVals, Common),
     maps:merge(Vals0, TypeOpts).
 
-method_values(MongoType, get) ->
-    Vals = method_values(MongoType, post),
-    maps:merge(?METRICS_EXAMPLE, Vals);
 method_values(MongoType, _) ->
     ConnectorType =
         case MongoType of

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl

@@ -5,7 +5,6 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
@@ -39,9 +38,7 @@ conn_bridge_examples(Method) ->
         }
     ].
 
-values(get) ->
-    maps:merge(values(post), ?METRICS_EXAMPLE);
-values(post) ->
+values(_Method) ->
     #{
         enable => true,
         type => mysql,
@@ -62,9 +59,7 @@ values(post) ->
             query_mode => async,
             max_queue_bytes => ?DEFAULT_QUEUE_SIZE
         }
-    };
-values(put) ->
-    values(post).
+    }.
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_pgsql.erl

@@ -5,7 +5,6 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
@@ -41,9 +40,7 @@ conn_bridge_examples(Method) ->
         }
     ].
 
-values(get, Type) ->
-    maps:merge(values(post, Type), ?METRICS_EXAMPLE);
-values(post, Type) ->
+values(_Method, Type) ->
     #{
         enable => true,
         type => Type,
@@ -64,9 +61,7 @@ values(post, Type) ->
             query_mode => async,
             max_queue_bytes => ?DEFAULT_QUEUE_SIZE
         }
-    };
-values(put, Type) ->
-    values(post, Type).
+    }.
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_redis.erl

@@ -3,7 +3,6 @@
 %%--------------------------------------------------------------------
 -module(emqx_ee_bridge_redis).
 
--include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 
@@ -46,7 +45,7 @@ conn_bridge_examples(Method) ->
     ].
 
 values(Protocol, get) ->
-    maps:merge(values(Protocol, post), ?METRICS_EXAMPLE);
+    values(Protocol, post);
 values("single", post) ->
     SpecificOpts = #{
         server => <<"127.0.0.1:6379">>,

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_tdengine.erl

@@ -5,7 +5,6 @@
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
--include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").
 
 -import(hoconsc, [mk/2, enum/1, ref/2]).
@@ -40,9 +39,7 @@ conn_bridge_examples(Method) ->
         }
     ].
 
-values(get) ->
-    maps:merge(values(post), ?METRICS_EXAMPLE);
-values(post) ->
+values(_Method) ->
     #{
         enable => true,
         type => tdengine,
@@ -63,9 +60,7 @@ values(post) ->
             query_mode => sync,
             max_queue_bytes => ?DEFAULT_QUEUE_SIZE
         }
-    };
-values(put) ->
-    values(post).
+    }.
 
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_redis_SUITE.erl

@@ -210,8 +210,7 @@ t_check_values(_Config) ->
     lists:foreach(
         fun(Method) ->
             lists:foreach(
-                fun({RedisType, #{value := Value0}}) ->
-                    Value = maps:without(maps:keys(?METRICS_EXAMPLE), Value0),
+                fun({RedisType, #{value := Value}}) ->
                     MethodBin = atom_to_binary(Method),
                     Type = string:slice(RedisType, length("redis_")),
                     RefName = binary_to_list(<<MethodBin/binary, "_", Type/binary>>),