Merge pull request #6608 from emqx/merge-5.0-beta.3-to-master

Merge 5.0 beta.3 to master

.github/workflows/build_slim_packages.yaml

@@ -74,7 +74,7 @@ jobs:
         - macos-11
         - macos-10.15
 
-    runs-on: ${{  matrix.macos }}
+    runs-on: ${{ matrix.macos }}
 
     steps:
     - uses: actions/checkout@v2

.github/workflows/run_api_tests.yaml

@@ -61,7 +61,7 @@ jobs:
     - uses: actions/checkout@v2
       with:
         repository: emqx/emqx-fvt
-        ref: 1.0.2-dev1
+        ref: 1.0.3-dev1
         path: .
     - uses: actions/setup-java@v1
       with:
@@ -93,7 +93,7 @@ jobs:
       run: |
         /opt/jmeter/bin/jmeter.sh \
           -Jjmeter.save.saveservice.output_format=xml -n \
-          -t .ci/api-test-suite/${{ matrix.script_name }}.jmx \
+          -t api-test-suite/${{ matrix.script_name }}.jmx \
           -Demqx_ip="127.0.0.1" \
           -l jmeter_logs/${{ matrix.script_name }}.jtl \
           -j jmeter_logs/logs/${{ matrix.script_name }}.log

Makefile

@@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/4.4-2:23.3.4.9-3-alpine3
 export EMQX_DEFAULT_RUNNER = alpine:3.14
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v0.10.0
+export EMQX_DASHBOARD_VERSION ?= v0.14.0
 export DOCKERFILE := deploy/docker/Dockerfile
 export DOCKERFILE_TESTING := deploy/docker/Dockerfile.testing
 ifeq ($(OS),Windows_NT)

apps/emqx/include/logger.hrl

@@ -59,15 +59,32 @@
 
 %% structured logging
 -define(SLOG(Level, Data),
-        %% check 'allow' here, only evaluate Data when necessary
-        case logger:allow(Level, ?MODULE) of
-            true ->
-                logger:log(Level, (Data), #{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
-                                           , line => ?LINE
-                                           });
-            false ->
-                ok
-        end).
+        ?SLOG(Level, Data, #{})).
+
+%% structured logging, meta is for handler's filter.
+-define(SLOG(Level, Data, Meta),
+%% check 'allow' here, only evaluate Data and Meta when necessary
+    case logger:allow(Level, ?MODULE) of
+        true ->
+            logger:log(Level, (Data), (Meta#{ mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}
+                                            , line => ?LINE
+            }));
+        false ->
+            ok
+    end).
+
+-define(TRACE_FILTER, emqx_trace_filter).
+
+%% Only evaluate when necessary
+-define(TRACE(Event, Msg, Meta),
+    begin
+    case persistent_term:get(?TRACE_FILTER, undefined) of
+        undefined -> ok;
+        [] -> ok;
+        List ->
+           emqx_trace:log(List, Event, Msg, Meta)
+    end
+    end).
 
 %% print to 'user' group leader
 -define(ULOG(Fmt, Args), io:format(user, Fmt, Args)).

apps/emqx/rebar.config

@@ -11,7 +11,7 @@
 {deps,
     [ {lc, {git, "https://github.com/qzhuyan/lc.git", {tag, "0.1.2"}}}
     , {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
-    , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}}
+    , {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.6"}}}
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.0"}}}

apps/emqx/src/emqx_authentication_config.erl

@@ -187,7 +187,7 @@ convert_certs(CertsDir, Config) ->
         {ok, SSL} ->
             new_ssl_config(Config, SSL);
         {error, Reason} ->
-            ?SLOG(error, Reason#{msg => bad_ssl_config}),
+            ?SLOG(error, Reason#{msg => "bad_ssl_config"}),
             throw({bad_ssl_config, Reason})
     end.
 
@@ -199,7 +199,7 @@ convert_certs(CertsDir, NewConfig, OldConfig) ->
             ok = emqx_tls_lib:delete_ssl_files(CertsDir, NewSSL1, OldSSL),
             new_ssl_config(NewConfig, NewSSL1);
         {error, Reason} ->
-            ?SLOG(error, Reason#{msg => bad_ssl_config}),
+            ?SLOG(error, Reason#{msg => "bad_ssl_config"}),
             throw({bad_ssl_config, Reason})
     end.
 

apps/emqx/src/emqx_banned.erl

@@ -37,7 +37,6 @@
         , info/1
         , format/1
         , parse/1
-        , to_timestamp/1
         ]).
 
 %% gen_server callbacks
@@ -53,6 +52,11 @@
 
 -define(BANNED_TAB, ?MODULE).
 
+-ifdef(TEST).
+-compile(export_all).
+-compile(nowarn_export_all).
+-endif.
+
 %%--------------------------------------------------------------------
 %% Mnesia bootstrap
 %%--------------------------------------------------------------------
@@ -106,32 +110,36 @@ format(#banned{who = Who0,
     }.
 
 parse(Params) ->
-    Who    = pares_who(Params),
-    By     = maps:get(<<"by">>, Params, <<"mgmt_api">>),
-    Reason = maps:get(<<"reason">>, Params, <<"">>),
-    At     = parse_time(maps:get(<<"at">>, Params, undefined), erlang:system_time(second)),
-    Until  = parse_time(maps:get(<<"until">>, Params, undefined), At + 5 * 60),
-    #banned{
-        who    = Who,
-        by     = By,
-        reason = Reason,
-        at     = At,
-        until  = Until
-    }.
-
+    case pares_who(Params) of
+        {error, Reason} -> {error, Reason};
+        Who  ->
+            By     = maps:get(<<"by">>, Params, <<"mgmt_api">>),
+            Reason = maps:get(<<"reason">>, Params, <<"">>),
+            At     = maps:get(<<"at">>, Params, erlang:system_time(second)),
+            Until  = maps:get(<<"until">>, Params, At + 5 * 60),
+            case Until > erlang:system_time(second) of
+                true ->
+                    #banned{
+                        who    = Who,
+                        by     = By,
+                        reason = Reason,
+                        at     = At,
+                        until  = Until
+                    };
+                false ->
+                    {error, "already_expired"}
+            end
+    end.
 pares_who(#{as := As, who := Who}) ->
     pares_who(#{<<"as">> => As, <<"who">> => Who});
 pares_who(#{<<"as">> := peerhost, <<"who">> := Peerhost0}) ->
-    {ok, Peerhost} = inet:parse_address(binary_to_list(Peerhost0)),
-    {peerhost, Peerhost};
+    case inet:parse_address(binary_to_list(Peerhost0)) of
+        {ok, Peerhost} -> {peerhost, Peerhost};
+        {error, einval} -> {error, "bad peerhost"}
+    end;
 pares_who(#{<<"as">> := As, <<"who">> := Who}) ->
     {As, Who}.
 
-parse_time(undefined, Default) ->
-    Default;
-parse_time(Rfc3339, _Default) ->
-    to_timestamp(Rfc3339).
-
 maybe_format_host({peerhost, Host}) ->
     AddrBinary = list_to_binary(inet:ntoa(Host)),
     {peerhost, AddrBinary};
@@ -141,11 +149,6 @@ maybe_format_host({As, Who}) ->
 to_rfc3339(Timestamp) ->
     list_to_binary(calendar:system_time_to_rfc3339(Timestamp, [{unit, second}])).
 
-to_timestamp(Rfc3339) when is_binary(Rfc3339) ->
-    to_timestamp(binary_to_list(Rfc3339));
-to_timestamp(Rfc3339) ->
-    calendar:rfc3339_to_system_time(Rfc3339, [{unit, second}]).
-
 -spec(create(emqx_types:banned() | map()) ->
     {ok, emqx_types:banned()} | {error, {already_exist, emqx_types:banned()}}).
 create(#{who    := Who,
@@ -168,10 +171,11 @@ create(Banned = #banned{who = Who})  ->
             mria:dirty_write(?BANNED_TAB, Banned),
             {ok, Banned};
         [OldBanned = #banned{until = Until}] ->
-            case Until < erlang:system_time(second) of
-                true ->
-                    {error, {already_exist, OldBanned}};
-                false ->
+            %% Don't support shorten or extend the until time by overwrite.
+            %% We don't support update api yet, user must delete then create new one.
+            case Until > erlang:system_time(second) of
+                true -> {error, {already_exist, OldBanned}};
+                false -> %% overwrite expired one is ok.
                     mria:dirty_write(?BANNED_TAB, Banned),
                     {ok, Banned}
             end

apps/emqx/src/emqx_broker.erl

@@ -204,9 +204,9 @@ publish(Msg) when is_record(Msg, message) ->
     _ = emqx_trace:publish(Msg),
     emqx_message:is_sys(Msg) orelse emqx_metrics:inc('messages.publish'),
     case emqx_hooks:run_fold('message.publish', [], emqx_message:clean_dup(Msg)) of
-        #message{headers = #{allow_publish := false}} ->
-            ?SLOG(debug, #{msg => "message_not_published",
-                           payload => emqx_message:to_log_map(Msg)}),
+        #message{headers = #{allow_publish := false}, topic = Topic} ->
+            ?TRACE("MQTT", "msg_publish_not_allowed", #{message => emqx_message:to_log_map(Msg),
+                topic => Topic}),
             [];
         Msg1 = #message{topic = Topic} ->
             emqx_persistent_session:persist_message(Msg1),
@@ -226,7 +226,9 @@ safe_publish(Msg) when is_record(Msg, message) ->
                 reason => Reason,
                 payload => emqx_message:to_log_map(Msg),
                 stacktrace => Stk
-            }),
+            },
+                #{topic => Msg#message.topic}
+            ),
             []
     end.
 
@@ -280,7 +282,7 @@ forward(Node, To, Delivery, async) ->
                 msg => "async_forward_msg_to_node_failed",
                 node => Node,
                 reason => Reason
-            }),
+            }, #{topic => To}),
             {error, badrpc}
     end;
 
@@ -291,7 +293,7 @@ forward(Node, To, Delivery, sync) ->
                 msg => "sync_forward_msg_to_node_failed",
                 node => Node,
                 reason => Reason
-            }),
+            }, #{topic => To}),
             {error, badrpc};
         Result ->
             emqx_metrics:inc('messages.forward'), Result

apps/emqx/src/emqx_channel.erl

@@ -292,7 +292,7 @@ handle_in(?CONNECT_PACKET(ConnPkt) = Packet, Channel) ->
                    fun check_banned/2
                   ], ConnPkt, Channel#channel{conn_state = connecting}) of
         {ok, NConnPkt, NChannel = #channel{clientinfo = ClientInfo}} ->
-            ?SLOG(debug, #{msg => "recv_packet", packet => emqx_packet:format(Packet)}),
+            ?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
             NChannel1 = NChannel#channel{
                                         will_msg = emqx_packet:will_msg(NConnPkt),
                                         alias_maximum = init_alias_maximum(NConnPkt, ClientInfo)
@@ -550,9 +550,8 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
         {error, Rc = ?RC_NOT_AUTHORIZED, NChannel} ->
             ?SLOG(warning, #{
                 msg => "cannot_publish_to_topic",
-                topic => Topic,
                 reason => emqx_reason_codes:name(Rc)
-            }),
+            }, #{topic => Topic}),
             case emqx:get_config([authorization, deny_action], ignore) of
                 ignore ->
                     case QoS of
@@ -568,9 +567,8 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
         {error, Rc = ?RC_QUOTA_EXCEEDED, NChannel} ->
             ?SLOG(warning, #{
                 msg => "cannot_publish_to_topic",
-                topic => Topic,
                 reason => emqx_reason_codes:name(Rc)
-            }),
+            }, #{topic => Topic}),
             case QoS of
                 ?QOS_0 ->
                     ok = emqx_metrics:inc('packets.publish.dropped'),
@@ -585,7 +583,7 @@ process_publish(Packet = ?PUBLISH_PACKET(QoS, Topic, PacketId), Channel) ->
                 msg => "cannot_publish_to_topic",
                 topic => Topic,
                 reason => emqx_reason_codes:name(Rc)
-            }),
+            }, #{topic => Topic}),
             handle_out(disconnect, Rc, NChannel)
     end.
 
@@ -635,7 +633,7 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2},
                 msg => "dropped_qos2_packet",
                 reason => emqx_reason_codes:name(RC),
                 packet_id => PacketId
-            }),
+            }, #{topic => Msg#message.topic}),
             ok = emqx_metrics:inc('packets.publish.dropped'),
             handle_out(disconnect, RC, Channel)
     end.
@@ -687,7 +685,7 @@ process_subscribe([Topic = {TopicFilter, SubOpts} | More], SubProps, Channel, Ac
             ?SLOG(warning, #{
                 msg => "cannot_subscribe_topic_filter",
                 reason => emqx_reason_codes:name(ReasonCode)
-            }),
+            }, #{topic => TopicFilter}),
             process_subscribe(More, SubProps, Channel, [{Topic, ReasonCode} | Acc])
     end.
 
@@ -703,7 +701,7 @@ do_subscribe(TopicFilter, SubOpts = #{qos := QoS}, Channel =
             ?SLOG(warning, #{
                 msg => "cannot_subscribe_topic_filter",
                 reason => emqx_reason_codes:text(RC)
-            }),
+            }, #{topic => NTopicFilter}),
             {RC, Channel}
     end.
 

apps/emqx/src/emqx_cm.erl

@@ -375,7 +375,7 @@ discard_session(ClientId) when is_binary(ClientId) ->
 -spec kick_or_kill(kick | discard, module(), pid()) -> ok.
 kick_or_kill(Action, ConnMod, Pid) ->
     try
-        %% this is essentailly a gen_server:call implemented in emqx_connection
+        %% this is essentially a gen_server:call implemented in emqx_connection
         %% and emqx_ws_connection.
         %% the handle_call is implemented in emqx_channel
         ok = apply(ConnMod, call, [Pid, Action, ?T_KICK])
@@ -390,19 +390,12 @@ kick_or_kill(Action, ConnMod, Pid) ->
             ok = ?tp(debug, "session_already_shutdown", #{pid => Pid, action => Action});
         _ : {timeout, {gen_server, call, _}} ->
             ?tp(warning, "session_kick_timeout",
-                #{pid => Pid,
-                  action => Action,
-                  stale_channel => stale_channel_info(Pid)
-                 }),
+                #{pid => Pid, action => Action, stale_channel => stale_channel_info(Pid)}),
             ok = force_kill(Pid);
         _ : Error : St ->
             ?tp(error, "session_kick_exception",
-                #{pid => Pid,
-                  action => Action,
-                  reason => Error,
-                  stacktrace => St,
-                  stale_channel => stale_channel_info(Pid)
-                 }),
+                #{pid => Pid, action => Action, reason => Error, stacktrace => St,
+                    stale_channel => stale_channel_info(Pid)}),
             ok = force_kill(Pid)
     end.
 
@@ -448,20 +441,22 @@ kick_session(Action, ClientId, ChanPid) ->
                           , action => Action
                           , error => Error
                           , reason => Reason
-                          })
+                          },
+                #{clientid => ClientId})
     end.
 
 kick_session(ClientId) ->
     case lookup_channels(ClientId) of
         [] ->
-            ?SLOG(warning, #{msg => "kicked_an_unknown_session",
-                             clientid => ClientId}),
+            ?SLOG(warning, #{msg => "kicked_an_unknown_session"},
+                #{clientid => ClientId}),
             ok;
         ChanPids ->
             case length(ChanPids) > 1 of
                 true ->
                     ?SLOG(warning, #{msg => "more_than_one_channel_found",
-                                     chan_pids => ChanPids});
+                                     chan_pids => ChanPids},
+                        #{clientid => ClientId});
                 false -> ok
             end,
             lists:foreach(fun(Pid) -> kick_session(ClientId, Pid) end, ChanPids)
@@ -478,12 +473,12 @@ with_channel(ClientId, Fun) ->
         Pids  -> Fun(lists:last(Pids))
     end.
 
-%% @doc Get all registed channel pids. Debugg/test interface
+%% @doc Get all registered channel pids. Debug/test interface
 all_channels() ->
     Pat = [{{'_', '$1'}, [], ['$1']}],
     ets:select(?CHAN_TAB, Pat).
 
-%% @doc Get all registed clientIDs. Debugg/test interface
+%% @doc Get all registered clientIDs. Debug/test interface
 all_client_ids() ->
     Pat = [{{'$1', '_'}, [], ['$1']}],
     ets:select(?CHAN_TAB, Pat).
@@ -511,7 +506,7 @@ lookup_channels(local, ClientId) ->
 rpc_call(Node, Fun, Args, Timeout) ->
     case rpc:call(Node, ?MODULE, Fun, Args, 2 * Timeout) of
         {badrpc, Reason} ->
-            %% since eqmx app 4.3.10, the 'kick' and 'discard' calls hanndler
+            %% since emqx app 4.3.10, the 'kick' and 'discard' calls handler
             %% should catch all exceptions and always return 'ok'.
             %% This leaves 'badrpc' only possible when there is problem
             %% calling the remote node.

apps/emqx/src/emqx_config.erl

@@ -262,8 +262,9 @@ init_load(SchemaMod, Conf) when is_list(Conf) orelse is_binary(Conf) ->
         {ok, RawRichConf} ->
             init_load(SchemaMod, RawRichConf);
         {error, Reason} ->
-            ?SLOG(error, #{msg => failed_to_load_hocon_conf,
+            ?SLOG(error, #{msg => "failed_to_load_hocon_conf",
                            reason => Reason,
+                           pwd => file:get_cwd(),
                            include_dirs => IncDir
                           }),
             error(failed_to_load_hocon_conf)
@@ -396,7 +397,7 @@ save_to_override_conf(RawConf, Opts) ->
             case file:write_file(FileName, hocon_pp:do(RawConf, #{})) of
                 ok -> ok;
                 {error, Reason} ->
-                    ?SLOG(error, #{msg => failed_to_write_override_file,
+                    ?SLOG(error, #{msg => "failed_to_write_override_file",
                                    filename => FileName,
                                    reason => Reason}),
                     {error, Reason}

apps/emqx/src/emqx_connection.erl

@@ -449,14 +449,12 @@ handle_msg({'$gen_cast', Req}, State) ->
     {ok, NewState};
 
 handle_msg({Inet, _Sock, Data}, State) when Inet == tcp; Inet == ssl ->
-    ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => Inet}),
     Oct = iolist_size(Data),
     inc_counter(incoming_bytes, Oct),
     ok = emqx_metrics:inc('bytes.received', Oct),
     when_bytes_in(Oct, Data, State);
 
 handle_msg({quic, Data, _Sock, _, _, _}, State) ->
-    ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => quic}),
     Oct = iolist_size(Data),
     inc_counter(incoming_bytes, Oct),
     ok = emqx_metrics:inc('bytes.received', Oct),
@@ -528,7 +526,7 @@ handle_msg({connack, ConnAck}, State) ->
     handle_outgoing(ConnAck, State);
 
 handle_msg({close, Reason}, State) ->
-    ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}),
+    ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}),
     handle_info({sock_closed, Reason}, close_socket(State));
 
 handle_msg({event, connected}, State = #state{channel = Channel}) ->
@@ -566,7 +564,8 @@ terminate(Reason, State = #state{channel = Channel, transport = Transport,
         Channel1 = emqx_channel:set_conn_state(disconnected, Channel),
         emqx_congestion:cancel_alarms(Socket, Transport, Channel1),
         emqx_channel:terminate(Reason, Channel1),
-        close_socket_ok(State)
+        close_socket_ok(State),
+        ?TRACE("SOCKET", "tcp_socket_terminated", #{reason => Reason})
     catch
         E : C : S ->
             ?tp(warning, unclean_terminate, #{exception => E, context => C, stacktrace => S})
@@ -716,7 +715,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
 
 handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) ->
     ok = inc_incoming_stats(Packet),
-    ?SLOG(debug, #{msg => "RECV_packet", packet => emqx_packet:format(Packet)}),
+    ?TRACE("MQTT", "mqtt_packet_received", #{packet => Packet}),
     with_channel(handle_in, [Packet], State);
 
 handle_incoming(FrameError, State) ->
@@ -755,15 +754,13 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
             <<>> -> ?SLOG(warning, #{
                         msg => "packet_is_discarded",
                         reason => "frame_is_too_large",
-                        packet => emqx_packet:format(Packet)
+                        packet => emqx_packet:format(Packet, hidden)
                     }),
                     ok = emqx_metrics:inc('delivery.dropped.too_large'),
                     ok = emqx_metrics:inc('delivery.dropped'),
                     <<>>;
-            Data -> ?SLOG(debug, #{
-                        msg => "SEND_packet",
-                        packet => emqx_packet:format(Packet)
-                    }),
+            Data ->
+                    ?TRACE("MQTT", "mqtt_packet_sent", #{packet => Packet}),
                     ok = inc_outgoing_stats(Packet),
                     Data
         catch
@@ -875,7 +872,7 @@ check_limiter(Needs,
                 {ok, Limiter2} ->
                     WhenOk(Data, Msgs, State#state{limiter = Limiter2});
                 {pause, Time, Limiter2} ->
-                    ?SLOG(warning, #{msg => "pause time dueto rate limit",
+                    ?SLOG(warning, #{msg => "pause_time_dueto_rate_limit",
                                      needs => Needs,
                                      time_in_ms => Time}),
 
@@ -915,7 +912,7 @@ retry_limiter(#state{limiter = Limiter} = State) ->
                                 , limiter_timer = undefined
                                 });
             {pause, Time, Limiter2} ->
-                ?SLOG(warning, #{msg => "pause time dueto rate limit",
+                ?SLOG(warning, #{msg => "pause_time_dueto_rate_limit",
                                  types => Types,
                                  time_in_ms => Time}),
 

apps/emqx/src/emqx_flapping.erl

@@ -118,11 +118,10 @@ handle_cast({detected, #flapping{clientid   = ClientId,
         true -> %% Flapping happened:(
             ?SLOG(warning, #{
                 msg => "flapping_detected",
-                client_id => ClientId,
                 peer_host => fmt_host(PeerHost),
                 detect_cnt => DetectCnt,
                 wind_time_in_ms => WindTime
-            }),
+            }, #{clientid => ClientId}),
             Now = erlang:system_time(second),
             Banned = #banned{who    = {clientid, ClientId},
                              by     = <<"flapping detector">>,
@@ -134,11 +133,10 @@ handle_cast({detected, #flapping{clientid   = ClientId,
         false ->
             ?SLOG(warning, #{
                 msg => "client_disconnected",
-                client_id => ClientId,
                 peer_host => fmt_host(PeerHost),
                 detect_cnt => DetectCnt,
                 interval => Interval
-            })
+            }, #{clientid => ClientId})
     end,
     {noreply, State};
 

apps/emqx/src/emqx_logger.erl

@@ -197,15 +197,7 @@ critical(Metadata, Format, Args) when is_map(Metadata) ->
 set_metadata_clientid(<<>>) ->
     ok;
 set_metadata_clientid(ClientId) ->
-    try
-        %% try put string format client-id metadata so
-        %% so the log is not like <<"...">>
-        Id = unicode:characters_to_list(ClientId, utf8),
-        set_proc_metadata(#{clientid => Id})
-    catch
-        _: _->
-            ok
-    end.
+    set_proc_metadata(#{clientid => ClientId}).
 
 -spec(set_metadata_peername(peername_str()) -> ok).
 set_metadata_peername(Peername) ->

apps/emqx/src/emqx_logger_textfmt.erl

@@ -18,22 +18,77 @@
 
 -export([format/2]).
 -export([check_config/1]).
+-export([try_format_unicode/1]).
 
 check_config(X) -> logger_formatter:check_config(X).
 
-format(#{msg := {report, Report}, meta := Meta} = Event, Config) when is_map(Report) ->
-    logger_formatter:format(Event#{msg := {report, enrich(Report, Meta)}}, Config);
-format(#{msg := Msg, meta := Meta} = Event, Config) ->
-    NewMsg = enrich_fmt(Msg, Meta),
-    logger_formatter:format(Event#{msg := NewMsg}, Config).
+format(#{msg := {report, Report0}, meta := Meta} = Event, Config) when is_map(Report0) ->
+    Report1 = enrich_report_mfa(Report0, Meta),
+    Report2 = enrich_report_clientid(Report1, Meta),
+    Report3 = enrich_report_peername(Report2, Meta),
+    Report4 = enrich_report_topic(Report3, Meta),
+    logger_formatter:format(Event#{msg := {report, Report4}}, Config);
+format(#{msg := {string, String}} = Event, Config) ->
+    format(Event#{msg => {"~ts ", String}}, Config);
+format(#{msg := Msg0, meta := Meta} = Event, Config) ->
+    Msg1 = enrich_client_info(Msg0, Meta),
+    Msg2 = enrich_mfa(Msg1, Meta),
+    Msg3 = enrich_topic(Msg2, Meta),
+    logger_formatter:format(Event#{msg := Msg3}, Config).
 
-enrich(Report, #{mfa := Mfa, line := Line}) ->
+try_format_unicode(Char) ->
+    List =
+        try
+            case unicode:characters_to_list(Char) of
+                {error, _, _} -> error;
+                {incomplete, _, _} -> error;
+                Binary -> Binary
+            end
+        catch _:_ ->
+            error
+        end,
+    case List of
+        error -> io_lib:format("~0p", [Char]);
+        _ -> List
+    end.
+
+enrich_report_mfa(Report, #{mfa := Mfa, line := Line}) ->
     Report#{mfa => mfa(Mfa), line => Line};
-enrich(Report, _) -> Report.
+enrich_report_mfa(Report, _) -> Report.
+
+enrich_report_clientid(Report, #{clientid := ClientId}) ->
+    Report#{clientid => try_format_unicode(ClientId)};
+enrich_report_clientid(Report, _) -> Report.
+
+enrich_report_peername(Report, #{peername := Peername}) ->
+    Report#{peername => Peername};
+enrich_report_peername(Report, _) -> Report.
+
+%% clientid and peername always in emqx_conn's process metadata.
+%% topic can be put in meta using ?SLOG/3, or put in msg's report by ?SLOG/2
+enrich_report_topic(Report, #{topic := Topic}) ->
+    Report#{topic => try_format_unicode(Topic)};
+enrich_report_topic(Report = #{topic := Topic}, _) ->
+    Report#{topic => try_format_unicode(Topic)};
+enrich_report_topic(Report, _) -> Report.
 
-enrich_fmt({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) ->
+enrich_mfa({Fmt, Args}, #{mfa := Mfa, line := Line}) when is_list(Fmt) ->
     {Fmt ++ " mfa: ~ts line: ~w", Args ++ [mfa(Mfa), Line]};
-enrich_fmt(Msg, _) ->
+enrich_mfa(Msg, _) ->
+    Msg.
+
+enrich_client_info({Fmt, Args}, #{clientid := ClientId, peername := Peer}) when is_list(Fmt) ->
+    {" ~ts@~ts " ++ Fmt, [ClientId, Peer | Args] };
+enrich_client_info({Fmt, Args}, #{clientid := ClientId}) when is_list(Fmt) ->
+    {" ~ts " ++ Fmt, [ClientId | Args]};
+enrich_client_info({Fmt, Args}, #{peername := Peer}) when is_list(Fmt) ->
+    {" ~ts " ++ Fmt, [Peer | Args]};
+enrich_client_info(Msg, _) ->
+    Msg.
+
+enrich_topic({Fmt, Args}, #{topic := Topic}) when is_list(Fmt) ->
+    {" topic: ~ts" ++ Fmt, [Topic | Args]};
+enrich_topic(Msg, _) ->
     Msg.
 
 mfa({M, F, A}) -> atom_to_list(M) ++ ":" ++ atom_to_list(F) ++ "/" ++ integer_to_list(A).

apps/emqx/src/emqx_packet.erl

@@ -44,7 +44,11 @@
         , will_msg/1
         ]).
 
--export([format/1]).
+-export([ format/1
+        , format/2
+        ]).
+
+-export([encode_hex/1]).
 
 -define(TYPE_NAMES,
         { 'CONNECT'
@@ -435,25 +439,28 @@ will_msg(#mqtt_packet_connect{clientid     = ClientId,
 
 %% @doc Format packet
 -spec(format(emqx_types:packet()) -> iolist()).
-format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}) ->
-    format_header(Header, format_variable(Variable, Payload)).
+format(Packet) -> format(Packet, emqx_trace_handler:payload_encode()).
+
+%% @doc Format packet
+-spec(format(emqx_types:packet(), hex | text | hidden) -> iolist()).
+format(#mqtt_packet{header = Header, variable = Variable, payload = Payload}, PayloadEncode) ->
+    HeaderIO = format_header(Header),
+    case format_variable(Variable, Payload, PayloadEncode) of
+        "" -> HeaderIO;
+        VarIO -> [HeaderIO,",", VarIO]
+    end.
 
 format_header(#mqtt_packet_header{type = Type,
                                   dup = Dup,
                                   qos = QoS,
-                                  retain = Retain}, S) ->
-    S1 = case S == undefined of
-             true -> <<>>;
-             false -> [", ", S]
-         end,
-    io_lib:format("~ts(Q~p, R~p, D~p~ts)", [type_name(Type), QoS, i(Retain), i(Dup), S1]).
-
-format_variable(undefined, _) ->
-    undefined;
-format_variable(Variable, undefined) ->
-    format_variable(Variable);
-format_variable(Variable, Payload) ->
-    io_lib:format("~ts, Payload=~0p", [format_variable(Variable), Payload]).
+                                  retain = Retain}) ->
+    io_lib:format("~ts(Q~p, R~p, D~p)", [type_name(Type), QoS, i(Retain), i(Dup)]).
+
+format_variable(undefined, _, _) -> "";
+format_variable(Variable, undefined, PayloadEncode) ->
+    format_variable(Variable, PayloadEncode);
+format_variable(Variable, Payload, PayloadEncode) ->
+    [format_variable(Variable, PayloadEncode), format_payload(Payload, PayloadEncode)].
 
 format_variable(#mqtt_packet_connect{
                  proto_ver    = ProtoVer,
@@ -467,57 +474,140 @@ format_variable(#mqtt_packet_connect{
                  will_topic   = WillTopic,
                  will_payload = WillPayload,
                  username     = Username,
-                 password     = Password}) ->
-    Format = "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts",
-    Args = [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)],
-    {Format1, Args1} = if
-                        WillFlag -> {Format ++ ", Will(Q~p, R~p, Topic=~ts, Payload=~0p)",
-                                     Args ++ [WillQoS, i(WillRetain), WillTopic, WillPayload]};
-                        true -> {Format, Args}
-                       end,
-    io_lib:format(Format1, Args1);
+                 password     = Password},
+    PayloadEncode) ->
+    Base = io_lib:format(
+        "ClientId=~ts, ProtoName=~ts, ProtoVsn=~p, CleanStart=~ts, KeepAlive=~p, Username=~ts, Password=~ts",
+        [ClientId, ProtoName, ProtoVer, CleanStart, KeepAlive, Username, format_password(Password)]),
+    case WillFlag of
+        true ->
+            [Base, io_lib:format(", Will(Q~p, R~p, Topic=~ts ",
+                [WillQoS, i(WillRetain), WillTopic]),
+                format_payload(WillPayload, PayloadEncode), ")"];
+        false ->
+            Base
+    end;
 
 format_variable(#mqtt_packet_disconnect
-                {reason_code = ReasonCode}) ->
+                {reason_code = ReasonCode}, _) ->
     io_lib:format("ReasonCode=~p", [ReasonCode]);
 
 format_variable(#mqtt_packet_connack{ack_flags   = AckFlags,
-                                     reason_code = ReasonCode}) ->
+                                     reason_code = ReasonCode}, _) ->
     io_lib:format("AckFlags=~p, ReasonCode=~p", [AckFlags, ReasonCode]);
 
 format_variable(#mqtt_packet_publish{topic_name = TopicName,
-                                     packet_id  = PacketId}) ->
+                                     packet_id  = PacketId}, _) ->
     io_lib:format("Topic=~ts, PacketId=~p", [TopicName, PacketId]);
 
 format_variable(#mqtt_packet_puback{packet_id = PacketId,
-                                    reason_code = ReasonCode}) ->
+                                    reason_code = ReasonCode}, _) ->
     io_lib:format("PacketId=~p, ReasonCode=~p", [PacketId, ReasonCode]);
 
 format_variable(#mqtt_packet_subscribe{packet_id     = PacketId,
-                                       topic_filters = TopicFilters}) ->
-    io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, TopicFilters]);
+                                       topic_filters = TopicFilters}, _) ->
+    [io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=",
+        format_topic_filters(TopicFilters)];
 
 format_variable(#mqtt_packet_unsubscribe{packet_id     = PacketId,
-                                         topic_filters = Topics}) ->
-    io_lib:format("PacketId=~p, TopicFilters=~0p", [PacketId, Topics]);
+                                         topic_filters = Topics}, _) ->
+    [io_lib:format("PacketId=~p ", [PacketId]), "TopicFilters=",
+        format_topic_filters(Topics)];
 
 format_variable(#mqtt_packet_suback{packet_id = PacketId,
-                                    reason_codes = ReasonCodes}) ->
+                                    reason_codes = ReasonCodes}, _) ->
     io_lib:format("PacketId=~p, ReasonCodes=~p", [PacketId, ReasonCodes]);
 
-format_variable(#mqtt_packet_unsuback{packet_id = PacketId}) ->
+format_variable(#mqtt_packet_unsuback{packet_id = PacketId}, _) ->
     io_lib:format("PacketId=~p", [PacketId]);
 
-format_variable(#mqtt_packet_auth{reason_code = ReasonCode}) ->
+format_variable(#mqtt_packet_auth{reason_code = ReasonCode}, _) ->
     io_lib:format("ReasonCode=~p", [ReasonCode]);
 
-format_variable(PacketId) when is_integer(PacketId) ->
+format_variable(PacketId, _) when is_integer(PacketId) ->
     io_lib:format("PacketId=~p", [PacketId]).
 
-format_password(undefined) -> undefined;
-format_password(_Password) -> '******'.
+format_password(undefined) -> "undefined";
+format_password(_Password) -> "******".
+
+format_payload(Payload, text) -> ["Payload=", io_lib:format("~ts", [Payload])];
+format_payload(Payload, hex) -> ["Payload(hex)=", encode_hex(Payload)];
+format_payload(_, hidden) -> "Payload=******".
 
 i(true)  -> 1;
 i(false) -> 0;
 i(I) when is_integer(I) -> I.
 
+format_topic_filters(Filters) ->
+    ["[",
+        lists:join(",",
+            lists:map(
+                fun({TopicFilter, SubOpts}) ->
+                    io_lib:format("~ts(~p)", [TopicFilter, SubOpts]);
+                    (TopicFilter) ->
+                        io_lib:format("~ts", [TopicFilter])
+                end, Filters)),
+        "]"].
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%% Hex encoding functions
+%% Copy from binary:encode_hex/1 (was only introduced in OTP24).
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+-define(HEX(X), (hex(X)):16).
+-compile({inline,[hex/1]}).
+-spec encode_hex(Bin) -> Bin2 when
+    Bin :: binary(),
+    Bin2 :: <<_:_*16>>.
+encode_hex(Data) when byte_size(Data) rem 8 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G),?HEX(H)>> || <<A,B,C,D,E,F,G,H>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 7 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F),?HEX(G)>> || <<A,B,C,D,E,F,G>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 6 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E),?HEX(F)>> || <<A,B,C,D,E,F>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 5 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D),?HEX(E)>> || <<A,B,C,D,E>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 4 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C),?HEX(D)>> || <<A,B,C,D>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 3 =:= 0 ->
+    << <<?HEX(A),?HEX(B),?HEX(C)>> || <<A,B,C>> <= Data >>;
+encode_hex(Data) when byte_size(Data) rem 2 =:= 0 ->
+    << <<?HEX(A),?HEX(B)>> || <<A,B>> <= Data >>;
+encode_hex(Data) when is_binary(Data) ->
+    << <<?HEX(N)>> || <<N>> <= Data >>;
+encode_hex(Bin) ->
+    erlang:error(badarg, [Bin]).
+
+hex(X) ->
+    element(
+        X+1, {16#3030, 16#3031, 16#3032, 16#3033, 16#3034, 16#3035, 16#3036, 16#3037, 16#3038, 16#3039, 16#3041,
+            16#3042, 16#3043, 16#3044, 16#3045, 16#3046,
+            16#3130, 16#3131, 16#3132, 16#3133, 16#3134, 16#3135, 16#3136, 16#3137, 16#3138, 16#3139, 16#3141,
+            16#3142, 16#3143, 16#3144, 16#3145, 16#3146,
+            16#3230, 16#3231, 16#3232, 16#3233, 16#3234, 16#3235, 16#3236, 16#3237, 16#3238, 16#3239, 16#3241,
+            16#3242, 16#3243, 16#3244, 16#3245, 16#3246,
+            16#3330, 16#3331, 16#3332, 16#3333, 16#3334, 16#3335, 16#3336, 16#3337, 16#3338, 16#3339, 16#3341,
+            16#3342, 16#3343, 16#3344, 16#3345, 16#3346,
+            16#3430, 16#3431, 16#3432, 16#3433, 16#3434, 16#3435, 16#3436, 16#3437, 16#3438, 16#3439, 16#3441,
+            16#3442, 16#3443, 16#3444, 16#3445, 16#3446,
+            16#3530, 16#3531, 16#3532, 16#3533, 16#3534, 16#3535, 16#3536, 16#3537, 16#3538, 16#3539, 16#3541,
+            16#3542, 16#3543, 16#3544, 16#3545, 16#3546,
+            16#3630, 16#3631, 16#3632, 16#3633, 16#3634, 16#3635, 16#3636, 16#3637, 16#3638, 16#3639, 16#3641,
+            16#3642, 16#3643, 16#3644, 16#3645, 16#3646,
+            16#3730, 16#3731, 16#3732, 16#3733, 16#3734, 16#3735, 16#3736, 16#3737, 16#3738, 16#3739, 16#3741,
+            16#3742, 16#3743, 16#3744, 16#3745, 16#3746,
+            16#3830, 16#3831, 16#3832, 16#3833, 16#3834, 16#3835, 16#3836, 16#3837, 16#3838, 16#3839, 16#3841,
+            16#3842, 16#3843, 16#3844, 16#3845, 16#3846,
+            16#3930, 16#3931, 16#3932, 16#3933, 16#3934, 16#3935, 16#3936, 16#3937, 16#3938, 16#3939, 16#3941,
+            16#3942, 16#3943, 16#3944, 16#3945, 16#3946,
+            16#4130, 16#4131, 16#4132, 16#4133, 16#4134, 16#4135, 16#4136, 16#4137, 16#4138, 16#4139, 16#4141,
+            16#4142, 16#4143, 16#4144, 16#4145, 16#4146,
+            16#4230, 16#4231, 16#4232, 16#4233, 16#4234, 16#4235, 16#4236, 16#4237, 16#4238, 16#4239, 16#4241,
+            16#4242, 16#4243, 16#4244, 16#4245, 16#4246,
+            16#4330, 16#4331, 16#4332, 16#4333, 16#4334, 16#4335, 16#4336, 16#4337, 16#4338, 16#4339, 16#4341,
+            16#4342, 16#4343, 16#4344, 16#4345, 16#4346,
+            16#4430, 16#4431, 16#4432, 16#4433, 16#4434, 16#4435, 16#4436, 16#4437, 16#4438, 16#4439, 16#4441,
+            16#4442, 16#4443, 16#4444, 16#4445, 16#4446,
+            16#4530, 16#4531, 16#4532, 16#4533, 16#4534, 16#4535, 16#4536, 16#4537, 16#4538, 16#4539, 16#4541,
+            16#4542, 16#4543, 16#4544, 16#4545, 16#4546,
+            16#4630, 16#4631, 16#4632, 16#4633, 16#4634, 16#4635, 16#4636, 16#4637, 16#4638, 16#4639, 16#4641,
+            16#4642, 16#4643, 16#4644, 16#4645, 16#4646}).

apps/emqx/src/emqx_schema.erl

@@ -38,7 +38,6 @@
 -type ip_port() :: tuple().
 -type cipher() :: map().
 -type rfc3339_system_time() :: integer().
--type unicode_binary() :: binary().
 
 -typerefl_from_string({duration/0, emqx_schema, to_duration}).
 -typerefl_from_string({duration_s/0, emqx_schema, to_duration_s}).
@@ -52,7 +51,6 @@
 -typerefl_from_string({cipher/0, emqx_schema, to_erl_cipher_suite}).
 -typerefl_from_string({comma_separated_atoms/0, emqx_schema, to_comma_separated_atoms}).
 -typerefl_from_string({rfc3339_system_time/0, emqx_schema, rfc3339_to_system_time}).
--typerefl_from_string({unicode_binary/0, emqx_schema, to_unicode_binary}).
 
 -export([ validate_heap_size/1
         , parse_user_lookup_fun/1
@@ -66,8 +64,7 @@
          to_bar_separated_list/1, to_ip_port/1,
          to_erl_cipher_suite/1,
          to_comma_separated_atoms/1,
-         rfc3339_to_system_time/1,
-         to_unicode_binary/1]).
+         rfc3339_to_system_time/1]).
 
 -behaviour(hocon_schema).
 
@@ -76,8 +73,7 @@
                 comma_separated_list/0, bar_separated_list/0, ip_port/0,
                 cipher/0,
                 comma_separated_atoms/0,
-                rfc3339_system_time/0,
-                unicode_binary/0]).
+                rfc3339_system_time/0]).
 
 -export([namespace/0, roots/0, roots/1, fields/1]).
 -export([conf_get/2, conf_get/3, keys/2, filter/1]).
@@ -184,6 +180,12 @@ roots(low) ->
     , {"latency_stats",
        sc(ref("latency_stats"),
           #{})}
+    , {"trace",
+       sc(ref("trace"),
+          #{desc => """
+Real-time filtering logs for the ClientID or Topic or IP for debugging.
+"""
+          })}
     ].
 
 fields("persistent_session_store") ->
@@ -1044,6 +1046,17 @@ when deactivated, but after the retention time.
 fields("latency_stats") ->
     [ {"samples", sc(integer(), #{default => 10,
                                   desc => "the number of smaples for calculate the average latency of delivery"})}
+    ];
+fields("trace") ->
+    [ {"payload_encode", sc(hoconsc:enum([hex, text, hidden]), #{
+        default => text,
+        desc => """
+Determine the format of the payload format in the trace file.<br>
+`text`: Text-based protocol or plain text protocol. It is recommended when payload is json encode.<br>
+`hex`: Binary hexadecimal encode. It is recommended when payload is a custom binary protocol.<br>
+`hidden`: payload is obfuscated as `******`
+        """
+    })}
     ].
 
 mqtt_listener() ->
@@ -1453,9 +1466,6 @@ rfc3339_to_system_time(DateTime) ->
         {error, bad_rfc3339_timestamp}
     end.
 
-to_unicode_binary(Str) ->
-    {ok, unicode:characters_to_binary(Str)}.
-
 to_bar_separated_list(Str) ->
     {ok, string:tokens(Str, "| ")}.
 

apps/emqx/src/emqx_session.erl

@@ -535,16 +535,20 @@ enqueue(Msg, Session = #session{mqueue = Q}) when is_record(Msg, message) ->
     (Dropped =/= undefined) andalso log_dropped(Dropped, Session),
     Session#session{mqueue = NewQ}.
 
-log_dropped(Msg = #message{qos = QoS}, #session{mqueue = Q}) ->
-    case (QoS == ?QOS_0) andalso (not emqx_mqueue:info(store_qos0, Q)) of
+log_dropped(Msg = #message{qos = QoS, topic = Topic}, #session{mqueue = Q}) ->
+    Payload = emqx_message:to_log_map(Msg),
+     #{store_qos0 := StoreQos0} = QueueInfo = emqx_mqueue:info(Q),
+    case (QoS == ?QOS_0) andalso (not StoreQos0) of
         true  ->
             ok = emqx_metrics:inc('delivery.dropped.qos0_msg'),
             ?SLOG(warning, #{msg => "dropped_qos0_msg",
-                             payload => emqx_message:to_log_map(Msg)});
+                             queue => QueueInfo,
+                             payload => Payload}, #{topic => Topic});
         false ->
             ok = emqx_metrics:inc('delivery.dropped.queue_full'),
             ?SLOG(warning, #{msg => "dropped_msg_due_to_mqueue_is_full",
-                             payload => emqx_message:to_log_map(Msg)})
+                             queue => QueueInfo,
+                             payload => Payload}, #{topic => Topic})
     end.
 
 enrich_fun(Session = #session{subscriptions = Subs}) ->

apps/emqx/src/emqx_session_router.erl

@@ -260,7 +260,7 @@ code_change(_OldVsn, State, _Extra) ->
 init_resume_worker(RemotePid, SessionID, #{ pmon := Pmon } = State) ->
     case emqx_session_router_worker_sup:start_worker(SessionID, RemotePid) of
         {error, What} ->
-            ?SLOG(error, #{msg => "Could not start resume worker", reason => What}),
+            ?SLOG(error, #{msg => "failed_to_start_resume_worker", reason => What}),
             error;
         {ok, Pid} ->
             Pmon1 = emqx_pmon:monitor(Pid, Pmon),

apps/emqx/src/emqx_trace/emqx_trace.erl

@@ -26,6 +26,7 @@
 -export([ publish/1
         , subscribe/3
         , unsubscribe/2
+        , log/4
         ]).
 
 -export([ start_link/0
@@ -36,6 +37,7 @@
         , delete/1
         , clear/0
         , update/2
+        , check/0
         ]).
 
 -export([ format/1
@@ -50,6 +52,7 @@
 
 -define(TRACE, ?MODULE).
 -define(MAX_SIZE, 30).
+-define(OWN_KEYS, [level, filters, filter_default, handlers]).
 
 -ifdef(TEST).
 -export([ log_file/2
@@ -80,27 +83,53 @@ mnesia(boot) ->
 publish(#message{topic = <<"$SYS/", _/binary>>}) -> ignore;
 publish(#message{from = From, topic = Topic, payload = Payload}) when
     is_binary(From); is_atom(From) ->
-    emqx_logger:info(
-        #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
-        "PUBLISH to ~s: ~0p",
-        [Topic, Payload]
-    ).
+    ?TRACE("PUBLISH", "publish_to", #{topic => Topic, payload => Payload}).
 
 subscribe(<<"$SYS/", _/binary>>, _SubId, _SubOpts) -> ignore;
 subscribe(Topic, SubId, SubOpts) ->
-    emqx_logger:info(
-        #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
-        "~ts SUBSCRIBE ~ts: Options: ~0p",
-        [SubId, Topic, SubOpts]
-    ).
+    ?TRACE("SUBSCRIBE", "subscribe", #{topic => Topic, sub_opts => SubOpts, sub_id => SubId}).
 
 unsubscribe(<<"$SYS/", _/binary>>, _SubOpts) -> ignore;
 unsubscribe(Topic, SubOpts) ->
-    emqx_logger:info(
-        #{topic => Topic, mfa => {?MODULE, ?FUNCTION_NAME, ?FUNCTION_ARITY}},
-        "~ts UNSUBSCRIBE ~ts: Options: ~0p",
-        [maps:get(subid, SubOpts, ""), Topic, SubOpts]
-    ).
+    ?TRACE("UNSUBSCRIBE", "unsubscribe", #{topic => Topic, sub_opts => SubOpts}).
+
+log(List, Event, Msg, Meta0) ->
+    Meta =
+        case logger:get_process_metadata() of
+            undefined -> Meta0;
+            ProcMeta -> maps:merge(ProcMeta, Meta0)
+        end,
+    Log = #{level => trace, event => Event, meta => Meta, msg => Msg},
+    log_filter(List, Log).
+
+log_filter([], _Log) -> ok;
+log_filter([{Id, FilterFun, Filter, Name} | Rest], Log0) ->
+    case FilterFun(Log0, {Filter, Name}) of
+        stop -> stop;
+        ignore -> ignore;
+        Log ->
+            case logger_config:get(ets:whereis(logger), Id) of
+                {ok, #{module := Module} = HandlerConfig0} ->
+                    HandlerConfig = maps:without(?OWN_KEYS, HandlerConfig0),
+                    try Module:log(Log, HandlerConfig)
+                    catch C:R:S ->
+                        case logger:remove_handler(Id) of
+                            ok ->
+                                logger:internal_log(error, {removed_failing_handler, Id, C, R, S});
+                            {error,{not_found,_}} ->
+                                %% Probably already removed by other client
+                                %% Don't report again
+                                ok;
+                            {error,Reason} ->
+                                logger:internal_log(error,
+                                    {removed_handler_failed, Id, Reason, C, R, S})
+                        end
+                    end;
+                {error, {not_found, Id}} -> ok;
+                {error, Reason} -> logger:internal_log(error, {find_handle_id_failed, Id, Reason})
+            end
+    end,
+    log_filter(Rest, Log0).
 
 -spec(start_link() -> emqx_types:startlink_ret()).
 start_link() ->
@@ -161,6 +190,9 @@ update(Name, Enable) ->
            end,
     transaction(Tran).
 
+check() ->
+    gen_server:call(?MODULE, check).
+
 -spec get_trace_filename(Name :: binary()) ->
     {ok, FileName :: string()} | {error, not_found}.
 get_trace_filename(Name) ->
@@ -196,15 +228,17 @@ format(Traces) ->
 init([]) ->
     ok = mria:wait_for_tables([?TRACE]),
     erlang:process_flag(trap_exit, true),
-    OriginLogLevel = emqx_logger:get_primary_log_level(),
     ok = filelib:ensure_dir(trace_dir()),
     ok = filelib:ensure_dir(zip_dir()),
     {ok, _} = mnesia:subscribe({table, ?TRACE, simple}),
     Traces = get_enable_trace(),
-    ok = update_log_primary_level(Traces, OriginLogLevel),
     TRef = update_trace(Traces),
-    {ok, #{timer => TRef, monitors => #{}, primary_log_level => OriginLogLevel}}.
+    update_trace_handler(),
+    {ok, #{timer => TRef, monitors => #{}}}.
 
+handle_call(check, _From, State) ->
+    {_, NewState} = handle_info({mnesia_table_event, check}, State),
+    {reply, ok, NewState};
 handle_call(Req, _From, State) ->
     ?SLOG(error, #{unexpected_call => Req}),
     {reply, ok, State}.
@@ -223,11 +257,10 @@ handle_info({'DOWN', _Ref, process, Pid, _Reason}, State = #{monitors := Monitor
             lists:foreach(fun file:delete/1, Files),
             {noreply, State#{monitors => NewMonitors}}
     end;
-handle_info({timeout, TRef, update_trace},
-    #{timer := TRef, primary_log_level := OriginLogLevel} = State) ->
+handle_info({timeout, TRef, update_trace}, #{timer := TRef} = State) ->
     Traces = get_enable_trace(),
-    ok = update_log_primary_level(Traces, OriginLogLevel),
     NextTRef = update_trace(Traces),
+    update_trace_handler(),
     {noreply, State#{timer => NextTRef}};
 
 handle_info({mnesia_table_event, _Events}, State = #{timer := TRef}) ->
@@ -238,11 +271,11 @@ handle_info(Info, State) ->
     ?SLOG(error, #{unexpected_info => Info}),
     {noreply, State}.
 
-terminate(_Reason, #{timer := TRef, primary_log_level := OriginLogLevel}) ->
-    ok = set_log_primary_level(OriginLogLevel),
+terminate(_Reason, #{timer := TRef}) ->
     _ = mnesia:unsubscribe({table, ?TRACE, simple}),
     emqx_misc:cancel_timer(TRef),
     stop_all_trace_handler(),
+    update_trace_handler(),
     _ = file:del_dir_r(zip_dir()),
     ok.
 
@@ -270,7 +303,7 @@ update_trace(Traces) ->
     disable_finished(Finished),
     Started = emqx_trace_handler:running(),
     {NeedRunning, AllStarted} = start_trace(Running, Started),
-    NeedStop = AllStarted -- NeedRunning,
+    NeedStop = filter_cli_handler(AllStarted) -- NeedRunning,
     ok = stop_trace(NeedStop, Started),
     clean_stale_trace_files(),
     NextTime = find_closest_time(Traces, Now),
@@ -308,10 +341,10 @@ disable_finished(Traces) ->
 
 start_trace(Traces, Started0) ->
     Started = lists:map(fun(#{name := Name}) -> Name end, Started0),
-    lists:foldl(fun(#?TRACE{name = Name} = Trace, {Running, StartedAcc}) ->
+    lists:foldl(fun(#?TRACE{name = Name} = Trace,
+        {Running, StartedAcc}) ->
         case lists:member(Name, StartedAcc) of
-            true ->
-                {[Name | Running], StartedAcc};
+            true -> {[Name | Running], StartedAcc};
             false ->
                 case start_trace(Trace) of
                     ok -> {[Name | Running], [Name | StartedAcc]};
@@ -330,9 +363,11 @@ start_trace(Trace) ->
     emqx_trace_handler:install(Who, debug, log_file(Name, Start)).
 
 stop_trace(Finished, Started) ->
-    lists:foreach(fun(#{name := Name, type := Type}) ->
+    lists:foreach(fun(#{name := Name, type := Type, filter := Filter}) ->
         case lists:member(Name, Finished) of
-            true -> emqx_trace_handler:uninstall(Type, Name);
+            true ->
+                ?TRACE("API", "trace_stopping", #{Type => Filter}),
+                emqx_trace_handler:uninstall(Type, Name);
             false -> ok
         end
                   end, Started).
@@ -419,7 +454,7 @@ to_trace(#{type := ip_address, ip_address := Filter} = Trace, Rec) ->
     case validate_ip_address(Filter) of
         ok ->
             Trace0 = maps:without([type, ip_address], Trace),
-            to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = Filter});
+            to_trace(Trace0, Rec#?TRACE{type = ip_address, filter = binary_to_list(Filter)});
         Error -> Error
     end;
 to_trace(#{type := Type}, _Rec) -> {error, io_lib:format("required ~s field", [Type])};
@@ -481,11 +516,20 @@ transaction(Tran) ->
         {aborted, Reason} -> {error, Reason}
     end.
 
-update_log_primary_level([], OriginLevel) -> set_log_primary_level(OriginLevel);
-update_log_primary_level(_, _) -> set_log_primary_level(debug).
-
-set_log_primary_level(NewLevel) ->
-    case NewLevel =/= emqx_logger:get_primary_log_level() of
-        true -> emqx_logger:set_primary_log_level(NewLevel);
-        false -> ok
+update_trace_handler() ->
+    case emqx_trace_handler:running() of
+        [] -> persistent_term:erase(?TRACE_FILTER);
+        Running ->
+            List = lists:map(fun(#{id := Id, filter_fun := FilterFun,
+                filter := Filter, name := Name}) ->
+                {Id, FilterFun, Filter, Name} end, Running),
+            case List =/= persistent_term:get(?TRACE_FILTER, undefined) of
+                true -> persistent_term:put(?TRACE_FILTER, List);
+                false -> ok
+            end
     end.
+
+filter_cli_handler(Names) ->
+    lists:filter(fun(Name) ->
+        nomatch =:= re:run(Name, "^CLI-+.", [])
+                 end, Names).

apps/emqx/src/emqx_trace/emqx_trace_formatter.erl

@@ -0,0 +1,62 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_trace_formatter).
+
+-export([format/2]).
+
+%%%-----------------------------------------------------------------
+%%% API
+-spec format(LogEvent, Config) -> unicode:chardata() when
+    LogEvent :: logger:log_event(),
+    Config :: logger:config().
+format(#{level := trace, event := Event, meta := Meta, msg := Msg},
+    #{payload_encode := PEncode}) ->
+    Time = calendar:system_time_to_rfc3339(erlang:system_time(second)),
+    ClientId = to_iolist(maps:get(clientid, Meta, "")),
+    Peername = maps:get(peername, Meta, ""),
+    MetaBin = format_meta(Meta, PEncode),
+    [Time, " [", Event, "] ", ClientId, "@", Peername, " msg: ", Msg, MetaBin, "\n"];
+
+format(Event, Config) ->
+    emqx_logger_textfmt:format(Event, Config).
+
+format_meta(Meta0, Encode) ->
+    Packet = format_packet(maps:get(packet, Meta0, undefined), Encode),
+    Payload = format_payload(maps:get(payload, Meta0, undefined), Encode),
+    Meta1 = maps:without([msg, clientid, peername, packet, payload], Meta0),
+    case Meta1 =:= #{} of
+        true -> [Packet, Payload];
+        false -> [Packet, ", ", map_to_iolist(Meta1), Payload]
+    end.
+
+format_packet(undefined, _) -> "";
+format_packet(Packet, Encode) -> [", packet: ", emqx_packet:format(Packet, Encode)].
+
+format_payload(undefined, _) -> "";
+format_payload(Payload, text) -> [", payload: ", io_lib:format("~ts", [Payload])];
+format_payload(Payload, hex) -> [", payload(hex): ", emqx_packet:encode_hex(Payload)];
+format_payload(_, hidden) -> ", payload=******".
+
+to_iolist(Atom) when is_atom(Atom) -> atom_to_list(Atom);
+to_iolist(Int) when is_integer(Int) -> integer_to_list(Int);
+to_iolist(Float) when is_float(Float) -> float_to_list(Float, [{decimals, 2}]);
+to_iolist(SubMap) when is_map(SubMap) -> ["[", map_to_iolist(SubMap), "]"];
+to_iolist(Char) -> emqx_logger_textfmt:try_format_unicode(Char).
+
+map_to_iolist(Map) ->
+    lists:join(",",
+        lists:map(fun({K, V}) -> [to_iolist(K), ": ", to_iolist(V)] end,
+            maps:to_list(Map))).

apps/emqx/src/emqx_trace/emqx_trace_handler.erl

@@ -25,6 +25,7 @@
 -export([ running/0
         , install/3
         , install/4
+        , install/5
         , uninstall/1
         , uninstall/2
         ]).
@@ -36,6 +37,7 @@
         ]).
 
 -export([handler_id/2]).
+-export([payload_encode/0]).
 
 -type tracer() :: #{
          name := binary(),
@@ -77,22 +79,18 @@ install(Type, Filter, Level, LogFile) ->
 -spec install(tracer(), logger:level() | all, string()) -> ok | {error, term()}.
 install(Who, all, LogFile) ->
     install(Who, debug, LogFile);
-install(Who, Level, LogFile) ->
-    PrimaryLevel = emqx_logger:get_primary_log_level(),
-    try logger:compare_levels(Level, PrimaryLevel) of
-        lt ->
-            {error,
-                io_lib:format(
-                    "Cannot trace at a log level (~s) "
-                    "lower than the primary log level (~s)",
-                    [Level, PrimaryLevel]
-                )};
-        _GtOrEq ->
-            install_handler(Who, Level, LogFile)
-    catch
-        error:badarg ->
-            {error, {invalid_log_level, Level}}
-    end.
+install(Who = #{name := Name, type := Type}, Level, LogFile) ->
+    HandlerId = handler_id(Name, Type),
+    Config = #{
+        level => Level,
+        formatter => formatter(Who),
+        filter_default => stop,
+        filters => filters(Who),
+        config => ?CONFIG(LogFile)
+    },
+    Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
+    show_prompts(Res, Who, "start_trace"),
+    Res.
 
 -spec uninstall(Type :: clientid | topic | ip_address,
     Name :: binary() | list()) -> ok | {error, term()}.
@@ -121,83 +119,59 @@ uninstall(HandlerId) ->
 running() ->
     lists:foldl(fun filter_traces/2, [], emqx_logger:get_log_handlers(started)).
 
--spec filter_clientid(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore.
+-spec filter_clientid(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop.
 filter_clientid(#{meta := #{clientid := ClientId}} = Log, {ClientId, _Name}) -> Log;
-filter_clientid(_Log, _ExpectId) -> ignore.
+filter_clientid(_Log, _ExpectId) -> stop.
 
--spec filter_topic(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore.
+-spec filter_topic(logger:log_event(), {binary(), atom()}) -> logger:log_event() | stop.
 filter_topic(#{meta := #{topic := Topic}} = Log, {TopicFilter, _Name}) ->
     case emqx_topic:match(Topic, TopicFilter) of
         true -> Log;
-        false -> ignore
+        false -> stop
     end;
-filter_topic(_Log, _ExpectId) -> ignore.
+filter_topic(_Log, _ExpectId) -> stop.
 
--spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | ignore.
+-spec filter_ip_address(logger:log_event(), {string(), atom()}) -> logger:log_event() | stop.
 filter_ip_address(#{meta := #{peername := Peername}} = Log, {IP, _Name}) ->
     case lists:prefix(IP, Peername) of
         true -> Log;
-        false -> ignore
+        false -> stop
     end;
-filter_ip_address(_Log, _ExpectId) -> ignore.
-
-install_handler(Who = #{name := Name, type := Type}, Level, LogFile) ->
-    HandlerId = handler_id(Name, Type),
-    Config = #{
-        level => Level,
-        formatter => formatter(Who),
-        filter_default => stop,
-        filters => filters(Who),
-        config => ?CONFIG(LogFile)
-    },
-    Res = logger:add_handler(HandlerId, logger_disk_log_h, Config),
-    show_prompts(Res, Who, "start_trace"),
-    Res.
+filter_ip_address(_Log, _ExpectId) -> stop.
 
 filters(#{type := clientid, filter := Filter, name := Name}) ->
-    [{clientid, {fun ?MODULE:filter_clientid/2, {ensure_list(Filter), Name}}}];
+    [{clientid, {fun ?MODULE:filter_clientid/2, {Filter, Name}}}];
 filters(#{type := topic, filter := Filter, name := Name}) ->
     [{topic, {fun ?MODULE:filter_topic/2, {ensure_bin(Filter), Name}}}];
 filters(#{type := ip_address, filter := Filter, name := Name}) ->
     [{ip_address, {fun ?MODULE:filter_ip_address/2, {ensure_list(Filter), Name}}}].
 
-formatter(#{type := Type}) ->
-    {logger_formatter,
+formatter(#{type := _Type}) ->
+    {emqx_trace_formatter,
         #{
-            template => template(Type),
-            single_line => false,
+            %% template is for ?SLOG message not ?TRACE.
+            template => [time," [",level,"] ", msg,"\n"],
+            single_line => true,
             max_size => unlimited,
-            depth => unlimited
+            depth => unlimited,
+            payload_encode => payload_encode()
         }
     }.
 
-%% Don't log clientid since clientid only supports exact match, all client ids are the same.
-%% if clientid is not latin characters. the logger_formatter restricts the output must be `~tp`
-%% (actually should use `~ts`), the utf8 characters clientid will become very difficult to read.
-template(clientid) ->
-    [time, " [", level, "] ", {peername, [peername, " "], []}, msg, "\n"];
-%% TODO better format when clientid is utf8.
-template(_) ->
-    [time, " [", level, "] ",
-        {clientid,
-            [{peername, [clientid, "@", peername, " "], [clientid, " "]}],
-            [{peername, [peername, " "], []}]
-        },
-        msg, "\n"
-    ].
-
 filter_traces(#{id := Id, level := Level, dst := Dst, filters := Filters}, Acc) ->
     Init = #{id => Id, level => Level, dst => Dst},
     case Filters of
-        [{Type, {_FilterFun, {Filter, Name}}}] when
+        [{Type, {FilterFun, {Filter, Name}}}] when
             Type =:= topic orelse
                 Type =:= clientid orelse
                 Type =:= ip_address ->
-            [Init#{type => Type, filter => Filter, name => Name} | Acc];
+            [Init#{type => Type, filter => Filter, name => Name, filter_fun => FilterFun} | Acc];
         _ ->
             Acc
     end.
 
+payload_encode() -> emqx_config:get([trace, payload_encode], text).
+
 handler_id(Name, Type) ->
     try
         do_handler_id(Name, Type)

apps/emqx/src/emqx_ws_connection.erl

@@ -347,7 +347,6 @@ websocket_handle({binary, Data}, State) when is_list(Data) ->
     websocket_handle({binary, iolist_to_binary(Data)}, State);
 
 websocket_handle({binary, Data}, State) ->
-    ?SLOG(debug, #{msg => "RECV_data", data => Data, transport => websocket}),
     State2 = ensure_stats_timer(State),
     {Packets, State3} = parse_incoming(Data, [], State2),
     LenMsg = erlang:length(Packets),
@@ -432,11 +431,11 @@ websocket_info(Info, State) ->
 websocket_close({_, ReasonCode, _Payload}, State) when is_integer(ReasonCode) ->
     websocket_close(ReasonCode, State);
 websocket_close(Reason, State) ->
-    ?SLOG(debug, #{msg => "websocket_closed", reason => Reason}),
+    ?TRACE("SOCKET", "websocket_closed", #{reason => Reason}),
     handle_info({sock_closed, Reason}, State).
 
 terminate(Reason, _Req, #state{channel = Channel}) ->
-    ?SLOG(debug, #{msg => "terminated", reason => Reason}),
+    ?TRACE("SOCKET", "websocket_terminated", #{reason => Reason}),
     emqx_channel:terminate(Reason, Channel);
 
 terminate(_Reason, _Req, _UnExpectedState) ->
@@ -480,7 +479,7 @@ handle_info({connack, ConnAck}, State) ->
     return(enqueue(ConnAck, State));
 
 handle_info({close, Reason}, State) ->
-    ?SLOG(debug, #{msg => "force_socket_close", reason => Reason}),
+    ?TRACE("SOCKET", "socket_force_closed", #{reason => Reason}),
     return(enqueue({close, Reason}, State));
 
 handle_info({event, connected}, State = #state{channel = Channel}) ->
@@ -550,7 +549,7 @@ check_limiter(Needs,
                 {ok, Limiter2} ->
                     WhenOk(Data, Msgs, State#state{limiter = Limiter2});
                 {pause, Time, Limiter2} ->
-                    ?SLOG(warning, #{msg => "pause time dueto rate limit",
+                    ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit",
                                      needs => Needs,
                                      time_in_ms => Time}),
 
@@ -586,7 +585,7 @@ retry_limiter(#state{limiter = Limiter} = State) ->
                             , limiter_timer = undefined
                             });
         {pause, Time, Limiter2} ->
-            ?SLOG(warning, #{msg => "pause time dueto rate limit",
+            ?SLOG(warning, #{msg => "pause_time_due_to_rate_limit",
                              types => Types,
                              time_in_ms => Time}),
 
@@ -663,7 +662,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
 
 handle_incoming(Packet, State = #state{listener = {Type, Listener}})
   when is_record(Packet, mqtt_packet) ->
-    ?SLOG(debug, #{msg => "RECV", packet => emqx_packet:format(Packet)}),
+    ?TRACE("WS-MQTT", "mqtt_packet_received", #{packet => Packet}),
     ok = inc_incoming_stats(Packet),
     NState = case emqx_pd:get_counter(incoming_pubs) >
                   get_active_n(Type, Listener) of
@@ -727,7 +726,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
                     ok = emqx_metrics:inc('delivery.dropped.too_large'),
                     ok = emqx_metrics:inc('delivery.dropped'),
                     <<>>;
-            Data -> ?SLOG(debug, #{msg => "SEND", packet => Packet}),
+            Data -> ?TRACE("WS-MQTT", "mqtt_packet_sent", #{packet => Packet}),
                     ok = inc_outgoing_stats(Packet),
                     Data
         catch

apps/emqx/test/emqx_banned_SUITE.erl

@@ -39,9 +39,13 @@ t_add_delete(_) ->
                      by = <<"banned suite">>,
                      reason = <<"test">>,
                      at = erlang:system_time(second),
-                     until = erlang:system_time(second) + 1000
+                     until = erlang:system_time(second) + 1
                     },
     {ok, _} = emqx_banned:create(Banned),
+    {error, {already_exist, Banned}} = emqx_banned:create(Banned),
+    ?assertEqual(1, emqx_banned:info(size)),
+    {error, {already_exist, Banned}} =
+        emqx_banned:create(Banned#banned{until = erlang:system_time(second) + 100}),
     ?assertEqual(1, emqx_banned:info(size)),
 
     ok = emqx_banned:delete({clientid, <<"TestClient">>}),
@@ -68,10 +72,14 @@ t_check(_) ->
                     username => <<"user">>,
                     peerhost => {127,0,0,1}
                    },
+    ClientInfo5 = #{},
+    ClientInfo6 = #{clientid => <<"client1">>},
     ?assert(emqx_banned:check(ClientInfo1)),
     ?assert(emqx_banned:check(ClientInfo2)),
     ?assert(emqx_banned:check(ClientInfo3)),
     ?assertNot(emqx_banned:check(ClientInfo4)),
+    ?assertNot(emqx_banned:check(ClientInfo5)),
+    ?assertNot(emqx_banned:check(ClientInfo6)),
     ok = emqx_banned:delete({clientid, <<"BannedClient">>}),
     ok = emqx_banned:delete({username, <<"BannedUser">>}),
     ok = emqx_banned:delete({peerhost, {192,168,0,1}}),
@@ -83,8 +91,10 @@ t_check(_) ->
 
 t_unused(_) ->
     {ok, Banned} = emqx_banned:start_link(),
-    {ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient">>},
-                                    until = erlang:system_time(second)}),
+    {ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient1">>},
+        until = erlang:system_time(second)}),
+    {ok, _} = emqx_banned:create(#banned{who = {clientid, <<"BannedClient2">>},
+        until = erlang:system_time(second) - 1}),
     ?assertEqual(ignored, gen_server:call(Banned, unexpected_req)),
     ?assertEqual(ok, gen_server:cast(Banned, unexpected_msg)),
     ?assertEqual(ok, Banned ! ok),

apps/emqx/test/emqx_trace_handler_SUITE.erl

@@ -39,32 +39,29 @@ end_per_suite(_Config) ->
     emqx_common_test_helpers:stop_apps([]).
 
 init_per_testcase(t_trace_clientid, Config) ->
+    init(),
     Config;
 init_per_testcase(_Case, Config) ->
-    ok = emqx_logger:set_log_level(debug),
     _ = [logger:remove_handler(Id) ||#{id := Id} <- emqx_trace_handler:running()],
+    init(),
     Config.
 
 end_per_testcase(_Case, _Config) ->
-    ok = emqx_logger:set_log_level(warning),
+    terminate(),
     ok.
 
 t_trace_clientid(_Config) ->
     %% Start tracing
-    emqx_logger:set_log_level(error),
-    {error, _} = emqx_trace_handler:install(clientid, <<"client">>, debug, "tmp/client.log"),
-    emqx_logger:set_log_level(debug),
     %% add list clientid
-    ok = emqx_trace_handler:install(clientid, "client", debug, "tmp/client.log"),
-    ok = emqx_trace_handler:install(clientid, <<"client2">>, all, "tmp/client2.log"),
-    ok = emqx_trace_handler:install(clientid, <<"client3">>, all, "tmp/client3.log"),
-    {error, {invalid_log_level, bad_level}} =
-        emqx_trace_handler:install(clientid, <<"client4">>, bad_level, "tmp/client4.log"),
+    ok = emqx_trace_handler:install("CLI-client1", clientid, "client", debug, "tmp/client.log"),
+    ok = emqx_trace_handler:install("CLI-client2", clientid, <<"client2">>, all, "tmp/client2.log"),
+    ok = emqx_trace_handler:install("CLI-client3", clientid, <<"client3">>, all, "tmp/client3.log"),
     {error, {handler_not_added, {file_error, ".", eisdir}}} =
         emqx_trace_handler:install(clientid, <<"client5">>, debug, "."),
-    ok = filesync(<<"client">>, clientid),
-    ok = filesync(<<"client2">>, clientid),
-    ok = filesync(<<"client3">>, clientid),
+    emqx_trace:check(),
+    ok = filesync(<<"CLI-client1">>, clientid),
+    ok = filesync(<<"CLI-client2">>, clientid),
+    ok = filesync(<<"CLI-client3">>, clientid),
 
     %% Verify the tracing file exits
     ?assert(filelib:is_regular("tmp/client.log")),
@@ -72,11 +69,11 @@ t_trace_clientid(_Config) ->
     ?assert(filelib:is_regular("tmp/client3.log")),
 
     %% Get current traces
-    ?assertMatch([#{type := clientid, filter := "client", name := <<"client">>,
+    ?assertMatch([#{type := clientid, filter := <<"client">>, name := <<"CLI-client1">>,
         level := debug, dst := "tmp/client.log"},
-        #{type := clientid, filter := "client2", name := <<"client2">>
+        #{type := clientid, filter := <<"client2">>, name := <<"CLI-client2">>
             , level := debug, dst := "tmp/client2.log"},
-        #{type := clientid, filter := "client3", name := <<"client3">>,
+        #{type := clientid, filter := <<"client3">>, name := <<"CLI-client3">>,
             level := debug, dst := "tmp/client3.log"}
     ], emqx_trace_handler:running()),
 
@@ -85,9 +82,9 @@ t_trace_clientid(_Config) ->
     emqtt:connect(T),
     emqtt:publish(T, <<"a/b/c">>, <<"hi">>),
     emqtt:ping(T),
-    ok = filesync(<<"client">>, clientid),
-    ok = filesync(<<"client2">>, clientid),
-    ok = filesync(<<"client3">>, clientid),
+    ok = filesync(<<"CLI-client1">>, clientid),
+    ok = filesync(<<"CLI-client2">>, clientid),
+    ok = filesync(<<"CLI-client3">>, clientid),
 
     %% Verify messages are logged to "tmp/client.log" but not "tmp/client2.log".
     {ok, Bin} = file:read_file("tmp/client.log"),
@@ -98,25 +95,24 @@ t_trace_clientid(_Config) ->
     ?assert(filelib:file_size("tmp/client2.log") == 0),
 
     %% Stop tracing
-    ok = emqx_trace_handler:uninstall(clientid, <<"client">>),
-    ok = emqx_trace_handler:uninstall(clientid, <<"client2">>),
-    ok = emqx_trace_handler:uninstall(clientid, <<"client3">>),
+    ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client1">>),
+    ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client2">>),
+    ok = emqx_trace_handler:uninstall(clientid, <<"CLI-client3">>),
 
     emqtt:disconnect(T),
     ?assertEqual([], emqx_trace_handler:running()).
 
 t_trace_clientid_utf8(_) ->
-    emqx_logger:set_log_level(debug),
-
     Utf8Id = <<"client 漢字編碼"/utf8>>,
-    ok = emqx_trace_handler:install(clientid, Utf8Id, debug, "tmp/client-utf8.log"),
+    ok = emqx_trace_handler:install("CLI-UTF8", clientid, Utf8Id, debug, "tmp/client-utf8.log"),
+    emqx_trace:check(),
     {ok, T} = emqtt:start_link([{clientid, Utf8Id}]),
     emqtt:connect(T),
     [begin emqtt:publish(T, <<"a/b/c">>, <<"hi">>) end|| _ <- lists:seq(1, 10)],
     emqtt:ping(T),
 
-    ok = filesync(Utf8Id, clientid),
-    ok = emqx_trace_handler:uninstall(clientid, Utf8Id),
+    ok = filesync("CLI-UTF8", clientid),
+    ok = emqx_trace_handler:uninstall(clientid, "CLI-UTF8"),
     emqtt:disconnect(T),
     ?assertEqual([], emqx_trace_handler:running()),
     ok.
@@ -126,11 +122,11 @@ t_trace_topic(_Config) ->
     emqtt:connect(T),
 
     %% Start tracing
-    emqx_logger:set_log_level(debug),
-    ok = emqx_trace_handler:install(topic, <<"x/#">>, all, "tmp/topic_trace_x.log"),
-    ok = emqx_trace_handler:install(topic, <<"y/#">>, all, "tmp/topic_trace_y.log"),
-    ok = filesync(<<"x/#">>, topic),
-    ok = filesync(<<"y/#">>, topic),
+    ok = emqx_trace_handler:install("CLI-TOPIC-1", topic, <<"x/#">>, all, "tmp/topic_trace_x.log"),
+    ok = emqx_trace_handler:install("CLI-TOPIC-2", topic, <<"y/#">>, all, "tmp/topic_trace_y.log"),
+    emqx_trace:check(),
+    ok = filesync("CLI-TOPIC-1", topic),
+    ok = filesync("CLI-TOPIC-2", topic),
 
     %% Verify the tracing file exits
     ?assert(filelib:is_regular("tmp/topic_trace_x.log")),
@@ -138,9 +134,9 @@ t_trace_topic(_Config) ->
 
     %% Get current traces
     ?assertMatch([#{type := topic, filter := <<"x/#">>,
-                    level := debug, dst := "tmp/topic_trace_x.log", name := <<"x/#">>},
+                    level := debug, dst := "tmp/topic_trace_x.log", name := <<"CLI-TOPIC-1">>},
                   #{type := topic, filter := <<"y/#">>,
-                      name := <<"y/#">>, level := debug, dst := "tmp/topic_trace_y.log"}
+                      name := <<"CLI-TOPIC-2">>, level := debug, dst := "tmp/topic_trace_y.log"}
                  ],
         emqx_trace_handler:running()),
 
@@ -149,8 +145,8 @@ t_trace_topic(_Config) ->
     emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
     emqtt:subscribe(T, <<"x/y/z">>),
     emqtt:unsubscribe(T, <<"x/y/z">>),
-    ok = filesync(<<"x/#">>, topic),
-    ok = filesync(<<"y/#">>, topic),
+    ok = filesync("CLI-TOPIC-1", topic),
+    ok = filesync("CLI-TOPIC-2", topic),
 
     {ok, Bin} = file:read_file("tmp/topic_trace_x.log"),
     ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@@ -161,8 +157,8 @@ t_trace_topic(_Config) ->
     ?assert(filelib:file_size("tmp/topic_trace_y.log") =:= 0),
 
     %% Stop tracing
-    ok = emqx_trace_handler:uninstall(topic, <<"x/#">>),
-    ok = emqx_trace_handler:uninstall(topic, <<"y/#">>),
+    ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-1">>),
+    ok = emqx_trace_handler:uninstall(topic, <<"CLI-TOPIC-2">>),
     {error, _Reason} = emqx_trace_handler:uninstall(topic, <<"z/#">>),
     ?assertEqual([], emqx_trace_handler:running()),
     emqtt:disconnect(T).
@@ -172,10 +168,12 @@ t_trace_ip_address(_Config) ->
     emqtt:connect(T),
 
     %% Start tracing
-    ok = emqx_trace_handler:install(ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"),
-    ok = emqx_trace_handler:install(ip_address, "192.168.1.1", all, "tmp/ip_trace_y.log"),
-    ok = filesync(<<"127.0.0.1">>, ip_address),
-    ok = filesync(<<"192.168.1.1">>, ip_address),
+    ok = emqx_trace_handler:install("CLI-IP-1", ip_address, "127.0.0.1", all, "tmp/ip_trace_x.log"),
+    ok = emqx_trace_handler:install("CLI-IP-2", ip_address,
+        "192.168.1.1", all, "tmp/ip_trace_y.log"),
+    emqx_trace:check(),
+    ok = filesync(<<"CLI-IP-1">>, ip_address),
+    ok = filesync(<<"CLI-IP-2">>, ip_address),
 
     %% Verify the tracing file exits
     ?assert(filelib:is_regular("tmp/ip_trace_x.log")),
@@ -183,10 +181,10 @@ t_trace_ip_address(_Config) ->
 
     %% Get current traces
     ?assertMatch([#{type := ip_address, filter := "127.0.0.1",
-                    name := <<"127.0.0.1">>,
+                    name := <<"CLI-IP-1">>,
                     level := debug, dst := "tmp/ip_trace_x.log"},
                   #{type := ip_address, filter := "192.168.1.1",
-                      name := <<"192.168.1.1">>,
+                      name := <<"CLI-IP-2">>,
                     level := debug, dst := "tmp/ip_trace_y.log"}
                  ],
         emqx_trace_handler:running()),
@@ -196,8 +194,8 @@ t_trace_ip_address(_Config) ->
     emqtt:publish(T, <<"x/y/z">>, <<"hi2">>),
     emqtt:subscribe(T, <<"x/y/z">>),
     emqtt:unsubscribe(T, <<"x/y/z">>),
-    ok = filesync(<<"127.0.0.1">>, ip_address),
-    ok = filesync(<<"192.168.1.1">>, ip_address),
+    ok = filesync(<<"CLI-IP-1">>, ip_address),
+    ok = filesync(<<"CLI-IP-2">>, ip_address),
 
     {ok, Bin} = file:read_file("tmp/ip_trace_x.log"),
     ?assertNotEqual(nomatch, binary:match(Bin, [<<"hi1">>])),
@@ -208,8 +206,8 @@ t_trace_ip_address(_Config) ->
     ?assert(filelib:file_size("tmp/ip_trace_y.log") =:= 0),
 
     %% Stop tracing
-    ok = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.1">>),
-    ok = emqx_trace_handler:uninstall(ip_address, <<"192.168.1.1">>),
+    ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-1">>),
+    ok = emqx_trace_handler:uninstall(ip_address, <<"CLI-IP-2">>),
     {error, _Reason} = emqx_trace_handler:uninstall(ip_address, <<"127.0.0.2">>),
     emqtt:disconnect(T),
     ?assertEqual([], emqx_trace_handler:running()).
@@ -221,7 +219,12 @@ filesync(Name, Type) ->
 
 %% sometime the handler process is not started yet.
 filesync(_Name, _Type, 0) -> ok;
-filesync(Name, Type, Retry) ->
+filesync(Name0, Type, Retry) ->
+    Name =
+        case is_binary(Name0) of
+            true -> Name0;
+            false -> list_to_binary(Name0)
+        end,
     try
         Handler = binary_to_atom(<<"trace_",
             (atom_to_binary(Type))/binary, "_", Name/binary>>),
@@ -231,3 +234,9 @@ filesync(Name, Type, Retry) ->
         ct:sleep(100),
         filesync(Name, Type, Retry - 1)
     end.
+
+init() ->
+    emqx_trace:start_link().
+
+terminate() ->
+    catch ok = gen_server:stop(emqx_trace, normal, 5000).

apps/emqx_authn/src/emqx_authn_api.erl

@@ -726,7 +726,7 @@ with_chain(ListenerID, Fun) ->
 create_authenticator(ConfKeyPath, ChainName, Config) ->
     case update_config(ConfKeyPath, {create_authenticator, ChainName, Config}) of
         {ok, #{post_config_update := #{emqx_authentication := #{id := ID}},
-            raw_config := AuthenticatorsConfig}} ->
+               raw_config := AuthenticatorsConfig}} ->
             {ok, AuthenticatorConfig} = find_config(ID, AuthenticatorsConfig),
             {200, maps:put(id, ID, convert_certs(fill_defaults(AuthenticatorConfig)))};
         {error, {_PrePostConfigUpdate, emqx_authentication, Reason}} ->
@@ -872,7 +872,7 @@ fill_defaults(Configs) when is_list(Configs) ->
 fill_defaults(Config) ->
     emqx_authn:check_config(Config, #{only_fill_defaults => true}).
 
-convert_certs(#{ssl := SSLOpts} = Config) ->
+convert_certs(#{ssl := #{enable := true} = SSLOpts} = Config) ->
     NSSLOpts = lists:foldl(fun(K, Acc) ->
                                case maps:get(K, Acc, undefined) of
                                    undefined -> Acc;
@@ -979,7 +979,7 @@ authenticator_examples() ->
                 mechanism => <<"password-based">>,
                 backend => <<"http">>,
                 method => <<"post">>,
-                url => <<"http://127.0.0.2:8080">>,
+                url => <<"http://127.0.0.1:18083">>,
                 headers => #{
                     <<"content-type">> => <<"application/json">>
                 },

apps/emqx_authn/src/simple_authn/emqx_authn_pgsql.erl

@@ -106,7 +106,7 @@ authenticate(#{password := Password} = Credential,
                resource_id := ResourceId,
                password_hash_algorithm := Algorithm}) ->
     Params = emqx_authn_utils:replace_placeholders(PlaceHolders, Credential),
-    case emqx_resource:query(ResourceId, {sql, Query, Params}) of
+    case emqx_resource:query(ResourceId, {prepared_query, ResourceId, Query, Params}) of
         {ok, _Columns, []} -> ignore;
         {ok, Columns, [Row | _]} ->
             NColumns = [Name || #column{name = Name} <- Columns],

apps/emqx_authn/test/emqx_authn_api_SUITE.erl

@@ -67,7 +67,7 @@ init_per_suite(Config) ->
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_authn, emqx_dashboard]),
+    emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_authn]),
     ok.
 
 set_special_configs(emqx_dashboard) ->

apps/emqx_authn/test/emqx_authn_http_SUITE.erl

@@ -153,9 +153,8 @@ t_destroy(_Config) ->
       ?GLOBAL),
 
     % Authenticator should not be usable anymore
-    ?assertException(
-       error,
-       _,
+    ?assertMatch(
+       ignore,
        emqx_authn_http:authenticate(
          Credentials,
          State)).

apps/emqx_authn/test/emqx_authn_mongo_SUITE.erl

@@ -146,9 +146,8 @@ t_destroy(_Config) ->
       ?GLOBAL),
 
     % Authenticator should not be usable anymore
-    ?assertException(
-       error,
-       _,
+    ?assertMatch(
+       ignore,
        emqx_authn_mongodb:authenticate(
          #{username => <<"plain">>,
            password => <<"plain">>

apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl

@@ -91,7 +91,7 @@ t_create_invalid_server_name(_Config) ->
        create_mongo_auth_with_ssl_opts(
          #{<<"server_name_indication">> => <<"authn-server-unknown-host">>,
            <<"verify">> => <<"verify_peer">>}),
-       fun({ok, _}, Trace) ->
+       fun({error, _}, Trace) ->
                ?assertEqual(
                   [failed],
                   ?projection(
@@ -109,7 +109,7 @@ t_create_invalid_version(_Config) ->
          #{<<"server_name_indication">> => <<"authn-server">>,
            <<"verify">> => <<"verify_peer">>,
            <<"versions">> => [<<"tlsv1.1">>]}),
-       fun({ok, _}, Trace) ->
+       fun({error, _}, Trace) ->
                ?assertEqual(
                   [failed],
                   ?projection(
@@ -118,7 +118,7 @@ t_create_invalid_version(_Config) ->
        end).
 
 
-%% docker-compose-mongo-single-tls.yaml: 
+%% docker-compose-mongo-single-tls.yaml:
 %% --setParameter opensslCipherConfig='HIGH:!EXPORT:!aNULL:!DHE:!kDHE@STRENGTH'
 
 t_invalid_ciphers(_Config) ->
@@ -128,7 +128,7 @@ t_invalid_ciphers(_Config) ->
            <<"verify">> => <<"verify_peer">>,
            <<"versions">> => [<<"tlsv1.2">>],
            <<"ciphers">> => [<<"DHE-RSA-AES256-GCM-SHA384">>]}),
-       fun({ok, _}, Trace) ->
+       fun({error, _}, Trace) ->
                ?assertEqual(
                   [failed],
                   ?projection(

apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl

@@ -157,9 +157,8 @@ t_destroy(_Config) ->
       ?GLOBAL),
 
     % Authenticator should not be usable anymore
-    ?assertException(
-       error,
-       _,
+    ?assertMatch(
+       ignore,
        emqx_authn_mysql:authenticate(
          #{username => <<"plain">>,
            password => <<"plain">>

apps/emqx_authn/test/emqx_authn_pgsql_SUITE.erl

@@ -158,9 +158,8 @@ t_destroy(_Config) ->
       ?GLOBAL),
 
     % Authenticator should not be usable anymore
-    ?assertException(
-       error,
-       _,
+    ?assertMatch(
+       ignore,
        emqx_authn_pgsql:authenticate(
          #{username => <<"plain">>,
            password => <<"plain">>
@@ -440,12 +439,12 @@ create_user(Values) ->
 q(Sql) ->
     emqx_resource:query(
       ?PGSQL_RESOURCE,
-      {sql, Sql}).
+      {query, Sql}).
 
 q(Sql, Params) ->
     emqx_resource:query(
       ?PGSQL_RESOURCE,
-      {sql, Sql, Params}).
+      {query, Sql, Params}).
 
 drop_seeds() ->
     {ok, _, _} = q("DROP TABLE IF EXISTS users"),

apps/emqx_authn/test/emqx_authn_redis_SUITE.erl

@@ -164,9 +164,8 @@ t_destroy(_Config) ->
       ?GLOBAL),
 
     % Authenticator should not be usable anymore
-    ?assertException(
-       error,
-       _,
+    ?assertMatch(
+       ignore,
        emqx_authn_redis:authenticate(
          #{username => <<"plain">>,
            password => <<"plain">>

apps/emqx_authz/src/emqx_authz.erl

@@ -31,9 +31,7 @@
         , lookup/0
         , lookup/1
         , move/2
-        , move/3
         , update/2
-        , update/3
         , authorize/5
         ]).
 
@@ -112,28 +110,19 @@ lookup(Type) ->
     {Source, _Front, _Rear} = take(Type),
     Source.
 
-move(Type, Cmd) ->
-    move(Type, Cmd, #{}).
-
-move(Type, #{<<"before">> := Before}, Opts) ->
-    emqx:update_config( ?CONF_KEY_PATH
-                      , {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))}, Opts);
-move(Type, #{<<"after">> := After}, Opts) ->
-    emqx:update_config( ?CONF_KEY_PATH
-                      , {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))}, Opts);
-move(Type, Position, Opts) ->
-    emqx:update_config( ?CONF_KEY_PATH
-                      , {?CMD_MOVE, type(Type), Position}, Opts).
-
+move(Type, #{<<"before">> := Before}) ->
+    emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_BEFORE(type(Before))});
+move(Type, #{<<"after">> := After}) ->
+    emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), ?CMD_MOVE_AFTER(type(After))});
+move(Type, Position) ->
+    emqx_authz_utils:update_config(?CONF_KEY_PATH, {?CMD_MOVE, type(Type), Position}).
+
+update({?CMD_REPLACE, Type}, Sources) ->
+    emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources});
+update({?CMD_DELETE, Type}, Sources) ->
+    emqx_authz_utils:update_config(?CONF_KEY_PATH, {{?CMD_DELETE, type(Type)}, Sources});
 update(Cmd, Sources) ->
-    update(Cmd, Sources, #{}).
-
-update({?CMD_REPLACE, Type}, Sources, Opts) ->
-    emqx:update_config(?CONF_KEY_PATH, {{?CMD_REPLACE, type(Type)}, Sources}, Opts);
-update({?CMD_DELETE, Type}, Sources, Opts) ->
-    emqx:update_config(?CONF_KEY_PATH, {{?CMD_DELETE, type(Type)}, Sources}, Opts);
-update(Cmd, Sources, Opts) ->
-    emqx:update_config(?CONF_KEY_PATH, {Cmd, Sources}, Opts).
+    emqx_authz_utils:update_config(?CONF_KEY_PATH, {Cmd, Sources}).
 
 do_update({?CMD_MOVE, Type, ?CMD_MOVE_TOP}, Conf) when is_list(Conf) ->
     {Source, Front, Rear} = take(Type, Conf),
@@ -167,7 +156,8 @@ do_update({{?CMD_REPLACE, Type}, #{<<"enable">> := Enable} = Source}, Conf)
             NConf;
         {error, _} = Error -> Error
     end;
-do_update({{?CMD_REPLACE, Type}, Source}, Conf) when is_map(Source), is_list(Conf) ->
+do_update({{?CMD_REPLACE, Type}, Source}, Conf)
+  when is_map(Source), is_list(Conf) ->
     {_Old, Front, Rear} = take(Type, Conf),
     NConf = Front ++ [Source | Rear],
     ok = check_dup_types(NConf),

apps/emqx_authz/src/emqx_authz_api_schema.erl

@@ -182,8 +182,7 @@ definitions() ->
             mongo_type => #{type => string,
                             enum => [<<"rs">>],
                             example => <<"rs">>},
-            servers => #{type => array,
-                         items => #{type => string,example => <<"127.0.0.1:27017">>}},
+            servers => #{type => string, example => <<"127.0.0.1:27017, 127.0.0.2:27017">>},
             replica_set_name => #{type => string},
             pool_size => #{type => integer},
             username => #{type => string},
@@ -240,8 +239,7 @@ definitions() ->
             mongo_type => #{type => string,
                             enum => [<<"sharded">>],
                             example => <<"sharded">>},
-            servers => #{type => array,
-                         items => #{type => string,example => <<"127.0.0.1:27017">>}},
+            servers => #{type => string,example => <<"127.0.0.1:27017, 127.0.0.2:27017">>},
             pool_size => #{type => integer},
             username => #{type => string},
             password => #{type => string},
@@ -401,8 +399,7 @@ definitions() ->
                 type => string,
                 example => <<"HGETALL mqtt_authz">>
             },
-            servers => #{type => array,
-                         items => #{type => string,example => <<"127.0.0.1:3306">>}},
+            servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>},
             redis_type => #{type => string,
                             enum => [<<"sentinel">>],
                             example => <<"sentinel">>},
@@ -438,8 +435,7 @@ definitions() ->
                 type => string,
                 example => <<"HGETALL mqtt_authz">>
             },
-            servers => #{type => array,
-                         items => #{type => string, example => <<"127.0.0.1:3306">>}},
+            servers => #{type => string, example => <<"127.0.0.1:6379, 127.0.0.2:6379">>},
             redis_type => #{type => string,
                             enum => [<<"cluster">>],
                             example => <<"cluster">>},

apps/emqx_authz/src/emqx_authz_api_settings.erl

@@ -54,8 +54,9 @@ settings(get, _Params) ->
 settings(put, #{body := #{<<"no_match">> := NoMatch,
                           <<"deny_action">> := DenyAction,
                           <<"cache">> := Cache}}) ->
-    {ok, _} = emqx:update_config([authorization, no_match], NoMatch),
-    {ok, _} = emqx:update_config([authorization, deny_action], DenyAction),
-    {ok, _} = emqx:update_config([authorization, cache], Cache),
+    {ok, _} = emqx_authz_utils:update_config([authorization, no_match], NoMatch),
+    {ok, _} = emqx_authz_utils:update_config(
+                [authorization, deny_action], DenyAction),
+    {ok, _} = emqx_authz_utils:update_config([authorization, cache], Cache),
     ok = emqx_authz_cache:drain_cache(),
     {200, authorization_settings()}.

apps/emqx_authz/src/emqx_authz_http.erl

@@ -46,10 +46,10 @@ init(Source) ->
     end.
 
 destroy(#{annotations := #{id := Id}}) ->
-    ok = emqx_resource:remove(Id).
+    ok = emqx_resource:remove_local(Id).
 
 dry_run(Source) ->
-    emqx_resource:create_dry_run(emqx_connector_http, Source).
+    emqx_resource:create_dry_run_local(emqx_connector_http, Source).
 
 authorize(Client, PubSub, Topic,
             #{type := http,

apps/emqx_authz/src/emqx_authz_mongodb.erl

@@ -46,10 +46,10 @@ init(Source) ->
     end.
 
 dry_run(Source) ->
-    emqx_resource:create_dry_run(emqx_connector_mongo, Source).
+    emqx_resource:create_dry_run_local(emqx_connector_mongo, Source).
 
 destroy(#{annotations := #{id := Id}}) ->
-    ok = emqx_resource:remove(Id).
+    ok = emqx_resource:remove_local(Id).
 
 authorize(Client, PubSub, Topic,
             #{collection := Collection,

apps/emqx_authz/src/emqx_authz_mysql.erl

@@ -48,10 +48,10 @@ init(#{query := SQL} = Source) ->
     end.
 
 dry_run(Source) ->
-    emqx_resource:create_dry_run(emqx_connector_mysql, Source).
+    emqx_resource:create_dry_run_local(emqx_connector_mysql, Source).
 
 destroy(#{annotations := #{id := Id}}) ->
-    ok = emqx_resource:remove(Id).
+    ok = emqx_resource:remove_local(Id).
 
 authorize(Client, PubSub, Topic,
             #{annotations := #{id := ResourceID,

apps/emqx_authz/src/emqx_authz_postgresql.erl

@@ -48,10 +48,10 @@ init(#{query := SQL} = Source) ->
     end.
 
 destroy(#{annotations := #{id := Id}}) ->
-    ok = emqx_resource:remove(Id).
+    ok = emqx_resource:remove_local(Id).
 
 dry_run(Source) ->
-    emqx_resource:create_dry_run(emqx_connector_pgsql, Source).
+    emqx_resource:create_dry_run_local(emqx_connector_pgsql, Source).
 
 parse_query(Sql) ->
     case re:run(Sql, ?RE_PLACEHOLDER, [global, {capture, all, list}]) of
@@ -73,7 +73,7 @@ authorize(Client, PubSub, Topic,
                                query := {Query, Params}
                               }
              }) ->
-    case emqx_resource:query(ResourceID, {sql, Query, replvar(Params, Client)}) of
+    case emqx_resource:query(ResourceID, {prepared_query, ResourceID, Query, replvar(Params, Client)}) of
         {ok, _Columns, []} -> nomatch;
         {ok, Columns, Rows} ->
             do_authorize(Client, PubSub, Topic, Columns, Rows);

apps/emqx_authz/src/emqx_authz_redis.erl

@@ -46,10 +46,10 @@ init(Source) ->
     end.
 
 destroy(#{annotations := #{id := Id}}) ->
-    ok = emqx_resource:remove(Id).
+    ok = emqx_resource:remove_local(Id).
 
 dry_run(Source) ->
-    emqx_resource:create_dry_run(emqx_connector_redis, Source).
+    emqx_resource:create_dry_run_local(emqx_connector_redis, Source).
 
 authorize(Client, PubSub, Topic,
             #{cmd := CMD,

apps/emqx_authz/src/emqx_authz_utils.erl

@@ -18,9 +18,11 @@
 
 -include_lib("emqx/include/emqx_placeholder.hrl").
 
--export([cleanup_resources/0,
-         make_resource_id/1,
-         create_resource/2]).
+-export([ cleanup_resources/0
+        , make_resource_id/1
+        , create_resource/2
+        , update_config/2
+        ]).
 
 -define(RESOURCE_GROUP, <<"emqx_authz">>).
 
@@ -30,7 +32,7 @@
 
 create_resource(Module, Config) ->
     ResourceID = make_resource_id(Module),
-    case emqx_resource:create(ResourceID, Module, Config) of
+    case emqx_resource:create_local(ResourceID, Module, Config) of
         {ok, already_created} -> {ok, ResourceID};
         {ok, _} -> {ok, ResourceID};
         {error, Reason} -> {error, Reason}
@@ -38,13 +40,17 @@ create_resource(Module, Config) ->
 
 cleanup_resources() ->
     lists:foreach(
-      fun emqx_resource:remove/1,
+      fun emqx_resource:remove_local/1,
       emqx_resource:list_group_instances(?RESOURCE_GROUP)).
 
 make_resource_id(Name) ->
     NameBin = bin(Name),
     emqx_resource:generate_id(?RESOURCE_GROUP, NameBin).
 
+update_config(Path, ConfigRequest) ->
+    emqx_conf:update(Path, ConfigRequest, #{rawconf_with_defaults => true,
+                                            override_to => cluster}).
+
 %%------------------------------------------------------------------------------
 %% Internal functions
 %%------------------------------------------------------------------------------

apps/emqx_authz/test/emqx_authz_SUITE.erl

@@ -31,10 +31,9 @@ groups() ->
 
 init_per_suite(Config) ->
     meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
-    meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end),
-    meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end),
-    meck:expect(emqx_resource, remove, fun(_) -> ok end),
-    meck:expect(emqx_resource, create_dry_run, fun(_, _) -> ok end),
+    meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, meck_data} end),
+    meck:expect(emqx_resource, remove_local, fun(_) -> ok end),
+    meck:expect(emqx_resource, create_dry_run_local, fun(_, _) -> ok end),
 
     ok = emqx_common_test_helpers:start_apps(
            [emqx_connector, emqx_conf, emqx_authz],
@@ -105,6 +104,7 @@ set_special_configs(_App) ->
                    <<"query">> => <<"abcb">>
                   }).
 -define(SOURCE5, #{<<"type">> => <<"redis">>,
+                   <<"redis_type">> => <<"single">>,
                    <<"enable">> => true,
                    <<"server">> => <<"127.0.0.1:27017">>,
                    <<"pool_size">> => 1,

apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl

@@ -26,6 +26,7 @@
 -define(HOST, "http://127.0.0.1:18083/").
 -define(API_VERSION, "v5").
 -define(BASE_PATH, "api").
+-define(MONGO_SINGLE_HOST, "mongo:27017").
 
 -define(SOURCE1, #{<<"type">> => <<"http">>,
                    <<"enable">> => true,
@@ -38,8 +39,8 @@
                   }).
 -define(SOURCE2, #{<<"type">> => <<"mongodb">>,
                    <<"enable">> => true,
-                   <<"mongo_type">> => <<"sharded">>,
-                   <<"servers">> => <<"127.0.0.1:27017,192.168.0.1:27017">>,
+                   <<"mongo_type">> => <<"single">>,
+                   <<"server">> => <<?MONGO_SINGLE_HOST>>,
                    <<"pool_size">> => 1,
                    <<"database">> => <<"mqtt">>,
                    <<"ssl">> => #{<<"enable">> => false},
@@ -48,7 +49,7 @@
                   }).
 -define(SOURCE3, #{<<"type">> => <<"mysql">>,
                    <<"enable">> => true,
-                   <<"server">> => <<"127.0.0.1:3306">>,
+                   <<"server">> => <<"mysql:3306">>,
                    <<"pool_size">> => 1,
                    <<"database">> => <<"mqtt">>,
                    <<"username">> => <<"xx">>,
@@ -59,7 +60,7 @@
                   }).
 -define(SOURCE4, #{<<"type">> => <<"postgresql">>,
                    <<"enable">> => true,
-                   <<"server">> => <<"127.0.0.1:5432">>,
+                   <<"server">> => <<"pgsql:5432">>,
                    <<"pool_size">> => 1,
                    <<"database">> => <<"mqtt">>,
                    <<"username">> => <<"xx">>,
@@ -70,9 +71,7 @@
                   }).
 -define(SOURCE5, #{<<"type">> => <<"redis">>,
                    <<"enable">> => true,
-                   <<"servers">> => [<<"127.0.0.1:6379">>,
-                                     <<"127.0.0.1:6380">>
-                                    ],
+                   <<"servers">> => <<"redis:6379,127.0.0.1:6380">>,
                    <<"pool_size">> => 1,
                    <<"database">> => 0,
                    <<"password">> => <<"ee">>,
@@ -98,14 +97,14 @@ groups() ->
 
 init_per_suite(Config) ->
     meck:new(emqx_resource, [non_strict, passthrough, no_history, no_link]),
-    meck:expect(emqx_resource, create, fun(_, _, _) -> {ok, meck_data} end),
-    meck:expect(emqx_resource, create_dry_run,
+    meck:expect(emqx_resource, create_local, fun(_, _, _) -> {ok, meck_data} end),
+    meck:expect(emqx_resource, create_dry_run_local,
                 fun(emqx_connector_mysql, _) -> ok;
+                   (emqx_connector_mongo, _) -> ok;
                    (T, C) -> meck:passthrough([T, C])
                 end),
-    meck:expect(emqx_resource, update, fun(_, _, _, _) -> {ok, meck_data} end),
-    meck:expect(emqx_resource, health_check, fun(_) -> ok end),
-    meck:expect(emqx_resource, remove, fun(_) -> ok end ),
+    meck:expect(emqx_resource, health_check, fun(St) -> {ok, St} end),
+    meck:expect(emqx_resource, remove_local, fun(_) -> ok end ),
 
     ok = emqx_common_test_helpers:start_apps(
            [emqx_conf, emqx_authz, emqx_dashboard],

apps/emqx_authz/test/emqx_authz_http_SUITE.erl

@@ -343,17 +343,16 @@ t_create_replace(_Config) ->
                    listener => {tcp, default}
                   },
 
-    %% Bad URL
+    %% Create with valid URL
     ok = setup_handler_and_config(
            fun(Req0, State) ->
                    Req = cowboy_req:reply(200, Req0),
                    {ok, Req, State}
            end,
-           #{<<"base_url">> => <<"http://127.0.0.1:33331/authz">>}),
-
+           #{<<"base_url">> => <<"http://127.0.0.1:33333/authz">>}),
 
     ?assertEqual(
-        deny,
+        allow,
         emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
 
     %% Changing to other bad config does not work
@@ -366,14 +365,14 @@ t_create_replace(_Config) ->
         emqx_authz:update({?CMD_REPLACE, http}, BadConfig)),
 
     ?assertEqual(
-        deny,
+        allow,
         emqx_access_control:authorize(ClientInfo, publish, <<"t">>)),
 
     %% Changing to valid config
     OkConfig = maps:merge(
                   raw_http_authz_config(),
                   #{<<"base_url">> => <<"http://127.0.0.1:33333/authz">>}),
-    
+
     ?assertMatch(
         {ok, _},
         emqx_authz:update({?CMD_REPLACE, http}, OkConfig)),

apps/emqx_authz/test/emqx_authz_postgresql_SUITE.erl

@@ -228,12 +228,12 @@ raw_pgsql_authz_config() ->
 q(Sql) ->
     emqx_resource:query(
       ?PGSQL_RESOURCE,
-      {sql, Sql}).
+      {query, Sql}).
 
 insert(Sql, Params) ->
     {ok, _} = emqx_resource:query(
                 ?PGSQL_RESOURCE,
-                {sql, Sql, Params}),
+                {query, Sql, Params}),
     ok.
 
 init_table() ->

apps/emqx_auto_subscribe/src/emqx_auto_subscribe.erl

@@ -80,8 +80,15 @@ format(Rule = #{topic := Topic}) when is_map(Rule) ->
     }.
 
 update_(Topics) when length(Topics) =< ?MAX_AUTO_SUBSCRIBE ->
-    {ok, _} = emqx:update_config([auto_subscribe, topics], Topics),
-    update_hook();
+    case emqx_conf:update([auto_subscribe, topics],
+                          Topics,
+                          #{rawconf_with_defaults => true, override_to => cluster}) of
+        {ok, #{raw_config := NewTopics}} ->
+            ok = update_hook(),
+            {ok, NewTopics};
+        {error, Reason} ->
+            {error, Reason}
+    end;
 update_(_Topics) ->
     {error, quota_exceeded}.
 

apps/emqx_auto_subscribe/src/emqx_auto_subscribe_api.erl

@@ -22,6 +22,7 @@
 
 -export([auto_subscribe/2]).
 
+-define(INTERNAL_ERROR, 'INTERNAL_ERROR').
 -define(EXCEED_LIMIT, 'EXCEED_LIMIT').
 -define(BAD_REQUEST, 'BAD_REQUEST').
 
@@ -90,6 +91,9 @@ auto_subscribe(put, #{body := Params}) ->
             Message = list_to_binary(io_lib:format("Max auto subscribe topic count is  ~p",
                                         [emqx_auto_subscribe:max_limit()])),
             {409, #{code => ?EXCEED_LIMIT, message => Message}};
-        ok ->
-            {200, emqx_auto_subscribe:list()}
+        {error, Reason} ->
+            Message = list_to_binary(io_lib:format("Update config failed ~p", [Reason])),
+            {500, #{code => ?INTERNAL_ERROR, message => Message}};
+        {ok, NewTopics} ->
+            {200, NewTopics}
     end.

apps/emqx_auto_subscribe/test/emqx_auto_subscribe_SUITE.erl

@@ -85,7 +85,7 @@ init_per_suite(Config) ->
                 }
             ]
         }">>),
-    emqx_common_test_helpers:start_apps([emqx_dashboard, ?APP], fun set_special_configs/1),
+    emqx_common_test_helpers:start_apps([emqx_dashboard, emqx_conf, ?APP], fun set_special_configs/1),
     Config.
 
 set_special_configs(emqx_dashboard) ->
@@ -113,15 +113,17 @@ topic_config(T) ->
 
 end_per_suite(_) ->
     application:unload(emqx_management),
+    application:unload(emqx_conf),
     application:unload(?APP),
     meck:unload(emqx_resource),
     meck:unload(emqx_schema),
-    emqx_common_test_helpers:stop_apps([emqx_dashboard, ?APP]).
+    emqx_common_test_helpers:stop_apps([emqx_dashboard, emqx_conf, ?APP]).
 
 t_auto_subscribe(_) ->
+    emqx_auto_subscribe:update([#{<<"topic">> => Topic} || Topic <- ?TOPICS]),
     {ok, Client} = emqtt:start_link(#{username => ?CLIENT_USERNAME, clientid => ?CLIENT_ID}),
     {ok, _} = emqtt:connect(Client),
-    timer:sleep(100),
+    timer:sleep(200),
     ?assertEqual(check_subs(length(?TOPICS)), ok),
     emqtt:disconnect(Client),
     ok.
@@ -148,6 +150,7 @@ t_update(_) ->
 
 check_subs(Count) ->
     Subs = ets:tab2list(emqx_suboption),
+    ct:pal("--->  ~p ~p ~n", [Subs, Count]),
     ?assert(length(Subs) >= Count),
     check_subs((Subs), ?ENSURE_TOPICS).
 

apps/emqx_bridge/etc/emqx_bridge.conf

@@ -34,8 +34,8 @@
 #    direction = egress
 #    ## NOTE: we cannot use placehodler variables in the `scheme://host:port` part of the url
 #    url = "http://localhost:9901/messages/${topic}"
-#    request_timeout = "30s"
-#    connect_timeout = "30s"
+#    request_timeout = "15s"
+#    connect_timeout = "15s"
 #    max_retries = 3
 #    retry_interval = "10s"
 #    pool_type = "random"

apps/emqx_bridge/src/emqx_bridge.erl

@@ -35,15 +35,19 @@
         ]).
 
 -export([ load/0
+        , lookup/1
         , lookup/2
         , lookup/3
         , list/0
         , list_bridges_by_connector/1
+        , create/2
         , create/3
         , recreate/2
         , recreate/3
         , create_dry_run/2
+        , remove/1
         , remove/3
+        , update/2
         , update/3
         , start/2
         , stop/2
@@ -80,17 +84,36 @@ unload_hook() ->
 on_message_publish(Message = #message{topic = Topic, flags = Flags}) ->
     case maps:get(sys, Flags, false) of
         false ->
-            lists:foreach(fun (Id) ->
-                    send_message(Id, emqx_rule_events:eventmsg_publish(Message))
-                end, get_matched_bridges(Topic));
+            Msg = emqx_rule_events:eventmsg_publish(Message),
+            send_to_matched_egress_bridges(Topic, Msg);
         true -> ok
     end,
     {ok, Message}.
 
+send_to_matched_egress_bridges(Topic, Msg) ->
+    lists:foreach(fun (Id) ->
+        try send_message(Id, Msg) of
+            ok -> ok;
+            Error -> ?SLOG(error, #{msg => "send_message_to_bridge_failed",
+                        bridge => Id, error => Error})
+        catch Err:Reason:ST ->
+            ?SLOG(error, #{msg => "send_message_to_bridge_crash",
+                bridge => Id, error => Err, reason => Reason,
+                stacktrace => ST})
+        end
+    end, get_matched_bridges(Topic)).
+
 send_message(BridgeId, Message) ->
     {BridgeType, BridgeName} = parse_bridge_id(BridgeId),
     ResId = emqx_bridge:resource_id(BridgeType, BridgeName),
-    emqx_resource:query(ResId, {send_message, Message}).
+    case emqx:get_config([bridges, BridgeType, BridgeName], not_found) of
+        not_found ->
+            {error, {bridge_not_found, BridgeId}};
+        #{enable := true} ->
+            emqx_resource:query(ResId, {send_message, Message});
+        #{enable := false} ->
+            {error, {bridge_stopped, BridgeId}}
+    end.
 
 config_key_path() ->
     [bridges].
@@ -169,6 +192,10 @@ list_bridges_by_connector(ConnectorId) ->
     [B || B = #{raw_config := #{<<"connector">> := Id}} <- list(),
          ConnectorId =:= Id].
 
+lookup(Id) ->
+    {Type, Name} = parse_bridge_id(Id),
+    lookup(Type, Name).
+
 lookup(Type, Name) ->
     RawConf = emqx:get_raw_config([bridges, Type, Name], #{}),
     lookup(Type, Name, RawConf).
@@ -188,16 +215,24 @@ stop(Type, Name) ->
 restart(Type, Name) ->
     emqx_resource:restart(resource_id(Type, Name)).
 
+create(BridgeId, Conf) ->
+    {BridgeType, BridgeName} = parse_bridge_id(BridgeId),
+    create(BridgeType, BridgeName, Conf).
+
 create(Type, Name, Conf) ->
     ?SLOG(info, #{msg => "create bridge", type => Type, name => Name,
         config => Conf}),
     case emqx_resource:create_local(resource_id(Type, Name), emqx_bridge:resource_type(Type),
-            parse_confs(Type, Name, Conf), #{force_create => true}) of
+            parse_confs(Type, Name, Conf), #{async_create => true}) of
         {ok, already_created} -> maybe_disable_bridge(Type, Name, Conf);
         {ok, _} -> maybe_disable_bridge(Type, Name, Conf);
         {error, Reason} -> {error, Reason}
     end.
 
+update(BridgeId, {OldConf, Conf}) ->
+    {BridgeType, BridgeName} = parse_bridge_id(BridgeId),
+    update(BridgeType, BridgeName, {OldConf, Conf}).
+
 update(Type, Name, {OldConf, Conf}) ->
     %% TODO: sometimes its not necessary to restart the bridge connection.
     %%
@@ -214,23 +249,27 @@ update(Type, Name, {OldConf, Conf}) ->
             case recreate(Type, Name, Conf) of
                 {ok, _} -> maybe_disable_bridge(Type, Name, Conf);
                 {error, not_found} ->
-                    ?SLOG(warning, #{ msg => "updating a non-exist bridge, create a new one"
+                    ?SLOG(warning, #{ msg => "updating_a_non-exist_bridge_need_create_a_new_one"
                                     , type => Type, name => Name, config => Conf}),
                     create(Type, Name, Conf);
-                {error, Reason} -> {update_bridge_failed, Reason}
+                {error, Reason} -> {error, {update_bridge_failed, Reason}}
             end;
         true ->
             %% we don't need to recreate the bridge if this config change is only to
             %% toggole the config 'bridge.{type}.{name}.enable'
-            ok
+            case maps:get(enable, Conf, true) of
+                false -> stop(Type, Name);
+                true -> start(Type, Name)
+            end
     end.
 
 recreate(Type, Name) ->
-    recreate(Type, Name, emqx:get_raw_config([bridges, Type, Name])).
+    recreate(Type, Name, emqx:get_config([bridges, Type, Name])).
 
 recreate(Type, Name, Conf) ->
     emqx_resource:recreate_local(resource_id(Type, Name),
-        emqx_bridge:resource_type(Type), parse_confs(Type, Name, Conf), []).
+        emqx_bridge:resource_type(Type), parse_confs(Type, Name, Conf),
+        #{async_create => true}).
 
 create_dry_run(Type, Conf) ->
     Conf0 = Conf#{<<"ingress">> => #{<<"remote_topic">> => <<"t">>}},
@@ -241,8 +280,12 @@ create_dry_run(Type, Conf) ->
             Error
     end.
 
+remove(BridgeId) ->
+    {BridgeType, BridgeName} = parse_bridge_id(BridgeId),
+    remove(BridgeType, BridgeName, #{}).
+
 remove(Type, Name, _Conf) ->
-    ?SLOG(info, #{msg => "remove bridge", type => Type, name => Name}),
+    ?SLOG(info, #{msg => "remove_bridge", type => Type, name => Name}),
     case emqx_resource:remove_local(resource_id(Type, Name)) of
         ok -> ok;
         {error, not_found} -> ok;
@@ -276,6 +319,8 @@ get_matched_bridges(Topic) ->
         end, Acc0, Conf)
     end, [], Bridges).
 
+get_matched_bridge_id(#{enable := false}, _Topic, _BType, _BName, Acc) ->
+    Acc;
 get_matched_bridge_id(#{local_topic := Filter}, Topic, BType, BName, Acc) ->
     case emqx_topic:match(Topic, Filter) of
         true -> [bridge_id(BType, BName) | Acc];
@@ -306,21 +351,21 @@ parse_confs(Type, Name, #{connector := ConnId, direction := Direction} = Conf)
         {Type, ConnName} ->
             ConnectorConfs = emqx:get_config([connectors, Type, ConnName]),
             make_resource_confs(Direction, ConnectorConfs,
-                maps:without([connector, direction], Conf), Name);
+                maps:without([connector, direction], Conf), Type, Name);
         {_ConnType, _ConnName} ->
             error({cannot_use_connector_with_different_type, ConnId})
     end;
-parse_confs(_Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf)
+parse_confs(Type, Name, #{connector := ConnectorConfs, direction := Direction} = Conf)
         when is_map(ConnectorConfs) ->
     make_resource_confs(Direction, ConnectorConfs,
-        maps:without([connector, direction], Conf), Name).
+        maps:without([connector, direction], Conf), Type, Name).
 
-make_resource_confs(ingress, ConnectorConfs, BridgeConf, Name) ->
-    BName = bin(Name),
+make_resource_confs(ingress, ConnectorConfs, BridgeConf, Type, Name) ->
+    BName = bridge_id(Type, Name),
     ConnectorConfs#{
         ingress => BridgeConf#{hookpoint => <<"$bridges/", BName/binary>>}
     };
-make_resource_confs(egress, ConnectorConfs, BridgeConf, _Name) ->
+make_resource_confs(egress, ConnectorConfs, BridgeConf, _Type, _Name) ->
     ConnectorConfs#{
         egress => BridgeConf
     }.

apps/emqx_bridge/src/emqx_bridge_api.erl

@@ -158,8 +158,8 @@ method_example(_Type, _Direction, put) ->
 info_example_basic(http, _) ->
     #{
         url => <<"http://localhost:9901/messages/${topic}">>,
-        request_timeout => <<"30s">>,
-        connect_timeout => <<"30s">>,
+        request_timeout => <<"15s">>,
+        connect_timeout => <<"15s">>,
         max_retries => 3,
         retry_interval => <<"10s">>,
         pool_type => <<"random">>,
@@ -276,7 +276,7 @@ schema("/bridges/:id/operation/:operation") ->
 
 '/bridges'(post, #{body := #{<<"type">> := BridgeType} = Conf0}) ->
     Conf = filter_out_request_body(Conf0),
-    BridgeName = maps:get(<<"name">>, Conf, emqx_misc:gen_id()),
+    BridgeName = emqx_misc:gen_id(),
     case emqx_bridge:lookup(BridgeType, BridgeName) of
         {ok, _} ->
             {400, error_msg('ALREADY_EXISTS', <<"bridge already exists">>)};
@@ -356,9 +356,8 @@ operation_to_conf_req(<<"restart">>) -> restart;
 operation_to_conf_req(_) -> invalid.
 
 ensure_bridge_created(BridgeType, BridgeName, Conf) ->
-    Conf1 = maps:without([<<"type">>, <<"name">>], Conf),
     case emqx_conf:update(emqx_bridge:config_key_path() ++ [BridgeType, BridgeName],
-            Conf1, #{override_to => cluster}) of
+            Conf, #{override_to => cluster}) of
         {ok, _} -> ok;
         {error, Reason} ->
             {error, error_msg('BAD_ARG', Reason)}
@@ -411,12 +410,12 @@ aggregate_metrics(AllMetrics) ->
 
 format_resp(#{id := Id, raw_config := RawConf,
               resource_data := #{status := Status, metrics := Metrics}}) ->
-    {Type, Name} = emqx_bridge:parse_bridge_id(Id),
+    {Type, BridgeName} = emqx_bridge:parse_bridge_id(Id),
     IsConnected = fun(started) -> connected; (_) -> disconnected end,
     RawConf#{
         id => Id,
         type => Type,
-        name => Name,
+        name => maps:get(<<"name">>, RawConf, BridgeName),
         node => node(),
         status => IsConnected(Status),
         metrics => Metrics
@@ -431,8 +430,8 @@ rpc_multicall(Func, Args) ->
     end.
 
 filter_out_request_body(Conf) ->
-    ExtraConfs = [<<"id">>, <<"status">>, <<"node_status">>, <<"node_metrics">>,
-        <<"metrics">>, <<"node">>],
+    ExtraConfs = [<<"id">>, <<"type">>, <<"status">>, <<"node_status">>,
+        <<"node_metrics">>, <<"metrics">>, <<"node">>],
     maps:without(ExtraConfs, Conf).
 
 rpc_call(Node, Fun, Args) ->

apps/emqx_bridge/src/emqx_bridge_http_schema.erl

@@ -59,7 +59,7 @@ Template with variables is allowed.
 """
            })}
     , {request_timeout, mk(emqx_schema:duration_ms(),
-          #{ default => <<"30s">>
+          #{ default => <<"15s">>
            , desc =>"""
 How long will the HTTP request timeout.
 """
@@ -68,7 +68,6 @@ How long will the HTTP request timeout.
 
 fields("post") ->
     [ type_field()
-    , name_field()
     ] ++ fields("bridge");
 
 fields("put") ->
@@ -84,9 +83,14 @@ basic_config() ->
            #{ desc => "Enable or disable this bridge"
             , default => true
             })}
+    , {name,
+       mk(binary(),
+          #{ desc => "Bridge name, used as a human-readable description of the bridge."
+           })}
     , {direction,
         mk(egress,
            #{ desc => "The direction of this bridge, MUST be egress"
+            , default => egress
             })}
     ]
     ++ proplists:delete(base_url, emqx_connector_http:fields(config)).
@@ -98,8 +102,5 @@ id_field() ->
 type_field() ->
     {type, mk(http, #{desc => "The Bridge Type"})}.
 
-name_field() ->
-    {name, mk(binary(), #{desc => "The Bridge Name"})}.
-
 method() ->
     enum([post, put, get, delete]).

apps/emqx_bridge/src/emqx_bridge_mqtt_schema.erl

@@ -24,11 +24,9 @@ fields("egress") ->
 
 fields("post_ingress") ->
     [ type_field()
-    , name_field()
     ] ++ proplists:delete(enable, fields("ingress"));
 fields("post_egress") ->
     [ type_field()
-    , name_field()
     ] ++ proplists:delete(enable, fields("egress"));
 
 fields("put_ingress") ->
@@ -49,9 +47,3 @@ id_field() ->
 
 type_field() ->
     {type, mk(mqtt, #{desc => "The Bridge Type"})}.
-
-name_field() ->
-    {name, mk(binary(),
-        #{ desc => "The Bridge Name"
-         , example => "some_bridge_name"
-         })}.

apps/emqx_bridge/src/emqx_bridge_schema.erl

@@ -43,9 +43,13 @@ http_schema(Method) ->
 common_bridge_fields() ->
     [ {enable,
         mk(boolean(),
-           #{ desc =>"Enable or disable this bridge"
+           #{ desc => "Enable or disable this bridge"
             , default => true
             })}
+    , {name,
+       mk(binary(),
+          #{ desc => "Bridge name, used as a human-readable description of the bridge."
+           })}
     , {connector,
         mk(binary(),
            #{ nullable => false
@@ -71,6 +75,7 @@ metrics_status_fields() ->
 direction_field(Dir, Desc) ->
     {direction, mk(Dir,
         #{ nullable => false
+         , default => egress
          , desc => "The direction of the bridge. Can be one of 'ingress' or 'egress'.<br>"
             ++ Desc
          })}.

apps/emqx_bridge/test/emqx_bridge_api_SUITE.erl

@@ -23,12 +23,13 @@
 -define(CONF_DEFAULT, <<"bridges: {}">>).
 -define(BRIDGE_TYPE, <<"http">>).
 -define(BRIDGE_NAME, <<"test_bridge">>).
--define(BRIDGE_ID, <<"http:test_bridge">>).
 -define(URL(PORT, PATH), list_to_binary(
     io_lib:format("http://localhost:~s/~s",
                   [integer_to_list(PORT), PATH]))).
--define(HTTP_BRIDGE(URL),
+-define(HTTP_BRIDGE(URL, TYPE, NAME),
 #{
+    <<"type">> => TYPE,
+    <<"name">> => NAME,
     <<"url">> => URL,
     <<"local_topic">> => <<"emqx_http/#">>,
     <<"method">> => <<"post">>,
@@ -47,7 +48,7 @@ groups() ->
     [].
 
 suite() ->
-	[{timetrap,{seconds,30}}].
+	[{timetrap,{seconds,60}}].
 
 init_per_suite(Config) ->
     ok = emqx_config:put([emqx_dashboard], #{
@@ -84,7 +85,7 @@ start_http_server(HandleFun) ->
     spawn_link(fun() ->
         {Port, Sock} = listen_on_random_port(),
         Parent ! {port, Port},
-        loop(Sock, HandleFun)
+        loop(Sock, HandleFun, Parent)
     end),
     receive
         {port, Port} -> Port
@@ -95,40 +96,49 @@ start_http_server(HandleFun) ->
 listen_on_random_port() ->
     Min = 1024, Max = 65000,
     Port = rand:uniform(Max - Min) + Min,
-    case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}]) of
+    case gen_tcp:listen(Port, [{active, false}, {reuseaddr, true}, binary]) of
         {ok, Sock} -> {Port, Sock};
         {error, eaddrinuse} -> listen_on_random_port()
     end.
 
-loop(Sock, HandleFun) ->
+loop(Sock, HandleFun, Parent) ->
     {ok, Conn} = gen_tcp:accept(Sock),
-    Handler = spawn(fun () -> HandleFun(Conn) end),
+    Handler = spawn(fun () -> HandleFun(Conn, Parent) end),
     gen_tcp:controlling_process(Conn, Handler),
-    loop(Sock, HandleFun).
+    loop(Sock, HandleFun, Parent).
 
 make_response(CodeStr, Str) ->
     B = iolist_to_binary(Str),
     iolist_to_binary(
       io_lib:fwrite(
-         "HTTP/1.0 ~s\nContent-Type: text/html\nContent-Length: ~p\n\n~s",
+         "HTTP/1.0 ~s\r\nContent-Type: text/html\r\nContent-Length: ~p\r\n\r\n~s",
          [CodeStr, size(B), B])).
 
-handle_fun_200_ok(Conn) ->
+handle_fun_200_ok(Conn, Parent) ->
     case gen_tcp:recv(Conn, 0) of
-        {ok, Request} ->
+        {ok, ReqStr} ->
+            ct:pal("the http handler got request: ~p", [ReqStr]),
+            Req = parse_http_request(ReqStr),
+            Parent ! {http_server, received, Req},
             gen_tcp:send(Conn, make_response("200 OK", "Request OK")),
-            self() ! {http_server, received, Request},
-            handle_fun_200_ok(Conn);
+            handle_fun_200_ok(Conn, Parent);
         {error, closed} ->
             gen_tcp:close(Conn)
     end.
 
+parse_http_request(ReqStr0) ->
+    [Method, ReqStr1] = string:split(ReqStr0, " ", leading),
+    [Path, ReqStr2] = string:split(ReqStr1, " ", leading),
+    [_ProtoVsn, ReqStr3] = string:split(ReqStr2, "\r\n", leading),
+    [_HeaderStr, Body] = string:split(ReqStr3, "\r\n\r\n", leading),
+    #{method => Method, path => Path, body => Body}.
+
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
 
 t_http_crud_apis(_) ->
-    Port = start_http_server(fun handle_fun_200_ok/1),
+    Port = start_http_server(fun handle_fun_200_ok/2),
     %% assert we there's no bridges at first
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
@@ -136,38 +146,39 @@ t_http_crud_apis(_) ->
     %% POST /bridges/ will create a bridge
     URL1 = ?URL(Port, "path1"),
     {ok, 201, Bridge} = request(post, uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1)#{
-            <<"type">> => ?BRIDGE_TYPE,
-            <<"name">> => ?BRIDGE_NAME
-        }),
+        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
 
     %ct:pal("---bridge: ~p", [Bridge]),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID
-                  , <<"type">> := ?BRIDGE_TYPE
-                  , <<"name">> := ?BRIDGE_NAME
-                  , <<"status">> := _
-                  , <<"node_status">> := [_|_]
-                  , <<"metrics">> := _
-                  , <<"node_metrics">> := [_|_]
-                  , <<"url">> := URL1
-                  }, jsx:decode(Bridge)),
-
-    %% create a again returns an error
-    {ok, 400, RetMsg} = request(post, uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1)#{
-            <<"type">> => ?BRIDGE_TYPE,
-            <<"name">> => ?BRIDGE_NAME
-        }),
-    ?assertMatch(
-        #{ <<"code">> := _
-         , <<"message">> := <<"bridge already exists">>
-         }, jsx:decode(RetMsg)),
+    #{ <<"id">> := BridgeID
+     , <<"type">> := ?BRIDGE_TYPE
+     , <<"name">> := ?BRIDGE_NAME
+     , <<"status">> := _
+     , <<"node_status">> := [_|_]
+     , <<"metrics">> := _
+     , <<"node_metrics">> := [_|_]
+     , <<"url">> := URL1
+     } = jsx:decode(Bridge),
 
+    %% send an message to emqx and the message should be forwarded to the HTTP server
+    wait_for_resource_ready(BridgeID, 5),
+    Body = <<"my msg">>,
+    emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)),
+    ?assert(
+        receive
+            {http_server, received, #{method := <<"POST">>, path := <<"/path1">>,
+                    body := Body}} ->
+                true;
+            Msg ->
+                ct:pal("error: http got unexpected request: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end),
     %% update the request-path of the bridge
     URL2 = ?URL(Port, "path2"),
-    {ok, 200, Bridge2} = request(put, uri(["bridges", ?BRIDGE_ID]),
-                                 ?HTTP_BRIDGE(URL2)),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID
+    {ok, 200, Bridge2} = request(put, uri(["bridges", BridgeID]),
+                                 ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
+    ?assertMatch(#{ <<"id">> := BridgeID
                   , <<"type">> := ?BRIDGE_TYPE
                   , <<"name">> := ?BRIDGE_NAME
                   , <<"status">> := _
@@ -179,7 +190,7 @@ t_http_crud_apis(_) ->
 
     %% list all bridges again, assert Bridge2 is in it
     {ok, 200, Bridge2Str} = request(get, uri(["bridges"]), []),
-    ?assertMatch([#{ <<"id">> := ?BRIDGE_ID
+    ?assertMatch([#{ <<"id">> := BridgeID
                    , <<"type">> := ?BRIDGE_TYPE
                    , <<"name">> := ?BRIDGE_NAME
                    , <<"status">> := _
@@ -190,8 +201,8 @@ t_http_crud_apis(_) ->
                    }], jsx:decode(Bridge2Str)),
 
     %% get the bridge by id
-    {ok, 200, Bridge3Str} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID
+    {ok, 200, Bridge3Str} = request(get, uri(["bridges", BridgeID]), []),
+    ?assertMatch(#{ <<"id">> := BridgeID
                   , <<"type">> := ?BRIDGE_TYPE
                   , <<"name">> := ?BRIDGE_NAME
                   , <<"status">> := _
@@ -201,13 +212,27 @@ t_http_crud_apis(_) ->
                   , <<"url">> := URL2
                   }, jsx:decode(Bridge3Str)),
 
+    %% send an message to emqx again, check the path has been changed
+    wait_for_resource_ready(BridgeID, 5),
+    emqx:publish(emqx_message:make(<<"emqx_http/1">>, Body)),
+    ?assert(
+        receive
+            {http_server, received, #{path := <<"/path2">>}} ->
+                true;
+            Msg2 ->
+                ct:pal("error: http got unexpected request: ~p", [Msg2]),
+                false
+        after 100 ->
+            false
+        end),
+
     %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
     %% update a deleted bridge returns an error
-    {ok, 404, ErrMsg2} = request(put, uri(["bridges", ?BRIDGE_ID]),
-                                 ?HTTP_BRIDGE(URL2)),
+    {ok, 404, ErrMsg2} = request(put, uri(["bridges", BridgeID]),
+                                 ?HTTP_BRIDGE(URL2, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
     ?assertMatch(
         #{ <<"code">> := _
          , <<"message">> := <<"bridge not found">>
@@ -215,52 +240,51 @@ t_http_crud_apis(_) ->
     ok.
 
 t_start_stop_bridges(_) ->
-    Port = start_http_server(fun handle_fun_200_ok/1),
+    %% assert we there's no bridges at first
+    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
+
+    Port = start_http_server(fun handle_fun_200_ok/2),
     URL1 = ?URL(Port, "abc"),
     {ok, 201, Bridge} = request(post, uri(["bridges"]),
-        ?HTTP_BRIDGE(URL1)#{
-            <<"type">> => ?BRIDGE_TYPE,
-            <<"name">> => ?BRIDGE_NAME
-        }),
+        ?HTTP_BRIDGE(URL1, ?BRIDGE_TYPE, ?BRIDGE_NAME)),
     %ct:pal("the bridge ==== ~p", [Bridge]),
-    ?assertMatch(
-        #{ <<"id">> := ?BRIDGE_ID
-         , <<"type">> := ?BRIDGE_TYPE
-         , <<"name">> := ?BRIDGE_NAME
-         , <<"status">> := _
-         , <<"node_status">> := [_|_]
-         , <<"metrics">> := _
-         , <<"node_metrics">> := [_|_]
-         , <<"url">> := URL1
-         }, jsx:decode(Bridge)),
+    #{ <<"id">> := BridgeID
+     , <<"type">> := ?BRIDGE_TYPE
+     , <<"name">> := ?BRIDGE_NAME
+     , <<"status">> := _
+     , <<"node_status">> := [_|_]
+     , <<"metrics">> := _
+     , <<"node_metrics">> := [_|_]
+     , <<"url">> := URL1
+     } = jsx:decode(Bridge),
     %% stop it
-    {ok, 200, <<>>} = request(post, operation_path(stop), <<"">>),
-    {ok, 200, Bridge2} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID
+    {ok, 200, <<>>} = request(post, operation_path(stop, BridgeID), <<"">>),
+    {ok, 200, Bridge2} = request(get, uri(["bridges", BridgeID]), []),
+    ?assertMatch(#{ <<"id">> := BridgeID
                   , <<"status">> := <<"disconnected">>
                   }, jsx:decode(Bridge2)),
     %% start again
-    {ok, 200, <<>>} = request(post, operation_path(start), <<"">>),
-    {ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID
+    {ok, 200, <<>>} = request(post, operation_path(start, BridgeID), <<"">>),
+    {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
+    ?assertMatch(#{ <<"id">> := BridgeID
                   , <<"status">> := <<"connected">>
                   }, jsx:decode(Bridge3)),
     %% restart an already started bridge
-    {ok, 200, <<>>} = request(post, operation_path(restart), <<"">>),
-    {ok, 200, Bridge3} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID
+    {ok, 200, <<>>} = request(post, operation_path(restart, BridgeID), <<"">>),
+    {ok, 200, Bridge3} = request(get, uri(["bridges", BridgeID]), []),
+    ?assertMatch(#{ <<"id">> := BridgeID
                   , <<"status">> := <<"connected">>
                   }, jsx:decode(Bridge3)),
     %% stop it again
-    {ok, 200, <<>>} = request(post, operation_path(stop), <<"">>),
+    {ok, 200, <<>>} = request(post, operation_path(stop, BridgeID), <<"">>),
     %% restart a stopped bridge
-    {ok, 200, <<>>} = request(post, operation_path(restart), <<"">>),
-    {ok, 200, Bridge4} = request(get, uri(["bridges", ?BRIDGE_ID]), []),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID
+    {ok, 200, <<>>} = request(post, operation_path(restart, BridgeID), <<"">>),
+    {ok, 200, Bridge4} = request(get, uri(["bridges", BridgeID]), []),
+    ?assertMatch(#{ <<"id">> := BridgeID
                   , <<"status">> := <<"connected">>
                   }, jsx:decode(Bridge4)),
     %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []).
 
 %%--------------------------------------------------------------------
@@ -296,5 +320,16 @@ auth_header_() ->
     {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
     {"Authorization", "Bearer " ++ binary_to_list(Token)}.
 
-operation_path(Oper) ->
-    uri(["bridges", ?BRIDGE_ID, "operation", Oper]).
+operation_path(Oper, BridgeID) ->
+    uri(["bridges", BridgeID, "operation", Oper]).
+
+wait_for_resource_ready(InstId, 0) ->
+    ct:pal("--- bridge ~p: ~p", [InstId, emqx_bridge:lookup(InstId)]),
+    ct:fail(wait_resource_timeout);
+wait_for_resource_ready(InstId, Retry) ->
+    case emqx_bridge:lookup(InstId) of
+        {ok, #{resource_data := #{status := started}}} -> ok;
+        _ ->
+            timer:sleep(100),
+            wait_for_resource_ready(InstId, Retry-1)
+    end.

apps/emqx_conf/src/emqx_cluster_rpc.erl

@@ -236,7 +236,7 @@ catch_up(#{node := Node, retry_interval := RetryMs} = State, SkipResult) ->
                 false -> RetryMs
             end;
         {aborted, Reason} ->
-            ?SLOG(error, #{msg => "read_next_mfa transaction failed", error => Reason}),
+            ?SLOG(error, #{msg => "read_next_mfa_transaction_failed", error => Reason}),
             RetryMs
     end.
 
@@ -248,7 +248,7 @@ read_next_mfa(Node) ->
                 TnxId = max(LatestId - 1, 0),
                 commit(Node, TnxId),
                 ?SLOG(notice, #{
-                    msg => "New node first catch up and start commit.",
+                    msg => "new_node_first_catch_up_and_start_commit.",
                     node => Node, tnx_id => TnxId}),
                 TnxId;
             [#cluster_rpc_commit{tnx_id = LastAppliedID}] -> LastAppliedID + 1
@@ -277,7 +277,7 @@ do_catch_up(ToTnxId, Node) ->
                 io_lib:format("~p catch up failed by LastAppliedId(~p) > ToTnxId(~p)",
                 [Node, LastAppliedId, ToTnxId])),
             ?SLOG(error, #{
-                msg => "catch up failed!",
+                msg => "catch_up_failed!",
                 last_applied_id => LastAppliedId,
                 to_tnx_id => ToTnxId
             }),

apps/emqx_conf/src/emqx_conf.erl

@@ -144,7 +144,7 @@ multicall(M, F, Args) ->
         {retry, TnxId, Res, Nodes} ->
             %% The init MFA return ok, but other nodes failed.
             %% We return ok and alert an alarm.
-            ?SLOG(error, #{msg => "failed to update config in cluster", nodes => Nodes,
+            ?SLOG(error, #{msg => "failed_to_update_config_in_cluster", nodes => Nodes,
                 tnx_id => TnxId, mfa => {M, F, Args}}),
             Res;
         {error, Error} -> %% all MFA return not ok or {ok, term()}.

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -730,16 +730,7 @@ do_formatter(json, CharsLimit, SingleLine, TimeOffSet, Depth) ->
         }};
 do_formatter(text, CharsLimit, SingleLine, TimeOffSet, Depth) ->
     {emqx_logger_textfmt,
-        #{template =>
-            [time," [",level,"] ",
-                {clientid,
-                    [{peername,
-                        [clientid,"@",peername," "],
-                        [clientid, " "]}],
-                    [{peername,
-                        [peername," "],
-                        []}]},
-                msg,"\n"],
+        #{template => [time," [",level,"] ", msg,"\n"],
           chars_limit => CharsLimit,
           single_line => SingleLine,
           time_offset => TimeOffSet,

apps/emqx_conf/test/emqx_cluster_rpc_SUITE.erl

@@ -74,9 +74,19 @@ t_base_test(_Config) ->
     ?assertEqual(node(), maps:get(initiator, Query)),
     ?assert(maps:is_key(created_at, Query)),
     ?assertEqual(ok, receive_msg(3, test)),
+    ?assertEqual({ok, 2, ok}, emqx_cluster_rpc:multicall(M, F, A)),
     {atomic, Status} = emqx_cluster_rpc:status(),
-    ?assertEqual(3, length(Status)),
-    ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 1 end, Status)),
+    case length(Status) =:= 3 of
+        true -> ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status));
+        false ->
+            %% wait for mnesia to write in.
+            ct:sleep(42),
+            {atomic, Status1} = emqx_cluster_rpc:status(),
+            ct:pal("status: ~p", Status),
+            ct:pal("status1: ~p", Status1),
+            ?assertEqual(3, length(Status1)),
+            ?assert(lists:all(fun(I) -> maps:get(tnx_id, I) =:= 2 end, Status))
+    end,
     ok.
 
 t_commit_fail_test(_Config) ->

apps/emqx_connector/rebar.config

@@ -7,7 +7,7 @@
   {emqx, {path, "../emqx"}},
   {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
   {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}},
-  {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.6.0"}}},
+  {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7-emqx.1"}}},
   %% NOTE: mind poolboy version when updating mongodb-erlang version
   {mongodb, {git,"https://github.com/emqx/mongodb-erlang", {tag, "v3.0.11"}}},
   %% NOTE: mind poolboy version when updating eredis_cluster version

apps/emqx_connector/src/emqx_connector.erl

@@ -37,31 +37,26 @@
 config_key_path() ->
     [connectors].
 
+-dialyzer([{nowarn_function, [post_config_update/5]}, error_handling]).
 post_config_update([connectors, Type, Name], '$remove', _, _OldConf, _AppEnvs) ->
     ConnId = connector_id(Type, Name),
-    LinkedBridgeIds = lists:foldl(fun
-        (#{id := BId, raw_config := #{<<"connector">> := ConnId0}}, Acc)
-                when ConnId0 == ConnId ->
-            [BId | Acc];
-        (_, Acc) -> Acc
-    end, [], emqx_bridge:list()),
-    case LinkedBridgeIds of
-        [] -> ok;
-        _ -> {error, {dependency_bridges_exist, LinkedBridgeIds}}
+    try foreach_linked_bridges(ConnId, fun(#{id := BId}) ->
+            throw({dependency_bridges_exist, BId})
+        end)
+    catch throw:Error -> {error, Error}
     end;
-post_config_update([connectors, Type, Name], _Req, NewConf, _OldConf, _AppEnvs) ->
+post_config_update([connectors, Type, Name], _Req, NewConf, OldConf, _AppEnvs) ->
     ConnId = connector_id(Type, Name),
-    lists:foreach(fun
-        (#{id := BId, raw_config := #{<<"connector">> := ConnId0}}) when ConnId0 == ConnId ->
+    foreach_linked_bridges(ConnId,
+        fun(#{id := BId}) ->
             {BType, BName} = emqx_bridge:parse_bridge_id(BId),
             BridgeConf = emqx:get_config([bridges, BType, BName]),
-            case emqx_bridge:recreate(BType, BName, BridgeConf#{connector => NewConf}) of
-                {ok, _} -> ok;
+            case emqx_bridge:update(BType, BName, {BridgeConf#{connector => OldConf},
+                    BridgeConf#{connector => NewConf}}) of
+                ok -> ok;
                 {error, Reason} -> error({update_bridge_error, Reason})
-            end;
-        (_) ->
-            ok
-    end, emqx_bridge:list()).
+            end
+        end).
 
 connector_id(Type0, Name0) ->
     Type = bin(Type0),
@@ -112,3 +107,10 @@ delete(Type, Name) ->
 bin(Bin) when is_binary(Bin) -> Bin;
 bin(Str) when is_list(Str) -> list_to_binary(Str);
 bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8).
+
+foreach_linked_bridges(ConnId, Do) ->
+    lists:foreach(fun
+        (#{raw_config := #{<<"connector">> := ConnId0}} = Bridge) when ConnId0 == ConnId ->
+            Do(Bridge);
+        (_) -> ok
+    end, emqx_bridge:list()).

apps/emqx_connector/src/emqx_connector_api.erl

@@ -107,14 +107,14 @@ info_example_basic(mqtt) ->
     #{
         mode => cluster_shareload,
         server => <<"127.0.0.1:1883">>,
-        reconnect_interval => <<"30s">>,
+        reconnect_interval => <<"15s">>,
         proto_ver => <<"v4">>,
         username => <<"foo">>,
         password => <<"bar">>,
         clientid => <<"foo">>,
         clean_start => true,
         keepalive => <<"300s">>,
-        retry_interval => <<"30s">>,
+        retry_interval => <<"15s">>,
         max_inflight => 100,
         ssl => #{
             enable => false
@@ -155,8 +155,7 @@ schema("/connectors") ->
         },
         post => #{
             tags => [<<"connectors">>],
-            description => <<"Create a new connector by given Id <br>"
-                             "The ID must be of format '{type}:{name}'">>,
+            description => <<"Create a new connector">>,
             summary => <<"Create connector">>,
             requestBody => post_request_body_schema(),
             responses => #{
@@ -212,13 +211,13 @@ schema("/connectors/:id") ->
     {200, [format_resp(Conn) || Conn <- emqx_connector:list()]};
 
 '/connectors'(post, #{body := #{<<"type">> := ConnType} = Params}) ->
-    ConnName = maps:get(<<"name">>, Params, emqx_misc:gen_id()),
+    ConnName = emqx_misc:gen_id(),
     case emqx_connector:lookup(ConnType, ConnName) of
         {ok, _} ->
             {400, error_msg('ALREADY_EXISTS', <<"connector already exists">>)};
         {error, not_found} ->
             case emqx_connector:update(ConnType, ConnName,
-                    maps:without([<<"type">>, <<"name">>], Params)) of
+                    filter_out_request_body(Params)) of
                 {ok, #{raw_config := RawConf}} ->
                     Id = emqx_connector:connector_id(ConnType, ConnName),
                     {201, format_resp(Id, RawConf)};
@@ -254,6 +253,10 @@ schema("/connectors/:id") ->
             {ok, _} ->
                 case emqx_connector:delete(ConnType, ConnName) of
                     {ok, _} -> {204};
+                    {error, {post_config_update, _, {dependency_bridges_exist, BridgeID}}} ->
+                        {403, error_msg('DEPENDENCY_EXISTS',
+                             <<"Cannot remove the connector as it's in use by a bridge: ",
+                                BridgeID/binary>>)};
                     {error, Error} -> {400, error_msg('BAD_ARG', Error)}
                 end;
             {error, not_found} ->
@@ -270,16 +273,16 @@ format_resp(#{<<"id">> := Id} = RawConf) ->
 
 format_resp(ConnId, RawConf) ->
     NumOfBridges = length(emqx_bridge:list_bridges_by_connector(ConnId)),
-    {Type, Name} = emqx_connector:parse_connector_id(ConnId),
+    {Type, ConnName} = emqx_connector:parse_connector_id(ConnId),
     RawConf#{
         <<"id">> => ConnId,
         <<"type">> => Type,
-        <<"name">> => Name,
+        <<"name">> => maps:get(<<"name">>, RawConf, ConnName),
         <<"num_of_bridges">> => NumOfBridges
     }.
 
 filter_out_request_body(Conf) ->
-    ExtraConfs = [<<"num_of_bridges">>, <<"type">>, <<"name">>],
+    ExtraConfs = [<<"clientid">>, <<"num_of_bridges">>, <<"type">>],
     maps:without(ExtraConfs, Conf).
 
 bin(S) when is_list(S) ->

apps/emqx_connector/src/emqx_connector_http.erl

@@ -75,7 +75,7 @@ For example: http://localhost:9901/
            })}
     , {connect_timeout,
         sc(emqx_schema:duration_ms(),
-           #{ default => "30s"
+           #{ default => "15s"
             , desc => "The timeout when connecting to the HTTP server"
             })}
     , {max_retries,
@@ -143,7 +143,7 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
                    retry_interval := RetryInterval,
                    pool_type := PoolType,
                    pool_size := PoolSize} = Config) ->
-    ?SLOG(info, #{msg => "starting http connector",
+    ?SLOG(info, #{msg => "starting_http_connector",
                   connector => InstId, config => Config}),
     {Transport, TransportOpts} = case Scheme of
                                      http ->
@@ -181,13 +181,13 @@ on_start(InstId, #{base_url := #{scheme := Scheme,
     end.
 
 on_stop(InstId, #{pool_name := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping http connector",
+    ?SLOG(info, #{msg => "stopping_http_connector",
                   connector => InstId}),
     ehttpc_sup:stop_pool(PoolName).
 
 on_query(InstId, {send_message, Msg}, AfterQuery, State) ->
     case maps:get(request, State, undefined) of
-        undefined -> ?SLOG(error, #{msg => "request not found", connector => InstId});
+        undefined -> ?SLOG(error, #{msg => "request_not_found", connector => InstId});
         Request ->
             #{method := Method, path := Path, body := Body, headers := Headers,
               request_timeout := Timeout} = process_request(Request, Msg),
@@ -199,23 +199,32 @@ on_query(InstId, {Method, Request, Timeout}, AfterQuery, State) ->
     on_query(InstId, {undefined, Method, Request, Timeout}, AfterQuery, State);
 on_query(InstId, {KeyOrNum, Method, Request, Timeout}, AfterQuery,
         #{pool_name := PoolName, base_path := BasePath} = State) ->
-    ?SLOG(debug, #{msg => "http connector received request",
-                   request => Request, connector => InstId,
-                   state => State}),
-    NRequest = update_path(BasePath, Request),
-    Name = case KeyOrNum of
-               undefined -> PoolName;
-               _ -> {PoolName, KeyOrNum}
-           end,
-    Result = ehttpc:request(Name, Method, NRequest, Timeout),
-    case Result of
+    ?TRACE("QUERY", "http_connector_received",
+        #{request => Request, connector => InstId, state => State}),
+    NRequest = formalize_request(Method, BasePath, Request),
+    case Result = ehttpc:request(case KeyOrNum of
+                                     undefined -> PoolName;
+                                     _ -> {PoolName, KeyOrNum}
+                                 end, Method, NRequest, Timeout) of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "http connector do reqeust failed",
+            ?SLOG(error, #{msg => "http_connector_do_reqeust_failed",
                            request => NRequest, reason => Reason,
                            connector => InstId}),
             emqx_resource:query_failed(AfterQuery);
-        _ ->
-            emqx_resource:query_success(AfterQuery)
+        {ok, StatusCode, _} when StatusCode >= 200 andalso StatusCode < 300 ->
+            emqx_resource:query_success(AfterQuery);
+        {ok, StatusCode, _, _} when StatusCode >= 200 andalso StatusCode < 300 ->
+            emqx_resource:query_success(AfterQuery);
+        {ok, StatusCode, _} ->
+            ?SLOG(error, #{msg => "http connector do reqeust, received error response",
+                           request => NRequest, connector => InstId,
+                           status_code => StatusCode}),
+            emqx_resource:query_failed(AfterQuery);
+        {ok, StatusCode, _, _} ->
+            ?SLOG(error, #{msg => "http connector do reqeust, received error response",
+                           request => NRequest, connector => InstId,
+                           status_code => StatusCode}),
+            emqx_resource:query_failed(AfterQuery)
     end,
     Result.
 
@@ -268,11 +277,16 @@ process_request(#{
         } = Conf, Msg) ->
     Conf#{ method => make_method(emqx_plugin_libs_rule:proc_tmpl(MethodTks, Msg))
          , path => emqx_plugin_libs_rule:proc_tmpl(PathTks, Msg)
-         , body => emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg)
+         , body => process_request_body(BodyTks, Msg)
          , headers => maps:to_list(proc_headers(HeadersTks, Msg))
          , request_timeout => ReqTimeout
          }.
 
+process_request_body([], Msg) ->
+    emqx_json:encode(Msg);
+process_request_body(BodyTks, Msg) ->
+    emqx_plugin_libs_rule:proc_tmpl(BodyTks, Msg).
+
 proc_headers(HeaderTks, Msg) ->
     maps:fold(fun(K, V, Acc) ->
             Acc#{emqx_plugin_libs_rule:proc_tmpl(K, Msg) =>
@@ -296,10 +310,14 @@ check_ssl_opts(URLFrom, Conf) ->
         {_, _} -> false
     end.
 
-update_path(BasePath, {Path, Headers}) ->
-    {filename:join(BasePath, Path), Headers};
-update_path(BasePath, {Path, Headers, Body}) ->
-    {filename:join(BasePath, Path), Headers, Body}.
+formalize_request(Method, BasePath, {Path, Headers, _Body})
+        when Method =:= get; Method =:= delete ->
+    formalize_request(Method, BasePath, {Path, Headers});
+formalize_request(_Method, BasePath, {Path, Headers, Body}) ->
+    {filename:join(BasePath, Path), Headers, Body};
+
+formalize_request(_Method, BasePath, {Path, Headers}) ->
+    {filename:join(BasePath, Path), Headers}.
 
 bin(Bin) when is_binary(Bin) ->
     Bin;

apps/emqx_connector/src/emqx_connector_ldap.erl

@@ -55,7 +55,7 @@ on_start(InstId, #{servers := Servers0,
                    pool_size := PoolSize,
                    auto_reconnect := AutoReconn,
                    ssl := SSL} = Config) ->
-    ?SLOG(info, #{msg => "starting ldap connector",
+    ?SLOG(info, #{msg => "starting_ldap_connector",
                   connector => InstId, config => Config}),
     Servers = [begin proplists:get_value(host, S) end || S <- Servers0],
     SslOpts = case maps:get(enable, SSL) of
@@ -81,23 +81,21 @@ on_start(InstId, #{servers := Servers0,
     {ok, #{poolname => PoolName}}.
 
 on_stop(InstId, #{poolname := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping ldap connector",
+    ?SLOG(info, #{msg => "stopping_ldap_connector",
                   connector => InstId}),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
 on_query(InstId, {search, Base, Filter, Attributes}, AfterQuery, #{poolname := PoolName} = State) ->
     Request = {Base, Filter, Attributes},
-    ?SLOG(debug, #{msg => "ldap connector received request",
-                   request => Request, connector => InstId,
-                   state => State}),
+    ?TRACE("QUERY", "ldap_connector_received",
+        #{request => Request, connector => InstId, state => State}),
     case Result = ecpool:pick_and_do(
                     PoolName,
                     {?MODULE, search, [Base, Filter, Attributes]},
                     no_handover) of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "ldap connector do request failed",
-                           request => Request, connector => InstId,
-                           reason => Reason}),
+            ?SLOG(error, #{msg => "ldap_connector_do_request_failed",
+                request => Request, connector => InstId, reason => Reason}),
             emqx_resource:query_failed(AfterQuery);
         _ ->
             emqx_resource:query_success(AfterQuery)

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -34,6 +34,8 @@
         , on_jsonify/1
         ]).
 
+
+%% ecpool callback
 -export([connect/1]).
 
 -export([roots/0, fields/1]).
@@ -125,11 +127,11 @@ on_start(InstId, Config = #{mongo_type := Type,
             {options, init_topology_options(maps:to_list(Topology), [])},
             {worker_options, init_worker_options(maps:to_list(NConfig), SslOpts)}],
     PoolName = emqx_plugin_libs_pool:pool_name(InstId),
-    _ = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts),
+    ok = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Opts),
     {ok, #{poolname => PoolName, type => Type}}.
 
 on_stop(InstId, #{poolname := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping mongodb connector",
+    ?SLOG(info, #{msg => "stopping_mongodb_connector",
                   connector => InstId}),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
@@ -138,14 +140,13 @@ on_query(InstId,
          AfterQuery,
          #{poolname := PoolName} = State) ->
     Request = {Action, Collection, Selector, Docs},
-    ?SLOG(debug, #{msg => "mongodb connector received request",
-        request => Request, connector => InstId,
-        state => State}),
+    ?TRACE("QUERY", "mongodb_connector_received",
+        #{request => Request, connector => InstId, state => State}),
     case ecpool:pick_and_do(PoolName,
                             {?MODULE, mongo_query, [Action, Collection, Selector, Docs]},
                             no_handover) of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "mongodb connector do query failed",
+            ?SLOG(error, #{msg => "mongodb_connector_do_query_failed",
                 request => Request, reason => Reason,
                 connector => InstId}),
             emqx_resource:query_failed(AfterQuery),
@@ -178,18 +179,22 @@ health_check(PoolName) ->
 
 %% ===================================================================
 
-check_worker_health(Worker) -> 
+%% TODO: log reasons
+check_worker_health(Worker) ->
     case ecpool_worker:client(Worker) of
         {ok, Conn} ->
             %% we don't care if this returns something or not, we just to test the connection
             try mongo_api:find_one(Conn, <<"foo">>, #{}, #{}) of
-                {error, _} -> false;
+                {error, _Reason} ->
+                    false;
                 _ ->
                     true
             catch
-                _Class:_Error -> false
+                _ : _ ->
+                    false
             end;
-        _ -> false
+        _ ->
+            false
     end.
 
 connect(Opts) ->

apps/emqx_connector/src/emqx_connector_mqtt.erl

@@ -29,7 +29,7 @@
         , bridges/0
         ]).
 
--export([on_message_received/2]).
+-export([on_message_received/3]).
 
 %% callbacks of behaviour emqx_resource
 -export([ on_start/2
@@ -68,10 +68,6 @@ fields("put") ->
 
 fields("post") ->
     [ {type, mk(mqtt, #{desc => "The Connector Type"})}
-    , {name, mk(binary(),
-        #{ desc => "The Connector Name"
-         , example => <<"my_mqtt_connector">>
-         })}
     ] ++ fields("put").
 
 %% ===================================================================
@@ -105,26 +101,29 @@ drop_bridge(Name) ->
     case supervisor:terminate_child(?MODULE, Name) of
         ok ->
             supervisor:delete_child(?MODULE, Name);
+        {error, not_found} ->
+            ok;
         {error, Error} ->
             {error, Error}
     end.
 
 %% ===================================================================
-%% When use this bridge as a data source, ?MODULE:on_message_received/2 will be called
+%% When use this bridge as a data source, ?MODULE:on_message_received will be called
 %% if the bridge received msgs from the remote broker.
-on_message_received(Msg, HookPoint) ->
+on_message_received(Msg, HookPoint, InstId) ->
+    _ = emqx_resource:query(InstId, {message_received, Msg}),
     emqx:run_hook(HookPoint, [Msg]).
 
 %% ===================================================================
 on_start(InstId, Conf) ->
     InstanceId = binary_to_atom(InstId, utf8),
-    ?SLOG(info, #{msg => "starting mqtt connector",
+    ?SLOG(info, #{msg => "starting_mqtt_connector",
                   connector => InstanceId, config => Conf}),
     BasicConf = basic_config(Conf),
     BridgeConf = BasicConf#{
         name => InstanceId,
-        clientid => clientid(maps:get(clientid, Conf, InstId)),
-        subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined)),
+        clientid => clientid(InstId),
+        subscriptions => make_sub_confs(maps:get(ingress, Conf, undefined), InstId),
         forwards => make_forward_confs(maps:get(egress, Conf, undefined))
     },
     case ?MODULE:create_bridge(BridgeConf) of
@@ -139,19 +138,21 @@ on_start(InstId, Conf) ->
     end.
 
 on_stop(_InstId, #{name := InstanceId}) ->
-    ?SLOG(info, #{msg => "stopping mqtt connector",
+    ?SLOG(info, #{msg => "stopping_mqtt_connector",
                   connector => InstanceId}),
     case ?MODULE:drop_bridge(InstanceId) of
         ok -> ok;
         {error, not_found} -> ok;
         {error, Reason} ->
-            ?SLOG(error, #{msg => "stop mqtt connector",
+            ?SLOG(error, #{msg => "stop_mqtt_connector",
                 connector => InstanceId, reason => Reason})
     end.
 
+on_query(_InstId, {message_received, _Msg}, AfterQuery, _State) ->
+    emqx_resource:query_success(AfterQuery);
+
 on_query(_InstId, {send_message, Msg}, AfterQuery, #{name := InstanceId}) ->
-    ?SLOG(debug, #{msg => "send msg to remote node", message => Msg,
-        connector => InstanceId}),
+    ?TRACE("QUERY", "send_msg_to_remote_node", #{message => Msg, connector => InstanceId}),
     emqx_connector_mqtt_worker:send_to_remote(InstanceId, Msg),
     emqx_resource:query_success(AfterQuery).
 
@@ -167,15 +168,15 @@ ensure_mqtt_worker_started(InstanceId) ->
         {error, Reason} -> {error, Reason}
     end.
 
-make_sub_confs(EmptyMap) when map_size(EmptyMap) == 0 ->
+make_sub_confs(EmptyMap, _) when map_size(EmptyMap) == 0 ->
     undefined;
-make_sub_confs(undefined) ->
+make_sub_confs(undefined, _) ->
     undefined;
-make_sub_confs(SubRemoteConf) ->
+make_sub_confs(SubRemoteConf, InstId) ->
     case maps:take(hookpoint, SubRemoteConf) of
         error -> SubRemoteConf;
         {HookPoint, SubConf} ->
-            MFA = {?MODULE, on_message_received, [HookPoint]},
+            MFA = {?MODULE, on_message_received, [HookPoint, InstId]},
             SubConf#{on_message_received => MFA}
     end.
 
@@ -208,7 +209,7 @@ basic_config(#{
         username => User,
         password => Password,
         clean_start => CleanStart,
-        keepalive => KeepAlive,
+        keepalive => ms_to_s(KeepAlive),
         retry_interval => RetryIntv,
         max_inflight => MaxInflight,
         ssl => EnableSsl,
@@ -216,5 +217,8 @@ basic_config(#{
         if_record_metrics => true
     }.
 
+ms_to_s(Ms) ->
+    erlang:ceil(Ms / 1000).
+
 clientid(Id) ->
     iolist_to_binary([Id, ":", atom_to_list(node())]).

apps/emqx_connector/src/emqx_connector_mysql.erl

@@ -56,7 +56,7 @@ on_start(InstId, #{server := {Host, Port},
                    auto_reconnect := AutoReconn,
                    pool_size := PoolSize,
                    ssl := SSL } = Config) ->
-    ?SLOG(info, #{msg => "starting mysql connector",
+    ?SLOG(info, #{msg => "starting_mysql_connector",
                   connector => InstId, config => Config}),
     SslOpts = case maps:get(enable, SSL) of
         true ->
@@ -75,7 +75,7 @@ on_start(InstId, #{server := {Host, Port},
     {ok, #{poolname => PoolName}}.
 
 on_stop(InstId, #{poolname := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping mysql connector",
+    ?SLOG(info, #{msg => "stopping_mysql_connector",
                   connector => InstId}),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
@@ -84,14 +84,13 @@ on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
 on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := _PoolName} = State) ->
     on_query(InstId, {sql, SQL, Params, default_timeout}, AfterQuery, State);
 on_query(InstId, {sql, SQL, Params, Timeout}, AfterQuery, #{poolname := PoolName} = State) ->
-    ?SLOG(debug, #{msg => "mysql connector received sql query",
-        connector => InstId, sql => SQL, state => State}),
+    ?TRACE("QUERY", "mysql_connector_received", #{connector => InstId, sql => SQL, state => State}),
     case Result = ecpool:pick_and_do(
                     PoolName,
                     {mysql, query, [SQL, Params, Timeout]},
                     no_handover) of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "mysql connector do sql query failed",
+            ?SLOG(error, #{msg => "mysql_connector_do_sql_query_failed",
                 connector => InstId, sql => SQL, reason => Reason}),
             emqx_resource:query_failed(AfterQuery);
         _ ->

apps/emqx_connector/src/emqx_connector_pgsql.erl

@@ -32,7 +32,9 @@
 
 -export([connect/1]).
 
--export([query/3]).
+-export([ query/3
+        , prepared_query/4
+        ]).
 
 -export([do_health_check/1]).
 
@@ -56,7 +58,7 @@ on_start(InstId, #{server := {Host, Port},
                    auto_reconnect := AutoReconn,
                    pool_size := PoolSize,
                    ssl := SSL } = Config) ->
-    ?SLOG(info, #{msg => "starting postgresql connector",
+    ?SLOG(info, #{msg => "starting_postgresql_connector",
                   connector => InstId, config => Config}),
     SslOpts = case maps:get(enable, SSL) of
                   true ->
@@ -65,7 +67,7 @@ on_start(InstId, #{server := {Host, Port},
                         emqx_plugin_libs_ssl:save_files_return_opts(SSL, "connectors", InstId)}];
                   false ->
                       [{ssl, false}]
-    end,
+              end,
     Options = [{host, Host},
                {port, Port},
                {username, User},
@@ -82,15 +84,19 @@ on_stop(InstId, #{poolname := PoolName}) ->
                   connector => InstId}),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
-on_query(InstId, {sql, SQL}, AfterQuery, #{poolname := _PoolName} = State) ->
-    on_query(InstId, {sql, SQL, []}, AfterQuery, State);
-on_query(InstId, {sql, SQL, Params}, AfterQuery, #{poolname := PoolName} = State) ->
-    ?SLOG(debug, #{msg => "postgresql connector received sql query",
-        connector => InstId, sql => SQL, state => State}),
-    case Result = ecpool:pick_and_do(PoolName, {?MODULE, query, [SQL, Params]}, no_handover) of
+on_query(InstId, QueryParams, AfterQuery, #{poolname := PoolName} = State) ->
+    {Command, Args} = case QueryParams of
+                          {query, SQL} -> {query, [SQL, []]};
+                          {query, SQL, Params} -> {query, [SQL, Params]};
+                          {prepared_query, Name, SQL} -> {prepared_query, [Name, SQL, []]};
+                          {prepared_query, Name, SQL, Params} -> {prepared_query, [Name, SQL, Params]}
+                      end,
+    ?TRACE("QUERY", "postgresql_connector_received",
+        #{connector => InstId, command => Command, args => Args, state => State}),
+    case Result = ecpool:pick_and_do(PoolName, {?MODULE, Command, Args}, no_handover) of
         {error, Reason} ->
             ?SLOG(error, #{
-                msg => "postgresql connector do sql query failed",
+                msg => "postgresql_connector_do_sql_query_failed",
                 connector => InstId, sql => SQL, reason => Reason}),
             emqx_resource:query_failed(AfterQuery);
         _ ->
@@ -117,6 +123,9 @@ connect(Opts) ->
 query(Conn, SQL, Params) ->
     epgsql:equery(Conn, SQL, Params).
 
+prepared_query(Conn, Name, SQL, Params) ->
+    epgsql:prepared_query2(Conn, Name, SQL, Params).
+
 conn_opts(Opts) ->
     conn_opts(Opts, []).
 conn_opts([], Acc) ->

apps/emqx_connector/src/emqx_connector_redis.erl

@@ -20,12 +20,19 @@
 -include_lib("emqx/include/logger.hrl").
 
 -type server() :: tuple().
-
+%% {"127.0.0.1", 7000}
+%% For eredis:start_link/1~7
 -reflect_type([server/0]).
-
 -typerefl_from_string({server/0, ?MODULE, to_server}).
 
--export([to_server/1]).
+-type servers() :: list().
+%% [{"127.0.0.1", 7000}, {"127.0.0.2", 7000}]
+%% For eredis_cluster
+-reflect_type([servers/0]).
+-typerefl_from_string({servers/0, ?MODULE, to_servers}).
+
+-export([ to_server/1
+        , to_servers/1]).
 
 -export([roots/0, fields/1]).
 
@@ -63,14 +70,14 @@ fields(single) ->
     redis_fields() ++
     emqx_connector_schema_lib:ssl_fields();
 fields(cluster) ->
-    [ {servers, #{type => hoconsc:array(server())}}
+    [ {servers, #{type => servers()}}
     , {redis_type, #{type => hoconsc:enum([cluster]),
                      default => cluster}}
     ] ++
     redis_fields() ++
     emqx_connector_schema_lib:ssl_fields();
 fields(sentinel) ->
-    [ {servers, #{type => hoconsc:array(server())}}
+    [ {servers, #{type => servers()}}
     , {redis_type, #{type => hoconsc:enum([sentinel]),
                      default => sentinel}}
     , {sentinel, #{type => string()}}
@@ -87,7 +94,7 @@ on_start(InstId, #{redis_type := Type,
                    pool_size := PoolSize,
                    auto_reconnect := AutoReconn,
                    ssl := SSL } = Config) ->
-    ?SLOG(info, #{msg => "starting redis connector",
+    ?SLOG(info, #{msg => "starting_redis_connector",
                   connector => InstId, config => Config}),
     Servers = case Type of
                 single -> [{servers, [maps:get(server, Config)]}];
@@ -120,20 +127,20 @@ on_start(InstId, #{redis_type := Type,
     {ok, #{poolname => PoolName, type => Type}}.
 
 on_stop(InstId, #{poolname := PoolName}) ->
-    ?SLOG(info, #{msg => "stopping redis connector",
+    ?SLOG(info, #{msg => "stopping_redis_connector",
                   connector => InstId}),
     emqx_plugin_libs_pool:stop_pool(PoolName).
 
 on_query(InstId, {cmd, Command}, AfterCommand, #{poolname := PoolName, type := Type} = State) ->
-    ?SLOG(debug, #{msg => "redis connector received cmd query",
-        connector => InstId, sql => Command, state => State}),
+    ?TRACE("QUERY", "redis_connector_received",
+        #{connector => InstId, sql => Command, state => State}),
     Result = case Type of
                  cluster -> eredis_cluster:q(PoolName, Command);
                  _ -> ecpool:pick_and_do(PoolName, {?MODULE, cmd, [Type, Command]}, no_handover)
              end,
     case Result of
         {error, Reason} ->
-            ?SLOG(error, #{msg => "redis connector do cmd query failed",
+            ?SLOG(error, #{msg => "redis_connector_do_cmd_query_failed",
                 connector => InstId, sql => Command, reason => Reason}),
             emqx_resource:query_failed(AfterCommand);
         _ ->
@@ -181,7 +188,23 @@ redis_fields() ->
     ].
 
 to_server(Server) ->
-    case string:tokens(Server, ":") of
-        [Host, Port] -> {ok, {Host, list_to_integer(Port)}};
-        _ -> {error, Server}
+    try {ok, parse_server(Server)}
+    catch
+        throw : Error  ->
+            Error
+    end.
+
+to_servers(Servers) ->
+    try {ok, lists:map(fun parse_server/1, string:tokens(Servers, ", "))}
+    catch
+        throw : _Reason ->
+            {error, Servers}
+    end.
+
+parse_server(Server) ->
+    case string:tokens(Server, ": ") of
+        [Host, Port] ->
+            {Host, list_to_integer(Port)};
+        _ ->
+            throw({error, Server})
     end.

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_mod.erl

@@ -158,27 +158,23 @@ handle_puback(#{packet_id := PktId, reason_code := RC}, Parent)
        RC =:= ?RC_NO_MATCHING_SUBSCRIBERS ->
     Parent ! {batch_ack, PktId}, ok;
 handle_puback(#{packet_id := PktId, reason_code := RC}, _Parent) ->
-    ?SLOG(warning, #{msg => "publish to remote node falied",
+    ?SLOG(warning, #{msg => "publish_to_remote_node_falied",
         packet_id => PktId, reason_code => RC}).
 
 handle_publish(Msg, undefined) ->
-    ?SLOG(error, #{msg => "cannot publish to local broker as"
-                          " 'ingress' is not configured",
+    ?SLOG(error, #{msg => "cannot_publish_to_local_broker_as"
+                          "_'ingress'_is_not_configured",
                    message => Msg});
-handle_publish(Msg, Vars) ->
-    ?SLOG(debug, #{msg => "publish to local broker",
+handle_publish(Msg0, Vars) ->
+    Msg = format_msg_received(Msg0),
+    ?SLOG(debug, #{msg => "publish_to_local_broker",
                    message => Msg, vars => Vars}),
-    emqx_metrics:inc('bridge.mqtt.message_received_from_remote', 1),
     case Vars of
         #{on_message_received := {Mod, Func, Args}} ->
             _ = erlang:apply(Mod, Func, [Msg | Args]);
         _ -> ok
     end,
-    case maps:get(local_topic, Vars, undefined) of
-        undefined -> ok;
-        _Topic ->
-            emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars))
-    end.
+    maybe_publish_to_local_broker(Msg0, Vars).
 
 handle_disconnected(Reason, Parent) ->
     Parent ! {disconnected, self(), Reason}.
@@ -198,3 +194,45 @@ sub_remote_topics(ClientPid, #{remote_topic := FromTopic, remote_qos := QoS}) ->
 
 process_config(Config) ->
     maps:without([conn_type, address, receive_mountpoint, subscriptions, name], Config).
+
+maybe_publish_to_local_broker(#{topic := Topic} = Msg, #{remote_topic := SubTopic} = Vars) ->
+    case maps:get(local_topic, Vars, undefined) of
+        undefined ->
+            ok; %% local topic is not set, discard it
+        _ ->
+            case emqx_topic:match(Topic, SubTopic) of
+                true ->
+                    _ = emqx_broker:publish(emqx_connector_mqtt_msg:to_broker_msg(Msg, Vars)),
+                    ok;
+                false ->
+                    ?SLOG(warning, #{msg => "discard_message_as_topic_not_matched",
+                        message => Msg, subscribed => SubTopic, got_topic => Topic})
+            end
+    end.
+
+format_msg_received(#{dup := Dup, payload := Payload, properties := Props,
+        qos := QoS, retain := Retain, topic := Topic}) ->
+    #{event => '$bridges/mqtt',
+      id => emqx_guid:to_hexstr(emqx_guid:gen()),
+      payload => Payload,
+      topic => Topic,
+      qos => QoS,
+      dup => Dup,
+      retain => Retain,
+      pub_props => printable_maps(Props),
+      timestamp => erlang:system_time(millisecond)
+    }.
+
+printable_maps(undefined) -> #{};
+printable_maps(Headers) ->
+    maps:fold(
+        fun ('User-Property', V0, AccIn) when is_list(V0) ->
+                AccIn#{
+                    'User-Property' => maps:from_list(V0),
+                    'User-Property-Pairs' => [#{
+                        key => Key,
+                        value => Value
+                     } || {Key, Value} <- V0]
+                };
+            (K, V0, AccIn) -> AccIn#{K => V0}
+        end, #{}, Headers).

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_msg.erl

@@ -61,12 +61,12 @@ make_pub_vars(Mountpoint, Conf) when is_map(Conf) ->
         -> exp_msg().
 to_remote_msg(#message{flags = Flags0} = Msg, Vars) ->
     Retain0 = maps:get(retain, Flags0, false),
-    MapMsg = maps:put(retain, Retain0, emqx_message:to_map(Msg)),
+    MapMsg = maps:put(retain, Retain0, emqx_rule_events:eventmsg_publish(Msg)),
     to_remote_msg(MapMsg, Vars);
 to_remote_msg(MapMsg, #{remote_topic := TopicToken, payload := PayloadToken,
         remote_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) when is_map(MapMsg) ->
     Topic = replace_vars_in_str(TopicToken, MapMsg),
-    Payload = replace_vars_in_str(PayloadToken, MapMsg),
+    Payload = process_payload(PayloadToken, MapMsg),
     QoS = replace_simple_var(QoSToken, MapMsg),
     Retain = replace_simple_var(RetainToken, MapMsg),
     #mqtt_msg{qos = QoS,
@@ -82,13 +82,18 @@ to_broker_msg(#{dup := Dup, properties := Props} = MapMsg,
             #{local_topic := TopicToken, payload := PayloadToken,
               local_qos := QoSToken, retain := RetainToken, mountpoint := Mountpoint}) ->
     Topic = replace_vars_in_str(TopicToken, MapMsg),
-    Payload = replace_vars_in_str(PayloadToken, MapMsg),
+    Payload = process_payload(PayloadToken, MapMsg),
     QoS = replace_simple_var(QoSToken, MapMsg),
     Retain = replace_simple_var(RetainToken, MapMsg),
     set_headers(Props,
         emqx_message:set_flags(#{dup => Dup, retain => Retain},
             emqx_message:make(bridge, QoS, topic(Mountpoint, Topic), Payload))).
 
+process_payload([], Msg) ->
+    emqx_json:encode(Msg);
+process_payload(Tks, Msg) ->
+    replace_vars_in_str(Tks, Msg).
+
 %% Replace a string contains vars to another string in which the placeholders are replace by the
 %% corresponding values. For example, given "a: ${var}", if the var=1, the result string will be:
 %% "a: 1".

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_schema.erl

@@ -39,7 +39,7 @@ fields("config") ->
 
 fields("connector") ->
     [ {mode,
-        sc(hoconsc:enum([cluster_singleton, cluster_shareload]),
+        sc(hoconsc:enum([cluster_shareload]),
            #{ default => cluster_shareload
             , desc => """
 The mode of the MQTT Bridge. Can be one of 'cluster_singleton' or 'cluster_shareload'<br>
@@ -55,12 +55,17 @@ clientid conflicts between different nodes. And we can only use shared subscript
 topic filters for 'remote_topic' of ingress connections.
 """
             })}
+    , {name,
+       sc(binary(),
+          #{ nullable => true
+           , desc => "Connector name, used as a human-readable description of the connector."
+           })}
     , {server,
         sc(emqx_schema:ip_port(),
            #{ default => "127.0.0.1:1883"
             , desc => "The host and port of the remote MQTT broker"
             })}
-    , {reconnect_interval, mk_duration("reconnect interval", #{default => "30s"})}
+    , {reconnect_interval, mk_duration("reconnect interval", #{default => "15s"})}
     , {proto_ver,
         sc(hoconsc:enum([v3, v4, v5]),
            #{ default => v4
@@ -76,17 +81,13 @@ topic filters for 'remote_topic' of ingress connections.
            #{ default => "emqx"
             , desc => "The password of the MQTT protocol"
             })}
-    , {clientid,
-        sc(binary(),
-           #{ desc => "The clientid of the MQTT protocol"
-            })}
     , {clean_start,
         sc(boolean(),
            #{ default => true
             , desc => "The clean-start or the clean-session of the MQTT protocol"
             })}
     , {keepalive, mk_duration("keepalive", #{default => "300s"})}
-    , {retry_interval, mk_duration("retry interval", #{default => "30s"})}
+    , {retry_interval, mk_duration("retry interval", #{default => "15s"})}
     , {max_inflight,
         sc(integer(),
            #{ default => 32

apps/emqx_connector/src/mqtt/emqx_connector_mqtt_worker.erl

@@ -188,7 +188,7 @@ callback_mode() -> [state_functions].
 
 %% @doc Config should be a map().
 init(#{name := Name} = ConnectOpts) ->
-    ?SLOG(debug, #{msg => "starting bridge worker",
+    ?SLOG(debug, #{msg => "starting_bridge_worker",
                    name => Name}),
     erlang:process_flag(trap_exit, true),
     Queue = open_replayq(Name, maps:get(replayq, ConnectOpts, #{})),
@@ -335,7 +335,7 @@ common(_StateName, cast, {send_to_remote, Msg}, #{replayq := Q} = State) ->
     NewQ = replayq:append(Q, [Msg]),
     {keep_state, State#{replayq => NewQ}, {next_event, internal, maybe_send}};
 common(StateName, Type, Content, #{name := Name} = State) ->
-    ?SLOG(notice, #{msg => "Bridge discarded event",
+    ?SLOG(notice, #{msg => "bridge_discarded_event",
         name => Name, type => Type, state_name => StateName,
         content => Content}),
     {keep_state, State}.
@@ -349,7 +349,7 @@ do_connect(#{connect_opts := ConnectOpts,
             {ok, State#{connection => Conn}};
         {error, Reason} ->
             ConnectOpts1 = obfuscate(ConnectOpts),
-            ?SLOG(error, #{msg => "Failed to connect",
+            ?SLOG(error, #{msg => "failed_to_connect",
                 config => ConnectOpts1, reason => Reason}),
             {error, Reason, State}
     end.
@@ -386,8 +386,8 @@ pop_and_send_loop(#{replayq := Q} = State, N) ->
     end.
 
 do_send(#{connect_opts := #{forwards := undefined}}, _QAckRef, Msg) ->
-    ?SLOG(error, #{msg => "cannot forward messages to remote broker"
-                          " as 'egress' is not configured",
+    ?SLOG(error, #{msg => "cannot_forward_messages_to_remote_broker"
+                          "_as_'egress'_is_not_configured",
                    messages => Msg});
 do_send(#{inflight := Inflight,
           connection := Connection,
@@ -398,7 +398,7 @@ do_send(#{inflight := Inflight,
                     emqx_metrics:inc('bridge.mqtt.message_sent_to_remote'),
                     emqx_connector_mqtt_msg:to_remote_msg(Message, Vars)
                 end,
-    ?SLOG(debug, #{msg => "publish to remote broker",
+    ?SLOG(debug, #{msg => "publish_to_remote_broker",
         message => Msg, vars => Vars}),
     case emqx_connector_mqtt_mod:send(Connection, [ExportMsg(Msg)]) of
         {ok, Refs} ->

apps/emqx_connector/test/emqx_connector_api_SUITE.erl

@@ -22,15 +22,15 @@
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("common_test/include/ct.hrl").
 
--define(CONF_DEFAULT, <<"connectors: {}">>).
+%% output functions
+-export([ inspect/3
+        ]).
+
 -define(BRIDGE_CONF_DEFAULT, <<"bridges: {}">>).
 -define(CONNECTR_TYPE, <<"mqtt">>).
 -define(CONNECTR_NAME, <<"test_connector">>).
--define(CONNECTR_ID, <<"mqtt:test_connector">>).
 -define(BRIDGE_NAME_INGRESS, <<"ingress_test_bridge">>).
 -define(BRIDGE_NAME_EGRESS, <<"egress_test_bridge">>).
--define(BRIDGE_ID_INGRESS, <<"mqtt:ingress_test_bridge">>).
--define(BRIDGE_ID_EGRESS, <<"mqtt:egress_test_bridge">>).
 -define(MQTT_CONNECOTR(Username),
 #{
     <<"server">> => <<"127.0.0.1:1883">>,
@@ -70,6 +70,9 @@
       <<"failed">> := FAILED, <<"rate">> := SPEED,
       <<"rate_last5m">> := SPEED5M, <<"rate_max">> := SPEEDMAX}).
 
+inspect(Selected, _Envs, _Args) ->
+    persistent_term:put(?MODULE, #{inspect => Selected}).
+
 all() ->
     emqx_common_test_helpers:all(?MODULE).
 
@@ -92,21 +95,38 @@ init_per_suite(Config) ->
     %% some testcases (may from other app) already get emqx_connector started
     _ = application:stop(emqx_resource),
     _ = application:stop(emqx_connector),
-    ok = emqx_common_test_helpers:start_apps([emqx_connector, emqx_bridge, emqx_dashboard]),
-    ok = emqx_config:init_load(emqx_connector_schema, ?CONF_DEFAULT),
+    ok = emqx_common_test_helpers:start_apps([emqx_rule_engine, emqx_connector,
+        emqx_bridge, emqx_dashboard]),
+    ok = emqx_config:init_load(emqx_connector_schema, <<"connectors: {}">>),
+    ok = emqx_config:init_load(emqx_rule_engine_schema, <<"rule_engine {rules {}}">>),
     ok = emqx_config:init_load(emqx_bridge_schema, ?BRIDGE_CONF_DEFAULT),
     Config.
 
 end_per_suite(_Config) ->
-    emqx_common_test_helpers:stop_apps([emqx_connector, emqx_bridge, emqx_dashboard]),
+    emqx_common_test_helpers:stop_apps([emqx_rule_engine, emqx_connector, emqx_bridge, emqx_dashboard]),
     ok.
 
 init_per_testcase(_, Config) ->
     {ok, _} = emqx_cluster_rpc:start_link(node(), emqx_cluster_rpc, 1000),
+    %% assert we there's no connectors and no bridges at first
+    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
+    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
     Config.
 end_per_testcase(_, _Config) ->
+    clear_resources(),
     ok.
 
+clear_resources() ->
+    lists:foreach(fun(#{id := Id}) ->
+            ok = emqx_rule_engine:delete_rule(Id)
+        end, emqx_rule_engine:get_rules()),
+    lists:foreach(fun(#{id := Id}) ->
+            ok = emqx_bridge:remove(Id)
+        end, emqx_bridge:list()),
+    lists:foreach(fun(#{<<"id">> := Id}) ->
+            ok = emqx_connector:delete(Id)
+        end, emqx_connector:list()).
+
 %%------------------------------------------------------------------------------
 %% Testcases
 %%------------------------------------------------------------------------------
@@ -123,32 +143,21 @@ t_mqtt_crud_apis(_) ->
                                , <<"name">> => ?CONNECTR_NAME
                                }),
 
-    %ct:pal("---connector: ~p", [Connector]),
-    ?assertMatch(#{ <<"id">> := ?CONNECTR_ID
-                  , <<"type">> := ?CONNECTR_TYPE
-                  , <<"name">> := ?CONNECTR_NAME
-                  , <<"server">> := <<"127.0.0.1:1883">>
-                  , <<"username">> := User1
-                  , <<"password">> := <<"">>
-                  , <<"proto_ver">> := <<"v4">>
-                  , <<"ssl">> := #{<<"enable">> := false}
-                  }, jsx:decode(Connector)),
-
-    %% create a again returns an error
-    {ok, 400, RetMsg} = request(post, uri(["connectors"]),
-        ?MQTT_CONNECOTR(User1)#{ <<"type">> => ?CONNECTR_TYPE
-                               , <<"name">> => ?CONNECTR_NAME
-                               }),
-    ?assertMatch(
-        #{ <<"code">> := _
-         , <<"message">> := <<"connector already exists">>
-         }, jsx:decode(RetMsg)),
+    #{ <<"id">> := ConnctorID
+     , <<"type">> := ?CONNECTR_TYPE
+     , <<"name">> := ?CONNECTR_NAME
+     , <<"server">> := <<"127.0.0.1:1883">>
+     , <<"username">> := User1
+     , <<"password">> := <<"">>
+     , <<"proto_ver">> := <<"v4">>
+     , <<"ssl">> := #{<<"enable">> := false}
+     } = jsx:decode(Connector),
 
     %% update the request-path of the connector
     User2 = <<"user2">>,
-    {ok, 200, Connector2} = request(put, uri(["connectors", ?CONNECTR_ID]),
+    {ok, 200, Connector2} = request(put, uri(["connectors", ConnctorID]),
                                  ?MQTT_CONNECOTR(User2)),
-    ?assertMatch(#{ <<"id">> := ?CONNECTR_ID
+    ?assertMatch(#{ <<"id">> := ConnctorID
                   , <<"server">> := <<"127.0.0.1:1883">>
                   , <<"username">> := User2
                   , <<"password">> := <<"">>
@@ -158,7 +167,7 @@ t_mqtt_crud_apis(_) ->
 
     %% list all connectors again, assert Connector2 is in it
     {ok, 200, Connector2Str} = request(get, uri(["connectors"]), []),
-    ?assertMatch([#{ <<"id">> := ?CONNECTR_ID
+    ?assertMatch([#{ <<"id">> := ConnctorID
                    , <<"type">> := ?CONNECTR_TYPE
                    , <<"name">> := ?CONNECTR_NAME
                    , <<"server">> := <<"127.0.0.1:1883">>
@@ -169,8 +178,8 @@ t_mqtt_crud_apis(_) ->
                    }], jsx:decode(Connector2Str)),
 
     %% get the connector by id
-    {ok, 200, Connector3Str} = request(get, uri(["connectors", ?CONNECTR_ID]), []),
-    ?assertMatch(#{ <<"id">> := ?CONNECTR_ID
+    {ok, 200, Connector3Str} = request(get, uri(["connectors", ConnctorID]), []),
+    ?assertMatch(#{ <<"id">> := ConnctorID
                   , <<"type">> := ?CONNECTR_TYPE
                   , <<"name">> := ?CONNECTR_NAME
                   , <<"server">> := <<"127.0.0.1:1883">>
@@ -181,11 +190,11 @@ t_mqtt_crud_apis(_) ->
                   }, jsx:decode(Connector3Str)),
 
     %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
+    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
 
     %% update a deleted connector returns an error
-    {ok, 404, ErrMsg2} = request(put, uri(["connectors", ?CONNECTR_ID]),
+    {ok, 404, ErrMsg2} = request(put, uri(["connectors", ConnctorID]),
                                  ?MQTT_CONNECOTR(User2)),
     ?assertMatch(
         #{ <<"code">> := _
@@ -194,10 +203,6 @@ t_mqtt_crud_apis(_) ->
     ok.
 
 t_mqtt_conn_bridge_ingress(_) ->
-    %% assert we there's no connectors and no bridges at first
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
-
     %% then we add a mqtt connector, using POST
     User1 = <<"user1">>,
     {ok, 201, Connector} = request(post, uri(["connectors"]),
@@ -205,28 +210,28 @@ t_mqtt_conn_bridge_ingress(_) ->
                                , <<"name">> => ?CONNECTR_NAME
                                }),
 
-    ?assertMatch(#{ <<"id">> := ?CONNECTR_ID
-                  , <<"server">> := <<"127.0.0.1:1883">>
-                  , <<"num_of_bridges">> := 0
-                  , <<"username">> := User1
-                  , <<"password">> := <<"">>
-                  , <<"proto_ver">> := <<"v4">>
-                  , <<"ssl">> := #{<<"enable">> := false}
-                  }, jsx:decode(Connector)),
+    #{ <<"id">> := ConnctorID
+     , <<"server">> := <<"127.0.0.1:1883">>
+     , <<"num_of_bridges">> := 0
+     , <<"username">> := User1
+     , <<"password">> := <<"">>
+     , <<"proto_ver">> := <<"v4">>
+     , <<"ssl">> := #{<<"enable">> := false}
+     } = jsx:decode(Connector),
 
     %% ... and a MQTT bridge, using POST
     %% we bind this bridge to the connector created just now
     {ok, 201, Bridge} = request(post, uri(["bridges"]),
-        ?MQTT_BRIDGE_INGRESS(?CONNECTR_ID)#{
+        ?MQTT_BRIDGE_INGRESS(ConnctorID)#{
             <<"type">> => ?CONNECTR_TYPE,
             <<"name">> => ?BRIDGE_NAME_INGRESS
         }),
 
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_INGRESS
-                  , <<"type">> := <<"mqtt">>
-                  , <<"status">> := <<"connected">>
-                  , <<"connector">> := ?CONNECTR_ID
-                  }, jsx:decode(Bridge)),
+    #{ <<"id">> := BridgeIDIngress
+     , <<"type">> := <<"mqtt">>
+     , <<"status">> := <<"connected">>
+     , <<"connector">> := ConnctorID
+     } = jsx:decode(Bridge),
 
     %% we now test if the bridge works as expected
 
@@ -236,8 +241,8 @@ t_mqtt_conn_bridge_ingress(_) ->
     emqx:subscribe(LocalTopic),
     %% PUBLISH a message to the 'remote' broker, as we have only one broker,
     %% the remote broker is also the local one.
+    wait_for_resource_ready(BridgeIDIngress, 5),
     emqx:publish(emqx_message:make(RemoteTopic, Payload)),
-
     %% we should receive a message on the local broker, with specified topic
     ?assert(
         receive
@@ -252,25 +257,21 @@ t_mqtt_conn_bridge_ingress(_) ->
         end),
 
     %% get the connector by id, verify the num_of_bridges now is 1
-    {ok, 200, Connector1Str} = request(get, uri(["connectors", ?CONNECTR_ID]), []),
-    ?assertMatch(#{ <<"id">> := ?CONNECTR_ID
+    {ok, 200, Connector1Str} = request(get, uri(["connectors", ConnctorID]), []),
+    ?assertMatch(#{ <<"id">> := ConnctorID
                   , <<"num_of_bridges">> := 1
                   }, jsx:decode(Connector1Str)),
 
     %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_INGRESS]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
     %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
+    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
     ok.
 
 t_mqtt_conn_bridge_egress(_) ->
-    %% assert we there's no connectors and no bridges at first
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
-
     %% then we add a mqtt connector, using POST
     User1 = <<"user1">>,
     {ok, 201, Connector} = request(post, uri(["connectors"]),
@@ -279,29 +280,28 @@ t_mqtt_conn_bridge_egress(_) ->
                                }),
 
     %ct:pal("---connector: ~p", [Connector]),
-    ?assertMatch(#{ <<"id">> := ?CONNECTR_ID
-                  , <<"server">> := <<"127.0.0.1:1883">>
-                  , <<"username">> := User1
-                  , <<"password">> := <<"">>
-                  , <<"proto_ver">> := <<"v4">>
-                  , <<"ssl">> := #{<<"enable">> := false}
-                  }, jsx:decode(Connector)),
+    #{ <<"id">> := ConnctorID
+     , <<"server">> := <<"127.0.0.1:1883">>
+     , <<"username">> := User1
+     , <<"password">> := <<"">>
+     , <<"proto_ver">> := <<"v4">>
+     , <<"ssl">> := #{<<"enable">> := false}
+     } = jsx:decode(Connector),
 
     %% ... and a MQTT bridge, using POST
     %% we bind this bridge to the connector created just now
     {ok, 201, Bridge} = request(post, uri(["bridges"]),
-        ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{
+        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
             <<"type">> => ?CONNECTR_TYPE,
             <<"name">> => ?BRIDGE_NAME_EGRESS
         }),
 
-    %ct:pal("---bridge: ~p", [Bridge]),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
-                  , <<"type">> := ?CONNECTR_TYPE
-                  , <<"name">> := ?BRIDGE_NAME_EGRESS
-                  , <<"status">> := <<"connected">>
-                  , <<"connector">> := ?CONNECTR_ID
-                  }, jsx:decode(Bridge)),
+    #{ <<"id">> := BridgeIDEgress
+     , <<"type">> := ?CONNECTR_TYPE
+     , <<"name">> := ?BRIDGE_NAME_EGRESS
+     , <<"status">> := <<"connected">>
+     , <<"connector">> := ConnctorID
+     } = jsx:decode(Bridge),
 
     %% we now test if the bridge works as expected
     LocalTopic = <<"local_topic/1">>,
@@ -310,6 +310,7 @@ t_mqtt_conn_bridge_egress(_) ->
     emqx:subscribe(RemoteTopic),
     %% PUBLISH a message to the 'local' broker, as we have only one broker,
     %% the remote broker is also the local one.
+    wait_for_resource_ready(BridgeIDEgress, 5),
     emqx:publish(emqx_message:make(LocalTopic, Payload)),
 
     %% we should receive a message on the "remote" broker, with specified topic
@@ -326,19 +327,19 @@ t_mqtt_conn_bridge_egress(_) ->
         end),
 
     %% verify the metrics of the bridge
-    {ok, 200, BridgeStr} = request(get, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
+    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    ?assertMatch(#{ <<"id">> := BridgeIDEgress
                   , <<"metrics">> := ?metrics(1, 1, 0, _, _, _)
                   , <<"node_metrics">> :=
                       [#{<<"node">> := _, <<"metrics">> := ?metrics(1, 1, 0, _, _, _)}]
                   }, jsx:decode(BridgeStr)),
 
     %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
     %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
+    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
     ok.
 
@@ -346,10 +347,6 @@ t_mqtt_conn_bridge_egress(_) ->
 %% - update a connector should also update all of the the bridges
 %% - cannot delete a connector that is used by at least one bridge
 t_mqtt_conn_update(_) ->
-    %% assert we there's no connectors and no bridges at first
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
-
     %% then we add a mqtt connector, using POST
     {ok, 201, Connector} = request(post, uri(["connectors"]),
                             ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)
@@ -358,44 +355,41 @@ t_mqtt_conn_update(_) ->
                                  }),
 
     %ct:pal("---connector: ~p", [Connector]),
-    ?assertMatch(#{ <<"id">> := ?CONNECTR_ID
-                  , <<"server">> := <<"127.0.0.1:1883">>
-                  }, jsx:decode(Connector)),
+    #{ <<"id">> := ConnctorID
+     , <<"server">> := <<"127.0.0.1:1883">>
+     } = jsx:decode(Connector),
 
     %% ... and a MQTT bridge, using POST
     %% we bind this bridge to the connector created just now
     {ok, 201, Bridge} = request(post, uri(["bridges"]),
-        ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{
+        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
             <<"type">> => ?CONNECTR_TYPE,
             <<"name">> => ?BRIDGE_NAME_EGRESS
         }),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
-                  , <<"type">> := <<"mqtt">>
-                  , <<"name">> := ?BRIDGE_NAME_EGRESS
-                  , <<"status">> := <<"connected">>
-                  , <<"connector">> := ?CONNECTR_ID
-                  }, jsx:decode(Bridge)),
+    #{ <<"id">> := BridgeIDEgress
+     , <<"type">> := <<"mqtt">>
+     , <<"name">> := ?BRIDGE_NAME_EGRESS
+     , <<"status">> := <<"connected">>
+     , <<"connector">> := ConnctorID
+     } = jsx:decode(Bridge),
+    wait_for_resource_ready(BridgeIDEgress, 2),
 
     %% then we try to update 'server' of the connector, to an unavailable IP address
     %% the update should fail because of 'unreachable' or 'connrefused'
-    {ok, 400, _ErrorMsg} = request(put, uri(["connectors", ?CONNECTR_ID]),
+    {ok, 400, _ErrorMsg} = request(put, uri(["connectors", ConnctorID]),
                                  ?MQTT_CONNECOTR2(<<"127.0.0.1:2603">>)),
     %% we fix the 'server' parameter to a normal one, it should work
-    {ok, 200, _} = request(put, uri(["connectors", ?CONNECTR_ID]),
+    {ok, 200, _} = request(put, uri(["connectors", ConnctorID]),
                                  ?MQTT_CONNECOTR2(<<"127.0.0.1 : 1883">>)),
     %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
     %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
+    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []).
 
 t_mqtt_conn_update2(_) ->
-    %% assert we there's no connectors and no bridges at first
-    {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []),
-    {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
-
     %% then we add a mqtt connector, using POST
     %% but this connector is point to a unreachable server "2603"
     {ok, 201, Connector} = request(post, uri(["connectors"]),
@@ -404,38 +398,71 @@ t_mqtt_conn_update2(_) ->
                                  , <<"name">> => ?CONNECTR_NAME
                                  }),
 
-    ?assertMatch(#{ <<"id">> := ?CONNECTR_ID
-                  , <<"server">> := <<"127.0.0.1:2603">>
-                  }, jsx:decode(Connector)),
+    #{ <<"id">> := ConnctorID
+     , <<"server">> := <<"127.0.0.1:2603">>
+     } = jsx:decode(Connector),
 
     %% ... and a MQTT bridge, using POST
     %% we bind this bridge to the connector created just now
     {ok, 201, Bridge} = request(post, uri(["bridges"]),
-        ?MQTT_BRIDGE_EGRESS(?CONNECTR_ID)#{
+        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
             <<"type">> => ?CONNECTR_TYPE,
             <<"name">> => ?BRIDGE_NAME_EGRESS
         }),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
-                  , <<"type">> := <<"mqtt">>
-                  , <<"name">> := ?BRIDGE_NAME_EGRESS
-                  , <<"status">> := <<"disconnected">>
-                  , <<"connector">> := ?CONNECTR_ID
-                  }, jsx:decode(Bridge)),
+    #{ <<"id">> := BridgeIDEgress
+     , <<"type">> := <<"mqtt">>
+     , <<"name">> := ?BRIDGE_NAME_EGRESS
+     , <<"status">> := <<"disconnected">>
+     , <<"connector">> := ConnctorID
+     } = jsx:decode(Bridge),
+    %% We try to fix the 'server' parameter, to another unavailable server..
+    %% The update should success: we don't check the connectivity of the new config
+    %% if the resource is now disconnected.
+    {ok, 200, _} = request(put, uri(["connectors", ConnctorID]),
+                                 ?MQTT_CONNECOTR2(<<"127.0.0.1:2604">>)),
     %% we fix the 'server' parameter to a normal one, it should work
-    {ok, 200, _} = request(put, uri(["connectors", ?CONNECTR_ID]),
+    {ok, 200, _} = request(put, uri(["connectors", ConnctorID]),
                                  ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)),
-    {ok, 200, BridgeStr} = request(get, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
-    ?assertMatch(#{ <<"id">> := ?BRIDGE_ID_EGRESS
+    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    ?assertMatch(#{ <<"id">> := BridgeIDEgress
                   , <<"status">> := <<"connected">>
                   }, jsx:decode(BridgeStr)),
     %% delete the bridge
-    {ok, 204, <<>>} = request(delete, uri(["bridges", ?BRIDGE_ID_EGRESS]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["bridges"]), []),
 
     %% delete the connector
-    {ok, 204, <<>>} = request(delete, uri(["connectors", ?CONNECTR_ID]), []),
+    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []),
     {ok, 200, <<"[]">>} = request(get, uri(["connectors"]), []).
 
+t_mqtt_conn_update3(_) ->
+    %% we add a mqtt connector, using POST
+    {ok, 201, Connector} = request(post, uri(["connectors"]),
+                            ?MQTT_CONNECOTR2(<<"127.0.0.1:1883">>)
+                                #{ <<"type">> => ?CONNECTR_TYPE
+                                 , <<"name">> => ?CONNECTR_NAME
+                                 }),
+    #{ <<"id">> := ConnctorID } = jsx:decode(Connector),
+
+    %% ... and a MQTT bridge, using POST
+    %% we bind this bridge to the connector created just now
+    {ok, 201, Bridge} = request(post, uri(["bridges"]),
+        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
+            <<"type">> => ?CONNECTR_TYPE,
+            <<"name">> => ?BRIDGE_NAME_EGRESS
+        }),
+    #{ <<"id">> := BridgeIDEgress
+     , <<"connector">> := ConnctorID
+     } = jsx:decode(Bridge),
+    wait_for_resource_ready(BridgeIDEgress, 2),
+
+    %% delete the connector should fail because it is in use by a bridge
+    {ok, 403, _} = request(delete, uri(["connectors", ConnctorID]), []),
+    %% delete the bridge
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
+    %% the connector now can be deleted without problems
+    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
+
 t_mqtt_conn_testing(_) ->
     %% APIs for testing the connectivity
     %% then we add a mqtt connector, using POST
@@ -450,6 +477,153 @@ t_mqtt_conn_testing(_) ->
             <<"name">> => ?BRIDGE_NAME_EGRESS
         }).
 
+t_ingress_mqtt_bridge_with_rules(_) ->
+    {ok, 201, Connector} = request(post, uri(["connectors"]),
+        ?MQTT_CONNECOTR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE
+                               , <<"name">> => ?CONNECTR_NAME
+                               }),
+    #{ <<"id">> := ConnctorID } = jsx:decode(Connector),
+
+    {ok, 201, Bridge} = request(post, uri(["bridges"]),
+        ?MQTT_BRIDGE_INGRESS(ConnctorID)#{
+            <<"type">> => ?CONNECTR_TYPE,
+            <<"name">> => ?BRIDGE_NAME_INGRESS
+        }),
+    #{ <<"id">> := BridgeIDIngress } = jsx:decode(Bridge),
+
+    {ok, 201, Rule} = request(post, uri(["rules"]),
+        #{<<"name">> => <<"A rule get messages from a source mqtt bridge">>,
+          <<"enable">> => true,
+          <<"outputs">> => [#{<<"function">> => "emqx_connector_api_SUITE:inspect"}],
+          <<"sql">> => <<"SELECT * from \"$bridges/", BridgeIDIngress/binary, "\"">>
+        }),
+    #{<<"id">> := RuleId} = jsx:decode(Rule),
+
+    %% we now test if the bridge works as expected
+
+    RemoteTopic = <<"remote_topic/1">>,
+    LocalTopic = <<"local_topic/", RemoteTopic/binary>>,
+    Payload = <<"hello">>,
+    emqx:subscribe(LocalTopic),
+    %% PUBLISH a message to the 'remote' broker, as we have only one broker,
+    %% the remote broker is also the local one.
+    wait_for_resource_ready(BridgeIDIngress, 5),
+    emqx:publish(emqx_message:make(RemoteTopic, Payload)),
+    %% we should receive a message on the local broker, with specified topic
+    ?assert(
+        receive
+            {deliver, LocalTopic, #message{payload = Payload}} ->
+                ct:pal("local broker got message: ~p on topic ~p", [Payload, LocalTopic]),
+                true;
+            Msg ->
+                ct:pal("Msg: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end),
+    %% and also the rule should be matched, with matched + 1:
+    {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
+    #{ <<"id">> := RuleId
+     , <<"metrics">> := #{<<"matched">> := 1}
+     } = jsx:decode(Rule1),
+    %% we also check if the outputs of the rule is triggered
+    ?assertMatch(#{inspect := #{
+        event := '$bridges/mqtt',
+        id := MsgId,
+        payload := Payload,
+        topic := RemoteTopic,
+        qos := 0,
+        dup := false,
+        retain := false,
+        pub_props := #{},
+        timestamp := _
+    }} when is_binary(MsgId), persistent_term:get(?MODULE)),
+
+    {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDIngress]), []),
+    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
+
+t_egress_mqtt_bridge_with_rules(_) ->
+    {ok, 201, Connector} = request(post, uri(["connectors"]),
+        ?MQTT_CONNECOTR(<<"user1">>)#{ <<"type">> => ?CONNECTR_TYPE
+                               , <<"name">> => ?CONNECTR_NAME
+                               }),
+    #{ <<"id">> := ConnctorID } = jsx:decode(Connector),
+
+    {ok, 201, Bridge} = request(post, uri(["bridges"]),
+        ?MQTT_BRIDGE_EGRESS(ConnctorID)#{
+            <<"type">> => ?CONNECTR_TYPE,
+            <<"name">> => ?BRIDGE_NAME_EGRESS
+        }),
+    #{ <<"id">> := BridgeIDEgress } = jsx:decode(Bridge),
+
+    {ok, 201, Rule} = request(post, uri(["rules"]),
+        #{<<"name">> => <<"A rule send messages to a sink mqtt bridge">>,
+          <<"enable">> => true,
+          <<"outputs">> => [BridgeIDEgress],
+          <<"sql">> => <<"SELECT * from \"t/1\"">>
+        }),
+    #{<<"id">> := RuleId} = jsx:decode(Rule),
+
+    %% we now test if the bridge works as expected
+    LocalTopic = <<"local_topic/1">>,
+    RemoteTopic = <<"remote_topic/", LocalTopic/binary>>,
+    Payload = <<"hello">>,
+    emqx:subscribe(RemoteTopic),
+    %% PUBLISH a message to the 'local' broker, as we have only one broker,
+    %% the remote broker is also the local one.
+    wait_for_resource_ready(BridgeIDEgress, 5),
+    emqx:publish(emqx_message:make(LocalTopic, Payload)),
+    %% we should receive a message on the "remote" broker, with specified topic
+    ?assert(
+        receive
+            {deliver, RemoteTopic, #message{payload = Payload}} ->
+                ct:pal("local broker got message: ~p on topic ~p", [Payload, RemoteTopic]),
+                true;
+            Msg ->
+                ct:pal("Msg: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end),
+    emqx:unsubscribe(RemoteTopic),
+
+    %% PUBLISH a message to the rule.
+    Payload2 = <<"hi">>,
+    RuleTopic = <<"t/1">>,
+    RemoteTopic2 = <<"remote_topic/", RuleTopic/binary>>,
+    emqx:subscribe(RemoteTopic2),
+    wait_for_resource_ready(BridgeIDEgress, 5),
+    emqx:publish(emqx_message:make(RuleTopic, Payload2)),
+    {ok, 200, Rule1} = request(get, uri(["rules", RuleId]), []),
+    #{ <<"id">> := RuleId
+     , <<"metrics">> := #{<<"matched">> := 1}
+     } = jsx:decode(Rule1),
+    %% we should receive a message on the "remote" broker, with specified topic
+    ?assert(
+        receive
+            {deliver, RemoteTopic2, #message{payload = Payload2}} ->
+                ct:pal("local broker got message: ~p on topic ~p", [Payload2, RemoteTopic2]),
+                true;
+            Msg ->
+                ct:pal("Msg: ~p", [Msg]),
+                false
+        after 100 ->
+            false
+        end),
+
+    %% verify the metrics of the bridge
+    {ok, 200, BridgeStr} = request(get, uri(["bridges", BridgeIDEgress]), []),
+    ?assertMatch(#{ <<"id">> := BridgeIDEgress
+                  , <<"metrics">> := ?metrics(2, 2, 0, _, _, _)
+                  , <<"node_metrics">> :=
+                      [#{<<"node">> := _, <<"metrics">> := ?metrics(2, 2, 0, _, _, _)}]
+                  }, jsx:decode(BridgeStr)),
+
+    {ok, 204, <<>>} = request(delete, uri(["rules", RuleId]), []),
+    {ok, 204, <<>>} = request(delete, uri(["bridges", BridgeIDEgress]), []),
+    {ok, 204, <<>>} = request(delete, uri(["connectors", ConnctorID]), []).
+
 %%--------------------------------------------------------------------
 %% HTTP Request
 %%--------------------------------------------------------------------
@@ -483,3 +657,13 @@ auth_header_() ->
     {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
     {"Authorization", "Bearer " ++ binary_to_list(Token)}.
 
+wait_for_resource_ready(InstId, 0) ->
+    ct:pal("--- bridge ~p: ~p", [InstId, emqx_bridge:lookup(InstId)]),
+    ct:fail(wait_resource_timeout);
+wait_for_resource_ready(InstId, Retry) ->
+    case emqx_bridge:lookup(InstId) of
+        {ok, #{resource_data := #{status := started}}} -> ok;
+        _ ->
+            timer:sleep(100),
+            wait_for_resource_ready(InstId, Retry-1)
+    end.

apps/emqx_dashboard/rebar.config

@@ -1,4 +1,4 @@
-{deps, [ {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.5"}}}
+{deps, [ {typerefl, {git, "https://github.com/k32/typerefl", {tag, "0.8.6"}}}
        , {emqx, {path, "../emqx"}}
        ]}.
 

apps/emqx_dashboard/src/emqx_dashboard.erl

@@ -151,9 +151,9 @@ authorize(Req) ->
                 ok ->
                     ok;
                 {error, token_timeout} ->
-                    return_unauthorized(<<"TOKEN_TIME_OUT">>, <<"POST '/login', get new token">>);
+                    {401, 'TOKEN_TIME_OUT', <<"Token expired, get new token by POST /login">>};
                 {error, not_found} ->
-                    return_unauthorized(<<"BAD_TOKEN">>, <<"POST '/login'">>)
+                    {401, 'BAD_TOKEN', <<"Get a token by POST /login">>}
             end;
         _ ->
             return_unauthorized(<<"AUTHORIZATION_HEADER_ERROR">>,

apps/emqx_dashboard/src/emqx_dashboard_api.erl

@@ -123,7 +123,7 @@ schema("/users/:username") ->
                 #{in => path, example => <<"admin">>})}],
             'requestBody' => [
                 { description
-                , mk(emqx_schema:unicode_binary(),
+                , mk(binary(),
                     #{desc => <<"User description">>, example => <<"administrator">>})}
             ],
             responses => #{
@@ -176,7 +176,7 @@ schema("/users/:username/change_pwd") ->
 fields(user) ->
     [
         {description,
-            mk(emqx_schema:unicode_binary(),
+            mk(binary(),
                 #{desc => <<"User description">>, example => "administrator"})},
         {username,
             mk(binary(),

apps/emqx_dashboard/src/emqx_dashboard_swagger.erl

@@ -312,6 +312,9 @@ responses(Responses, Module) ->
 
 response(Status, Bin, {Acc, RefsAcc, Module}) when is_binary(Bin) ->
     {Acc#{integer_to_binary(Status) => #{description => Bin}}, RefsAcc, Module};
+%% Support swagger raw object(file download).
+response(Status, #{content := _} = Content, {Acc, RefsAcc, Module}) ->
+    {Acc#{integer_to_binary(Status) => Content}, RefsAcc, Module};
 response(Status, ?REF(StructName), {Acc, RefsAcc, Module}) ->
     response(Status, ?R_REF(Module, StructName), {Acc, RefsAcc, Module});
 response(Status, ?R_REF(_Mod, _Name) = RRef, {Acc, RefsAcc, Module}) ->
@@ -423,8 +426,10 @@ typename_to_spec("duration_ms()", _Mod) -> #{type => string, example => <<"32s">
 typename_to_spec("percent()", _Mod) -> #{type => number, example => <<"12%">>};
 typename_to_spec("file()", _Mod) -> #{type => string, example => <<"/path/to/file">>};
 typename_to_spec("ip_port()", _Mod) -> #{type => string, example => <<"127.0.0.1:80">>};
+typename_to_spec("ip_ports()", _Mod) -> #{type => string, example => <<"127.0.0.1:80, 127.0.0.2:80">>};
 typename_to_spec("url()", _Mod) -> #{type => string, example => <<"http://127.0.0.1">>};
 typename_to_spec("server()", Mod) -> typename_to_spec("ip_port()", Mod);
+typename_to_spec("servers()", Mod) -> typename_to_spec("ip_ports()", Mod);
 typename_to_spec("connect_timeout()", Mod) -> typename_to_spec("timeout()", Mod);
 typename_to_spec("timeout()", _Mod) -> #{<<"oneOf">> => [#{type => string, example => infinity},
     #{type => integer, example => 100}], example => infinity};

apps/emqx_gateway/src/coap/emqx_coap_impl.erl

@@ -16,9 +16,16 @@
 
 -module(emqx_coap_impl).
 
+-behaviour(emqx_gateway_impl).
+
+-include_lib("emqx/include/logger.hrl").
 -include_lib("emqx_gateway/include/emqx_gateway.hrl").
 
--behaviour(emqx_gateway_impl).
+-import(emqx_gateway_utils,
+        [ normalize_config/1
+        , start_listeners/4
+        , stop_listeners/2
+        ]).
 
 %% APIs
 -export([ reg/0
@@ -30,8 +37,6 @@
         , on_gateway_unload/2
         ]).
 
--include_lib("emqx/include/logger.hrl").
-
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -51,12 +56,20 @@ unreg() ->
 on_gateway_load(_Gateway = #{name := GwName,
                              config := Config
                             }, Ctx) ->
-    Listeners = emqx_gateway_utils:normalize_config(Config),
-    ListenerPids = lists:map(fun(Lis) ->
-                                     start_listener(GwName, Ctx, Lis)
-                             end, Listeners),
-
-    {ok, ListenerPids,  #{ctx => Ctx}}.
+    Listeners = normalize_config(Config),
+    ModCfg = #{frame_mod => emqx_coap_frame,
+               chann_mod => emqx_coap_channel
+              },
+    case start_listeners(
+           Listeners, GwName, Ctx, ModCfg) of
+        {ok, ListenerPids} ->
+            {ok, ListenerPids,  #{ctx => Ctx}};
+        {error, {Reason, Listener}} ->
+            throw({badconf, #{ key => listeners
+                             , vallue => Listener
+                             , reason => Reason
+                             }})
+    end.
 
 on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
     GwName = maps:get(name, Gateway),
@@ -76,63 +89,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
 on_gateway_unload(_Gateway = #{ name := GwName,
                                 config := Config
                               }, _GwState) ->
-    Listeners = emqx_gateway_utils:normalize_config(Config),
-    lists:foreach(fun(Lis) ->
-        stop_listener(GwName, Lis)
-    end, Listeners).
-
-%%--------------------------------------------------------------------
-%% Internal funcs
-%%--------------------------------------------------------------------
-
-start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
-        {ok, Pid} ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                          [GwName, Type, LisName, ListenOnStr]),
-            Pid;
-        {error, Reason} ->
-            ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason]),
-            throw({badconf, Reason})
-    end.
-
-start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    NCfg = Cfg#{ctx => Ctx,
-                listener => {GwName, Type, LisName},
-                frame_mod => emqx_coap_frame,
-                chann_mod => emqx_coap_channel
-               },
-    MFA = {emqx_gateway_conn, start_link, [NCfg]},
-    do_start_listener(Type, Name, ListenOn, SocketOpts, MFA).
-
-do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) ->
-    esockd:open_udp(Name, ListenOn, SocketOpts, MFA);
-
-do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) ->
-    esockd:open_dtls(Name, ListenOn, SocketOpts, MFA).
-
-stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case StopRet of
-        ok ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
-                          [GwName, Type, LisName, ListenOnStr]);
-        {error, Reason} ->
-            ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason])
-    end,
-    StopRet.
-
-stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    esockd:close(Name, ListenOn).
-
--ifndef(TEST).
-console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
--else.
-console_print(_Fmt, _Args) -> ok.
--endif.
+    Listeners = normalize_config(Config),
+    stop_listeners(GwName, Listeners).

apps/emqx_gateway/src/emqx_gateway_api_clients.erl

@@ -532,7 +532,21 @@ params_client_searching_in_qs() ->
     , {lte_connected_at,
        mk(binary(),
           M#{desc => <<"Match the client socket connected datatime less than "
-                       " a certain value">>})}
+                       "a certain value">>})}
+    , {endpoint_name,
+       mk(binary(),
+          M#{desc => <<"Match the lwm2m client's endpoint name">>})}
+    , {like_endpoint_name,
+       mk(binary(),
+          M#{desc => <<"Use sub-string to match lwm2m client's endpoint name">>})}
+    , {gte_lifetime,
+       mk(binary(),
+          M#{desc => <<"Match the lwm2m client registered lifetime greater "
+                        "than a certain value">>})}
+    , {lte_lifetime,
+       mk(binary(),
+          M#{desc => <<"Match the lwm2m client registered lifetime less than "
+                       "a certain value">>})}
     ].
 
 params_paging() ->

apps/emqx_gateway/src/emqx_gateway_api_listeners.erl

@@ -580,7 +580,7 @@ common_listener_opts() ->
           #{ nullable => {true, recursively}
            , desc => <<"The authenticatior for this listener">>
            })}
-    ].
+    ] ++ emqx_gateway_schema:proxy_protocol_opts().
 
 %%--------------------------------------------------------------------
 %% examples

apps/emqx_gateway/src/emqx_gateway_cli.erl

@@ -28,6 +28,8 @@
         %, 'gateway-banned'/1
         ]).
 
+-elvis([{elvis_style, function_naming_convention, disable}]).
+
 -spec load() -> ok.
 load() ->
     Cmds = [Fun || {Fun, _} <- ?MODULE:module_info(exports), is_cmd(Fun)],
@@ -50,18 +52,24 @@ is_cmd(Fun) ->
 %% Cmds
 
 gateway(["list"]) ->
-    lists:foreach(fun(#{name := Name} = Gateway) ->
-        %% TODO: More infos: listeners?, connected?
-        Status = maps:get(status, Gateway, stopped),
-        print("Gateway(name=~ts, status=~ts)~n", [Name, Status])
-    end, emqx_gateway:list());
+    lists:foreach(
+      fun (#{name := Name, status := unloaded}) ->
+            print("Gateway(name=~ts, status=unloaded)\n", [Name]);
+          (#{name := Name, status := stopped, stopped_at := StoppedAt}) ->
+            print("Gateway(name=~ts, status=stopped, stopped_at=~ts)\n",
+                  [Name, StoppedAt]);
+          (#{name := Name, status := running, current_connections := ConnCnt,
+             started_at := StartedAt}) ->
+            print("Gateway(name=~ts, status=running, clients=~w, started_at=~ts)\n",
+                  [Name, ConnCnt, StartedAt])
+    end, emqx_gateway_http:gateways(all));
 
 gateway(["lookup", Name]) ->
     case emqx_gateway:lookup(atom(Name)) of
         undefined ->
-            print("undefined~n");
+            print("undefined\n");
         Info ->
-            print("~p~n", [Info])
+            print("~p\n", [Info])
     end;
 
 gateway(["load", Name, Conf]) ->
@@ -70,17 +78,17 @@ gateway(["load", Name, Conf]) ->
            emqx_json:decode(Conf, [return_maps])
           ) of
         {ok, _} ->
-            print("ok~n");
+            print("ok\n");
         {error, Reason} ->
-            print("Error: ~p~n", [Reason])
+            print("Error: ~p\n", [Reason])
     end;
 
 gateway(["unload", Name]) ->
     case emqx_gateway_conf:unload_gateway(bin(Name)) of
         ok ->
-            print("ok~n");
+            print("ok\n");
         {error, Reason} ->
-            print("Error: ~p~n", [Reason])
+            print("Error: ~p\n", [Reason])
     end;
 
 gateway(["stop", Name]) ->
@@ -89,9 +97,9 @@ gateway(["stop", Name]) ->
            #{<<"enable">> => <<"false">>}
           ) of
         {ok, _} ->
-            print("ok~n");
+            print("ok\n");
         {error, Reason} ->
-            print("Error: ~p~n", [Reason])
+            print("Error: ~p\n", [Reason])
     end;
 
 gateway(["start", Name]) ->
@@ -100,9 +108,9 @@ gateway(["start", Name]) ->
            #{<<"enable">> => <<"true">>}
           ) of
         {ok, _} ->
-            print("ok~n");
+            print("ok\n");
         {error, Reason} ->
-            print("Error: ~p~n", [Reason])
+            print("Error: ~p\n", [Reason])
     end;
 
 gateway(_) ->
@@ -123,7 +131,7 @@ gateway(_) ->
 'gateway-registry'(["list"]) ->
     lists:foreach(
       fun({Name, #{cbkmod := CbMod}}) ->
-        print("Registered Name: ~ts, Callback Module: ~ts~n", [Name, CbMod])
+        print("Registered Name: ~ts, Callback Module: ~ts\n", [Name, CbMod])
       end,
     emqx_gateway_registry:list());
 
@@ -137,15 +145,15 @@ gateway(_) ->
     InfoTab = emqx_gateway_cm:tabname(info, Name),
     case ets:info(InfoTab) of
         undefined ->
-            print("Bad Gateway Name.~n");
+            print("Bad Gateway Name.\n");
         _ ->
-        dump(InfoTab, client)
+            dump(InfoTab, client)
     end;
 
 'gateway-clients'(["lookup", Name, ClientId]) ->
     ChanTab = emqx_gateway_cm:tabname(chan, Name),
     case ets:lookup(ChanTab, bin(ClientId)) of
-        [] -> print("Not Found.~n");
+        [] -> print("Not Found.\n");
         [Chann] ->
             InfoTab = emqx_gateway_cm:tabname(info, Name),
             [ChannInfo] = ets:lookup(InfoTab, Chann),
@@ -154,8 +162,8 @@ gateway(_) ->
 
 'gateway-clients'(["kick", Name, ClientId]) ->
     case emqx_gateway_cm:kick_session(Name, bin(ClientId)) of
-        ok -> print("ok~n");
-        _ -> print("Not Found.~n")
+        ok -> print("ok\n");
+        _ -> print("Not Found.\n")
     end;
 
 'gateway-clients'(_) ->
@@ -171,11 +179,11 @@ gateway(_) ->
     Tab = emqx_gateway_metrics:tabname(Name),
     case ets:info(Tab) of
         undefined ->
-            print("Bad Gateway Name.~n");
+            print("Bad Gateway Name.\n");
         _ ->
             lists:foreach(
               fun({K, V}) ->
-                print("~-30s: ~w~n", [K, V])
+                print("~-30s: ~w\n", [K, V])
               end, lists:sort(ets:tab2list(Tab)))
     end;
 
@@ -232,7 +240,7 @@ print_record({client, {_, Infos, Stats}}) ->
     print("Client(~ts, username=~ts, peername=~ts, "
           "clean_start=~ts, keepalive=~w, "
           "subscriptions=~w, delivered_msgs=~w, "
-          "connected=~ts, created_at=~w, connected_at=~w)~n",
+          "connected=~ts, created_at=~w, connected_at=~w)\n",
           [format(K, maps:get(K, Info)) || K <- InfoKeys]).
 
 print(S) -> emqx_ctl:print(S).

apps/emqx_gateway/src/emqx_gateway_schema.erl

@@ -50,6 +50,8 @@
 
 -export([namespace/0, roots/0 , fields/1]).
 
+-export([proxy_protocol_opts/0]).
+
 namespace() -> gateway.
 
 roots() -> [gateway].

apps/emqx_gateway/src/emqx_gateway_utils.erl

@@ -18,6 +18,7 @@
 -module(emqx_gateway_utils).
 
 -include("emqx_gateway.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -export([ childspec/2
         , childspec/3
@@ -26,6 +27,12 @@
         , find_sup_child/2
         ]).
 
+-export([ start_listeners/4
+        , start_listener/4
+        , stop_listeners/2
+        , stop_listener/2
+        ]).
+
 -export([ apply/2
         , format_listenon/1
         , parse_listenon/1
@@ -89,9 +96,15 @@ childspec(Id, Type, Mod, Args) ->
 -spec supervisor_ret(supervisor:startchild_ret())
     -> {ok, pid()}
      | {error, supervisor:startchild_err()}.
-supervisor_ret({ok, Pid, _Info}) -> {ok, Pid};
-supervisor_ret({error, {Reason, _Child}}) -> {error, Reason};
-supervisor_ret(Ret) -> Ret.
+supervisor_ret({ok, Pid, _Info}) ->
+    {ok, Pid};
+supervisor_ret({error, {Reason, Child}}) ->
+    case element(1, Child) == child of
+        true -> {error, Reason};
+        _ -> {error, {Reason, Child}}
+    end;
+supervisor_ret(Ret) ->
+    Ret.
 
 -spec find_sup_child(Sup :: pid() | atom(), ChildId :: supervisor:child_id())
     -> false
@@ -102,6 +115,120 @@ find_sup_child(Sup, ChildId) ->
         {_Id, Pid, _Type, _Mods} -> {ok, Pid}
     end.
 
+%% @doc start listeners. close all listeners if someone failed
+-spec start_listeners(Listeners :: list(),
+                      GwName :: atom(),
+                      Ctx :: map(),
+                      ModCfg)
+    -> {ok, [pid()]}
+     | {error, term()}
+    when ModCfg :: #{frame_mod := atom(), chann_mod := atom()}.
+start_listeners(Listeners, GwName, Ctx, ModCfg) ->
+    start_listeners(Listeners, GwName, Ctx, ModCfg, []).
+
+start_listeners([], _, _, _, Acc) ->
+    {ok, lists:map(fun({listener, {_, Pid}}) -> Pid end, Acc)};
+start_listeners([L | Ls], GwName, Ctx, ModCfg, Acc) ->
+    case start_listener(GwName, Ctx, L, ModCfg) of
+        {ok, {ListenerId, ListenOn, Pid}} ->
+            NAcc = Acc ++ [{listener, {{ListenerId, ListenOn}, Pid}}],
+            start_listeners(Ls, GwName, Ctx, ModCfg, NAcc);
+        {error, Reason} ->
+            lists:foreach(fun({listener, {{ListenerId, ListenOn}, _}}) ->
+                esockd:close({ListenerId, ListenOn})
+            end, Acc),
+            {error, {Reason, L}}
+    end.
+
+-spec start_listener(GwName :: atom(),
+                     Ctx :: emqx_gateway_ctx:context(),
+                     Listener :: tuple(),
+                     ModCfg :: map())
+    -> {ok, {ListenerId :: atom(), esockd:listen_on(), pid()}}
+     | {error, term()}.
+start_listener(GwName, Ctx,
+               {Type, LisName, ListenOn, SocketOpts, Cfg}, ModCfg) ->
+    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
+    ListenerId  = emqx_gateway_utils:listener_id(GwName, Type, LisName),
+
+    NCfg = maps:merge(Cfg, ModCfg),
+    case start_listener(GwName, Ctx, Type,
+                        LisName, ListenOn, SocketOpts, NCfg) of
+        {ok, Pid} ->
+            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
+                          [GwName, Type, LisName, ListenOnStr]),
+            {ok, {ListenerId, ListenOn, Pid}};
+        {error, Reason} ->
+            ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
+                  [GwName, Type, LisName, ListenOnStr, Reason]),
+            emqx_gateway_utils:supervisor_ret({error, Reason})
+    end.
+
+start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
+    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
+    NCfg = Cfg#{ ctx => Ctx
+               , listener => {GwName, Type, LisName}
+               },
+    NSocketOpts = merge_default(Type, SocketOpts),
+    MFA = {emqx_gateway_conn, start_link, [NCfg]},
+    do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA).
+
+merge_default(Udp, Options) ->
+    {Key, Default} = case Udp of
+                         udp ->
+                             {udp_options, default_udp_options()};
+                         dtls ->
+                             {udp_options, default_udp_options()};
+                         tcp ->
+                             {tcp_options, default_tcp_options()};
+                         ssl ->
+                             {tcp_options, default_tcp_options()}
+                     end,
+    case lists:keytake(Key, 1, Options) of
+        {value, {Key, TcpOpts}, Options1} ->
+            [{Key, emqx_misc:merge_opts(Default, TcpOpts)}
+             | Options1];
+        false ->
+            [{Key, Default} | Options]
+    end.
+
+do_start_listener(Type, Name, ListenOn, SocketOpts, MFA)
+  when Type == tcp;
+       Type == ssl ->
+    esockd:open(Name, ListenOn, SocketOpts, MFA);
+do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) ->
+    esockd:open_udp(Name, ListenOn, SocketOpts, MFA);
+do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) ->
+    esockd:open_dtls(Name, ListenOn, SocketOpts, MFA).
+
+-spec stop_listeners(GwName :: atom(), Listeners :: list()) -> ok.
+stop_listeners(GwName, Listeners) ->
+    lists:foreach(fun(L) -> stop_listener(GwName, L) end, Listeners).
+
+-spec stop_listener(GwName :: atom(), Listener :: tuple()) -> ok.
+stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
+    StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
+    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
+    case StopRet of
+        ok ->
+            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
+                          [GwName, Type, LisName, ListenOnStr]);
+        {error, Reason} ->
+            ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
+                  [GwName, Type, LisName, ListenOnStr, Reason])
+    end,
+    StopRet.
+
+stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
+    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
+    esockd:close(Name, ListenOn).
+
+-ifndef(TEST).
+console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
+-else.
+console_print(_Fmt, _Args) -> ok.
+-endif.
+
 apply({M, F, A}, A2) when is_atom(M),
                           is_atom(M),
                           is_list(A),

apps/emqx_gateway/src/exproto/emqx_exproto_impl.erl

@@ -19,6 +19,14 @@
 
 -behaviour(emqx_gateway_impl).
 
+-include_lib("emqx/include/logger.hrl").
+
+-import(emqx_gateway_utils,
+        [ normalize_config/1
+        , start_listeners/4
+        , stop_listeners/2
+        ]).
+
 %% APIs
 -export([ reg/0
         , unreg/0
@@ -29,8 +37,6 @@
         , on_gateway_unload/2
         ]).
 
--include_lib("emqx/include/logger.hrl").
-
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -47,6 +53,73 @@ unreg() ->
 %% emqx_gateway_registry callbacks
 %%--------------------------------------------------------------------
 
+on_gateway_load(_Gateway = #{ name := GwName,
+                              config := Config
+                            }, Ctx) ->
+    %% XXX: How to monitor it ?
+    %% Start grpc client pool & client channel
+    PoolName = pool_name(GwName),
+    PoolSize = emqx_vm:schedulers() * 2,
+    {ok, PoolSup} = emqx_pool_sup:start_link(
+                      PoolName, hash, PoolSize,
+                      {emqx_exproto_gcli, start_link, []}),
+    _ = start_grpc_client_channel(GwName,
+                                  maps:get(handler, Config, undefined)
+                                 ),
+    %% XXX: How to monitor it ?
+    _ = start_grpc_server(GwName, maps:get(server, Config, undefined)),
+
+    NConfig = maps:without(
+                 [server, handler],
+                 Config#{pool_name => PoolName}
+                ),
+    Listeners = emqx_gateway_utils:normalize_config(
+                  NConfig#{handler => GwName}
+                 ),
+
+    ModCfg = #{frame_mod => emqx_exproto_frame,
+               chann_mod => emqx_exproto_channel
+              },
+    case start_listeners(
+           Listeners, GwName, Ctx, ModCfg) of
+        {ok, ListenerPids} ->
+            {ok, ListenerPids, _GwState = #{ctx => Ctx, pool => PoolSup}};
+        {error, {Reason, Listener}} ->
+            throw({badconf, #{ key => listeners
+                             , vallue => Listener
+                             , reason => Reason
+                             }})
+    end.
+
+on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
+    GwName = maps:get(name, Gateway),
+    try
+        %% XXX: 1. How hot-upgrade the changes ???
+        %% XXX: 2. Check the New confs first before destroy old instance ???
+        on_gateway_unload(Gateway, GwState),
+        on_gateway_load(Gateway#{config => Config}, Ctx)
+    catch
+        Class : Reason : Stk ->
+            logger:error("Failed to update ~ts; "
+                         "reason: {~0p, ~0p} stacktrace: ~0p",
+                         [GwName, Class, Reason, Stk]),
+            {error, {Class, Reason}}
+    end.
+
+on_gateway_unload(_Gateway = #{ name := GwName,
+                                config := Config
+                              }, _GwState = #{pool := PoolSup}) ->
+    Listeners = emqx_gateway_utils:normalize_config(Config),
+    %% Stop funcs???
+    exit(PoolSup, kill),
+    stop_grpc_server(GwName),
+    stop_grpc_client_channel(GwName),
+    stop_listeners(GwName, Listeners).
+
+%%--------------------------------------------------------------------
+%% Internal funcs
+%%--------------------------------------------------------------------
+
 start_grpc_server(_GwName, undefined) ->
     undefined;
 start_grpc_server(GwName, Options = #{bind := ListenOn}) ->
@@ -103,140 +176,9 @@ stop_grpc_client_channel(GwName) ->
     _ = grpc_client_sup:stop_channel_pool(GwName),
     ok.
 
-on_gateway_load(_Gateway = #{ name := GwName,
-                              config := Config
-                            }, Ctx) ->
-    %% XXX: How to monitor it ?
-    %% Start grpc client pool & client channel
-    PoolName = pool_name(GwName),
-    PoolSize = emqx_vm:schedulers() * 2,
-    {ok, PoolSup} = emqx_pool_sup:start_link(
-                      PoolName, hash, PoolSize,
-                      {emqx_exproto_gcli, start_link, []}),
-    _ = start_grpc_client_channel(GwName,
-                                  maps:get(handler, Config, undefined)
-                                 ),
-    %% XXX: How to monitor it ?
-    _ = start_grpc_server(GwName, maps:get(server, Config, undefined)),
-
-    NConfig = maps:without(
-                 [server, handler],
-                 Config#{pool_name => PoolName}
-                ),
-    Listeners = emqx_gateway_utils:normalize_config(
-                  NConfig#{handler => GwName}
-                 ),
-    ListenerPids = lists:map(fun(Lis) ->
-                     start_listener(GwName, Ctx, Lis)
-                   end, Listeners),
-    {ok, ListenerPids, _GwState = #{ctx => Ctx, pool => PoolSup}}.
-
-on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
-    GwName = maps:get(name, Gateway),
-    try
-        %% XXX: 1. How hot-upgrade the changes ???
-        %% XXX: 2. Check the New confs first before destroy old instance ???
-        on_gateway_unload(Gateway, GwState),
-        on_gateway_load(Gateway#{config => Config}, Ctx)
-    catch
-        Class : Reason : Stk ->
-            logger:error("Failed to update ~ts; "
-                         "reason: {~0p, ~0p} stacktrace: ~0p",
-                         [GwName, Class, Reason, Stk]),
-            {error, {Class, Reason}}
-    end.
-
-on_gateway_unload(_Gateway = #{ name := GwName,
-                                config := Config
-                              }, _GwState = #{pool := PoolSup}) ->
-    Listeners = emqx_gateway_utils:normalize_config(Config),
-    %% Stop funcs???
-    exit(PoolSup, kill),
-    stop_grpc_server(GwName),
-    stop_grpc_client_channel(GwName),
-    lists:foreach(fun(Lis) ->
-        stop_listener(GwName, Lis)
-    end, Listeners).
-
 pool_name(GwName) ->
     list_to_atom(lists:concat([GwName, "_gcli_pool"])).
 
-%%--------------------------------------------------------------------
-%% Internal funcs
-%%--------------------------------------------------------------------
-
-start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
-        {ok, Pid} ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                          [GwName, Type, LisName, ListenOnStr]),
-            Pid;
-        {error, Reason} ->
-            ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason]),
-            throw({badconf, Reason})
-    end.
-
-start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    NCfg = Cfg#{
-             ctx => Ctx,
-             listener => {GwName, Type, LisName},
-             frame_mod => emqx_exproto_frame,
-             chann_mod => emqx_exproto_channel
-            },
-    MFA = {emqx_gateway_conn, start_link, [NCfg]},
-    NSockOpts = merge_default_by_type(Type, SocketOpts),
-    do_start_listener(Type, Name, ListenOn, NSockOpts, MFA).
-
-do_start_listener(Type, Name, ListenOn, Opts, MFA)
-  when Type == tcp;
-       Type == ssl ->
-    esockd:open(Name, ListenOn, Opts, MFA);
-do_start_listener(udp, Name, ListenOn, Opts, MFA) ->
-    esockd:open_udp(Name, ListenOn, Opts, MFA);
-do_start_listener(dtls, Name, ListenOn, Opts, MFA) ->
-    esockd:open_dtls(Name, ListenOn, Opts, MFA).
-
-merge_default_by_type(Type, Options) when Type =:= tcp;
-                                          Type =:= ssl ->
-    Default = emqx_gateway_utils:default_tcp_options(),
-    case lists:keytake(tcp_options, 1, Options) of
-        {value, {tcp_options, TcpOpts}, Options1} ->
-            [{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)}
-             | Options1];
-        false ->
-            [{tcp_options, Default} | Options]
-    end;
-merge_default_by_type(Type, Options) when Type =:= udp;
-                                          Type =:= dtls ->
-    Default = emqx_gateway_utils:default_udp_options(),
-    case lists:keytake(udp_options, 1, Options) of
-        {value, {udp_options, TcpOpts}, Options1} ->
-            [{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
-             | Options1];
-        false ->
-            [{udp_options, Default} | Options]
-    end.
-
-stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case StopRet of
-        ok ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
-                          [GwName, Type, LisName, ListenOnStr]);
-        {error, Reason} ->
-            ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason])
-    end,
-    StopRet.
-
-stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    esockd:close(Name, ListenOn).
-
 -ifndef(TEST).
 console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
 -else.

apps/emqx_gateway/src/lwm2m/emqx_lwm2m_impl.erl

@@ -19,6 +19,8 @@
 
 -behaviour(emqx_gateway_impl).
 
+-include_lib("emqx/include/logger.hrl").
+
 %% APIs
 -export([ reg/0
         , unreg/0
@@ -29,8 +31,6 @@
         , on_gateway_unload/2
         ]).
 
--include_lib("emqx/include/logger.hrl").
-
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -54,10 +54,20 @@ on_gateway_load(_Gateway = #{ name := GwName,
     case emqx_lwm2m_xml_object_db:start_link(XmlDir) of
         {ok, RegPid} ->
             Listeners = emqx_gateway_utils:normalize_config(Config),
-            ListenerPids = lists:map(fun(Lis) ->
-                                             start_listener(GwName, Ctx, Lis)
-                                     end, Listeners),
-            {ok, ListenerPids, _GwState = #{ctx => Ctx, registry => RegPid}};
+            ModCfg = #{frame_mod => emqx_coap_frame,
+                       chann_mod => emqx_lwm2m_channel
+                      },
+            case emqx_gateway_utils:start_listeners(
+                   Listeners, GwName, Ctx, ModCfg) of
+                {ok, ListenerPids} ->
+                    {ok, ListenerPids, #{ctx => Ctx, registry => RegPid}};
+                {error, {Reason, Listener}} ->
+                    _ = emqx_lwm2m_xml_object_db:stop(),
+                    throw({badconf, #{ key => listeners
+                                     , vallue => Listener
+                                     , reason => Reason
+                                     }})
+            end;
         {error, Reason} ->
             throw({badconf, #{ key => xml_dir
                              , value => XmlDir
@@ -85,73 +95,4 @@ on_gateway_unload(_Gateway = #{ name := GwName,
                               }, _GwState = #{registry := RegPid}) ->
     exit(RegPid, kill),
     Listeners = emqx_gateway_utils:normalize_config(Config),
-    lists:foreach(fun(Lis) ->
-        stop_listener(GwName, Lis)
-    end, Listeners).
-
-%%--------------------------------------------------------------------
-%% Internal funcs
-%%--------------------------------------------------------------------
-
-start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
-        {ok, Pid} ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                          [GwName, Type, LisName, ListenOnStr]),
-            Pid;
-        {error, Reason} ->
-            ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason]),
-            throw({badconf, Reason})
-    end.
-
-start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    NCfg = Cfg#{ ctx => Ctx
-               , listener => {GwName, Type, LisName}
-               , frame_mod => emqx_coap_frame
-               , chann_mod => emqx_lwm2m_channel
-               },
-    NSocketOpts = merge_default(SocketOpts),
-    MFA = {emqx_gateway_conn, start_link, [NCfg]},
-    do_start_listener(Type, Name, ListenOn, NSocketOpts, MFA).
-
-merge_default(Options) ->
-    Default = emqx_gateway_utils:default_udp_options(),
-    case lists:keytake(udp_options, 1, Options) of
-        {value, {udp_options, TcpOpts}, Options1} ->
-            [{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
-             | Options1];
-        false ->
-            [{udp_options, Default} | Options]
-    end.
-
-do_start_listener(udp, Name, ListenOn, SocketOpts, MFA) ->
-    esockd:open_udp(Name, ListenOn, SocketOpts, MFA);
-
-do_start_listener(dtls, Name, ListenOn, SocketOpts, MFA) ->
-    esockd:open_dtls(Name, ListenOn, SocketOpts, MFA).
-
-stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case StopRet of
-        ok ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
-                          [GwName, Type, LisName, ListenOnStr]);
-        {error, Reason} ->
-            ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason])
-    end,
-    StopRet.
-
-stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    esockd:close(Name, ListenOn).
-
--ifndef(TEST).
-console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
--else.
-console_print(_Fmt, _Args) -> ok.
--endif.
+    emqx_gateway_utils:stop_listeners(GwName, Listeners).

apps/emqx_gateway/src/mqttsn/emqx_sn_impl.erl

@@ -19,6 +19,14 @@
 
 -behaviour(emqx_gateway_impl).
 
+-include_lib("emqx/include/logger.hrl").
+
+-import(emqx_gateway_utils,
+        [ normalize_config/1
+        , start_listeners/4
+        , stop_listeners/2
+        ]).
+
 %% APIs
 -export([ reg/0
         , unreg/0
@@ -29,8 +37,6 @@
         , on_gateway_unload/2
         ]).
 
--include_lib("emqx/include/logger.hrl").
-
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -70,12 +76,23 @@ on_gateway_load(_Gateway = #{ name := GwName,
                  [broadcast, predefined],
                  Config#{registry => emqx_sn_registry:lookup_name(RegistrySvr)}
                 ),
+
     Listeners = emqx_gateway_utils:normalize_config(NConfig),
 
-    ListenerPids = lists:map(fun(Lis) ->
-                     start_listener(GwName, Ctx, Lis)
-                   end, Listeners),
-    {ok, ListenerPids, _InstaState = #{ctx => Ctx}}.
+    ModCfg = #{frame_mod => emqx_sn_frame,
+               chann_mod => emqx_sn_channel
+              },
+
+    case start_listeners(
+           Listeners, GwName, Ctx, ModCfg) of
+        {ok, ListenerPids} ->
+            {ok, ListenerPids, _GwState = #{ctx => Ctx}};
+        {error, {Reason, Listener}} ->
+            throw({badconf, #{ key => listeners
+                             , vallue => Listener
+                             , reason => Reason
+                             }})
+    end.
 
 on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
     GwName = maps:get(name, Gateway),
@@ -95,68 +112,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
 on_gateway_unload(_Gateway = #{ name := GwName,
                                 config := Config
                               }, _GwState) ->
-    Listeners = emqx_gateway_utils:normalize_config(Config),
-    lists:foreach(fun(Lis) ->
-        stop_listener(GwName, Lis)
-    end, Listeners).
-
-%%--------------------------------------------------------------------
-%% Internal funcs
-%%--------------------------------------------------------------------
-
-start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
-        {ok, Pid} ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                          [GwName, Type, LisName, ListenOnStr]),
-            Pid;
-        {error, Reason} ->
-            ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason]),
-            throw({badconf, Reason})
-    end.
-
-start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    NCfg = Cfg#{
-             ctx => Ctx,
-             listene => {GwName, Type, LisName},
-             frame_mod => emqx_sn_frame,
-             chann_mod => emqx_sn_channel
-            },
-    esockd:open_udp(Name, ListenOn, merge_default(SocketOpts),
-                    {emqx_gateway_conn, start_link, [NCfg]}).
-
-merge_default(Options) ->
-    Default = emqx_gateway_utils:default_udp_options(),
-    case lists:keytake(udp_options, 1, Options) of
-        {value, {udp_options, TcpOpts}, Options1} ->
-            [{udp_options, emqx_misc:merge_opts(Default, TcpOpts)}
-             | Options1];
-        false ->
-            [{udp_options, Default} | Options]
-    end.
-
-stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case StopRet of
-        ok ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
-                          [GwName, Type, LisName, ListenOnStr]);
-        {error, Reason} ->
-            ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason])
-    end,
-    StopRet.
-
-stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    esockd:close(Name, ListenOn).
-
--ifndef(TEST).
-console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
--else.
-console_print(_Fmt, _Args) -> ok.
--endif.
+    Listeners = normalize_config(Config),
+    stop_listeners(GwName, Listeners).

apps/emqx_gateway/src/stomp/emqx_stomp_impl.erl

@@ -18,6 +18,15 @@
 
 -behaviour(emqx_gateway_impl).
 
+-include_lib("emqx/include/logger.hrl").
+-include_lib("emqx_gateway/include/emqx_gateway.hrl").
+
+-import(emqx_gateway_utils,
+        [ normalize_config/1
+        , start_listeners/4
+        , stop_listeners/2
+        ]).
+
 %% APIs
 -export([ reg/0
         , unreg/0
@@ -28,9 +37,6 @@
         , on_gateway_unload/2
         ]).
 
--include_lib("emqx_gateway/include/emqx_gateway.hrl").
--include_lib("emqx/include/logger.hrl").
-
 %%--------------------------------------------------------------------
 %% APIs
 %%--------------------------------------------------------------------
@@ -52,15 +58,22 @@ unreg() ->
 on_gateway_load(_Gateway = #{ name := GwName,
                               config := Config
                             }, Ctx) ->
-    %% Step1. Fold the config to listeners
-    Listeners = emqx_gateway_utils:normalize_config(Config),
-    %% Step2. Start listeners or escokd:specs
-    ListenerPids = lists:map(fun(Lis) ->
-                     start_listener(GwName, Ctx, Lis)
-                   end, Listeners),
-    %% FIXME: How to throw an exception to interrupt the restart logic ?
-    %% FIXME: Assign ctx to GwState
-    {ok, ListenerPids, _GwState = #{ctx => Ctx}}.
+    Listeners = normalize_config(Config),
+    ModCfg = #{frame_mod => emqx_stomp_frame,
+               chann_mod => emqx_stomp_channel
+              },
+    case start_listeners(
+           Listeners, GwName, Ctx, ModCfg) of
+        {ok, ListenerPids} ->
+            %% FIXME: How to throw an exception to interrupt the restart logic ?
+            %% FIXME: Assign ctx to GwState
+            {ok, ListenerPids, _GwState = #{ctx => Ctx}};
+        {error, {Reason, Listener}} ->
+            throw({badconf, #{ key => listeners
+                             , vallue => Listener
+                             , reason => Reason
+                             }})
+    end.
 
 on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
     GwName = maps:get(name, Gateway),
@@ -80,68 +93,5 @@ on_gateway_update(Config, Gateway, GwState = #{ctx := Ctx}) ->
 on_gateway_unload(_Gateway = #{ name := GwName,
                                 config := Config
                               }, _GwState) ->
-    Listeners = emqx_gateway_utils:normalize_config(Config),
-    lists:foreach(fun(Lis) ->
-        stop_listener(GwName, Lis)
-    end, Listeners).
-
-%%--------------------------------------------------------------------
-%% Internal funcs
-%%--------------------------------------------------------------------
-
-start_listener(GwName, Ctx, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) of
-        {ok, Pid} ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts started.~n",
-                          [GwName, Type, LisName, ListenOnStr]),
-            Pid;
-        {error, Reason} ->
-            ?ELOG("Failed to start gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason]),
-            throw({badconf, Reason})
-    end.
-
-start_listener(GwName, Ctx, Type, LisName, ListenOn, SocketOpts, Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    NCfg = Cfg#{
-             ctx => Ctx,
-             listener => {GwName, Type, LisName},   %% Used for authn
-             frame_mod => emqx_stomp_frame,
-             chann_mod => emqx_stomp_channel
-            },
-    esockd:open(Name, ListenOn, merge_default(SocketOpts),
-                {emqx_gateway_conn, start_link, [NCfg]}).
-
-merge_default(Options) ->
-    Default = emqx_gateway_utils:default_tcp_options(),
-    case lists:keytake(tcp_options, 1, Options) of
-        {value, {tcp_options, TcpOpts}, Options1} ->
-            [{tcp_options, emqx_misc:merge_opts(Default, TcpOpts)}
-             | Options1];
-        false ->
-            [{tcp_options, Default} | Options]
-    end.
-
-stop_listener(GwName, {Type, LisName, ListenOn, SocketOpts, Cfg}) ->
-    StopRet = stop_listener(GwName, Type, LisName, ListenOn, SocketOpts, Cfg),
-    ListenOnStr = emqx_gateway_utils:format_listenon(ListenOn),
-    case StopRet of
-        ok ->
-            console_print("Gateway ~ts:~ts:~ts on ~ts stopped.~n",
-                          [GwName, Type, LisName, ListenOnStr]);
-        {error, Reason} ->
-            ?ELOG("Failed to stop gateway ~ts:~ts:~ts on ~ts: ~0p~n",
-                  [GwName, Type, LisName, ListenOnStr, Reason])
-    end,
-    StopRet.
-
-stop_listener(GwName, Type, LisName, ListenOn, _SocketOpts, _Cfg) ->
-    Name = emqx_gateway_utils:listener_id(GwName, Type, LisName),
-    esockd:close(Name, ListenOn).
-
--ifndef(TEST).
-console_print(Fmt, Args) -> ?ULOG(Fmt, Args).
--else.
-console_print(_Fmt, _Args) -> ok.
--endif.
+    Listeners = normalize_config(Config),
+    stop_listeners(GwName, Listeners).

apps/emqx_gateway/test/emqx_gateway_cli_SUITE.erl

@@ -0,0 +1,150 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_gateway_cli_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+-define(GP(S), begin S, receive {fmt, P} -> P; O -> O end end).
+
+%% this parses to #{}, will not cause config cleanup
+%% so we will need call emqx_config:erase
+-define(CONF_DEFAULT, <<"
+gateway {}
+">>).
+
+%%--------------------------------------------------------------------
+%% Setup
+%%--------------------------------------------------------------------
+
+all() -> emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Conf) ->
+    emqx_config:erase(gateway),
+    emqx_config:init_load(emqx_gateway_schema, ?CONF_DEFAULT),
+    emqx_mgmt_api_test_util:init_suite([emqx_conf, emqx_authn, emqx_gateway]),
+    Conf.
+
+end_per_suite(Conf) ->
+    emqx_mgmt_api_test_util:end_suite([emqx_gateway, emqx_authn, emqx_conf]),
+    Conf.
+
+init_per_testcase(_, Conf) ->
+    Self = self(),
+    ok = meck:new(emqx_ctl, [passthrough, no_history, no_link]),
+    ok = meck:expect(emqx_ctl, usage,
+                     fun(L) -> emqx_ctl:format_usage(L) end),
+    ok = meck:expect(emqx_ctl, print,
+                     fun(Fmt) ->
+                        Self ! {fmt, emqx_ctl:format(Fmt)}
+                     end),
+    ok = meck:expect(emqx_ctl, print,
+                     fun(Fmt, Args) ->
+                        Self ! {fmt, emqx_ctl:format(Fmt, Args)}
+                     end),
+    Conf.
+
+end_per_testcase(_, _) ->
+    meck:unload([emqx_ctl]),
+    ok.
+
+%%--------------------------------------------------------------------
+%% Cases
+%%--------------------------------------------------------------------
+
+%% TODO:
+
+t_load_unload(_) ->
+    ok.
+
+t_gateway_registry_usage(_) ->
+    ?assertEqual(
+       ["gateway-registry list # List all registered gateways\n"],
+       emqx_gateway_cli:'gateway-registry'(usage)).
+
+t_gateway_registry_list(_) ->
+    emqx_gateway_cli:'gateway-registry'(["list"]),
+    ?assertEqual(
+       "Registered Name: coap, Callback Module: emqx_coap_impl\n"
+       "Registered Name: exproto, Callback Module: emqx_exproto_impl\n"
+       "Registered Name: lwm2m, Callback Module: emqx_lwm2m_impl\n"
+       "Registered Name: mqttsn, Callback Module: emqx_sn_impl\n"
+       "Registered Name: stomp, Callback Module: emqx_stomp_impl\n"
+       , acc_print()).
+
+t_gateway_usage(_) ->
+    ?assertEqual(
+       ["gateway list                     # List all gateway\n",
+        "gateway lookup <Name>            # Lookup a gateway detailed informations\n",
+        "gateway load   <Name> <JsonConf> # Load a gateway with config\n",
+        "gateway unload <Name>            # Unload the gateway\n",
+        "gateway stop   <Name>            # Stop the gateway\n",
+        "gateway start  <Name>            # Start the gateway\n"],
+       emqx_gateway_cli:gateway(usage)
+     ).
+
+t_gateway_list(_) ->
+    emqx_gateway_cli:gateway(["list"]),
+    ?assertEqual(
+      "Gateway(name=coap, status=unloaded)\n"
+      "Gateway(name=exproto, status=unloaded)\n"
+      "Gateway(name=lwm2m, status=unloaded)\n"
+      "Gateway(name=mqttsn, status=unloaded)\n"
+      "Gateway(name=stomp, status=unloaded)\n"
+      , acc_print()).
+
+t_gateway_load(_) ->
+    ok.
+
+t_gateway_unload(_) ->
+    ok.
+
+t_gateway_start(_) ->
+    ok.
+
+t_gateway_stop(_) ->
+    ok.
+
+t_gateway_clients_usage(_) ->
+    ok.
+
+t_gateway_clients_list(_) ->
+    ok.
+
+t_gateway_clients_lookup(_) ->
+    ok.
+
+t_gateway_clients_kick(_) ->
+    ok.
+
+t_gateway_metrcis_usage(_) ->
+    ok.
+
+t_gateway_metrcis(_) ->
+    ok.
+
+acc_print() ->
+    lists:concat(lists:reverse(acc_print([]))).
+
+acc_print(Acc) ->
+    receive
+        {fmt, S} -> acc_print([S|Acc])
+    after 200 ->
+        Acc
+    end.

apps/emqx_machine/src/emqx_machine_boot.erl

@@ -85,7 +85,6 @@ reboot_apps() ->
     , esockd
     , ranch
     , cowboy
-    , emqx_conf
     , emqx
     , emqx_prometheus
     , emqx_modules
@@ -96,7 +95,6 @@ reboot_apps() ->
     , emqx_resource
     , emqx_rule_engine
     , emqx_bridge
-    , emqx_bridge_mqtt
     , emqx_plugin_libs
     , emqx_management
     , emqx_retainer
@@ -112,17 +110,18 @@ sorted_reboot_apps() ->
 
 app_deps(App) ->
     case application:get_key(App, applications) of
-        undefined -> [];
+        undefined -> undefined;
         {ok, List} -> lists:filter(fun(A) -> lists:member(A, reboot_apps()) end, List)
     end.
 
 sorted_reboot_apps(Apps) ->
     G = digraph:new(),
     try
-        lists:foreach(fun({App, Deps}) -> add_app(G, App, Deps) end, Apps),
+        NoDepApps = add_apps_to_digraph(G, Apps),
         case digraph_utils:topsort(G) of
             Sorted when is_list(Sorted) ->
-                Sorted;
+                %% ensure emqx_conf boot up first
+                [emqx_conf | Sorted ++ (NoDepApps -- Sorted)];
             false ->
                 Loops = find_loops(G),
                 error({circular_application_dependency, Loops})
@@ -131,23 +130,33 @@ sorted_reboot_apps(Apps) ->
         digraph:delete(G)
     end.
 
-add_app(G, App, undefined) ->
+%% Build a dependency graph from the provided application list.
+%% Return top-sort result of the apps.
+%% Isolated apps without which are not dependency of any other apps are
+%% put to the end of the list in the original order.
+add_apps_to_digraph(G, Apps) ->
+    lists:foldl(fun
+            ({App, undefined}, Acc) ->
+                ?SLOG(debug, #{msg => "app_is_not_loaded", app => App}),
+                Acc;
+            ({App, []}, Acc) ->
+                Acc ++ [App]; %% use '++' to keep the original order
+            ({App, Deps}, Acc) ->
+                add_app_deps_to_digraph(G, App, Deps),
+                Acc
+        end, [], Apps).
+
+add_app_deps_to_digraph(G, App, undefined) ->
     ?SLOG(debug, #{msg => "app_is_not_loaded", app => App}),
     %% not loaded
-    add_app(G, App, []);
-% We ALWAYS want to add `emqx_conf', even if no other app declare a
-% dependency on it.  Otherwise, emqx may fail to load the config
-% schemas, especially in the test profile.
-add_app(G, App = emqx_conf, []) ->
-    digraph:add_vertex(G, App),
-    ok;
-add_app(_G, _App, []) ->
+    add_app_deps_to_digraph(G, App, []);
+add_app_deps_to_digraph(_G, _App, []) ->
     ok;
-add_app(G, App, [Dep | Deps]) ->
+add_app_deps_to_digraph(G, App, [Dep | Deps]) ->
     digraph:add_vertex(G, App),
     digraph:add_vertex(G, Dep),
     digraph:add_edge(G, Dep, App), %% dep -> app as dependency
-    add_app(G, App, Deps).
+    add_app_deps_to_digraph(G, App, Deps).
 
 find_loops(G) ->
     lists:filtermap(

apps/emqx_machine/test/emqx_machine_tests.erl

@@ -38,7 +38,7 @@ sorted_reboot_apps_cycle_test() ->
 
 check_order(Apps) ->
     AllApps = lists:usort(lists:append([[A | Deps] || {A, Deps} <- Apps])),
-    Sorted = emqx_machine_boot:sorted_reboot_apps(Apps),
+    [emqx_conf | Sorted] = emqx_machine_boot:sorted_reboot_apps(Apps),
     case length(AllApps) =:= length(Sorted) of
         true -> ok;
         false -> error({AllApps, Sorted})

apps/emqx_management/src/emqx_mgmt_api.erl

@@ -30,6 +30,7 @@
 -export([ node_query/5
         , cluster_query/4
         , select_table_with_count/5
+        , b2i/1
         ]).
 
 -export([do_query/6]).

apps/emqx_management/src/emqx_mgmt_api_app.erl

@@ -91,16 +91,17 @@ fields(app) ->
             """They are useful for accessing public data anonymously,"""
             """and are used to associate API requests.""",
                 example => <<"MzAyMjk3ODMwMDk0NjIzOTUxNjcwNzQ0NzQ3MTE2NDYyMDI">>})},
-        {expired_at, hoconsc:mk(emqx_schema:rfc3339_system_time(),
+        {expired_at, hoconsc:mk(hoconsc:union([undefined, emqx_schema:rfc3339_system_time()]),
             #{desc => "No longer valid datetime",
                 example => <<"2021-12-05T02:01:34.186Z">>,
-                nullable => true
+                nullable => true,
+                default => undefined
             })},
         {created_at, hoconsc:mk(emqx_schema:rfc3339_system_time(),
             #{desc => "ApiKey create datetime",
                 example => <<"2021-12-01T00:00:00.000Z">>
             })},
-        {desc, hoconsc:mk(emqx_schema:unicode_binary(),
+        {desc, hoconsc:mk(binary(),
             #{example => <<"Note">>, nullable => true})},
         {enable, hoconsc:mk(boolean(), #{desc => "Enable/Disable", nullable => true})}
     ];
@@ -136,13 +137,19 @@ api_key(post, #{body := App}) ->
     #{
         <<"name">> := Name,
         <<"desc">> := Desc0,
-        <<"expired_at">> := ExpiredAt,
         <<"enable">> := Enable
     } = App,
+    %% undefined is never expired
+    ExpiredAt0 = maps:get(<<"expired_at">>, App, <<"undefined">>),
+    ExpiredAt =
+        case ExpiredAt0 of
+            <<"undefined">> -> undefined;
+            _ -> ExpiredAt0
+        end,
     Desc = unicode:characters_to_binary(Desc0, unicode),
     case emqx_mgmt_auth:create(Name, Enable, ExpiredAt, Desc) of
         {ok, NewApp} -> {200, format(NewApp)};
-        {error, Reason} -> {400, Reason}
+        {error, Reason} -> {400, io_lib:format("~p", [Reason])}
     end.
 
 api_key_by_name(get, #{bindings := #{name := Name}}) ->
@@ -164,8 +171,13 @@ api_key_by_name(put, #{bindings := #{name := Name}, body := Body}) ->
         {error, not_found} -> {404, <<"NOT_FOUND">>}
     end.
 
-format(App = #{expired_at := ExpiredAt, created_at := CreateAt}) ->
+format(App = #{expired_at := ExpiredAt0, created_at := CreateAt}) ->
+    ExpiredAt =
+        case ExpiredAt0 of
+            undefined -> <<"undefined">>;
+            _ -> list_to_binary(calendar:system_time_to_rfc3339(ExpiredAt0))
+        end,
     App#{
-        expired_at => list_to_binary(calendar:system_time_to_rfc3339(ExpiredAt)),
+        expired_at => ExpiredAt,
         created_at => list_to_binary(calendar:system_time_to_rfc3339(CreateAt))
     }.

apps/emqx_management/src/emqx_mgmt_api_banned.erl

@@ -101,7 +101,7 @@ fields(ban) ->
             desc => <<"Banned type clientid, username, peerhost">>,
             nullable => false,
             example => username})},
-        {who, hoconsc:mk(emqx_schema:unicode_binary(), #{
+        {who, hoconsc:mk(binary(), #{
             desc => <<"Client info as banned type">>,
             nullable => false,
             example => <<"Badass坏"/utf8>>})},
@@ -109,19 +109,17 @@ fields(ban) ->
             desc => <<"Commander">>,
             nullable => true,
             example => <<"mgmt_api">>})},
-        {reason, hoconsc:mk(emqx_schema:unicode_binary(), #{
+        {reason, hoconsc:mk(binary(), #{
             desc => <<"Banned reason">>,
             nullable => true,
             example => <<"Too many requests">>})},
-        {at, hoconsc:mk(binary(), #{
+        {at, hoconsc:mk(emqx_schema:rfc3339_system_time(), #{
             desc => <<"Create banned time, rfc3339, now if not specified">>,
             nullable => true,
-            validator => fun is_rfc3339/1,
             example => <<"2021-10-25T21:48:47+08:00">>})},
-        {until, hoconsc:mk(binary(), #{
+        {until, hoconsc:mk(emqx_schema:rfc3339_system_time(), #{
             desc => <<"Cancel banned time, rfc3339, now + 5 minute if not specified">>,
             nullable => true,
-            validator => fun is_rfc3339/1,
             example => <<"2021-10-25T21:53:47+08:00">>})
         }
     ];
@@ -130,22 +128,19 @@ fields(meta) ->
         emqx_dashboard_swagger:fields(limit) ++
         [{count, hoconsc:mk(integer(), #{example => 1})}].
 
-is_rfc3339(Time) ->
-    try
-        emqx_banned:to_timestamp(Time),
-        ok
-    catch _:_ -> {error, Time}
-    end.
-
 banned(get, #{query_string := Params}) ->
     Response = emqx_mgmt_api:paginate(?TAB, Params, ?FORMAT_FUN),
     {200, Response};
 banned(post, #{body := Body}) ->
-    case emqx_banned:create(emqx_banned:parse(Body)) of
-        {ok, Banned} ->
-            {200, format(Banned)};
-        {error, {already_exist, Old}} ->
-            {400, #{code => 'ALREADY_EXISTED', message => format(Old)}}
+    case emqx_banned:parse(Body) of
+        {error, Reason} ->
+            {400, #{code => 'PARAMS_ERROR', message => list_to_binary(Reason)}};
+        Ban ->
+            case emqx_banned:create(Ban) of
+                {ok, Banned} -> {200, format(Banned)};
+                {error, {already_exist, Old}} ->
+                    {400, #{code => 'ALREADY_EXISTED', message => format(Old)}}
+            end
     end.
 
 delete_banned(delete, #{bindings := Params}) ->

apps/emqx_management/src/emqx_mgmt_api_trace.erl

@@ -107,9 +107,14 @@ schema("/trace/:name/download") ->
         get => #{
             description => "Download trace log by name",
             parameters => [hoconsc:ref(name)],
-            %% todo zip file octet-stream
             responses => #{
-                200 => <<"TODO octet-stream">>
+                200 =>
+                #{description => "A trace zip file",
+                    content => #{
+                        'application/octet-stream' =>
+                             #{schema => #{type => "string", format => "binary"}}
+                    }
+                }
             }
         }
     };
@@ -124,9 +129,12 @@ schema("/trace/:name/log") ->
                 hoconsc:ref(position),
                 hoconsc:ref(node)
             ],
-            %% todo response data
             responses => #{
-                200 => <<"TODO">>
+                200 =>
+                [
+                    {items, hoconsc:mk(binary(), #{example => "TEXT-LOG-ITEMS"})}
+                    | fields(bytes) ++ fields(position)
+                ]
             }
         }
     }.
@@ -209,6 +217,7 @@ fields(position) ->
             default => 0
         })}].
 
+
 -define(NAME_RE, "^[A-Za-z]+[A-Za-z0-9-_]*$").
 
 validate_name(Name) ->
@@ -296,7 +305,12 @@ download_trace_log(get, #{bindings := #{name := Name}}) ->
             ZipFileName = ZipDir ++ binary_to_list(Name) ++ ".zip",
             {ok, ZipFile} = zip:zip(ZipFileName, Zips, [{cwd, ZipDir}]),
             emqx_trace:delete_files_after_send(ZipFileName, Zips),
-            {200, ZipFile};
+            Headers = #{
+                <<"content-type">> => <<"application/x-zip">>,
+                <<"content-disposition">> =>
+                iolist_to_binary("attachment; filename=" ++ filename:basename(ZipFile))
+            },
+            {200, Headers, {file, ZipFile}};
         {error, not_found} -> ?NOT_FOUND(Name)
     end.
 
@@ -324,11 +338,10 @@ cluster_call(Mod, Fun, Args, Timeout) ->
     BadNodes =/= [] andalso ?LOG(error, "rpc call failed on ~p ~p", [BadNodes, {Mod, Fun, Args}]),
     GoodRes.
 
-stream_log_file(get, #{bindings := #{name := Name}, query_string := Query} = T) ->
+stream_log_file(get, #{bindings := #{name := Name}, query_string := Query}) ->
     Node0 = maps:get(<<"node">>, Query, atom_to_binary(node())),
     Position = maps:get(<<"position">>, Query, 0),
     Bytes = maps:get(<<"bytes">>, Query, 1000),
-    logger:error("~p", [T]),
     case to_node(Node0) of
         {ok, Node} ->
             case rpc:call(Node, ?MODULE, read_trace_file, [Name, Position, Bytes]) of

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -37,7 +37,7 @@
     api_secret_hash = <<>> :: binary() | '_',
     enable = true :: boolean() | '_',
     desc = <<>> :: binary() | '_',
-    expired_at = 0 :: integer() | '_',
+    expired_at = 0 :: integer() | undefined | '_',
     created_at = 0 :: integer() | '_'
               }).
 

apps/emqx_management/src/emqx_mgmt_cli.erl

@@ -18,6 +18,7 @@
 
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("emqx/include/emqx_mqtt.hrl").
+-include_lib("emqx/include/logger.hrl").
 
 -include("emqx_mgmt.hrl").
 
@@ -386,18 +387,20 @@ trace(["list"]) ->
         emqx_ctl:print("Trace(~s=~s, level=~s, destination=~p)~n", [Type, Filter, Level, Dst])
                   end, emqx_trace_handler:running());
 
-trace(["stop", Operation, ClientId]) ->
-    case trace_type(Operation) of
-        {ok, Type} -> trace_off(Type, ClientId);
+trace(["stop", Operation, Filter0]) ->
+    case trace_type(Operation, Filter0) of
+        {ok, Type, Filter} -> trace_off(Type, Filter);
         error -> trace([])
     end;
 
 trace(["start", Operation, ClientId, LogFile]) ->
     trace(["start", Operation, ClientId, LogFile, "all"]);
 
-trace(["start", Operation, ClientId, LogFile, Level]) ->
-    case trace_type(Operation) of
-        {ok, Type} -> trace_on(Type, ClientId, list_to_existing_atom(Level), LogFile);
+trace(["start", Operation, Filter0, LogFile, Level]) ->
+    case trace_type(Operation, Filter0) of
+        {ok, Type, Filter} ->
+            trace_on(name(Filter0), Type, Filter,
+                list_to_existing_atom(Level), LogFile);
         error -> trace([])
     end;
 
@@ -417,20 +420,23 @@ trace(_) ->
             "Stop tracing for a client ip on local node"}
     ]).
 
-trace_on(Who, Name, Level, LogFile) ->
-    case emqx_trace_handler:install(Who, Name, Level, LogFile) of
+trace_on(Name, Type, Filter, Level, LogFile) ->
+    case emqx_trace_handler:install(Name, Type, Filter, Level, LogFile) of
         ok ->
-            emqx_ctl:print("trace ~s ~s successfully~n", [Who, Name]);
+            emqx_trace:check(),
+            emqx_ctl:print("trace ~s ~s successfully~n", [Filter, Name]);
         {error, Error} ->
-            emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Who, Name, Error])
+            emqx_ctl:print("[error] trace ~s ~s: ~p~n", [Filter, Name, Error])
     end.
 
-trace_off(Who, Name) ->
-    case emqx_trace_handler:uninstall(Who, Name) of
+trace_off(Type, Filter) ->
+    ?TRACE("CLI", "trace_stopping", #{Type => Filter}),
+    case emqx_trace_handler:uninstall(Type, name(Filter)) of
         ok ->
-            emqx_ctl:print("stop tracing ~s ~s successfully~n", [Who, Name]);
+            emqx_trace:check(),
+            emqx_ctl:print("stop tracing ~s ~s successfully~n", [Type, Filter]);
         {error, Error} ->
-            emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Who, Name, Error])
+            emqx_ctl:print("[error] stop tracing ~s ~s: ~p~n", [Type, Filter, Error])
     end.
 
 %%--------------------------------------------------------------------
@@ -459,9 +465,9 @@ traces(["delete", Name]) ->
 traces(["start", Name, Operation, Filter]) ->
     traces(["start", Name, Operation, Filter, "900"]);
 
-traces(["start", Name, Operation, Filter, DurationS]) ->
-    case trace_type(Operation) of
-        {ok, Type} ->  trace_cluster_on(Name, Type, Filter, DurationS);
+traces(["start", Name, Operation, Filter0, DurationS]) ->
+    case trace_type(Operation, Filter0) of
+        {ok, Type, Filter} ->  trace_cluster_on(Name, Type, Filter, DurationS);
         error -> traces([])
     end;
 
@@ -503,10 +509,10 @@ trace_cluster_off(Name) ->
         {error, Error} -> emqx_ctl:print("[error] Stop cluster_trace ~s: ~p~n", [Name, Error])
     end.
 
-trace_type("client") -> {ok, clientid};
-trace_type("topic") -> {ok, topic};
-trace_type("ip_address") -> {ok, ip_address};
-trace_type(_) -> error.
+trace_type("client", ClientId) -> {ok, clientid, list_to_binary(ClientId)};
+trace_type("topic", Topic) -> {ok, topic, list_to_binary(Topic)};
+trace_type("ip_address", IP) -> {ok, ip_address, IP};
+trace_type(_, _) -> error.
 
 %%--------------------------------------------------------------------
 %% @doc Listeners Command
@@ -716,3 +722,6 @@ format_listen_on({Addr, Port}) when is_list(Addr) ->
     io_lib:format("~ts:~w", [Addr, Port]);
 format_listen_on({Addr, Port}) when is_tuple(Addr) ->
     io_lib:format("~ts:~w", [inet:ntoa(Addr), Port]).
+
+name(Filter) ->
+    iolist_to_binary(["CLI-", Filter]).