Merge remote-tracking branch 'origin/develop'

etc/emqx.conf

@@ -1,5 +1,5 @@
 ##====================================================================
-## EMQ X Configuration R3.0
+## EMQ X Configuration R4.0
 ##====================================================================
 
 ##--------------------------------------------------------------------

include/emqx.hrl

@@ -69,9 +69,9 @@
           topic :: binary(),
           %% Message Payload
           payload :: binary(),
-          %% Timestamp
-          timestamp :: erlang:timestamp()
-        }).
+          %% Timestamp (Unit: millisecond)
+          timestamp :: integer()
+         }).
 
 -record(delivery, {
           sender  :: pid(),      %% Sender of the delivery

rebar.config

@@ -2,8 +2,8 @@
 
 {deps,
     [{jsx, "2.10.0"},
-     {cowboy, "2.7.0"},
      {gproc, "0.8.0"},
+     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.7.1"}}},
      {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.6.0"}}},
      {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.7.1"}}},
      {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.4.1"}}},

src/emqx.app.src.script

@@ -1,6 +1,6 @@
 %%-*- mode: erlang -*-
-	%% .app.src.script
-	
+%% .app.src.script
+
 Config = case os:getenv("EMQX_DESC") of
 	false -> CONFIG; % env var not defined
 	[]    -> CONFIG; % env var set to empty string
@@ -31,4 +31,4 @@ case os:getenv("EMQX_DEPS_DEFAULT_VSN") of
 			AppConf0 = lists:keystore(vsn, 1, AppConf, {vsn, RemoveLeadingV(Tag)}),
 			{application, App, AppConf0}
 		end || Conf = {application, App, AppConf} <- Config]
-end.
+end.

src/emqx_alarm_handler.erl

@@ -105,7 +105,7 @@ init(_) ->
     {ok, []}.
 
 handle_event({set_alarm, {AlarmId, AlarmDesc = #alarm{timestamp = undefined}}}, State) ->
-    handle_event({set_alarm, {AlarmId, AlarmDesc#alarm{timestamp = os:timestamp()}}}, State);
+    handle_event({set_alarm, {AlarmId, AlarmDesc#alarm{timestamp = erlang:system_time(second)}}}, State);
 handle_event({set_alarm, Alarm = {AlarmId, AlarmDesc}}, State) ->
     ?LOG(warning, "New Alarm: ~p, Alarm Info: ~p", [AlarmId, AlarmDesc]),
     case encode_alarm(Alarm) of
@@ -158,7 +158,7 @@ encode_alarm({AlarmId, #alarm{severity  = Severity,
                            {desc, [{severity, Severity},
                                    {title, iolist_to_binary(Title)},
                                    {summary, iolist_to_binary(Summary)},
-                                   {timestamp, emqx_misc:now_to_secs(Ts)}]}]);
+                                   {timestamp, Ts}]}]);
 encode_alarm({AlarmId, undefined}) ->
     emqx_json:safe_encode([{id, maybe_to_binary(AlarmId)}]);
 encode_alarm({AlarmId, AlarmDesc}) ->
@@ -194,5 +194,5 @@ clear_alarm_(Id) ->
 set_alarm_history(Id, Desc) ->
     His = #alarm_history{id = Id,
                          desc = Desc,
-                         clear_at = os:timestamp()},
+                         clear_at = erlang:system_time(second)},
     mnesia:dirty_write(?ALARM_HISTORY_TAB, His).

src/emqx_broker.erl

@@ -191,20 +191,21 @@ do_unsubscribe(undefined, Topic, SubPid, SubOpts) ->
 do_unsubscribe(Group, Topic, SubPid, _SubOpts) ->
     emqx_shared_sub:unsubscribe(Group, Topic, SubPid).
 
-%%------------------------------------------------------------------------------
+%%--------------------------------------------------------------------
 %% Publish
-%%------------------------------------------------------------------------------
+%%--------------------------------------------------------------------
 
 -spec(publish(emqx_types:message()) -> emqx_types:publish_result()).
 publish(Msg) when is_record(Msg, message) ->
     _ = emqx_tracer:trace(publish, Msg),
-    Headers = Msg#message.headers,
-    case emqx_hooks:run_fold('message.publish', [], Msg#message{headers = Headers#{allow_publish => true}}) of
+    Msg1 = emqx_message:set_header(allow_publish, true,
+                                   emqx_message:clean_dup(Msg)),
+    case emqx_hooks:run_fold('message.publish', [], Msg1) of
         #message{headers = #{allow_publish := false}} ->
-            ?LOG(notice, "Publishing interrupted: ~s", [emqx_message:format(Msg)]),
+            ?LOG(notice, "Stop publishing: ~s", [emqx_message:format(Msg1)]),
             [];
-        #message{topic = Topic} = Msg1 ->
-            route(aggre(emqx_router:match_routes(Topic)), delivery(Msg1))
+        #message{topic = Topic} = Msg2 ->
+            route(aggre(emqx_router:match_routes(Topic)), delivery(Msg2))
     end.
 
 %% Called internally

src/emqx_channel.erl

@@ -118,6 +118,8 @@ info(Keys, Channel) when is_list(Keys) ->
     [{Key, info(Key, Channel)} || Key <- Keys];
 info(conninfo, #channel{conninfo = ConnInfo}) ->
     ConnInfo;
+info(zone, #channel{clientinfo = #{zone := Zone}}) ->
+    Zone;
 info(clientid, #channel{clientinfo = #{clientid := ClientId}}) ->
     ClientId;
 info(clientinfo, #channel{clientinfo = ClientInfo}) ->
@@ -147,11 +149,6 @@ stats(#channel{session = Session})->
 caps(#channel{clientinfo = #{zone := Zone}}) ->
     emqx_mqtt_caps:get_caps(Zone).
 
-%% For tests
-set_field(Name, Val, Channel) ->
-    Fields = record_info(fields, channel),
-    Pos = emqx_misc:index_of(Name, Fields),
-    setelement(Pos+1, Channel, Val).
 
 %%--------------------------------------------------------------------
 %% Init the channel
@@ -324,9 +321,13 @@ handle_in(?AUTH_PACKET(), Channel) ->
 handle_in({frame_error, Reason}, Channel = #channel{conn_state = idle}) ->
     shutdown(Reason, Channel);
 
+handle_in({frame_error, frame_too_large}, Channel = #channel{conn_state = connecting}) ->
+    shutdown(frame_too_large, ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE), Channel);
 handle_in({frame_error, Reason}, Channel = #channel{conn_state = connecting}) ->
     shutdown(Reason, ?CONNACK_PACKET(?RC_MALFORMED_PACKET), Channel);
 
+handle_in({frame_error, frame_too_large}, Channel = #channel{conn_state = connected}) ->
+    handle_out(disconnect, {?RC_PACKET_TOO_LARGE, frame_too_large}, Channel);
 handle_in({frame_error, Reason}, Channel = #channel{conn_state = connected}) ->
     handle_out(disconnect, {?RC_MALFORMED_PACKET, Reason}, Channel);
 
@@ -375,7 +376,7 @@ process_publish(Packet = ?PUBLISH_PACKET(_QoS, Topic, PacketId), Channel) ->
                    fun check_pub_caps/2
                   ], Packet, Channel) of
         {ok, NPacket, NChannel} ->
-            Msg = pub_to_msg(NPacket, NChannel),
+            Msg = packet_to_msg(NPacket, NChannel),
             do_publish(PacketId, Msg, NChannel);
         {error, ReasonCode, NChannel} ->
             ?LOG(warning, "Cannot publish message to ~s due to ~s",
@@ -383,9 +384,9 @@ process_publish(Packet = ?PUBLISH_PACKET(_QoS, Topic, PacketId), Channel) ->
             handle_out(disconnect, ReasonCode, NChannel)
     end.
 
-pub_to_msg(Packet, #channel{conninfo   = #{proto_ver := ProtoVer},
-                            clientinfo = ClientInfo =
-                            #{mountpoint := MountPoint}}) ->
+packet_to_msg(Packet, #channel{conninfo   = #{proto_ver := ProtoVer},
+                               clientinfo = ClientInfo =
+                               #{mountpoint := MountPoint}}) ->
     emqx_mountpoint:mount(
       MountPoint, emqx_packet:to_message(
                     ClientInfo, #{proto_ver => ProtoVer}, Packet)).
@@ -417,8 +418,8 @@ do_publish(PacketId, Msg = #message{qos = ?QOS_2},
     end.
 
 -compile({inline, [puback_reason_code/1]}).
-puback_reason_code([]) -> ?RC_NO_MATCHING_SUBSCRIBERS;
-puback_reason_code(_)  -> ?RC_SUCCESS.
+puback_reason_code([])    -> ?RC_NO_MATCHING_SUBSCRIBERS;
+puback_reason_code([_|_]) -> ?RC_SUCCESS.
 
 %%--------------------------------------------------------------------
 %% Process Subscribe
@@ -1210,7 +1211,7 @@ maybe_resume_session(#channel{resuming = false}) ->
 maybe_resume_session(#channel{session  = Session,
                               resuming = true,
                               pendings = Pendings}) ->
-    {ok, Publishes, Session1} = emqx_session:redeliver(Session),
+    {ok, Publishes, Session1} = emqx_session:replay(Session),
     case emqx_session:deliver(Pendings, Session1) of
         {ok, Session2} ->
             {ok, Publishes, Session2};
@@ -1309,3 +1310,11 @@ sp(false) -> 0.
 flag(true)  -> 1;
 flag(false) -> 0.
 
+%%--------------------------------------------------------------------
+%% For CT tests
+%%--------------------------------------------------------------------
+
+set_field(Name, Value, Channel) ->
+    Pos = emqx_misc:index_of(Name, record_info(fields, channel)),
+    setelement(Pos+1, Channel, Value).
+

src/emqx_connection.erl

@@ -51,7 +51,10 @@
         ]).
 
 %% Internal callback
--export([wakeup_from_hib/3]).
+-export([wakeup_from_hib/2]).
+
+%% Export for CT
+-export([set_field/3]).
 
 -import(emqx_misc,
         [ maybe_apply/2
@@ -85,6 +88,8 @@
           gc_state :: maybe(emqx_gc:gc_state()),
           %% Stats Timer
           stats_timer :: disabled | maybe(reference()),
+          %% Idle Timeout
+          idle_timeout :: integer(),
           %% Idle Timer
           idle_timer :: maybe(reference())
         }).
@@ -163,13 +168,13 @@ stop(Pid) ->
 init(Parent, Transport, RawSocket, Options) ->
     case Transport:wait(RawSocket) of
         {ok, Socket} ->
-            do_init(Parent, Transport, Socket, Options);
+            run_loop(Parent, init_state(Transport, Socket, Options));
         {error, Reason} ->
             ok = Transport:fast_close(RawSocket),
             exit_on_sock_error(Reason)
     end.
 
-do_init(Parent, Transport, Socket, Options) ->
+init_state(Transport, Socket, Options) ->
     {ok, Peername} = Transport:ensure_ok_or_exit(peername, [Socket]),
     {ok, Sockname} = Transport:ensure_ok_or_exit(sockname, [Socket]),
     Peercert = Transport:ensure_ok_or_exit(peercert, [Socket]),
@@ -181,7 +186,8 @@ do_init(Parent, Transport, Socket, Options) ->
                 },
     Zone = proplists:get_value(zone, Options),
     ActiveN = proplists:get_value(active_n, Options, ?ACTIVE_N),
-    Limiter = emqx_limiter:init(Options),
+    PubLimit = emqx_zone:publish_limit(Zone),
+    Limiter = emqx_limiter:init([{pub_limit, PubLimit}|Options]),
     FrameOpts = emqx_zone:mqtt_frame_options(Zone),
     ParseState = emqx_frame:initial_parse_state(FrameOpts),
     Serialize = emqx_frame:serialize_fun(),
@@ -190,25 +196,31 @@ do_init(Parent, Transport, Socket, Options) ->
     StatsTimer = emqx_zone:stats_timer(Zone),
     IdleTimeout = emqx_zone:idle_timeout(Zone),
     IdleTimer = start_timer(IdleTimeout, idle_timeout),
-    emqx_misc:tune_heap_size(emqx_zone:oom_policy(Zone)),
+    #state{transport    = Transport,
+           socket       = Socket,
+           peername     = Peername,
+           sockname     = Sockname,
+           sockstate    = idle,
+           active_n     = ActiveN,
+           limiter      = Limiter,
+           parse_state  = ParseState,
+           serialize    = Serialize,
+           channel      = Channel,
+           gc_state     = GcState,
+           stats_timer  = StatsTimer,
+           idle_timeout = IdleTimeout,
+           idle_timer   = IdleTimer
+          }.
+
+run_loop(Parent, State = #state{transport = Transport,
+                                socket    = Socket,
+                                peername  = Peername,
+                                channel   = Channel}) ->
     emqx_logger:set_metadata_peername(esockd:format(Peername)),
-    State = #state{transport   = Transport,
-                   socket      = Socket,
-                   peername    = Peername,
-                   sockname    = Sockname,
-                   sockstate   = idle,
-                   active_n    = ActiveN,
-                   limiter     = Limiter,
-                   parse_state = ParseState,
-                   serialize   = Serialize,
-                   channel     = Channel,
-                   gc_state    = GcState,
-                   stats_timer = StatsTimer,
-                   idle_timer  = IdleTimer
-                  },
+    emqx_misc:tune_heap_size(emqx_zone:oom_policy(
+                               emqx_channel:info(zone, Channel))),
     case activate_socket(State) of
-        {ok, NState} ->
-            hibernate(Parent, NState, #{idle_timeout => IdleTimeout});
+        {ok, NState} -> hibernate(Parent, NState);
         {error, Reason} ->
             ok = Transport:fast_close(Socket),
             exit_on_sock_error(Reason)
@@ -226,28 +238,24 @@ exit_on_sock_error(Reason) ->
 %%--------------------------------------------------------------------
 %% Recv Loop
 
-recvloop(Parent, State, Options = #{idle_timeout := IdleTimeout}) ->
+recvloop(Parent, State = #state{idle_timeout = IdleTimeout}) ->
     receive
         {system, From, Request} ->
-            sys:handle_system_msg(Request, From, Parent,
-                                  ?MODULE, [], {State, Options});
+            sys:handle_system_msg(Request, From, Parent, ?MODULE, [], State);
         {'EXIT', Parent, Reason} ->
             terminate(Reason, State);
         Msg ->
-            NState = ensure_stats_timer(IdleTimeout, State),
-            process_msg([Msg], Parent, NState, Options)
+            process_msg([Msg], Parent, ensure_stats_timer(IdleTimeout, State))
     after
         IdleTimeout ->
-            NState = cancel_stats_timer(State),
-            hibernate(Parent, NState, Options)
+            hibernate(Parent, cancel_stats_timer(State))
     end.
 
-hibernate(Parent, State, Options) ->
-    proc_lib:hibernate(?MODULE, wakeup_from_hib, [Parent, State, Options]).
+hibernate(Parent, State) ->
+    proc_lib:hibernate(?MODULE, wakeup_from_hib, [Parent, State]).
 
-wakeup_from_hib(Parent, State, Options) ->
-    %% Maybe do something later here.
-    recvloop(Parent, State, Options).
+%% Maybe do something here later.
+wakeup_from_hib(Parent, State) -> recvloop(Parent, State).
 
 %%--------------------------------------------------------------------
 %% Ensure/cancel stats timer
@@ -266,17 +274,16 @@ cancel_stats_timer(State) -> State.
 %%--------------------------------------------------------------------
 %% Process next Msg
 
-process_msg([], Parent, State, Options) ->
-    recvloop(Parent, State, Options);
+process_msg([], Parent, State) -> recvloop(Parent, State);
 
-process_msg([Msg|More], Parent, State, Options) ->
+process_msg([Msg|More], Parent, State) ->
     case catch handle_msg(Msg, State) of
         ok ->
-            process_msg(More, Parent, State, Options);
+            process_msg(More, Parent, State);
         {ok, NState} ->
-            process_msg(More, Parent, NState, Options);
+            process_msg(More, Parent, NState);
         {ok, Msgs, NState} ->
-            process_msg(append_msg(Msgs, More), Parent, NState, Options);
+            process_msg(append_msg(More, Msgs), Parent, NState);
         {stop, Reason} ->
             terminate(Reason, State);
         {stop, Reason, NState} ->
@@ -285,6 +292,15 @@ process_msg([Msg|More], Parent, State, Options) ->
             terminate(Reason, State)
     end.
 
+-compile({inline, [append_msg/2]}).
+append_msg([], Msgs) when is_list(Msgs) ->
+    Msgs;
+append_msg([], Msg) -> [Msg];
+append_msg(Q, Msgs) when is_list(Msgs) ->
+    lists:append(Q, Msgs);
+append_msg(Q, Msg) ->
+    lists:append(Q, [Msg]).
+
 %%--------------------------------------------------------------------
 %% Handle a Msg
 
@@ -343,11 +359,10 @@ handle_msg({Passive, _Sock}, State)
     NState1 = check_oom(run_gc(InStats, NState)),
     handle_info(activate_socket, NState1);
 
-handle_msg(Deliver = {deliver, _Topic, _Msg}, State =
-           #state{active_n = ActiveN, channel = Channel}) ->
+handle_msg(Deliver = {deliver, _Topic, _Msg},
+           State = #state{active_n = ActiveN}) ->
     Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)],
-    Ret = emqx_channel:handle_deliver(Delivers, Channel),
-    handle_chan_return(Ret, State);
+    with_channel(handle_deliver, [Delivers], State);
 
 %% Something sent
 handle_msg({inet_reply, _Sock, ok}, State = #state{active_n = ActiveN}) ->
@@ -407,18 +422,17 @@ terminate(Reason, State = #state{channel = Channel}) ->
 %%--------------------------------------------------------------------
 %% Sys callbacks
 
-system_continue(Parent, _Deb, {State, Options}) ->
-    recvloop(Parent, State, Options).
+system_continue(Parent, _Debug, State) ->
+    recvloop(Parent, State).
 
-system_terminate(Reason, _Parent, _Deb, {State, _}) ->
+system_terminate(Reason, _Parent, _Debug, State) ->
     terminate(Reason, State).
 
-system_code_change(Misc, _, _, _) ->
-    {ok, Misc}.
-
-system_get_state({State, _Options}) ->
+system_code_change(State, _Mod, _OldVsn, _Extra) ->
     {ok, State}.
 
+system_get_state(State) -> {ok, State}.
+
 %%--------------------------------------------------------------------
 %% Handle call
 
@@ -467,9 +481,8 @@ handle_timeout(TRef, keepalive, State =
             handle_info({sock_error, Reason}, State)
     end;
 
-handle_timeout(TRef, Msg, State = #state{channel = Channel}) ->
-    Ret = emqx_channel:handle_timeout(TRef, Msg, Channel),
-    handle_chan_return(Ret, State).
+handle_timeout(TRef, Msg, State) ->
+    with_channel(handle_timeout, [TRef, Msg], State).
 
 %%--------------------------------------------------------------------
 %% Parse incoming data
@@ -491,7 +504,7 @@ parse_incoming(Data, Packets, State = #state{parse_state = ParseState}) ->
             parse_incoming(Rest, [Packet|Packets], NState)
     catch
         error:Reason:Stk ->
-            ?LOG(error, "~nParse failed for ~p~nStacktrace: ~p~nFrame data:~p",
+            ?LOG(error, "~nParse failed for ~p~n~p~nFrame data:~p",
                  [Reason, Stk, Data]),
             {[{frame_error, Reason}|Packets], State}
     end.
@@ -505,30 +518,31 @@ next_incoming_msgs(Packets) ->
 %%--------------------------------------------------------------------
 %% Handle incoming packet
 
-handle_incoming(Packet, State = #state{channel = Channel})
-  when is_record(Packet, mqtt_packet) ->
+handle_incoming(Packet, State) when is_record(Packet, mqtt_packet) ->
     ok = inc_incoming_stats(Packet),
     ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]),
-    handle_chan_return(emqx_channel:handle_in(Packet, Channel), State);
+    with_channel(handle_in, [Packet], State);
 
-handle_incoming(FrameError, State = #state{channel = Channel}) ->
-    handle_chan_return(emqx_channel:handle_in(FrameError, Channel), State).
+handle_incoming(FrameError, State) ->
+    with_channel(handle_in, [FrameError], State).
 
 %%--------------------------------------------------------------------
-%% Handle channel return
-
-handle_chan_return(ok, State) ->
-    {ok, State};
-handle_chan_return({ok, NChannel}, State) ->
-    {ok, State#state{channel = NChannel}};
-handle_chan_return({ok, Replies, NChannel}, State) ->
-    {ok, next_msgs(Replies), State#state{channel = NChannel}};
-handle_chan_return({shutdown, Reason, NChannel}, State) ->
-    shutdown(Reason, State#state{channel = NChannel});
-handle_chan_return({shutdown, Reason, OutPacket, NChannel}, State) ->
-    NState = State#state{channel = NChannel},
-    ok = handle_outgoing(OutPacket, NState),
-    shutdown(Reason, NState).
+%% With Channel
+
+with_channel(Fun, Args, State = #state{channel = Channel}) ->
+    case erlang:apply(emqx_channel, Fun, Args ++ [Channel]) of
+        ok -> {ok, State};
+        {ok, NChannel} ->
+            {ok, State#state{channel = NChannel}};
+        {ok, Replies, NChannel} ->
+            {ok, next_msgs(Replies), State#state{channel = NChannel}};
+        {shutdown, Reason, NChannel} ->
+            shutdown(Reason, State#state{channel = NChannel});
+        {shutdown, Reason, Packet, NChannel} ->
+            NState = State#state{channel = NChannel},
+            ok = handle_outgoing(Packet, NState),
+            shutdown(Reason, NState)
+    end.
 
 %%--------------------------------------------------------------------
 %% Handle outgoing packets
@@ -585,9 +599,8 @@ handle_info({sock_error, Reason}, State) ->
     ?LOG(debug, "Socket error: ~p", [Reason]),
     handle_info({sock_closed, Reason}, close_socket(State));
 
-handle_info(Info, State = #state{channel = Channel}) ->
-    Ret = emqx_channel:handle_info(Info, Channel),
-    handle_chan_return(Ret, State).
+handle_info(Info, State) ->
+    with_channel(handle_info, [Info], State).
 
 %%--------------------------------------------------------------------
 %% Ensure rate limit
@@ -613,12 +626,12 @@ run_gc(Stats, State = #state{gc_state = GcSt}) ->
     case ?ENABLED(GcSt) andalso emqx_gc:run(Stats, GcSt) of
         false -> State;
         {IsGC, GcSt1} ->
-            IsGC andalso emqx_metrics:inc('channel.gc.cnt'),
+            IsGC andalso emqx_metrics:inc('channel.gc'),
             State#state{gc_state = GcSt1}
     end.
 
 check_oom(State = #state{channel = Channel}) ->
-    #{zone := Zone} = emqx_channel:info(clientinfo, Channel),
+    Zone = emqx_channel:info(zone, Channel),
     OomPolicy = emqx_zone:oom_policy(Zone),
     case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of
         Shutdown = {shutdown, _Reason} ->
@@ -679,18 +692,13 @@ inc_outgoing_stats(Packet = ?PACKET(Type)) ->
 %%--------------------------------------------------------------------
 %% Helper functions
 
--compile({inline, [append_msg/2]}).
-append_msg(Msgs, Q) when is_list(Msgs) ->
-    lists:append(Msgs, Q);
-append_msg(Msg, Q) -> [Msg|Q].
-
 -compile({inline, [next_msgs/1]}).
 next_msgs(Packet) when is_record(Packet, mqtt_packet) ->
     {outgoing, Packet};
-next_msgs(Action) when is_tuple(Action) ->
-    Action;
-next_msgs(Actions) when is_list(Actions) ->
-    Actions.
+next_msgs(Event) when is_tuple(Event) ->
+    Event;
+next_msgs(More) when is_list(More) ->
+    More.
 
 -compile({inline, [shutdown/2, shutdown/3]}).
 shutdown(Reason, State) ->
@@ -706,3 +714,11 @@ stop(Reason, State) ->
 stop(Reason, Reply, State) ->
     {stop, Reason, Reply, State}.
 
+%%--------------------------------------------------------------------
+%% For CT tests
+%%--------------------------------------------------------------------
+
+set_field(Name, Value, State) ->
+    Pos = emqx_misc:index_of(Name, record_info(fields, state)),
+    setelement(Pos+1, State, Value).
+

src/emqx_guid.erl

@@ -67,14 +67,7 @@ next(NPid, Seq) ->
 bin({Ts, NPid, Seq}) ->
     <<Ts:64, NPid:48, Seq:16>>.
 
-ts() ->
-    case erlang:function_exported(erlang, system_time, 1) of
-        true -> %% R18
-            erlang:system_time(micro_seconds);
-        false ->
-            {MegaSeconds, Seconds, MicroSeconds} = os:timestamp(),
-            (MegaSeconds * 1000000 + Seconds) * 1000000 + MicroSeconds
-    end.
+ts() -> erlang:system_time(micro_seconds).
 
 %% Copied from https://github.com/okeuday/uuid.git.
 npid() ->

src/emqx_limiter.erl

@@ -37,8 +37,7 @@
 
 -spec(init(proplists:proplist()) -> maybe(limiter())).
 init(Options) ->
-    Zone = proplists:get_value(zone, Options),
-    Pl = emqx_zone:publish_limit(Zone),
+    Pl = proplists:get_value(pub_limit, Options),
     Rl = proplists:get_value(rate_limit, Options),
     case ?ENABLED(Pl) or ?ENABLED(Rl) of
         true  -> #limiter{pub_limit  = init_limit(Pl),

src/emqx_message.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_message).
 
+-compile(inline).
+
 -include("emqx.hrl").
 -include("emqx_mqtt.hrl").
 -include("types.hrl").
@@ -36,7 +38,8 @@
         ]).
 
 %% Flags
--export([ get_flag/2
+-export([ clean_dup/1
+        , get_flag/2
         , get_flag/3
         , get_flags/1
         , set_flag/2
@@ -71,25 +74,25 @@
 make(Topic, Payload) ->
     make(undefined, Topic, Payload).
 
--spec(make(atom() | emqx_types:clientid(),
+-spec(make(emqx_types:clientid(),
            emqx_topic:topic(),
            emqx_types:payload()) -> emqx_types:message()).
 make(From, Topic, Payload) ->
     make(From, ?QOS_0, Topic, Payload).
 
--spec(make(atom() | emqx_types:clientid(),
+-spec(make(emqx_types:clientid(),
            emqx_types:qos(),
            emqx_topic:topic(),
            emqx_types:payload()) -> emqx_types:message()).
 make(From, QoS, Topic, Payload) when ?QOS_0 =< QoS, QoS =< ?QOS_2 ->
+    Now = erlang:system_time(millisecond),
     #message{id = emqx_guid:gen(),
              qos = QoS,
              from = From,
-             flags = #{dup => false},
-             headers = #{},
              topic = Topic,
              payload = Payload,
-             timestamp = os:timestamp()}.
+             timestamp = Now
+            }.
 
 -spec(id(emqx_types:message()) -> maybe(binary())).
 id(#message{id = Id}) -> Id.
@@ -106,9 +109,14 @@ topic(#message{topic = Topic}) -> Topic.
 -spec(payload(emqx_types:message()) -> emqx_types:payload()).
 payload(#message{payload = Payload}) -> Payload.
 
--spec(timestamp(emqx_types:message()) -> erlang:timestamp()).
+-spec(timestamp(emqx_types:message()) -> integer()).
 timestamp(#message{timestamp = TS}) -> TS.
 
+-spec(clean_dup(emqx_types:message()) -> emqx_types:message()).
+clean_dup(Msg = #message{flags = Flags = #{dup := true}}) ->
+    Msg#message{flags = Flags#{dup => false}};
+clean_dup(Msg) -> Msg.
+
 -spec(set_flags(map(), emqx_types:message()) -> emqx_types:message()).
 set_flags(Flags, Msg = #message{flags = undefined}) when is_map(Flags) ->
     Msg#message{flags = Flags};
@@ -116,8 +124,13 @@ set_flags(New, Msg = #message{flags = Old}) when is_map(New) ->
     Msg#message{flags = maps:merge(Old, New)}.
 
 -spec(get_flag(flag(), emqx_types:message()) -> boolean()).
+get_flag(_Flag, #message{flags = undefined}) ->
+    false;
 get_flag(Flag, Msg) ->
     get_flag(Flag, Msg, false).
+
+get_flag(_Flag, #message{flags = undefined}, Default) ->
+    Default;
 get_flag(Flag, #message{flags = Flags}, Default) ->
     maps:get(Flag, Flags, Default).
 
@@ -140,25 +153,27 @@ set_flag(Flag, Val, Msg = #message{flags = Flags}) when is_atom(Flag) ->
 -spec(unset_flag(flag(), emqx_types:message()) -> emqx_types:message()).
 unset_flag(Flag, Msg = #message{flags = Flags}) ->
     case maps:is_key(Flag, Flags) of
-        true ->
-            Msg#message{flags = maps:remove(Flag, Flags)};
+        true  -> Msg#message{flags = maps:remove(Flag, Flags)};
         false -> Msg
     end.
 
--spec(set_headers(undefined | map(), emqx_types:message()) -> emqx_types:message()).
+-spec(set_headers(map(), emqx_types:message()) -> emqx_types:message()).
 set_headers(Headers, Msg = #message{headers = undefined}) when is_map(Headers) ->
     Msg#message{headers = Headers};
 set_headers(New, Msg = #message{headers = Old}) when is_map(New) ->
     Msg#message{headers = maps:merge(Old, New)}.
 
--spec(get_headers(emqx_types:message()) -> map()).
-get_headers(Msg) ->
-    Msg#message.headers.
+-spec(get_headers(emqx_types:message()) -> maybe(map())).
+get_headers(Msg) -> Msg#message.headers.
 
 -spec(get_header(term(), emqx_types:message()) -> term()).
+get_header(_Hdr, #message{headers = undefined}) ->
+    undefined;
 get_header(Hdr, Msg) ->
     get_header(Hdr, Msg, undefined).
--spec(get_header(term(), emqx_types:message(), Default :: term()) -> term()).
+-spec(get_header(term(), emqx_types:message(), term()) -> term()).
+get_header(_Hdr, #message{headers = undefined}, Default) ->
+    Default;
 get_header(Hdr, #message{headers = Headers}, Default) ->
     maps:get(Hdr, Headers, Default).
 
@@ -169,10 +184,11 @@ set_header(Hdr, Val, Msg = #message{headers = Headers}) ->
     Msg#message{headers = maps:put(Hdr, Val, Headers)}.
 
 -spec(remove_header(term(), emqx_types:message()) -> emqx_types:message()).
+remove_header(_Hdr, Msg = #message{headers = undefined}) ->
+    Msg;
 remove_header(Hdr, Msg = #message{headers = Headers}) ->
     case maps:is_key(Hdr, Headers) of
-        true ->
-            Msg#message{headers = maps:remove(Hdr, Headers)};
+        true  -> Msg#message{headers = maps:remove(Hdr, Headers)};
         false -> Msg
     end.
 
@@ -180,15 +196,15 @@ remove_header(Hdr, Msg = #message{headers = Headers}) ->
 is_expired(#message{headers = #{'Message-Expiry-Interval' := Interval},
                     timestamp = CreatedAt}) ->
     elapsed(CreatedAt) > timer:seconds(Interval);
-is_expired(_Msg) ->
-    false.
+is_expired(_Msg) -> false.
 
 -spec(update_expiry(emqx_types:message()) -> emqx_types:message()).
 update_expiry(Msg = #message{headers = #{'Message-Expiry-Interval' := Interval},
                              timestamp = CreatedAt}) ->
     case elapsed(CreatedAt) of
         Elapsed when Elapsed > 0 ->
-            set_header('Message-Expiry-Interval', max(1, Interval - (Elapsed div 1000)), Msg);
+            Interval1 = max(1, Interval - (Elapsed div 1000)),
+            set_header('Message-Expiry-Interval', Interval1, Msg);
         _ -> Msg
     end;
 update_expiry(Msg) -> Msg.
@@ -196,30 +212,29 @@ update_expiry(Msg) -> Msg.
 %% @doc Message to PUBLISH Packet.
 -spec(to_packet(emqx_types:packet_id(), emqx_types:message())
       -> emqx_types:packet()).
-to_packet(PacketId, #message{qos = QoS, flags = Flags, headers = Headers,
-                                topic = Topic, payload = Payload}) ->
-    Flags1 = if Flags =:= undefined -> #{};
-                true -> Flags
-             end,
-    Dup = maps:get(dup, Flags1, false),
-    Retain = maps:get(retain, Flags1, false),
-    Publish = #mqtt_packet_publish{topic_name = Topic,
-                                   packet_id  = PacketId,
-                                   properties = publish_props(Headers)},
-    #mqtt_packet{header = #mqtt_packet_header{type   = ?PUBLISH,
-                                              dup    = Dup,
-                                              qos    = QoS,
-                                              retain = Retain},
-                 variable = Publish, payload = Payload}.
-
-publish_props(Headers) ->
-    maps:with(['Payload-Format-Indicator',
-               'Response-Topic',
-               'Correlation-Data',
-               'User-Property',
-               'Subscription-Identifier',
-               'Content-Type',
-               'Message-Expiry-Interval'], Headers).
+to_packet(PacketId, Msg = #message{qos = QoS, headers = Headers,
+                                   topic = Topic, payload = Payload}) ->
+    #mqtt_packet{header   = #mqtt_packet_header{type   = ?PUBLISH,
+                                                dup    = get_flag(dup, Msg),
+                                                qos    = QoS,
+                                                retain = get_flag(retain, Msg)
+                                               },
+                 variable = #mqtt_packet_publish{topic_name = Topic,
+                                                 packet_id  = PacketId,
+                                                 properties = props(Headers)
+                                                },
+                 payload  = Payload
+                }.
+
+props(undefined) -> undefined;
+props(Headers)   -> maps:with(['Payload-Format-Indicator',
+                               'Response-Topic',
+                               'Correlation-Data',
+                               'User-Property',
+                               'Subscription-Identifier',
+                               'Content-Type',
+                               'Message-Expiry-Interval'
+                              ], Headers).
 
 %% @doc Message to map
 -spec(to_map(emqx_types:message()) -> map()).
@@ -240,7 +255,8 @@ to_map(#message{
       headers => Headers,
       topic => Topic,
       payload => Payload,
-      timestamp => Timestamp}.
+      timestamp => Timestamp
+     }.
 
 %% @doc Message to tuple list
 -spec(to_list(emqx_types:message()) -> map()).
@@ -249,7 +265,7 @@ to_list(Msg) ->
 
 %% MilliSeconds
 elapsed(Since) ->
-    max(0, timer:now_diff(os:timestamp(), Since) div 1000).
+    max(0, erlang:system_time(millisecond) - Since).
 
 format(#message{id = Id, qos = QoS, topic = Topic, from = From, flags = Flags, headers = Headers}) ->
     io_lib:format("Message(Id=~s, QoS=~w, Topic=~s, From=~p, Flags=~s, Headers=~s)",

src/emqx_metrics.erl

@@ -137,7 +137,7 @@
 ]).
 
 -define(CHAN_METRICS, [
-    {counter, 'channel.gc.cnt'}
+    {counter, 'channel.gc'}
 ]).
 
 -define(MQTT_METRICS, [
@@ -377,7 +377,7 @@ handle_call({create, Type, Name}, _From, State = #state{next_idx = ?MAX_SIZE}) -
 handle_call({create, Type, Name}, _From, State = #state{next_idx = NextIdx}) ->
     case ets:lookup(?TAB, Name) of
         [#metric{idx = Idx}] ->
-            ?LOG(warning, "~s already exists.", [Name]),
+            ?LOG(info, "~s already exists.", [Name]),
             {reply, {ok, Idx}, State};
         [] ->
             Metric = #metric{name = Name, type = Type, idx = NextIdx},
@@ -459,7 +459,7 @@ reserved_idx('messages.dropped')             -> 49;
 reserved_idx('messages.expired')             -> 50;
 reserved_idx('messages.forward')             -> 51;
 reserved_idx('auth.mqtt.anonymous')          -> 52;
-reserved_idx('channel.gc.cnt')               -> 53;
+reserved_idx('channel.gc')                   -> 53;
 reserved_idx('packets.pubrec.inuse')         -> 54;
 reserved_idx('packets.pubcomp.inuse')        -> 55;
 reserved_idx(_)                              -> undefined.

src/emqx_misc.erl

@@ -72,7 +72,7 @@ pipeline([], Input, State) ->
     {ok, Input, State};
 
 pipeline([Fun|More], Input, State) ->
-    try apply_fun(Fun, Input, State) of
+    case apply_fun(Fun, Input, State) of
         ok -> pipeline(More, Input, State);
         {ok, NState} ->
             pipeline(More, Input, NState);
@@ -82,11 +82,6 @@ pipeline([Fun|More], Input, State) ->
             {error, Reason, State};
         {error, Reason, NState} ->
             {error, Reason, NState}
-    catch
-        Error:Reason:Stacktrace ->
-            ?LOG(error, "pipeline ~p failed: ~p,\nstacktrace: ~0p",
-                [{Fun, Input, State}, {Error, Reason}, Stacktrace]),
-            {error, Reason, State}
     end.
 
 -compile({inline, [apply_fun/3]}).

src/emqx_session.erl

@@ -80,7 +80,7 @@
 
 -export([ takeover/1
         , resume/2
-        , redeliver/1
+        , replay/1
         ]).
 
 -export([expire/2]).
@@ -340,8 +340,7 @@ return_with(Msg, {ok, Publishes, Session}) ->
 pubrec(PacketId, Session = #session{inflight = Inflight}) ->
     case emqx_inflight:lookup(PacketId, Inflight) of
         {value, {Msg, _Ts}} when is_record(Msg, message) ->
-            Inflight1 = emqx_inflight:update(
-                          PacketId, {pubrel, os:timestamp()}, Inflight),
+            Inflight1 = emqx_inflight:update(PacketId, with_ts(pubrel), Inflight),
             {ok, Msg, Session#session{inflight = Inflight1}};
         {value, {pubrel, _Ts}} ->
             {error, ?RC_PACKET_IDENTIFIER_IN_USE};
@@ -418,6 +417,10 @@ acc_msg(Msg, Msgs) ->
 
 -spec(deliver(list(emqx_types:deliver()), session())
       -> {ok, session()} | {ok, replies(), session()}).
+deliver([Deliver], Session) -> %% Optimize
+    Enrich = enrich_fun(Session),
+    deliver_msg(Enrich(Deliver), Session);
+
 deliver(Delivers, Session) ->
     Msgs = lists:map(enrich_fun(Session), Delivers),
     deliver(Msgs, [], Session).
@@ -425,12 +428,19 @@ deliver(Delivers, Session) ->
 deliver([], Publishes, Session) ->
     {ok, lists:reverse(Publishes), Session};
 
-deliver([Msg = #message{qos = ?QOS_0}|More], Acc, Session) ->
-    Publish = {undefined, maybe_ack(Msg)},
-    deliver(More, [Publish|Acc], Session);
+deliver([Msg|More], Acc, Session) ->
+    case deliver_msg(Msg, Session) of
+        {ok, Session1} ->
+            deliver(More, Acc, Session1);
+        {ok, [Publish], Session1} ->
+            deliver(More, [Publish|Acc], Session1)
+    end.
+
+deliver_msg(Msg = #message{qos = ?QOS_0}, Session) ->
+    {ok, [{undefined, maybe_ack(Msg)}], Session};
 
-deliver([Msg = #message{qos = QoS}|More], Acc, Session =
-        #session{next_pkt_id = PacketId, inflight = Inflight})
+deliver_msg(Msg = #message{qos = QoS}, Session =
+            #session{next_pkt_id = PacketId, inflight = Inflight})
     when QoS =:= ?QOS_1 orelse QoS =:= ?QOS_2 ->
     case emqx_inflight:is_full(Inflight) of
         true ->
@@ -438,15 +448,19 @@ deliver([Msg = #message{qos = QoS}|More], Acc, Session =
                            true  -> Session;
                            false -> enqueue(Msg, Session)
                        end,
-            deliver(More, Acc, Session1);
+            {ok, Session1};
         false ->
             Publish = {PacketId, maybe_ack(Msg)},
             Session1 = await(PacketId, Msg, Session),
-            deliver(More, [Publish|Acc], next_pkt_id(Session1))
+            {ok, [Publish], next_pkt_id(Session1)}
     end.
 
--spec(enqueue(list(emqx_types:deliver())|emqx_types:message(), session())
-      -> session()).
+-spec(enqueue(list(emqx_types:deliver())|emqx_types:message(),
+              session()) -> session()).
+enqueue([Deliver], Session) -> %% Optimize
+    Enrich = enrich_fun(Session),
+    enqueue(Enrich(Deliver), Session);
+
 enqueue(Delivers, Session) when is_list(Delivers) ->
     Msgs = lists:map(enrich_fun(Session), Delivers),
     lists:foldl(fun enqueue/2, Session, Msgs);
@@ -510,7 +524,7 @@ enrich_subopts([{subid, SubId}|Opts], Msg, Session) ->
 %%--------------------------------------------------------------------
 
 await(PacketId, Msg, Session = #session{inflight = Inflight}) ->
-    Inflight1 = emqx_inflight:insert(PacketId, {Msg, os:timestamp()}, Inflight),
+    Inflight1 = emqx_inflight:insert(PacketId, with_ts(Msg), Inflight),
     Session#session{inflight = Inflight1}.
 
 %%--------------------------------------------------------------------
@@ -521,9 +535,8 @@ await(PacketId, Msg, Session = #session{inflight = Inflight}) ->
 retry(Session = #session{inflight = Inflight}) ->
     case emqx_inflight:is_empty(Inflight) of
         true  -> {ok, Session};
-        false ->
-            retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight),
-                           [], os:timestamp(), Session)
+        false -> retry_delivery(emqx_inflight:to_list(sort_fun(), Inflight),
+                                [], erlang:system_time(millisecond), Session)
     end.
 
 retry_delivery([], Acc, _Now, Session = #session{retry_interval = Interval}) ->
@@ -561,53 +574,48 @@ retry_delivery(PacketId, pubrel, Now, Acc, Inflight) ->
 expire(awaiting_rel, Session = #session{awaiting_rel = AwaitingRel}) ->
     case maps:size(AwaitingRel) of
         0 -> {ok, Session};
-        _ -> expire_awaiting_rel(os:timestamp(), Session)
+        _ -> expire_awaiting_rel(erlang:system_time(millisecond), Session)
     end.
 
 expire_awaiting_rel(Now, Session = #session{awaiting_rel = AwaitingRel,
                                             await_rel_timeout = Timeout}) ->
-
     NotExpired = fun(_PacketId, Ts) -> age(Now, Ts) < Timeout end,
-    case maps:filter(NotExpired, AwaitingRel) of
-        [] -> {ok, Session};
-        AwaitingRel1 ->
-            {ok, Timeout, Session#session{awaiting_rel = AwaitingRel1}}
+    AwaitingRel1 = maps:filter(NotExpired, AwaitingRel),
+    NSession = Session#session{awaiting_rel = AwaitingRel1},
+    case maps:size(AwaitingRel1) of
+        0 -> {ok, NSession};
+        _ -> {ok, Timeout, NSession}
     end.
 
 %%--------------------------------------------------------------------
-%% Takeover, Resume and Redeliver
+%% Takeover, Resume and Replay
 %%--------------------------------------------------------------------
 
 -spec(takeover(session()) -> ok).
 takeover(#session{subscriptions = Subs}) ->
-    lists:foreach(fun({TopicFilter, _SubOpts}) ->
-                          ok = emqx_broker:unsubscribe(TopicFilter)
-                  end, maps:to_list(Subs)).
+    lists:foreach(fun emqx_broker:unsubscribe/1, maps:keys(Subs)).
 
 -spec(resume(emqx_types:clientid(), session()) -> ok).
 resume(ClientId, #session{subscriptions = Subs}) ->
-    %% 1. Subscribe again.
     lists:foreach(fun({TopicFilter, SubOpts}) ->
-                          ok = emqx_broker:subscribe(TopicFilter, ClientId, SubOpts)
+                      ok = emqx_broker:subscribe(TopicFilter, ClientId, SubOpts)
                   end, maps:to_list(Subs)).
-    %% 2. Run hooks.
-    %% ok = emqx_hooks:run('session.resumed', [#{clientid => ClientId}, attrs(Session)]),
-    %% TODO: 3. Redeliver: Replay delivery and Dequeue pending messages
-    %%Session.
-
--spec(redeliver(session()) -> {ok, replies(), session()}).
-redeliver(Session = #session{inflight = Inflight}) ->
-    Pubs = lists:map(fun to_pub/1, emqx_inflight:to_list(Inflight)),
+
+-spec(replay(session()) -> {ok, replies(), session()}).
+replay(Session = #session{inflight = Inflight}) ->
+    Pubs = replay(Inflight),
     case dequeue(Session) of
         {ok, NSession} -> {ok, Pubs, NSession};
         {ok, More, NSession} ->
             {ok, lists:append(Pubs, More), NSession}
-    end.
+    end;
 
-to_pub({PacketId, {pubrel, _Ts}}) ->
-    {pubrel, PacketId};
-to_pub({PacketId, {Msg, _Ts}}) ->
-    {PacketId, emqx_message:set_flag(dup, true, Msg)}.
+replay(Inflight) ->
+    lists:map(fun({PacketId, {pubrel, _Ts}}) ->
+                      {pubrel, PacketId};
+                 ({PacketId, {Msg, _Ts}}) ->
+                      {PacketId, emqx_message:set_flag(dup, true, Msg)}
+              end, emqx_inflight:to_list(Inflight)).
 
 %%--------------------------------------------------------------------
 %% Next Packet Id
@@ -623,10 +631,10 @@ next_pkt_id(Session = #session{next_pkt_id = Id}) ->
 %% Helper functions
 %%--------------------------------------------------------------------
 
--compile({inline, [sort_fun/0, batch_n/1, age/2]}).
+-compile({inline, [sort_fun/0, batch_n/1, with_ts/1, age/2]}).
 
 sort_fun() ->
-    fun({_, {_, Ts1}}, {_, {_, Ts2}}) -> Ts1 < Ts2 end.
+    fun({_, {_, Ts1}}, {_, {_, Ts2}}) -> Ts1 =< Ts2 end.
 
 batch_n(Inflight) ->
     case emqx_inflight:max_size(Inflight) of
@@ -634,14 +642,16 @@ batch_n(Inflight) ->
         Sz -> Sz - emqx_inflight:size(Inflight)
     end.
 
-age(Now, Ts) -> timer:now_diff(Now, Ts) div 1000.
+with_ts(Msg) ->
+    {Msg, erlang:system_time(millisecond)}.
+
+age(Now, Ts) -> Now - Ts.
 
 %%--------------------------------------------------------------------
 %% For CT tests
 %%--------------------------------------------------------------------
 
-set_field(Name, Val, Channel) ->
-    Fields = record_info(fields, session),
-    Pos = emqx_misc:index_of(Name, Fields),
-    setelement(Pos+1, Channel, Val).
+set_field(Name, Value, Session) ->
+    Pos = emqx_misc:index_of(Name, record_info(fields, session)),
+    setelement(Pos+1, Session, Value).
 

src/emqx_stats.erl

@@ -58,7 +58,7 @@
 -record(update, {name, countdown, interval, func}).
 
 -record(state, {
-          timer :: reference(),
+          timer   :: reference(),
           updates :: [#update{}],
           tick_ms :: timeout()
          }).
@@ -66,48 +66,53 @@
 -type(stats() :: list({atom(), non_neg_integer()})).
 
 %% Connection stats
--define(CONNECTION_STATS, [
-    'connections.count', % current connections
-    'connections.max'    % maximum connections connected
-]).
+-define(CONNECTION_STATS,
+        ['connections.count', %% Count of Concurrent Connections
+         'connections.max'    %% Maximum Number of Concurrent Connections
+        ]).
+
+%% Channel stats
+-define(CHANNEL_STATS,
+        ['channels.count', %% Count of Concurrent Channels
+         'channels.max'    %% Maximum Number of Concurrent Channels
+        ]).
 
 %% Session stats
--define(SESSION_STATS, [
-    'sessions.count',
-    'sessions.max',
-    'sessions.persistent.count',
-    'sessions.persistent.max'
-]).
-
-%% Subscribers, Subscriptions stats
--define(PUBSUB_STATS, [
-    'topics.count',
-    'topics.max',
-    'suboptions.count',
-    'suboptions.max',
-    'subscribers.count',
-    'subscribers.max',
-    'subscriptions.count',
-    'subscriptions.max',
-    'subscriptions.shared.count',
-    'subscriptions.shared.max'
-]).
-
--define(ROUTE_STATS, [
-    'routes.count',
-    'routes.max'
-]).
+-define(SESSION_STATS,
+        ['sessions.count', %% Count of Concurrent Sessions
+         'sessions.max'    %% Maximum Number of Concurrent Sessions
+        ]).
+
+%% PubSub stats
+-define(PUBSUB_STATS,
+        ['topics.count',
+         'topics.max',
+         'suboptions.count',
+         'suboptions.max',
+         'subscribers.count',
+         'subscribers.max',
+         'subscriptions.count',
+         'subscriptions.max',
+         'subscriptions.shared.count',
+         'subscriptions.shared.max'
+        ]).
+
+%% Route stats
+-define(ROUTE_STATS,
+        ['routes.count',
+         'routes.max'
+        ]).
 
 %% Retained stats
--define(RETAINED_STATS, [
-    'retained.count',
-    'retained.max'
-]).
+-define(RETAINED_STATS,
+        ['retained.count',
+         'retained.max'
+        ]).
 
 -define(TAB, ?MODULE).
 -define(SERVER, ?MODULE).
 
--type opts() :: #{tick_ms := timeout()}.
+-type(opts() :: #{tick_ms := timeout()}).
 
 %% @doc Start stats server
 -spec(start_link() -> startlink_ret()).
@@ -122,7 +127,7 @@ start_link(Opts) ->
 stop() ->
     gen_server:call(?SERVER, stop, infinity).
 
-%% @doc Generate stats fun
+%% @doc Generate stats fun.
 -spec(statsfun(Stat :: atom()) -> fun()).
 statsfun(Stat) ->
     fun(Val) -> setstat(Stat, Val) end.
@@ -131,7 +136,7 @@ statsfun(Stat) ->
 statsfun(Stat, MaxStat) ->
     fun(Val) -> setstat(Stat, MaxStat, Val) end.
 
-%% @doc Get all statistics
+%% @doc Get all statistics.
 -spec(getstats() -> stats()).
 getstats() ->
     case ets:info(?TAB, name) of
@@ -139,7 +144,7 @@ getstats() ->
         _ -> ets:tab2list(?TAB)
     end.
 
-%% @doc Get stats by name
+%% @doc Get stats by name.
 -spec(getstat(atom()) -> maybe(non_neg_integer())).
 getstat(Name) ->
     case ets:lookup(?TAB, Name) of
@@ -173,8 +178,7 @@ cancel_update(Name) ->
 rec(Name, Secs, UpFun) ->
     #update{name = Name, countdown = Secs, interval = Secs, func = UpFun}.
 
-cast(Msg) ->
-    gen_server:cast(?SERVER, Msg).
+cast(Msg) -> gen_server:cast(?SERVER, Msg).
 
 %%--------------------------------------------------------------------
 %% gen_server callbacks
@@ -182,8 +186,13 @@ cast(Msg) ->
 
 init(#{tick_ms := TickMs}) ->
     ok = emqx_tables:new(?TAB, [public, set, {write_concurrency, true}]),
-    Stats = lists:append([?CONNECTION_STATS, ?SESSION_STATS, ?PUBSUB_STATS,
-                          ?ROUTE_STATS, ?RETAINED_STATS]),
+    Stats = lists:append([?CONNECTION_STATS,
+                          ?CHANNEL_STATS,
+                          ?SESSION_STATS,
+                          ?PUBSUB_STATS,
+                          ?ROUTE_STATS,
+                          ?RETAINED_STATS
+                         ]),
     true = ets:insert(?TAB, [{Name, 0} || Name <- Stats]),
     {ok, start_timer(#state{updates = [], tick_ms = TickMs}), hibernate}.
 
@@ -211,16 +220,17 @@ handle_cast({setstat, Stat, MaxStat, Val}, State) ->
 
 handle_cast({update_interval, Update = #update{name = Name}},
             State = #state{updates = Updates}) ->
-    case lists:keyfind(Name, #update.name, Updates) of
-        #update{} ->
-            ?LOG(warning, "Duplicated update: ~s", [Name]),
-            {noreply, State};
-        false ->
-            {noreply, State#state{updates = [Update | Updates]}}
-    end;
+    NState = case lists:keyfind(Name, #update.name, Updates) of
+                 #update{} ->
+                     ?LOG(warning, "Duplicated update: ~s", [Name]),
+                     State;
+                 false -> State#state{updates = [Update|Updates]}
+             end,
+    {noreply, NState};
 
 handle_cast({cancel_update, Name}, State = #state{updates = Updates}) ->
-    {noreply, State#state{updates = lists:keydelete(Name, #update.name, Updates)}};
+    Updates1 = lists:keydelete(Name, #update.name, Updates),
+    {noreply, State#state{updates = Updates1}};
 
 handle_cast(Msg, State) ->
     ?LOG(error, "Unexpected cast: ~p", [Msg]),
@@ -233,7 +243,7 @@ handle_info({timeout, TRef, tick}, State = #state{timer = TRef, updates = Update
                          try UpFun()
                          catch
                              _:Error ->
-                                 ?LOG(error, "update ~s failed: ~p", [Name, Error])
+                                 ?LOG(error, "Update ~s failed: ~p", [Name, Error])
                          end,
                          [Update#update{countdown = I} | Acc];
                     (Update = #update{countdown = C}, Acc) ->
@@ -259,10 +269,9 @@ safe_update_element(Key, Val) ->
     try ets:update_element(?TAB, Key, {2, Val}) of
         false ->
             ets:insert_new(?TAB, {Key, Val});
-        true ->
-            true
+        true -> true
     catch
         error:badarg ->
-            ?LOG(warning, "Update ~p to ~p failed", [Key, Val])
+            ?LOG(warning, "Failed to update ~p to ~p", [Key, Val])
     end.
 

src/emqx_types.erl

@@ -176,8 +176,8 @@
 -type(deliver() :: {deliver, topic(), message()}).
 -type(delivery() :: #delivery{}).
 -type(deliver_result() :: ok | {error, term()}).
--type(publish_result() :: [ {node(), topic(), deliver_result()}
-                          | {share, topic(), deliver_result()}]).
+-type(publish_result() :: [{node(), topic(), deliver_result()} |
+                           {share, topic(), deliver_result()}]).
 -type(route() :: #route{}).
 -type(sub_group() :: tuple() | binary()).
 -type(route_entry() :: {topic(), node()} | {topic, sub_group()}).

src/emqx_vm.erl

@@ -18,7 +18,6 @@
 
 -export([ schedulers/0
         , scheduler_usage/1
-        , microsecs/0
         , system_info_keys/0
         , get_system_info/0
         , get_system_info/1
@@ -171,10 +170,6 @@
 schedulers() ->
     erlang:system_info(schedulers).
 
-microsecs() ->
-    {Mega, Sec, Micro} = os:timestamp(),
-    (Mega * 1000000 + Sec) * 1000000 + Micro.
-
 loads() ->
     [{load1,  ftos(avg1()/256)},
      {load5,  ftos(avg5()/256)},

src/emqx_ws_connection.erl

@@ -45,13 +45,16 @@
         , terminate/3
         ]).
 
+%% Export for CT
+-export([set_field/3]).
+
 -import(emqx_misc,
         [ maybe_apply/2
         , start_timer/2
         ]).
 
 -record(state, {
-          %% Peername of the ws connection.
+          %% Peername of the ws connection
           peername :: emqx_types:peername(),
           %% Sockname of the ws connection
           sockname :: emqx_types:peername(),
@@ -65,26 +68,26 @@
           limit_timer :: maybe(reference()),
           %% Parse State
           parse_state :: emqx_frame:parse_state(),
-          %% Serialize function
+          %% Serialize Fun
           serialize :: emqx_frame:serialize_fun(),
           %% Channel
           channel :: emqx_channel:channel(),
           %% GC State
           gc_state :: maybe(emqx_gc:gc_state()),
-          %% Out Pending Packets
-          pendings :: list(emqx_types:packet()),
+          %% Postponed Packets|Cmds|Events
+          postponed :: list(emqx_types:packet()|ws_cmd()|tuple()),
           %% Stats Timer
           stats_timer :: disabled | maybe(reference()),
           %% Idle Timeout
           idle_timeout :: timeout(),
           %% Idle Timer
-          idle_timer :: reference(),
-          %% The stop reason
-          stop_reason :: term()
+          idle_timer :: reference()
         }).
 
 -type(state() :: #state{}).
 
+-type(ws_cmd() :: {active, boolean()}|close).
+
 -define(ACTIVE_N, 100).
 -define(INFO_KEYS, [socktype, peername, sockname, sockstate, active_n]).
 -define(SOCK_STATS, [recv_oct, recv_cnt, send_oct, send_cnt]).
@@ -93,7 +96,7 @@
 -define(ENABLED(X), (X =/= undefined)).
 
 %%--------------------------------------------------------------------
-%% API
+%% Info, Stats
 %%--------------------------------------------------------------------
 
 -spec(info(pid()|state()) -> emqx_types:infos()).
@@ -120,8 +123,16 @@ info(limiter, #state{limiter = Limiter}) ->
     maybe_apply(fun emqx_limiter:info/1, Limiter);
 info(channel, #state{channel = Channel}) ->
     emqx_channel:info(Channel);
-info(stop_reason, #state{stop_reason = Reason}) ->
-    Reason.
+info(gc_state, #state{gc_state = GcSt}) ->
+    maybe_apply(fun emqx_gc:info/1, GcSt);
+info(postponed, #state{postponed = Postponed}) ->
+    Postponed;
+info(stats_timer, #state{stats_timer = TRef}) ->
+    TRef;
+info(idle_timeout, #state{idle_timeout = Timeout}) ->
+    Timeout;
+info(idle_timer, #state{idle_timer = TRef}) ->
+    TRef.
 
 -spec(stats(pid()|state()) -> emqx_types:stats()).
 stats(WsPid) when is_pid(WsPid) ->
@@ -201,8 +212,9 @@ websocket_init([Req, Opts]) ->
                  conn_mod  => ?MODULE
                 },
     Zone = proplists:get_value(zone, Opts),
+    PubLimit = emqx_zone:publish_limit(Zone),
+    Limiter = emqx_limiter:init([{pub_limit, PubLimit}|Opts]),
     ActiveN = proplists:get_value(active_n, Opts, ?ACTIVE_N),
-    Limiter = emqx_limiter:init(Opts),
     FrameOpts = emqx_zone:mqtt_frame_options(Zone),
     ParseState = emqx_frame:initial_parse_state(FrameOpts),
     Serialize = emqx_frame:serialize_fun(),
@@ -223,7 +235,7 @@ websocket_init([Req, Opts]) ->
                 serialize    = Serialize,
                 channel      = Channel,
                 gc_state     = GcState,
-                pendings     = [],
+                postponed    = [],
                 stats_timer  = StatsTimer,
                 idle_timeout = IdleTimeout,
                 idle_timer   = IdleTimer
@@ -235,145 +247,141 @@ websocket_handle({binary, Data}, State) when is_list(Data) ->
 websocket_handle({binary, Data}, State) ->
     ?LOG(debug, "RECV ~p", [Data]),
     ok = inc_recv_stats(1, iolist_size(Data)),
-    parse_incoming(Data, ensure_stats_timer(State));
+    NState = ensure_stats_timer(State),
+    return(parse_incoming(Data, NState));
 
 %% Pings should be replied with pongs, cowboy does it automatically
 %% Pongs can be safely ignored. Clause here simply prevents crash.
 websocket_handle(Frame, State) when Frame =:= ping; Frame =:= pong ->
-    {ok, State};
+    return(State);
 
-websocket_handle({FrameType, _}, State) when FrameType =:= ping;
-                                             FrameType =:= pong ->
-    {ok, State};
+websocket_handle({Frame, _}, State) when Frame =:= ping; Frame =:= pong ->
+    return(State);
 
-websocket_handle({FrameType, _}, State) ->
-    ?LOG(error, "Unexpected frame - ~p", [FrameType]),
-    stop({shutdown, unexpected_ws_frame}, State).
+websocket_handle({Frame, _}, State) ->
+    %% TODO: should not close the ws connection
+    ?LOG(error, "Unexpected frame - ~p", [Frame]),
+    shutdown(unexpected_ws_frame, State).
 
 websocket_info({call, From, Req}, State) ->
     handle_call(From, Req, State);
 
-websocket_info({cast, Msg}, State = #state{channel = Channel}) ->
-    handle_chan_return(emqx_channel:handle_info(Msg, Channel), State);
+websocket_info({cast, rate_limit}, State) ->
+    Stats = #{cnt => emqx_pd:reset_counter(incoming_pubs),
+              oct => emqx_pd:reset_counter(incoming_bytes)
+             },
+    NState = postpone({check_gc, Stats}, State),
+    return(ensure_rate_limit(Stats, NState));
 
-websocket_info({incoming, Packet = ?CONNECT_PACKET(ConnPkt)},
-               State = #state{idle_timer = IdleTimer}) ->
-    ok = emqx_misc:cancel_timer(IdleTimer),
+websocket_info({cast, Msg}, State) ->
+    handle_info(Msg, State);
+
+websocket_info({incoming, Packet = ?CONNECT_PACKET(ConnPkt)}, State) ->
     Serialize = emqx_frame:serialize_fun(ConnPkt),
-    NState = State#state{serialize  = Serialize,
-                         idle_timer = undefined
-                        },
-    handle_incoming(Packet, NState);
+    NState = State#state{serialize = Serialize},
+    handle_incoming(Packet, cancel_idle_timer(NState));
 
 websocket_info({incoming, ?PACKET(?PINGREQ)}, State) ->
-    reply(?PACKET(?PINGRESP), State);
+    return(enqueue(?PACKET(?PINGRESP), State));
 
 websocket_info({incoming, Packet}, State) ->
     handle_incoming(Packet, State);
 
-websocket_info(rate_limit, State) ->
-    InStats = #{cnt => emqx_pd:reset_counter(incoming_pubs),
-                oct => emqx_pd:reset_counter(incoming_bytes)
-               },
-    erlang:send(self(), {check_gc, InStats}),
-    ensure_rate_limit(InStats, State);
-
 websocket_info({check_gc, Stats}, State) ->
-    {ok, check_oom(run_gc(Stats, State))};
+    return(check_oom(run_gc(Stats, State)));
 
 websocket_info(Deliver = {deliver, _Topic, _Msg},
-               State = #state{active_n = ActiveN, channel = Channel}) ->
+               State = #state{active_n = ActiveN}) ->
     Delivers = [Deliver|emqx_misc:drain_deliver(ActiveN)],
-    Ret = emqx_channel:handle_deliver(Delivers, Channel),
-    handle_chan_return(Ret, State);
+    with_channel(handle_deliver, [Delivers], State);
 
-websocket_info({timeout, TRef, limit_timeout}, State = #state{limit_timer = TRef}) ->
+websocket_info({timeout, TRef, limit_timeout},
+               State = #state{limit_timer = TRef}) ->
     NState = State#state{sockstate   = running,
                          limit_timer = undefined
                         },
-    {reply, [{active, true}], NState};
+    return(enqueue({active, true}, NState));
 
 websocket_info({timeout, TRef, Msg}, State) when is_reference(TRef) ->
     handle_timeout(TRef, Msg, State);
 
-websocket_info(Close = {close, _Reason}, State) ->
-    handle_info(Close, State);
-
 websocket_info({shutdown, Reason}, State) ->
-    stop({shutdown, Reason}, State);
+    shutdown(Reason, State);
 
 websocket_info({stop, Reason}, State) ->
-    stop(Reason, State);
+    shutdown(Reason, State);
 
 websocket_info(Info, State) ->
     handle_info(Info, State).
 
 websocket_close(Reason, State) ->
-    ?LOG(debug, "WebSocket closed due to ~p~n", [Reason]),
+    ?LOG(debug, "Websocket closed due to ~p~n", [Reason]),
     handle_info({sock_closed, Reason}, State).
 
-terminate(SockError, _Req, #state{channel    = Channel,
-                                  stop_reason = Reason}) ->
-    ?LOG(debug, "Terminated for ~p, sockerror: ~p", [Reason, SockError]),
+terminate(Reason, _Req, #state{channel = Channel}) ->
+    ?LOG(debug, "Terminated due to ~p", [Reason]),
     emqx_channel:terminate(Reason, Channel).
 
 %%--------------------------------------------------------------------
 %% Handle call
+%%--------------------------------------------------------------------
 
 handle_call(From, info, State) ->
     gen_server:reply(From, info(State)),
-    {ok, State};
+    return(State);
 
 handle_call(From, stats, State) ->
     gen_server:reply(From, stats(State)),
-    {ok, State};
+    return(State);
 
 handle_call(From, Req, State = #state{channel = Channel}) ->
     case emqx_channel:handle_call(Req, Channel) of
         {reply, Reply, NChannel} ->
-            _ = gen_server:reply(From, Reply),
-            {ok, State#state{channel = NChannel}};
-        {stop, Reason, Reply, NChannel} ->
-            _ = gen_server:reply(From, Reply),
-            stop(Reason, State#state{channel = NChannel});
-        {stop, Reason, Reply, OutPacket, NChannel} ->
+            gen_server:reply(From, Reply),
+            return(State#state{channel = NChannel});
+        {shutdown, Reason, Reply, NChannel} ->
+            gen_server:reply(From, Reply),
+            shutdown(Reason, State#state{channel = NChannel});
+        {shutdown, Reason, Reply, Packet, NChannel} ->
             gen_server:reply(From, Reply),
             NState = State#state{channel = NChannel},
-            stop(Reason, enqueue(OutPacket, NState))
+            shutdown(Reason, enqueue(Packet, NState))
     end.
 
 %%--------------------------------------------------------------------
 %% Handle Info
+%%--------------------------------------------------------------------
 
 handle_info({connack, ConnAck}, State) ->
-    reply(enqueue(ConnAck, State));
+    return(enqueue(ConnAck, State));
 
 handle_info({close, Reason}, State) ->
-    stop({shutdown, Reason}, State);
+    ?LOG(debug, "Force to close the socket due to ~p", [Reason]),
+    return(enqueue(close, State));
 
 handle_info({event, connected}, State = #state{channel = Channel}) ->
     ClientId = emqx_channel:info(clientid, Channel),
-    emqx_cm:register_channel(ClientId, info(State), stats(State)),
-    reply(State);
+    ok = emqx_cm:register_channel(ClientId, info(State), stats(State)),
+    return(State);
 
 handle_info({event, disconnected}, State = #state{channel = Channel}) ->
     ClientId = emqx_channel:info(clientid, Channel),
     emqx_cm:set_chan_info(ClientId, info(State)),
     emqx_cm:connection_closed(ClientId),
-    reply(State);
+    return(State);
 
 handle_info({event, _Other}, State = #state{channel = Channel}) ->
     ClientId = emqx_channel:info(clientid, Channel),
     emqx_cm:set_chan_info(ClientId, info(State)),
     emqx_cm:set_chan_stats(ClientId, stats(State)),
-    reply(State);
+    return(State);
 
-handle_info(Info, State = #state{channel = Channel}) ->
-    Ret = emqx_channel:handle_info(Info, Channel),
-    handle_chan_return(Ret, State).
+handle_info(Info, State) ->
+    with_channel(handle_info, [Info], State).
 
 %%--------------------------------------------------------------------
 %% Handle timeout
+%%--------------------------------------------------------------------
 
 handle_timeout(TRef, idle_timeout, State = #state{idle_timer = TRef}) ->
     shutdown(idle_timeout, State);
@@ -382,33 +390,24 @@ handle_timeout(TRef, keepalive, State) when is_reference(TRef) ->
     RecvOct = emqx_pd:get_counter(recv_oct),
     handle_timeout(TRef, {keepalive, RecvOct}, State);
 
-handle_timeout(TRef, emit_stats, State =
-               #state{channel = Channel, stats_timer = TRef}) ->
+handle_timeout(TRef, emit_stats, State = #state{channel = Channel,
+                                                stats_timer = TRef}) ->
     ClientId = emqx_channel:info(clientid, Channel),
     emqx_cm:set_chan_stats(ClientId, stats(State)),
-    reply(State#state{stats_timer = undefined});
+    return(State#state{stats_timer = undefined});
 
-handle_timeout(TRef, TMsg, State = #state{channel = Channel}) ->
-    Ret = emqx_channel:handle_timeout(TRef, TMsg, Channel),
-    handle_chan_return(Ret, State).
-
-%%--------------------------------------------------------------------
-%% Ensure stats timer
-
--compile({inline, [ensure_stats_timer/1]}).
-ensure_stats_timer(State = #state{idle_timeout = Timeout,
-                                  stats_timer  = undefined}) ->
-    State#state{stats_timer = start_timer(Timeout, emit_stats)};
-ensure_stats_timer(State) -> State.
+handle_timeout(TRef, TMsg, State) ->
+    with_channel(handle_timeout, [TRef, TMsg], State).
 
 %%--------------------------------------------------------------------
 %% Ensure rate limit
+%%--------------------------------------------------------------------
 
 ensure_rate_limit(Stats, State = #state{limiter = Limiter}) ->
     case ?ENABLED(Limiter) andalso emqx_limiter:check(Stats, Limiter) of
-        false -> {ok, State};
+        false -> State;
         {ok, Limiter1} ->
-            {ok, State#state{limiter = Limiter1}};
+            State#state{limiter = Limiter1};
         {pause, Time, Limiter1} ->
             ?LOG(debug, "Pause ~pms due to rate limit", [Time]),
             TRef = start_timer(Time, limit_timeout),
@@ -416,102 +415,108 @@ ensure_rate_limit(Stats, State = #state{limiter = Limiter}) ->
                                  limiter     = Limiter1,
                                  limit_timer = TRef
                                 },
-            {reply, [{active, false}], NState}
+            enqueue({active, false}, NState)
     end.
 
 %%--------------------------------------------------------------------
-%% Run GC and Check OOM
+%% Run GC, Check OOM
+%%--------------------------------------------------------------------
 
 run_gc(Stats, State = #state{gc_state = GcSt}) ->
     case ?ENABLED(GcSt) andalso emqx_gc:run(Stats, GcSt) of
-        false -> State;
         {IsGC, GcSt1} ->
-            IsGC andalso emqx_metrics:inc('channel.gc.cnt'),
-            State#state{gc_state = GcSt1}
+            IsGC andalso emqx_metrics:inc('channel.gc'),
+            State#state{gc_state = GcSt1};
+        false -> State
     end.
 
 check_oom(State = #state{channel = Channel}) ->
-    #{zone := Zone} = emqx_channel:info(clientinfo, Channel),
-    OomPolicy = emqx_zone:oom_policy(Zone),
+    OomPolicy = emqx_zone:oom_policy(emqx_channel:info(zone, Channel)),
     case ?ENABLED(OomPolicy) andalso emqx_misc:check_oom(OomPolicy) of
         Shutdown = {shutdown, _Reason} ->
-            erlang:send(self(), Shutdown);
-        _Other -> ok
-    end,
-    State.
+            postpone(Shutdown, State);
+        _Other -> State
+    end.
 
 %%--------------------------------------------------------------------
 %% Parse incoming data
+%%--------------------------------------------------------------------
 
 parse_incoming(<<>>, State) ->
-    {ok, State};
+    State;
 
 parse_incoming(Data, State = #state{parse_state = ParseState}) ->
     try emqx_frame:parse(Data, ParseState) of
         {more, NParseState} ->
-            {ok, State#state{parse_state = NParseState}};
+            State#state{parse_state = NParseState};
         {ok, Packet, Rest, NParseState} ->
-            erlang:send(self(), {incoming, Packet}),
-            parse_incoming(Rest, State#state{parse_state = NParseState})
+            NState = State#state{parse_state = NParseState},
+            parse_incoming(Rest, postpone({incoming, Packet}, NState))
     catch
         error:Reason:Stk ->
-            ?LOG(error, "~nParse failed for ~p~nStacktrace: ~p~nFrame data: ~p",
+            ?LOG(error, "~nParse failed for ~p~n~p~nFrame data: ~p",
                  [Reason, Stk, Data]),
-            self() ! {incoming, {frame_error, Reason}},
-            {ok, State}
+            FrameError = {frame_error, Reason},
+            postpone({incoming, FrameError}, State)
     end.
 
 %%--------------------------------------------------------------------
 %% Handle incoming packet
+%%--------------------------------------------------------------------
 
-handle_incoming(Packet, State = #state{active_n = ActiveN, channel = Channel})
+handle_incoming(Packet, State = #state{active_n = ActiveN})
   when is_record(Packet, mqtt_packet) ->
     ?LOG(debug, "RECV ~s", [emqx_packet:format(Packet)]),
     ok = inc_incoming_stats(Packet),
-    (emqx_pd:get_counter(incoming_pubs) > ActiveN)
-        andalso erlang:send(self(), rate_limit),
-    Ret = emqx_channel:handle_in(Packet, Channel),
-    handle_chan_return(Ret, State);
+    NState = case emqx_pd:get_counter(incoming_pubs) > ActiveN of
+                 true  -> postpone({cast, rate_limit}, State);
+                 false -> State
+             end,
+    with_channel(handle_in, [Packet], NState);
 
-handle_incoming(FrameError, State = #state{channel = Channel}) ->
-    handle_chan_return(emqx_channel:handle_in(FrameError, Channel), State).
+handle_incoming(FrameError, State) ->
+    with_channel(handle_in, [FrameError], State).
 
 %%--------------------------------------------------------------------
-%% Handle channel return
+%% With Channel
+%%--------------------------------------------------------------------
 
-handle_chan_return(ok, State) ->
-    reply(State);
-handle_chan_return({ok, NChannel}, State) ->
-    reply(State#state{channel= NChannel});
-handle_chan_return({ok, Replies, NChannel}, State) ->
-    reply(Replies, State#state{channel= NChannel});
-handle_chan_return({shutdown, Reason, NChannel}, State) ->
-    stop(Reason, State#state{channel = NChannel});
-handle_chan_return({shutdown, Reason, OutPacket, NChannel}, State) ->
-    NState = State#state{channel = NChannel},
-    stop(Reason, enqueue(OutPacket, NState)).
+with_channel(Fun, Args, State = #state{channel = Channel}) ->
+    case erlang:apply(emqx_channel, Fun, Args ++ [Channel]) of
+        ok -> return(State);
+        {ok, NChannel} ->
+            return(State#state{channel = NChannel});
+        {ok, Replies, NChannel} ->
+            return(postpone(Replies, State#state{channel= NChannel}));
+        {shutdown, Reason, NChannel} ->
+            shutdown(Reason, State#state{channel = NChannel});
+        {shutdown, Reason, Packet, NChannel} ->
+            NState = State#state{channel = NChannel},
+            shutdown(Reason, postpone(Packet, NState))
+    end.
 
 %%--------------------------------------------------------------------
 %% Handle outgoing packets
+%%--------------------------------------------------------------------
 
 handle_outgoing(Packets, State = #state{active_n = ActiveN}) ->
     IoData = lists:map(serialize_and_inc_stats_fun(State), Packets),
     Oct = iolist_size(IoData),
     ok = inc_sent_stats(length(Packets), Oct),
-    case emqx_pd:get_counter(outgoing_pubs) > ActiveN of
-        true ->
-            OutStats = #{cnt => emqx_pd:reset_counter(outgoing_pubs),
-                         oct => emqx_pd:reset_counter(outgoing_bytes)
-                        },
-            erlang:send(self(), {check_gc, OutStats});
-        false -> ok
-    end,
-    {{binary, IoData}, ensure_stats_timer(State)}.
+    NState = case emqx_pd:get_counter(outgoing_pubs) > ActiveN of
+                 true ->
+                     Stats = #{cnt => emqx_pd:reset_counter(outgoing_pubs),
+                               oct => emqx_pd:reset_counter(outgoing_bytes)
+                              },
+                     postpone({check_gc, Stats}, State);
+                 false -> State
+             end,
+    {{binary, IoData}, ensure_stats_timer(NState)}.
 
 serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
     fun(Packet) ->
         case Serialize(Packet) of
-            <<>> -> ?LOG(warning, "~s is discarded due to the frame is too large!",
+            <<>> -> ?LOG(warning, "~s is discarded due to the frame is too large.",
                          [emqx_packet:format(Packet)]),
                     <<>>;
             Data -> ?LOG(debug, "SEND ~s", [emqx_packet:format(Packet)]),
@@ -522,6 +527,7 @@ serialize_and_inc_stats_fun(#state{serialize = Serialize}) ->
 
 %%--------------------------------------------------------------------
 %% Inc incoming/outgoing stats
+%%--------------------------------------------------------------------
 
 -compile({inline,
           [ inc_recv_stats/2
@@ -561,46 +567,86 @@ inc_sent_stats(Cnt, Oct) ->
     emqx_metrics:inc('bytes.sent', Oct).
 
 %%--------------------------------------------------------------------
-%% Reply or Stop
+%% Helper functions
+%%--------------------------------------------------------------------
+
+-compile({inline, [cancel_idle_timer/1, ensure_stats_timer/1]}).
+
+%%--------------------------------------------------------------------
+%% Cancel idle timer
+
+cancel_idle_timer(State = #state{idle_timer = IdleTimer}) ->
+    ok = emqx_misc:cancel_timer(IdleTimer),
+    State#state{idle_timer = undefined}.
 
-reply(Packet, State) when is_record(Packet, mqtt_packet) ->
-    reply(enqueue(Packet, State));
-reply({outgoing, Packets}, State) ->
-    reply(enqueue(Packets, State));
-reply(Other, State) when is_tuple(Other) ->
-    self() ! Other,
-    reply(State);
+%%--------------------------------------------------------------------
+%% Ensure stats timer
 
-reply([], State) ->
-    reply(State);
-reply([Packet|More], State) when is_record(Packet, mqtt_packet) ->
-    reply(More, enqueue(Packet, State));
-reply([{outgoing, Packets}|More], State) ->
-    reply(More, enqueue(Packets, State));
-reply([Other|More], State) ->
-    self() ! Other,
-    reply(More, State).
+ensure_stats_timer(State = #state{idle_timeout = Timeout,
+                                  stats_timer  = undefined}) ->
+    State#state{stats_timer = start_timer(Timeout, emit_stats)};
+ensure_stats_timer(State) -> State.
 
--compile({inline, [reply/1, enqueue/2]}).
+-compile({inline, [postpone/2, enqueue/2, return/1, shutdown/2]}).
 
-reply(State = #state{pendings = []}) ->
+%%--------------------------------------------------------------------
+%% Postpone the packet, cmd or event
+
+postpone(Packet, State) when is_record(Packet, mqtt_packet) ->
+    enqueue(Packet, State);
+postpone({outgoing, Packets}, State) ->
+    enqueue(Packets, State);
+postpone(Event, State) when is_tuple(Event) ->
+    enqueue(Event, State);
+postpone(More, State) when is_list(More) ->
+    lists:foldl(fun postpone/2, State, More).
+
+enqueue([Packet], State = #state{postponed = Postponed}) ->
+    State#state{postponed = [Packet|Postponed]};
+enqueue(Packets, State = #state{postponed = Postponed})
+  when is_list(Packets) ->
+    State#state{postponed = lists:reverse(Packets) ++ Postponed};
+enqueue(Other, State = #state{postponed = Postponed}) ->
+    State#state{postponed = [Other|Postponed]}.
+
+shutdown(Reason, State = #state{postponed = Postponed}) ->
+    return(State#state{postponed = [{shutdown, Reason}|Postponed]}).
+
+return(State = #state{postponed = []}) ->
     {ok, State};
-reply(State = #state{pendings = Pendings}) ->
-    {Reply, NState} = handle_outgoing(Pendings, State),
-    {reply, Reply, NState#state{pendings = []}}.
-
-enqueue(Packet, State) when is_record(Packet, mqtt_packet) ->
-    enqueue([Packet], State);
-enqueue(Packets, State = #state{pendings = Pendings}) ->
-    State#state{pendings = lists:append(Pendings, Packets)}.
-
-shutdown(Reason, State) ->
-    stop({shutdown, Reason}, State).
-
-stop(Reason, State = #state{pendings = []}) ->
-    {stop, State#state{stop_reason = Reason}};
-stop(Reason, State = #state{pendings = Pendings}) ->
-    {Reply, State1} = handle_outgoing(Pendings, State),
-    State2 = State1#state{pendings = [], stop_reason = Reason},
-    {reply, [Reply, close], State2}.
+return(State = #state{postponed = Postponed}) ->
+    {Packets, Cmds, Events} = classify(Postponed, [], [], []),
+    ok = lists:foreach(fun trigger/1, Events),
+    State1 = State#state{postponed = []},
+    case {Packets, Cmds} of
+        {[], []}   -> {ok, State1};
+        {[], Cmds} -> {Cmds, State1};
+        {Packets, Cmds} ->
+            {Frame, State2} = handle_outgoing(Packets, State1),
+            {[Frame|Cmds], State2}
+    end.
+
+classify([], Packets, Cmds, Events) ->
+    {Packets, Cmds, Events};
+classify([Packet|More], Packets, Cmds, Events)
+  when is_record(Packet, mqtt_packet) ->
+    classify(More, [Packet|Packets], Cmds, Events);
+classify([Cmd = {active, _}|More], Packets, Cmds, Events) ->
+    classify(More, Packets, [Cmd|Cmds], Events);
+classify([Cmd = {shutdown, _Reason}|More], Packets, Cmds, Events) ->
+    classify(More, Packets, [Cmd|Cmds], Events);
+classify([Cmd = close|More], Packets, Cmds, Events) ->
+    classify(More, Packets, [Cmd|Cmds], Events);
+classify([Event|More], Packets, Cmds, Events) ->
+    classify(More, Packets, Cmds, [Event|Events]).
+
+trigger(Event) -> erlang:send(self(), Event).
+
+%%--------------------------------------------------------------------
+%% For CT tests
+%%--------------------------------------------------------------------
+
+set_field(Name, Value, State) ->
+    Pos = emqx_misc:index_of(Name, record_info(fields, state)),
+    setelement(Pos+1, State, Value).
 

test/emqx_broker_SUITE.erl

@@ -89,6 +89,7 @@ t_subscribers(_) ->
 
 t_subscriptions(_) ->
     emqx_broker:subscribe(<<"topic">>, <<"clientid">>, #{qos => 1}),
+    ok = timer:sleep(100),
     ?assertEqual(#{qos => 1, subid => <<"clientid">>},
                  proplists:get_value(<<"topic">>, emqx_broker:subscriptions(self()))),
     ?assertEqual(#{qos => 1, subid => <<"clientid">>},

test/emqx_broker_helper_SUITE.erl

@@ -65,3 +65,11 @@ t_shards_num(_) ->
     
 t_get_sub_shard(_) ->
     ?assertEqual(0, emqx_broker_helper:get_sub_shard(self(), <<"topic">>)).
+
+t_terminate(_) ->
+    ?assertEqual(ok, gen_server:stop(emqx_broker_helper)).
+
+t_uncovered_func(_) ->
+    gen_server:call(emqx_broker_helper, test),
+    gen_server:cast(emqx_broker_helper, test),
+    emqx_broker_helper ! test.

test/emqx_channel_SUITE.erl

@@ -23,20 +23,6 @@
 -include("emqx_mqtt.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
--define(DEFAULT_CONNINFO,
-        #{peername => {{127,0,0,1}, 3456},
-          sockname => {{127,0,0,1}, 1883},
-          conn_mod => emqx_connection,
-          proto_name => <<"MQTT">>,
-          proto_ver => ?MQTT_PROTO_V5,
-          clean_start => true,
-          keepalive => 30,
-          clientid => <<"clientid">>,
-          username => <<"username">>,
-          conn_props => #{},
-          receive_maximum => 100,
-          expiry_interval => 0
-         }).
 
 all() -> emqx_ct:all(?MODULE).
 
@@ -45,40 +31,40 @@ all() -> emqx_ct:all(?MODULE).
 %%--------------------------------------------------------------------
 
 init_per_suite(Config) ->
-    Config.
-
-end_per_suite(_Config) ->
-    ok.
-
-init_per_testcase(_TestCase, Config) ->
     %% CM Meck
-    ok = meck:new(emqx_cm, [passthrough, no_history]),
+    ok = meck:new(emqx_cm, [passthrough, no_history, no_link]),
     %% Access Control Meck
-    ok = meck:new(emqx_access_control, [passthrough, no_history]),
+    ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
     ok = meck:expect(emqx_access_control, authenticate,
                      fun(_) -> {ok, #{auth_result => success}} end),
     ok = meck:expect(emqx_access_control, check_acl, fun(_, _, _) -> allow end),
     %% Broker Meck
-    ok = meck:new(emqx_broker, [passthrough, no_history]),
+    ok = meck:new(emqx_broker, [passthrough, no_history, no_link]),
     %% Hooks Meck
-    ok = meck:new(emqx_hooks, [passthrough, no_history]),
+    ok = meck:new(emqx_hooks, [passthrough, no_history, no_link]),
     ok = meck:expect(emqx_hooks, run, fun(_Hook, _Args) -> ok end),
     ok = meck:expect(emqx_hooks, run_fold, fun(_Hook, _Args, Acc) -> Acc end),
     %% Session Meck
-    ok = meck:new(emqx_session, [passthrough, no_history]),
+    ok = meck:new(emqx_session, [passthrough, no_history, no_link]),
     %% Metrics
-    ok = meck:new(emqx_metrics, [passthrough, no_history]),
+    ok = meck:new(emqx_metrics, [passthrough, no_history, no_link]),
     ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end),
     ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end),
     Config.
 
-end_per_testcase(_TestCase, Config) ->
+end_per_suite(_Config) ->
     ok = meck:unload(emqx_access_control),
     ok = meck:unload(emqx_metrics),
     ok = meck:unload(emqx_session),
     ok = meck:unload(emqx_broker),
     ok = meck:unload(emqx_hooks),
     ok = meck:unload(emqx_cm),
+    ok.
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, Config) ->
     Config.
 
 %%--------------------------------------------------------------------
@@ -92,17 +78,15 @@ t_chan_info(_) ->
     ?assertEqual(clientinfo(), ClientInfo).
 
 t_chan_caps(_) ->
-    Caps = emqx_mqtt_caps:default(),
-    ?assertEqual(Caps#{max_packet_size => 1048576},
-                 emqx_channel:caps(channel())).
-
-%%--------------------------------------------------------------------
-%% Test cases for channel init
-%%--------------------------------------------------------------------
-
-%% TODO:
-t_chan_init(_) ->
-    _Channel = channel().
+     #{max_clientid_len := 65535,
+       max_qos_allowed := 2,
+       max_topic_alias := 65535,
+       max_topic_levels := 0,
+       retain_available := true,
+       shared_subscription := true,
+       subscription_identifiers := true,
+       wildcard_subscription := true
+      } = emqx_channel:caps(channel()).
 
 %%--------------------------------------------------------------------
 %% Test cases for channel handle_in
@@ -114,8 +98,8 @@ t_handle_in_connect_packet_sucess(_) ->
                              {ok, #{session => session(), present => false}}
                      end),
     IdleChannel = channel(#{conn_state => idle}),
-    {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, _)}], Channel}
-      = emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), IdleChannel),
+    {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, _)}], Channel} =
+        emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), IdleChannel),
     ClientInfo = emqx_channel:info(clientinfo, Channel),
     ?assertMatch(#{clientid := <<"clientid">>,
                    username := <<"username">>
@@ -125,32 +109,47 @@ t_handle_in_connect_packet_sucess(_) ->
 t_handle_in_unexpected_connect_packet(_) ->
     Channel = emqx_channel:set_field(conn_state, connected, channel()),
     Packet = ?DISCONNECT_PACKET(?RC_PROTOCOL_ERROR),
-    {ok, [{outgoing, Packet}, {close, protocol_error}], Channel}
-      = emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), Channel).
+    {ok, [{outgoing, Packet}, {close, protocol_error}], Channel} =
+        emqx_channel:handle_in(?CONNECT_PACKET(connpkt()), Channel).
 
 t_handle_in_qos0_publish(_) ->
-    ok = meck:expect(emqx_broker, publish, fun(_) -> ok end),
+    ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
     Channel = channel(#{conn_state => connected}),
     Publish = ?PUBLISH_PACKET(?QOS_0, <<"topic">>, undefined, <<"payload">>),
     {ok, _NChannel} = emqx_channel:handle_in(Publish, Channel).
-    % ?assertEqual(#{publish_in => 1}, emqx_channel:info(pub_stats, NChannel)).
 
 t_handle_in_qos1_publish(_) ->
-    ok = meck:expect(emqx_broker, publish, fun(_) -> ok end),
-    Channel = channel(#{conn_state => connected}),
+    ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
     Publish = ?PUBLISH_PACKET(?QOS_1, <<"topic">>, 1, <<"payload">>),
-    {ok, ?PUBACK_PACKET(1, RC), _NChannel} = emqx_channel:handle_in(Publish, Channel),
-    ?assert((RC == ?RC_SUCCESS) orelse (RC == ?RC_NO_MATCHING_SUBSCRIBERS)).
-    % ?assertEqual(#{publish_in => 1, puback_out => 1}, emqx_channel:info(pub_stats, NChannel)).
+    {ok, ?PUBACK_PACKET(1, ?RC_NO_MATCHING_SUBSCRIBERS), _Channel} =
+        emqx_channel:handle_in(Publish, channel(#{conn_state => connected})).
 
 t_handle_in_qos2_publish(_) ->
-    ok = meck:expect(emqx_session, publish, fun(_, _Msg, Session) -> {ok, [], Session} end),
-    ok = meck:expect(emqx_session, info, fun(await_rel_timeout, _Session) -> 300 end),
-    Channel = channel(#{conn_state => connected}),
-    Publish = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 1, <<"payload">>),
-    {ok, ?PUBREC_PACKET(1, RC), _NChannel} = emqx_channel:handle_in(Publish, Channel),
-    ?assert((RC == ?RC_SUCCESS) orelse (RC == ?RC_NO_MATCHING_SUBSCRIBERS)).
-    % ?assertEqual(#{publish_in => 1, pubrec_out => 1}, emqx_channel:info(pub_stats, NChannel)).
+    ok = meck:expect(emqx_broker, publish, fun(_) -> [{node(), <<"topic">>, 1}] end),
+    Channel = channel(#{conn_state => connected, session => session()}),
+    Publish1 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 1, <<"payload">>),
+    {ok, ?PUBREC_PACKET(1, ?RC_SUCCESS), Channel1} =
+        emqx_channel:handle_in(Publish1, Channel),
+    ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
+    Publish2 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 2, <<"payload">>),
+    {ok, ?PUBREC_PACKET(2, ?RC_NO_MATCHING_SUBSCRIBERS), Channel2} =
+        emqx_channel:handle_in(Publish2, Channel1),
+    ?assertEqual(2, proplists:get_value(awaiting_rel_cnt, emqx_channel:stats(Channel2))).
+
+t_handle_in_qos2_publish_with_error_return(_) ->
+    ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end),
+    ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
+    Session = session(#{max_awaiting_rel => 2, awaiting_rel => #{1 => 1}}),
+    Channel = channel(#{conn_state => connected, session => Session}),
+    Publish1 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 1, <<"payload">>),
+    {ok, ?PUBREC_PACKET(1, ?RC_PACKET_IDENTIFIER_IN_USE), Channel} =
+        emqx_channel:handle_in(Publish1, Channel),
+    Publish2 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 2, <<"payload">>),
+    {ok, ?PUBREC_PACKET(2, ?RC_NO_MATCHING_SUBSCRIBERS), Channel1} =
+        emqx_channel:handle_in(Publish2, Channel),
+    Publish3 = ?PUBLISH_PACKET(?QOS_2, <<"topic">>, 3, <<"payload">>),
+    {ok, ?PUBREC_PACKET(3, ?RC_RECEIVE_MAXIMUM_EXCEEDED), Channel1} =
+        emqx_channel:handle_in(Publish3, Channel1).
 
 t_handle_in_puback_ok(_) ->
     Msg = emqx_message:make(<<"t">>, <<"payload">>),
@@ -180,46 +179,38 @@ t_handle_in_pubrec_ok(_) ->
     Msg = emqx_message:make(test,?QOS_2, <<"t">>, <<"payload">>),
     ok = meck:expect(emqx_session, pubrec, fun(_, Session) -> {ok, Msg, Session} end),
     Channel = channel(#{conn_state => connected}),
-    {ok, ?PUBREL_PACKET(1, ?RC_SUCCESS), _Channel1}
-        = emqx_channel:handle_in(?PUBREC_PACKET(1, ?RC_SUCCESS), Channel).
-    % ?assertEqual(#{pubrec_in => 1, pubrel_out => 1},
-    %              emqx_channel:info(pub_stats, Channel1)).
+    {ok, ?PUBREL_PACKET(1, ?RC_SUCCESS), _Channel1} =
+        emqx_channel:handle_in(?PUBREC_PACKET(1, ?RC_SUCCESS), Channel).
 
 t_handle_in_pubrec_id_in_use(_) ->
     ok = meck:expect(emqx_session, pubrec,
                      fun(_, _Session) ->
                              {error, ?RC_PACKET_IDENTIFIER_IN_USE}
                      end),
-    {ok, ?PUBREL_PACKET(1, ?RC_PACKET_IDENTIFIER_IN_USE), _Channel}
-        = emqx_channel:handle_in(?PUBREC_PACKET(1, ?RC_SUCCESS), channel()).
-    % ?assertEqual(#{pubrec_in => 1, pubrel_out => 1},
-    %              emqx_channel:info(pub_stats, Channel)).
+    {ok, ?PUBREL_PACKET(1, ?RC_PACKET_IDENTIFIER_IN_USE), _Channel} =
+        emqx_channel:handle_in(?PUBREC_PACKET(1, ?RC_SUCCESS), channel()).
 
 t_handle_in_pubrec_id_not_found(_) ->
     ok = meck:expect(emqx_session, pubrec,
                      fun(_, _Session) ->
                              {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
                      end),
-    {ok, ?PUBREL_PACKET(1, ?RC_PACKET_IDENTIFIER_NOT_FOUND), _Channel}
-        = emqx_channel:handle_in(?PUBREC_PACKET(1, ?RC_SUCCESS), channel()).
-    % ?assertEqual(#{pubrec_in => 1, pubrel_out => 1},
-    %              emqx_channel:info(pub_stats, Channel)).
+    {ok, ?PUBREL_PACKET(1, ?RC_PACKET_IDENTIFIER_NOT_FOUND), _Channel} =
+        emqx_channel:handle_in(?PUBREC_PACKET(1, ?RC_SUCCESS), channel()).
 
 t_handle_in_pubrel_ok(_) ->
     ok = meck:expect(emqx_session, pubrel, fun(_, Session) -> {ok, Session} end),
     Channel = channel(#{conn_state => connected}),
-    {ok, ?PUBCOMP_PACKET(1, ?RC_SUCCESS), _Channel1}
-        = emqx_channel:handle_in(?PUBREL_PACKET(1, ?RC_SUCCESS), Channel).
-    % ?assertEqual(#{pubrel_in => 1, pubcomp_out => 1},
-    %              emqx_channel:info(pub_stats, Channel1)).
+    {ok, ?PUBCOMP_PACKET(1, ?RC_SUCCESS), _Channel1} =
+        emqx_channel:handle_in(?PUBREL_PACKET(1, ?RC_SUCCESS), Channel).
 
 t_handle_in_pubrel_not_found_error(_) ->
     ok = meck:expect(emqx_session, pubrel,
                      fun(_PacketId, _Session) ->
                              {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND}
                      end),
-    {ok, ?PUBCOMP_PACKET(1, ?RC_PACKET_IDENTIFIER_NOT_FOUND), _Channel}
-        = emqx_channel:handle_in(?PUBREL_PACKET(1, ?RC_SUCCESS), channel()).
+    {ok, ?PUBCOMP_PACKET(1, ?RC_PACKET_IDENTIFIER_NOT_FOUND), _Channel} =
+        emqx_channel:handle_in(?PUBREL_PACKET(1, ?RC_SUCCESS), channel()).
 
 t_handle_in_pubcomp_ok(_) ->
     ok = meck:expect(emqx_session, pubcomp, fun(_, Session) -> {ok, Session} end),
@@ -233,7 +224,6 @@ t_handle_in_pubcomp_not_found_error(_) ->
                      end),
     Channel = channel(#{conn_state => connected}),
     {ok, _Channel1} = emqx_channel:handle_in(?PUBCOMP_PACKET(1, ?RC_SUCCESS), Channel).
-    % ?assertEqual(#{pubcomp_in => 1}, emqx_channel:info(pub_stats, Channel1)).
 
 t_handle_in_subscribe(_) ->
     ok = meck:expect(emqx_session, subscribe,
@@ -250,12 +240,12 @@ t_handle_in_unsubscribe(_) ->
                              {ok, Session}
                      end),
     Channel = channel(#{conn_state => connected}),
-    {ok, [{outgoing, ?UNSUBACK_PACKET(1)}, {event, updated}], _Chan}
-      = emqx_channel:handle_in(?UNSUBSCRIBE_PACKET(1, #{}, [<<"+">>]), Channel).
+    {ok, [{outgoing, ?UNSUBACK_PACKET(1)}, {event, updated}], _Chan} =
+        emqx_channel:handle_in(?UNSUBSCRIBE_PACKET(1, #{}, [<<"+">>]), Channel).
 
 t_handle_in_pingreq(_) ->
-    {ok, ?PACKET(?PINGRESP), _Channel}
-        = emqx_channel:handle_in(?PACKET(?PINGREQ), channel()).
+    {ok, ?PACKET(?PINGRESP), _Channel} =
+        emqx_channel:handle_in(?PACKET(?PINGREQ), channel()).
 
 t_handle_in_disconnect(_) ->
     Packet = ?DISCONNECT_PACKET(?RC_SUCCESS),
@@ -266,38 +256,37 @@ t_handle_in_disconnect(_) ->
 t_handle_in_auth(_) ->
     Channel = channel(#{conn_state => connected}),
     Packet = ?DISCONNECT_PACKET(?RC_IMPLEMENTATION_SPECIFIC_ERROR),
-    {ok, [{outgoing, Packet},
-          {close, implementation_specific_error}], Channel}
-    = emqx_channel:handle_in(?AUTH_PACKET(), Channel).
+    {ok, [{outgoing, Packet}, {close, implementation_specific_error}], Channel} =
+        emqx_channel:handle_in(?AUTH_PACKET(), Channel).
 
 t_handle_in_frame_error(_) ->
     IdleChannel = channel(#{conn_state => idle}),
-    {shutdown, frame_too_large, _}
-      = emqx_channel:handle_in({frame_error, frame_too_large}, IdleChannel),
+    {shutdown, frame_too_large, _Chan} =
+        emqx_channel:handle_in({frame_error, frame_too_large}, IdleChannel),
     ConnectingChan =  channel(#{conn_state => connecting}),
-    ConnackPacket = ?CONNACK_PACKET(?RC_MALFORMED_PACKET),
-    {shutdown, frame_too_large, ConnackPacket, _}
-      = emqx_channel:handle_in({frame_error, frame_too_large}, ConnectingChan),
-    DisconnectPacket = ?DISCONNECT_PACKET(?RC_MALFORMED_PACKET),
+    ConnackPacket = ?CONNACK_PACKET(?RC_PACKET_TOO_LARGE),
+    {shutdown, frame_too_large, ConnackPacket, _} =
+        emqx_channel:handle_in({frame_error, frame_too_large}, ConnectingChan),
+    DisconnectPacket = ?DISCONNECT_PACKET(?RC_PACKET_TOO_LARGE),
     ConnectedChan = channel(#{conn_state => connected}),
-    {ok, [{outgoing, DisconnectPacket}, {close, frame_too_large}], _}
-      = emqx_channel:handle_in({frame_error, frame_too_large}, ConnectedChan),
+    {ok, [{outgoing, DisconnectPacket}, {close, frame_too_large}], _} =
+        emqx_channel:handle_in({frame_error, frame_too_large}, ConnectedChan),
     DisconnectedChan = channel(#{conn_state => disconnected}),
-    {ok, DisconnectedChan}
-      = emqx_channel:handle_in({frame_error, frame_too_large}, DisconnectedChan).
+    {ok, DisconnectedChan} =
+        emqx_channel:handle_in({frame_error, frame_too_large}, DisconnectedChan).
 
 t_handle_in_expected_packet(_) ->
     Packet = ?DISCONNECT_PACKET(?RC_PROTOCOL_ERROR),
-    {ok, [{outgoing, Packet}, {close, protocol_error}], _Chan}
-      = emqx_channel:handle_in(packet, channel()).
+    {ok, [{outgoing, Packet}, {close, protocol_error}], _Chan} =
+        emqx_channel:handle_in(packet, channel()).
 
 t_process_connect(_) ->
     ok = meck:expect(emqx_cm, open_session,
                      fun(true, _ClientInfo, _ConnInfo) ->
                              {ok, #{session => session(), present => false}}
                      end),
-    {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS)}], _Chan}
-      = emqx_channel:process_connect(connpkt(), channel(#{conn_state => idle})).
+    {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS)}], _Chan} =
+        emqx_channel:process_connect(connpkt(), channel(#{conn_state => idle})).
 
 t_process_publish_qos0(_) ->
     ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
@@ -307,8 +296,8 @@ t_process_publish_qos0(_) ->
 t_process_publish_qos1(_) ->
     ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
     Publish = ?PUBLISH_PACKET(?QOS_1, <<"t">>, 1, <<"payload">>),
-    {ok, ?PUBACK_PACKET(1, ?RC_NO_MATCHING_SUBSCRIBERS), _Channel}
-      = emqx_channel:process_publish(Publish, channel()).
+    {ok, ?PUBACK_PACKET(1, ?RC_NO_MATCHING_SUBSCRIBERS), _Channel} =
+        emqx_channel:process_publish(Publish, channel()).
 
 t_process_subscribe(_) ->
     ok = meck:expect(emqx_session, subscribe, fun(_, _, _, Session) -> {ok, Session} end),
@@ -325,15 +314,6 @@ t_process_unsubscribe(_) ->
 %%--------------------------------------------------------------------
 
 t_handle_deliver(_) ->
-    WithPacketId = fun(Msgs) ->
-                           lists:zip(lists:seq(1, length(Msgs)), Msgs)
-                   end,
-    ok = meck:expect(emqx_session, deliver,
-                     fun(Delivers, Session) ->
-                             Publishes = WithPacketId([Msg || {deliver, _, Msg} <- Delivers]),
-                             {ok, Publishes, Session}
-                     end),
-    ok = meck:expect(emqx_session, info, fun(retry_interval, _Session) -> 20 end),
     Msg0 = emqx_message:make(test, ?QOS_1, <<"t1">>, <<"qos1">>),
     Msg1 = emqx_message:make(test, ?QOS_2, <<"t2">>, <<"qos2">>),
     Delivers = [{deliver, <<"+">>, Msg0}, {deliver, <<"+">>, Msg1}],
@@ -348,14 +328,14 @@ t_handle_out_publish(_) ->
     Channel = channel(#{conn_state => connected}),
     Pub0 = {undefined, emqx_message:make(<<"t">>, <<"qos0">>)},
     Pub1 = {1, emqx_message:make(<<"c">>, ?QOS_1, <<"t">>, <<"qos1">>)},
-    {ok, {outgoing, Packets}, _NChannel}
-      = emqx_channel:handle_out(publish, [Pub0, Pub1], Channel),
+    {ok, {outgoing, Packets}, _NChannel} =
+        emqx_channel:handle_out(publish, [Pub0, Pub1], Channel),
     ?assertEqual(2, length(Packets)).
 
 t_handle_out_publish_1(_) ->
     Msg = emqx_message:make(<<"clientid">>, ?QOS_1, <<"t">>, <<"payload">>),
-    {ok, ?PUBLISH_PACKET(?QOS_1, <<"t">>, 1, <<"payload">>), _Chan}
-      = emqx_channel:handle_out(publish, [{1, Msg}], channel()).
+    {ok, ?PUBLISH_PACKET(?QOS_1, <<"t">>, 1, <<"payload">>), _Chan} =
+        emqx_channel:handle_out(publish, [{1, Msg}], channel()).
 
 t_handle_out_publish_nl(_) ->
     ClientInfo = clientinfo(#{clientid => <<"clientid">>}),
@@ -365,50 +345,47 @@ t_handle_out_publish_nl(_) ->
     {ok, Channel} = emqx_channel:handle_out(publish, Pubs, Channel).
 
 t_handle_out_connack_sucess(_) ->
-    {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, _)}], Channel}
-      = emqx_channel:handle_out(connack, {?RC_SUCCESS, 0, connpkt()}, channel()),
+    {ok, [{event, connected}, {connack, ?CONNACK_PACKET(?RC_SUCCESS, 0, _)}], Channel} =
+        emqx_channel:handle_out(connack, {?RC_SUCCESS, 0, connpkt()}, channel()),
     ?assertEqual(connected, emqx_channel:info(conn_state, Channel)).
 
 t_handle_out_connack_failure(_) ->
-    {shutdown, not_authorized, ?CONNACK_PACKET(?RC_NOT_AUTHORIZED), _Chan}
-      = emqx_channel:handle_out(connack, {?RC_NOT_AUTHORIZED, connpkt()}, channel()).
+    {shutdown, not_authorized, ?CONNACK_PACKET(?RC_NOT_AUTHORIZED), _Chan} =
+        emqx_channel:handle_out(connack, {?RC_NOT_AUTHORIZED, connpkt()}, channel()).
 
 t_handle_out_puback(_) ->
     Channel = channel(#{conn_state => connected}),
-    {ok, ?PUBACK_PACKET(1, ?RC_SUCCESS), _NChannel}
-        = emqx_channel:handle_out(puback, {1, ?RC_SUCCESS}, Channel).
-    % ?assertEqual(#{puback_out => 1}, emqx_channel:info(pub_stats, NChannel)).
+    {ok, ?PUBACK_PACKET(1, ?RC_SUCCESS), _NChannel} =
+        emqx_channel:handle_out(puback, {1, ?RC_SUCCESS}, Channel).
 
 t_handle_out_pubrec(_) ->
     Channel = channel(#{conn_state => connected}),
-    {ok, ?PUBREC_PACKET(1, ?RC_SUCCESS), _NChannel}
-      = emqx_channel:handle_out(pubrec, {1, ?RC_SUCCESS}, Channel).
+    {ok, ?PUBREC_PACKET(1, ?RC_SUCCESS), _NChannel} =
+        emqx_channel:handle_out(pubrec, {1, ?RC_SUCCESS}, Channel).
 
 t_handle_out_pubrel(_) ->
     Channel = channel(#{conn_state => connected}),
-    {ok, ?PUBREL_PACKET(1), Channel1}
-      = emqx_channel:handle_out(pubrel, {1, ?RC_SUCCESS}, Channel),
-    {ok, ?PUBREL_PACKET(2, ?RC_SUCCESS), _Channel2}
-      = emqx_channel:handle_out(pubrel, {2, ?RC_SUCCESS}, Channel1).
+    {ok, ?PUBREL_PACKET(1), Channel1} =
+        emqx_channel:handle_out(pubrel, {1, ?RC_SUCCESS}, Channel),
+    {ok, ?PUBREL_PACKET(2, ?RC_SUCCESS), _Channel2} =
+        emqx_channel:handle_out(pubrel, {2, ?RC_SUCCESS}, Channel1).
 
 t_handle_out_pubcomp(_) ->
-    {ok, ?PUBCOMP_PACKET(1, ?RC_SUCCESS), _Channel}
-      = emqx_channel:handle_out(pubcomp, {1, ?RC_SUCCESS}, channel()).
+    {ok, ?PUBCOMP_PACKET(1, ?RC_SUCCESS), _Channel} =
+        emqx_channel:handle_out(pubcomp, {1, ?RC_SUCCESS}, channel()).
 
 t_handle_out_suback(_) ->
     Replies = [{outgoing, ?SUBACK_PACKET(1, [?QOS_2])}, {event, updated}],
-    {ok, Replies, _Channel}
-      = emqx_channel:handle_out(suback, {1, [?QOS_2]}, channel()).
+    {ok, Replies, _Chan} = emqx_channel:handle_out(suback, {1, [?QOS_2]}, channel()).
 
 t_handle_out_unsuback(_) ->
     Replies = [{outgoing, ?UNSUBACK_PACKET(1, [?RC_SUCCESS])}, {event, updated}],
-    {ok, Replies, _Channel}
-      = emqx_channel:handle_out(unsuback, {1, [?RC_SUCCESS]}, channel()).
+    {ok, Replies, _Chan} = emqx_channel:handle_out(unsuback, {1, [?RC_SUCCESS]}, channel()).
 
 t_handle_out_disconnect(_) ->
     Packet = ?DISCONNECT_PACKET(?RC_SUCCESS),
-    {ok, [{outgoing, Packet}, {close, normal}], _Chan}
-      = emqx_channel:handle_out(disconnect, ?RC_SUCCESS, channel()).
+    {ok, [{outgoing, Packet}, {close, normal}], _Chan} =
+        emqx_channel:handle_out(disconnect, ?RC_SUCCESS, channel()).
 
 t_handle_out_unexpected(_) ->
     {ok, _Channel} = emqx_channel:handle_out(unexpected, <<"data">>, channel()).
@@ -422,20 +399,19 @@ t_handle_call_kick(_) ->
 
 t_handle_call_discard(_) ->
     Packet = ?DISCONNECT_PACKET(?RC_SESSION_TAKEN_OVER),
-    {shutdown, discarded, ok, Packet, _Channel}
-        = emqx_channel:handle_call(discard, channel()).
+    {shutdown, discarded, ok, Packet, _Channel} =
+        emqx_channel:handle_call(discard, channel()).
 
 t_handle_call_takeover_begin(_) ->
-    {reply, undefined, _Channel}
-        = emqx_channel:handle_call({takeover, 'begin'}, channel()).
+    {reply, _Session, _Chan} = emqx_channel:handle_call({takeover, 'begin'}, channel()).
 
 t_handle_call_takeover_end(_) ->
     ok = meck:expect(emqx_session, takeover, fun(_) -> ok end),
-    {shutdown, takeovered, [], _Channel}
-        = emqx_channel:handle_call({takeover, 'end'}, channel()).
+    {shutdown, takeovered, [], _Chan} =
+        emqx_channel:handle_call({takeover, 'end'}, channel()).
 
 t_handle_call_unexpected(_) ->
-    {reply, ignored, _Channel} = emqx_channel:handle_call(unexpected_req, channel()).
+    {reply, ignored, _Chan} = emqx_channel:handle_call(unexpected_req, channel()).
 
 %%--------------------------------------------------------------------
 %% Test cases for handle_info
@@ -507,8 +483,8 @@ t_auth_connect(_) ->
 t_process_alias(_) ->
     Publish = #mqtt_packet_publish{topic_name = <<>>, properties = #{'Topic-Alias' => 1}},
     Channel = emqx_channel:set_field(topic_aliases, #{1 => <<"t">>}, channel()),
-    {ok, #mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"t">>}}, _Chan}
-        = emqx_channel:process_alias(#mqtt_packet{variable = Publish}, Channel).
+    {ok, #mqtt_packet{variable = #mqtt_packet_publish{topic_name = <<"t">>}}, _Chan} =
+        emqx_channel:process_alias(#mqtt_packet{variable = Publish}, Channel).
 
 t_check_pub_acl(_) ->
     ok = meck:new(emqx_zone, [passthrough, no_history]),
@@ -566,14 +542,27 @@ t_terminate(_) ->
 
 channel() -> channel(#{}).
 channel(InitFields) ->
+    ConnInfo = #{peername => {{127,0,0,1}, 3456},
+                 sockname => {{127,0,0,1}, 1883},
+                 conn_mod => emqx_connection,
+                 proto_name => <<"MQTT">>,
+                 proto_ver => ?MQTT_PROTO_V5,
+                 clean_start => true,
+                 keepalive => 30,
+                 clientid => <<"clientid">>,
+                 username => <<"username">>,
+                 conn_props => #{},
+                 receive_maximum => 100,
+                 expiry_interval => 0
+                },
     maps:fold(fun(Field, Value, Channel) ->
                       emqx_channel:set_field(Field, Value, Channel)
-              end, default_channel(), InitFields).
-
-default_channel() ->
-    Channel = emqx_channel:init(?DEFAULT_CONNINFO, [{zone, zone}]),
-    Channel1 = emqx_channel:set_field(conn_state, connected, Channel),
-    emqx_channel:set_field(clientinfo, clientinfo(), Channel1).
+              end,
+              emqx_channel:init(ConnInfo, [{zone, zone}]),
+              maps:merge(#{clientinfo => clientinfo(),
+                           session    => session(),
+                           conn_state => connected
+                          }, InitFields)).
 
 clientinfo() -> clientinfo(#{}).
 clientinfo(InitProps) ->
@@ -608,6 +597,6 @@ session(InitFields) when is_map(InitFields) ->
     maps:fold(fun(Field, Value, Session) ->
                       emqx_session:set_field(Field, Value, Session)
               end,
-              emqx_session:init(#{zone => zone}, #{receive_maximum => 0}),
+              emqx_session:init(#{zone => channel}, #{receive_maximum => 0}),
               InitFields).
 

test/emqx_cm_SUITE.erl

@@ -22,6 +22,8 @@
 -include("emqx.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
+-define(CM, emqx_cm).
+
 %%--------------------------------------------------------------------
 %% CT callbacks
 %%--------------------------------------------------------------------
@@ -81,10 +83,55 @@ t_open_session(_) ->
     ?assertEqual(100, emqx_session:info(inflight_max, Session2)).
 
 t_discard_session(_) ->
-    ok = emqx_cm:discard_session(<<"clientid">>).
+    ok = meck:new(emqx_connection, [passthrough, no_history]),
+    ok = meck:expect(emqx_connection, call, fun(_, _) -> ok end),
+    ok = emqx_cm:discard_session(<<"clientid">>),
+    ok = emqx_cm:register_channel(<<"clientid">>),
+    ok = emqx_cm:discard_session(<<"clientid">>),
+    ok = emqx_cm:unregister_channel(<<"clientid">>),
+    ok = emqx_cm:register_channel(<<"clientid">>, #{conninfo => #{conn_mod => emqx_connection}}, []),
+    ok = emqx_cm:discard_session(<<"clientid">>),
+    ok = meck:expect(emqx_connection, call, fun(_, _) -> error(testing) end),
+    ok = emqx_cm:discard_session(<<"clientid">>),
+    ok = emqx_cm:unregister_channel(<<"clientid">>),
+    ok = meck:unload(emqx_connection).
 
 t_takeover_session(_) ->
-    {error, not_found} = emqx_cm:takeover_session(<<"clientid">>).
+    ok = meck:new(emqx_connection, [passthrough, no_history]),
+    ok = meck:expect(emqx_connection, call, fun(_, _) -> test end),
+    {error, not_found} = emqx_cm:takeover_session(<<"clientid">>),
+    ok = emqx_cm:register_channel(<<"clientid">>),
+    {error, not_found} = emqx_cm:takeover_session(<<"clientid">>),
+    ok = emqx_cm:unregister_channel(<<"clientid">>),
+    ok = emqx_cm:register_channel(<<"clientid">>, #{conninfo => #{conn_mod => emqx_connection}}, []),
+    Pid = self(),
+    {ok, emqx_connection, Pid, test} = emqx_cm:takeover_session(<<"clientid">>),
+    erlang:spawn(fun() ->
+                     ok = emqx_cm:register_channel(<<"clientid">>, #{conninfo => #{conn_mod => emqx_connection}}, []),
+                     timer:sleep(1000)
+                 end),
+    ct:sleep(100),
+    {ok, emqx_connection, _, test} = emqx_cm:takeover_session(<<"clientid">>),
+    ok = emqx_cm:unregister_channel(<<"clientid">>),
+    ok = meck:unload(emqx_connection).
+
+t_kick_session(_) ->
+    ok = meck:new(emqx_connection, [passthrough, no_history]),
+    ok = meck:expect(emqx_connection, call, fun(_, _) -> test end),
+    {error, not_found} = emqx_cm:kick_session(<<"clientid">>),
+    ok = emqx_cm:register_channel(<<"clientid">>),
+    {error, not_found} = emqx_cm:kick_session(<<"clientid">>),
+    ok = emqx_cm:unregister_channel(<<"clientid">>),
+    ok = emqx_cm:register_channel(<<"clientid">>, #{conninfo => #{conn_mod => emqx_connection}}, []),
+    test = emqx_cm:kick_session(<<"clientid">>),
+    erlang:spawn(fun() ->
+                     ok = emqx_cm:register_channel(<<"clientid">>, #{conninfo => #{conn_mod => emqx_connection}}, []),
+                     timer:sleep(1000)
+                 end),
+    ct:sleep(100),
+    test = emqx_cm:kick_session(<<"clientid">>),
+    ok = emqx_cm:unregister_channel(<<"clientid">>),
+    ok = meck:unload(emqx_connection).
 
 t_lock_clientid(_) ->
     {true, _Nodes} = emqx_cm_locker:lock(<<"clientid">>),
@@ -92,27 +139,7 @@ t_lock_clientid(_) ->
     {true, _Nodes} = emqx_cm_locker:unlock(<<"clientid">>),
     {true, _Nodes} = emqx_cm_locker:unlock(<<"clientid">>).
 
-
-% t_unregister_channel(_) ->
-%     error('TODO').
-
-% t_get_chan_attrs(_) ->
-%     error('TODO').
-
-% t_get_chan_stats(_) ->
-%     error('TODO').
-
-% t_lookup_channels(_) ->
-%     error('TODO').
-
-% t_set_chan_stats(_) ->
-%     error('TODO').
-
-% t_set_chan_attrs(_) ->
-%     error('TODO').
-
-% t_register_channel(_) ->
-%     error('TODO').
-
-% t_stats_fun(_) ->
-%     error('TODO').
+t_message(_) ->
+    ?CM ! testing,
+    gen_server:cast(?CM, testing),
+    gen_server:call(?CM, testing).

test/emqx_connection_SUITE.erl

@@ -22,43 +22,34 @@
 -include("emqx_mqtt.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
--define(STATS_KYES, [recv_pkt, recv_msg, send_pkt, send_msg,
-                     recv_oct, recv_cnt, send_oct, send_cnt,
-                     send_pend
-                    ]).
-
-all() -> emqx_ct:all(?MODULE) ++ [{group, real_client}].
-
-groups() ->
-    [{real_client, [non_parallel_tests],
-        [
-            g_get_conn_stats,
-            g_handle_sock_passive
-        ]}].
+all() -> emqx_ct:all(?MODULE).
 
 %%--------------------------------------------------------------------
 %% CT callbacks
 %%--------------------------------------------------------------------
 
 init_per_suite(Config) ->
+    %% Meck Transport
+    ok = meck:new(emqx_transport, [non_strict, passthrough, no_history, no_link]),
+    %% Meck Channel
+    ok = meck:new(emqx_channel, [passthrough, no_history, no_link]),
+    %% Meck Cm
+    ok = meck:new(emqx_cm, [passthrough, no_history, no_link]),
+    %% Meck Metrics
+    ok = meck:new(emqx_metrics, [passthrough, no_history, no_link]),
+    ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end),
+    ok = meck:expect(emqx_metrics, inc_recv, fun(_) -> ok end),
+    ok = meck:expect(emqx_metrics, inc_sent, fun(_) -> ok end),
     Config.
 
 end_per_suite(_Config) ->
+    ok = meck:unload(emqx_transport),
+    ok = meck:unload(emqx_channel),
+    ok = meck:unload(emqx_cm),
+    ok = meck:unload(emqx_metrics),
     ok.
 
-init_per_group(real_client, Config) ->
-    emqx_ct_helpers:boot_modules(all),
-    emqx_ct_helpers:start_apps([]),
-    Config;
-init_per_group(_, Config) -> Config.
-
-end_per_group(real_client, _Config) ->
-    emqx_ct_helpers:stop_apps([]);
-end_per_group(_, Config) -> Config.
-
 init_per_testcase(_TestCase, Config) ->
-    %% Meck Transport
-    ok = meck:new(emqx_transport, [non_strict, passthrough, no_history]),
     ok = meck:expect(emqx_transport, wait, fun(Sock) -> {ok, Sock} end),
     ok = meck:expect(emqx_transport, type, fun(_Sock) -> tcp end),
     ok = meck:expect(emqx_transport, ensure_ok_or_exit,
@@ -72,19 +63,9 @@ init_per_testcase(_TestCase, Config) ->
                                               end),
     ok = meck:expect(emqx_transport, async_send, fun(_Sock, _Data) -> ok end),
     ok = meck:expect(emqx_transport, fast_close, fun(_Sock) -> ok end),
-    %% Meck Channel
-    ok = meck:new(emqx_channel, [passthrough, no_history]),
-    %% Meck Metrics
-    ok = meck:new(emqx_metrics, [passthrough, no_history]),
-    ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end),
-    ok = meck:expect(emqx_metrics, inc_recv, fun(_) -> ok end),
-    ok = meck:expect(emqx_metrics, inc_sent, fun(_) -> ok end),
     Config.
 
 end_per_testcase(_TestCase, Config) ->
-    ok = meck:unload(emqx_transport),
-    ok = meck:unload(emqx_channel),
-    ok = meck:unload(emqx_metrics),
     Config.
 
 %%--------------------------------------------------------------------
@@ -92,9 +73,7 @@ end_per_testcase(_TestCase, Config) ->
 %%--------------------------------------------------------------------
 
 t_start_link_ok(_) ->
-    with_connection(fun(CPid) ->
-                            state = element(1, sys:get_state(CPid))
-                    end).
+    with_conn(fun(CPid) -> state = element(1, sys:get_state(CPid)) end).
 
 t_start_link_exit_on_wait(_) ->
     ok = exit_on_wait_error(enotconn, normal),
@@ -110,108 +89,114 @@ t_start_link_exit_on_activate(_) ->
     ok = exit_on_activate_error(econnreset, {shutdown, econnreset}).
 
 t_get_conn_info(_) ->
-    with_connection(fun(CPid) ->
-                            #{sockinfo := SockInfo} = emqx_connection:info(CPid),
-                            ?assertEqual(#{active_n => 100,
-                                           peername => {{127,0,0,1},3456},
-                                           sockname => {{127,0,0,1},1883},
-                                           sockstate => running,
-                                           socktype => tcp}, SockInfo)
-                    end).
-
-g_get_conn_stats(_) ->
-    with_client(fun(CPid) ->
-                            Stats = emqx_connection:stats(CPid),
-                            ct:pal("==== stats: ~p", [Stats]),
-                            [?assert(proplists:get_value(Key, Stats) >= 0) || Key <- ?STATS_KYES]
-                    end, []).
+    with_conn(fun(CPid) ->
+                      #{sockinfo := SockInfo} = emqx_connection:info(CPid),
+                      ?assertEqual(#{active_n => 100,
+                                     peername => {{127,0,0,1},3456},
+                                     sockname => {{127,0,0,1},1883},
+                                     sockstate => running,
+                                     socktype => tcp
+                                    }, SockInfo)
+              end).
 
 t_handle_call_discard(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_call,
-                                             fun(discard, Channel) ->
-                                                     {shutdown, discarded, ok, Channel}
-                                             end),
-                            ok = emqx_connection:call(CPid, discard),
-                            timer:sleep(100),
-                            ok = trap_exit(CPid, {shutdown, discarded})
-                    end, #{trap_exit => true}).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_call,
+                                       fun(discard, Channel) ->
+                                               {shutdown, discarded, ok, Channel}
+                                       end),
+                      ok = emqx_connection:call(CPid, discard),
+                      timer:sleep(100),
+                      ok = trap_exit(CPid, {shutdown, discarded})
+              end, #{trap_exit => true}),
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_call,
+                                       fun(discard, Channel) ->
+                                               {shutdown, discarded, ok, ?DISCONNECT_PACKET(?RC_SESSION_TAKEN_OVER), Channel}
+                                       end),
+                      ok = emqx_connection:call(CPid, discard),
+                      timer:sleep(100),
+                      ok = trap_exit(CPid, {shutdown, discarded})
+              end, #{trap_exit => true}).
 
 t_handle_call_takeover(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_call,
-                                              fun({takeover, 'begin'}, Channel) ->
-                                                      {reply, session, Channel};
-                                                 ({takeover, 'end'}, Channel) ->
-                                                      {shutdown, takeovered, [], Channel}
-                                              end),
-                            session = emqx_connection:call(CPid, {takeover, 'begin'}),
-                            [] = emqx_connection:call(CPid, {takeover, 'end'}),
-                            timer:sleep(100),
-                            ok = trap_exit(CPid, {shutdown, takeovered})
-                    end, #{trap_exit => true}).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_call,
+                                       fun({takeover, 'begin'}, Channel) ->
+                                               {reply, session, Channel};
+                                          ({takeover, 'end'}, Channel) ->
+                                               {shutdown, takeovered, [], Channel}
+                                       end),
+                      session = emqx_connection:call(CPid, {takeover, 'begin'}),
+                      [] = emqx_connection:call(CPid, {takeover, 'end'}),
+                      timer:sleep(100),
+                      ok = trap_exit(CPid, {shutdown, takeovered})
+              end, #{trap_exit => true}).
 
 t_handle_call_any(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_call,
-                                             fun(_Req, Channel) -> {reply, ok, Channel} end),
-                            ok = emqx_connection:call(CPid, req)
-                    end).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_call,
+                                       fun(_Req, Channel) -> {reply, ok, Channel} end),
+                      ok = emqx_connection:call(CPid, req)
+              end).
 
 t_handle_incoming_connect(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
-                            ConnPkt = #mqtt_packet_connect{proto_ver   = ?MQTT_PROTO_V5,
-                                                           proto_name  = <<"MQTT">>,
-                                                           clientid    = <<>>,
-                                                           clean_start = true,
-                                                           keepalive   = 60
-                                                          },
-                            Frame = make_frame(?CONNECT_PACKET(ConnPkt)),
-                            CPid ! {tcp, sock, Frame}
-                    end).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
+                      ConnPkt = #mqtt_packet_connect{proto_ver   = ?MQTT_PROTO_V5,
+                                                     proto_name  = <<"MQTT">>,
+                                                     clientid    = <<>>,
+                                                     clean_start = true,
+                                                     keepalive   = 60
+                                                    },
+                      Frame = make_frame(?CONNECT_PACKET(ConnPkt)),
+                      CPid ! {tcp, sock, Frame}
+              end).
 
 t_handle_incoming_publish(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
-                            Frame = make_frame(?PUBLISH_PACKET(?QOS_1, <<"t">>, 1, <<"payload">>)),
-                            CPid ! {tcp, sock, Frame}
-                    end).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
+                      Frame = make_frame(?PUBLISH_PACKET(?QOS_1, <<"t">>, 1, <<"payload">>)),
+                      CPid ! {tcp, sock, Frame}
+              end).
 
 t_handle_incoming_subscribe(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
-                            Frame = <<?SUBSCRIBE:4,2:4,11,0,2,0,6,84,111,112,105,99,65,2>>,
-                            CPid ! {tcp, sock, Frame}
-                    end).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
+                      Frame = <<?SUBSCRIBE:4,2:4,11,0,2,0,6,84,111,112,105,99,65,2>>,
+                      CPid ! {tcp, sock, Frame}
+              end).
 
 t_handle_incoming_unsubscribe(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
-                            Frame = <<?UNSUBSCRIBE:4,2:4,10,0,2,0,6,84,111,112,105,99,65>>,
-                            CPid ! {tcp, sock, Frame}
-                    end).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
+                      Frame = <<?UNSUBSCRIBE:4,2:4,10,0,2,0,6,84,111,112,105,99,65>>,
+                      CPid ! {tcp, sock, Frame}
+              end).
+
+t_handle_incoming_undefined(_) ->
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
+                      CPid ! {incoming, undefined}
+              end).
 
 t_handle_sock_error(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_info,
-                                             fun({_, Reason}, Channel) ->
-                                                     {shutdown, Reason, Channel}
-                                             end),
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_info,
+                                       fun({_, Reason}, Channel) ->
+                                               {shutdown, Reason, Channel}
+                                       end),
                             %% TODO: fixme later
                             CPid ! {tcp_error, sock, econnreset},
                             timer:sleep(100),
                             trap_exit(CPid, {shutdown, econnreset})
-                    end, #{trap_exit => true}).
-
-g_handle_sock_passive(_) ->
-    with_client(fun(CPid) -> CPid ! {tcp_passive, sock} end, []).
+              end, #{trap_exit => true}).
 
 t_handle_sock_activate(_) ->
-    with_connection(fun(CPid) -> CPid ! activate_socket end).
+    with_conn(fun(CPid) -> CPid ! activate_socket end).
 
 t_handle_sock_closed(_) ->
-    with_connection(fun(CPid) ->
+    with_conn(fun(CPid) ->
                             ok = meck:expect(emqx_channel, handle_info,
                                              fun({sock_closed, Reason}, Channel) ->
                                                      {shutdown, Reason, Channel}
@@ -219,75 +204,156 @@ t_handle_sock_closed(_) ->
                             CPid ! {tcp_closed, sock},
                             timer:sleep(100),
                             trap_exit(CPid, {shutdown, tcp_closed})
-                    end, #{trap_exit => true}).
+                    end, #{trap_exit => true}),
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_info,
+                                       fun({sock_closed, Reason}, Channel) ->
+                                               {shutdown, Reason, ?DISCONNECT_PACKET(), Channel}
+                                       end),
+                      CPid ! {tcp_closed, sock},
+                      timer:sleep(100),
+                      trap_exit(CPid, {shutdown, tcp_closed})
+              end, #{trap_exit => true}).
 
 t_handle_outgoing(_) ->
-    with_connection(fun(CPid) ->
-                            Publish = ?PUBLISH_PACKET(?QOS_2, <<"Topic">>, 1, <<>>),
-                            CPid ! {outgoing, Publish},
-                            CPid ! {outgoing, ?PUBREL_PACKET(1)},
-                            CPid ! {outgoing, [?PUBCOMP_PACKET(1)]}
-                    end).
+    with_conn(fun(CPid) ->
+                      Publish = ?PUBLISH_PACKET(?QOS_2, <<"Topic">>, 1, <<>>),
+                      CPid ! {outgoing, Publish},
+                      CPid ! {outgoing, ?PUBREL_PACKET(1)},
+                      CPid ! {outgoing, [?PUBCOMP_PACKET(1)]}
+              end).
 
 t_conn_rate_limit(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_in, fun(_, Channel) -> {ok, Channel} end),
-                            lists:foreach(fun(I) ->
-                                                  Publish = ?PUBLISH_PACKET(?QOS_0, <<"Topic">>, I, payload(2000)),
-                                                  CPid ! {tcp, sock, make_frame(Publish)}
-                                          end, [1, 2])
-                            %%#{sockinfo := #{sockstate := blocked}} = emqx_connection:info(CPid)
-                    end, #{active_n => 1, rate_limit => {1, 1024}}).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_in, fun(_, Channel) -> {ok, Channel} end),
+                      lists:foreach(fun(I) ->
+                                            Publish = ?PUBLISH_PACKET(?QOS_0, <<"Topic">>, I, payload(2000)),
+                                            CPid ! {tcp, sock, make_frame(Publish)}
+                                    end, [1, 2])
+              end, #{active_n => 1, rate_limit => {1, 1024}}).
 
 t_conn_pub_limit(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_in, fun(_, Channel) -> {ok, Channel} end),
-                            ok = lists:foreach(fun(I) ->
-                                                       CPid ! {incoming, ?PUBLISH_PACKET(?QOS_0, <<"Topic">>, I, <<>>)}
-                                               end, lists:seq(1, 3))
-                            %%#{sockinfo := #{sockstate := blocked}} = emqx_connection:info(CPid)
-                    end, #{active_n => 1, publish_limit => {1, 2}}).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_in, fun(_, Channel) -> {ok, Channel} end),
+                      ok = lists:foreach(fun(I) ->
+                                                 CPid ! {incoming, ?PUBLISH_PACKET(?QOS_0, <<"Topic">>, I, <<>>)}
+                                         end, lists:seq(1, 3))
+                      %%#{sockinfo := #{sockstate := blocked}} = emqx_connection:info(CPid)
+              end, #{active_n => 1, publish_limit => {1, 2}}).
+
+t_conn_pingreq(_) ->
+    with_conn(fun(CPid) -> CPid ! {incoming, ?PACKET(?PINGREQ)} end).
+
+t_inet_reply(_) ->
+    ok = meck:new(emqx_pd, [passthrough, no_history]),
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_pd, get_counter, fun(_) -> 10 end),
+                      CPid ! {inet_reply, for_testing, ok},
+                      timer:sleep(100)
+              end, #{active_n => 1, trap_exit => true}),
+    ok = meck:unload(emqx_pd),
+    with_conn(fun(CPid) ->
+                      CPid ! {inet_reply, for_testing, {error, for_testing}},
+                      timer:sleep(100),
+                      trap_exit(CPid, {shutdown, for_testing})
+              end, #{trap_exit => true}).
+
+t_deliver(_) ->
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_deliver,
+                                       fun(_, Channel) -> {ok, Channel} end),
+                      CPid ! {deliver, topic, msg}
+              end).
+
+t_event_disconnected(_) ->
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_cm, set_chan_info, fun(_, _) -> ok end),
+                      ok = meck:expect(emqx_cm, connection_closed, fun(_) -> ok end),
+                      CPid ! {event, disconnected}
+              end).
+
+t_event_undefined(_) ->
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, stats, fun(_Channel) -> [] end),
+                      ok = meck:expect(emqx_cm, set_chan_info, fun(_, _) -> ok end),
+                      ok = meck:expect(emqx_cm, set_chan_stats, fun(_, _) -> true end),
+                      CPid ! {event, undefined}
+              end).
+
+t_cloes(_) ->
+    with_conn(fun(CPid) ->
+                      CPid ! {close, normal},
+                      timer:sleep(100),
+                      trap_exit(CPid, {shutdown, normal})
+              end, #{trap_exit => true}).
 
 t_oom_shutdown(_) ->
-    with_connection(fun(CPid) ->
-                            CPid ! {shutdown, message_queue_too_long},
-                            timer:sleep(100),
-                            trap_exit(CPid, {shutdown, message_queue_too_long})
-                    end, #{trap_exit => true}).
+    with_conn(fun(CPid) ->
+                      CPid ! {shutdown, message_queue_too_long},
+                      timer:sleep(100),
+                      trap_exit(CPid, {shutdown, message_queue_too_long})
+              end, #{trap_exit => true}).
 
 t_handle_idle_timeout(_) ->
     ok = emqx_zone:set_env(external, idle_timeout, 10),
-    with_connection(fun(CPid) ->
-                            timer:sleep(100),
-                            trap_exit(CPid, {shutdown, idle_timeout})
-                    end, #{zone => external, trap_exit => true}).
+    with_conn(fun(CPid) ->
+                      timer:sleep(100),
+                      trap_exit(CPid, {shutdown, idle_timeout})
+              end, #{zone => external, trap_exit => true}).
 
 t_handle_emit_stats(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_timeout,
-                                             fun(_TRef, _TMsg, Channel) ->
-                                                     {ok, Channel}
-                                             end),
-                            CPid ! {timeout, make_ref(), emit_stats}
-                    end).
+    ok = emqx_zone:set_env(external, idle_timeout, 1000),
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, stats, fun(_Channel) -> [] end),
+                      ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
+                      ok = meck:expect(emqx_cm, set_chan_stats, fun(_, _) -> true end),
+                      CPid ! {incoming, ?CONNECT_PACKET(#{strict_mode => false,
+                                                          max_size    => ?MAX_PACKET_SIZE,
+                                                          version     => ?MQTT_PROTO_V4
+                                                         })},
+                      timer:sleep(1000)
+              end,#{zone => external, trap_exit => true}).
+
+t_handle_limit_timeout(_) ->
+    with_conn(fun(CPid) ->
+                      CPid ! {timeout, undefined, limit_timeout},
+                      timer:sleep(100),
+                      true = erlang:is_process_alive(CPid)
+              end).
 
 t_handle_keepalive_timeout(_) ->
-    with_connection(fun(CPid) ->
-                            ok = meck:expect(emqx_channel, handle_timeout,
-                                             fun(_TRef, _TMsg, Channel) ->
-                                                     {shutdown, keepalive_timeout, Channel}
-                                             end),
-                            CPid ! {timeout, make_ref(), keepalive},
-                            timer:sleep(100),
-                            trap_exit(CPid, {shutdown, keepalive_timeout})
-                    end, #{trap_exit => true}).
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_channel, handle_timeout,
+                                       fun(_TRef, _TMsg, Channel) ->
+                                               {shutdown, keepalive_timeout, Channel}
+                                       end),
+                      CPid ! {timeout, make_ref(), keepalive},
+                      timer:sleep(100),
+                      trap_exit(CPid, {shutdown, keepalive_timeout})
+              end, #{trap_exit => true}),
+    with_conn(fun(CPid) ->
+                      ok = meck:expect(emqx_transport, getstat, fun(_Sock, _Options) -> {error, for_testing} end),
+                      ok = meck:expect(emqx_channel, handle_timeout,
+                                       fun(_TRef, _TMsg, Channel) ->
+                                               {shutdown, keepalive_timeout, Channel}
+                                       end),
+                      CPid ! {timeout, make_ref(), keepalive},
+                      timer:sleep(100),
+                      false = erlang:is_process_alive(CPid)
+              end, #{trap_exit => true}).
 
 t_handle_shutdown(_) ->
-    with_connection(fun(CPid) ->
-                            CPid ! Shutdown = {shutdown, reason},
-                            timer:sleep(100),
-                            trap_exit(CPid, Shutdown)
-                    end, #{trap_exit => true}).
+    with_conn(fun(CPid) ->
+                      CPid ! Shutdown = {shutdown, reason},
+                      timer:sleep(100),
+                      trap_exit(CPid, Shutdown)
+              end, #{trap_exit => true}).
+
+t_exit_message(_) ->
+    with_conn(fun(CPid) ->
+                      CPid ! {'EXIT', CPid, for_testing},
+                      timer:sleep(1000)
+              end, #{trap_exit => true}).
 
 %%--------------------------------------------------------------------
 %% Helper functions
@@ -298,27 +364,28 @@ exit_on_wait_error(SockErr, Reason) ->
                      fun(_Sock) ->
                              {error, SockErr}
                      end),
-    with_connection(fun(CPid) ->
-                            timer:sleep(100),
-                            trap_exit(CPid, Reason)
-                    end, #{trap_exit => true}).
+    with_conn(fun(CPid) ->
+                      timer:sleep(100),
+                      trap_exit(CPid, Reason)
+              end, #{trap_exit => true}).
 
 exit_on_activate_error(SockErr, Reason) ->
     ok = meck:expect(emqx_transport, setopts,
                      fun(_Sock, _Opts) ->
                              {error, SockErr}
                      end),
-    with_connection(fun(CPid) ->
-                            timer:sleep(100),
-                            trap_exit(CPid, Reason)
-                    end, #{trap_exit => true}).
+    with_conn(fun(CPid) ->
+                      timer:sleep(100),
+                      trap_exit(CPid, Reason)
+              end, #{trap_exit => true}).
+
+with_conn(TestFun) ->
+    with_conn(TestFun, #{trap_exit => false}).
 
-with_connection(TestFun) ->
-    with_connection(TestFun, #{trap_exit => false}).
+with_conn(TestFun, Options) when is_map(Options) ->
+    with_conn(TestFun, maps:to_list(Options));
 
-with_connection(TestFun, Options) when is_map(Options) ->
-    with_connection(TestFun, maps:to_list(Options));
-with_connection(TestFun, Options) ->
+with_conn(TestFun, Options) ->
     TrapExit = proplists:get_value(trap_exit, Options, false),
     process_flag(trap_exit, TrapExit),
     {ok, CPid} = emqx_connection:start_link(emqx_transport, sock, Options),
@@ -326,18 +393,6 @@ with_connection(TestFun, Options) ->
     TrapExit orelse emqx_connection:stop(CPid),
     ok.
 
-with_client(TestFun, _Options) ->
-    ClientId = <<"t_conn">>,
-    {ok, C} = emqtt:start_link([{clientid, ClientId}]),
-    {ok, _} = emqtt:connect(C),
-    timer:sleep(50),
-    case emqx_cm:lookup_channels(ClientId) of
-        [] -> ct:fail({client_not_started, ClientId});
-        [ChanPid] ->
-            TestFun(ChanPid),
-            emqtt:stop(C)
-    end.
-
 trap_exit(Pid, Reason) ->
     receive
         {'EXIT', Pid, Reason} -> ok;

test/emqx_flapping_SUITE.erl

@@ -53,5 +53,10 @@ t_detect_check(_) ->
     true = emqx_banned:check(ClientInfo),
     timer:sleep(200),
     false = emqx_banned:check(ClientInfo),
+    Childrens = supervisor:which_children(emqx_cm_sup),
+    {flapping, Pid, _, _} = lists:keyfind(flapping, 1, Childrens),
+    gen_server:call(Pid, test),
+    gen_server:cast(Pid, test),
+    Pid ! test,
     ok = emqx_flapping:stop().
 

test/emqx_hooks_SUITE.erl

@@ -99,6 +99,14 @@ t_run_hooks(_) ->
 
     ok = emqx_hooks:stop().
 
+t_uncovered_func(_) ->
+    {ok, _} = emqx_hooks:start_link(),
+    Pid = erlang:whereis(emqx_hooks),
+    gen_server:call(Pid, test),
+    gen_server:cast(Pid, test),
+    Pid ! test,
+    ok = emqx_hooks:stop().
+
 %%--------------------------------------------------------------------
 %% Hook fun
 %%--------------------------------------------------------------------

test/emqx_limiter_SUITE.erl

@@ -0,0 +1,58 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2019 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_limiter_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("eunit/include/eunit.hrl").
+
+all() -> emqx_ct:all(?MODULE).
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, _Config) ->
+    ok.
+
+t_info(_) ->
+    #{pub_limit := #{rate   := 1,
+                     burst  := 10,
+                     tokens := 10
+                    },
+      rate_limit := #{rate   := 100,
+                      burst  := 1000,
+                      tokens := 1000
+                     }
+     } = emqx_limiter:info(limiter()).
+
+t_check(_) ->
+    lists:foreach(fun(I) ->
+                          {ok, Limiter} = emqx_limiter:check(#{cnt => I, oct => I*100}, limiter()),
+                          #{pub_limit  := #{tokens := Cnt},
+                            rate_limit := #{tokens := Oct}
+                           } = emqx_limiter:info(Limiter),
+                          ?assertEqual({10 - I, 1000 - I*100}, {Cnt, Oct})
+                  end, lists:seq(1, 10)).
+
+t_check_pause(_) ->
+    {pause, 1000, _} = emqx_limiter:check(#{cnt => 11, oct => 2000}, limiter()),
+    {pause, 2000, _} = emqx_limiter:check(#{cnt => 10, oct => 1200}, limiter()).
+
+limiter() ->
+    emqx_limiter:init([{pub_limit, {1, 10}}, {rate_limit, {100, 1000}}]).
+

test/emqx_message_SUITE.erl

@@ -43,17 +43,56 @@ t_make(_) ->
     ?assertEqual(<<"topic">>, emqx_message:topic(Msg2)),
     ?assertEqual(<<"payload">>, emqx_message:payload(Msg2)).
 
+t_id(_) ->
+    Msg = emqx_message:make(<<"topic">>, <<"payload">>),
+    ?assert(is_binary(emqx_message:id(Msg))).
+
+t_qos(_) ->
+    Msg = emqx_message:make(<<"topic">>, <<"payload">>),
+    ?assertEqual(?QOS_0, emqx_message:qos(Msg)),
+    Msg1 = emqx_message:make(id, ?QOS_1, <<"t">>, <<"payload">>),
+    ?assertEqual(?QOS_1, emqx_message:qos(Msg1)),
+    Msg2 = emqx_message:make(id, ?QOS_2, <<"t">>, <<"payload">>),
+    ?assertEqual(?QOS_2, emqx_message:qos(Msg2)).
+
+t_topic(_) ->
+    Msg = emqx_message:make(<<"t">>, <<"payload">>),
+    ?assertEqual(<<"t">>, emqx_message:topic(Msg)).
+
+t_payload(_) ->
+    Msg = emqx_message:make(<<"t">>, <<"payload">>),
+    ?assertEqual(<<"payload">>, emqx_message:payload(Msg)).
+
+t_timestamp(_) ->
+    Msg = emqx_message:make(<<"t">>, <<"payload">>),
+    timer:sleep(1),
+    ?assert(erlang:system_time(millisecond) > emqx_message:timestamp(Msg)).
+
+t_clean_dup(_) ->
+    Msg = emqx_message:make(<<"topic">>, <<"payload">>),
+    ?assertNot(emqx_message:get_flag(dup, Msg)),
+    Msg = emqx_message:clean_dup(Msg),
+    Msg1 = emqx_message:set_flag(dup, Msg),
+    ?assert(emqx_message:get_flag(dup, Msg1)),
+    Msg2 = emqx_message:clean_dup(Msg1),
+    ?assertNot(emqx_message:get_flag(dup, Msg2)).
+
 t_get_set_flags(_) ->
     Msg = #message{id = <<"id">>, qos = ?QOS_1, flags = undefined},
     Msg1 = emqx_message:set_flags(#{retain => true}, Msg),
-    ?assertEqual(#{retain => true}, emqx_message:get_flags(Msg1)).
+    ?assertEqual(#{retain => true}, emqx_message:get_flags(Msg1)),
+    Msg2 = emqx_message:set_flags(#{dup => true}, Msg1),
+    ?assertEqual(#{retain => true, dup => true}, emqx_message:get_flags(Msg2)).
 
 t_get_set_flag(_) ->
     Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>),
-    Msg2 = emqx_message:set_flag(retain, false, Msg),
+    ?assertNot(emqx_message:get_flag(dup, Msg)),
+    ?assertNot(emqx_message:get_flag(retain, Msg)),
+    Msg1 = emqx_message:set_flag(dup, true, Msg),
+    Msg2 = emqx_message:set_flag(retain, true, Msg1),
     Msg3 = emqx_message:set_flag(dup, Msg2),
     ?assert(emqx_message:get_flag(dup, Msg3)),
-    ?assertNot(emqx_message:get_flag(retain, Msg3)),
+    ?assert(emqx_message:get_flag(retain, Msg3)),
     Msg4 = emqx_message:unset_flag(dup, Msg3),
     Msg5 = emqx_message:unset_flag(retain, Msg4),
     Msg5 = emqx_message:unset_flag(badflag, Msg5),
@@ -76,6 +115,8 @@ t_get_set_headers(_) ->
 
 t_get_set_header(_) ->
     Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>),
+    Msg = emqx_message:remove_header(x, Msg),
+    ?assertEqual(undefined, emqx_message:get_header(a, Msg)),
     Msg1 = emqx_message:set_header(a, 1, Msg),
     Msg2 = emqx_message:set_header(b, 2, Msg1),
     Msg3 = emqx_message:set_header(c, 3, Msg2),
@@ -95,11 +136,8 @@ t_undefined_headers(_) ->
 t_format(_) ->
     Msg = emqx_message:make(<<"clientid">>, <<"topic">>, <<"payload">>),
     io:format("~s~n", [emqx_message:format(Msg)]),
-    Msg1 = #message{id = <<"id">>,
-                    qos = ?QOS_0,
-                    flags = undefined,
-                    headers = undefined
-                   },
+    Msg1 = emqx_message:set_header('Subscription-Identifier', 1,
+                                   emqx_message:set_flag(dup, Msg)),
     io:format("~s~n", [emqx_message:format(Msg1)]).
 
 t_is_expired(_) ->
@@ -117,28 +155,49 @@ t_is_expired(_) ->
 
 % t_to_list(_) ->
 %     error('TODO').
-    
+
 t_to_packet(_) ->
-    Pkt = #mqtt_packet{header = #mqtt_packet_header{type   = ?PUBLISH,
-                                                    qos    = ?QOS_0,
-                                                    retain = false,
-                                                    dup    = false},
+    Pkt = #mqtt_packet{header   = #mqtt_packet_header{type   = ?PUBLISH,
+                                                      qos    = ?QOS_0,
+                                                      retain = false,
+                                                      dup    = false
+                                                     },
                        variable = #mqtt_packet_publish{topic_name = <<"topic">>,
                                                        packet_id  = 10,
-                                                       properties = #{}},
-                       payload = <<"payload">>},
+                                                       properties = undefined
+                                                      },
+                       payload  = <<"payload">>
+                      },
     Msg = emqx_message:make(<<"clientid">>, ?QOS_0, <<"topic">>, <<"payload">>),
     ?assertEqual(Pkt, emqx_message:to_packet(10, Msg)).
 
+t_to_packet_with_props(_) ->
+    Props = #{'Subscription-Identifier' => 1},
+    Pkt = #mqtt_packet{header   = #mqtt_packet_header{type   = ?PUBLISH,
+                                                      qos    = ?QOS_0,
+                                                      retain = false,
+                                                      dup    = false
+                                                     },
+                       variable = #mqtt_packet_publish{topic_name = <<"topic">>,
+                                                       packet_id  = 10,
+                                                       properties = Props
+                                                      },
+                       payload  = <<"payload">>
+                      },
+    Msg = emqx_message:make(<<"clientid">>, ?QOS_0, <<"topic">>, <<"payload">>),
+    Msg1 = emqx_message:set_header('Subscription-Identifier', 1, Msg),
+    ?assertEqual(Pkt, emqx_message:to_packet(10, Msg1)).
+
 t_to_map(_) ->
     Msg = emqx_message:make(<<"clientid">>, ?QOS_1, <<"topic">>, <<"payload">>),
     List = [{id, emqx_message:id(Msg)},
             {qos, ?QOS_1},
             {from, <<"clientid">>},
-            {flags, #{dup => false}},
-            {headers, #{}},
+            {flags, undefined},
+            {headers, undefined},
             {topic, <<"topic">>},
             {payload, <<"payload">>},
             {timestamp, emqx_message:timestamp(Msg)}],
     ?assertEqual(List, emqx_message:to_list(Msg)),
     ?assertEqual(maps:from_list(List), emqx_message:to_map(Msg)).
+

test/emqx_misc_SUITE.erl

@@ -66,8 +66,7 @@ t_pipeline(_) ->
             fun(I, St)   -> {ok, I*2, St*2} end],
     ?assertEqual({ok, 4, 6}, emqx_misc:pipeline(Funs, 1, 1)),
     ?assertEqual({error, undefined, 1}, emqx_misc:pipeline([fun(_I) -> {error, undefined} end], 1, 1)),
-    ?assertEqual({error, undefined, 2}, emqx_misc:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1)),
-    ?assertEqual({error, undefined, 1}, emqx_misc:pipeline([fun(_I, _St) -> erlang:error(undefined) end], 1, 1)).
+    ?assertEqual({error, undefined, 2}, emqx_misc:pipeline([fun(_I, _St) -> {error, undefined, 2} end], 1, 1)).
 
 t_start_timer(_) ->
     TRef = emqx_misc:start_timer(1, tmsg),

test/emqx_mod_subscription_SUITE.erl

@@ -32,10 +32,8 @@ init_per_suite(Config) ->
 end_per_suite(_Config) ->
     emqx_ct_helpers:stop_apps([emqx]).
 
-t_load(_) ->
-    ?assertEqual(ok, emqx_mod_subscription:load([{<<"connected/%c/%u">>, ?QOS_0}])).
-
 t_on_client_connected(_) ->
+    ?assertEqual(ok, emqx_mod_subscription:load([{<<"connected/%c/%u">>, ?QOS_0}])),
     {ok, C} = emqtt:start_link([{host, "localhost"},
                             {clientid, "myclient"},
                             {username, "admin"}]),
@@ -44,11 +42,20 @@ t_on_client_connected(_) ->
     {ok, #{topic := Topic, payload := Payload}} = receive_publish(100),
     ?assertEqual(<<"connected/myclient/admin">>, Topic),
     ?assertEqual(<<"Hello world">>, Payload),
-    ok = emqtt:disconnect(C).
-
-t_unload(_) ->
+    ok = emqtt:disconnect(C),
     ?assertEqual(ok, emqx_mod_subscription:unload([{<<"connected/%c/%u">>, ?QOS_0}])).
 
+t_on_undefined_client_connected(_) ->
+    ?assertEqual(ok, emqx_mod_subscription:load([{<<"connected/undefined">>, ?QOS_0}])),
+    {ok, C} = emqtt:start_link([{host, "localhost"}]),
+    {ok, _} = emqtt:connect(C),
+    emqtt:publish(C, <<"connected/undefined">>, <<"Hello world">>, ?QOS_0),
+    {ok, #{topic := Topic, payload := Payload}} = receive_publish(100),
+    ?assertEqual(<<"connected/undefined">>, Topic),
+    ?assertEqual(<<"Hello world">>, Payload),
+    ok = emqtt:disconnect(C),
+    ?assertEqual(ok, emqx_mod_subscription:unload([{<<"connected/undefined">>, ?QOS_0}])).
+
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------

test/emqx_msg_expiry_interval_SUITE.erl

@@ -14,23 +14,40 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--module(emqx_msg_expiry_interval_SUITE).
+-module(emqx_mqtt_SUITE).
 
 -compile(export_all).
 -compile(nowarn_export_all).
 
+-include("emqx.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-define(STATS_KYES, [recv_pkt, recv_msg, send_pkt, send_msg,
+                     recv_oct, recv_cnt, send_oct, send_cnt,
+                     send_pend
+                    ]).
 
 all() -> emqx_ct:all(?MODULE).
 
 init_per_suite(Config) ->
-	emqx_ct_helpers:boot_modules(all),
+    emqx_ct_helpers:boot_modules(all),
     emqx_ct_helpers:start_apps([]),
     Config.
 
 end_per_suite(_Config) ->
     emqx_ct_helpers:stop_apps([]).
 
+t_conn_stats(_) ->
+    with_client(fun(CPid) ->
+                            Stats = emqx_connection:stats(CPid),
+                            ct:pal("==== stats: ~p", [Stats]),
+                            [?assert(proplists:get_value(Key, Stats) >= 0) || Key <- ?STATS_KYES]
+                    end, []).
+
+t_tcp_sock_passive(_) ->
+    with_client(fun(CPid) -> CPid ! {tcp_passive, sock} end, []).
+
 t_message_expiry_interval_1(_) ->
 	ClientA = message_expiry_interval_init(),
 	[message_expiry_interval_exipred(ClientA, QoS) || QoS <- [0,1,2]],
@@ -92,3 +109,16 @@ message_expiry_interval_not_exipred(ClientA, QoS) ->
 		ct:fail(no_publish_received)
 	end,
 	emqtt:stop(ClientB1).
+
+with_client(TestFun, _Options) ->
+    ClientId = <<"t_conn">>,
+    {ok, C} = emqtt:start_link([{clientid, ClientId}]),
+    {ok, _} = emqtt:connect(C),
+    timer:sleep(50),
+    case emqx_cm:lookup_channels(ClientId) of
+        [] -> ct:fail({client_not_started, ClientId});
+        [ChanPid] ->
+            TestFun(ChanPid),
+            emqtt:stop(C)
+    end.
+

test/emqx_mqtt_caps_SUITE.erl

@@ -24,12 +24,6 @@
 
 all() -> emqx_ct:all(?MODULE).
 
-% t_get_caps(_) ->
-%     error('TODO').
-
-% t_default(_) ->
-%     error('TODO').
-
 t_check_pub(_) ->
     PubCaps = #{max_qos_allowed => ?QOS_1,
                 retain_available => false

test/emqx_packet_SUITE.erl

@@ -153,7 +153,7 @@ t_check_connect(_) ->
 
 t_from_to_message(_) ->
     ExpectedMsg = emqx_message:make(<<"clientid">>, ?QOS_0, <<"topic">>, <<"payload">>),
-    ExpectedMsg1 = emqx_message:set_flag(retain, false, ExpectedMsg),
+    ExpectedMsg1 = emqx_message:set_flags(#{dup => false, retain => false}, ExpectedMsg),
     ExpectedMsg2 = emqx_message:set_headers(#{peerhost => {127,0,0,1},
                                               protocol => mqtt,
                                               username => <<"test">>

test/emqx_router_helper_SUITE.erl

@@ -21,20 +21,30 @@
 
 -include_lib("eunit/include/eunit.hrl").
 
+-define(ROUTER_HELPER, emqx_router_helper).
+
 all() -> emqx_ct:all(?MODULE).
 
 init_per_testcase(_TestCase, Config) ->
+    emqx_ct_helpers:start_apps([emqx]),
     Config.
 
 end_per_testcase(_TestCase, Config) ->
     Config.
 
-% t_mnesia(_) ->
-%     error('TODO').
-
-% t_monitor(_) ->
-%     error('TODO').
-
-% t_stats_fun(_) ->
-%     error('TODO').
-
+t_monitor(_) ->
+    ok = emqx_router_helper:monitor({undefined, node()}),
+    emqx_router_helper:monitor(undefined).
+
+t_mnesia(_) ->
+    ?ROUTER_HELPER ! {mnesia_table_event, {delete, {emqx_routing_node, node()}, undefined}},
+    ?ROUTER_HELPER ! {mnesia_table_event, testing},
+    ?ROUTER_HELPER ! {mnesia_table_event, {write, {emqx_routing_node, node()}, undefined}},
+    ?ROUTER_HELPER ! {membership, testing},
+    ?ROUTER_HELPER ! {membership, {mnesia, down, node()}},
+    ct:sleep(200).
+
+t_message(_) ->
+    ?ROUTER_HELPER ! testing,
+    gen_server:cast(?ROUTER_HELPER, testing),
+    gen_server:call(?ROUTER_HELPER, testing).

test/emqx_session_SUITE.erl

@@ -23,24 +23,27 @@
 -include_lib("proper/include/proper.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
--import(emqx_session, [set_field/3]).
-
 all() -> emqx_ct:all(?MODULE).
 
 %%--------------------------------------------------------------------
 %% CT callbacks
 %%--------------------------------------------------------------------
 
-init_per_testcase(_TestCase, Config) ->
-    %% Meck Broker
-    ok = meck:new(emqx_broker, [passthrough, no_history]),
-    ok = meck:new(emqx_hooks, [passthrough, no_history]),
+init_per_suite(Config) ->
+    %% Broker
+    ok = meck:new(emqx_broker, [passthrough, no_history, no_link]),
+    ok = meck:new(emqx_hooks, [passthrough, no_history, no_link]),
     ok = meck:expect(emqx_hooks, run, fun(_Hook, _Args) -> ok end),
     Config.
 
-end_per_testcase(_TestCase, Config) ->
+end_per_suite(_Config) ->
     ok = meck:unload(emqx_broker),
-    ok = meck:unload(emqx_hooks),
+    ok = meck:unload(emqx_hooks).
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
+end_per_testcase(_TestCase, Config) ->
     Config.
 
 %%--------------------------------------------------------------------
@@ -57,6 +60,7 @@ t_session_init(_) ->
     ?assertEqual(64, emqx_session:info(inflight_max, Session)),
     ?assertEqual(1, emqx_session:info(next_pkt_id, Session)),
     ?assertEqual(0, emqx_session:info(retry_interval, Session)),
+    ?assertEqual(0, emqx_mqueue:len(emqx_session:info(mqueue, Session))),
     ?assertEqual(0, emqx_session:info(awaiting_rel_cnt, Session)),
     ?assertEqual(100, emqx_session:info(awaiting_rel_max, Session)),
     ?assertEqual(300, emqx_session:info(await_rel_timeout, Session)),
@@ -86,12 +90,11 @@ t_session_stats(_) ->
                   }, maps:from_list(Stats)).
 
 %%--------------------------------------------------------------------
-%% Test cases for pub/sub
+%% Test cases for sub/unsub
 %%--------------------------------------------------------------------
 
 t_subscribe(_) ->
     ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end),
-    ok = meck:expect(emqx_broker, set_subopts, fun(_, _) -> ok end),
     {ok, Session} = emqx_session:subscribe(
                       clientinfo(), <<"#">>, subopts(), session()),
     ?assertEqual(1, emqx_session:info(subscriptions_cnt, Session)).
@@ -101,110 +104,217 @@ t_is_subscriptions_full_false(_) ->
     ?assertNot(emqx_session:is_subscriptions_full(Session)).
 
 t_is_subscriptions_full_true(_) ->
+    ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end),
     Session = session(#{max_subscriptions => 1}),
     ?assertNot(emqx_session:is_subscriptions_full(Session)),
-    Subs = #{<<"t1">> => subopts(), <<"t2">> => subopts()},
-    NSession = set_field(subscriptions, Subs, Session),
-    ?assert(emqx_session:is_subscriptions_full(NSession)).
+    {ok, Session1} = emqx_session:subscribe(
+                       clientinfo(), <<"t1">>, subopts(), Session),
+    ?assert(emqx_session:is_subscriptions_full(Session1)),
+    {error, ?RC_QUOTA_EXCEEDED} =
+        emqx_session:subscribe(clientinfo(), <<"t2">>, subopts(), Session1).
 
 t_unsubscribe(_) ->
     ok = meck:expect(emqx_broker, unsubscribe, fun(_) -> ok end),
     Session = session(#{subscriptions => #{<<"#">> => subopts()}}),
-    {ok, NSession} = emqx_session:unsubscribe(clientinfo(), <<"#">>, Session),
-    Error = emqx_session:unsubscribe(clientinfo(), <<"#">>, NSession),
-    ?assertEqual({error, ?RC_NO_SUBSCRIPTION_EXISTED}, Error).
+    {ok, Session1} = emqx_session:unsubscribe(clientinfo(), <<"#">>, Session),
+    {error, ?RC_NO_SUBSCRIPTION_EXISTED} =
+        emqx_session:unsubscribe(clientinfo(), <<"#">>, Session1).
 
-t_publish_qos2(_) ->
+t_publish_qos0(_) ->
     ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
-    Msg = emqx_message:make(test, ?QOS_2, <<"t">>, <<"payload">>),
-    {ok, [], Session} = emqx_session:publish(1, Msg, session()),
-    ?assertEqual(1, emqx_session:info(awaiting_rel_cnt, Session)).
+    Msg = emqx_message:make(clientid, ?QOS_0, <<"t">>, <<"payload">>),
+    {ok, [], Session} = emqx_session:publish(1, Msg, Session = session()),
+    {ok, [], Session} = emqx_session:publish(undefined, Msg, Session).
 
 t_publish_qos1(_) ->
     ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
-    Msg = emqx_message:make(test, ?QOS_1, <<"t">>, <<"payload">>),
-    {ok, [], _Session} = emqx_session:publish(1, Msg, session()).
+    Msg = emqx_message:make(clientid, ?QOS_1, <<"t">>, <<"payload">>),
+    {ok, [], Session} = emqx_session:publish(1, Msg, Session = session()),
+    {ok, [], Session} = emqx_session:publish(2, Msg, Session).
 
-t_publish_qos0(_) ->
+t_publish_qos2(_) ->
     ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
-    Msg = emqx_message:make(test, ?QOS_1, <<"t">>, <<"payload">>),
-    {ok, [], _Session} = emqx_session:publish(0, Msg, session()).
+    Msg = emqx_message:make(clientid, ?QOS_2, <<"t">>, <<"payload">>),
+    {ok, [], Session} = emqx_session:publish(1, Msg, session()),
+    ?assertEqual(1, emqx_session:info(awaiting_rel_cnt, Session)),
+    {ok, Session1} = emqx_session:pubrel(1, Session),
+    ?assertEqual(0, emqx_session:info(awaiting_rel_cnt, Session1)),
+    {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrel(1, Session1).
+
+t_publish_qos2_with_error_return(_) ->
+    ok = meck:expect(emqx_broker, publish, fun(_) -> [] end),
+    Session = session(#{max_awaiting_rel => 2,
+                        awaiting_rel => #{1 => ts(millisecond)}
+                       }),
+    Msg = emqx_message:make(clientid, ?QOS_2, <<"t">>, <<"payload">>),
+    {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:publish(1, Msg, Session),
+    {ok, [], Session1} = emqx_session:publish(2, Msg, Session),
+    ?assertEqual(2, emqx_session:info(awaiting_rel_cnt, Session1)),
+    {error, ?RC_RECEIVE_MAXIMUM_EXCEEDED} = emqx_session:publish(3, Msg, Session1).
 
 t_is_awaiting_full_false(_) ->
-    ?assertNot(emqx_session:is_awaiting_full(session(#{max_awaiting_rel => 0}))).
+    Session = session(#{max_awaiting_rel => 0}),
+    ?assertNot(emqx_session:is_awaiting_full(Session)).
 
 t_is_awaiting_full_true(_) ->
     Session = session(#{max_awaiting_rel => 1,
-                        awaiting_rel => #{1 => 1}
+                        awaiting_rel => #{1 => ts(millisecond)}
                        }),
     ?assert(emqx_session:is_awaiting_full(Session)).
 
 t_puback(_) ->
     Msg = emqx_message:make(test, ?QOS_1, <<"t">>, <<>>),
-    Inflight = emqx_inflight:insert(1, {Msg, os:timestamp()}, emqx_inflight:new()),
-    Session = set_field(inflight, Inflight, session()),
-    {ok, Msg, NSession} = emqx_session:puback(1, Session),
-    ?assertEqual(0, emqx_session:info(inflight_cnt, NSession)).
+    Inflight = emqx_inflight:insert(1, {Msg, ts(millisecond)}, emqx_inflight:new()),
+    Session = session(#{inflight => Inflight, mqueue => mqueue()}),
+    {ok, Msg, Session1} = emqx_session:puback(1, Session),
+    ?assertEqual(0, emqx_session:info(inflight_cnt, Session1)).
+
+t_puback_with_dequeue(_) ->
+    Msg1 = emqx_message:make(clientid, ?QOS_1, <<"t1">>, <<"payload1">>),
+    Inflight = emqx_inflight:insert(1, {Msg1, ts(millisecond)}, emqx_inflight:new()),
+    Msg2 = emqx_message:make(clientid, ?QOS_1, <<"t2">>, <<"payload2">>),
+    {_, Q} = emqx_mqueue:in(Msg2, mqueue(#{max_len => 10})),
+    Session = session(#{inflight => Inflight, mqueue => Q}),
+    {ok, Msg1, [{_, Msg3}], Session1} = emqx_session:puback(1, Session),
+    ?assertEqual(1, emqx_session:info(inflight_cnt, Session1)),
+    ?assertEqual(0, emqx_session:info(mqueue_len, Session1)),
+    ?assertEqual(<<"t2">>, emqx_message:topic(Msg3)).
 
 t_puback_error_packet_id_in_use(_) ->
-    Inflight = emqx_inflight:insert(1, {pubrel, os:timestamp()}, emqx_inflight:new()),
-    Session = set_field(inflight, Inflight, session()),
-    {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:puback(1, Session).
+    Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()),
+    {error, ?RC_PACKET_IDENTIFIER_IN_USE} =
+        emqx_session:puback(1, session(#{inflight => Inflight})).
 
 t_puback_error_packet_id_not_found(_) ->
     {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:puback(1, session()).
 
 t_pubrec(_) ->
     Msg = emqx_message:make(test, ?QOS_2, <<"t">>, <<>>),
-    Inflight = emqx_inflight:insert(2, {Msg, os:timestamp()}, emqx_inflight:new()),
-    Session = set_field(inflight, Inflight, session()),
-    {ok, Msg, NSession} = emqx_session:pubrec(2, Session),
-    ?assertMatch([{pubrel, _}], emqx_inflight:values(emqx_session:info(inflight, NSession))).
+    Inflight = emqx_inflight:insert(2, {Msg, ts(millisecond)}, emqx_inflight:new()),
+    Session = session(#{inflight => Inflight}),
+    {ok, Msg, Session1} = emqx_session:pubrec(2, Session),
+    ?assertMatch([{pubrel, _}], emqx_inflight:values(emqx_session:info(inflight, Session1))).
 
 t_pubrec_packet_id_in_use_error(_) ->
-    Inflight = emqx_inflight:insert(1, {pubrel, ts()}, emqx_inflight:new()),
-    Session = set_field(inflight, Inflight, session()),
-    {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:puback(1, Session).
+    Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()),
+    {error, ?RC_PACKET_IDENTIFIER_IN_USE} =
+        emqx_session:pubrec(1, session(#{inflight => Inflight})).
 
 t_pubrec_packet_id_not_found_error(_) ->
     {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrec(1, session()).
 
 t_pubrel(_) ->
-    Session = set_field(awaiting_rel, #{1 => os:timestamp()}, session()),
-    {ok, NSession} = emqx_session:pubrel(1, Session),
-    ?assertEqual(#{}, emqx_session:info(awaiting_rel, NSession)).
+    Session = session(#{awaiting_rel => #{1 => ts(millisecond)}}),
+    {ok, Session1} = emqx_session:pubrel(1, Session),
+    ?assertEqual(#{}, emqx_session:info(awaiting_rel, Session1)).
 
-t_pubrel_id_not_found(_) ->
+t_pubrel_error_packetid_not_found(_) ->
     {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubrel(1, session()).
 
 t_pubcomp(_) ->
-    Inflight = emqx_inflight:insert(2, {pubrel, os:timestamp()}, emqx_inflight:new()),
-    Session = emqx_session:set_field(inflight, Inflight, session()),
-    {ok, NSession} = emqx_session:pubcomp(2, Session),
-    ?assertEqual(0, emqx_session:info(inflight_cnt, NSession)).
+    Inflight = emqx_inflight:insert(1, {pubrel, ts(millisecond)}, emqx_inflight:new()),
+    Session = session(#{inflight => Inflight}),
+    {ok, Session1} = emqx_session:pubcomp(1, Session),
+    ?assertEqual(0, emqx_session:info(inflight_cnt, Session1)).
+
+t_pubcomp_error_packetid_in_use(_) ->
+    Msg = emqx_message:make(test, ?QOS_2, <<"t">>, <<>>),
+    Inflight = emqx_inflight:insert(1, {Msg, ts(millisecond)}, emqx_inflight:new()),
+    Session = session(#{inflight => Inflight}),
+    {error, ?RC_PACKET_IDENTIFIER_IN_USE} = emqx_session:pubcomp(1, Session).
 
-t_pubcomp_id_not_found(_) ->
-    {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubcomp(2, session()).
+t_pubcomp_error_packetid_not_found(_) ->
+    {error, ?RC_PACKET_IDENTIFIER_NOT_FOUND} = emqx_session:pubcomp(1, session()).
 
 %%--------------------------------------------------------------------
 %% Test cases for deliver/retry
 %%--------------------------------------------------------------------
 
 t_dequeue(_) ->
-    {ok, _Session} = emqx_session:dequeue(session()).
-
-t_deliver(_) ->
+    Q = mqueue(#{store_qos0 => true}),
+    {ok, Session} = emqx_session:dequeue(session(#{mqueue => Q})),
+    Msgs = [emqx_message:make(clientid, ?QOS_0, <<"t0">>, <<"payload">>),
+            emqx_message:make(clientid, ?QOS_1, <<"t1">>, <<"payload">>),
+            emqx_message:make(clientid, ?QOS_2, <<"t2">>, <<"payload">>)
+           ],
+    Session1 = lists:foldl(fun emqx_session:enqueue/2, Session, Msgs),
+    {ok, [{undefined, Msg0}, {1, Msg1}, {2, Msg2}], Session2} =
+        emqx_session:dequeue(Session1),
+    ?assertEqual(0, emqx_session:info(mqueue_len, Session2)),
+    ?assertEqual(2, emqx_session:info(inflight_cnt, Session2)),
+    ?assertEqual(<<"t0">>, emqx_message:topic(Msg0)),
+    ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)),
+    ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)).
+
+t_deliver_qos0(_) ->
+    ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end),
+    {ok, Session} = emqx_session:subscribe(
+                      clientinfo(), <<"t0">>, subopts(), session()),
+    {ok, Session1} = emqx_session:subscribe(
+                       clientinfo(), <<"t1">>, subopts(), Session),
+    Deliveries = [delivery(?QOS_0, T) || T <- [<<"t0">>, <<"t1">>]],
+    {ok, [{undefined, Msg1}, {undefined, Msg2}], Session1} =
+        emqx_session:deliver(Deliveries, Session1),
+    ?assertEqual(<<"t0">>, emqx_message:topic(Msg1)),
+    ?assertEqual(<<"t1">>, emqx_message:topic(Msg2)).
+
+t_deliver_qos1(_) ->
+    ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end),
+    {ok, Session} = emqx_session:subscribe(
+                       clientinfo(), <<"t1">>, subopts(#{qos => ?QOS_1}), session()),
+    Delivers = [delivery(?QOS_1, T) || T <- [<<"t1">>, <<"t2">>]],
+    {ok, [{1, Msg1}, {2, Msg2}], Session1} = emqx_session:deliver(Delivers, Session),
+    ?assertEqual(2, emqx_session:info(inflight_cnt, Session1)),
+    ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)),
+    ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)),
+    {ok, Msg1, Session2} = emqx_session:puback(1, Session1),
+    ?assertEqual(1, emqx_session:info(inflight_cnt, Session2)),
+    {ok, Msg2, Session3} = emqx_session:puback(2, Session2),
+    ?assertEqual(0, emqx_session:info(inflight_cnt, Session3)).
+
+t_deliver_qos2(_) ->
+    ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end),
+    Delivers = [delivery(?QOS_2, <<"t0">>), delivery(?QOS_2, <<"t1">>)],
+    {ok, [{1, Msg1}, {2, Msg2}], Session} =
+        emqx_session:deliver(Delivers, session()),
+    ?assertEqual(2, emqx_session:info(inflight_cnt, Session)),
+    ?assertEqual(<<"t0">>, emqx_message:topic(Msg1)),
+    ?assertEqual(<<"t1">>, emqx_message:topic(Msg2)).
+
+t_deliver_one_msg(_) ->
+    {ok, [{1, Msg}], Session} =
+        emqx_session:deliver([delivery(?QOS_1, <<"t1">>)], session()),
+    ?assertEqual(1, emqx_session:info(inflight_cnt, Session)),
+    ?assertEqual(<<"t1">>, emqx_message:topic(Msg)).
+
+t_deliver_when_inflight_is_full(_) ->
     Delivers = [delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)],
-    {ok, Publishes, _Session} = emqx_session:deliver(Delivers, session()),
-    ?assertEqual(2, length(Publishes)).
+    Session = session(#{inflight => emqx_inflight:new(1)}),
+    {ok, Publishes, Session1} = emqx_session:deliver(Delivers, Session),
+    ?assertEqual(1, length(Publishes)),
+    ?assertEqual(1, emqx_session:info(inflight_cnt, Session1)),
+    ?assertEqual(1, emqx_session:info(mqueue_len, Session1)),
+    {ok, Msg1, [{2, Msg2}], Session2} = emqx_session:puback(1, Session1),
+    ?assertEqual(1, emqx_session:info(inflight_cnt, Session2)),
+    ?assertEqual(0, emqx_session:info(mqueue_len, Session2)),
+    ?assertEqual(<<"t1">>, emqx_message:topic(Msg1)),
+    ?assertEqual(<<"t2">>, emqx_message:topic(Msg2)).
 
 t_enqueue(_) ->
-    Delivers = [delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)],
-    Session = emqx_session:enqueue(Delivers, session()),
-    ?assertEqual(2, emqx_session:info(mqueue_len, Session)).
+    %% store_qos0 = true
+    Session = emqx_session:enqueue([delivery(?QOS_0, <<"t0">>)], session()),
+    Session1 = emqx_session:enqueue([delivery(?QOS_1, <<"t1">>),
+                                     delivery(?QOS_2, <<"t2">>)], Session),
+    ?assertEqual(3, emqx_session:info(mqueue_len, Session1)).
 
 t_retry(_) ->
-    {ok, _Session} = emqx_session:retry(session()).
+    Delivers = [delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)],
+    Session = session(#{retry_interval => 100}),
+    {ok, Pubs, Session1} = emqx_session:deliver(Delivers, Session),
+    ok = timer:sleep(200),
+    Msgs1 = [{I, emqx_message:set_flag(dup, Msg)} || {I, Msg} <- Pubs],
+    {ok, Msgs1, 100, Session2} = emqx_session:retry(Session1),
+    ?assertEqual(2, emqx_session:info(inflight_cnt, Session2)).
 
 %%--------------------------------------------------------------------
 %% Test cases for takeover/resume
@@ -220,22 +330,53 @@ t_resume(_) ->
     Session = session(#{subscriptions => #{<<"t">> => ?DEFAULT_SUBOPTS}}),
     ok = emqx_session:resume(<<"clientid">>, Session).
 
-t_redeliver(_) ->
-    {ok, [], _Session} = emqx_session:redeliver(session()).
+t_replay(_) ->
+    Delivers = [delivery(?QOS_1, <<"t1">>), delivery(?QOS_2, <<"t2">>)],
+    {ok, Pubs, Session1} = emqx_session:deliver(Delivers, session()),
+    Msg = emqx_message:make(clientid, ?QOS_1, <<"t1">>, <<"payload">>),
+    Session2 = emqx_session:enqueue(Msg, Session1),
+    Pubs1 = [{I, emqx_message:set_flag(dup, M)} || {I, M} <- Pubs],
+    {ok, ReplayPubs, Session3} = emqx_session:replay(Session2),
+    ?assertEqual(Pubs1 ++ [{3, Msg}], ReplayPubs),
+    ?assertEqual(3, emqx_session:info(inflight_cnt, Session3)).
+
+t_expire_awaiting_rel(_) ->
+    {ok, Session} = emqx_session:expire(awaiting_rel, session()),
+    Timeout = emqx_session:info(await_rel_timeout, Session) * 1000,
+    Session1 = emqx_session:set_field(awaiting_rel, #{1 => Ts = ts(millisecond)}, Session),
+    {ok, Timeout, Session2} = emqx_session:expire(awaiting_rel, Session1),
+    ?assertEqual(#{1 => Ts}, emqx_session:info(awaiting_rel, Session2)).
+
+t_expire_awaiting_rel_all(_) ->
+    Session = session(#{awaiting_rel => #{1 => 1, 2 => 2}}),
+    {ok, Session1} = emqx_session:expire(awaiting_rel, Session),
+    ?assertEqual(#{}, emqx_session:info(awaiting_rel, Session1)).
+
+%%--------------------------------------------------------------------
+%% CT for utility functions
+%%--------------------------------------------------------------------
 
-t_expire(_) ->
-    {ok, _Session} = emqx_session:expire(awaiting_rel, session()).
+t_next_pakt_id(_) ->
+    Session = session(#{next_pkt_id => 16#FFFF}),
+    Session1 = emqx_session:next_pkt_id(Session),
+    ?assertEqual(1, emqx_session:info(next_pkt_id, Session1)),
+    Session2 = emqx_session:next_pkt_id(Session1),
+    ?assertEqual(2, emqx_session:info(next_pkt_id, Session2)).
 
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------
 
+mqueue() -> mqueue(#{}).
+mqueue(Opts) ->
+    emqx_mqueue:init(maps:merge(#{max_len => 0, store_qos0 => false}, Opts)).
+
 session() -> session(#{}).
 session(InitFields) when is_map(InitFields) ->
     maps:fold(fun(Field, Value, Session) ->
                       emqx_session:set_field(Field, Value, Session)
               end,
-              emqx_session:init(#{zone => zone}, #{receive_maximum => 0}),
+              emqx_session:init(#{zone => channel}, #{receive_maximum => 0}),
               InitFields).
 
 
@@ -252,5 +393,8 @@ subopts(Init) ->
 delivery(QoS, Topic) ->
     {deliver, Topic, emqx_message:make(test, QoS, Topic, <<"payload">>)}.
 
-ts() -> erlang:system_time(second).
+ts(second) ->
+    erlang:system_time(second);
+ts(millisecond) ->
+    erlang:system_time(millisecond).
 

test/emqx_shared_sub_SUITE.erl

@@ -41,7 +41,7 @@ init_per_suite(Config) ->
 
 end_per_suite(_Config) ->
     emqx_ct_helpers:stop_apps([]).
-    
+
 t_is_ack_required(_) ->
     ?assertEqual(false, emqx_shared_sub:is_ack_required(#message{headers = #{}})).
 

test/emqx_vm_SUITE.erl

@@ -95,9 +95,6 @@ t_scheduler_usage(_Config) ->
 t_get_memory(_Config) ->
     emqx_vm:get_memory().
 
-t_microsecs(_Config) ->
-    emqx_vm:microsecs().
-
 t_schedulers(_Config) ->
     emqx_vm:schedulers().
 

test/emqx_ws_connection_SUITE.erl

@@ -16,6 +16,7 @@
 
 -module(emqx_ws_connection_SUITE).
 
+-include("emqx.hrl").
 -include("emqx_mqtt.hrl").
 -include_lib("eunit/include/eunit.hrl").
 
@@ -25,12 +26,15 @@
 -import(emqx_ws_connection,
         [ websocket_handle/2
         , websocket_info/2
+        , websocket_close/2
         ]).
 
 -define(STATS_KEYS, [recv_oct, recv_cnt, send_oct, send_cnt,
                      recv_pkt, recv_msg, send_pkt, send_msg
                     ]).
 
+-define(ws_conn, emqx_ws_connection).
+
 all() -> emqx_ct:all(?MODULE).
 
 %%--------------------------------------------------------------------
@@ -38,184 +42,358 @@ all() -> emqx_ct:all(?MODULE).
 %%--------------------------------------------------------------------
 
 init_per_suite(Config) ->
-    Config.
-
-end_per_suite(_Config) ->
-    ok.
-
-init_per_testcase(_TestCase, Config) ->
-    %% Meck CowboyReq
-    ok = meck:new(cowboy_req, [passthrough, no_history]),
+    %% Mock cowboy_req
+    ok = meck:new(cowboy_req, [passthrough, no_history, no_link]),
     ok = meck:expect(cowboy_req, peer, fun(_) -> {{127,0,0,1}, 3456} end),
-    ok = meck:expect(cowboy_req, sock, fun(_) -> {{127,0,0,1}, 8883} end),
+    ok = meck:expect(cowboy_req, sock, fun(_) -> {{127,0,0,1}, 18083} end),
     ok = meck:expect(cowboy_req, cert, fun(_) -> undefined end),
-    ok = meck:expect(cowboy_req, parse_cookies, fun(_) -> undefined end),
-    %% Meck Channel
-    ok = meck:new(emqx_channel, [passthrough, no_history]),
-    %% Meck Metrics
-    ok = meck:new(emqx_metrics, [passthrough, no_history]),
+    ok = meck:expect(cowboy_req, parse_cookies, fun(_) -> error(badarg) end),
+    %% Mock emqx_zone
+    ok = meck:new(emqx_zone, [passthrough, no_history, no_link]),
+    ok = meck:expect(emqx_zone, oom_policy,
+                     fun(_) -> #{max_heap_size => 838860800,
+                                 message_queue_len => 8000
+                                }
+                     end),
+    %% Mock emqx_access_control
+    ok = meck:new(emqx_access_control, [passthrough, no_history, no_link]),
+    ok = meck:expect(emqx_access_control, check_acl, fun(_, _, _) -> allow end),
+    %% Mock emqx_hooks
+    ok = meck:new(emqx_hooks, [passthrough, no_history, no_link]),
+    ok = meck:expect(emqx_hooks, run, fun(_Hook, _Args) -> ok end),
+    ok = meck:expect(emqx_hooks, run_fold, fun(_Hook, _Args, Acc) -> Acc end),
+    %% Mock emqx_broker
+    ok = meck:new(emqx_broker, [passthrough, no_history, no_link]),
+    ok = meck:expect(emqx_broker, subscribe, fun(_, _, _) -> ok end),
+    ok = meck:expect(emqx_broker, publish, fun(#message{topic = Topic}) ->
+                                                   [{node(), Topic, 1}]
+                                           end),
+    ok = meck:expect(emqx_broker, unsubscribe, fun(_) -> ok end),
+    %% Mock emqx_metrics
+    ok = meck:new(emqx_metrics, [passthrough, no_history, no_link]),
+    ok = meck:expect(emqx_metrics, inc, fun(_) -> ok end),
     ok = meck:expect(emqx_metrics, inc, fun(_, _) -> ok end),
     ok = meck:expect(emqx_metrics, inc_recv, fun(_) -> ok end),
     ok = meck:expect(emqx_metrics, inc_sent, fun(_) -> ok end),
     Config.
 
+end_per_suite(_Config) ->
+    lists:foreach(fun meck:unload/1,
+                  [cowboy_req,
+                   emqx_zone,
+                   emqx_access_control,
+                   emqx_broker,
+                   emqx_hooks,
+                   emqx_metrics
+                  ]).
+
+init_per_testcase(_TestCase, Config) ->
+    Config.
+
 end_per_testcase(_TestCase, Config) ->
-    ok = meck:unload(cowboy_req),
-    ok = meck:unload(emqx_channel),
-    ok = meck:unload(emqx_metrics),
     Config.
 
 %%--------------------------------------------------------------------
 %% Test Cases
 %%--------------------------------------------------------------------
 
-%%TODO:...
-t_ws_conn_init(_) ->
-    with_ws_conn(fun(_WsConn) -> ok end).
-
-t_ws_conn_info(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         #{sockinfo := SockInfo} = emqx_ws_connection:info(WsConn),
-                         #{socktype  := ws,
-                           peername  := {{127,0,0,1}, 3456},
-                           sockname  := {{127,0,0,1}, 8883},
-                           sockstate := running} = SockInfo
-                 end).
-
-t_websocket_init(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         #{sockinfo := SockInfo} = emqx_ws_connection:info(WsConn),
-                         #{socktype  := ws,
-                           peername  := {{127,0,0,1}, 3456},
-                           sockname  := {{127,0,0,1}, 8883},
-                           sockstate := running
-                          } = SockInfo
-                 end).
+t_info(_) ->
+    WsPid = spawn(fun() ->
+                      receive {call, From, info} ->
+                                  gen_server:reply(From, ?ws_conn:info(st()))
+                      end
+                  end),
+    #{sockinfo := SockInfo} = ?ws_conn:call(WsPid, info),
+    #{socktype  := ws,
+      active_n  := 100,
+      peername  := {{127,0,0,1}, 3456},
+      sockname  := {{127,0,0,1}, 18083},
+      sockstate := running
+     } = SockInfo.
+
+t_info_limiter(_) ->
+    St = st(#{limiter => emqx_limiter:init([])}),
+    ?assertEqual(undefined, ?ws_conn:info(limiter, St)).
+
+t_info_channel(_) ->
+    #{conn_state := connected} = ?ws_conn:info(channel, st()).
+
+t_info_gc_state(_) ->
+    GcSt = emqx_gc:init(#{count => 10, bytes => 1000}),
+    GcInfo = ?ws_conn:info(gc_state, st(#{gc_state => GcSt})),
+    ?assertEqual(#{cnt => {10,10}, oct => {1000,1000}}, GcInfo).
+
+t_info_postponed(_) ->
+    ?assertEqual([], ?ws_conn:info(postponed, st())),
+    St = ?ws_conn:postpone({active, false}, st()),
+    ?assertEqual([{active, false}], ?ws_conn:info(postponed, St)).
+
+t_stats(_) ->
+    WsPid = spawn(fun() ->
+                      receive {call, From, stats} ->
+                                  gen_server:reply(From, ?ws_conn:stats(st()))
+                      end
+                  end),
+    Stats = ?ws_conn:call(WsPid, stats),
+    [{recv_oct, 0}, {recv_cnt, 0}, {send_oct, 0}, {send_cnt, 0},
+     {recv_pkt, 0}, {recv_msg, 0}, {send_pkt, 0}, {send_msg, 0}|_] = Stats.
+
+t_call(_) ->
+    Info = ?ws_conn:info(st()),
+    WsPid = spawn(fun() ->
+                      receive {call, From, info} -> gen_server:reply(From, Info) end
+                  end),
+    ?assertEqual(Info, ?ws_conn:call(WsPid, info)).
+
+t_init(_) ->
+    Opts = [{idle_timeout, 300000}],
+    WsOpts = #{compress       => false,
+               deflate_opts   => #{},
+               max_frame_size => infinity,
+               idle_timeout   => 300000
+              },
+    ok = meck:expect(cowboy_req, parse_header, fun(_, req) -> undefined end),
+    {cowboy_websocket, req, [req, Opts], WsOpts} = ?ws_conn:init(req, Opts),
+    ok = meck:expect(cowboy_req, parse_header, fun(_, req) -> [<<"mqtt">>] end),
+    ok = meck:expect(cowboy_req, set_resp_header, fun(_, <<"mqtt">>, req) -> resp end),
+    {cowboy_websocket, resp, [req, Opts], WsOpts} = ?ws_conn:init(req, Opts).
 
 t_websocket_handle_binary(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         {ok, _} = websocket_handle({binary, [<<>>]}, WsConn)
-                 end).
-
-t_websocket_handle_ping_pong(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         {ok, WsConn} = websocket_handle(ping, WsConn),
-                         {ok, WsConn} = websocket_handle(pong, WsConn),
-                         {ok, WsConn} = websocket_handle({ping, <<>>}, WsConn),
-                         {ok, WsConn} = websocket_handle({pong, <<>>}, WsConn)
-                 end).
+    {ok, _} = websocket_handle({binary, <<>>}, st()),
+    {ok, _} = websocket_handle({binary, [<<>>]}, st()),
+    {ok, _} = websocket_handle({binary, <<192,0>>}, st()),
+    receive {incoming, ?PACKET(?PINGREQ)} -> ok
+    after 0 -> error(expect_incoming_pingreq)
+    end.
+
+t_websocket_handle_ping(_) ->
+    {ok, St} = websocket_handle(ping, St = st()),
+    {ok, St} = websocket_handle({ping, <<>>}, St).
+
+t_websocket_handle_pong(_) ->
+    {ok, St} = websocket_handle(pong, St = st()),
+    {ok, St} = websocket_handle({pong, <<>>}, St).
 
 t_websocket_handle_bad_frame(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         {stop, WsConn1} = websocket_handle({badframe, <<>>}, WsConn),
-                         ?assertEqual({shutdown, unexpected_ws_frame}, stop_reason(WsConn1))
-                 end).
+    {[{shutdown, unexpected_ws_frame}], _St} = websocket_handle({badframe, <<>>}, st()).
 
 t_websocket_info_call(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         From = {make_ref(), self()},
-                         Call = {call, From, badreq},
-                         websocket_info(Call, WsConn)
-                 end).
+    From = {make_ref(), self()},
+    Call = {call, From, badreq},
+    {ok, _St} = websocket_info(Call, st()).
+
+t_websocket_info_rate_limit(_) ->
+    {ok, _} = websocket_info({cast, rate_limit}, st()),
+    ok = timer:sleep(1),
+    receive
+        {check_gc, Stats} ->
+            ?assertEqual(#{cnt => 0, oct => 0}, Stats)
+    after 0 -> error(expect_check_gc)
+    end.
 
 t_websocket_info_cast(_) ->
-    ok = meck:expect(emqx_channel, handle_info, fun(_Msg, Channel) -> {ok, Channel} end),
-    with_ws_conn(fun(WsConn) -> websocket_info({cast, msg}, WsConn) end).
+    {ok, _St} = websocket_info({cast, msg}, st()).
 
 t_websocket_info_incoming(_) ->
-    ok = meck:expect(emqx_channel, handle_in, fun(_Packet, Channel) -> {ok, Channel} end),
-    with_ws_conn(fun(WsConn) ->
-                         Connect = ?CONNECT_PACKET(
-                                      #mqtt_packet_connect{proto_ver   = ?MQTT_PROTO_V5,
-                                                           proto_name  = <<"MQTT">>,
-                                                           clientid    = <<>>,
-                                                           clean_start = true,
-                                                           keepalive   = 60}),
-                         {ok, WsConn1} = websocket_info({incoming, Connect}, WsConn),
-                         Publish = ?PUBLISH_PACKET(?QOS_1, <<"t">>, 1, <<"payload">>),
-                         {ok, _WsConn2} = websocket_info({incoming, Publish}, WsConn1)
-                 end).
+    ConnPkt = #mqtt_packet_connect{
+                 proto_name  = <<"MQTT">>,
+                 proto_ver   = ?MQTT_PROTO_V5,
+                 is_bridge   = false,
+                 clean_start = true,
+                 keepalive   = 60,
+                 properties  = undefined,
+                 clientid    = <<"clientid">>,
+                 username    = <<"username">>,
+                 password    = <<"passwd">>
+                },
+    {[{binary, IoData1}], St1} =
+        websocket_info({incoming, ?CONNECT_PACKET(ConnPkt)}, st()),
+    ?assertEqual(<<224,2,130,0>>, iolist_to_binary(IoData1)),
+    %% PINGREQ
+    {[{binary, IoData2}], St2} =
+        websocket_info({incoming, ?PACKET(?PINGREQ)}, St1),
+    ?assertEqual(<<208,0>>, iolist_to_binary(IoData2)),
+    %% PUBLISH
+    Publish = ?PUBLISH_PACKET(?QOS_1, <<"t">>, 1, <<"payload">>),
+    {[{binary, IoData3}], _St3} =
+        websocket_info({incoming, Publish}, St2),
+    ?assertEqual(<<64,4,0,1,0,0>>, iolist_to_binary(IoData3)).
+
+t_websocket_info_check_gc(_) ->
+    Stats = #{cnt => 10, oct => 1000},
+    {ok, _St} = websocket_info({check_gc, Stats}, st()).
 
 t_websocket_info_deliver(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         ok = meck:expect(emqx_channel, handle_deliver,
-                                          fun(Delivers, Channel) ->
-                                                  Packets = [emqx_message:to_packet(1, Msg) || {deliver, _, Msg} <- Delivers],
-                                                  {ok, {outgoing, Packets}, Channel}
-                                          end),
-                         Deliver = {deliver, <<"#">>, emqx_message:make(<<"topic">>, <<"payload">>)},
-                         {reply, {binary, _Data}, _WsConn1} = websocket_info(Deliver, WsConn)
-                 end).
-
-t_websocket_info_timeout(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         websocket_info({timeout, make_ref(), keepalive}, WsConn),
-                         websocket_info({timeout, make_ref(), emit_stats}, WsConn),
-                         websocket_info({timeout, make_ref(), retry_delivery}, WsConn)
-                 end).
+    Msg0 = emqx_message:make(clientid, ?QOS_0, <<"t">>, <<"">>),
+    Msg1 = emqx_message:make(clientid, ?QOS_1, <<"t">>, <<"">>),
+    self() ! {deliver, <<"#">>, Msg1},
+    {[{binary, IoData}], _St} =
+        websocket_info({deliver, <<"#">>, Msg0}, st()),
+    ?assertEqual(<<48,3,0,1,116,50,5,0,1,116,0,1>>, iolist_to_binary(IoData)).
+
+t_websocket_info_timeout_limiter(_) ->
+    Ref = make_ref(),
+    Event = {timeout, Ref, limit_timeout},
+    {[{active, true}], St} = websocket_info(Event, st(#{limit_timer => Ref})),
+    ?assertEqual([], ?ws_conn:info(postponed, St)).
+
+t_websocket_info_timeout_keepalive(_) ->
+    {ok, _St} = websocket_info({timeout, make_ref(), keepalive}, st()).
+
+t_websocket_info_timeout_emit_stats(_) ->
+    Ref = make_ref(),
+    St = st(#{stats_timer => Ref}),
+    {ok, St1} = websocket_info({timeout, Ref, emit_stats}, St),
+    ?assertEqual(undefined, ?ws_conn:info(stats_timer, St1)).
+
+t_websocket_info_timeout_retry(_) ->
+    {ok, _St} = websocket_info({timeout, make_ref(), retry_delivery}, st()).
 
 t_websocket_info_close(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         {stop, WsConn1} = websocket_info({close, sock_error}, WsConn),
-                         ?assertEqual({shutdown, sock_error}, stop_reason(WsConn1))
-                 end).
+    {[close], _St} = websocket_info({close, sock_error}, st()).
 
 t_websocket_info_shutdown(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         {stop, WsConn1} = websocket_info({shutdown, reason}, WsConn),
-                         ?assertEqual({shutdown, reason}, stop_reason(WsConn1))
-                 end).
-
+    {[{shutdown, reason}], _St} = websocket_info({shutdown, reason}, st()).
 
 t_websocket_info_stop(_) ->
-    with_ws_conn(fun(WsConn) ->
-                         {stop, WsConn1} = websocket_info({stop, normal}, WsConn),
-                         ?assertEqual(normal, stop_reason(WsConn1))
-                 end).
+    {[{shutdown, normal}], _St} = websocket_info({stop, normal}, st()).
 
 t_websocket_close(_) ->
-    ok = meck:expect(emqx_channel, handle_info,
-                     fun({sock_closed, badframe}, Channel) ->
-                             {shutdown, sock_closed, Channel}
-                     end),
-    with_ws_conn(fun(WsConn) ->
-                         {stop, WsConn1} = emqx_ws_connection:websocket_close(badframe, WsConn),
-                         ?assertEqual(sock_closed, stop_reason(WsConn1))
-                 end).
-
-t_handle_call(_) ->
-    with_ws_conn(fun(WsConn) -> ok end).
-
-t_handle_info(_) ->
-    with_ws_conn(fun(WsConn) -> ok end).
-
-t_handle_timeout(_) ->
-    with_ws_conn(fun(WsConn) -> ok end).
+    {[{shutdown, badframe}], _St} = websocket_close(badframe, st()).
+
+t_handle_info_connack(_) ->
+    ConnAck = ?CONNACK_PACKET(?RC_SUCCESS),
+    {[{binary, IoData}], _St} =
+        ?ws_conn:handle_info({connack, ConnAck}, st()),
+    ?assertEqual(<<32,2,0,0>>, iolist_to_binary(IoData)).
+
+t_handle_info_close(_) ->
+    {[close], _St} = ?ws_conn:handle_info({close, protocol_error}, st()).
+
+t_handle_info_event(_) ->
+    ok = meck:new(emqx_cm, [passthrough, no_history]),
+    ok = meck:expect(emqx_cm, register_channel, fun(_,_,_) -> ok end),
+    ok = meck:expect(emqx_cm, connection_closed, fun(_) -> true end),
+    {ok, _} = ?ws_conn:handle_info({event, connected}, st()),
+    {ok, _} = ?ws_conn:handle_info({event, disconnected}, st()),
+    {ok, _} = ?ws_conn:handle_info({event, updated}, st()),
+    ok = meck:unload(emqx_cm).
+
+t_handle_timeout_idle_timeout(_) ->
+    TRef = make_ref(),
+    St = st(#{idle_timer => TRef}),
+    {[{shutdown, idle_timeout}], _St} = ?ws_conn:handle_timeout(TRef, idle_timeout, St).
+
+t_handle_timeout_keepalive(_) ->
+    {ok, _St} = ?ws_conn:handle_timeout(make_ref(), keepalive, st()).
+
+t_handle_timeout_emit_stats(_) ->
+    TRef = make_ref(),
+    {ok, St} = ?ws_conn:handle_timeout(
+                  TRef, emit_stats, st(#{stats_timer => TRef})),
+    ?assertEqual(undefined, ?ws_conn:info(stats_timer, St)).
+
+t_ensure_rate_limit(_) ->
+    Limiter = emqx_limiter:init([{pub_limit, {1, 10}},
+                                 {rate_limit, {100, 1000}}
+                                ]),
+    St = st(#{limiter => Limiter}),
+    St1 = ?ws_conn:ensure_rate_limit(#{cnt => 0, oct => 0}, St),
+    St2 = ?ws_conn:ensure_rate_limit(#{cnt => 11, oct => 1200}, St1),
+    ?assertEqual(blocked, ?ws_conn:info(sockstate, St2)),
+    ?assertEqual([{active, false}], ?ws_conn:info(postponed, St2)).
 
 t_parse_incoming(_) ->
-    with_ws_conn(fun(WsConn) -> ok end).
-
-t_handle_incoming(_) ->
-    with_ws_conn(fun(WsConn) -> ok end).
-
-t_handle_return(_) ->
-    with_ws_conn(fun(WsConn) -> ok end).
+    St = ?ws_conn:parse_incoming(<<48,3>>, st()),
+    St1 = ?ws_conn:parse_incoming(<<0,1,116>>, St),
+    Packet = ?PUBLISH_PACKET(?QOS_0, <<"t">>, undefined, <<>>),
+    [{incoming, Packet}] = ?ws_conn:info(postponed, St1).
+
+t_parse_incoming_frame_error(_) ->
+    St = ?ws_conn:parse_incoming(<<3,2,1,0>>, st()),
+    FrameError = {frame_error, function_clause},
+    [{incoming, FrameError}] = ?ws_conn:info(postponed, St).
+
+t_handle_incomming_frame_error(_) ->
+    FrameError = {frame_error, bad_qos},
+    Serialize = emqx_frame:serialize_fun(#{version => 5, max_size => 16#FFFF}),
+    {[{binary, IoData}], _St} =
+        ?ws_conn:handle_incoming(FrameError, st(#{serialize => Serialize})),
+    ?assertEqual(<<224,2,129,0>>, iolist_to_binary(IoData)).
 
 t_handle_outgoing(_) ->
-    with_ws_conn(fun(WsConn) -> ok end).
+    Packets = [?PUBLISH_PACKET(?QOS_1, <<"t1">>, 1, <<"payload">>),
+               ?PUBLISH_PACKET(?QOS_2, <<"t2">>, 2, <<"payload">>)
+              ],
+    {{binary, IoData}, _St} = ?ws_conn:handle_outgoing(Packets, st()),
+    ?assert(is_binary(iolist_to_binary(IoData))).
+
+t_run_gc(_) ->
+    GcSt = emqx_gc:init(#{count => 10, bytes => 100}),
+    WsSt = st(#{gc_state => GcSt}),
+    ?ws_conn:run_gc(#{cnt => 100, oct => 10000}, WsSt).
+
+t_check_oom(_) ->
+    %%Policy = #{max_heap_size => 10, message_queue_len => 10},
+    %%meck:expect(emqx_zone, oom_policy, fun(_) -> Policy end),
+    _St = ?ws_conn:check_oom(st()),
+    ok = timer:sleep(10).
+    %%receive {shutdown, proc_heap_too_large} -> ok
+    %%after 0 -> error(expect_shutdown)
+    %%end.
+
+t_enqueue(_) ->
+    Packet = ?PUBLISH_PACKET(?QOS_0),
+    St = ?ws_conn:enqueue(Packet, st()),
+    [Packet] = ?ws_conn:info(postponed, St).
+
+t_shutdown(_) ->
+    {[{shutdown, closed}], _St} = ?ws_conn:shutdown(closed, st()).
 
 %%--------------------------------------------------------------------
 %% Helper functions
 %%--------------------------------------------------------------------
 
-with_ws_conn(TestFun) ->
-    with_ws_conn(TestFun, []).
-
-with_ws_conn(TestFun, Opts) ->
-    {ok, WsConn, _} = emqx_ws_connection:websocket_init(
-                     [req, emqx_misc:merge_opts([{zone, external}], Opts)]),
-    TestFun(WsConn).
-
-stop_reason(WsConn) ->
-    emqx_ws_connection:info(stop_reason, WsConn).
+st() -> st(#{}).
+st(InitFields) when is_map(InitFields) ->
+    {ok, St, _} = ?ws_conn:websocket_init([req, [{zone, external}]]),
+    maps:fold(fun(N, V, S) -> ?ws_conn:set_field(N, V, S) end,
+              ?ws_conn:set_field(channel, channel(), St),
+              InitFields
+             ).
+
+channel() -> channel(#{}).
+channel(InitFields) ->
+    ConnInfo = #{peername => {{127,0,0,1}, 3456},
+                 sockname => {{127,0,0,1}, 18083},
+                 conn_mod => emqx_ws_connection,
+                 proto_name => <<"MQTT">>,
+                 proto_ver => ?MQTT_PROTO_V5,
+                 clean_start => true,
+                 keepalive => 30,
+                 clientid => <<"clientid">>,
+                 username => <<"username">>,
+                 receive_maximum => 100,
+                 expiry_interval => 0
+                },
+    ClientInfo = #{zone       => zone,
+                   protocol   => mqtt,
+                   peerhost   => {127,0,0,1},
+                   clientid   => <<"clientid">>,
+                   username   => <<"username">>,
+                   is_superuser => false,
+                   peercert   => undefined,
+                   mountpoint => undefined
+                  },
+    Session = emqx_session:init(#{zone => external},
+                                #{receive_maximum => 0}
+                               ),
+    maps:fold(fun(Field, Value, Channel) ->
+                      emqx_channel:set_field(Field, Value, Channel)
+              end,
+              emqx_channel:init(ConnInfo, [{zone, zone}]),
+              maps:merge(#{clientinfo => ClientInfo,
+                           session    => Session,
+                           conn_state => connected
+                          }, InitFields)).