feat(emqx_machine): add a kill signal handler

Now the signal from kill PID can also be handled gracefully

apps/emqx_machine/src/emqx_machine.erl

@@ -49,7 +49,7 @@ start() ->
     ok = emqx_machine_terminator:start().
 
 graceful_shutdown() ->
-    emqx_machine_terminator:graceful().
+    emqx_machine_terminator:graceful_wait().
 
 set_backtrace_depth() ->
     {ok, Depth} = application:get_env(emqx_machine, backtrace_depth),

apps/emqx_machine/src/emqx_machine_signal_handler.erl

@@ -0,0 +1,61 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% This module implements a gen_event handler which
+%% swap-in replaces the default one from OTP.
+%% The kill signal (sigterm) is captured so we can
+%% perform graceful shutdown.
+-module(emqx_machine_signal_handler).
+
+-export([start/0, init/1, format_status/2,
+         handle_event/2, handle_call/2, handle_info/2,
+         terminate/2, code_change/3]).
+
+-include_lib("emqx/include/logger.hrl").
+
+start() ->
+    ok = gen_event:swap_sup_handler(
+           erl_signal_server,
+           {erl_signal_handler, []},
+           {?MODULE, []}).
+
+init({[], _}) -> {ok, #{}}.
+
+handle_event(sigterm, State) ->
+    ?ULOG("Received terminate signal, shutting down now~n", []),
+    emqx_machine_terminator:graceful(),
+    {ok, State};
+handle_event(Event, State) ->
+    %% delegate other events back to erl_signal_handler
+    %% erl_signal_handler does not make use of the State
+    %% so we can pass whatever from here
+    erl_signal_handler:handle_event(Event, State),
+    {ok, State}.
+
+handle_info(stop, State) ->
+    {ok, State}.
+
+handle_call(_Request, State) ->
+    {ok, ok, State}.
+
+format_status(_Opt, [_Pdict,_S]) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+terminate(_Args, _State) ->
+    ok.

apps/emqx_machine/src/emqx_machine_terminator.erl

@@ -16,43 +16,40 @@
 
 -module(emqx_machine_terminator).
 
+-behaviour(gen_server).
+
 -export([ start/0
         , graceful/0
-        , terminator_loop/0
+        , graceful_wait/0
         ]).
 
+-export([init/1, format_status/2,
+         handle_cast/2, handle_call/3, handle_info/2,
+         terminate/2, code_change/3]).
+
 -define(TERMINATOR, ?MODULE).
+-define(DO_IT, graceful_shutdown).
 
 %% @doc This API is called to shutdown the Erlang VM by RPC call from remote shell node.
 %% The shutown of apps is delegated to a to a process instead of doing it in the RPC spawned
 %% process which has a remote group leader.
 start() ->
-    _ = spawn_link(
-          fun() ->
-                  register(?TERMINATOR, self()),
-                  terminator_loop()
-          end),
+    {ok, _} = gen_server:start_link({local, ?TERMINATOR}, ?MODULE, [], []),
+    %% NOTE: Do not link this process under any supervision tree
     ok.
 
-%% internal use
-terminator_loop() ->
-    receive
-        graceful_shutdown ->
-            ok = emqx_machine:stop_apps(normal),
-            exit_loop()
-    after
-        1000 ->
-            %% keep looping for beam reload
-            ?MODULE:terminator_loop()
-    end.
-
-%% @doc Shutdown the Erlang VM.
+%% @doc Send a signal to activate the terminator.
 graceful() ->
+    ?TERMINATOR ! ?DO_IT,
+    ok.
+
+%% @doc Shutdown the Erlang VM and wait until the terminator dies or the VM dies.
+graceful_wait() ->
     case whereis(?TERMINATOR) of
         undefined ->
             exit(emqx_machine_not_started);
         Pid ->
-            Pid ! graceful_shutdown,
+            ok = graceful(),
             Ref = monitor(process, Pid),
             %% NOTE: not exactly sure, but maybe there is a chance that
             %% Erlang VM goes down before this receive.
@@ -60,8 +57,28 @@ graceful() ->
             receive {'DOWN', Ref, process, Pid, _} -> ok end
     end.
 
-%% Loop until Erlang VM exits
-exit_loop() ->
+init(_) ->
+    ok = emqx_machine_signal_handler:start(),
+    {ok, #{}}.
+
+handle_info(?DO_IT, State) ->
+    ok = emqx_machine:stop_apps(normal),
     init:stop(),
-    timer:sleep(100),
-    exit_loop().
+    {noreply, State};
+handle_info(_, State) ->
+    {noreply, State}.
+
+handle_cast(_Cast, State) ->
+    {noreply, State}.
+
+handle_call(_Call, _From, State) ->
+    {noreply, State}.
+
+format_status(_Opt, [_Pdict,_S]) ->
+    ok.
+
+code_change(_OldVsn, State, _Extra) ->
+    {ok, State}.
+
+terminate(_Args, _State) ->
+    ok.