fix(emqx_machine): handle early shutdown

apps/emqx_machine/src/emqx_machine.erl

@@ -16,8 +16,9 @@
 
 -module(emqx_machine).
 
--export([start/0,
-         graceful_shutdown/0
+-export([ start/0
+        , graceful_shutdown/0
+        , is_ready/0
         ]).
 
 -export([ stop_apps/1
@@ -40,7 +41,6 @@ start() ->
     _ = load_modules(),
     ok = load_config_files(),
 
-
     ok = ensure_apps_started(),
 
     _ = emqx_plugins:load(),
@@ -48,6 +48,7 @@ start() ->
     ok = print_vsn(),
 
     ok = start_autocluster(),
+    %% NOTE: keep this to the end
     ok = emqx_machine_terminator:start().
 
 graceful_shutdown() ->
@@ -58,6 +59,10 @@ set_backtrace_depth() ->
     _ = erlang:system_flag(backtrace_depth, Depth),
     ok.
 
+%% @doc Return true if boot is complete.
+is_ready() ->
+    emqx_machine_terminator:is_running().
+
 -if(?OTP_RELEASE > 22).
 print_otp_version_warning() -> ok.
 -else.

apps/emqx_machine/src/emqx_machine_terminator.erl

@@ -21,6 +21,7 @@
 -export([ start/0
         , graceful/0
         , graceful_wait/0
+        , is_running/0
         ]).
 
 -export([init/1, format_status/2,
@@ -40,6 +41,8 @@ start() ->
     %% NOTE: Do not link this process under any supervision tree
     ok.
 
+is_running() -> is_pid(whereis(?TERMINATOR)).
+
 %% @doc Send a signal to activate the terminator.
 graceful() ->
     ?TERMINATOR ! ?DO_IT,
@@ -49,7 +52,8 @@ graceful() ->
 graceful_wait() ->
     case whereis(?TERMINATOR) of
         undefined ->
-            exit(emqx_machine_not_started);
+            ?SLOG(warning, #{msg => "shutdown_before_boot_is_complete"}),
+            exit_loop();
         Pid ->
             ok = graceful(),
             Ref = monitor(process, Pid),
@@ -59,6 +63,11 @@ graceful_wait() ->
             receive {'DOWN', Ref, process, Pid, _} -> ok end
     end.
 
+exit_loop() ->
+    init:stop(),
+    timer:sleep(100),
+    exit_loop().
+
 init(_) ->
     ok = emqx_machine_signal_handler:start(),
     {ok, #{}}.