emqx4.3/bin/emqx

#!/usr/bin/env bash
# -*- tab-width:4;indent-tabs-mode:nil -*-
# ex: ts=4 sw=4 et

set -euo pipefail

DEBUG="${DEBUG:-0}"
if [ "$DEBUG" -eq 1 ]; then
    set -x
fi

ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
# shellcheck disable=SC1090,SC1091
. "$ROOT_DIR"/releases/emqx_vars

# defined in emqx_vars
export RUNNER_ROOT_DIR
export RUNNER_ETC_DIR
export REL_VSN
export SCHEMA_MOD

RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME"
CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}"
REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN"

WHOAMI=$(whoami)

# Make sure log directory exists
mkdir -p "$RUNNER_LOG_DIR"

# Make sure data directory exists
mkdir -p "$RUNNER_DATA_DIR"

# Make sure data/configs exists
CONFIGS_DIR="$RUNNER_DATA_DIR/configs"
mkdir -p "$CONFIGS_DIR"

# hocon try to read environment variables starting with "EMQX_"
export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'

export ROOTDIR="$RUNNER_ROOT_DIR"
export ERTS_DIR="$ROOTDIR/erts-$ERTS_VSN"
export BINDIR="$ERTS_DIR/bin"
export EMU="beam"
export PROGNAME="erl"
export ERTS_LIB_DIR="$ERTS_DIR/../lib"
DYNLIBS_DIR="$RUNNER_ROOT_DIR/dynlibs"

# Echo to stderr on errors
echoerr() { echo "ERROR: $*" 1>&2; }

die() {
    echoerr "ERROR: $1"
    errno=${2:-1}
    exit "$errno"
}

assert_node_alive() {
    if ! relx_nodetool "ping" > /dev/null; then
        die "node_is_not_running!" 1
    fi
}


# Echo to stderr on errors
echoerr() { echo "$*" 1>&2; }

check_erlang_start() {
  # RELEASE_LIB is used by Elixir
  "$BINDIR/$PROGNAME" \
    -noshell \
    -boot_var RELEASE_LIB "$ERTS_LIB_DIR/lib" \
    -boot "$REL_DIR/start_clean" \
    -s crypto start \
    -s erlang halt
}

usage() {
    local command="$1"

    case "$command" in
    start)
        echo "Start EMQX service in daemon mode"
        ;;
    stop)
        echo "Stop the running EMQX program"
        ;;
    console)
        echo "Boot up EMQX service in an interactive Erlang or Elixir shell"
        echo "This command needs a tty"
        ;;
    console_clean)
        echo "This command does NOT boot up the EMQX service"
        echo "It only starts an interactive Erlang or Elixir console with all the"
        echo "EMQX code available"
        ;;
    foreground)
        echo "Start EMQX in foreground mode without an interactive shell"
        ;;
    pid)
        echo "Print out EMQX process identifier"
        ;;
    ping)
        echo "Check if the EMQX node is up and running"
        echo "This command exit with 0 silently if node is running"
        ;;
    escript)
        echo "Execute a escript using the Erlang runtime from EMQX package installation"
        echo "For example $REL_NAME escript /path/to/my/escript my_arg1 my_arg2"
        ;;
    attach)
        echo "This command is applicable when EMQX is started in daemon"
        echo "mode. it attaches the current shell to EMQX's control console"
        echo "through a named pipe"
        echo "WARNING: try to use the safer alternative, remote_console command."
        ;;
    remote_console)
        echo "Start a dummy Erlang or Elixir node and hidden-connect EMQX to"
        echo "with an interactive Erlang or Elixir shell"
        ;;
    ertspath)
        echo "Print path to Erlang runtime dir"
        ;;
    rpc)
        echo "Usge $REL_NAME rpc MODULE FUNCTION [ARGS, ...]"
        echo "Connect to the EMQX node and make an Erlang RPC"
        echo "This command blocks for at most 60 seconds."
        echo "It exits with non-zero code in case of any RPC failure"
        echo "including connection error and runtime exception"
        ;;
    rpcterms)
        echo "Usge $REL_NAME rpcterms MODULE FUNCTION [ARGS, ...]"
        echo "Connect to the EMQX node and make an Erlang RPC"
        echo "The result of the RPC call is pretty-printed as an "
        echo "Erlang term"
        ;;
    root_dir)
        echo "Print EMQX installation root dir"
        ;;
    eval)
        echo "Evaluate an Erlang or Elxir expression in the EMQX node"
        ;;
    eval-erl)
        echo "Evaluate an Erlang expression in the EMQX node, even on Elixir node"
        ;;
    versions)
        echo "List installed EMQX versions and their status"
        ;;
    unpack)
        echo "Usage: $REL_NAME unpack [VERSION]"
        echo "Unpacks a release package VERSION, it assumes that this"
        echo "release package tarball has already been deployed at one"
        echo "of the following locations:"
        echo "      releases/<relname>-<version>.tar.gz"
        ;;
    install)
        echo "Usage: $REL_NAME install [VERSION]"
        echo "Installs a release package VERSION, it assumes that this"
        echo "release package tarball has already been deployed at one"
        echo "of the following locations:"
        echo "      releases/<relname>-<version>.tar.gz"
        echo ""
        echo "     --no-permanent   Install release package VERSION but"
        echo "                      don't make it permanent"
        ;;
    uninstall)
        echo "Usage: $REL_NAME uninstall [VERSION]"
        echo "Uninstalls a release VERSION, it will only accept"
        echo "versions that are not currently in use"
        ;;
    upgrade)
        echo "Usage: $REL_NAME upgrade [VERSION]"
        echo "Upgrades the currently running release to VERSION, it assumes"
        echo "that a release package tarball has already been deployed at one"
        echo "of the following locations:"
        echo "      releases/<relname>-<version>.tar.gz"
        echo ""
        echo "     --no-permanent   Install release package VERSION but"
        echo "                      don't make it permanent"
        ;;
    downgrade)
        echo "Usage: $REL_NAME downgrade [VERSION]"
        echo "Downgrades the currently running release to VERSION, it assumes"
        echo "that a release package tarball has already been deployed at one"
        echo "of the following locations:"
        echo "      releases/<relname>-<version>.tar.gz"
        echo ""
        echo "     --no-permanent   Install release package VERSION but"
        echo "                      don't make it permanent"
        ;;
    *)
        echo "Usage: $REL_NAME COMMAND [help]"
        echo ''
        echo "Commonly used COMMANDs:"
        echo "  start:      Start EMQX in daemon mode"
        echo "  console:    Start EMQX in an interactive Erlang or Elixir shell"
        echo "  foreground: Start EMQX in foreground mode without an interactive shell"
        echo "  stop:       Stop the running EMQX node"
        echo "  ctl:        Administration commands, execute '$REL_NAME ctl help' for more details"
        echo ''
        echo "More:"
        echo "  Shell attach:  remote_console | attach"
        echo "  Up/Down-grade: upgrade | downgrade | install | uninstall"
        echo "  Install info:  ertspath | root_dir | versions | root_dir"
        echo "  Runtime info:  pid | ping | versions"
        echo "  Advanced:      console_clean | escript | rpc | rpcterms | eval | eval-erl"
        echo ''
        echo "Execute '$REL_NAME COMMAND help' for more information"
    ;;
    esac
}

COMMAND="${1:-}"

if [ -z "$COMMAND" ]; then
    usage 'nil'
    exit 1
fi

if [ "${2:-}" = 'help' ]; then
    ## 'ctl' command has its own usage info
    if [ "$COMMAND" != 'ctl' ]; then
        usage "$COMMAND"
        exit 0
    fi
fi

if ! check_erlang_start >/dev/null 2>&1; then
    BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")"
    ## failed to start, might be due to missing libs, try to be portable
    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-$DYNLIBS_DIR}"
    if [ "$LD_LIBRARY_PATH" != "$DYNLIBS_DIR" ]; then
        export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH"
    fi
    if ! check_erlang_start; then
        ## it's hopeless
        echoerr "FATAL: Unable to start Erlang."
        echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed."
        echoerr "Also ensure it's running on the correct platform,"
        echoerr "this EMQX release is built for $BUILT_ON"
        exit 1
    fi
    echoerr "WARNING: There seem to be missing dynamic libs from the OS. Using libs from ${DYNLIBS_DIR}"
fi

## backward compatible
if [ -d "$ERTS_DIR/lib" ]; then
    export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH"
fi

# Simple way to check the correct user and fail early
check_user() {
    # Validate that the user running the script is the owner of the
    # RUN_DIR.
    if [ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]; then
        if [ "x$WHOAMI" != "xroot" ]; then
            echo "You need to be root or use sudo to run this command"
            exit 1
        fi
        CMD="DEBUG=$DEBUG \"$RUNNER_SCRIPT\" "
        for ARG in "$@"; do
            CMD="${CMD} \"$ARG\""
        done
        # This will drop privileges into the runner user
        # It exec's in a new shell and the current shell will exit
        exec su - "$RUNNER_USER" -c "$CMD"
    fi
}

# Make sure the user running this script is the owner and/or su to that user
check_user "$@"
ES=$?
if [ "$ES" -ne 0 ]; then
    exit $ES
fi

# EPMD_ARG="-start_epmd true $PROTO_DIST_ARG"
NO_EPMD="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka"
EPMD_ARG="${EPMD_ARG:-${NO_EPMD}}"

# Warn the user if ulimit -n is less than 1024
ULIMIT_F=$(ulimit -n)
if [ "$ULIMIT_F" -lt 1024 ]; then
    echo "!!!!"
    echo "!!!! WARNING: ulimit -n is ${ULIMIT_F}; 1024 is the recommended minimum."
    echo "!!!!"
fi

SED_REPLACE="sed -i "
case $(sed --help 2>&1) in
    *GNU*) SED_REPLACE="sed -i ";;
    *BusyBox*) SED_REPLACE="sed -i ";;
    *) SED_REPLACE="sed -i '' ";;
esac

# Get node pid
relx_get_pid() {
    if output="$(relx_nodetool rpcterms os getpid)"
    then
        # shellcheck disable=SC2001 # Escaped quote taken as closing quote in editor
        echo "$output" | sed -e 's/"//g'
        return 0
    else
        echo "$output"
        return 1
    fi
}

# Connect to a remote node
relx_rem_sh() {
    # Generate a unique id used to allow multiple remsh to the same node
    # transparently
    id="remsh$(relx_gen_id)-${NAME}"
    # Get the node's ticktime so that we use the same thing.
    TICKTIME="$(relx_nodetool rpcterms net_kernel get_net_ticktime)"

    # shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace
    # Setup remote shell command to control node
    if [ "$IS_ELIXIR" = "yes" ]
    then
      exec "$REL_DIR/iex" \
           --remsh "$NAME" \
           --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
           --cookie "$COOKIE" \
           --hidden \
           --erl "-kernel net_ticktime $TICKTIME" \
           --erl "$EPMD_ARG" \
           --erl "$NAME_TYPE $id" \
           --boot "$REL_DIR/start_clean"
    else
      exec "$BINDIR/erl" "$NAME_TYPE" "$id" \
           -remsh "$NAME" -boot "$REL_DIR/start_clean" \
           -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
           -setcookie "$COOKIE" -hidden -kernel net_ticktime "$TICKTIME" \
           $EPMD_ARG
    fi
}

# Generate a random id
relx_gen_id() {
    od -t x -N 4 /dev/urandom | head -n1 | awk '{print $2}'
}

call_nodetool() {
    "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$@"
}

# Control a node
relx_nodetool() {
    command="$1"; shift
    ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARG" \
    call_nodetool "$NAME_TYPE" "$NAME" \
                                -setcookie "$COOKIE" "$command" "$@"
}

call_hocon() {
    call_nodetool hocon "$@" \
        || die "call_hocon_failed: $*" $?
}

get_config_value() {
    path_to_value="$1"
    call_hocon -s "$SCHEMA_MOD" -I "$CONFIGS_DIR/" -c "$RUNNER_ETC_DIR"/emqx.conf get "$path_to_value" | tr -d \"
}

check_license() {
    if [ "$IS_ENTERPRISE" == "no" ]; then
        return 0
    fi

    file_license="${EMQX_LICENSE__FILE:-$(get_config_value license.file)}"

    if [[ -n "$file_license" && ("$file_license" != "undefined") ]]; then
        call_nodetool check_license_file "$file_license"
    else
        key_license="${EMQX_LICENSE__KEY:-$(get_config_value license.key)}"

        if [[ -n "$key_license" && ("$key_license" != "undefined") ]]; then
            call_nodetool check_license_key "$key_license"
        else
            echoerr "License not found."
            echoerr "Please specify one via EMQX_LICENSE__KEY or EMQX_LICENSE__FILE variables"
            echoerr "or via license.key|file in emqx_enterprise.conf."
            return 1
        fi
    fi
}

# Run an escript in the node's environment
relx_escript() {
    shift; scriptpath="$1"; shift
    "$ERTS_DIR/bin/escript" "$ROOTDIR/$scriptpath" "$@"
}

# Output a start command for the last argument of run_erl
relx_start_command() {
    printf "exec \"%s\" \"%s\"" "$RUNNER_SCRIPT" \
           "$START_OPTION"
}

# Function to generate app.config and vm.args
generate_config() {
    local name_type="$1"
    local node_name="$2"
    ## Delete the *.siz files first or it can't start after
    ## changing the config 'log.rotation.size'
    rm -rf "${RUNNER_LOG_DIR}"/*.siz

    ## timestamp for each generation
    local NOW_TIME
    NOW_TIME="$(call_hocon now_time)"

    ## this command populates two files: app.<time>.config and vm.<time>.args
    ## NOTE: the generate command merges environment variables to the base config (emqx.conf),
    ## but does not include the cluster-override.conf and local-override.conf
    ## meaning, certain overrides will not be mapped to app.<time>.config file
    call_hocon -v -t "$NOW_TIME" -I "$CONFIGS_DIR/" -s "$SCHEMA_MOD" -c "$RUNNER_ETC_DIR"/emqx.conf -d "$RUNNER_DATA_DIR"/configs generate

    ## filenames are per-hocon convention
    local CONF_FILE="$CONFIGS_DIR/app.$NOW_TIME.config"
    local HOCON_GEN_ARG_FILE="$CONFIGS_DIR/vm.$NOW_TIME.args"

    # This is needed by the Elixir scripts.
    # Do NOT append `.config`.
    RELEASE_SYS_CONFIG="$CONFIGS_DIR/app.$NOW_TIME"
    export RELEASE_SYS_CONFIG

    CONFIG_ARGS="-config $CONF_FILE -args_file $HOCON_GEN_ARG_FILE"

    ## Merge hocon generated *.args into the vm.args
    TMP_ARG_FILE="$CONFIGS_DIR/vm.args.tmp"
    cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE"
    echo "" >> "$TMP_ARG_FILE"
    echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE"
    ## read lines from generated vm.<time>.args file
    ## drop comment lines, and empty lines using sed
    ## pipe the lines to a while loop
    sed '/^#/d' "$HOCON_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
        ## in the loop, split the 'key[:space:]value' pair
        ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}')
        ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}')
        ## use the key to look up in vm.args file for the value
        TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" || true | awk '{print $NF}')
        ## compare generated (to override) value to original (to be overridden) value
        if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then
            ## if they are different
            if [ -n "$TMP_ARG_VALUE" ]; then
                ## if the old value is present, replace it with generated value
                sh -c "$SED_REPLACE 's|^$ARG_KEY.*$|$ARG_LINE|' $TMP_ARG_FILE"
            else
                ## otherwise append generated value to the end
                echo "$ARG_LINE" >> "$TMP_ARG_FILE"
            fi
        fi
    done
    echo "$name_type $node_name" >> "$TMP_ARG_FILE"
    ## rename the generated vm.<time>.args file
    mv -f "$TMP_ARG_FILE" "$HOCON_GEN_ARG_FILE"

    # shellcheck disable=SC2086
    if ! relx_nodetool chkconfig $CONFIG_ARGS; then
        die "failed_to_check_config $CONFIG_ARGS"
    fi
}

# check if a PID is down
is_down() {
    PID="$1"
    if ps -p "$PID" >/dev/null; then
        # still around
        # shellcheck disable=SC2009 # this grep pattern is not a part of the progra names
        if ps -p "$PID" | grep -q 'defunct'; then
            # zombie state, print parent pid
            parent="$(ps -o ppid= -p "$PID" | tr -d ' ')"
            echo "WARN: $PID is marked <defunct>, parent:"
            ps -p "$parent"
            return 0
        fi
        return 1
    fi
    # it's gone
    return 0
}

wait_for() {
    local WAIT_TIME
    local CMD
    WAIT_TIME="$1"
    shift
    CMD="$*"
    while true; do
        if $CMD >/dev/null 2>&1; then
            return 0
        fi
        if [ "$WAIT_TIME" -le 0 ]; then
            return 1
        fi
        WAIT_TIME=$((WAIT_TIME - 1))
        sleep 1
    done
}

wait_until_return_val() {
    local RESULT
    local WAIT_TIME
    local CMD
    RESULT="$1"
    WAIT_TIME="$2"
    shift 2
    CMD="$*"
    while true; do
        if [ "$($CMD 2>/dev/null)" = "$RESULT" ]; then
            return 0
        fi
        if [ "$WAIT_TIME" -le 0 ]; then
            return 1
        fi
        WAIT_TIME=$((WAIT_TIME - 1))
        sleep 1
    done
}

latest_vm_args() {
    local hint_var_name="$1"
    local vm_args_file
    vm_args_file="$(find "$CONFIGS_DIR" -type f -name "vm.*.args" | sort | tail -1)"
    if [ -f "$vm_args_file" ]; then
        echo "$vm_args_file"
    else
        echoerr "ERROR: node not initialized?"
        echoerr "Generated config file vm.*.args is not found for command '$COMMAND'"
        echoerr "in config dir: $CONFIGS_DIR"
        echoerr "In case the file has been deleted while the node is running,"
        echoerr "set environment variable '$hint_var_name' to continue"
        exit 1
    fi
}

## IS_BOOT_COMMAND is set for later to inspect node name and cookie from hocon config (or env variable)
case "${COMMAND}" in
    start|console|console_clean|foreground)
        IS_BOOT_COMMAND='yes'
        ;;
    ertspath)
        echo "$ERTS_PATH"
        exit 0
        ;;
    *)
        IS_BOOT_COMMAND='no'
        ;;
esac

## make EMQX_NODE_COOKIE right
if [ -n "${EMQX_NODE_NAME:-}" ]; then
    export EMQX_NODE__NAME="${EMQX_NODE_NAME}"
    unset EMQX_NODE_NAME
fi
## Possible ways to configure emqx node name:
## 1. configure node.name in emqx.conf
## 2. override with environment variable EMQX_NODE__NAME
## Node name is either short-name (without '@'), e.g. 'emqx'
## or long name (with '@') e.g. 'emqx@example.net' or 'emqx@127.0.0.1'
NAME="${EMQX_NODE__NAME:-}"
if [ -z "$NAME" ]; then
    if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
        # for boot commands, inspect emqx.conf for node name
        NAME="$(get_config_value node.name)"
    else
        vm_args_file="$(latest_vm_args 'EMQX_NODE__NAME')"
        NAME="$(grep -E '^-s?name' "${vm_args_file}" | awk '{print $2}')"
    fi
fi

# force to use 'emqx' short name
[ -z "$NAME" ] && NAME='emqx'
MNESIA_DATA_DIR="$RUNNER_DATA_DIR/mnesia/$NAME"

case "$NAME" in
    *@*)
        NAME_TYPE='-name'
       ;;
    *)
        NAME_TYPE='-sname'
esac
SHORT_NAME="$(echo "$NAME" | awk -F'@' '{print $1}')"
export ESCRIPT_NAME="$SHORT_NAME"

PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"

## make EMQX_NODE_COOKIE right
if [ -n "${EMQX_NODE_COOKIE:-}" ]; then
    export EMQX_NODE__COOKIE="${EMQX_NODE_COOKIE}"
    unset EMQX_NODE_COOKIE
fi
COOKIE="${EMQX_NODE__COOKIE:-}"
if [ -z "$COOKIE" ]; then
    if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
        COOKIE="$(get_config_value node.cookie)"
    else
        vm_args_file="$(latest_vm_args 'EMQX_NODE__COOKIE')"
        COOKIE="$(grep -E '^-setcookie' "${vm_args_file}" | awk '{print $2}')"
    fi
fi

if [ -z "$COOKIE" ]; then
    die "Please set node.cookie in $RUNNER_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE"
fi

cd "$ROOTDIR"

case "${COMMAND}" in
    start)
        # Make sure a node IS not running
        if relx_nodetool "ping" >/dev/null 2>&1; then
            die "node_is_already_running!"
        fi

        # this flag passes down to console mode
        # so we know it's intended to be run in daemon mode
        export _EMQX_START_MODE="$COMMAND"

        case "$COMMAND" in
            start)
                shift
                START_OPTION="console"
                HEART_OPTION="start"
                ;;
        esac
        RUN_PARAM="$*"

        # Set arguments for the heart command
        set -- "$RUNNER_SCRIPT" "$HEART_OPTION"
        [ "$RUN_PARAM" ] && set -- "$@" "$RUN_PARAM"

        # Export the HEART_COMMAND
        HEART_COMMAND="$RUNNER_SCRIPT $COMMAND"
        export HEART_COMMAND

        ## See: http://erlang.org/doc/man/run_erl.html
        # Export the RUN_ERL_LOG_GENERATIONS
        export RUN_ERL_LOG_GENERATIONS=${RUN_ERL_LOG_GENERATIONS:-"5"}

        # Export the RUN_ERL_LOG_MAXSIZE
        export RUN_ERL_LOG_MAXSIZE=${RUN_ERL_LOG_MAXSIZE:-"10485760"}

        mkdir -p "$PIPE_DIR"

        "$BINDIR/run_erl" -daemon "$PIPE_DIR" "$RUNNER_LOG_DIR" \
                          "$(relx_start_command)"

        WAIT_TIME=${EMQX_WAIT_FOR_START:-120}
        if wait_until_return_val "true" "$WAIT_TIME" 'relx_nodetool' \
                'eval' 'emqx:is_running()'; then
            echo "$EMQX_DESCRIPTION $REL_VSN is started successfully!"
            exit 0
        else
            echo "$EMQX_DESCRIPTION $REL_VSN failed to start in ${WAIT_TIME} seconds."
            echo "Please find more information in erlang.log.N"
            echo "Or run 'DEBUG=1 $0 console' to have logs printed to console."
            exit 1
        fi
        ;;

    stop)
        # Wait for the node to completely stop...
        PID="$(relx_get_pid)"
        if ! relx_nodetool "stop"; then
            echoerr "Graceful shutdown failed PID=[$PID]"
            exit 1
        fi
        WAIT_TIME="${EMQX_WAIT_FOR_STOP:-120}"
        if ! wait_for "$WAIT_TIME" 'is_down' "$PID"; then
            msg="dangling after ${WAIT_TIME} seconds"
            # also log to syslog
            logger -t "${REL_NAME}[${PID}]" "STOP: $msg"
            # log to user console
            echoerr "Stop failed, $msg"
            echo "ERROR: $PID is still around"
            ps -p "$PID"
            exit 1
        fi
        echo "ok"
        logger -t "${REL_NAME}[${PID}]" "STOP: OK"
        ;;

    pid)
        ## Get the VM's pid
        if ! relx_get_pid; then
            exit 1
        fi
        ;;

    ping)
        assert_node_alive
        echo pong
        ;;

    escript)
        ## Run an escript under the node's environment
        if ! relx_escript "$@"; then
            exit 1
        fi
        ;;

    attach)
        assert_node_alive

        shift
        exec "$BINDIR/to_erl" "$PIPE_DIR"
        ;;

    remote_console)
        assert_node_alive

        shift
        relx_rem_sh
        ;;

    upgrade|downgrade|install|unpack|uninstall)
        if [ -z "${2:-}" ]; then
            echo "Missing version argument"
            echo "Usage: $REL_NAME $COMMAND {version}"
            exit 1
        fi

        shift

        assert_node_alive

        ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARG" \
        exec "$BINDIR/escript" "$ROOTDIR/bin/install_upgrade.escript" \
             "$COMMAND" "{'$REL_NAME', \"$NAME_TYPE\", '$NAME', '$COOKIE'}" "$@"
        ;;

    versions)
        assert_node_alive

        shift

        ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARG" \
        exec "$BINDIR/escript" "$ROOTDIR/bin/install_upgrade.escript" \
             "versions" "{'$REL_NAME', \"$NAME_TYPE\", '$NAME', '$COOKIE'}" "$@"
        ;;

    console|console_clean)
        # .boot file typically just $REL_NAME (ie, the app name)
        # however, for debugging, sometimes start_clean.boot is useful.
        # For e.g. 'setup', one may even want to name another boot script.
        case "$COMMAND" in
            console)
                if [ -f "$REL_DIR/$REL_NAME.boot" ]; then
                  BOOTFILE="$REL_DIR/$REL_NAME"
                else
                  BOOTFILE="$REL_DIR/start"
                fi
                ;;
            console_clean)
                BOOTFILE="$REL_DIR/start_clean"
                ;;
        esac

        # set before generate_config
        if [ "${_EMQX_START_MODE:-}" = '' ]; then
            export EMQX_LOG__CONSOLE_HANDLER__ENABLE="${EMQX_LOG__CONSOLE_HANDLER__ENABLE:-true}"
        fi

        #generate app.config and vm.args
        generate_config "$NAME_TYPE" "$NAME"

        check_license

        # Setup beam-required vars
        EMU="beam"
        PROGNAME="${0#*/}"

        export EMU
        export PROGNAME

        # Store passed arguments since they will be erased by `set`
        ARGS="$*"

        # shellcheck disable=SC2086 # $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
        # Build an array of arguments to pass to exec later on
        # Build it here because this command will be used for logging.
        if [ "$IS_ELIXIR" = yes ]
        then
          set -- "$REL_DIR/iex" \
              --boot "$BOOTFILE" \
              --erl "-mode $CODE_LOADING_MODE" \
              --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
              --erl "-mnesia dir \"${MNESIA_DATA_DIR}\"" \
              --erl "$CONFIG_ARGS" \
              --erl "$EPMD_ARG" \
              --werl
        else
          set -- "$BINDIR/erlexec" \
              -boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \
              -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
              -mnesia dir "\"${MNESIA_DATA_DIR}\"" \
              $CONFIG_ARGS $EPMD_ARG
        fi

        # Log the startup
        logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS}"

        # Start the VM
        exec "$@" -- ${1+$ARGS}
        ;;

    foreground)
        # start up the release in the foreground for use by runit
        # or other supervision services

        # set before generate_config
        export EMQX_LOG__CONSOLE_HANDLER__ENABLE="${EMQX_LOG__CONSOLE_HANDLER__ENABLE:-true}"

        #generate app.config and vm.args
        generate_config "$NAME_TYPE" "$NAME"

        check_license

        [ -f "$REL_DIR/$REL_NAME.boot" ] && BOOTFILE="$REL_NAME" || BOOTFILE=start
        FOREGROUNDOPTIONS="-noshell -noinput +Bd"

        # Setup beam-required vars
        EMU=beam
        PROGNAME="${0#*/}"

        export EMU
        export PROGNAME

        # Store passed arguments since they will be erased by `set`
        ARGS="$*"

        # shellcheck disable=SC2086 # $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
        # Build an array of arguments to pass to exec later on
        # Build it here because this command will be used for logging.
        if [ "$IS_ELIXIR" = yes ]
        then
          set -- "$REL_DIR/elixir" \
              --boot "$REL_DIR/start" \
              --erl "$FOREGROUNDOPTIONS" \
              --erl "-mode $CODE_LOADING_MODE" \
              --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
              --boot-var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
              --erl "-mnesia dir \"${MNESIA_DATA_DIR}\"" \
              --erl "$CONFIG_ARGS" \
              --erl "$EPMD_ARG" \
              --no-halt
        else
          set -- "$BINDIR/erlexec" $FOREGROUNDOPTIONS \
              -boot "$REL_DIR/$BOOTFILE" -mode "$CODE_LOADING_MODE" \
              -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
              -mnesia dir "\"${MNESIA_DATA_DIR}\"" \
              $CONFIG_ARGS $EPMD_ARG
        fi

        # Log the startup
        logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS}"

        # Start the VM
        exec "$@" -- ${1+$ARGS}
        ;;
    ctl)
        assert_node_alive

        shift

        relx_nodetool rpc_infinity emqx_ctl run_command "$@"
        ;;
    rpc)
        assert_node_alive

        shift

        relx_nodetool rpc "$@"
        ;;
    rpcterms)
        assert_node_alive

        shift

        relx_nodetool rpcterms "$@"
        ;;
    root_dir)
        assert_node_alive

        shift
        relx_nodetool "eval" 'code:root_dir()'
        ;;
    eval)
        assert_node_alive

        shift
        if [ "$IS_ELIXIR" = "yes" ]
        then
          "$REL_DIR/elixir" \
              --hidden \
              --name "rand-$(relx_gen_id)-$NAME" \
              --cookie "$COOKIE" \
              --boot "$REL_DIR/start_clean" \
              --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
              --vm-args "$REL_DIR/remote.vm.args" \
              --erl "-start_epmd false -epmd_module ekka_epmd" \
              --rpc-eval "$NAME" "$@"
        else
          relx_nodetool "eval" "$@"
        fi
        ;;
    eval-erl)
        assert_node_alive

        shift
        relx_nodetool "eval" "$@"
        ;;
    *)
        usage "$COMMAND"
        exit 1
        ;;
esac

exit 0