emqx4.3/bin/emqx

#!/usr/bin/env bash
# -*- tab-width:4;indent-tabs-mode:nil -*-
# ex: ts=4 sw=4 et

set -euo pipefail

DEBUG="${DEBUG:-0}"
if [ "$DEBUG" -eq 1 ]; then
    set -x
fi

ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
# shellcheck disable=SC1090,SC1091
. "$ROOT_DIR"/releases/emqx_vars

# defined in emqx_vars
export RUNNER_ROOT_DIR
export RUNNER_ETC_DIR
export REL_VSN

RUNNER_SCRIPT="$RUNNER_BIN_DIR/$REL_NAME"
CODE_LOADING_MODE="${CODE_LOADING_MODE:-embedded}"
REL_DIR="$RUNNER_ROOT_DIR/releases/$REL_VSN"
SCHEMA_MOD=emqx_conf_schema

WHOAMI=$(whoami)

# Make sure log directory exists
mkdir -p "$RUNNER_LOG_DIR"

# Make sure data directory exists
mkdir -p "$RUNNER_DATA_DIR"

# Make sure data/configs exists
CONFIGS_DIR="$RUNNER_DATA_DIR/configs"
mkdir -p "$CONFIGS_DIR"

# hocon try to read environment variables starting with "EMQX_"
export HOCON_ENV_OVERRIDE_PREFIX='EMQX_'

export ROOTDIR="$RUNNER_ROOT_DIR"
export ERTS_DIR="$ROOTDIR/erts-$ERTS_VSN"
export BINDIR="$ERTS_DIR/bin"
export EMU="beam"
export PROGNAME="erl"
export ERTS_LIB_DIR="$ERTS_DIR/../lib"
DYNLIBS_DIR="$RUNNER_ROOT_DIR/dynlibs"

# Echo to stderr on errors
echoerr() { echo "ERROR: $*" 1>&2; }

die() {
    echoerr "ERROR: $1"
    errno=${2:-1}
    exit "$errno"
}

assert_node_alive() {
    if ! relx_nodetool "ping" > /dev/null; then
        die "node_is_not_running!" 1
    fi
}


# Echo to stderr on errors
echoerr() { echo "$*" 1>&2; }

check_erlang_start() {
  # RELEASE_LIB is used by Elixir
  "$BINDIR/$PROGNAME" \
    -noshell \
    -boot_var RELEASE_LIB "$ERTS_LIB_DIR/lib" \
    -boot "$REL_DIR/start_clean" \
    -s crypto start \
    -s erlang halt
}

usage() {
    local command="$1"

    case "$command" in
    start)
        echo "Start EMQ X service in daemon mode"
        ;;
    stop)
        echo "Stop the running EMQ X program"
        ;;
    restart|reboot)
        echo "Restart $EMQX_DESCRIPTION"
        ;;
    pid)
        echo "Print out $EMQX_DESCRIPTION process identifier"
        ;;
    ping)
        echo "Check if the $EMQX_DESCRIPTION node is up and running"
        echo "This command exit with 0 silently if node is running"
        ;;
    escript)
        echo "Execute a escript using the Erlang runtime from EMQ X package installation"
        echo "For example $REL_NAME escript /path/to/my/escript my_arg1 my_arg2"
        ;;
    attach)
        echo "This command is applicable when $EMQX_DESCRIPTION is started in daemon"
        echo "mode. it attaches the current shell to EMQ X's control console"
        echo "through a named pipe"
        echo "WARNING: try to use the safer alternative, remote_console command."
        ;;
    remote_console)
        echo "Start a dummy Erlang node and hidden-connect $EMQX_DESCRIPTION to"
        echo "with an interactive Erlang shell"
        ;;
    console)
        echo "Boot up $EMQX_DESCRIPTION service in an interactive Erlang shell"
        echo "This command is useful for troubleshooting"
        ;;
    console_clean)
        echo "This command does NOT boot up the $EMQX_DESCRIPTION service"
        echo "It only starts an interactive Erlang console with all the"
        echo "EMQ X code available"
        ;;
    foreground)
        echo "Start $EMQX_DESCRIPTION in foreground mode"
        ;;
    ertspath)
        echo "Print path to Erlang runtime dir"
        ;;
    rpc)
        echo "Usge $REL_NAME rpc MODULE FUNCTION [ARGS, ...]"
        echo "Connect to the $EMQX_DESCRIPTION node and make an Erlang RPC"
        echo "The result of the RPC call must be 'ok'"
        echo "This command blocks for at most 60 seconds in case the node"
        echo "does not reply the call in time"
        ;;
    rpcterms)
        echo "Usge $REL_NAME rpcterms MODULE FUNCTION [ARGS, ...]"
        echo "Connect to the $EMQX_DESCRIPTION node and make an Erlang RPC"
        echo "The result of the RPC call is pretty-printed as an Erlang term"
        ;;
    root_dir)
        echo "Print EMQ X installation root dir"
        ;;
    eval)
        echo "Evaluate an Erlang expression in the EMQ X node"
        ;;
    versions)
        echo "List installed EMQ X versions and their status"
        ;;
    unpack)
        echo "Usage: $REL_NAME unpack [VERSION]"
        echo "Unpacks a release package VERSION, it assumes that this"
        echo "release package tarball has already been deployed at one"
        echo "of the following locations:"
        echo "      releases/<relname>-<version>.tar.gz"
        echo "      releases/<relname>-<version>.zip"
        ;;
    install)
        echo "Usage: $REL_NAME install [VERSION]"
        echo "Installs a release package VERSION, it assumes that this"
        echo "release package tarball has already been deployed at one"
        echo "of the following locations:"
        echo "      releases/<relname>-<version>.tar.gz"
        echo "      releases/<relname>-<version>.zip"
        echo ""
        echo "     --no-permanent   Install release package VERSION but"
        echo "                      don't make it permanent"
        ;;
    uninstall)
        echo "Usage: $REL_NAME uninstall [VERSION]"
        echo "Uninstalls a release VERSION, it will only accept"
        echo "versions that are not currently in use"
        ;;
    upgrade)
        echo "Usage: $REL_NAME upgrade [VERSION]"
        echo "Upgrades the currently running release to VERSION, it assumes"
        echo "that a release package tarball has already been deployed at one"
        echo "of the following locations:"
        echo "      releases/<relname>-<version>.tar.gz"
        echo "      releases/<relname>-<version>.zip"
        echo ""
        echo "     --no-permanent   Install release package VERSION but"
        echo "                      don't make it permanent"
        ;;
    downgrade)
        echo "Usage: $REL_NAME downgrade [VERSION]"
        echo "Downgrades the currently running release to VERSION, it assumes"
        echo "that a release package tarball has already been deployed at one"
        echo "of the following locations:"
        echo "      releases/<relname>-<version>.tar.gz"
        echo "      releases/<relname>-<version>.zip"
        echo ""
        echo "     --no-permanent   Install release package VERSION but"
        echo "                      don't make it permanent"
        ;;
    *)
        echo "Usage: $REL_NAME {start|ertspath|foreground|stop|pid|ping|console|console_clean|attach|remote_console|upgrade|downgrade|install|uninstall|versions|escript|ctl|rpc|rpcterms|eval|root_dir} <help>"
    ;;
    esac
}

COMMAND="${1:-}"

if [ "${2:-}" = 'help' ]; then
    ## 'ctl' command has its own usage info
    if [ "$COMMAND" != 'ctl' ]; then
        usage "$COMMAND"
        exit 0
    fi
fi

if ! check_erlang_start >/dev/null 2>&1; then
    BUILT_ON="$(head -1 "${REL_DIR}/BUILT_ON")"
    ## failed to start, might be due to missing libs, try to be portable
    export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-$DYNLIBS_DIR}"
    if [ "$LD_LIBRARY_PATH" != "$DYNLIBS_DIR" ]; then
        export LD_LIBRARY_PATH="$DYNLIBS_DIR:$LD_LIBRARY_PATH"
    fi
    if ! check_erlang_start; then
        ## it's hopeless
        echoerr "FATAL: Unable to start Erlang."
        echoerr "Please make sure openssl-1.1.1 (libcrypto) and libncurses are installed."
        echoerr "Also ensure it's running on the correct platform,"
        echoerr "this EMQ X release is built for $BUILT_ON"
        exit 1
    fi
    echoerr "WARNING: There seem to be missing dynamic libs from the OS. Using libs from ${DYNLIBS_DIR}"
fi

## backward compatible
if [ -d "$ERTS_DIR/lib" ]; then
    export LD_LIBRARY_PATH="$ERTS_DIR/lib:$LD_LIBRARY_PATH"
fi

# Simple way to check the correct user and fail early
check_user() {
    # Validate that the user running the script is the owner of the
    # RUN_DIR.
    if [ "$RUNNER_USER" ] && [ "x$WHOAMI" != "x$RUNNER_USER" ]; then
        if [ "x$WHOAMI" != "xroot" ]; then
            echo "You need to be root or use sudo to run this command"
            exit 1
        fi
        CMD="DEBUG=$DEBUG \"$RUNNER_SCRIPT\" "
        for ARG in "$@"; do
            CMD="${CMD} \"$ARG\""
        done
        # This will drop priviledges into the runner user
        # It exec's in a new shell and the current shell will exit
        exec su - "$RUNNER_USER" -c "$CMD"
    fi
}

# Make sure the user running this script is the owner and/or su to that user
check_user "$@"
ES=$?
if [ "$ES" -ne 0 ]; then
    exit $ES
fi

# EPMD_ARG="-start_epmd true $PROTO_DIST_ARG"
NO_EPMD="-start_epmd false -epmd_module ekka_epmd -proto_dist ekka"
EPMD_ARG="${EPMD_ARG:-${NO_EPMD}}"

# Warn the user if ulimit -n is less than 1024
ULIMIT_F=$(ulimit -n)
if [ "$ULIMIT_F" -lt 1024 ]; then
    echo "!!!!"
    echo "!!!! WARNING: ulimit -n is ${ULIMIT_F}; 1024 is the recommended minimum."
    echo "!!!!"
fi

SED_REPLACE="sed -i "
case $(sed --help 2>&1) in
    *GNU*) SED_REPLACE="sed -i ";;
    *BusyBox*) SED_REPLACE="sed -i ";;
    *) SED_REPLACE="sed -i '' ";;
esac

# Get node pid
relx_get_pid() {
    if output="$(relx_nodetool rpcterms os getpid)"
    then
        # shellcheck disable=SC2001 # Escaped quote taken as closing quote in editor
        echo "$output" | sed -e 's/"//g'
        return 0
    else
        echo "$output"
        return 1
    fi
}

# Connect to a remote node
relx_rem_sh() {
    # Generate a unique id used to allow multiple remsh to the same node
    # transparently
    id="remsh$(relx_gen_id)-${NAME}"
    # Get the node's ticktime so that we use the same thing.
    TICKTIME="$(relx_nodetool rpcterms net_kernel get_net_ticktime)"

    # shellcheck disable=SC2086 # $EPMD_ARG is supposed to be split by whitespace
    # Setup remote shell command to control node
    if [ "$IS_ELIXIR" = "yes" ]
    then
      exec "$REL_DIR/iex" \
           --remsh "$NAME" \
           --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
           --cookie "$COOKIE" \
           --hidden \
           --erl "-kernel net_ticktime $TICKTIME" \
           --erl "$EPMD_ARG" \
           --erl "$NAME_TYPE $id" \
           --boot "$REL_DIR/start_clean"
    else
      exec "$BINDIR/erl" "$NAME_TYPE" "$id" \
           -remsh "$NAME" -boot "$REL_DIR/start_clean" \
           -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
           -setcookie "$COOKIE" -hidden -kernel net_ticktime "$TICKTIME" \
           $EPMD_ARG
    fi
}

# Generate a random id
relx_gen_id() {
    od -t x -N 4 /dev/urandom | head -n1 | awk '{print $2}'
}

# Control a node
relx_nodetool() {
    command="$1"; shift
    ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARG" \
    "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" "$NAME_TYPE" "$NAME" \
                                -setcookie "$COOKIE" "$command" "$@"
}

call_hocon() {
    "$ERTS_DIR/bin/escript" "$ROOTDIR/bin/nodetool" hocon "$@" \
        || die "call_hocon_failed: $*" $?
}

# Run an escript in the node's environment
relx_escript() {
    shift; scriptpath="$1"; shift
    "$ERTS_DIR/bin/escript" "$ROOTDIR/$scriptpath" "$@"
}

# Output a start command for the last argument of run_erl
relx_start_command() {
    printf "exec \"%s\" \"%s\"" "$RUNNER_SCRIPT" \
           "$START_OPTION"
}

# Function to generate app.config and vm.args
generate_config() {
    local name_type="$1"
    local node_name="$2"
    ## Delete the *.siz files first or it cann't start after
    ## changing the config 'log.rotation.size'
    rm -rf "${RUNNER_LOG_DIR}"/*.siz

    EMQX_LICENSE_CONF_OPTION=""
    if [ "${EMQX_LICENSE_CONF:-}" != "" ]; then
        EMQX_LICENSE_CONF_OPTION="-c ${EMQX_LICENSE_CONF}"
    fi

    ## timestamp for each generation
    local NOW_TIME
    NOW_TIME="$(call_hocon now_time)"

    ## ths command populates two files: app.<time>.config and vm.<time>.args
    ## NOTE: the generate command merges environment variables to the base config (emqx.conf),
    ## but does not include the cluster-override.conf and local-override.conf
    ## meaning, certain overrides will not be mapped to app.<time>.config file
    ## disable SC2086 to allow EMQX_LICENSE_CONF_OPTION to split
    # shellcheck disable=SC2086
    call_hocon -v -t "$NOW_TIME" -I "$CONFIGS_DIR/" -s $SCHEMA_MOD -c "$RUNNER_ETC_DIR"/emqx.conf $EMQX_LICENSE_CONF_OPTION -d "$RUNNER_DATA_DIR"/configs generate

    ## filenames are per-hocon convention
    local CONF_FILE="$CONFIGS_DIR/app.$NOW_TIME.config"
    local HOCON_GEN_ARG_FILE="$CONFIGS_DIR/vm.$NOW_TIME.args"

    # This is needed by the Elixir scripts.
    # Do NOT append `.config`.
    RELEASE_SYS_CONFIG="$CONFIGS_DIR/app.$NOW_TIME"
    export RELEASE_SYS_CONFIG

    CONFIG_ARGS="-config $CONF_FILE -args_file $HOCON_GEN_ARG_FILE"

    ## Merge hocon generated *.args into the vm.args
    TMP_ARG_FILE="$CONFIGS_DIR/vm.args.tmp"
    cp "$RUNNER_ETC_DIR/vm.args" "$TMP_ARG_FILE"
    echo "" >> "$TMP_ARG_FILE"
    echo "-pa ${REL_DIR}/consolidated" >> "$TMP_ARG_FILE"
    ## read lines from generated vm.<time>.args file
    ## drop comment lines, and empty lines using sed
    ## pipe the lines to a while loop
    sed '/^#/d' "$HOCON_GEN_ARG_FILE" | sed '/^$/d' | while IFS='' read -r ARG_LINE || [ -n "$ARG_LINE" ]; do
        ## in the loop, split the 'key[:space:]value' pair
        ARG_KEY=$(echo "$ARG_LINE" | awk '{$NF="";print}')
        ARG_VALUE=$(echo "$ARG_LINE" | awk '{print $NF}')
        ## use the key to look up in vm.args file for the value
        TMP_ARG_VALUE=$(grep "^$ARG_KEY" "$TMP_ARG_FILE" || true | awk '{print $NF}')
        ## compare generated (to override) value to original (to be overriden) value
        if [ "$ARG_VALUE" != "$TMP_ARG_VALUE" ] ; then
            ## if they are different
            if [ -n "$TMP_ARG_VALUE" ]; then
                ## if the old value is present, replace it with generated value
                sh -c "$SED_REPLACE 's|^$ARG_KEY.*$|$ARG_LINE|' $TMP_ARG_FILE"
            else
                ## otherwise append generated value to the end
                echo "$ARG_LINE" >> "$TMP_ARG_FILE"
            fi
        fi
    done
    echo "$name_type $node_name" >> "$TMP_ARG_FILE"
    ## rename the generated vm.<time>.args file
    mv -f "$TMP_ARG_FILE" "$HOCON_GEN_ARG_FILE"

    # shellcheck disable=SC2086
    if ! relx_nodetool chkconfig $CONFIG_ARGS; then
        die "failed_to_check_config $CONFIG_ARGS"
    fi
}

# check if a PID is down
is_down() {
    PID="$1"
    if ps -p "$PID" >/dev/null; then
        # still around
        # shellcheck disable=SC2009 # this grep pattern is not a part of the progra names
        if ps -p "$PID" | grep -q 'defunct'; then
            # zombie state, print parent pid
            parent="$(ps -o ppid= -p "$PID" | tr -d ' ')"
            echo "WARN: $PID is marked <defunct>, parent:"
            ps -p "$parent"
            return 0
        fi
        return 1
    fi
    # it's gone
    return 0
}

wait_for() {
    local WAIT_TIME
    local CMD
    WAIT_TIME="$1"
    shift
    CMD="$*"
    while true; do
        if $CMD >/dev/null 2>&1; then
            return 0
        fi
        if [ "$WAIT_TIME" -le 0 ]; then
            return 1
        fi
        WAIT_TIME=$((WAIT_TIME - 1))
        sleep 1
    done
}

wait_until_return_val() {
    local RESULT
    local WAIT_TIME
    local CMD
    RESULT="$1"
    WAIT_TIME="$2"
    shift 2
    CMD="$*"
    while true; do
        if [ "$($CMD 2>/dev/null)" = "$RESULT" ]; then
            return 0
        fi
        if [ "$WAIT_TIME" -le 0 ]; then
            return 1
        fi
        WAIT_TIME=$((WAIT_TIME - 1))
        sleep 1
    done
}

latest_vm_args() {
    local hint_var_name="$1"
    local vm_args_file
    vm_args_file="$(find "$CONFIGS_DIR" -type f -name "vm.*.args" | sort | tail -1)"
    if [ -f "$vm_args_file" ]; then
        echo "$vm_args_file"
    else
        echoerr "ERRRO: node not initialized?"
        echoerr "Generated config file vm.*.args is not found for command '$COMMAND'"
        echoerr "in config dir: $CONFIGS_DIR"
        echoerr "In case the file has been deleted while the node is running,"
        echoerr "set environment variable '$hint_var_name' to continue"
        exit 1
    fi
}

## IS_BOOT_COMMAND is set for later to inspect node name and cookie from hocon config (or env variable)
case "${COMMAND}" in
    start|console|console_clean|foreground)
        IS_BOOT_COMMAND='yes'
        ;;
    *)
        IS_BOOT_COMMAND='no'
        ;;
esac

## make EMQX_NODE_COOKIE right
if [ -n "${EMQX_NODE_NAME:-}" ]; then
    export EMQX_NODE__NAME="${EMQX_NODE_NAME}"
    unset EMQX_NODE_NAME
fi
## Possible ways to configure emqx node name:
## 1. configure node.name in emqx.conf
## 2. override with environment variable EMQX_NODE__NAME
## Node name is either short-name (without '@'), e.g. 'emqx'
## or long name (with '@') e.g. 'emqx@example.net' or 'emqx@127.0.0.1'
NAME="${EMQX_NODE__NAME:-}"
if [ -z "$NAME" ]; then
    if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
        # for boot commands, inspect emqx.conf for node name
        NAME="$(call_hocon -s $SCHEMA_MOD -I "$CONFIGS_DIR/" -c "$RUNNER_ETC_DIR"/emqx.conf get node.name | tr -d \")"
    else
        vm_args_file="$(latest_vm_args 'EMQX_NODE__NAME')"
        NAME="$(grep -E '^-s?name' "${vm_args_file}" | awk '{print $2}')"
    fi
fi

# force to use 'emqx' short name
[ -z "$NAME" ] && NAME='emqx'
MNESIA_DATA_DIR="$RUNNER_DATA_DIR/mnesia/$NAME"

case "$NAME" in
    *@*)
        NAME_TYPE='-name'
       ;;
    *)
        NAME_TYPE='-sname'
esac
SHORT_NAME="$(echo "$NAME" | awk -F'@' '{print $1}')"
export ESCRIPT_NAME="$SHORT_NAME"

PIPE_DIR="${PIPE_DIR:-/$RUNNER_DATA_DIR/${WHOAMI}_erl_pipes/$NAME/}"

## make EMQX_NODE_COOKIE right
if [ -n "${EMQX_NODE_COOKIE:-}" ]; then
    export EMQX_NODE__COOKIE="${EMQX_NODE_COOKIE}"
    unset EMQX_NODE_COOKIE
fi
COOKIE="${EMQX_NODE__COOKIE:-}"
if [ -z "$COOKIE" ]; then
    if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
        COOKIE="$(call_hocon -s $SCHEMA_MOD -I "$CONFIGS_DIR/" -c "$RUNNER_ETC_DIR"/emqx.conf get node.cookie | tr -d \")"
    else
        vm_args_file="$(latest_vm_args 'EMQX_NODE__COOKIE')"
        COOKIE="$(grep -E '^-setcookie' "${vm_args_file}" | awk '{print $2}')"
    fi
fi

if [ -z "$COOKIE" ]; then
    die "Please set node.cookie in $RUNNER_ETC_DIR/emqx.conf or override from environment variable EMQX_NODE__COOKIE"
fi

cd "$ROOTDIR"

case "${COMMAND}" in
    start)
        # Make sure a node IS not running
        if relx_nodetool "ping" >/dev/null 2>&1; then
            die "node_is_already_running!"
        fi

        # this flag passes down to console mode
        # so we know it's intended to be run in daemon mode
        export _EMQX_START_MODE="$COMMAND"

        case "$COMMAND" in
            start)
                shift
                START_OPTION="console"
                HEART_OPTION="start"
                ;;
        esac
        RUN_PARAM="$*"

        # Set arguments for the heart command
        set -- "$RUNNER_SCRIPT" "$HEART_OPTION"
        [ "$RUN_PARAM" ] && set -- "$@" "$RUN_PARAM"

        # Export the HEART_COMMAND
        HEART_COMMAND="$RUNNER_SCRIPT $COMMAND"
        export HEART_COMMAND

        ## See: http://erlang.org/doc/man/run_erl.html
        # Export the RUN_ERL_LOG_GENERATIONS
        export RUN_ERL_LOG_GENERATIONS=${RUN_ERL_LOG_GENERATIONS:-"5"}

        # Export the RUN_ERL_LOG_MAXSIZE
        export RUN_ERL_LOG_MAXSIZE=${RUN_ERL_LOG_MAXSIZE:-"10485760"}

        mkdir -p "$PIPE_DIR"

        "$BINDIR/run_erl" -daemon "$PIPE_DIR" "$RUNNER_LOG_DIR" \
                          "$(relx_start_command)"

        WAIT_TIME=${EMQX_WAIT_FOR_START:-120}
        if wait_until_return_val "true" "$WAIT_TIME" 'relx_nodetool' \
                'eval' 'emqx:is_running()'; then
            echo "$EMQX_DESCRIPTION $REL_VSN is started successfully!"
            exit 0
        else
            echo "$EMQX_DESCRIPTION $REL_VSN failed to start in ${WAIT_TIME} seconds."
            echo "Please find more information in erlang.log.N"
            echo "Or run 'DEBUG=1 $0 console' to have logs printed to console."
            exit 1
        fi
        ;;

    stop)
        # Wait for the node to completely stop...
        PID="$(relx_get_pid)"
        if ! relx_nodetool "stop"; then
            echoerr "Graceful shutdown failed PID=[$PID]"
            exit 1
        fi
        WAIT_TIME="${EMQX_WAIT_FOR_STOP:-120}"
        if ! wait_for "$WAIT_TIME" 'is_down' "$PID"; then
            msg="dangling after ${WAIT_TIME} seconds"
            # also log to syslog
            logger -t "${REL_NAME}[${PID}]" "STOP: $msg"
            # log to user console
            echoerr "Stop failed, $msg"
            echo "ERROR: $PID is still around"
            ps -p "$PID"
            exit 1
        fi
        echo "ok"
        logger -t "${REL_NAME}[${PID}]" "STOP: OK"
        ;;

    restart|reboot)
        echo "$EMQX_DESCRIPTION $REL_VSN is stopped: $("$RUNNER_BIN_DIR/$REL_NAME" stop)"
        "$RUNNER_BIN_DIR/$REL_NAME" start
        ;;

    pid)
        ## Get the VM's pid
        if ! relx_get_pid; then
            exit 1
        fi
        ;;

    ping)
        assert_node_alive
        echo pong
        ;;

    escript)
        ## Run an escript under the node's environment
        if ! relx_escript "$@"; then
            exit 1
        fi
        ;;

    attach)
        assert_node_alive

        shift
        exec "$BINDIR/to_erl" "$PIPE_DIR"
        ;;

    remote_console)
        assert_node_alive

        shift
        relx_rem_sh
        ;;

    upgrade|downgrade|install|unpack|uninstall)
        if [ -z "${2:-}" ]; then
            echo "Missing version argument"
            echo "Usage: $REL_NAME $COMMAND {version}"
            exit 1
        fi

        shift

        assert_node_alive

        ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARG" \
        exec "$BINDIR/escript" "$ROOTDIR/bin/install_upgrade.escript" \
             "$COMMAND" "{'$REL_NAME', \"$NAME_TYPE\", '$NAME', '$COOKIE'}" "$@"
        ;;

    versions)
        assert_node_alive

        shift

        ERL_FLAGS="${ERL_FLAGS:-} $EPMD_ARG" \
        exec "$BINDIR/escript" "$ROOTDIR/bin/install_upgrade.escript" \
             "versions" "{'$REL_NAME', \"$NAME_TYPE\", '$NAME', '$COOKIE'}" "$@"
        ;;

    console|console_clean)
        # .boot file typically just $REL_NAME (ie, the app name)
        # however, for debugging, sometimes start_clean.boot is useful.
        # For e.g. 'setup', one may even want to name another boot script.
        case "$COMMAND" in
            console)
                if [ -f "$REL_DIR/$REL_NAME.boot" ]; then
                  BOOTFILE="$REL_DIR/$REL_NAME"
                else
                  BOOTFILE="$REL_DIR/start"
                fi
                ;;
            console_clean)
                BOOTFILE="$REL_DIR/start_clean"
                ;;
        esac

        # set before generate_config
        if [ "${_EMQX_START_MODE:-}" = '' ]; then
            export EMQX_LOG__CONSOLE_HANDLER__ENABLE="${EMQX_LOG__CONSOLE_HANDLER__ENABLE:-true}"
        fi

        #generate app.config and vm.args
        generate_config "$NAME_TYPE" "$NAME"

        # Setup beam-required vars
        EMU="beam"
        PROGNAME="${0#*/}"

        export EMU
        export PROGNAME

        # Store passed arguments since they will be erased by `set`
        ARGS="$*"

        # shellcheck disable=SC2086 # $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
        # Build an array of arguments to pass to exec later on
        # Build it here because this command will be used for logging.
        if [ "$IS_ELIXIR" = yes ]
        then
          set -- "$REL_DIR/iex" \
              --boot "$BOOTFILE" \
              --erl "-mode $CODE_LOADING_MODE" \
              --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
              --erl "-mnesia dir \"${MNESIA_DATA_DIR}\"" \
              --erl "$CONFIG_ARGS" \
              --erl "$EPMD_ARG" \
              --werl
        else
          set -- "$BINDIR/erlexec" \
              -boot "$BOOTFILE" -mode "$CODE_LOADING_MODE" \
              -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
              -mnesia dir "\"${MNESIA_DATA_DIR}\"" \
              $CONFIG_ARGS $EPMD_ARG
        fi

        # Log the startup
        logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS}"

        # Start the VM
        exec "$@" -- ${1+$ARGS}
        ;;

    foreground)
        # start up the release in the foreground for use by runit
        # or other supervision services

        # set before generate_config
        export EMQX_LOG__CONSOLE_HANDLER__ENABLE="${EMQX_LOG__CONSOLE_HANDLER__ENABLE:-true}"

        #generate app.config and vm.args
        generate_config "$NAME_TYPE" "$NAME"

        [ -f "$REL_DIR/$REL_NAME.boot" ] && BOOTFILE="$REL_NAME" || BOOTFILE=start
        FOREGROUNDOPTIONS="-noshell -noinput +Bd"

        # Setup beam-required vars
        EMU=beam
        PROGNAME="${0#*/}"

        export EMU
        export PROGNAME

        # Store passed arguments since they will be erased by `set`
        ARGS="$*"

        # shellcheck disable=SC2086 # $CONFIG_ARGS $EPMD_ARG are supposed to be split by whitespace
        # Build an array of arguments to pass to exec later on
        # Build it here because this command will be used for logging.
        if [ "$IS_ELIXIR" = yes ]
        then
          set -- "$REL_DIR/elixir" \
              --boot "$REL_DIR/start" \
              --erl "$FOREGROUNDOPTIONS" \
              --erl "-mode $CODE_LOADING_MODE" \
              --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
              --boot-var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
              --erl "-mnesia dir \"${MNESIA_DATA_DIR}\"" \
              --erl "$CONFIG_ARGS" \
              --erl "$EPMD_ARG" \
              --no-halt
        else
          set -- "$BINDIR/erlexec" $FOREGROUNDOPTIONS \
              -boot "$REL_DIR/$BOOTFILE" -mode "$CODE_LOADING_MODE" \
              -boot_var ERTS_LIB_DIR "$ERTS_LIB_DIR" \
              -mnesia dir "\"${MNESIA_DATA_DIR}\"" \
              $CONFIG_ARGS $EPMD_ARG
        fi

        # Log the startup
        logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS}"

        # Start the VM
        exec "$@" -- ${1+$ARGS}
        ;;
    ertspath)
        echo "$ERTS_PATH"
        ;;
    ctl)
        assert_node_alive

        shift

        relx_nodetool rpc_infinity emqx_ctl run_command "$@"
        ;;
    rpc)
        assert_node_alive

        shift

        relx_nodetool rpc "$@"
        ;;
    rpcterms)
        assert_node_alive

        shift

        relx_nodetool rpcterms "$@"
        ;;
    root_dir)
        assert_node_alive

        shift
        relx_nodetool "eval" 'code:root_dir()'
        ;;
    eval)
        assert_node_alive

        shift
        if [ "$IS_ELIXIR" = "yes" ]
        then
          "$REL_DIR/elixir" \
              --hidden \
              --cookie "$COOKIE" \
              --boot "$REL_DIR/start_clean" \
              --boot-var RELEASE_LIB "$ERTS_LIB_DIR" \
              --vm-args "$(latest_vm_args 'EMQX_NODE__NAME')"\
              --rpc-eval "$NAME" "$@"
        else
          relx_nodetool "eval" "$@"
        fi
        ;;
    *)
        usage "$COMMAND"
        exit 1
        ;;
esac

exit 0