Merge pull request #11734 from zmstone/1002-support-ipv6-only-clustering

Make IPv6 clustering options configurable

apps/emqx/rebar.config

@@ -28,8 +28,8 @@
     {gproc, {git, "https://github.com/emqx/gproc", {tag, "0.9.0.1"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.15"}}},
-    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.1"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}},
+    {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.0"}}},
     {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.39.16"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -195,7 +195,7 @@ fields("cluster") ->
             )},
         {"proto_dist",
             sc(
-                hoconsc:enum([inet_tcp, inet6_tcp, inet_tls]),
+                hoconsc:enum([inet_tcp, inet6_tcp, inet_tls, inet6_tls]),
                 #{
                     mapping => "ekka.proto_dist",
                     default => inet_tcp,
@@ -948,7 +948,26 @@ fields("rpc") ->
                 }
             )},
         {"ciphers", emqx_schema:ciphers_schema(tls_all_available)},
-        {"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)}
+        {"tls_versions", emqx_schema:tls_versions_schema(tls_all_available)},
+        {"listen_address",
+            sc(
+                string(),
+                #{
+                    default => "0.0.0.0",
+                    desc => ?DESC(rpc_listen_address),
+                    importance => ?IMPORTANCE_MEDIUM
+                }
+            )},
+        {"ipv6_only",
+            sc(
+                boolean(),
+                #{
+                    default => false,
+                    mapping => "gen_rpc.ipv6_only",
+                    desc => ?DESC(rpc_ipv6_only),
+                    importance => ?IMPORTANCE_LOW
+                }
+            )}
     ];
 fields("log") ->
     [
@@ -1133,7 +1152,16 @@ translation("gen_rpc") ->
     [
         {"default_client_driver", fun tr_default_config_driver/1},
         {"ssl_client_options", fun tr_gen_rpc_ssl_options/1},
-        {"ssl_server_options", fun tr_gen_rpc_ssl_options/1}
+        {"ssl_server_options", fun tr_gen_rpc_ssl_options/1},
+        {"socket_ip", fun(Conf) ->
+            Addr = conf_get("rpc.listen_address", Conf),
+            case inet:parse_address(Addr) of
+                {ok, Tuple} ->
+                    Tuple;
+                {error, _Reason} ->
+                    throw(#{bad_ip_address => Addr})
+            end
+        end}
     ];
 translation("prometheus") ->
     [

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.27"},
+    {vsn, "5.0.28"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl, uuid]},

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -80,14 +80,18 @@
     tanh/1
 ]).
 
-%% Bits Funcs
+%% Bitwise operations
 -export([
     bitnot/1,
     bitand/2,
     bitor/2,
     bitxor/2,
     bitsl/2,
-    bitsr/2,
+    bitsr/2
+]).
+
+%% binary and bitstring Funcs
+-export([
     bitsize/1,
     bytesize/1,
     subbits/2,

bin/emqx

@@ -522,17 +522,13 @@ else
         ## only one emqx node is running, get running args from 'ps -ef' output
         tmp_nodename=$(echo -e "$PS_LINE" | $GREP -oE "\s-s?name.*" | awk '{print $2}' || true)
         tmp_cookie=$(echo -e "$PS_LINE" | $GREP -oE "\s-setcookie.*" | awk '{print $2}' || true)
+        tmp_proto_dist=$(echo -e "$PS_LINE" | $GREP -oE '\s-ekka_proto_dist.*' | awk '{print $2}' || echo 'inet_tcp')
         SSL_DIST_OPTFILE="$(echo -e "$PS_LINE" | $GREP -oE '\-ssl_dist_optfile\s.+\s' | awk '{print $2}' || true)"
         tmp_ticktime="$(echo -e "$PS_LINE" | $GREP -oE '\s-kernel\snet_ticktime\s.+\s' | awk '{print $3}' || true)"
         # data_dir is actually not needed, but kept anyway
         tmp_datadir="$(echo -e "$PS_LINE" | $GREP -oE "\-emqx_data_dir.*" | sed -E 's#.+emqx_data_dir[[:blank:]]##g' | sed -E 's#[[:blank:]]--$##g' || true)"
-        if [ -z "$SSL_DIST_OPTFILE" ]; then
-            tmp_proto='inet_tcp'
-        else
-            tmp_proto='inet_tls'
-        fi
         ## Make the format like what call_hocon multi_get prints out, but only need 4 args
-        EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}"
+        EMQX_BOOT_CONFIGS="node.name=${tmp_nodename}\nnode.cookie=${tmp_cookie}\ncluster.proto_dist=${tmp_proto_dist}\nnode.dist_net_ticktime=$tmp_ticktime\nnode.data_dir=${tmp_datadir}"
     else
         if [ "$RUNNING_NODES_COUNT" -gt 1 ]; then
             if [ -z "${EMQX_NODE__NAME:-}" ]; then
@@ -567,7 +563,7 @@ TICKTIME="$(get_boot_config 'node.dist_net_ticktime' || echo '120')"
 # this environment variable is required by ekka_dist module
 # because proto_dist is overriden to ekka, and there is a lack of ekka_tls module
 export EKKA_PROTO_DIST_MOD="${PROTO_DIST:-inet_tcp}"
-if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ]; then
+if [ "$EKKA_PROTO_DIST_MOD" = 'inet_tls' ] || [ "$EKKA_PROTO_DIST_MOD" = 'inet6_tls' ]; then
     if [ "$IS_BOOT_COMMAND" = 'yes' ]; then
         SSL_DIST_OPTFILE=${EMQX_SSL_DIST_OPTFILE:-"$EMQX_ETC_DIR/ssl_dist.conf"}
         case "$SSL_DIST_OPTFILE" in
@@ -1216,7 +1212,6 @@ case "${COMMAND}" in
         export PROGNAME
 
         # Store passed arguments since they will be erased by `set`
-        # add emqx_data_dir to boot command so it is visible from 'ps -ef'
         ARGS="$*"
 
         # shellcheck disable=SC2086
@@ -1247,10 +1242,13 @@ case "${COMMAND}" in
         fi
 
         # Log the startup
-        logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -emqx_data_dir ${DATA_DIR}"
+        logger -t "${REL_NAME}[$$]" "EXEC: $* -- ${1+$ARGS} -ekka_proto_dist ${EKKA_PROTO_DIST_MOD} -emqx_data_dir ${DATA_DIR}"
 
         # Start the VM
-        exec "$@" -- ${1+$ARGS} -emqx_data_dir "${DATA_DIR}"
+        # add ekka_proto_dist emqx_data_dir to boot command so it is visible from 'ps -ef'
+        # NTOE: order matters! emqx_data_dir has to be positioned at the end of the line to simplify the
+        # line parsing when file path contains spaces
+        exec "$@" -- ${1+$ARGS} -ekka_proto_dist "${EKKA_PROTO_DIST_MOD}" -emqx_data_dir "${DATA_DIR}"
         ;;
 
     ctl)

build

@@ -407,10 +407,10 @@ make_docker() {
         PRODUCT_DESCRIPTION='Official docker image for EMQX Enterprise, an enterprise MQTT platform at scale. '
         DOCUMENTATION_URL='https://docs.emqx.com/en/enterprise/latest/'
     fi
-    # shellcheck disable=SC2155
-    local ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
-    # shellcheck disable=SC2155
-    local GIT_REVISION="$(git rev-parse HEAD)"
+    local ISO_8601_DATE GIT_REVISION
+    ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
+    GIT_REVISION="$(git rev-parse HEAD)"
+    export BUILDX_NO_DEFAULT_ATTESTATIONS=1
     local DOCKER_BUILDX_ARGS=(
        --build-arg BUILD_FROM="${EMQX_BUILDER}" \
        --build-arg RUN_FROM="${EMQX_RUNNER}" \
@@ -430,7 +430,6 @@ make_docker() {
        --label org.opencontainers.image.licenses="${LICENSE}" \
        --label org.opencontainers.image.otp.version="${EMQX_BUILDER_OTP}" \
        --tag "${EMQX_IMAGE_TAG}" \
-       --provenance false \
        --pull
     )
     if [ "${DOCKER_BUILD_NOCACHE:-false}" = true ]; then

changes/ce/fix-11734.en.md

@@ -0,0 +1,3 @@
+Fix clustering in IPv6 network.
+
+Added new configurations `rpc.listen_address` and `rpc.ipv6_only` to allow EMQX cluster RPC server and client to use IPv6.

mix.exs

@@ -55,8 +55,8 @@ defmodule EMQXUmbrella.MixProject do
       {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true},
       {:esockd, github: "emqx/esockd", tag: "5.9.7", override: true},
       {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true},
-      {:ekka, github: "emqx/ekka", tag: "0.15.15", override: true},
-      {:gen_rpc, github: "emqx/gen_rpc", tag: "3.1.1", override: true},
+      {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true},
+      {:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.0", override: true},
       {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.3.13", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true},

rebar.config

@@ -62,8 +62,8 @@
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.2"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.7"}}}
     , {rocksdb, {git, "https://github.com/emqx/erlang-rocksdb", {tag, "1.8.0-emqx-1"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.15"}}}
-    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.1.1"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.15.16"}}}
+    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.2.0"}}}
     , {grpc, {git, "https://github.com/emqx/grpc-erl", {tag, "0.6.8"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.13"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.4"}}}

rel/i18n/emqx_conf_schema.hocon

@@ -60,7 +60,9 @@ node_etc_dir.label:
 cluster_proto_dist.desc:
 """The Erlang distribution protocol for the cluster.<br/>
 - inet_tcp: IPv4 TCP <br/>
-- inet_tls: IPv4 TLS, works together with <code>etc/ssl_dist.conf</code>"""
+- inet_tls: IPv4 TLS, works together with <code>etc/ssl_dist.conf</code> <br/>
+- inet6_tcp: IPv6 TCP <br/>
+- inet6_tls: IPv6 TLS, works together with <code>etc/ssl_dist.conf</code>"""
 
 cluster_proto_dist.label:
 """Cluster Protocol Distribution"""
@@ -192,6 +194,20 @@ rpc_insecure_fallback.desc:
 rpc_insecure_fallback.label:
 """RPC insecure fallback"""
 
+rpc_listen_address.desc:
+"""Indicates the IP address for the RPC server to listen on. For example, use <code>"0.0.0.0"</code> for IPv4 or <code>"::"</code> for IPv6."""
+
+rpc_listen_address.label:
+"""RPC Listen IP Address"""
+
+rpc_ipv6_only.desc:
+"""This setting is effective only when <code>rpc.listen_address</code> is assigned an IPv6 address.
+If set to <code>true</code>, the RPC client will exclusively use IPv6 for connections.
+Otherwise, the client might opt for IPv4, even if the server is on IPv6."""
+
+rpc_ipv6_only.label:
+"""Use IPv6 Only"""
+
 cluster_mcast_buffer.desc:
 """Size of the user-level buffer."""
 

scripts/test/start-two-nodes-in-docker.sh

@@ -16,6 +16,7 @@ NET='emqx.io'
 NODE1="node1.$NET"
 NODE2="node2.$NET"
 COOKIE='this-is-a-secret'
+IPV6=0
 
 cleanup() {
     docker rm -f haproxy >/dev/null 2>&1 || true
@@ -24,31 +25,61 @@ cleanup() {
     docker network rm "$NET" >/dev/null 2>&1 || true
 }
 
-while getopts ":Pc" opt
+show_help() {
+    echo "Usage: $0 [options] EMQX_IMAGE1 [EMQX_IAMGE2]"
+    echo ""
+    echo "Specifiy which docker image to run with EMQX_IMAGE1"
+    echo "EMQX_IMAGE2 is the same as EMQX_IMAGE1 if not set"
+    echo ""
+    echo "Options:"
+    echo "  -h, --help  Show this help message and exit."
+    echo "  -P          Add -p options for docker run to expose more HAProxy container ports."
+    echo "  -6          Test with IPv6"
+    echo "  -c          Cleanup: delete docker network, force delete the containers."
+}
+
+while getopts "hc6P:" opt
 do
     case $opt in
         # -P option is treated similarly to docker run -P:
         # publish ports to random available host ports
         P) HAPROXY_PORTS=(-p 18083 -p 8883 -p 8084);;
         c) cleanup; exit 0;;
+        h) show_help; exit 0;;
+        6) IPV6=1;;
         *) ;;
     esac
 done
 shift $((OPTIND - 1))
 
-IMAGE1="${1}"
+IMAGE1="${1:-}"
 IMAGE2="${2:-${IMAGE1}}"
 
+if [ -z "${IMAGE1:-}" ] || [ -z "${IMAGE2:-}" ]; then
+    show_help
+    exit 1
+fi
+
 cleanup
 
-docker network create "$NET"
+if [ ${IPV6} = 1 ]; then
+    docker network create --ipv6 --subnet 2001:0DB8::/112 "$NET"
+    RPC_ADDRESS="::"
+    PROTO_DIST='inet6_tls'
+else
+    docker network create "$NET"
+    RPC_ADDRESS="0.0.0.0"
+    PROTO_DIST='inet_tls'
+fi
 
 docker run -d -t --restart=always --name "$NODE1" \
   --net "$NET" \
   -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \
   -e EMQX_NODE_NAME="emqx@$NODE1" \
   -e EMQX_NODE_COOKIE="$COOKIE" \
-  -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \
+  -e EMQX_CLUSTER__PROTO_DIST="${PROTO_DIST}" \
+  -e EMQX_RPC__LISTEN_ADDRESS="${RPC_ADDRESS}" \
+  -e EMQX_RPC__IPV6_ONLY="true" \
   -e EMQX_listeners__ssl__default__enable=false \
   -e EMQX_listeners__wss__default__enable=false \
   -e EMQX_listeners__tcp__default__proxy_protocol=true \
@@ -60,7 +91,9 @@ docker run -d -t --restart=always --name "$NODE2" \
   -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \
   -e EMQX_NODE_NAME="emqx@$NODE2" \
   -e EMQX_NODE_COOKIE="$COOKIE" \
-  -e EMQX_CLUSTER__PROTO_DIST='inet_tls' \
+  -e EMQX_CLUSTER__PROTO_DIST="${PROTO_DIST}" \
+  -e EMQX_RPC__LISTEN_ADDRESS="${RPC_ADDRESS}" \
+  -e EMQX_RPC__IPV6_ONLY="true" \
   -e EMQX_listeners__ssl__default__enable=false \
   -e EMQX_listeners__wss__default__enable=false \
   -e EMQX_listeners__tcp__default__proxy_protocol=true \