Merge pull request #8229 from zmstone/0615-delete-appname-for-k8s-and-dns-discovery

Delete app name config for k8s and dns discovery

apps/emqx/rebar.config

@@ -27,7 +27,7 @@
     {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}},
     {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}},
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}},
-    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.9"}}},
+    {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.0"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}},
     {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.28.2"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},

apps/emqx_conf/i18n/emqx_conf_schema.conf

@@ -245,11 +245,12 @@ Note: probe messages are broadcast to all the specified ports.
 
   cluster_dns_name {
     desc {
-      en: """The domain name of the EMQX cluster."""
-      zh: """指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表，
-然后拼接 cluster.dns.app 里指定的 APP 名得到集群中所有节点的列表。
-当 cluster.discovery_strategy 为 dns 时，此配置项才有效。
-          """
+      en: """The domain name from which to discover peer EMQX nodes' IP addresses.
+Applicable when <code>cluster.discovery_strategy = dns</code>
+"""
+      zh: """指定 DNS A 记录的名字。emqx 会通过访问这个 DNS A 记录来获取 IP 地址列表。
+当<code>cluster.discovery_strategy</code> 为 <code>dns</code> 时有效。
+"""
     }
     label {
       en: "Cluster Dns Name"
@@ -257,19 +258,6 @@ Note: probe messages are broadcast to all the specified ports.
     }
   }
 
-  cluster_dns_app {
-    desc {
-      en: """The symbolic name of the EMQX service."""
-      zh: """用来与从 cluster.dns.name 获取的 IP 列表拼接得到节点名列表。
-当 cluster.discovery_strategy 为 dns 时，此配置项才有效。
-         """
-    }
-    label {
-      en: "Cluster DNS App"
-      zh: "DNS前缀标识"
-    }
-  }
-
   cluster_etcd_server {
     desc {
       en: """List of endpoint URLs of the etcd cluster"""
@@ -353,17 +341,14 @@ It is refreshed automatically, as long as the node is alive.
 
   cluster_k8s_address_type {
     desc {
-      en: """Address type used for connecting to the discovered nodes."""
+      en: """Address type used for connecting to the discovered nodes.
+Setting <code>cluster.k8s.address_type</code> to <code>ip</code> will
+make EMQX to discover IP addresses of peer nodes from Kubernetes API.
+"""
       zh: """当使用 k8s 方式集群时，address_type 用来从 Kubernetes 接口的应答里获取什么形式的 Host 列表。
-指定 cluster.k8s.address_type 为 ip，则将从 Kubernetes 接口中获取 emqx 服务的 IP 地址列表:
-- 172.16.122.31 </br>
-- 172.16.122.32 </br>
-- 172.16.122.33 </br>
-然后与 cluster.k8s.app_name 配置指定的 app name 拼接，得到 emqx 节点列表: </br>
-- emqx@172.16.122.31
-- emqx@172.16.122.32
-- emqx@172.16.122.33
-          """
+指定 <code>cluster.k8s.address_type</code. 为 <code>ip</code>，则将从 Kubernetes 接口中获取集群中其他节点
+的IP地址。
+"""
     }
     label {
       en: "K8s Address Type"
@@ -371,22 +356,6 @@ It is refreshed automatically, as long as the node is alive.
     }
   }
 
-  cluster_k8s_app_name {
-    desc {
-      en: """This parameter should be set to the part of the <code>node.name</code>
-before the '@'.</br>
-For example, if the <code>node.name</code> is <code>emqx@127.0.0.1</code>, then this parameter
-should be set to <code>emqx</code>."""
-      zh: """app_name 用来跟获取的 Host 列表拼接，得到节点列表。</br>
-当 cluster.discovery_strategy 为 k8s 时，此配置项才有效。
-          """
-    }
-    label {
-      en: "K8s App Name"
-      zh: "K8s 节点名前缀"
-    }
-  }
-
   cluster_k8s_namespace {
     desc {
       en: """Kubernetes namespace."""

apps/emqx_conf/src/emqx_conf_schema.erl

@@ -287,15 +287,6 @@ fields(cluster_dns) ->
                     desc => ?DESC(cluster_dns_name),
                     'readOnly' => true
                 }
-            )},
-        {"app",
-            sc(
-                string(),
-                #{
-                    default => "emqx",
-                    desc => ?DESC(cluster_dns_app),
-                    'readOnly' => true
-                }
             )}
     ];
 fields(cluster_etcd) ->
@@ -364,15 +355,6 @@ fields(cluster_k8s) ->
                     'readOnly' => true
                 }
             )},
-        {"app_name",
-            sc(
-                string(),
-                #{
-                    default => "emqx",
-                    'readOnly' => true,
-                    desc => ?DESC(cluster_k8s_app_name)
-                }
-            )},
         {"namespace",
             sc(
                 string(),
@@ -1300,8 +1282,7 @@ options(mcast, Conf) ->
     ];
 options(dns, Conf) ->
     [
-        {name, conf_get("cluster.dns.name", Conf)},
-        {app, conf_get("cluster.dns.app", Conf)}
+        {name, conf_get("cluster.dns.name", Conf)}
     ];
 options(etcd, Conf) ->
     Namespace = "cluster.etcd.ssl",
@@ -1320,7 +1301,6 @@ options(k8s, Conf) ->
         {apiserver, conf_get("cluster.k8s.apiserver", Conf)},
         {service_name, conf_get("cluster.k8s.service_name", Conf)},
         {address_type, conf_get("cluster.k8s.address_type", Conf, ip)},
-        {app_name, conf_get("cluster.k8s.app_name", Conf)},
         {namespace, conf_get("cluster.k8s.namespace", Conf)},
         {suffix, conf_get("cluster.k8s.suffix", Conf, "")}
     ];

deploy/charts/emqx/templates/StatefulSet.yaml

@@ -117,8 +117,6 @@ spec:
           env:
           - name: EMQX_NAME
             value: {{ .Release.Name }}
-          - name: EMQX_CLUSTER__K8S__APP_NAME
-            value: {{ .Release.Name }}
           - name: EMQX_CLUSTER__DISCOVERY_STRATEGY
             value: k8s
           - name: EMQX_CLUSTER__K8S__SERVICE_NAME

mix.exs

@@ -52,7 +52,7 @@ defmodule EMQXUmbrella.MixProject do
       {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
       {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
       {:esockd, github: "emqx/esockd", tag: "5.9.3", override: true},
-      {:ekka, github: "emqx/ekka", tag: "0.12.9", override: true},
+      {:ekka, github: "emqx/ekka", tag: "0.13.0", override: true},
       {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
       {:minirest, github: "emqx/minirest", tag: "1.3.4", override: true},
       {:ecpool, github: "emqx/ecpool", tag: "0.5.2"},

rebar.config

@@ -54,7 +54,7 @@
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.9.0"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.9.3"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.12.9"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.13.0"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.8.1"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "1.3.4"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.2"}}}

scripts/make-docker-image-from-host-build.sh

@@ -22,7 +22,8 @@ if [ "$COMPILE" = '--compile' ]; then
     sync
 fi
 
-export DOCKER_BUILDKIT=1
+# cannot enable DOCKER_BUILDKIT because the COPY often gets stale layers
+#export DOCKER_BUILDKIT=1
 docker build --build-arg PROFILE="${PROFILE}" \
     -t "emqx/emqx:${PKG_VSN}-${DISTRO}" \
     -f "$EMQX_DOCKERFILE" .

scripts/test-node-discovery-dns.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+## Test two nodes-cluster discover each other using DNS A records lookup result.
+
+set -euo pipefail
+
+cd -P -- "$(dirname -- "$0")/.."
+
+IMAGE="${1}"
+
+NET='test_node_discovery_dns'
+NODE1='emqx1'
+NODE2='emqx2'
+COOKIE='this-is-a-secret'
+
+# cleanup
+docker rm -f dnsmasq >/dev/null 2>&1 || true
+docker rm -f "$NODE1" >/dev/null 2>&1 || true
+docker rm -f "$NODE2" >/dev/null 2>&1 || true
+docker network rm "$NET" >/dev/null 2>&1 || true
+
+docker network create --subnet=172.18.0.0/16 $NET
+
+IP0="172.18.0.100"
+IP1="172.18.0.101"
+IP2="172.18.0.102"
+DOMAIN="dnstest.mynet"
+
+# create configs for dnsmasq
+cat <<-EOF > "/tmp/dnsmasq.conf"
+conf-dir=/etc/dnsmasq,*.conf
+addn-hosts=/etc/hosts.$DOMAIN
+EOF
+
+cat <<-EOF > "/tmp/hosts.$DOMAIN"
+$IP1 $DOMAIN
+$IP2 $DOMAIN
+EOF
+
+cat <<-EOF > /tmp/dnsmasq.base.conf
+domain-needed
+bogus-priv
+no-hosts
+keep-in-foreground
+no-resolv
+expand-hosts
+server=8.8.8.8
+EOF
+
+docker run -d -t --name dnsmasq \
+    --net "$NET" \
+    --ip "$IP0" \
+    -v /tmp/dnsmasq.conf:/etc/dnsmasq.conf \
+    -v "/tmp/hosts.$DOMAIN:/etc/hosts.$DOMAIN" \
+    -v "/tmp/dnsmasq.base.conf:/etc/dnsmasq/0.base.conf" \
+    --cap-add=NET_ADMIN \
+    storytel/dnsmasq dnsmasq --no-daemon --log-queries
+
+start_emqx() {
+    NAME="$1"
+    IP="$2"
+    DASHBOARD_PORT="$3"
+    docker run -d -t \
+        --name "$NAME" \
+        --net "$NET" \
+        --ip "$IP" \
+        --dns "$IP0" \
+        -p "$DASHBOARD_PORT:18083" \
+        -e EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug \
+        -e EMQX_NODE_COOKIE="$COOKIE" \
+        -e EMQX_cluster__discovery_strategy='dns' \
+        -e EMQX_cluster__dns__name="$DOMAIN" \
+        "$IMAGE"
+}
+
+start_emqx "$NODE1" "$IP1" 18083
+start_emqx "$NODE2" "$IP2" 18084