chore(fvt test): update emqx cluster compose file

.ci/docker-compose-file/.env

@@ -3,3 +3,6 @@ REDIS_TAG=6
 MONGO_TAG=4
 PGSQL_TAG=13
 LDAP_TAG=2.4.50
+
+TARGET=emqx/emqx
+EMQX_TAG=build-alpine-amd64

.ci/fvt_tests/docker-compose.yaml

@@ -1,14 +1,40 @@
-version: '3'
+version: '3.9'
 
 services:
+  haproxy:
+    container_name: haproxy
+    image: haproxy:2.3
+    depends_on:
+      - emqx1
+      - emqx2
+    volumes:
+      - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+      - ../../etc/certs:/usr/local/etc/haproxy/certs
+#   ports:
+#     - "1883:1883"
+#     - "8883:8883"
+#     - "8083:8083"
+#     - "8084:8084"
+    networks:
+      - emqx_bridge
+    working_dir: /usr/local/etc/haproxy
+    command:
+      - bash
+      - -c
+      - |
+        cat /usr/local/etc/haproxy/certs/cert.pem /usr/local/etc/haproxy/certs/key.pem > /usr/local/etc/haproxy/certs/emqx.pem
+        haproxy -f /usr/local/etc/haproxy/haproxy.cfg
+
   emqx1:
     container_name: node1.emqx.io
-    image: ${TARGET}:build-alpine-amd64
+    image: $TARGET:$EMQX_TAG
     environment:
     - "EMQX_NAME=emqx"
     - "EMQX_HOST=node1.emqx.io"
     - "EMQX_CLUSTER__DISCOVERY=static"
     - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
+    - "EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on"
+    - "EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on"
     - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s"
     - "EMQX_MQTT__MAX_TOPIC_ALIAS=10"
     - "EMQX_LOG__LEVEL=debug"
@@ -25,18 +51,20 @@ services:
       timeout: 25s
       retries: 5
     networks:
-      emqx-bridge:
+      emqx_bridge:
         aliases:
         - node1.emqx.io
 
   emqx2:
     container_name: node2.emqx.io
-    image: ${TARGET}:build-alpine-amd64
+    image: $TARGET:$EMQX_TAG
     environment:
     - "EMQX_NAME=emqx"
     - "EMQX_HOST=node2.emqx.io"
     - "EMQX_CLUSTER__DISCOVERY=static"
     - "EMQX_CLUSTER__STATIC__SEEDS=emqx@node1.emqx.io, emqx@node2.emqx.io"
+    - "EMQX_LISTENER__TCP__EXTERNAL__PROXY_PROTOCOL=on"
+    - "EMQX_LISTENER__WS__EXTERNAL__PROXY_PROTOCOL=on"
     - "EMQX_ZONE__EXTERNAL__RETRY_INTERVAL=2s"
     - "EMQX_MQTT__MAX_TOPIC_ALIAS=10"
     - "EMQX_LOG__LEVEL=debug"
@@ -53,22 +81,16 @@ services:
       timeout: 25s
       retries: 5
     networks:
-      emqx-bridge:
+      emqx_bridge:
         aliases:
         - node2.emqx.io
 
-  client:
-    container_name: paho_client
-    image: python:3.7.2-alpine3.9
-    depends_on:
-      - emqx1
-      - emqx2
-    tty: true
-    networks:
-        emqx-bridge:
-    volumes:
-      - ./scripts:/scripts
-
 networks:
-  emqx-bridge:
+  emqx_bridge:
     driver: bridge
+    name: emqx_bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.100.239.0/24
+          gateway: 172.100.239.1

.ci/docker-compose-file/docker-compose-python.yaml

@@ -0,0 +1,15 @@
+version: '3.9'
+
+services:
+  python:
+    container_name: python 
+    image: python:3.7.2-alpine3.9
+    depends_on:
+      - emqx1
+      - emqx2
+    tty: true
+    networks:
+        emqx_bridge:
+    volumes:
+      - ./python:/scripts
+

.ci/fvt_tests/scripts/pytest.sh

@@ -6,17 +6,19 @@
 set -x
 set +e
 
-NODE1="node1.emqx.io"
-NODE2="node2.emqx.io"
+LB="haproxy"
 
 apk update && apk add git curl
 git clone -b develop-4.0 https://github.com/emqx/paho.mqtt.testing.git /paho.mqtt.testing
 pip install pytest
-pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$NODE1"
+
+pytest -v /paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "$LB"
 RESULT=$?
-pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "$NODE1" --host2 "$NODE2"
-RESULT=$(( RESULT + $? ))
-pytest -v /paho.mqtt.testing/interoperability/test_client --host "$NODE1"
+
+pytest -v /paho.mqtt.testing/interoperability/test_client --host "$LB"
 RESULT=$(( RESULT + $? ))
 
+# pytest -v /paho.mqtt.testing/interoperability/test_cluster --host1 "node1.emqx.io" --host2 "node2.emqx.io"
+# RESULT=$(( RESULT + $? ))
+
 exit $RESULT

.github/workflows/run_fvt_tests.yaml

@@ -24,8 +24,10 @@ jobs:
               echo "${{ secrets.CI_GIT_TOKEN }}" >> scripts/git-token
               make deps-emqx-ee
               echo "TARGET=emqx/emqx-ee" >> $GITHUB_ENV
+              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             else
               echo "TARGET=emqx/emqx" >> $GITHUB_ENV
+              echo "EMQX_TAG=$(./pkg-vsn.sh)" >> $GITHUB_ENV
             fi
         - name: make emqx image
           run: make docker
@@ -33,7 +35,10 @@ jobs:
           timeout-minutes: 5
           run: |
             set -e -u -x
-            docker-compose -f .ci/fvt_tests/docker-compose.yaml up -d
+            docker-compose \
+                -f .ci/docker-compose-file/docker-compose-emqx-cluster.yaml \
+                -f .ci/docker-compose-file/docker-compose-python.yaml \
+                up -d
             while [ "$(docker inspect -f '{{ .State.Health.Status}}' node1.emqx.io)" != "healthy" ] || [ "$(docker inspect -f '{{ .State.Health.Status}}' node2.emqx.io)" != "healthy" ]; do
               if [ $(docker ps -a -f name=fvt_tests_emqx -f status=exited -q | wc -l) -ne 0 ]; then
                   echo "['$(date -u +"%Y-%m-%dT%H:%M:%SZ")']:emqx stop";
@@ -45,7 +50,7 @@ jobs:
             done
         - name: make paho tests
           run: |
-            if ! docker exec -i paho_client /scripts/pytest.sh; then
+            if ! docker exec -i python /scripts/pytest.sh; then
               docker logs node1.emqx.io
               docker logs node2.emqx.io
               exit 1