emqx4.3/.github/workflows/run_fvt_tests.yaml

name: Functional Verification Tests

concurrency:
  group: fvt-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  push:
    branches:
      - '**'
    tags:
      - v*
  pull_request:

jobs:
  prepare:
    runs-on: ubuntu-20.04
    # prepare source with any OTP version, no need for a matrix
    container: ghcr.io/emqx/emqx-builder/5.0-16:1.13.4-24.2.1-1-alpine3.15.1

    steps:
      - uses: actions/checkout@v2
        with:
          path: source
          fetch-depth: 0
      - name: get deps
        run: |
          make -C source deps-all
          zip -ryq source.zip source/* source/.[^.]*
      - uses: actions/upload-artifact@v2
        with:
          name: source
          path: source.zip

  docker_test:
    runs-on: ubuntu-20.04
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        profile:
          - emqx
          - emqx-enterprise
          - emqx-elixir
        cluster_db_backend:
          - mnesia
          - rlog
        os:
          - ["alpine3.15.1", "alpine:3.15.1"]
        otp:
          - 24.2.1-1
        elixir:
          - 1.13.4
        arch:
          - amd64
    steps:
    - uses: erlef/setup-beam@v1
      with:
        otp-version: "24.2"
    - uses: actions/download-artifact@v2
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip

    - name: Get deps git refs for cache
      id: deps-refs
      run: |
        cd source
        make ensure-rebar3
        sudo cp rebar3 /usr/local/bin/rebar3
        scripts/get-dep-refs.sh
    - name: load quicer cache
      uses: actions/cache@v2
      with:
        path: |
          source/_build/default/lib/quicer/
          source/deps/quicer/
        key: ${{ matrix.os[0] }}-${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}

    - name: make docker image
      working-directory: source
      env:
        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
        EMQX_RUNNER: ${{ matrix.os[1] }}
      run: |
        make ${{ matrix.profile }}-docker
    - name: run emqx
      timeout-minutes: 5
      working-directory: source
      run: |
        set -x
        if [[ "${{ matrix.profile }}" = *-elixir ]]
        then
          export IS_ELIXIR=yes
          PROFILE=$(echo ${{ matrix.profile }} | sed -e "s/-elixir//g")
          IMAGE=emqx/$PROFILE:$(./pkg-vsn.sh ${{ matrix.profile }})-elixir
        else
          IMAGE=emqx/${{ matrix.profile }}:$(./pkg-vsn.sh ${{ matrix.profile }})
        fi
        ./.ci/docker-compose-file/scripts/run-emqx.sh $IMAGE ${{ matrix.cluster_db_backend }}
    - name: make paho tests
      run: |
        if ! docker exec -i python /scripts/pytest.sh "${{ matrix.cluster_db_backend }}"; then
          echo "DUMP_CONTAINER_LOGS_BGN"
          echo "============== haproxy =============="
          docker logs haproxy
          echo "==============  node1  =============="
          docker logs node1.emqx.io
          echo "==============  node2  =============="
          docker logs node2.emqx.io
          echo "DUMP_CONTAINER_LOGS_END"
          exit 1
        fi
    # simple smoke test for node_dump
    - name: test node_dump
      run: |
        docker exec node1.emqx.io node_dump

  helm_test:
    runs-on: ubuntu-20.04
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        discovery:
        - k8s
        - dns
        profile:
        - emqx
        os:
        - ["debian11", "debian:11-slim"]
        otp:
        - 24.2.1-1
        elixir:
        - 1.13.4
        arch:
        - amd64
      # - emqx-enterprise # TODO test enterprise

    steps:
    - uses: erlef/setup-beam@v1
      with:
        otp-version: "24.2"
    - uses: actions/download-artifact@v2
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip
    - name: Get deps git refs for cache
      id: deps-refs
      run: |
        cd source
        make ensure-rebar3
        sudo cp rebar3 /usr/local/bin/rebar3
        scripts/get-dep-refs.sh
    - name: load quicer cache
      uses: actions/cache@v2
      with:
        path: source/_build/default/lib/quicer/
        key: ${{ matrix.os[0] }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}

    - name: make docker image
      working-directory: source
      env:
        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-16:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
        EMQX_RUNNER: ${{ matrix.os[1] }}
      run: |
        make ${{ matrix.profile }}-docker
        echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV
        echo "EMQX_TAG=$(./pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
    - run: minikube start
    - name: setup helm chart
      working-directory: source
      run: |
        minikube image load $TARGET:$EMQX_TAG
        sed -i -r "s/^appVersion: .*$/appVersion: \"$EMQX_TAG\"/g" deploy/charts/emqx/Chart.yaml
    - name: run emqx on chart
      working-directory: source
      if: matrix.discovery == 'k8s'
      run: |
        helm install emqx \
            --set image.repository=$TARGET \
            --set image.pullPolicy=Never \
            --set emqxAclConfig="" \
            --set image.pullPolicy=Never \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \
            --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
            --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
            deploy/charts/emqx \
            --debug
    - name: run emqx on chart
      working-directory: source
      if: matrix.discovery == 'dns'
      run: |
        helm install emqx \
            --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \
            --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \
            --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \
            --set image.repository=$TARGET \
            --set image.pullPolicy=Never \
            --set emqxAclConfig="" \
            --set image.pullPolicy=Never \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__RETRY_INTERVAL=2s \
            --set emqxConfig.EMQX_ZONES__DEFAULT__MQTT__MAX_TOPIC_ALIAS=10 \
            --set emqxConfig.EMQX_AUTHORIZATION__SOURCES=[] \
            --set emqxConfig.EMQX_AUTHORIZATION__NO_MATCH=allow \
            deploy/charts/emqx \
            --debug
    - name: waiting emqx started
      timeout-minutes: 10
      run: |
        while [ "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.replicas}')" \
          != "$(kubectl get StatefulSet -l app.kubernetes.io/name=emqx -o jsonpath='{.items[0].status.readyReplicas}')" ]; do
          echo "==============================";
          kubectl get pods;
          echo "==============================";
          echo "waiting emqx started";
          sleep 10;
        done
    - name: Check cluster
      timeout-minutes: 10
      run: |
        kubectl port-forward svc/emqx 18083:18083 &
        while
          [ "$(curl --silent --basic -u admin:public -X GET http://127.0.0.1:18083/api/v5/cluster| jq '.nodes|length')" != "3" ];
        do
          echo "waiting emqx cluster scale"
          sleep 1
        done
    - name: get emqx-0 pods log
      if: failure()
      run: |
        kubectl describe pods emqx-0
        kubectl logs emqx-0
    - name: get emqx-1 pods log
      if: failure()
      run: |
        kubectl describe pods emqx-1
        kubectl logs emqx-1
    - name: get emqx-2 pods log
      if: failure()
      run: |
        kubectl describe pods emqx-2
        kubectl logs emqx-2
    - uses: actions/checkout@v2
      with:
        repository: emqx/paho.mqtt.testing
        ref: develop-4.0
        path: paho.mqtt.testing
    - name: install pytest
      run: |
        pip install pytest
        echo "$HOME/.local/bin" >> $GITHUB_PATH
    - name: run paho test
      run: |
        port_connected () {
          local server="$1"
          local port="$2"
          echo > /dev/tcp/${server}/${port} 2>/dev/null
        }

        kubectl port-forward service/emqx 1883:1883 > /dev/null &

        while ! port_connected localhost 1883; do
          echo server not listening yet...
          sleep 10
        done

        pytest -v paho.mqtt.testing/interoperability/test_client/V5/test_connect.py -k test_basic --host "127.0.0.1"