emqx4.3/.github/workflows/performance_test.yaml

name: Performance Test

on:
  workflow_dispatch:
    inputs:
      version:
        required: false
      download_url:
        required: false

permissions:
  contents: read

jobs:
  perftest:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 1
      matrix:
        scenario:
          - tests/ci/pubsub-2x2c4g-10k-20k-tps
    defaults:
      run:
        shell: bash

    steps:
    - name: Checkout tf-emqx-performance-test
      uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
      with:
        repository: emqx/tf-emqx-performance-test
        ref: v0.3.2

    - name: Setup Terraform
      uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd # v3.1.2
      with:
        terraform_version: 1.6.4
        terraform_wrapper: false

    - uses: actions/setup-python@v5
      with:
        python-version: '3.11'

    - run: pip install -r requirements.txt

    - name: Download emqx package (custom URL)
      if: github.event.inputs.version == '' && github.event.inputs.download_url != ''
      run: |
        wget "${{ github.event.inputs.download_url }}"

    - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}

    - name: Download emqx package (specific version)
      env:
        AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
      if: github.event.inputs.version != '' && github.event.inputs.download_url == ''
      run: |
        version=${{ github.event.inputs.version }}
        aws s3 cp s3://$AWS_S3_BUCKET/emqx-ee/${version}/emqx-enterprise-${version}-ubuntu22.04-amd64.deb .

    - name: Download emqx package (latest version)
      if: github.event.inputs.version == '' && github.event.inputs.download_url == ''
      env:
        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: |
        set -xeuo pipefail
        # get latest emqx version
        version=$(gh release list --repo emqx/emqx --limit 1 --json tagName --jq '.[] | .tagName')
        # remove 'v' prefix from the version
        version=${version:1}
        wget https://www.emqx.com/en/downloads/enterprise/${version}/emqx-enterprise-${version}-ubuntu22.04-amd64.deb

    - name: Configure AWS Credentials
      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_PERF_TEST }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_PERF_TEST }}
        aws-region: ${{ secrets.AWS_DEFAULT_REGION_PERF_TEST }}

    - name: Create infrastructure
      id: infra
      timeout-minutes: 30
      run: |
        mv emqx-enterprise-*.deb emqx-enterprise-ubuntu22.04-amd64.deb
        ls -lh *.deb

        echo "${{ secrets.EMQX_ENTERPRISE_LICENSE }}" > emqx5.lic
        cat ${{ matrix.scenario }}.env >> "$GITHUB_ENV"
        echo '{}' > slack-payload.json

        terraform init
        set +e
        terraform apply -var spec_file=${{ matrix.scenario }}.yaml -auto-approve -lock=false
        # retry once
        if [ $? != 0 ]; then
          echo "Retrying once"
          set -e
          terraform apply -var spec_file=${{ matrix.scenario }}.yaml -auto-approve -lock=false
        fi
        set -e
        echo "ssh_key_path=$(terraform output -raw ssh_key_path)" >> $GITHUB_OUTPUT

    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
      if: success()
      with:
        name: ssh_private_key
        path: |
          ${{ steps.infra.outputs.ssh_key_path }}

    - name: Report failure
      if: failure()
      run: |
        jq -n '[{"color": "#ff0000", "fields": [{"title": "Failed to provision infrastructure", "short": false}]}]' > attachments.json
        jq -n --argjson attachments "$(<attachments.json)" '{"attachments": $attachments}' > slack-payload.json

    - name: Run benchmark
      if: success()
      id: benchmark
      timeout-minutes: 60
      run: |
        success=0

        export TMPDIR=$(mktemp -d)
        echo "TMPDIR=$TMPDIR" >> $GITHUB_ENV
        echo '[]' > attachments.json

        PERIOD=1m scripts/summary.sh

        MEM_CORE_1=$(jq -r '.[] | select(.host == "emqx-core-1") | .mem' $TMPDIR/mem.json)
        MEM_CORE_2=$(jq -r '.[] | select(.host == "emqx-core-2") | .mem' $TMPDIR/mem.json)

        if [ $(echo "$MEM_CORE_1 > $INITIAL_RAM_BASELINE * (1 + $ALLOWED_DEVIATION_CPU_RAM)" | bc -l) -eq 1 ] \
        || [ $(echo "$MEM_CORE_2 > $INITIAL_RAM_BASELINE * (1 + $ALLOWED_DEVIATION_CPU_RAM)" | bc -l) -eq 1 ]; then
          success=1
          jq --arg mem1 "$MEM_CORE_1" --arg mem2 "$MEM_CORE_2" '. += [{"color": "#ff0000", "fields": [{"title": "Initial RAM usage is too high", "short": false, "value": "Core 1: \($mem1)%\nCore 2: \($mem2)%"}]}]' \
            attachments.json 1<> attachments.json
        fi

        EMQX_API_URL=$(terraform output -raw emqx_dashboard_url)
        ansible loadgen -m command -a 'systemctl start loadgen' --become --limit 'loadgen-emqtt_bench-1.*'
        echo "Waiting for subscribers to connect"
        subs=0
        while [ $subs -lt 10000 ]; do
          curl -s -u perftest:perftest "$EMQX_API_URL/api/v5/monitor_current" > "$TMPDIR/monitor_current.json"
          subs=$(jq -r '.subscriptions' "$TMPDIR/monitor_current.json")
          sleep 1
        done
        ansible loadgen -m command -a 'systemctl start loadgen' --become --limit 'loadgen-emqtt_bench-2.*'
        echo "Waiting for publishers to connect"
        conns=$(jq -r '.live_connections' "$TMPDIR/monitor_current.json")
        while [ $conns -lt 20000 ]; do
          curl -s -u perftest:perftest "$EMQX_API_URL/api/v5/monitor_current" > "$TMPDIR/monitor_current.json"
          conns=$(jq -r '.live_connections' "$TMPDIR/monitor_current.json")
          sleep 1
        done
        echo "All clients connected, sleep for $DURATION seconds"
        sleep $DURATION
        PERIOD="${DURATION}s" scripts/summary.sh | tee -a $GITHUB_STEP_SUMMARY

        echo "success=$success" >> $GITHUB_OUTPUT

    - name: Cleanup infrastructure
      if: always()
      run: |
        terraform destroy -var spec_file=${{ matrix.scenario }}.yaml -auto-approve

    - name: Analyze results
      if: success()
      run: |
        success=${{ steps.benchmark.outputs.success }}

        echo "## Test results analysis" >> $GITHUB_STEP_SUMMARY
        echo '' >> $GITHUB_STEP_SUMMARY

        CPU_CORE_1=$(jq -r '.[] | select(.host == "emqx-core-1") | .cpu' $TMPDIR/cpu.json)
        CPU_CORE_2=$(jq -r '.[] | select(.host == "emqx-core-2") | .cpu' $TMPDIR/cpu.json)

        if [ $(echo "$CPU_CORE_1 > $CPU_BASELINE * (1 + $ALLOWED_DEVIATION_CPU_RAM)" | bc -l) -eq 1 ] \
        || [ $(echo "$CPU_CORE_2 > $CPU_BASELINE * (1 + $ALLOWED_DEVIATION_CPU_RAM)" | bc -l) -eq 1 ]; then
          success=1
          jq --arg cpu1 "$CPU_CORE_1" --arg cpu2 "$CPU_CORE_2" '. += [{"color": "#ff0000", "fields": [{"title": "CPU utilization was too high", "short": false, "value": "Core 1: \($cpu1)%\nCore 2: \($cpu2)%"}]}]' \
            attachments.json 1<> attachments.json
          echo "* CPU utilization was too high: Core 1: $CPU_CORE_1%, Core 2: $CPU_CORE_2%" >> $GITHUB_STEP_SUMMARY
        fi

        MEM_CORE_1=$(jq -r '.[] | select(.host == "emqx-core-1") | .mem' $TMPDIR/mem.json)
        MEM_CORE_2=$(jq -r '.[] | select(.host == "emqx-core-2") | .mem' $TMPDIR/mem.json)

        if [ $(echo "$MEM_CORE_1 > $RAM_BASELINE * (1 + $ALLOWED_DEVIATION_CPU_RAM)" | bc -l) -eq 1 ] \
        || [ $(echo "$MEM_CORE_2 > $RAM_BASELINE * (1 + $ALLOWED_DEVIATION_CPU_RAM)" | bc -l) -eq 1 ]; then
          success=1
          jq --arg mem1 "$MEM_CORE_1" --arg mem2 "$MEM_CORE_2" '. += [{"color": "#ff0000", "fields": [{"title": "RAM usage was too high", "short": false, "value": "Core 1: \($mem1)%\nCore 2: \($mem2)%"}]}]' \
            attachments.json 1<> attachments.json
          echo "* RAM usage was too high: Core 1: $MEM_CORE_1%, Core 2: $MEM_CORE_2%" >> $GITHUB_STEP_SUMMARY
        fi

        RECEIVED_MSG_RATE=$(jq -r '.received_msg_rate' $TMPDIR/emqx_metrics.json)
        SENT_MSG_RATE=$(jq -r '.sent_msg_rate' $TMPDIR/emqx_metrics.json)

        if [ $(echo "$RECEIVED_MSG_RATE < $RECEIVED_MSG_RATE_BASELINE * (1 - $ALLOWED_DEVIATION_MSG_RATE)" | bc -l) -eq 1 ] \
        || [ $(echo "$SENT_MSG_RATE < $SENT_MSG_RATE_BASELINE * (1 - $ALLOWED_DEVIATION_MSG_RATE)" | bc -l) -eq 1 ]; then
          success=1
          jq --arg received_msg_rate "$RECEIVED_MSG_RATE" --arg sent_msg_rate "$SENT_MSG_RATE" \
            '. += [{"color": "#ff0000", "fields": [{"title": "Message rate was too low", "short": false, "value": "Received message rate: \($received_msg_rate)\nSent message rate: \($sent_msg_rate)"}]}]' \
            attachments.json 1<> attachments.json
          echo "* Message rate was too low: Received message rate: $RECEIVED_MSG_RATE, Sent message rate: $SENT_MSG_RATE" >> $GITHUB_STEP_SUMMARY
        fi

        MESSAGES_DROPPED=$(jq -r '.messages_dropped' $TMPDIR/emqx_metrics.json)
        if [ $(echo "$MESSAGES_DROPPED > 100" | bc) -eq 1 ]; then
          success=1
          jq --arg dropped "$MESSAGES_DROPPED" '. += [{"color": "#ff0000", "fields": [{"title": "Too many dropped messages", "short": false, "value": "Dropped: \($dropped)"}]}]' \
            attachments.json 1<> attachments.json
          echo "* Too many dropped messages: $MESSAGES_DROPPED" >> $GITHUB_STEP_SUMMARY
        fi

        jq -n --argjson attachments "$(<attachments.json)" '{"attachments": $attachments}' > slack-payload.json

        exit $success

    - name: Post to Slack
      if: failure()
      uses: slackapi/slack-github-action@v1.27.0
      env:
        SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
      with:
        channel-id: ${{ secrets.SLACK_PERFTEST_CHANNEL_ID }}
        slack-message: "EMQX performance test ${{ matrix.scenario }} failed. <${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}|Workflow Run>"
        payload-file-path: slack-payload.json
        payload-file-path-parsed: false

    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
      if: failure()
      with:
        name: terraform
        path: |
          .terraform
          *.tfstate