build: add env.sh to manage builder version in one place

.github/actions/prepare-jmeter/action.yaml

@@ -1,24 +1,8 @@
 name: 'Prepare jmeter'
 
-inputs:
-  version-emqx:
-    required: true
-    type: string
-
 runs:
   using: composite
   steps:
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
-      with:
-        name: emqx-docker
-        path: /tmp
-    - name: load docker image
-      shell: bash
-      env:
-        PKG_VSN: ${{ inputs.version-emqx }}
-      run: |
-        EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
-        echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
     - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       with:
         repository: emqx/emqx-fvt

.github/workflows/_pr_entrypoint.yaml

@@ -11,23 +11,42 @@ on:
       ref:
         required: false
 
+defaults:
+  run:
+    shell: bash
+
 env:
   IS_CI: "yes"
 
 jobs:
+  init:
+    runs-on: ubuntu-22.04
+    outputs:
+      BUILDER_VSN: ${{ steps.env.outputs.BUILDER_VSN }}
+      OTP_VSN: ${{ steps.env.outputs.OTP_VSN }}
+      ELIXIR_VSN: ${{ steps.env.outputs.ELIXIR_VSN }}
+      BUILDER: ${{ steps.env.outputs.BUILDER }}
+    steps:
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        with:
+          ref: ${{ github.event.inputs.ref }}
+      - name: Set up environment
+        id: env
+        run: |
+          source ./env.sh
+          echo "BUILDER_VSN=$EMQX_BUILDER_VSN" | tee -a "$GITHUB_OUTPUT"
+          echo "OTP_VSN=$OTP_VSN" | tee -a "$GITHUB_OUTPUT"
+          echo "ELIXIR_VSN=$ELIXIR_VSN" | tee -a "$GITHUB_OUTPUT"
+          echo "BUILDER=$EMQX_BUILDER" | tee -a "$GITHUB_OUTPUT"
+
   sanity-checks:
     runs-on: ubuntu-22.04
-    container: "ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04"
+    needs: init
+    container: ${{ needs.init.outputs.BUILDER }}
     outputs:
       ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
       ct-host: ${{ steps.matrix.outputs.ct-host }}
       ct-docker: ${{ steps.matrix.outputs.ct-docker }}
-      version-emqx: ${{ steps.matrix.outputs.version-emqx }}
-      version-emqx-enterprise: ${{ steps.matrix.outputs.version-emqx-enterprise }}
-      builder: "ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04"
-      builder_vsn: "5.3-9"
-      otp_vsn: "26.2.5-3"
-      elixir_vsn: "1.15.7"
 
     permissions:
       contents: read
@@ -92,35 +111,20 @@ jobs:
       - name: Generate CT Matrix
         id: matrix
         run: |
-          APPS="$(./scripts/find-apps.sh --ci)"
-          MATRIX="$(echo "${APPS}" | jq -c '
-            [
-              (.[] | select(.profile == "emqx") | . + {
-                builder: "5.3-9",
-                otp: "26.2.5-3",
-                elixir: "1.15.7"
-              }),
-              (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.3-9",
-                otp: ["26.2.5-3"][],
-                elixir: "1.15.7"
-              })
-            ]
-          ')"
+          MATRIX="$(./scripts/find-apps.sh --ci)"
           echo "${MATRIX}" | jq
-          CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
+          CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile}) | unique')"
           CT_HOST="$(echo "${MATRIX}"   | jq -c 'map(select(.runner == "host"))')"
           CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
           echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT
           echo "ct-host=${CT_HOST}"     | tee -a $GITHUB_OUTPUT
           echo "ct-docker=${CT_DOCKER}" | tee -a $GITHUB_OUTPUT
-          echo "version-emqx=$(./pkg-vsn.sh emqx)" | tee -a $GITHUB_OUTPUT
-          echo "version-emqx-enterprise=$(./pkg-vsn.sh emqx-enterprise)" | tee -a $GITHUB_OUTPUT
 
   compile:
     runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral-xl","linux","x64"]') }}
-    container: ${{ needs.sanity-checks.outputs.builder }}
+    container: ${{ needs.init.outputs.BUILDER }}
     needs:
+      - init
       - sanity-checks
     strategy:
       matrix:
@@ -156,53 +160,47 @@ jobs:
 
   run_emqx_app_tests:
     needs:
+      - init
       - sanity-checks
       - compile
     uses: ./.github/workflows/run_emqx_app_tests.yaml
     with:
-      builder: ${{ needs.sanity-checks.outputs.builder }}
+      builder: ${{ needs.init.outputs.BUILDER }}
       before_ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event.before }}
       after_ref: ${{ github.sha }}
 
   run_test_cases:
     needs:
+      - init
       - sanity-checks
       - compile
     uses: ./.github/workflows/run_test_cases.yaml
     with:
-      builder: ${{ needs.sanity-checks.outputs.builder }}
+      builder: ${{ needs.init.outputs.BUILDER }}
       ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
       ct-host: ${{ needs.sanity-checks.outputs.ct-host }}
       ct-docker: ${{ needs.sanity-checks.outputs.ct-docker }}
 
   static_checks:
     needs:
+      - init
       - sanity-checks
       - compile
     uses: ./.github/workflows/static_checks.yaml
     with:
-      builder: ${{ needs.sanity-checks.outputs.builder }}
+      builder: ${{ needs.init.outputs.BUILDER }}
       ct-matrix: ${{ needs.sanity-checks.outputs.ct-matrix }}
 
   build_slim_packages:
     needs:
       - sanity-checks
     uses: ./.github/workflows/build_slim_packages.yaml
-    with:
-      builder: ${{ needs.sanity-checks.outputs.builder }}
-      builder_vsn: ${{ needs.sanity-checks.outputs.builder_vsn }}
-      otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
-      elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }}
 
   build_docker_for_test:
     needs:
+      - init
       - sanity-checks
     uses: ./.github/workflows/build_docker_for_test.yaml
-    with:
-      otp_vsn: ${{ needs.sanity-checks.outputs.otp_vsn }}
-      elixir_vsn: ${{ needs.sanity-checks.outputs.elixir_vsn }}
-      version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
-      version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
 
   spellcheck:
     needs:
@@ -212,41 +210,35 @@ jobs:
 
   run_conf_tests:
     needs:
+      - init
       - sanity-checks
       - compile
     uses: ./.github/workflows/run_conf_tests.yaml
     with:
-      builder: ${{ needs.sanity-checks.outputs.builder }}
+      builder: ${{ needs.init.outputs.BUILDER }}
 
   check_deps_integrity:
     needs:
+      - init
       - sanity-checks
     uses: ./.github/workflows/check_deps_integrity.yaml
     with:
-      builder: ${{ needs.sanity-checks.outputs.builder }}
+      builder: ${{ needs.init.outputs.BUILDER }}
 
   run_jmeter_tests:
     needs:
       - sanity-checks
       - build_docker_for_test
     uses: ./.github/workflows/run_jmeter_tests.yaml
-    with:
-      version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
 
   run_docker_tests:
     needs:
       - sanity-checks
       - build_docker_for_test
     uses: ./.github/workflows/run_docker_tests.yaml
-    with:
-      version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
-      version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}
 
   run_helm_tests:
     needs:
       - sanity-checks
       - build_docker_for_test
     uses: ./.github/workflows/run_helm_tests.yaml
-    with:
-      version-emqx: ${{ needs.sanity-checks.outputs.version-emqx }}
-      version-emqx-enterprise: ${{ needs.sanity-checks.outputs.version-emqx-enterprise }}

.github/workflows/_push-entrypoint.yaml

@@ -17,13 +17,42 @@ on:
       ref:
         required: false
 
+defaults:
+  run:
+    shell: bash
+
 env:
   IS_CI: 'yes'
 
 jobs:
+  init:
+    runs-on: ubuntu-22.04
+    outputs:
+      BUILDER_VSN: ${{ steps.env.outputs.BUILDER_VSN }}
+      OTP_VSN: ${{ steps.env.outputs.OTP_VSN }}
+      ELIXIR_VSN: ${{ steps.env.outputs.ELIXIR_VSN }}
+      BUILDER: ${{ steps.env.outputs.BUILDER }}
+      BUILD_FROM: ${{ steps.env.outputs.BUILD_FROM }}
+      RUN_FROM: ${{ steps.env.outputs.BUILD_FROM }}
+    steps:
+      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+        with:
+          ref: ${{ github.event.inputs.ref }}
+      - name: Set up environment
+        id: env
+        run: |
+          source env.sh
+          echo "BUILDER_VSN=$EMQX_BUILDER_VSN" >> "$GITHUB_OUTPUT"
+          echo "OTP_VSN=$OTP_VSN" >> "$GITHUB_OUTPUT"
+          echo "ELIXIR_VSN=$ELIXIR_VSN" >> "$GITHUB_OUTPUT"
+          echo "BUILDER=$EMQX_BUILDER" >> "$GITHUB_OUTPUT"
+          echo "BUILD_FROM=$EMQX_DOCKER_BUILD_FROM" >> "$GITHUB_OUTPUT"
+          echo "RUN_FROM=$EMQX_DOCKER_RUN_FROM" >> "$GITHUB_OUTPUT"
+
   prepare:
     runs-on: ubuntu-22.04
-    container: 'ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04'
+    needs: init
+    container: ${{ needs.init.outputs.BUILDER }}
     outputs:
       profile: ${{ steps.parse-git-ref.outputs.profile }}
       release: ${{ steps.parse-git-ref.outputs.release }}
@@ -31,10 +60,6 @@ jobs:
       ct-matrix: ${{ steps.matrix.outputs.ct-matrix }}
       ct-host: ${{ steps.matrix.outputs.ct-host }}
       ct-docker: ${{ steps.matrix.outputs.ct-docker }}
-      builder: 'ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04'
-      builder_vsn: '5.3-9'
-      otp_vsn: '26.2.5-3'
-      elixir_vsn: '1.15.7'
 
     permissions:
       contents: read
@@ -61,23 +86,9 @@ jobs:
       - name: Build matrix
         id: matrix
         run: |
-          APPS="$(./scripts/find-apps.sh --ci)"
-          MATRIX="$(echo "${APPS}" | jq -c '
-            [
-              (.[] | select(.profile == "emqx") | . + {
-                builder: "5.3-9",
-                otp: "26.2.5-3",
-                elixir: "1.15.7"
-              }),
-              (.[] | select(.profile == "emqx-enterprise") | . + {
-                builder: "5.3-9",
-                otp: ["26.2.5-3"][],
-                elixir: "1.15.7"
-              })
-            ]
-          ')"
+          MATRIX="$(./scripts/find-apps.sh --ci)"
           echo "${MATRIX}" | jq
-          CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile, builder, otp, elixir}) | unique')"
+          CT_MATRIX="$(echo "${MATRIX}" | jq -c 'map({profile}) | unique')"
           CT_HOST="$(echo "${MATRIX}"   | jq -c 'map(select(.runner == "host"))')"
           CT_DOCKER="$(echo "${MATRIX}" | jq -c 'map(select(.runner == "docker"))')"
           echo "ct-matrix=${CT_MATRIX}" | tee -a $GITHUB_OUTPUT
@@ -87,46 +98,44 @@ jobs:
   build_packages:
     if: needs.prepare.outputs.release == 'true'
     needs:
+      - init
       - prepare
     uses: ./.github/workflows/build_packages.yaml
     with:
       profile: ${{ needs.prepare.outputs.profile }}
       publish: true
-      otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
-      elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
-      builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
+      otp_vsn: ${{ needs.init.outputs.OTP_VSN }}
+      elixir_vsn: ${{ needs.init.outputs.ELIXIR_VSN }}
+      builder_vsn: ${{ needs.init.outputs.BUILDER_VSN }}
     secrets: inherit
 
   build_and_push_docker_images:
     if: needs.prepare.outputs.release == 'true'
     needs:
+      - init
       - prepare
     uses: ./.github/workflows/build_and_push_docker_images.yaml
     with:
       profile: ${{ needs.prepare.outputs.profile }}
       publish: true
       latest: ${{ needs.prepare.outputs.latest }}
-      otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
-      elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
-      builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
+      build_from: ${{ needs.init.outputs.BUILD_FROM }}
+      run_from: ${{ needs.init.outputs.RUN_FROM }}
     secrets: inherit
 
   build_slim_packages:
     if: needs.prepare.outputs.release != 'true'
     needs:
+      - init
       - prepare
     uses: ./.github/workflows/build_slim_packages.yaml
-    with:
-      builder: ${{ needs.prepare.outputs.builder }}
-      builder_vsn: ${{ needs.prepare.outputs.builder_vsn }}
-      otp_vsn: ${{ needs.prepare.outputs.otp_vsn }}
-      elixir_vsn: ${{ needs.prepare.outputs.elixir_vsn }}
 
   compile:
     if: needs.prepare.outputs.release != 'true'
     runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
-    container: ${{ needs.prepare.outputs.builder }}
+    container: ${{ needs.init.outputs.BUILDER }}
     needs:
+      - init
       - prepare
     strategy:
       matrix:
@@ -162,22 +171,23 @@ jobs:
 
   run_emqx_app_tests:
     needs:
-      - prepare
+      - init
       - compile
     uses: ./.github/workflows/run_emqx_app_tests.yaml
     with:
-      builder: ${{ needs.prepare.outputs.builder }}
+      builder: ${{ needs.init.outputs.BUILDER }}
       before_ref: ${{ github.event.before }}
       after_ref: ${{ github.sha }}
 
   run_test_cases:
     if: needs.prepare.outputs.release != 'true'
     needs:
+      - init
       - prepare
       - compile
     uses: ./.github/workflows/run_test_cases.yaml
     with:
-      builder: ${{ needs.prepare.outputs.builder }}
+      builder: ${{ needs.init.outputs.BUILDER }}
       ct-matrix: ${{ needs.prepare.outputs.ct-matrix }}
       ct-host: ${{ needs.prepare.outputs.ct-host }}
       ct-docker: ${{ needs.prepare.outputs.ct-docker }}
@@ -185,18 +195,20 @@ jobs:
   run_conf_tests:
     if: needs.prepare.outputs.release != 'true'
     needs:
+      - init
       - prepare
       - compile
     uses: ./.github/workflows/run_conf_tests.yaml
     with:
-      builder: ${{ needs.prepare.outputs.builder }}
+      builder: ${{ needs.init.outputs.BUILDER }}
 
   static_checks:
     if: needs.prepare.outputs.release != 'true'
     needs:
+      - init
       - prepare
       - compile
     uses: ./.github/workflows/static_checks.yaml
     with:
-      builder: ${{ needs.prepare.outputs.builder }}
+      builder: ${{ needs.init.outputs.BUILDER }}
       ct-matrix: ${{ needs.prepare.outputs.ct-matrix }}

.github/workflows/build_and_push_docker_images.yaml

@@ -16,13 +16,10 @@ on:
       publish:
         required: true
         type: boolean
-      otp_vsn:
+      build_from:
         required: true
         type: string
-      elixir_vsn:
-        required: true
-        type: string
-      builder_vsn:
+      run_from:
         required: true
         type: string
     secrets:
@@ -50,18 +47,12 @@ on:
         required: false
         type: boolean
         default: false
-      otp_vsn:
-        required: false
-        type: string
-        default: '26.2.5-3'
-      elixir_vsn:
-        required: false
-        type: string
-        default: '1.15.7'
-      builder_vsn:
+      build_from:
         required: false
         type: string
-        default: '5.3-9'
+        default: ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-debian12
+      run_from:
+        default: public.ecr.aws/debian/debian:stable-20240612-slim
 
 permissions:
   contents: read
@@ -69,7 +60,7 @@ permissions:
 jobs:
   build:
     runs-on: ${{ github.repository_owner == 'emqx' && fromJSON(format('["self-hosted","ephemeral","linux","{0}"]', matrix.arch)) || 'ubuntu-22.04' }}
-    container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ inputs.otp_vsn }}-debian12"
+    container: ${{ inputs.build_from }}
     outputs:
       PKG_VSN: ${{ steps.build.outputs.PKG_VSN }}
 
@@ -164,14 +155,9 @@ jobs:
           DOCKER_LATEST: ${{ inputs.latest }}
           DOCKER_PUSH: false
           DOCKER_BUILD_NOCACHE: true
-          DOCKER_PLATFORMS: linux/amd64
-          DOCKER_LOAD: true
-          EMQX_RUNNER: 'public.ecr.aws/debian/debian:stable-20240612-slim'
-          EMQX_DOCKERFILE: 'deploy/docker/Dockerfile'
+          BUILD_FROM: ${{ inputs.build_from }}
+          RUN_FROM: ${{ inputs.run_from }}
           PKG_VSN: ${{ needs.build.outputs.PKG_VSN }}
-          EMQX_BUILDER_VERSION: ${{ inputs.builder_vsn }}
-          OTP_VSN: ${{ inputs.otp_vsn }}
-          ELIXIR_VSN: ${{ inputs.elixir_vsn }}
           EMQX_SOURCE_TYPE: tgz
         run: |
           ./build ${PROFILE} docker
@@ -215,12 +201,9 @@ jobs:
           DOCKER_BUILD_NOCACHE: false
           DOCKER_PLATFORMS: linux/amd64,linux/arm64
           DOCKER_LOAD: false
-          EMQX_RUNNER: 'public.ecr.aws/debian/debian:stable-20240612-slim'
-          EMQX_DOCKERFILE: 'deploy/docker/Dockerfile'
+          BUILD_FROM: ${{ inputs.build_from }}
+          RUN_FROM: ${{ inputs.run_from }}
           PKG_VSN: ${{ needs.build.outputs.PKG_VSN }}
-          EMQX_BUILDER_VERSION: ${{ inputs.builder_vsn }}
-          OTP_VSN: ${{ inputs.otp_vsn }}
-          ELIXIR_VSN: ${{ inputs.elixir_vsn }}
           EMQX_SOURCE_TYPE: tgz
         run: |
           ./build ${PROFILE} docker

.github/workflows/build_docker_for_test.yaml

@@ -6,19 +6,6 @@ concurrency:
 
 on:
   workflow_call:
-    inputs:
-      otp_vsn:
-        required: true
-        type: string
-      elixir_vsn:
-        required: true
-        type: string
-      version-emqx:
-        required: true
-        type: string
-      version-emqx-enterprise:
-        required: true
-        type: string
 
 permissions:
   contents: read
@@ -28,9 +15,6 @@ jobs:
     runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
     env:
       EMQX_NAME: ${{ matrix.profile }}
-      PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
-      OTP_VSN: ${{ inputs.otp_vsn }}
-      ELIXIR_VSN: ${{ inputs.elixir_vsn }}
 
     strategy:
       fail-fast: false
@@ -42,7 +26,13 @@ jobs:
           - emqx-enterprise-elixir
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - name: Set up environment
+        id: env
+        run: |
+          source env.sh
+          PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
+          echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
       - name: build and export to Docker
         id: build
         run: |

.github/workflows/build_packages_cron.yaml

@@ -16,19 +16,22 @@ jobs:
   linux:
     if: github.repository_owner == 'emqx'
     runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
-    container:
-      image: "ghcr.io/emqx/emqx-builder/${{ matrix.profile[2] }}-${{ matrix.os }}"
 
     strategy:
       fail-fast: false
       matrix:
         profile:
-          - ['emqx', 'master', '5.3-9:1.15.7-26.2.5-3']
-          - ['emqx', 'release-57', '5.3-9:1.15.7-26.2.5-3']
+          - ['emqx', 'master']
+          - ['emqx', 'release-57']
         os:
           - ubuntu22.04
           - amzn2023
 
+    env:
+      PROFILE: ${{ matrix.profile[0] }}
+      OS: ${{ matrix.os }}
+      BUILDER_SYSTEM: force_docker
+
     defaults:
       run:
         shell: bash
@@ -38,34 +41,19 @@ jobs:
         with:
           ref: ${{ matrix.profile[1] }}
           fetch-depth: 0
-
-      - name: fix workdir
+      - name: Set up environment
+        id: env
         run: |
-          set -eu
-          git config --global --add safe.directory "$GITHUB_WORKSPACE"
-          # Align path for CMake caches
-          if [ ! "$PWD" = "/emqx" ]; then
-            ln -s $PWD /emqx
-            cd /emqx
-          fi
-          echo "pwd is $PWD"
-
-      - name: build emqx packages
-        env:
-          PROFILE: ${{ matrix.profile[0] }}
-          ACLOCAL_PATH: "/usr/share/aclocal:/usr/local/share/aclocal"
+          source env.sh
+          BUILDER="ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VSN}:${ELIXIR_VSN}-${OTP_VSN}-${OS}"
+          echo "BUILDER=$BUILDER" >> "$GITHUB_ENV"
+      - name: build tgz
         run: |
-          set -eu
-          make "${PROFILE}-tgz"
-          make "${PROFILE}-pkg"
-      - name: test emqx packages
-        env:
-          PROFILE: ${{ matrix.profile[0] }}
+          ./scripts/buildx.sh --profile "$PROFILE" --pkgtype tgz --builder "$BUILDER"
+      - name: build pkg
         run: |
-          set -eu
-          ./scripts/pkg-tests.sh "${PROFILE}-tgz"
-          ./scripts/pkg-tests.sh "${PROFILE}-pkg"
-      - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+          ./scripts/buildx.sh --profile "$PROFILE" --pkgtype pkg --builder "$BUILDER"
+      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
         if: success()
         with:
           name: ${{ matrix.profile[0] }}-${{ matrix.os }}-${{ github.ref_name }}
@@ -91,20 +79,23 @@ jobs:
           - emqx
         branch:
           - master
-        otp:
-          - 26.2.5-3
         os:
-          - macos-12-arm64
+          - macos-14-arm64
 
     steps:
       - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
         with:
           ref: ${{ matrix.branch }}
           fetch-depth: 0
+      - name: Set up environment
+        id: env
+        run: |
+          source env.sh
+          echo "OTP_VSN=$OTP_VSN" >> "$GITHUB_OUTPUT"
       - uses: ./.github/actions/package-macos
         with:
           profile: ${{ matrix.profile }}
-          otp: ${{ matrix.otp }}
+          otp: ${{ steps.env.outputs.OTP_VSN }}
           os: ${{ matrix.os }}
           apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
           apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}

.github/workflows/build_slim_packages.yaml

@@ -6,97 +6,50 @@ concurrency:
 
 on:
   workflow_call:
-    inputs:
-      builder:
-        required: true
-        type: string
-      builder_vsn:
-        required: true
-        type: string
-      otp_vsn:
-        required: true
-        type: string
-      elixir_vsn:
-        required: true
-        type: string
-
   workflow_dispatch:
     inputs:
       ref:
         required: false
-      builder:
-        required: false
-        type: string
-        default: 'ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04'
-      builder_vsn:
-        required: false
-        type: string
-        default: '5.3-9'
-      otp_vsn:
-        required: false
-        type: string
-        default: '26.2.5-3'
-      elixir_vsn:
-        required: false
-        type: string
-        default: '1.15.7'
 
 permissions:
   contents: read
 
 jobs:
   linux:
-    runs-on: ${{ github.repository_owner == 'emqx' && fromJSON(format('["self-hosted","ephemeral","linux","{0}"]', matrix.profile[4])) || 'ubuntu-22.04' }}
+    runs-on: ${{ github.repository_owner == 'emqx' && fromJSON(format('["self-hosted","ephemeral","linux","{0}"]', matrix.profile[2])) || 'ubuntu-22.04' }}
     env:
-      EMQX_NAME: ${{ matrix.profile[0] }}
+      PROFILE: ${{ matrix.profile[0] }}
+      ELIXIR: ${{ matrix.profile[1] == 'elixir' && 'yes' || 'no' }}
+      ARCH: ${{ matrix.profile[2] == 'x64' && 'amd64' || 'arm64' }}
+      BUILDER_SYSTEM: force_docker
 
     strategy:
       fail-fast: false
       matrix:
         profile:
-          - ["emqx", "26.2.5-3", "ubuntu22.04", "elixir", "x64"]
-          - ["emqx", "26.2.5-3", "ubuntu22.04", "elixir", "arm64"]
-          - ["emqx-enterprise", "26.2.5-3", "ubuntu22.04", "erlang", "x64"]
-
-    container: "ghcr.io/emqx/emqx-builder/${{ inputs.builder_vsn }}:${{ inputs.elixir_vsn }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
+          - ["emqx", "elixir", "x64"]
+          - ["emqx", "elixir", "arm64"]
+          - ["emqx-enterprise", "erlang", "x64"]
 
     steps:
     - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       with:
         fetch-depth: 0
-    - name: Work around https://github.com/actions/checkout/issues/766
+    - name: build tgz
       run: |
-        git config --global --add safe.directory "$GITHUB_WORKSPACE"
-        echo "CODE_PATH=$GITHUB_WORKSPACE" >> $GITHUB_ENV
-    - name: build and test tgz package
-      if: matrix.profile[3] == 'erlang'
+        ./scripts/buildx.sh --profile $PROFILE --pkgtype tgz --elixir $ELIXIR --arch $ARCH
+    - name: build pkg
       run: |
-        make ${EMQX_NAME}-tgz
-        ./scripts/pkg-tests.sh ${EMQX_NAME}-tgz
-    - name: build and test deb/rpm packages
-      if: matrix.profile[3] == 'erlang'
-      run: |
-        make ${EMQX_NAME}-pkg
-        ./scripts/pkg-tests.sh ${EMQX_NAME}-pkg
-    - name: build and test tgz package (Elixir)
-      if: matrix.profile[3] == 'elixir'
-      run: |
-        make ${EMQX_NAME}-elixir-tgz
-        ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-tgz
-    - name: build and test deb/rpm packages (Elixir)
-      if: matrix.profile[3] == 'elixir'
-      run: |
-        make ${EMQX_NAME}-elixir-pkg
-        ./scripts/pkg-tests.sh ${EMQX_NAME}-elixir-pkg
-    - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
+        ./scripts/buildx.sh --profile $PROFILE --pkgtype pkg --elixir $ELIXIR --arch $ARCH
+    - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
       with:
-        name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}"
+        name: "${{ matrix.profile[0] }}-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
         path: _packages/${{ matrix.profile[0] }}/*
         retention-days: 7
         compression-level: 0
     - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
       with:
-        name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}-${{ matrix.profile[3] }}-${{ matrix.profile[4] }}"
+        name: "${{ matrix.profile[0] }}-schema-dump-${{ matrix.profile[1] }}-${{ matrix.profile[2] }}"
         path: |
           scripts/spellcheck
           _build/docgen/${{ matrix.profile[0] }}/schema-en.json
@@ -108,21 +61,24 @@ jobs:
       matrix:
         profile:
         - emqx
-        otp:
-        - ${{ inputs.otp_vsn }}
         os:
-        - macos-14
+        - macos-14-arm64
 
     runs-on: ${{ matrix.os }}
     env:
       EMQX_NAME: ${{ matrix.profile }}
 
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - name: Set up environment
+      id: env
+      run: |
+        source env.sh
+        echo "OTP_VSN=$OTP_VSN" >> "$GITHUB_OUTPUT"
     - uses: ./.github/actions/package-macos
       with:
         profile: ${{ matrix.profile }}
-        otp: ${{ matrix.otp }}
+        otp: ${{ steps.env.outputs.OTP_VSN }}
         os: ${{ matrix.os }}
         apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
         apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}

.github/workflows/codeql.yaml

@@ -17,8 +17,6 @@ jobs:
     permissions:
       actions: read
       security-events: write
-    container:
-      image: ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04
 
     strategy:
       fail-fast: false
@@ -36,11 +34,6 @@ jobs:
       with:
         ref: ${{ matrix.branch }}
 
-    - name: Ensure git safe dir
-      run: |
-        git config --global --add safe.directory "$GITHUB_WORKSPACE"
-        make ensure-rebar3
-
     - name: Initialize CodeQL
       uses: github/codeql-action/init@7e187e1c529d80bac7b87a16e7a792427f65cf02 # v2.15.5
       with:
@@ -51,14 +44,7 @@ jobs:
       env:
         PROFILE: emqx-enterprise
       run: |
-        make emqx-enterprise-compile
-
-    - name: Fetch deps
-      if: matrix.language == 'python'
-      env:
-        PROFILE: emqx-enterprise
-      run: |
-        make deps-emqx-enterprise
+        ./scripts/buildx.sh --profile emqx-enterprise --pkgtype rel
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@7e187e1c529d80bac7b87a16e7a792427f65cf02 # v2.15.5

.github/workflows/run_docker_tests.yaml

@@ -6,13 +6,6 @@ concurrency:
 
 on:
   workflow_call:
-    inputs:
-      version-emqx:
-        required: true
-        type: string
-      version-emqx-enterprise:
-        required: true
-        type: string
 
 permissions:
   contents: read
@@ -32,12 +25,17 @@ jobs:
 
     env:
       EMQX_NAME: ${{ matrix.profile[0] }}
-      PKG_VSN: ${{ matrix.profile[0] == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
       EMQX_IMAGE_OLD_VERSION_TAG: ${{ matrix.profile[1] }}
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - name: Set up environment
+        id: env
+        run: |
+          source env.sh
+          PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
+          echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
+      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           name: ${{ env.EMQX_NAME }}-docker
           path: /tmp
@@ -71,7 +69,6 @@ jobs:
         shell: bash
     env:
       EMQX_NAME: ${{ matrix.profile }}
-      PKG_VSN: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
       _EMQX_TEST_DB_BACKEND: ${{ matrix.cluster_db_backend }}
 
     strategy:
@@ -85,8 +82,14 @@ jobs:
           - mnesia
           - rlog
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-      - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+      - name: Set up environment
+        id: env
+        run: |
+          source env.sh
+          PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
+          echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
+      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
         with:
           name: ${{ env.EMQX_NAME }}-docker
           path: /tmp

.github/workflows/run_helm_tests.yaml

@@ -6,13 +6,6 @@ concurrency:
 
 on:
   workflow_call:
-    inputs:
-      version-emqx:
-        required: true
-        type: string
-      version-emqx-enterprise:
-        required: true
-        type: string
 
 permissions:
   contents: read
@@ -25,7 +18,6 @@ jobs:
         shell: bash
     env:
       EMQX_NAME: ${{ matrix.profile }}
-      EMQX_TAG: ${{ matrix.profile == 'emqx-enterprise' && inputs.version-emqx-enterprise || inputs.version-emqx }}
       REPOSITORY: "emqx/${{ matrix.profile }}"
 
     strategy:
@@ -45,7 +37,14 @@ jobs:
     - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
       with:
         path: source
-    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
+    - name: Set up environment
+      id: env
+      run: |
+        cd source
+        source env.sh
+        PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh "$EMQX_NAME")
+        echo "EMQX_TAG=$PKG_VSN" >> "$GITHUB_ENV"
+    - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
       with:
         name: "${{ env.EMQX_NAME }}-docker"
         path: /tmp

.github/workflows/run_jmeter_tests.yaml

@@ -2,10 +2,6 @@ name: JMeter integration tests
 
 on:
   workflow_call:
-    inputs:
-      version-emqx:
-        required: true
-        type: string
 
 permissions:
   contents: read
@@ -55,10 +51,23 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-    - uses: ./.github/actions/prepare-jmeter
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - name: Set up environment
+      id: env
+      run: |
+        source env.sh
+        PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
+        echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
+    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
       with:
-        version-emqx: ${{ inputs.version-emqx }}
+        name: emqx-docker
+        path: /tmp
+    - name: load docker image
+      shell: bash
+      run: |
+        EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
+        echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
+    - uses: ./.github/actions/prepare-jmeter
     - name: docker compose up
       timeout-minutes: 5
       run: |
@@ -111,10 +120,23 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-    - uses: ./.github/actions/prepare-jmeter
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - name: Set up environment
+      id: env
+      run: |
+        source env.sh
+        PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
+        echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
+    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
       with:
-        version-emqx: ${{ inputs.version-emqx }}
+        name: emqx-docker
+        path: /tmp
+    - name: load docker image
+      shell: bash
+      run: |
+        EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
+        echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
+    - uses: ./.github/actions/prepare-jmeter
     - name: docker compose up
       timeout-minutes: 5
       env:
@@ -175,10 +197,23 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-    - uses: ./.github/actions/prepare-jmeter
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - name: Set up environment
+      id: env
+      run: |
+        source env.sh
+        PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
+        echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
+    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
       with:
-        version-emqx: ${{ inputs.version-emqx }}
+        name: emqx-docker
+        path: /tmp
+    - name: load docker image
+      shell: bash
+      run: |
+        EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
+        echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
+    - uses: ./.github/actions/prepare-jmeter
     - name: docker compose up
       timeout-minutes: 5
       env:
@@ -231,10 +266,23 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-    - uses: ./.github/actions/prepare-jmeter
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - name: Set up environment
+      id: env
+      run: |
+        source env.sh
+        PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
+        echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
+    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
       with:
-        version-emqx: ${{ inputs.version-emqx }}
+        name: emqx-docker
+        path: /tmp
+    - name: load docker image
+      shell: bash
+      run: |
+        EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
+        echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
+    - uses: ./.github/actions/prepare-jmeter
     - name: docker compose up
       timeout-minutes: 5
       run: |
@@ -284,10 +332,23 @@ jobs:
 
     needs: jmeter_artifact
     steps:
-    - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2
-    - uses: ./.github/actions/prepare-jmeter
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
+    - name: Set up environment
+      id: env
+      run: |
+        source env.sh
+        PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_BUILDER" ./pkg-vsn.sh emqx)
+        echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_ENV"
+    - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
       with:
-        version-emqx: ${{ inputs.version-emqx }}
+        name: emqx-docker
+        path: /tmp
+    - name: load docker image
+      shell: bash
+      run: |
+        EMQX_DOCKER_IMAGE_TAG=$(docker load < /tmp/emqx-docker-${PKG_VSN}.tar.gz | sed 's/Loaded image: //g')
+        echo "_EMQX_DOCKER_IMAGE_TAG=$EMQX_DOCKER_IMAGE_TAG" >> $GITHUB_ENV
+    - uses: ./.github/actions/prepare-jmeter
     - name: docker compose up
       timeout-minutes: 5
       run: |

.github/workflows/run_test_cases.yaml

@@ -35,12 +35,12 @@ jobs:
     defaults:
       run:
         shell: bash
-    container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
+    container: ${{ inputs.builder }}
 
     env:
       PROFILE: ${{ matrix.profile }}
       ENABLE_COVER_COMPILE: 1
-      CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}
+      CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}
 
     permissions:
       contents: read
@@ -100,7 +100,7 @@ jobs:
         # produces $PROFILE-<app-name>-<otp-vsn>-sg<suitegroup>.coverdata
       - name: run common tests
         env:
-          DOCKER_CT_RUNNER_IMAGE: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
+          DOCKER_CT_RUNNER_IMAGE: ${{ inputs.builder }}
           MONGO_TAG: "5"
           MYSQL_TAG: "8"
           PGSQL_TAG: "13"
@@ -111,7 +111,7 @@ jobs:
           MINIO_TAG: "RELEASE.2023-03-20T20-16-18Z"
           SUITEGROUP: ${{ matrix.suitegroup }}
           ENABLE_COVER_COMPILE: 1
-          CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
+          CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-sg${{ matrix.suitegroup }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: ./scripts/ct/run.sh --ci --app ${{ matrix.app }} --keep-up
 
@@ -136,7 +136,7 @@ jobs:
       - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: failure()
         with:
-          name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
+          name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-sg${{ matrix.suitegroup }}
           path: logs.tar.gz
           compression-level: 0
           retention-days: 7
@@ -149,7 +149,7 @@ jobs:
       matrix:
         include: ${{ fromJson(inputs.ct-host) }}
 
-    container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
+    container: ${{ inputs.builder }}
     defaults:
       run:
         shell: bash
@@ -161,7 +161,7 @@ jobs:
       PROFILE: ${{ matrix.profile }}
       SUITEGROUP: ${{ matrix.suitegroup }}
       ENABLE_COVER_COMPILE: 1
-      CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
+      CT_COVER_EXPORT_PREFIX: ${{ matrix.profile }}-sg${{ matrix.suitegroup }}
 
     steps:
       - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
@@ -196,7 +196,7 @@ jobs:
       - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
         if: failure()
         with:
-          name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-${{ matrix.otp }}-sg${{ matrix.suitegroup }}
+          name: logs-${{ matrix.profile }}-${{ matrix.prefix }}-sg${{ matrix.suitegroup }}
           path: logs.tar.gz
           compression-level: 0
           retention-days: 7

.github/workflows/static_checks.yaml

@@ -28,7 +28,7 @@ jobs:
       fail-fast: false
       matrix:
         include: ${{ fromJson(inputs.ct-matrix) }}
-    container: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-ubuntu22.04"
+    container: "${{ inputs.builder }}"
     steps:
       - uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # v4.1.4
         with:
@@ -39,10 +39,10 @@ jobs:
           git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 # v4.0.1
         with:
-          path: "emqx_dialyzer_${{ matrix.otp }}_plt"
-          key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }}
+          path: "emqx_dialyzer_${{ matrix.profile }}_plt"
+          key: rebar3-dialyzer-plt-${{ matrix.profile }}-${{ hashFiles('rebar.*', 'apps/*/rebar.*') }}
           restore-keys: |
-            rebar3-dialyzer-plt-${{ matrix.profile }}-${{ matrix.otp }}-
+            rebar3-dialyzer-plt-${{ matrix.profile }}-
       - run: cat .env | tee -a $GITHUB_ENV
       - name: run static checks
         run: make static_checks

Makefile

@@ -6,23 +6,16 @@ endif
 REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
-export EMQX_RELUP ?= true
-export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-debian12
-export EMQX_DEFAULT_RUNNER = public.ecr.aws/debian/debian:stable-20240612-slim
-export EMQX_REL_FORM ?= tgz
-export QUICER_DOWNLOAD_FROM_RELEASE = 1
-ifeq ($(OS),Windows_NT)
-	export REBAR_COLOR=none
-	FIND=/usr/bin/find
-else
-	FIND=find
-endif
+include env.sh
 
 # Dashboard version
 # from https://github.com/emqx/emqx-dashboard5
 export EMQX_DASHBOARD_VERSION ?= v1.9.1
 export EMQX_EE_DASHBOARD_VERSION ?= e1.7.1
 
+export EMQX_RELUP ?= true
+export EMQX_REL_FORM ?= tgz
+
 -include default-profile.mk
 PROFILE ?= emqx
 REL_PROFILES := emqx emqx-enterprise
@@ -196,8 +189,8 @@ $(PROFILES:%=clean-%):
 	@if [ -d _build/$(@:clean-%=%) ]; then \
 		rm -f rebar.lock; \
 		rm -rf _build/$(@:clean-%=%)/rel; \
-		$(FIND) _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
-		$(FIND) _build/$(@:clean-%=%) -type l -delete; \
+		find _build/$(@:clean-%=%) -name '*.beam' -o -name '*.so' -o -name '*.app' -o -name '*.appup' -o -name '*.o' -o -name '*.d' -type f | xargs rm -f; \
+		find _build/$(@:clean-%=%) -type l -delete; \
 	fi
 
 .PHONY: clean-all

build

@@ -397,12 +397,14 @@ function is_ecr_and_enterprise() {
 
 ## Build the default docker image based on debian 12.
 make_docker() {
-    local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-9}"
-    local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian12}"
-    local OTP_VSN="${OTP_VSN:-26.2.5-3}"
-    local ELIXIR_VSN="${ELIXIR_VSN:-1.15.7}"
-    local EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${ELIXIR_VSN}-${OTP_VSN}-${EMQX_BUILDER_PLATFORM}}
-    local EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
+    # shellcheck disable=SC1091
+    source ./env.sh
+    local BUILD_FROM="${BUILD_FROM:-${EMQX_DOCKER_BUILD_FROM}}"
+    # shellcheck disable=SC2155
+    local OTP_VSN="$(docker run --rm "${BUILD_FROM}" erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)"
+    # shellcheck disable=SC2155
+    local ELIXIR_VSN="$(docker run --rm "${BUILD_FROM}" elixir --short-version)"
+    local RUN_FROM="${RUN_FROM:-${EMQX_DOCKER_RUN_FROM}}"
     local EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}"
     local EMQX_SOURCE_TYPE="${EMQX_SOURCE_TYPE:-src}"
     # shellcheck disable=SC2155
@@ -446,8 +448,8 @@ make_docker() {
     GIT_REVISION="$(git rev-parse HEAD)"
     export BUILDX_NO_DEFAULT_ATTESTATIONS=1
     local DOCKER_BUILDX_ARGS=(
-       --build-arg BUILD_FROM="${EMQX_BUILDER}" \
-       --build-arg RUN_FROM="${EMQX_RUNNER}" \
+       --build-arg BUILD_FROM="${BUILD_FROM}" \
+       --build-arg RUN_FROM="${RUN_FROM}" \
        --build-arg SOURCE_TYPE="${EMQX_SOURCE_TYPE}" \
        --build-arg PROFILE="${PROFILE%%-elixir}" \
        --build-arg IS_ELIXIR="$([[ "$PROFILE" = *-elixir ]] && echo yes || echo no)" \

env.sh

@@ -0,0 +1,8 @@
+# https://github.com/emqx/emqx-builder
+export EMQX_BUILDER_VSN=5.3-9
+export OTP_VSN=26.2.5-3
+export ELIXIR_VSN=1.15.7
+export EMQX_BUILDER=ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VSN}:${ELIXIR_VSN}-${OTP_VSN}-ubuntu22.04
+export EMQX_DOCKER_BUILD_FROM=ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VSN}:${ELIXIR_VSN}-${OTP_VSN}-debian12
+export EMQX_DOCKER_RUN_FROM=public.ecr.aws/debian/debian:stable-20240612-slim
+export QUICER_DOWNLOAD_FROM_RELEASE=1

scripts/buildx.sh

@@ -8,8 +8,7 @@
 ## i.e. will not work if docker command has to be executed with sudo
 
 ## example:
-## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
-##     --builder ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-debian12
+## ./scripts/buildx.sh --profile emqx --pkgtype tgz
 
 set -euo pipefail
 
@@ -17,14 +16,14 @@ help() {
     echo
     echo "-h|--help:                 To display this usage information"
     echo "--profile <PROFILE>:       EMQX profile to build (emqx|emqx-enterprise)"
-    echo "--pkgtype tgz|pkg:         Specify which package to build, tgz for .tar.gz,"
-    echo "                           pkg for .rpm or .deb"
+    echo "--pkgtype tgz|pkg|rel:     Specify which package to build, tgz for .tar.gz,"
+    echo "                           pkg for .rpm or .deb, rel for release only"
     echo "--elixir:                  Specify if the release should be built with Elixir, "
     echo "                           defaults to 'no'."
     echo "--arch amd64|arm64:        Target arch to build the EMQX package for"
     echo "--src_dir <SRC_DIR>:       EMQX source code in this dir, default to PWD"
-    echo "--builder <BUILDER>:       Builder image to pull"
-    echo "                           E.g. ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-debian12"
+    echo "--builder <BUILDER>:       Docker image to use for building"
+    echo "                           E.g. ghcr.io/emqx/emqx-builder/5.3-8:1.15.7-26.2.5-2-debian12"
 }
 
 die() {
@@ -34,6 +33,11 @@ die() {
     exit 1
 }
 
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+# shellcheck disable=SC1091
+source ./env.sh
+
 while [ "$#" -gt 0 ]; do
     case $1 in
     -h|--help)
@@ -53,7 +57,7 @@ while [ "$#" -gt 0 ]; do
         shift 2
         ;;
     --builder)
-        BUILDER="$2"
+        EMQX_BUILDER="$2"
         shift 2
         ;;
     --arch)
@@ -103,24 +107,21 @@ ARCH="${ARCH:-${NATIVE_ARCH:-}}"
 
 [ -z "${PROFILE:-}" ] && die "missing --profile"
 [ -z "${PKGTYPE:-}" ] && die "missing --pkgtype"
-[ -z "${BUILDER:-}" ] && die "missing --builder"
+[ -z "${EMQX_BUILDER:-}" ] && die "missing --builder"
 [ -z "${ARCH:-}" ] && die "missing --arch"
 
-# ensure dir
-cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
-
 set -x
 
 if [ -z "${IS_ELIXIR:-}" ]; then
   IS_ELIXIR=no
 fi
 
-case "$PKGTYPE" in
-  tgz|pkg)
+case "${PKGTYPE:-}" in
+  tgz|pkg|rel)
     true
     ;;
   *)
-    echo "Bad --pkgtype option, should be tgz or pkg"
+    echo "Bad --pkgtype option, should be tgz, pkg or rel"
     exit 1
     ;;
 esac
@@ -135,9 +136,13 @@ else
 fi
 
 HOST_SYSTEM="$(./scripts/get-distro.sh)"
-BUILDER_SYSTEM="${BUILDER_SYSTEM:-$(echo "$BUILDER" | awk -F'-' '{print $NF}')}"
+BUILDER_SYSTEM="${BUILDER_SYSTEM:-$(echo "$EMQX_BUILDER" | awk -F'-' '{print $NF}')}"
 
-CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}"
+if [ "${PKGTYPE}" != 'rel' ]; then
+  CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}"
+else
+  CMD_RUN="make ${MAKE_TARGET}"
+fi
 
 IS_NATIVE_SYSTEM='no'
 if [[ "$BUILDER_SYSTEM" != "force_docker" ]]; then
@@ -163,7 +168,7 @@ elif docker info; then
         --workdir /emqx \
         --platform="linux/$ARCH" \
         --env ACLOCAL_PATH="/usr/share/aclocal:/usr/local/share/aclocal" \
-        "$BUILDER" \
+        "$EMQX_BUILDER" \
         bash -euc "git config --global --add safe.directory /emqx && $CMD_RUN"
 else
     echo "Error: Docker not available on unsupported platform"

scripts/pr-sanity-checks.sh

@@ -12,11 +12,9 @@ if ! type "yq" > /dev/null; then
     exit 1
 fi
 
-EMQX_BUILDER_VERSION=${EMQX_BUILDER_VERSION:-5.3-9}
-OTP_VSN=${OTP_VSN:-26.2.5-3}
-ELIXIR_VSN=${ELIXIR_VSN:-1.15.7}
-EMQX_BUILDER_PLATFORM=${EMQX_BUILDER_PLATFORM:-ubuntu22.04}
-EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${ELIXIR_VSN}-${OTP_VSN}-${EMQX_BUILDER_PLATFORM}}
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")/.."
+# shellcheck disable=SC1091
+source ./env.sh
 
 commands=$(yq ".jobs.sanity-checks.steps[].run" .github/workflows/_pr_entrypoint.yaml | grep -v null)
 

scripts/relup-test/start-relup-test-cluster.sh

@@ -9,6 +9,8 @@ set -euo pipefail
 
 # ensure dir
 cd -P -- "$(dirname -- "$0")/../.."
+# shellcheck disable=SC1091
+source ./env.sh
 
 set -x
 
@@ -22,7 +24,7 @@ WEBHOOK="webhook.$NET"
 BENCH="bench.$NET"
 COOKIE='this-is-a-secret'
 ## Erlang image is needed to run webhook server and emqtt-bench
-ERLANG_IMAGE="ghcr.io/emqx/emqx-builder/5.3-9:1.15.7-26.2.5-3-ubuntu22.04"
+ERLANG_IMAGE="${EMQX_BUILDER}"
 # builder has emqtt-bench installed
 BENCH_IMAGE="$ERLANG_IMAGE"