ci: change to build and test with .tar.gz instead of .zip packages

.ci/build_packages/tests.sh

@@ -4,12 +4,12 @@ set -euo pipefail
 set -x
 
 if [ -z "${1:-}" ]; then
-    echo "Usage $0 <PACKAGE_NAME> zip|pkg"
+    echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
     exit 1
 fi
 
-if [ "${2:-}" != 'zip' ] && [ "${2:-}" != 'pkg' ]; then
-    echo "Usage $0 <PACKAGE_NAME> zip|pkg"
+if [ "${2:-}" != 'tgz' ] && [ "${2:-}" != 'pkg' ]; then
+    echo "Usage $0 <PACKAGE_NAME> tgz|pkg"
     exit 1
 fi
 
@@ -22,8 +22,8 @@ export EMQX_NAME=${EMQX_NAME:-"emqx"}
 export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
 export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
 
-if [ "$PACKAGE_TYPE" = 'zip' ]; then
-    PKG_SUFFIX="zip"
+if [ "$PACKAGE_TYPE" = 'tgz' ]; then
+    PKG_SUFFIX="tar.gz"
 else
     SYSTEM="$("$CODE_PATH"/scripts/get-distro.sh)"
     case "${SYSTEM:-}" in
@@ -37,9 +37,10 @@ else
 fi
 PACKAGE_FILE_NAME="${PACKAGE_NAME}.${PKG_SUFFIX}"
 
-PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}.${PKG_SUFFIX}"
+PACKAGE_FILE="${PACKAGE_PATH}/${PACKAGE_FILE_NAME}"
 if ! [ -f "$PACKAGE_FILE" ]; then
     echo "$PACKAGE_FILE is not a file"
+    exit 1
 fi
 
 case "$(uname -m)" in
@@ -68,8 +69,8 @@ emqx_test(){
     cd "${PACKAGE_PATH}"
     local packagename="${PACKAGE_FILE_NAME}"
         case "$PKG_SUFFIX" in
-            "zip")
-                unzip -q "${PACKAGE_PATH}/${packagename}"
+            "tar.gz")
+                tar -zxf "${PACKAGE_PATH}/${packagename}"
                 export EMQX_ZONES__DEFAULT__MQTT__SERVER_KEEPALIVE=60
                 export EMQX_MQTT__MAX_TOPIC_ALIAS=10
                 export EMQX_LOG__CONSOLE_HANDLER__LEVEL=debug
@@ -205,10 +206,10 @@ relup_test(){
     if [ -d "${RELUP_PACKAGE_PATH}" ];then
         cd "${RELUP_PACKAGE_PATH}"
 
-        find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.zip" |
+        find . -maxdepth 1 -name "${EMQX_NAME}-*-${ARCH}.tar.gz" |
             while read -r pkg; do
                 packagename=$(basename "${pkg}")
-                unzip "$packagename"
+                tar -zxf "$packagename"
                 if ! ./emqx/bin/emqx start; then
                     cat emqx/log/erlang.log.1 || true
                     cat emqx/log/emqx.log.1 || true
@@ -216,7 +217,7 @@ relup_test(){
                 fi
                 ./emqx/bin/emqx_ctl status
                 ./emqx/bin/emqx versions
-                cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases
+                cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".tar.gz ./emqx/releases
                 ./emqx/bin/emqx install "${TARGET_VERSION}"
                 [ "$(./emqx/bin/emqx versions |grep permanent | awk '{print $2}')" = "${TARGET_VERSION}" ] || exit 1
                 ./emqx/bin/emqx_ctl status

.ci/fvt_tests/relup.lux

@@ -22,7 +22,7 @@
 
 [shell emqx]
     !cd $PACKAGE_PATH
-    !unzip -q -o $PROFILE-ubuntu20.04-$(echo $old_vsn | sed  -r 's/[v|e]//g')-amd64.zip
+    !tar -zxf ${PROFILE}-$(echo $old_vsn | sed  -r 's/[v|e]//g')-*-ubuntu20.04-amd64.tar.gz
     ?SH-PROMPT
 
     !cd emqx
@@ -82,7 +82,7 @@
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
-    !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
+    !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/
 
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
@@ -107,7 +107,7 @@
     !echo "" > log/emqx.log.1
     ?SH-PROMPT
 
-    !cp -f ../$PROFILE-ubuntu20.04-$VSN-amd64.zip releases/
+    !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/
 
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"

.github/workflows/build_packages.yaml

@@ -85,11 +85,11 @@ jobs:
         $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
         if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
           $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
-          $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.zip"
-          }
+          $pkg_name = "${{ matrix.profile }}-$([regex]::matches($version, $regex).value)-otp${{ matrix.otp }}-windows-amd64.tar.gz"
+        }
         else {
-          $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.zip"
-          }
+          $pkg_name = "${{ matrix.profile }}-$($version -replace '/')-otp${{ matrix.otp }}-windows-amd64.tar.gz"
+        }
         ## We do not build/release bcrypt and quic for windows package
         Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
         Remove-Item -Recurse -Force -Path _build/default/lib/quicer/
@@ -192,12 +192,12 @@ jobs:
         make ensure-rebar3
         sudo cp rebar3 /usr/local/bin/rebar3
         rm -rf _build/${{ matrix.profile }}/lib
-        make ${{ matrix.profile }}-zip
+        make ${{ matrix.profile }}-tgz
     - name: test
       working-directory: source
       run: |
-        pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
-        unzip -q $pkg_name
+        pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz)
+        tar -zxf $pkg_name
         # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
         ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
         ready='no'
@@ -297,7 +297,7 @@ jobs:
       with:
         path: source/_build/default/lib/quicer/
         key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
-    - name: downloads old emqx zip packages
+    - name: download old emqx tgz packages
       env:
         OTP_VSN: ${{ matrix.otp }}
         PROFILE: ${{ matrix.profile }}
@@ -320,10 +320,10 @@ jobs:
         old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
         for tag in ${old_vsns[@]}; do
           package_name="${PROFILE}-${tag#[e|v]}-otp${OTP_VSN}-${SYSTEM}-${ARCH}"
-          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip) | grep -oE "^[23]+")" ]; then
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip
-            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.zip.sha256
-            echo "$(cat $package_name.zip.sha256) $package_name.zip" | sha256sum -c || exit 1
+          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz) | grep -oE "^[23]+")" ]; then
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz
+            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$package_name.tar.gz.sha256
+            echo "$(cat $package_name.tar.gz.sha256) $package_name.tar.gz" | sha256sum -c || exit 1
           fi
         done
     - name: build emqx packages
@@ -336,10 +336,10 @@ jobs:
       run: |
         ./scripts/buildx.sh \
           --profile "${PROFILE}" \
-          --pkgtype "zip" \
+          --pkgtype "tgz" \
           --arch "${ARCH}" \
           --builder "ghcr.io/emqx/emqx-builder/5.0-3:${OTP}-${SYSTEM}"
-        ## the pkg build is incremental on the zip build
+        ## the pkg build is incremental on the tgz build
         ./scripts/buildx.sh \
           --profile "${PROFILE}" \
           --pkgtype "pkg" \

.github/workflows/build_slim_packages.yaml

@@ -63,10 +63,10 @@ jobs:
       with:
         path: _build/default/lib/quicer/
         key: ${{ matrix.os }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
-    - name: build and test zip package
+    - name: build and test tgz package
       run: |
-        make ${EMQX_NAME}-zip
-        .ci/build_packages/tests.sh "$EMQX_PKG_NAME" zip
+        make ${EMQX_NAME}-tgz
+        .ci/build_packages/tests.sh "$EMQX_PKG_NAME" tgz
     - name: run static checks
       if: contains(matrix.os, 'ubuntu')
       run: |
@@ -78,7 +78,7 @@ jobs:
     - uses: actions/upload-artifact@v2
       with:
         name: ${{ matrix.profile}}-${{ matrix.otp }}-${{ matrix.os }}
-        path: _packages/**/*.zip
+        path: _packages/${{ matrix.profile}}/*.tar.gz
   mac:
     strategy:
       fail-fast: false
@@ -140,11 +140,11 @@ jobs:
         . $HOME/.kerl/${{ matrix.otp }}/activate
         make ensure-rebar3
         sudo cp rebar3 /usr/local/bin/rebar3
-        make ${{ matrix.profile }}-zip
+        make ${{ matrix.profile }}-tgz
     - name: test
       run: |
-        pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
-        unzip -q $pkg_name
+        pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz)
+        tar -zxf $pkg_name
         # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
         ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
         ready='no'
@@ -166,4 +166,4 @@ jobs:
     - uses: actions/upload-artifact@v2
       with:
         name: macos
-        path: _packages/**/*.zip
+        path: _packages/**/*.tar.gz

.github/workflows/run_api_tests.yaml

@@ -18,21 +18,18 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-    - name: zip emqx-broker
-      if: endsWith(github.repository, 'emqx')
+    - name: build EMQ X CE and EE tgz package
       run: |
-        make emqx-zip
-    - name: zip emqx-broker
-      if: endsWith(github.repository, 'enterprise')
-      run: |
-        echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
-        git config --global credential.helper store
-        make emqx-enterprise-zip
+        make emqx-tgz
+        make emqx-enterprise-tgz
     - uses: actions/upload-artifact@v2
       with:
-        name: emqx-broker
-        path: _packages/**/*.zip
-
+        name: emqx-ce
+        path: _packages/emqx/*.tar.gz
+    - uses: actions/upload-artifact@v2
+      with:
+        name: emqx-ee
+        path: _packages/emqx-enterprise/*.tar.gz
   api-test:
     needs: build
     runs-on: ubuntu-latest
@@ -57,6 +54,9 @@ jobs:
           - api_topic_rewrite
           - api_event_message
           - api_stats
+        edition:
+          - emqx-ce
+            #TODO: add emqx-ee
     steps:
     - uses: actions/checkout@v2
       with:
@@ -70,13 +70,13 @@ jobs:
         architecture: x64 # (x64 or x86) - defaults to x64
     - uses: actions/download-artifact@v2
       with:
-        name: emqx-broker
+        name: ${{ matrix.edition }}
         path: .
-    - name: start emqx-broker
+    - name: start EMQ X service
       env:
         EMQX_LISTENERS__WSS__DEFAULT__BIND: "0.0.0.0:8085"
       run: |
-        unzip ./emqx/*.zip
+        tar -zxf ./*.tar.gz
         ./emqx/bin/emqx start
     - name: install jmeter
       timeout-minutes: 10
@@ -117,4 +117,7 @@ jobs:
     steps:
     - uses: geekyeggo/delete-artifact@v1
       with:
-        name: emqx-broker
+        name: emqx-ce
+    - uses: geekyeggo/delete-artifact@v1
+      with:
+        name: emqx-ee

.github/workflows/run_relup_tests.yaml

@@ -85,10 +85,10 @@ jobs:
         cd emqx/_upgrade_base
         old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
         for old_vsn in ${old_vsns[@]}; do
-          wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.zip
+          wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$BROKER/$old_vsn/$PROFILE-${old_vsn#[e|v]}-otp${OTP_VSN}-ubuntu20.04-amd64.tar.gz
         done
     - name: build emqx
-      run: make -C emqx ${PROFILE}-zip
+      run: make -C emqx ${PROFILE}-tgz
     - name: build emqtt-bench
       run: make -C emqtt-bench
     - name: build lux
@@ -105,8 +105,8 @@ jobs:
         set -e -x -u
         if [ -n "$OLD_VSNS" ]; then
             mkdir -p packages
-            cp emqx/_packages/${PROFILE}/*.zip packages
-            cp emqx/_upgrade_base/*.zip packages
+            cp emqx/_packages/${PROFILE}/*.tar.gz packages
+            cp emqx/_upgrade_base/*.tar.gz packages
             lux \
             --case_timeout infinity \
             --var PROFILE=$PROFILE \