chore(CI): update build packages workflows for appup

.ci/build_packages/tests.sh

@@ -3,10 +3,23 @@ set -x -e -u
 export CODE_PATH=${CODE_PATH:-"/emqx"}
 export EMQX_NAME=${EMQX_NAME:-"emqx"}
 export PACKAGE_PATH="${CODE_PATH}/_packages/${EMQX_NAME}"
-export RELUP_PACKAGE_PATH="${CODE_PATH}/relup_packages/${EMQX_NAME}"
+export RELUP_PACKAGE_PATH="${CODE_PATH}/_upgrade_base"
 # export EMQX_NODE_NAME="emqx-on-$(uname -m)@127.0.0.1"
 # export EMQX_NODE_COOKIE=$(date +%s%N)
 
+case "$(uname -m)" in
+    x86_64)
+        ARCH='amd64'
+        ;;
+    aarch64)
+        ARCH='arm64'
+        ;;
+    arm*)
+        ARCH=arm
+        ;;
+esac
+export ARCH
+
 emqx_prepare(){
     mkdir -p "${PACKAGE_PATH}"
 
@@ -140,13 +153,13 @@ relup_test(){
     if [ -d "${RELUP_PACKAGE_PATH}" ];then
         cd "${RELUP_PACKAGE_PATH }"
 
-        for var in "${EMQX_NAME}"-*-"$(uname -m)".zip;do
+        for var in "${EMQX_NAME}"-*-"${ARCH}".zip;do
             packagename=$(basename "${var}")
             unzip "$packagename"
             ./emqx/bin/emqx start || ( tail emqx/log/emqx.log.1 && exit 1 )
             ./emqx/bin/emqx_ctl status
             ./emqx/bin/emqx versions
-            cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-$(uname -m)".zip ./emqx/releases
+            cp "${PACKAGE_PATH}/${EMQX_NAME}"-*-"${TARGET_VERSION}-${ARCH}".zip ./emqx/releases
             ./emqx/bin/emqx install "${TARGET_VERSION}"
             [ "$(./emqx/bin/emqx versions |grep permanent | grep -oE "[0-9].[0-9].[0-9]")" = "${TARGET_VERSION}" ] || exit 1
             ./emqx/bin/emqx_ctl status
@@ -158,4 +171,4 @@ relup_test(){
 
 emqx_prepare
 emqx_test
-# relup_test <TODO: parameterise relup target version>
+relup_test $CODE_PATH/pkg-vsn.sh

.github/workflows/build_packages.yaml

@@ -19,11 +19,13 @@ jobs:
 
     outputs:
       profiles: ${{ steps.set_profile.outputs.profiles}}
+      old_vsns: ${{ steps.get_old_vsns.outputs.old_vsns}}
 
     steps:
       - uses: actions/checkout@v2
         with:
           path: source
+          fetch-depth: 0
       - name: set profile
         id: set_profile
         shell: bash
@@ -37,7 +39,7 @@ jobs:
         if: endsWith(github.repository, 'emqx')
         run: |
           make -C source deps-all
-          zip -ryq source.zip source
+          zip -ryq source.zip source/* source/.[^.]*
       - name: get_all_deps
         if: endsWith(github.repository, 'enterprise')
         run: |
@@ -45,7 +47,20 @@ jobs:
           git config --global credential.helper store
           echo "${{ secrets.CI_GIT_TOKEN }}" >> source/scripts/git-token
           make -C source deps-all
-          zip -ryq source.zip source
+          zip -ryq source.zip source/* source/.[^.]*
+      - name: get old version
+        id: get_old_vsns
+        run: |
+          set -e -x -u
+          cd source
+          vsn="$(./pkg-vsn.sh)"
+          pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
+          if make -C source emqx-ee --dry-run > /dev/null 2>&1; then
+              old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
+          else
+              old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
+          fi
+          echo "::set-output name=old_vsns::$old_vsns"
       - uses: actions/upload-artifact@v2
         with:
           name: source
@@ -251,35 +266,29 @@ jobs:
         path: .
     - name: unzip source code
       run: unzip -q source.zip
-    - name: downloads emqx zip packages
+    - name: downloads old emqx zip packages
       env:
         PROFILE: ${{ matrix.profile }}
         ARCH: ${{ matrix.arch }}
         SYSTEM: ${{ matrix.os }}
+        OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
       run: |
-        set -e -u -x
-        cd source
-        if [ $PROFILE = "emqx" ];then broker="emqx-ce"; else broker="$PROFILE"; fi
-        if [ $PROFILE = "emqx-ee" ];then edition='enterprise'; else edition='opensource'; fi
-
-        vsn="$(./pkg-vsn.sh)"
-        pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
-        if [ $PROFILE = "emqx-ee" ]; then
-            old_vsns=($(git tag -l "e$pre_vsn.[0-9]" | sed "s/e$vsn//"))
-        else
-            old_vsns=($(git tag -l "v$pre_vsn.[0-9]" | sed "s/v$vsn//"))
+        set -e -x -u
+        broker=$PROFILE
+        if [ $PROFILE = "emqx" ];then
+            broker="emqx-ce"
         fi
 
-        mkdir -p _upgrade_base
-        cd _upgrade_base
-        for tag in ${old_vsns[@]};do
-          if [ ! -z "$(echo  $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-${{ secrets.AWS_DEFAULT_REGION }}.amazonaws.com/${{ secrets.AWS_S3_BUCKET }}/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
+        mkdir -p source/_upgrade_base
+        cd source/_upgrade_base
+        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
+        for tag in ${old_vsns[@]}; do
+          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
             wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
             wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
             echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
           fi
         done
-        cd -
     - name: build emqx packages
       env:
         ERL_OTP: erl23.2.7.2-emqx-2
@@ -295,7 +304,7 @@ jobs:
           -f .ci/build_packages/Dockerfile \
           --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \
           --build-arg EMQX_NAME=$PROFILE \
-          --output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar .
+          --output type=tar,dest=/tmp/cross-build-$PROFILE-for-$SYSTEM.tar . || df -h
 
         mkdir -p /tmp/packages/$PROFILE
         tar -xvf /tmp/cross-build-$PROFILE-for-$SYSTEM.tar --wildcards emqx/_packages/$PROFILE/*