emqx4.3/.github/workflows/build_packages.yaml

name: Cross build packages

on:
  schedule:
    - cron:  '0 */6 * * *'
  release:
    types:
      - published
  workflow_dispatch:

jobs:
  prepare:
    strategy:
      matrix:
        otp:
          - "23.2.7.2-emqx-2"
          - "24.0.5-emqx-1"

    runs-on: ubuntu-20.04
    container: "ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-ubuntu20.04"

    outputs:
      profiles: ${{ steps.set_profile.outputs.profiles }}
      old_vsns: ${{ steps.set_profile.outputs.old_vsns }}

    steps:
      - uses: actions/checkout@v2
        with:
          path: source
          fetch-depth: 0
      - name: set profile
        id: set_profile
        shell: bash
        working-directory: source
        run: |
          vsn="$(./pkg-vsn.sh)"
          pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
          if make emqx-ee --dry-run > /dev/null 2>&1; then
            old_vsns="$(git tag -l "e$pre_vsn.[0-9]" | xargs echo -n | sed "s/e$vsn//")"
            echo "::set-output name=old_vsns::$old_vsns"
            echo "::set-output name=profiles::[\"emqx-ee\"]"
          else
            old_vsns="$(git tag -l "v$pre_vsn.[0-9]" | xargs echo -n | sed "s/v$vsn//")"
            echo "::set-output name=old_vsns::$old_vsns"
            echo "::set-output name=profiles::[\"emqx\", \"emqx-edge\"]"
          fi
      - name: get otp version
        id: get_otp_version
        run: |
          otp="$(erl -eval '{ok, Version} = file:read_file(filename:join([code:root_dir(), "releases", erlang:system_info(otp_release), "OTP_VERSION"])), io:fwrite(Version), halt().' -noshell)"
          echo "::set-output name=otp::$otp"
      - name: set get token
        if: endsWith(github.repository, 'enterprise')
        run: |
          echo "https://ci%40emqx.io:${{ secrets.CI_GIT_TOKEN }}@github.com" > $HOME/.git-credentials
          git config --global credential.helper store
      - name: get deps
        working-directory: source
        run: |
          make ensure-rebar3
          ./rebar3 as default get-deps
          rm -rf rebar.lock
      - name: gen zip file
        run: zip -ryq source-${{ steps.get_otp_version.outputs.otp }}.zip source/* source/.[^.]*
      - uses: actions/upload-artifact@v2
        with:
          name: source-${{ steps.get_otp_version.outputs.otp }}
          path: source-${{ steps.get_otp_version.outputs.otp }}.zip

  windows:
    runs-on: windows-2019

    needs: prepare
    if: endsWith(github.repository, 'emqx')

    strategy:
      fail-fast: false
      matrix:
        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
        exclude:
          - profile: emqx-edge

    steps:
    - uses: actions/download-artifact@v2
      with:
        name: source-23.2.7.2-emqx-2
        path: .
    - name: unzip source code
      run: Expand-Archive -Path source-23.2.7.2-emqx-2.zip -DestinationPath ./
    - uses: ilammy/msvc-dev-cmd@v1
    - uses: gleam-lang/setup-erlang@v1.1.2
      id: install_erlang
      ## gleam-lang/setup-erlang does not yet support the installation of otp24 on windows
      with:
        otp-version: 23.2
    - name: build
      env:
        PYTHON: python
        DIAGNOSTIC: 1
      working-directory: source
      run: |
        $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"

        $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
        if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
          $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]+"
          $pkg_name = "${{ matrix.profile }}-windows-$([regex]::matches($version, $regex).value).zip"
          }
        else {
          $pkg_name = "${{ matrix.profile }}-windows-$($version -replace '/').zip"
          }
        ## We do not build/release bcrypt and quic for windows package
        Remove-Item -Recurse -Force -Path _build/default/lib/bcrypt/
        Remove-Item -Recurse -Force -Path _build/default/lib/quicer/
        if (Test-Path rebar.lock) {
            Remove-Item -Force -Path rebar.lock
        }
        make ensure-rebar3
        copy rebar3 "${{ steps.install_erlang.outputs.erlpath }}\bin"
        ls "${{ steps.install_erlang.outputs.erlpath }}\bin"
        rebar3 --help
        make ${{ matrix.profile }}
        mkdir -p _packages/${{ matrix.profile }}
        Compress-Archive -Path _build/${{ matrix.profile }}/rel/emqx -DestinationPath _build/${{ matrix.profile }}/rel/$pkg_name
        mv _build/${{ matrix.profile }}/rel/$pkg_name _packages/${{ matrix.profile }}
        Get-FileHash -Path "_packages/${{ matrix.profile }}/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}'  > _packages/${{ matrix.profile }}/$pkg_name.sha256
    - name: run emqx
      timeout-minutes: 1
      working-directory: source
      run: |
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
        Start-Sleep -s 5
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
    - uses: actions/upload-artifact@v1
      if: startsWith(github.ref, 'refs/tags/')
      with:
        name: ${{ matrix.profile }}-23.2.7.2-emqx-2
        path: source/_packages/${{ matrix.profile }}/.

  mac:
    runs-on: macos-10.15

    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
        otp:
          - 24.0.5-emqx-1
        exclude:
          - profile: emqx-edge

    steps:
    - uses: actions/download-artifact@v2
      with:
        name: source-${{ matrix.otp }}
        path: .
    - name: unzip source code
      run: unzip -q source-${{ matrix.otp }}.zip
    - name: prepare
      run: |
        brew update
        brew install curl zip unzip gnu-sed kerl unixodbc freetds
        echo "/usr/local/bin" >> $GITHUB_PATH
        git config --global credential.helper store
    - uses: actions/cache@v2
      id: cache
      with:
        path: ~/.kerl
        key: erl${{ matrix.otp }}-macos10.15
    - name: build erlang
      if: steps.cache.outputs.cache-hit != 'true'
      timeout-minutes: 60
      env:
        KERL_BUILD_BACKEND: git
        OTP_GITHUB_URL: https://github.com/emqx/otp
      run: |
        kerl update releases
        kerl build ${{ matrix.otp }}
        kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}
    - name: build
      working-directory: source
      run: |
        . $HOME/.kerl/${{ matrix.otp }}/activate
        make ensure-rebar3
        sudo cp rebar3 /usr/local/bin/rebar3
        make ${{ matrix.profile }}-zip
    - name: test
      working-directory: source
      run: |
        pkg_name=$(basename _packages/${{ matrix.profile }}/${{ matrix.profile }}-*.zip)
        unzip -q _packages/${{ matrix.profile }}/$pkg_name
        # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
        ready='no'
        for i in {1..10}; do
          if curl -fs 127.0.0.1:18083/api/v5/status > /dev/null; then
            ready='yes'
            break
          fi
          sleep 1
        done
        if [ "$ready" != "yes" ]; then
          echo "Timed out waiting for emqx to be ready"
          cat emqx/log/erlang.log.1
          exit 1
        fi
        ./emqx/bin/emqx_ctl status
        ./emqx/bin/emqx stop
        rm -rf emqx
        openssl dgst -sha256 ./_packages/${{ matrix.profile }}/$pkg_name | awk '{print $2}'  > ./_packages/${{ matrix.profile }}/$pkg_name.sha256
    - uses: actions/upload-artifact@v1
      if: startsWith(github.ref, 'refs/tags/')
      with:
        name: ${{ matrix.profile }}-${{ matrix.otp }}
        path: source/_packages/${{ matrix.profile }}/.

  linux:
    runs-on: ubuntu-20.04

    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
        os:
          - ubuntu20.04
          - ubuntu18.04
          - ubuntu16.04
          - debian10
          - debian9
          # - opensuse
          - centos8
          - centos7
          - centos6
          - raspbian10
          # - raspbian9
        arch:
          - amd64
          - arm64
        otp:
          - 23.2.7.2-emqx-2
          - 24.0.5-emqx-1
        exclude:
        - os: centos6
          arch: arm64
        - os: raspbian9
          arch: amd64
        - os: raspbian10
          arch: amd64
        - os: raspbian9
          profile: emqx
        - os: raspbian10
          profile: emqx
        - os: raspbian9
          profile: emqx-ee
        - os: raspbian10
          profile: emqx-ee

    defaults:
      run:
        shell: bash

    steps:
    - uses: docker/setup-buildx-action@v1
    - uses: docker/setup-qemu-action@v1
      with:
        image: tonistiigi/binfmt:latest
        platforms: all
    - uses: actions/download-artifact@v2
      with:
        name: source-${{ matrix.otp }}
        path: .
    - name: unzip source code
      run: unzip -q source-${{ matrix.otp }}.zip
    - name: downloads old emqx zip packages
      env:
        PROFILE: ${{ matrix.profile }}
        ARCH: ${{ matrix.arch }}
        SYSTEM: ${{ matrix.os }}
        OLD_VSNS: ${{ needs.prepare.outputs.old_vsns }}
      working-directory: source
      run: |
        set -e -x -u
        broker=$PROFILE
        if [ $PROFILE = "emqx" ];then
            broker="emqx-ce"
        fi
        if [ ! -z "$(echo $SYSTEM | grep -oE 'raspbian')" ]; then
            export ARCH="arm"
        fi

        mkdir -p _upgrade_base
        cd _upgrade_base
        old_vsns=($(echo $OLD_VSNS | tr ' ' ' '))
        for tag in ${old_vsns[@]}; do
          if [ ! -z "$(echo $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip) | grep -oE "^[23]+")" ];then
            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip
            wget --no-verbose https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/$tag/$PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256
            echo "$(cat $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip.sha256) $PROFILE-$SYSTEM-${tag#[e|v]}-$ARCH.zip" | sha256sum -c || exit 1
          fi
        done
    - name: build emqx packages
      env:
        OTP: ${{ matrix.otp }}
        PROFILE: ${{ matrix.profile }}
        ARCH: ${{ matrix.arch }}
        SYSTEM: ${{ matrix.os }}
      working-directory: source
      run: |
        docker run -i --rm \
        -v $(pwd):/emqx \
        --workdir /emqx \
        --platform linux/$ARCH \
        ghcr.io/emqx/emqx-builder-helper/5.0:$OTP-$SYSTEM \
        bash -euc "make $PROFILE-zip || cat rebar3.crashdump; \
                   make $PROFILE-pkg || cat rebar3.crashdump; \
                   EMQX_NAME=$PROFILE && .ci/build_packages/tests.sh"
    - name: create sha256
      env:
        PROFILE: ${{ matrix.profile}}
      working-directory: source
      run: |
        if [ -d _packages/$PROFILE ]; then
          cd _packages/$PROFILE
            for var in $(ls emqx-* ); do
              sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
            done
          cd -
        fi
    - uses: actions/upload-artifact@v1
      if: startsWith(github.ref, 'refs/tags/')
      with:
        name: ${{ matrix.profile }}-${{ matrix.otp }}
        path: source/_packages/${{ matrix.profile }}/.

  docker:
    runs-on: ubuntu-20.04
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
        otp:
          - 24.0.5-emqx-1

    steps:
    - uses: actions/download-artifact@v2
      with:
        name: source-${{ matrix.otp }}
        path: .
    - name: unzip source code
      run: unzip -q source-${{ matrix.otp }}.zip
    - name: get version
      id: version
      working-directory: source
      run: echo "::set-output name=version::$(./pkg-vsn.sh)"
    - uses: docker/setup-buildx-action@v1
    - uses: docker/setup-qemu-action@v1
      with:
        image: tonistiigi/binfmt:latest
        platforms: all
    - uses: docker/build-push-action@v2
      if: github.event_name != 'release'
      with:
        push: false
        pull: true
        no-cache: true
        platforms: linux/amd64,linux/arm64
        tags: emqx/${{ matrix.profile }}:${{ steps.version.outputs.version }}
        build-args: |
          PKG_VSN=${{ steps.version.outputs.version }}
          BUILD_FROM=ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-alpine3.14
          RUN_FROM=alpine:3.14
          EMQX_NAME=${{ matrix.profile }}
        file: source/deploy/docker/Dockerfile
        context: source
    - uses: docker/login-action@v1
      if: github.event_name == 'release'
      with:
        username: ${{ secrets.DOCKER_HUB_USER }}
        password: ${{ secrets.DOCKER_HUB_TOKEN }}
    - uses: docker/build-push-action@v2
      if: github.event_name == 'release'
      with:
        push: true
        pull: true
        no-cache: true
        platforms: linux/amd64,linux/arm64
        tags: emqx/${{ matrix.profile }}:${{ steps.version.outputs.version }}
        build-args: |
          PKG_VSN=${{ steps.version.outputs.version }}
          BUILD_FROM=ghcr.io/emqx/emqx-builder-helper/5.0:${{ matrix.otp }}-alpine3.14
          RUN_FROM=alpine:3.14
          EMQX_NAME=${{ matrix.profile }}
        file: source/deploy/docker/Dockerfile
        context: source

  delete-artifact:

    runs-on: ubuntu-20.04
    strategy:
      matrix:
        otp:
          - 23.2.7.2-emqx-2
          - 24.0.5-emqx-1
    needs: [prepare, mac, linux, docker]
    steps:
    - uses: geekyeggo/delete-artifact@v1
      with:
        name: source-${{ matrix.otp }}

  upload:
    runs-on: ubuntu-20.04

    if: startsWith(github.ref, 'refs/tags/')

    needs: [prepare, mac, linux, docker]

    strategy:
      matrix:
        profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
        otp:
          - 24.0.5-emqx-1

    steps:
    - uses: actions/checkout@v2
    - name: get_version
      run: |
        echo 'version<<EOF' >> $GITHUB_ENV
        echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV
        echo 'EOF' >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      with:
        name: ${{ matrix.profile }}-${{ matrix.otp }}
        path: ./_packages/${{ matrix.profile }}
    - name: install dos2unix
      run: sudo apt-get update && sudo apt install -y dos2unix
    - name: get packages
      run: |
        set -e -u
        cd _packages/${{ matrix.profile }}
        for var in $( ls |grep emqx |grep -v sha256); do
          dos2unix $var.sha256
          echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
        done
        cd -
    - name: upload aws s3
      run: |
        set -e -u
        if [ "${{ matrix.profile }}"  == "emqx" ];then
            broker="emqx-ce"
        else
            broker=${{ matrix.profile }}
        fi
        aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }}

        aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$broker/${{ env.version }}
        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$broker/${{ env.version }}/*"
    - uses: Rory-Z/upload-release-asset@v1
      if: github.event_name == 'release' && matrix.profile != 'emqx-ee'
      with:
        repo: emqx
        path: "_packages/${{ matrix.profile }}/emqx-*"
        token: ${{ github.token }}
    - uses: Rory-Z/upload-release-asset@v1
      if: github.event_name == 'release' && matrix.profile == 'emqx-ee'
      with:
        repo: emqx-enterprise
        path: "_packages/${{ matrix.profile }}/emqx-*"
        token: ${{ github.token }}
    - name: update to emqx.io
      if: github.event_name == 'release'
      run: |
        set -e -x -u
        curl -w %{http_code} \
             --insecure \
             -H "Content-Type: application/json" \
             -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
             -X POST \
             -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
             ${{ secrets.EMQX_IO_RELEASE_API }}
    - name: push docker image to docker hub
      if: github.event_name == 'release'
      run: |
        set -e -x -u
        sudo make docker-prepare
        cd _packages/${{ matrix.profile }} && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
        echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
        sudo TARGET=emqx/${{ matrix.profile }} make docker-push
        sudo TARGET=emqx/${{ matrix.profile }} make docker-manifest-list
    - name: update repo.emqx.io
      if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
      run: |
        curl --silent --show-error \
          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
          -H "Accept: application/vnd.github.v3+json" \
          -X POST \
          -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ee\": \"true\"}}" \
          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
    - name: update repo.emqx.io
      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
      run: |
        curl --silent --show-error \
          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
          -H "Accept: application/vnd.github.v3+json" \
          -X POST \
          -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\", \"emqx_ce\": \"true\"}}" \
          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
    - name: update homebrew packages
      if: github.event_name == 'release' && endsWith(github.repository, 'emqx') && matrix.profile == 'emqx'
      run: |
        if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
            curl --silent --show-error \
              -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
              -H "Accept: application/vnd.github.v3+json" \
              -X POST \
              -d "{\"ref\":\"v1.0.1\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
              "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
        fi
    - uses: geekyeggo/delete-artifact@v1
      with:
        name: ${{ matrix.profile }}