emqx4.3/.github/workflows/build_packages.yaml

name: Cross build packages

concurrency:
  group: build-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  schedule:
    - cron:  '0 */6 * * *'
  push:
    branches:
      - 'ci/**'
  release:
    types:
      - published
  workflow_dispatch:
    inputs:
      which_branch:
        required: false

jobs:
  prepare:
    runs-on: ubuntu-20.04
    # prepare source with any OTP version, no need for a matrix
    container: "ghcr.io/emqx/emqx-builder/5.0-7:1.13.3-24.2.1-1-ubuntu20.04"

    outputs:
      DEP_ROCKSDB_REF: ${{ steps.deps-refs.outputs.DEP_ROCKSDB_REF }}
      DEP_QUICER_REF: ${{ steps.deps-refs.outputs.DEP_QUICER_REF }}

    steps:
      - uses: actions/checkout@v2
        with:
          ref: ${{ github.event.inputs.which_branch }}
          path: source
          fetch-depth: 0

      - name: Get deps git refs for cache
        id: deps-refs
        working-directory: source
        run: |
          bash -x scripts/get-dep-refs.sh
          make clean-all
      - name: get_all_deps
        run: |
          make -C source deps-all
          zip -ryq source.zip source/* source/.[^.]*
      - uses: actions/upload-artifact@v2
        with:
          name: source
          path: source.zip

  windows:
    runs-on: windows-2019
    needs: prepare
    strategy:
      fail-fast: false
      matrix:
        profile: # for now only CE for windows
          - emqx
        otp:
          - 24.2.1
    steps:
    - uses: actions/download-artifact@v2
      with:
        name: source
        path: .
    - name: unzip source code
      run: Expand-Archive -Path source.zip -DestinationPath ./
    - uses: ilammy/msvc-dev-cmd@v1
    - uses: erlef/setup-beam@v1
      with:
        otp-version: ${{ matrix.otp }}
    - name: build
      env:
        PYTHON: python
        DIAGNOSTIC: 1
      working-directory: source
      run: |
        # ensure crypto app (openssl)
        erl -eval "erlang:display(crypto:info_lib())" -s init stop
        make ${{ matrix.profile }}-tgz
    - name: run emqx
      timeout-minutes: 5
      working-directory: source
      run: |
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
        Start-Sleep -s 5
        echo "EMQX started"
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
        echo "EMQX stopped"
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
        echo "EMQX installed"
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
        echo "EMQX uninstalled"
    - uses: actions/upload-artifact@v1
      if: startsWith(github.ref, 'refs/tags/')
      with:
        name: ${{ matrix.profile }}-windows
        path: source/_packages/${{ matrix.profile }}/.

  mac:
    needs: prepare
    strategy:
      fail-fast: false
      matrix:
        profile: # no EDGE for mac
          - emqx
          - emqx-enterprise
        otp:
          - 24.2.1-1
        os:
          - macos-11
          - macos-10.15
        exclude:
          - profile: emqx-edge
    runs-on: ${{ matrix.os }}
    steps:
    - uses: actions/download-artifact@v2
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip
    - name: prepare
      run: |
        brew update
        brew install curl zip unzip gnu-sed kerl unixodbc freetds
        echo "/usr/local/bin" >> $GITHUB_PATH
        git config --global credential.helper store
    - uses: actions/cache@v2
      id: cache
      with:
        path: ~/.kerl/${{ matrix.otp }}
        key: otp-install-${{ matrix.otp }}-${{ matrix.os }}
    - name: build erlang
      if: steps.cache.outputs.cache-hit != 'true'
      timeout-minutes: 60
      env:
        KERL_BUILD_BACKEND: git
        OTP_GITHUB_URL: https://github.com/emqx/otp
      run: |
        kerl update releases
        kerl build ${{ matrix.otp }}
        kerl install ${{ matrix.otp }} $HOME/.kerl/${{ matrix.otp }}

    - name: load rocksdb cache
      uses: actions/cache@v2
      with:
        path: source/_build/default/lib/rocksdb/
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_ROCKSDB_REF }}
    - name: load quicer cache
      uses: actions/cache@v2
      with:
        path: source/_build/default/lib/quicer/
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_QUICER_REF }}

    - name: build
      working-directory: source
      run: |
        . $HOME/.kerl/${{ matrix.otp }}/activate
        make ensure-rebar3
        sudo cp rebar3 /usr/local/bin/rebar3
        rm -rf _build/${{ matrix.profile }}/lib
        make ${{ matrix.profile }}-tgz
    - name: test
      working-directory: source
      run: |
        pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz)
        tar -zxf $pkg_name
        # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
        ready='no'
        for i in {1..10}; do
          if curl -fs 127.0.0.1:18083/api/v5/status > /dev/null; then
            ready='yes'
            break
          fi
          sleep 1
        done
        if [ "$ready" != "yes" ]; then
          echo "Timed out waiting for emqx to be ready"
          cat emqx/log/erlang.log.1
          exit 1
        fi
        ./emqx/bin/emqx_ctl status
        ./emqx/bin/emqx stop
        rm -rf emqx
    - uses: actions/upload-artifact@v1
      if: startsWith(github.ref, 'refs/tags/')
      with:
        name: ${{ matrix.profile }}-${{ matrix.otp }}
        path: source/_packages/${{ matrix.profile }}/.

  linux:
    needs: prepare
    runs-on: ${{ matrix.build_machine }}
    container:
      image: "ghcr.io/emqx/emqx-builder/5.0-7:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"

    strategy:
      fail-fast: false
      matrix:
        profile: ## all editions for linux
          - emqx-edge
          - emqx
          - emqx-enterprise
        otp:
          - 24.2.1-1 # we test with OTP 23, but only build package on OTP 24 versions
        elixir:
          - 1.13.3
        # used to split elixir packages into a separate job, since the
        # entire job may take a lot of time, especially on arm64
        # emulation.
        # we only want to build ubuntu and centos with elixir for the
        # time being, so it's easier to just include those with
        # `with_elixir` set.
        build_elixir:
          # - with_elixir
          - no_elixir
        arch:
          - amd64
          - arm64
        os:
          - ubuntu20.04
          - ubuntu18.04
          - ubuntu16.04
          - debian10
          - debian9
          # - opensuse
          - centos7
          - raspbian10
          # - raspbian9
        build_machine:
          - aws-arm64
          - ubuntu-20.04
        exclude:
        - arch: arm64
          build_machine: ubuntu-20.04
        - arch: amd64
          build_machine: aws-arm64
        - os: raspbian9
          arch: amd64
        - os: raspbian10
          arch: amd64
        - os: raspbian10 # we only have arm32 image
          arch: arm64
        - os: raspbian9
          profile: emqx
        - os: raspbian10
          profile: emqx
        - os: raspbian9
          profile: emqx-enterprise
        - os: raspbian10
          profile: emqx-enterprise
        include:
          - profile: emqx
            otp: 24.2.1-1
            elixir: 1.13.3
            build_elixir: with_elixir
            arch: amd64
            os: ubuntu20.04
            build_machine: ubuntu-20.04
          - profile: emqx
            otp: 24.2.1-1
            elixir: 1.13.3
            build_elixir: with_elixir
            arch: amd64
            os: centos7
            build_machine: ubuntu-20.04

    defaults:
      run:
        shell: bash

    steps:
    - uses: AutoModality/action-clean@v1
      if: matrix.build_machine == 'aws-arm64'
    - uses: actions/download-artifact@v2
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip
    - name: load rocksdb cache
      uses: actions/cache@v2
      with:
        path: |
          source/_build/default/lib/rocksdb/
          source/deps/rocksdb/
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_ROCKSDB_REF }}
    - name: load quicer cache
      uses: actions/cache@v2
      with:
        path: |
          source/_build/default/lib/quicer/
          source/deps/quicer/
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_QUICER_REF }}
    - name: build emqx packages
      working-directory: source
      env:
        OTP: ${{ matrix.otp }}
        ELIXIR: ${{ matrix.elixir }}
        PROFILE: ${{ matrix.profile }}
        ARCH: ${{ matrix.arch }}
        SYSTEM: ${{ matrix.os }}
      run: |
        set -eu
        # Align path for CMake caches
        if [ ! "$PWD" = "/emqx" ]; then
          ln -s $PWD /emqx
          cd /emqx
        fi
        echo "pwd is $PWD"
        PkgTypes="tgz pkg"
        WithElixir=""
        if [ ${{ matrix.build_elixir }} = "with_elixir" ]; then
          PkgTypes="tgz"
          # set Elixir build flag
          WithElixir="--with-elixir"
        fi
        for PKGTYPE in ${PkgTypes};
        do
          ./scripts/buildx.sh \
            --profile "${PROFILE}" \
            --pkgtype "${PKGTYPE}" \
            --arch "${ARCH}" \
            --otp "${OTP}" \
            --elixir "${ELIXIR}" "$WithElixir" \
            --system "${SYSTEM}" \
            --builder "ghcr.io/emqx/emqx-builder/5.0-7:${ELIXIR}-${OTP}-${SYSTEM}"
        done
    - uses: actions/upload-artifact@v1
      if: startsWith(github.ref, 'refs/tags/')
      with:
        name: ${{ matrix.profile }}-${{ matrix.otp }}
        path: source/_packages/${{ matrix.profile }}/.

  docker:
    runs-on: ${{ matrix.build_machine }}
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        os:
          - alpine3.14
        profile: # all editions for docker
          - emqx-edge
          - emqx
          - emqx-enterprise
        # NOTE: for docker, only support latest otp and elixir
        # versions, not a matrix
        otp:
          - 24.2.1-1 # update to latest
        elixir:
          - 1.13.3 # update to latest
        arch:
          - amd64
          - arm64
        build_elixir:
          - no_elixir
        build_machine:
          - aws-arm64
          - ubuntu-20.04
        exclude:
          - arch: arm64
            build_machine: ubuntu-20.04
          - arch: amd64
            build_machine: aws-arm64
        include:
          - os: alpine3.14
            profile: emqx
            otp: 24.2.1-1
            elixir: 1.13.3
            arch: amd64
            build_elixir: with_elixir
            build_machine: ubuntu-20.04

    steps:
    - uses: AutoModality/action-clean@v1
      if: matrix.build_machine == 'aws-arm64'
    - uses: actions/download-artifact@v2
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip

    - uses: docker/setup-buildx-action@v1

    - name: load rocksdb cache
      uses: actions/cache@v2
      with:
        path: |
          source/_build/default/lib/rocksdb/
          source/deps/rocksdb//
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_ROCKSDB_REF }}
    - name: load quicer cache
      uses: actions/cache@v2
      with:
        path: |
          source/_build/default/lib/quicer/
          source/deps/quicer/
        key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ needs.prepare.outputs.DEP_QUICER_REF }}

    - uses: docker/login-action@v1
      with:
        username: ${{ secrets.DOCKER_HUB_USER }}
        password: ${{ secrets.DOCKER_HUB_TOKEN }}

    - name: prepare for docker-action-parms
      id: pre-meta
      run: |
        img=$(echo ${{ matrix.os }} | sed 's#\([0-9.]\+\)$#:\1#g')
        emqx_name=${{ matrix.profile }}
        img_suffix=${{ matrix.arch }}
        img_labels="org.opencontainers.image.otp.version=${{ matrix.otp }}"

        if [ ${{ matrix.build_elixir }} = "with_elixir" ]; then
          emqx_name="emqx-elixir"
          img_suffix="elixir-${{ matrix.arch }}"
          img_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}\n${img_labels}"
        fi
        echo "::set-output name=img::${img}"
        echo "::set-output name=emqx_name::${emqx_name}"
        echo "::set-output name=img_suffix::${img_suffix}"
        echo "::set-output name=img_labels::${img_labels}"

    # NOTE, Pls make sure this is identical as the one in job 'docker-push-multi-arch-manifest'
    - uses: docker/metadata-action@v3
      id: meta
      with:
        images: ${{ github.repository_owner }}/${{ matrix.profile }}
        flavor: |
          latest=${{ github.event_name == 'release' && !github.event.release.prerelease }}
          suffix=-${{ steps.pre-meta.outputs.img_suffix }}
        tags: |
          type=ref,event=branch
          type=ref,event=pr
          type=ref,event=tag
          type=semver,pattern={{version}}
        labels:
          ${{ steps.pre-meta.outputs.img_labels }}

    - uses: docker/build-push-action@v2
      with:
        push: >
          ${{ (github.event_name == 'release' && !github.event.release.prerelease)
          || (github.event.repository.owner != 'emqx' && startsWith(github.ref_name, 'ci/')) }}
        pull: true
        no-cache: true
        platforms: linux/${{ matrix.arch }}
        tags: ${{ steps.meta.outputs.tags }}
        labels: ${{ steps.meta.outputs.labels }}
        build-args: |
          BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-7:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
          RUN_FROM=${{ steps.pre-meta.outputs.img }}
          EMQX_NAME=${{ steps.pre-meta.outputs.emqx_name }}
        file: source/deploy/docker/Dockerfile
        context: source

    - uses: aws-actions/configure-aws-credentials@v1
      if: matrix.profile == 'emqx'
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
    - name: Docker push to aws ecr
      if: >
        ${{ (github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx')
        || (github.event.repository.owner != 'emqx' && startsWith(github.ref_name, 'ci/')) }}
      run: |
        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
        for tag in $(echo ${{ steps.meta.outputs.json }} | jq  -c '.tags[]'); do
          docker pull "$tag"
          docker tag "$tag" "public.ecr.aws/$tag"
          docker push "public.ecr.aws/$tag"
        done

  docker-push-multi-arch-manifest:
    # note, we only run on amd64
    if: >
      (github.event_name == 'release' && !github.event.release.prerelease)
      || (github.event.repository.owner != 'emqx' && startsWith(github.ref_name, 'ci/'))
    needs:
      - prepare
      - docker
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        profile: # all editions for docker
          - emqx-edge
          - emqx
          - emqx-enterprise
        # NOTE: for docker, only support latest otp version, not a matrix
        otp:
          - 24.2.1-1 # update to latest
          #
        elixir:
          - 1.13.3 # update to latest
        arch:
          - amd64
          - arm64
        build_elixir:
          - no_elixir
        build_machine:
          - aws-arm64
          - ubuntu-20.04
        exclude:
          - arch: arm64
            build_machine: ubuntu-20.04
          - arch: amd64
            build_machine: aws-arm64
        include:
          - os: alpine3.14
            profile: emqx
            otp: 24.2.1-1
            elixir: 1.13.3
            arch: amd64
            build_elixir: with_elixir
            build_machine: ubuntu-20.04

    steps:
      - uses: actions/download-artifact@v2
        if: matrix.arch == 'amd64'
        with:
          name: source
          path: .

      - name: unzip source code
        if:  matrix.arch == 'amd64'
        run: unzip -q source.zip

      - uses: docker/login-action@v1
        if: matrix.arch == 'amd64'
        with:
          username: ${{ secrets.DOCKER_HUB_USER }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}

      - name: prepare for docker-action-parms
        id: pre-meta
        run: |
          img=$(echo ${{ matrix.os }} | sed 's#\([0-9.]\+\)$#:\1#g')
          emqx_name=${{ matrix.profile }}
          img_suffix=${{ matrix.arch }}
          img_labels="org.opencontainers.image.otp.version=${{ matrix.otp }}"

          if [ ${{ matrix.build_elixir }} = "with_elixir" ]; then
            emqx_name="emqx-elixir"
            img_suffix="elixir-${{ matrix.arch }}"
            img_labels="org.opencontainers.image.elixir.version=${{ matrix.elixir }}\n$img_labels"
          fi
          echo "::set-output name=img::${img}"
          echo "::set-output name=emqx_name::${emqx_name}"
          echo "::set-output name=img_suffix::${img_suffix}"
          echo "::set-output name=img_labels::${img_labels}"

      # NOTE, Pls make sure this is identical as the one in job 'docker'
      - uses: docker/metadata-action@v3
        if: matrix.arch == 'amd64'
        id: meta
        with:
          images: ${{ github.repository_owner }}/${{ matrix.profile }}
          flavor: |
            latest=false
            suffix=-${{ steps.pre-meta.outputs.img_suffix }}
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=ref,event=tag
            type=semver,pattern={{version}}
          labels:
            ${{ steps.pre-meta.outputs.img_labels }}

      - name: update manifest for multiarch image
        if: matrix.arch == 'amd64'
        working-directory: source
        run: |
          IsPushLatest=${{ github.event_name == 'release' && !github.event.release.prerelease }};
          scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" "$IsPushLatest"

      - uses: aws-actions/configure-aws-credentials@v1
        if: matrix.profile == 'emqx' && matrix.arch == 'amd64'
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
      - name: Docker push to aws ecr
        if: matrix.profile == 'emqx' && matrix.arch == 'amd64'
        run: |
          aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
          IsPushLatest=${{ github.event_name == 'release' && !github.event.release.prerelease }};
          scripts/docker-create-push-manifests.sh "public.ecr.aws/${{ steps.meta.outputs.tags }}" "$IsPushLatest"

  delete-artifact:
    runs-on: ubuntu-20.04
    needs: [prepare, mac, linux, docker]
    steps:
    - uses: geekyeggo/delete-artifact@v1
      with:
        name: source

  upload:
    runs-on: ubuntu-20.04

    if: startsWith(github.ref, 'refs/tags/')

    needs: [prepare, mac, linux, docker]

    strategy:
      fail-fast: false
      matrix:
        profile:
          - emqx-edge
          - emqx
          - emqx-enterprise
        otp:
          - 24.2.1-1
        include:
          - profile: emqx
            otp: windows # otp version on windows is rather fixed
    steps:
    - uses: actions/checkout@v2
    - name: get_version
      run: |
        echo 'version<<EOF' >> $GITHUB_ENV
        echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g" >> $GITHUB_ENV
        echo 'EOF' >> $GITHUB_ENV
    - uses: actions/download-artifact@v2
      with:
        name: ${{ matrix.profile }}-${{ matrix.otp }}
        path: ./_packages/${{ matrix.profile }}
    - name: install dos2unix
      run: sudo apt-get update && sudo apt install -y dos2unix
    - name: get packages
      run: |
        set -e -u
        cd _packages/${{ matrix.profile }}
        for var in $( ls |grep emqx |grep -v sha256); do
          dos2unix $var.sha256
          echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
        done
        cd -
    - name: upload aws s3
      run: |
        set -e -u
        PROFILE=${{ matrix.profile }}
        if [ $PROFILE = 'emqx' ]; then
            s3dir='emqx-ce'
        elif [ $PROFILE = 'emqx-enterprise' ]; then
            s3dir='emqx-ee'
        elif [ $PROFILE = 'emqx-edge' ]; then
            s3dir='emqx-edge'
        else
            echo "unknown profile $PROFILE"
            exit 1
        fi
        aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws configure set default.region ${{ secrets.AWS_DEFAULT_REGION }}

        aws s3 cp --recursive _packages/${{ matrix.profile }} s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ env.version }}
        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ env.version }}/*"
    - uses: Rory-Z/upload-release-asset@v1
      if: github.event_name == 'release'
      with:
        repo: emqx
        path: "_packages/${{ matrix.profile }}/emqx-*"
        token: ${{ github.token }}
    - name: update to emqx.io
      if: github.event_name == 'release'
      run: |
        set -e -x -u
        curl -w %{http_code} \
             --insecure \
             -H "Content-Type: application/json" \
             -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
             -X POST \
             -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
             ${{ secrets.EMQX_IO_RELEASE_API }}
    - name: update repo.emqx.io
      if: github.event_name == 'release'
      run: |
        if [ "${{ matrix.profile }}" = 'emqx-enterprise' ]; then
          BOOL_FLAG_NAME="emqx_ee"
        else
          BOOL_FLAG_NAME="emqx_ce"
        fi
        curl --silent --show-error \
          -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
          -H "Accept: application/vnd.github.v3+json" \
          -X POST \
          -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\", \"${BOOL_FLAG_NAME}\": \"true\"}}" \
          "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_repos.yaml/dispatches"
    - name: update homebrew packages
      if: github.event_name == 'release' && matrix.profile == 'emqx'
      run: |
        if [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
            curl --silent --show-error \
              -H "Authorization: token ${{ secrets.CI_GIT_TOKEN }}" \
              -H "Accept: application/vnd.github.v3+json" \
              -X POST \
              -d "{\"ref\":\"v1.0.4\",\"inputs\":{\"version\": \"${{ env.version }}\"}}" \
              "https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_emqx_homebrew.yaml/dispatches"
        fi
    - uses: geekyeggo/delete-artifact@v1
      with:
        name: ${{ matrix.profile }}