emqx4.3/.github/workflows/build_packages.yaml

name: Cross build packages

on:
  push:
    tags:
      - v*
  release:
    types:
      - published
  pull_request:
  workflow_dispatch:
  repository_dispatch:
    types: [run_actions]

jobs:
  windows:
    runs-on: windows-2019

    if: startsWith(github.ref, 'refs/tags/')

    steps:
    - uses: actions/checkout@v1
    - uses: ilammy/msvc-dev-cmd@v1
    - uses: gleam-lang/setup-erlang@v1.1.0
      id: install_erlang
      with:
        otp-version: 23.2
    - name: build
      run: |
        #     set-executionpolicy remotesigned -s cu
        #     iex (new-object net.webclient).downloadstring('https://get.scoop.sh')
        #     # $env:path + ";" + $env:USERPROFILE + "\scoop\shims" + ';C:\Program Files\erl10.4\bin'
        #     [environment]::SetEnvironmentvariable("Path", ";" + $env:USERPROFILE + "\scoop\shims")
        #     [environment]::SetEnvironmentvariable("Path", ';C:\Program Files\erl10.4\bin')
        #     scoop bucket add extras https://github.com/lukesampson/scoop-extras.git
        #     scoop update
        #     scoop install sudo curl vcredist2013

        $env:PATH = "${{ steps.install_erlang.outputs.erlpath }}\bin;$env:PATH"

        $version = $( "${{ github.ref }}" -replace "^(.*)/(.*)/" )
        if ($version -match "^v[0-9]+\.[0-9]+(\.[0-9]+)?") {
          $regex = "[0-9]+\.[0-9]+(-alpha|-beta|-rc)?\.[0-9]"
          $pkg_name = "emqx-windows-$([regex]::matches($version, $regex).value).zip"
          }
        else {
          $pkg_name = "emqx-windows-$($version -replace '/').zip"
          }

        make deps-emqx || cat rebar3.crashdump
        $rebar3 = $env:USERPROFILE + "\rebar3"
        (New-Object System.Net.WebClient).DownloadFile('https://s3.amazonaws.com/rebar3/rebar3', $rebar3)
        cd _build/emqx/lib/jiffy/
        escript $rebar3 compile
        cd ../../../../

        make emqx
        mkdir -p _packages/emqx
        Compress-Archive -Path _build/emqx/rel/emqx -DestinationPath _build/emqx/rel/$pkg_name
        mv _build/emqx/rel/$pkg_name _packages/emqx
        Get-FileHash -Path "_packages/emqx/$pkg_name" | Format-List | grep 'Hash' | awk '{print $3}'  > _packages/emqx/$pkg_name.sha256
    - name: run emqx
      run: |
        ./_build/emqx/rel/emqx/bin/emqx start
        ./_build/emqx/rel/emqx/bin/emqx stop
        ./_build/emqx/rel/emqx/bin/emqx install
        ./_build/emqx/rel/emqx/bin/emqx uninstall
    - uses: actions/upload-artifact@v1
      with:
        name: emqx
        path: ./_packages/emqx/.

  mac:
    runs-on: macos-10.15

    if: startsWith(github.ref, 'refs/tags/')

    steps:
    - uses: actions/checkout@v1
    - name: prepare
      run: |
        brew install curl zip unzip gnu-sed kerl unixodbc freetds
        echo "/usr/local/bin" >> $GITHUB_PATH
        git config --global credential.helper store
    - name: build erlang
      timeout-minutes: 60
      run: |
        kerl build 23.2.2
        kerl install 23.2.2 $HOME/.kerl/23.2.2
    - name: build
      run: |
        . $HOME/.kerl/23.2.2/activate
        make emqx-pkg
    - name: test
      run: |
        pkg_name=$(basename _packages/emqx/emqx-macos-*.zip)
        unzip _packages/emqx/$pkg_name
        gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
        ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
        ready='no'
        for i in {1..10}; do
          if curl -fs 127.0.0.1:18083 > /dev/null; then
            ready='yes'
            break
          fi
          sleep 1
        done
        if [ "$ready" != "yes" ]; then
          echo "Timed out waiting for emqx to be ready"
          cat emqx/log/erlang.log.1
          exit 1
        fi
        ./emqx/bin/emqx_ctl status
        ./emqx/bin/emqx stop
        rm -rf emqx
        openssl dgst -sha256 ./_packages/emqx/$pkg_name | awk '{print $2}'  > ./_packages/emqx/$pkg_name.sha256
    - uses: actions/upload-artifact@v1
      with:
        name: emqx
        path: ./_packages/emqx/.

  linux:
    runs-on: ubuntu-20.04

    strategy:
      matrix:
        arch:
        - amd64
        - arm64
        emqx:
        - emqx
        - emqx-edge
        os:
        - ubuntu20.04
        - ubuntu18.04
        - ubuntu16.04
        - debian10
        - debian9
        - opensuse
        - centos8
        - centos7
        - centos6
        - raspbian10
        - raspbian9
        exclude:
        - os: raspbian9
          arch: amd64
        - os: raspbian9
          emqx: emqx
        - os: raspbian10
          arch: amd64
        - os: raspbian10
          emqx: emqx
        - os: centos6
          arch: arm64

    defaults:
      run:
        shell: bash

    steps:
    - name: prepare docker
      run: |
        mkdir -p $HOME/.docker
        echo '{ "experimental": "enabled" }' | tee $HOME/.docker/config.json
        echo '{ "experimental": true, "storage-driver": "overlay2", "max-concurrent-downloads": 50, "max-concurrent-uploads": 50}' | sudo tee /etc/docker/daemon.json
        sudo systemctl restart docker
        docker info
        docker buildx create --use --name mybuild
        docker run --rm --privileged tonistiigi/binfmt --install all
    - uses: actions/checkout@v1
    - name: get deps
      env:
        ERL_OTP: erl23.2.2
      run: |
        docker run -i --rm \
            -e GITHUB_RUN_ID=$GITHUB_RUN_ID \
            -e GITHUB_REF=$GITHUB_REF \
            -v $(pwd):/emqx \
            -w /emqx \
            emqx/build-env:${ERL_OTP}-debian10 \
            bash -c "make deps-all"
    - name: downloads emqx zip packages
      env:
        EMQX: ${{ matrix.emqx }}
        ARCH: ${{ matrix.arch }}
        SYSTEM: ${{ matrix.os }}
      run: |
        set -e -u -x
        if [ $EMQX = "emqx-edge" ];then broker="emqx-edge"; else broker="emqx-ce"; fi
        if [ $ARCH = "arm64" ];then arch="aarch64"; else arch="x86_64"; fi

        vsn="$(grep -oE '\{vsn, (.*)\}' src/emqx.app.src | sed -r 's/\{vsn, (.*)\}/\1/g' | sed 's/\"//g')"
        pre_vsn="$(echo $vsn | grep -oE '^[0-9]+.[0-9]')"
        old_vsns=($(git tag -l "$pre_vsn.[0-9]" | sed "s/$vsn//"))

        mkdir -p tmp/relup_packages/$EMQX
        cd tmp/relup_packages/$EMQX
        for tag in ${old_vsns[@]};do
          if [ ! -z "$(echo  $(curl -I -m 10 -o /dev/null -s -w %{http_code} https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/v${tag#[e|v]}/$EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip) | grep -oE "^[23]+")" ];then
            wget https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/v${tag#[e|v]}/$EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip
            wget https://s3-us-west-2.amazonaws.com/packages.emqx/$broker/v${tag#[e|v]}/$EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip.sha256
            echo "$(cat $EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip.sha256) $EMQX-$SYSTEM-${tag#[e|v]}-$arch.zip" | sha256sum -c || exit 1
          fi
        done
        cd -
    - name: build emqx packages
      if: (matrix.arch  == 'amd64' && matrix.emqx == 'emqx') || startsWith(github.ref, 'refs/tags/')
      env:
        ERL_OTP: erl23.2.2
        EMQX: ${{ matrix.emqx }}
        ARCH: ${{ matrix.arch }}
        SYSTEM: ${{ matrix.os }}
      run: |
        set -e -u -x
        docker buildx build --no-cache \
          --platform=linux/$ARCH \
          -t cross_build_emqx_for_$SYSTEM \
          -f .ci/build_packages/Dockerfile \
          --build-arg BUILD_FROM=emqx/build-env:$ERL_OTP-$SYSTEM \
          --build-arg EMQX_NAME=$EMQX \
          --output type=tar,dest=/tmp/cross-build-$EMQX-for-$SYSTEM.tar .

        mkdir -p /tmp/packages/$EMQX
        tar -xvf /tmp/cross-build-$EMQX-for-$SYSTEM.tar --wildcards emqx/_packages/$EMQX/*
        mv emqx/_packages/$EMQX/* /tmp/packages/$EMQX/
        rm -rf /tmp/cross-build-$EMQX-for-$SYSTEM.tar

        docker rm -f $(docker ps -a -q)
        docker volume prune -f
    - name: create sha256
      env:
        EMQX: ${{ matrix.emqx }}
      run: |
        if [ -d /tmp/packages/$EMQX ]; then
          cd /tmp/packages/$EMQX
            for var in $(ls emqx-* ); do
              bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"
            done
          cd -
        fi
    - uses: actions/upload-artifact@v1
      if: startsWith(github.ref, 'refs/tags/')
      with:
        name: ${{ matrix.emqx }}
        path: /tmp/packages/${{ matrix.emqx }}/.

  docker:
    runs-on: ubuntu-20.04

    if: startsWith(github.ref, 'refs/tags/')

    strategy:
      matrix:
        arch:
        - [amd64, x86_64]
        - [arm64v8, aarch64]
        - [arm32v7, arm]
        - [i386, i386]
        - [s390x, s390x]

    steps:
    - uses: actions/checkout@v1
    - name: get deps
      env:
        ERL_OTP: erl23.2.2
      run: |
        docker run -i --rm \
            -e GITHUB_RUN_ID=$GITHUB_RUN_ID \
            -e GITHUB_REF=$GITHUB_REF \
            -v $(pwd):/emqx \
            -w /emqx \
            emqx/build-env:${ERL_OTP}-alpine-amd64 \
            sh -c "make deps-emqx"
    - name: build emqx docker image
      env:
        ARCH: ${{ matrix.arch[0] }}
        QEMU_ARCH: ${{ matrix.arch[1] }}
      run: |
        sudo docker run --rm --privileged multiarch/qemu-user-static --reset -p yes

        sudo TARGET=emqx/emqx ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
        cd _packages/emqx && for var in $(ls emqx-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -

        sudo TARGET=emqx/emqx-edge ARCH=$ARCH QEMU_ARCH=$QEMU_ARCH  make docker
        cd _packages/emqx-edge && for var in $(ls emqx-edge-docker-* ); do sudo bash -c "echo $(sha256sum $var | awk '{print $1}') > $var.sha256"; done && cd -
    - uses: actions/upload-artifact@v1
      with:
        name: emqx
        path: ./_packages/emqx/.
    - uses: actions/upload-artifact@v1
      with:
        name: emqx-edge
        path: ./_packages/emqx-edge/.

  upload:
    runs-on: ubuntu-20.04

    needs: [windows, mac, linux, docker]

    if: startsWith(github.ref, 'refs/tags/')

    steps:
    - uses: actions/download-artifact@v2
      with:
        name: emqx
        path: ./_packages/emqx
    - uses: actions/download-artifact@v2
      with:
        name: emqx-edge
        path: ./_packages/emqx-edge
    - name: install dos2unix
      run: sudo apt-get update && sudo apt install -y dos2unix
    - name: get packages
      run: |
        set -e -x -u
        for EMQX in emqx emqx-edge; do
          cd _packages/$EMQX
          for var in $( ls |grep emqx |grep -v sha256); do
            dos2unix $var.sha256
            echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
          done
          cd -
        done
    - name: upload aws s3
      run: |
        set -e -x -u
        version=$(echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g")
        curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
        unzip -q awscliv2.zip
        sudo ./aws/install
        aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws configure set default.region us-west-2

        aws s3 cp --recursive _packages/emqx s3://packages.emqx/emqx-ce/$version
        aws s3 cp --recursive _packages/emqx-edge s3://packages.emqx/emqx-edge/$version
        aws cloudfront create-invalidation --distribution-id E170YEULGLT8XB --paths "/emqx-ce/$version/*,/emqx-edge/$version/*"

        mkdir packages
        mv _packages/emqx/* packages
        mv _packages/emqx-edge/* packages
    - uses: actions/checkout@v2
      with:
        path: emqx
    - name: update to github and emqx.io
      if: github.event_name == 'release'
      run: |
        set -e -x -u
        version=$(echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g")
        cd packages
        for var in $(ls); do
          ../emqx/.ci/build_packages/upload_github_release_asset.sh owner=emqx repo=emqx tag=$version filename=$var github_api_token=$(echo ${{ secrets.AccessToken }})
          sleep 1
        done
        curl -w %{http_code} \
             --insecure \
             -H "Content-Type: application/json" \
             -H "token: ${{ secrets.EMQX_IO_TOKEN }}" \
             -X POST \
             -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${version}\" }" \
             ${{ secrets.EMQX_IO_RELEASE_API }}
    - name: push docker image to docker hub
      if: github.event_name == 'release'
      run: |
        set -e -x -u
        version=$(echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g")
        sudo make -C emqx docker-prepare
        cd packages && for var in $(ls |grep docker |grep -v sha256); do unzip $var; sudo docker load < ${var%.*}; rm -f ${var%.*}; done && cd -
        echo ${{ secrets.DOCKER_HUB_TOKEN }} |sudo docker login -u ${{ secrets.DOCKER_HUB_USER }} --password-stdin
        sudo TARGET=emqx/emqx make -C emqx docker-push
        sudo TARGET=emqx/emqx make -C emqx docker-manifest-list
        sudo TARGET=emqx/emqx-edge make -C emqx docker-push
        sudo TARGET=emqx/emqx-edge make -C emqx docker-manifest-list
    - name: update repo.emqx.io
      if: github.event_name == 'release'
      run: |
        set -e -x -u
        version=$(echo ${{ github.ref }} | sed -r  "s ^refs/heads/|^refs/tags/(.*) \1 g")
        curl \
          -H "Authorization: token ${{ secrets.AccessToken }}" \
          -H "Accept: application/vnd.github.v3+json" \
          -X POST \
          -d "{\"ref\":\"v1.0.0\",\"inputs\":{\"version\": \"${version}\", \"emqx_ce\": \"true\"}}" \
          https://api.github.com/repos/emqx/emqx-ci-helper/actions/workflows/update_repos.yaml/dispatches
    - uses: geekyeggo/delete-artifact@v1
      with:
        name: emqx
    - uses: geekyeggo/delete-artifact@v1
      with:
        name: emqx-edge
    # - name: update homebrew packages
    #   run: |
    #     version=$(echo ${{ github.ref }} | sed -r  "s .*/.*/(.*) \1 g")
    #     if [ ! -z $(echo $version | grep -oE "v[0-9]+\.[0-9]+(\.[0-9]+)?") ] && [ -z $(echo $version | grep -oE "(alpha|beta|rc)\.[0-9]") ]; then
    #       curl -H "Authorization: token ${{ secrets.AccessToken }}" -H "Accept: application/vnd.github.everest-preview+json" -H "Content-Type: application/json" -X POST -d "{\"event_type\":\"update_homebrew\",\"client_payload\":{\"version\": \"$version\"}}" https://api.github.com/repos/emqx/emqx-packages-docker/dispatches
    #     fi