emqx4.3/.github/workflows/build_packages.yaml

name: Cross build packages

concurrency:
  group: build-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  schedule:
  - cron:  '0 */6 * * *'
  push:
    branches:
    - 'ci/**'
    tags:
    - v*
    - e*
  workflow_dispatch:
    inputs:
      branch_or_tag:
        required: false
      profile:
        required: false

jobs:
  prepare:
    runs-on: ubuntu-22.04
    if: (github.repository_owner == 'emqx' && github.event_name == 'schedule') || github.event_name != 'schedule'
    container: ghcr.io/emqx/emqx-builder/5.0-33:1.13.4-24.3.4.2-3-ubuntu22.04
    outputs:
      BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
      IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
      VERSION: ${{ steps.get_profile.outputs.VERSION }}

    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.inputs.branch_or_tag }} # when input is not given, the event tag is used
          path: source
          fetch-depth: 0

      - name: Get profile to build
        id: get_profile
        run: |
          cd source
          git config --global --add safe.directory "$(pwd)"
          tag=${{ github.ref }}
          if git describe --tags --match "[v|e]*" --exact; then
            echo "WARN: This is an exact git tag, will publish release"
            is_exact_tag='true'
          else
            echo "WARN: This is NOT an exact git tag, will not publish release"
            is_exact_tag='false'
          fi
          echo "IS_EXACT_TAG=${is_exact_tag}" >> $GITHUB_OUTPUT
          case $tag in
            refs/tags/v*)
              PROFILE='emqx'
              ;;
            refs/tags/e*)
              PROFILE=emqx-enterprise
              ;;
            *)
              PROFILE=${{ github.event.inputs.profile }}
              case "$PROFILE" in
                emqx)
                  true
                  ;;
                emqx-enterprise)
                  true
                  ;;
                *)
                  # maybe triggered from schedule
                  echo "WARN: \"$PROFILE\" is not a valid profile."
                  echo "building the default profile 'emqx' instead"
                  PROFILE='emqx'
                  ;;
              esac
              ;;
          esac
          echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT
          echo "VERSION=$(./pkg-vsn.sh $PROFILE)" >> $GITHUB_OUTPUT
      - name: get_all_deps
        run: |
          make -C source deps-all
          zip -ryq source.zip source/* source/.[^.]*
      - uses: actions/upload-artifact@v3
        with:
          name: source
          path: source.zip

  windows:
    runs-on: windows-2019
    if: startsWith(github.ref_name, 'v')
    needs: prepare
    strategy:
      fail-fast: false
      matrix:
        profile: # for now only CE for windows
          - emqx
    steps:
    - uses: actions/download-artifact@v3
      with:
        name: source
        path: .
    - name: unzip source code
      run: Expand-Archive -Path source.zip -DestinationPath ./
    - uses: ilammy/msvc-dev-cmd@v1.12.0
    - uses: erlef/setup-beam@v1.15.2
      with:
        otp-version: 24.3.4.6
    - name: build
      env:
        PYTHON: python
        DIAGNOSTIC: 1
      working-directory: source
      run: |
        # ensure crypto app (openssl)
        erl -eval "erlang:display(crypto:info_lib())" -s init stop
        make ${{ matrix.profile }}-tgz
    - name: run emqx
      timeout-minutes: 5
      working-directory: source
      run: |
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx start
        Start-Sleep -s 5
        echo "EMQX started"
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx stop
        echo "EMQX stopped"
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx install
        echo "EMQX installed"
        ./_build/${{ matrix.profile }}/rel/emqx/bin/emqx uninstall
        echo "EMQX uninstalled"
    - uses: actions/upload-artifact@v3
      if: success()
      with:
        name: ${{ matrix.profile }}
        path: source/_packages/${{ matrix.profile }}/
    - name: Send notification to Slack
      uses: slackapi/slack-github-action@v1.23.0
      if: failure() && github.event_name == 'schedule'
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
      with:
        payload: |
          {"text": "Scheduled run of ${{ github.workflow }}@Windows failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}

  mac:
    needs: prepare
    strategy:
      fail-fast: false
      matrix:
        profile:
          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
        otp:
          - 24.3.4.2-3
        os:
          - macos-11
          - macos-12
          - macos-12-arm64
    runs-on: ${{ matrix.os }}
    steps:
    - uses: emqx/self-hosted-cleanup-action@v1.0.3
    - uses: actions/download-artifact@v3
      with:
        name: source
        path: .
    - name: unzip source code
      run: |
        ln -s . source
        unzip -o -q source.zip
        rm source source.zip
    - uses: ./.github/actions/package-macos
      with:
        profile: ${{ matrix.profile }}
        otp: ${{ matrix.otp }}
        os: ${{ matrix.os }}
        apple_id_password: ${{ secrets.APPLE_ID_PASSWORD }}
        apple_developer_identity: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
        apple_developer_id_bundle: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
        apple_developer_id_bundle_password: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
    - uses: actions/upload-artifact@v3
      if: success()
      with:
        name: ${{ matrix.profile }}
        path: _packages/${{ matrix.profile }}/
    - name: Send notification to Slack
      uses: slackapi/slack-github-action@v1.23.0
      if: failure() && github.event_name == 'schedule'
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
      with:
        payload: |
          {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}

  linux:
    needs: prepare
    runs-on: ${{ matrix.build_machine }}
    container:
      image: "ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"

    strategy:
      fail-fast: false
      matrix:
        profile:
          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
        otp:
          - 24.3.4.2-3
        arch:
          - amd64
          - arm64
        os:
          - ubuntu22.04
          - ubuntu20.04
          - ubuntu18.04
          - debian11
          - debian10
          - el9
          - el8
          - el7
          - amzn2
        build_machine:
          - aws-arm64
          - ubuntu-22.04
        builder:
          - 5.0-33
        elixir:
          - 1.13.4
        exclude:
        - arch: arm64
          build_machine: ubuntu-22.04
        - arch: amd64
          build_machine: aws-arm64
        include:
          - profile: emqx
            otp: 25.1.2-3
            arch: amd64
            os: ubuntu22.04
            build_machine: ubuntu-22.04
            builder: 5.0-33
            elixir: 1.13.4
            release_with: elixir
          - profile: emqx
            otp: 25.1.2-3
            arch: amd64
            os: amzn2
            build_machine: ubuntu-22.04
            builder: 5.0-33
            elixir: 1.13.4
            release_with: elixir

    defaults:
      run:
        shell: bash

    steps:
    - uses: AutoModality/action-clean@v1
      if: matrix.build_machine == 'aws-arm64'
    - uses: actions/download-artifact@v3
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip
    - name: tmp fix for el9
      if: matrix.os == 'el9'
      run: |
        set -eu
        dnf install -y krb5-devel
    - name: build emqx packages
      working-directory: source
      env:
        BUILDER: ${{ matrix.builder }}
        ELIXIR: ${{ matrix.elixir }}
        OTP: ${{ matrix.otp }}
        PROFILE: ${{ matrix.profile }}
        ARCH: ${{ matrix.arch }}
        SYSTEM: ${{ matrix.os }}
      run: |
        set -eu
        git config --global --add safe.directory "/__w/emqx/emqx"
        # Align path for CMake caches
        if [ ! "$PWD" = "/emqx" ]; then
          ln -s $PWD /emqx
          cd /emqx
        fi
        echo "pwd is $PWD"
        PKGTYPES="tgz pkg"
        IS_ELIXIR="no"
        if [ ${{ matrix.release_with }} == 'elixir' ]; then
          PKGTYPES="tgz"
          # set Elixir build flag
          IS_ELIXIR="yes"
        fi
        for PKGTYPE in ${PKGTYPES};
        do
          ./scripts/buildx.sh \
            --profile "${PROFILE}" \
            --pkgtype "${PKGTYPE}" \
            --arch "${ARCH}" \
            --elixir "${IS_ELIXIR}" \
            --builder "ghcr.io/emqx/emqx-builder/${BUILDER}:${ELIXIR}-${OTP}-${SYSTEM}"
        done
    - uses: actions/upload-artifact@v3
      if: success()
      with:
        name: ${{ matrix.profile }}
        path: source/_packages/${{ matrix.profile }}/
    - name: Send notification to Slack
      uses: slackapi/slack-github-action@v1.23.0
      if: failure() && github.event_name == 'schedule'
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
      with:
        payload: |
          {"text": "Scheduled run of ${{ github.workflow }}@${{ matrix.os }} failed: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"}

  publish_artifacts:
    runs-on: ubuntu-22.04
    needs: [prepare, mac, linux]
    if: needs.prepare.outputs.IS_EXACT_TAG && github.event_name != 'schedule'
    strategy:
      fail-fast: false
      matrix:
        profile:
          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
    steps:
    - uses: actions/download-artifact@v3
      with:
        name: ${{ matrix.profile }}
        path: packages/${{ matrix.profile }}
    - name: install dos2unix
      run: sudo apt-get update && sudo apt install -y dos2unix
    - name: get packages
      run: |
        set -e -u
        cd packages/${{ matrix.profile }}
        # fix the .sha256 file format
        for var in $(ls | grep emqx | grep -v sha256); do
          dos2unix $var.sha256
          echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
        done
        cd -
    - uses: aws-actions/configure-aws-credentials@v2
      with:
        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
    - name: upload to aws s3
      env:
        PROFILE: ${{ matrix.profile }}
      run: |
        set -e -u
        if [ $PROFILE = 'emqx' ]; then
            s3dir='emqx-ce'
        elif [ $PROFILE = 'emqx-enterprise' ]; then
            s3dir='emqx-ee'
        else
            echo "unknown profile $PROFILE"
            exit 1
        fi
        aws s3 cp --recursive packages/$PROFILE s3://${{ secrets.AWS_S3_BUCKET }}/$s3dir/${{ github.ref_name }}
        aws cloudfront create-invalidation --distribution-id ${{ secrets.AWS_CLOUDFRONT_ID }} --paths "/$s3dir/${{ github.ref_name }}/*"
    - name: Push to packagecloud.io
      env:
        PROFILE: ${{ matrix.profile }}
        VERSION: ${{ needs.prepare.outputs.VERSION }}
        PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
      run: |
        set -eu
        REPO=$PROFILE
        if [ $PROFILE = 'emqx-enterprise' ]; then
            REPO='emqx-enterprise5'
        fi
        function push() {
          docker run -t --rm -e PACKAGECLOUD_TOKEN=$PACKAGECLOUD_TOKEN -v $(pwd)/$2:/w/$2 -w /w ghcr.io/emqx/package_cloud push emqx/$REPO/$1 $2
        }
        push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-amd64.deb"
        push "debian/buster" "packages/$PROFILE/$PROFILE-$VERSION-debian10-arm64.deb"
        push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-amd64.deb"
        push "debian/bullseye" "packages/$PROFILE/$PROFILE-$VERSION-debian11-arm64.deb"
        push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-amd64.deb"
        push "ubuntu/bionic" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu18.04-arm64.deb"
        push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-amd64.deb"
        push "ubuntu/focal" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu20.04-arm64.deb"
        push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-amd64.deb"
        push "ubuntu/jammy" "packages/$PROFILE/$PROFILE-$VERSION-ubuntu22.04-arm64.deb"
        push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-amd64.rpm"
        push "el/6" "packages/$PROFILE/$PROFILE-$VERSION-amzn2-arm64.rpm"
        push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-amd64.rpm"
        push "el/7" "packages/$PROFILE/$PROFILE-$VERSION-el7-arm64.rpm"
        push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-amd64.rpm"
        push "el/8" "packages/$PROFILE/$PROFILE-$VERSION-el8-arm64.rpm"
        push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-amd64.rpm"
        push "el/9" "packages/$PROFILE/$PROFILE-$VERSION-el9-arm64.rpm"