emqx4.3/.github/workflows/build_and_push_docker_images.yaml

name: Build and push docker images

concurrency:
  group: docker-build-${{ github.event_name }}-${{ github.ref }}
  cancel-in-progress: true

on:
  push:
    tags:
    - v*
    - e*
  release:
    types:
      - published
  workflow_dispatch:
    inputs:
      branch_or_tag:
        required: false
      profile:
        required: false

jobs:
  prepare:
    runs-on: ubuntu-20.04
    # prepare source with any OTP version, no need for a matrix
    container: "ghcr.io/emqx/emqx-builder/5.0-26:1.13.4-24.3.4.2-1-ubuntu20.04"

    outputs:
      BUILD_PROFILE: ${{ steps.get_profile.outputs.BUILD_PROFILE }}
      IS_DOCKER_LATEST: ${{ steps.get_profile.outputs.IS_DOCKER_LATEST }}
      IS_EXACT_TAG: ${{ steps.get_profile.outputs.IS_EXACT_TAG }}
      DOCKER_TAG_VERSION: ${{ steps.get_profile.outputs.DOCKER_TAG_VERSION }}

    steps:
      - uses: actions/checkout@v3
        with:
          ref: ${{ github.event.inputs.branch_or_tag }} # when input is not given, the event tag is used
          path: source
          fetch-depth: 0

      - name: Get profiles to build
        id: get_profile
        run: |
          cd source
          tag=${{ github.ref }}
          # tag docker-latest-ce or docker-latest-ee
          if git describe --tags --exact --match 'docker-latest-*' 2>/dev/null; then
            echo 'docker_latest=true due to docker-latest-* tag'
            docker_latest=true
          elif [ "${{ github.event_name }}" = "release" ]; then
            echo 'docker_latest=true due to release'
            docker_latest=true
          else
            echo 'docker_latest=false'
            docker_latest=false
          fi
          if git describe --tags --match "[v|e]*" --exact; then
            echo "This is an exact git tag, will publish images"
            is_exact='true'
          else
            echo "This is NOT an exact git tag, will not publish images"
            is_exact='false'
          fi
          case $tag in
            refs/tags/v*)
              PROFILE='emqx'
              ;;
            refs/tags/e*)
              PROFILE=emqx-enterprise
              ;;
            *)
              PROFILE=${{ github.event.inputs.profile }}
              case "$PROFILE" in
                emqx)
                  true
                  ;;
                emqx-enterprise)
                  true
                  ;;
                *)
                  echo "ERROR: Failed to resolve build profile"
                  exit 1
                  ;;
              esac
              ;;
          esac
          VSN="$(./pkg-vsn.sh "$PROFILE")"
          echo "Building $PROFILE image with tag $VSN (latest=$docker_latest)"
          echo "IS_DOCKER_LATEST=$docker_latest" >> $GITHUB_OUTPUT
          echo "IS_EXACT_TAG=$is_exact" >> $GITHUB_OUTPUT
          echo "BUILD_PROFILE=$PROFILE" >> $GITHUB_OUTPUT
          echo "DOCKER_TAG_VERSION=$VSN" >> $GITHUB_OUTPUT
      - name: get_all_deps
        run: |
          make -C source deps-all
          zip -ryq source.zip source/* source/.[^.]*
      - uses: actions/upload-artifact@v3
        with:
          name: source
          path: source.zip

  docker:
    runs-on: ${{ matrix.arch[1] }}
    needs: prepare

    strategy:
      fail-fast: false
      matrix:
        arch:
          - [amd64, ubuntu-20.04]
          - [arm64, aws-arm64]
        profile:
          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
        registry:
          - 'docker.io'
          - 'public.ecr.aws'
        os:
          - [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"]
          - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
        # NOTE: 'otp' and 'elixir' are to configure emqx-builder image
        #       only support latest otp and elixir, not a matrix
        builder:
          - 5.0-26 # update to latest
        otp:
          - 24.3.4.2-1 # switch to 25 once ready to release 5.1
        elixir:
          - 1.13.4 # update to latest
        exclude: # TODO: publish enterprise to ecr too?
          - registry: 'public.ecr.aws'
            profile: emqx-enterprise
    steps:
    - uses: AutoModality/action-clean@v1
      if: matrix.arch[1] == 'aws-arm64'
    - uses: actions/download-artifact@v3
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip

    - uses: docker/setup-buildx-action@v2

    - name: Login for docker.
      uses: docker/login-action@v2
      if: matrix.registry == 'docker.io'
      with:
        username: ${{ secrets.DOCKER_HUB_USER }}
        password: ${{ secrets.DOCKER_HUB_TOKEN }}

    - name: Login for AWS ECR
      uses: docker/login-action@v2
      if: matrix.registry == 'public.ecr.aws'
      with:
        registry: public.ecr.aws
        username: ${{ secrets.AWS_ACCESS_KEY_ID }}
        password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        ecr: true

    - uses: ./source/.github/actions/docker-meta
      id: meta
      with:
        profile: ${{ matrix.profile }}
        registry: ${{ matrix.registry }}
        arch: ${{ matrix.arch[0] }}
        otp: ${{ matrix.otp }}
        builder_base: ${{ matrix.os[0] }}
        owner: ${{ github.repository_owner }}
        docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}

    - uses: docker/build-push-action@v3
      with:
        push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }}
        pull: true
        no-cache: true
        platforms: linux/${{ matrix.arch[0] }}
        tags: ${{ steps.meta.outputs.tags }}
        labels: ${{ steps.meta.outputs.labels }}
        build-args: |
          BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
          RUN_FROM=${{ matrix.os[1] }}
          EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
        file: source/${{ matrix.os[2] }}
        context: source

    - name: Docker Hub Description
      if: matrix.registry == 'docker.io'
      uses: peter-evans/dockerhub-description@v3
      with:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_PASSWORD }}
        repository: "emqx/${{ needs.prepare.outputs.BUILD_PROFILE }}"
        readme-filepath: ./source/deploy/docker/README.md
        short-description: "The most scalable open-source MQTT broker for IoT, IIoT, connected vehicles, and more."

  docker-elixir:
    runs-on: ${{ matrix.arch[1] }}
    needs: prepare
    # do not build elixir images for ee for now
    if: needs.prepare.outputs.BUILD_PROFILE == 'emqx'

    strategy:
      fail-fast: false
      matrix:
        arch:
          - [amd64, ubuntu-20.04]
          - [arm64, aws-arm64]
        profile:
          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
        registry:
          - 'docker.io'
        os:
          - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
        builder:
          - 5.0-26 # update to latest
        otp:
          - 25.1.2-2 # update to latest
        elixir:
          - 1.13.4 # update to latest

    steps:
    - uses: AutoModality/action-clean@v1
      if: matrix.arch[1] == 'aws-arm64'
    - uses: actions/download-artifact@v3
      with:
        name: source
        path: .
    - name: unzip source code
      run: unzip -q source.zip

    - uses: docker/setup-buildx-action@v2

    - name: Login for docker.
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKER_HUB_USER }}
        password: ${{ secrets.DOCKER_HUB_TOKEN }}

    - uses: ./source/.github/actions/docker-meta
      id: meta
      with:
        profile: ${{ matrix.profile }}
        registry: ${{ matrix.registry }}
        arch: ${{ matrix.arch[0] }}
        otp: ${{ matrix.otp }}
        elixir: ${{ matrix.elixir }}
        builder_base: ${{ matrix.os[0] }}
        owner: ${{ github.repository_owner }}
        docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}

    - uses: docker/build-push-action@v3
      with:
        push: ${{ needs.prepare.outputs.IS_EXACT_TAG == 'true' || github.repository_owner != 'emqx' }}
        pull: true
        no-cache: true
        platforms: linux/${{ matrix.arch[0] }}
        tags: ${{ steps.meta.outputs.tags }}
        labels: ${{ steps.meta.outputs.labels }}
        build-args: |
          BUILD_FROM=ghcr.io/emqx/emqx-builder/${{ matrix.builder }}:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os[0] }}
          RUN_FROM=${{ matrix.os[1] }}
          EMQX_NAME=${{ steps.meta.outputs.emqx_name }}
        file: source/${{ matrix.os[2] }}
        context: source

  docker-push-multi-arch-manifest:
    # note, we only run on amd64
    if: needs.prepare.outputs.IS_EXACT_TAG
    needs:
      - prepare
      - docker
    runs-on: ${{ matrix.arch[1] }}
    strategy:
      fail-fast: false
      matrix:
        arch:
          - [amd64, ubuntu-20.04]
        profile:
          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
        os:
          - [alpine3.15.1, "alpine:3.15.1", "deploy/docker/Dockerfile.alpine"]
          - [debian11, "debian:11-slim", "deploy/docker/Dockerfile"]
        # NOTE: only support latest otp version, not a matrix
        otp:
          - 24.3.4.2-1 # switch to 25 once ready to release 5.1
        registry:
          - 'docker.io'
          - 'public.ecr.aws'
        exclude:
          - registry: 'public.ecr.aws'
            profile: emqx-enterprise

    steps:
      - uses: actions/download-artifact@v3
        with:
          name: source
          path: .

      - name: unzip source code
        run: unzip -q source.zip

      - uses: docker/login-action@v2
        if: matrix.registry == 'docker.io'
        with:
          username: ${{ secrets.DOCKER_HUB_USER }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}

      - uses: docker/login-action@v2
        if: matrix.registry == 'public.ecr.aws'
        with:
          registry: public.ecr.aws
          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          ecr: true

      - uses: ./source/.github/actions/docker-meta
        id: meta
        with:
          profile: ${{ matrix.profile }}
          registry: ${{ matrix.registry }}
          arch: ${{ matrix.arch[0] }}
          otp: ${{ matrix.otp }}
          builder_base: ${{ matrix.os[0] }}
          owner: ${{ github.repository_owner }}
          docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}

      - name: update manifest for multiarch image
        working-directory: source
        run: |
          is_latest="${{ needs.prepare.outputs.IS_DOCKER_LATEST }}"
          scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" "$is_latest"

  docker-elixir-push-multi-arch-manifest:
    # note, we only run on amd64
    # do not build enterprise elixir images for now
    if: needs.prepare.outputs.IS_EXACT_TAG == 'true' && needs.prepare.outputs.BUILD_PROFILE == 'emqx'
    needs:
      - prepare
      - docker-elixir
    runs-on: ${{ matrix.arch[1] }}
    strategy:
      fail-fast: false
      matrix:
        arch:
          - [amd64, ubuntu-20.04]
        profile:
          - ${{ needs.prepare.outputs.BUILD_PROFILE }}
        # NOTE: for docker, only support latest otp version, not a matrix
        otp:
          - 25.1.2-2 # update to latest
        elixir:
          - 1.13.4 # update to latest
        registry:
          - 'docker.io'

    steps:
      - uses: actions/download-artifact@v3
        with:
          name: source
          path: .

      - name: unzip source code
        run: unzip -q source.zip

      - uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_HUB_USER }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}

      - uses: ./source/.github/actions/docker-meta
        id: meta
        with:
          profile: ${{ matrix.profile }}
          registry: ${{ matrix.registry }}
          arch: ${{ matrix.arch[0] }}
          otp: ${{ matrix.otp }}
          elixir: ${{ matrix.elixir }}
          builder_base: ${{ matrix.os[0] }}
          owner: ${{ github.repository_owner }}
          docker_tags: ${{ needs.prepare.outputs.DOCKER_TAG_VERSION }}

      - name: update manifest for multiarch image
        working-directory: source
        run: |
          scripts/docker-create-push-manifests.sh "${{ steps.meta.outputs.tags }}" false