emqx4.3/.github/workflows/build_and_push_docker_images.yaml

name: Build and push docker images

concurrency:
  group: docker-build-${{ github.event_name }}-${{ inputs.profile }}-${{ github.ref }}
  cancel-in-progress: true

on:
  workflow_call:
    inputs:
      profile:
        required: true
        type: string
      latest:
        required: false
        type: string
        default: false
      publish:
        required: false
        type: boolean
        default: false
    secrets:
      DOCKER_HUB_USER:
        required: false
      DOCKER_HUB_TOKEN:
        required: false
      AWS_ACCESS_KEY_ID:
        required: false
      AWS_SECRET_ACCESS_KEY:
        required: false
      AWS_DEFAULT_REGION:
        required: false
      AWS_S3_BUCKET:
        required: false
      AWS_CLOUDFRONT_ID:
        required: false
  workflow_dispatch:
    inputs:
      ref:
        required: false
      profile:
        required: false
        type: string
        default: 'emqx'
      latest:
        required: false
        type: boolean
        default: false
      publish:
        required: false
        type: boolean
        default: false

permissions:
  contents: read

jobs:
  build:
    runs-on: ${{ github.repository_owner == 'emqx' && fromJSON(format('["self-hosted","ephemeral","linux","{0}"]', matrix.arch)) || 'ubuntu-22.04' }}
    outputs:
      PKG_VSN: ${{ steps.build.outputs.PKG_VSN }}

    strategy:
      fail-fast: false
      matrix:
        profile:
          - ${{ inputs.profile }}
          - ${{ inputs.profile }}-elixir
        arch:
          - x64
          - arm64

    steps:
      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
        with:
          ref: ${{ github.event.inputs.ref }}
      - name: build release tarball
        id: build
        env:
          BUILDER_SYSTEM: force_docker
        run: |
          source ./env.sh
          ./scripts/buildx.sh --profile ${{ matrix.profile }} --pkgtype tgz --builder "$EMQX_DOCKER_BUILD_FROM"
          PKG_VSN=$(docker run --rm -v $(pwd):$(pwd) -w $(pwd) -u $(id -u) "$EMQX_DOCKER_BUILD_FROM" ./pkg-vsn.sh "${{ matrix.profile }}")
          echo "PKG_VSN=$PKG_VSN" >> "$GITHUB_OUTPUT"
      - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
        with:
          name: "${{ matrix.profile }}-${{ matrix.arch == 'x64' && 'amd64' || 'arm64' }}.tar.gz"
          path: "_packages/emqx*/emqx-*.tar.gz"
          retention-days: 7
          overwrite: true
          if-no-files-found: error

  docker:
    runs-on: ${{ endsWith(github.repository, '/emqx') && 'ubuntu-22.04' || fromJSON('["self-hosted","ephemeral","linux","x64"]') }}
    needs:
      - build
    defaults:
      run:
        shell: bash

    strategy:
      fail-fast: false
      matrix:
        profile:
          - ["${{ inputs.profile }}", "${{ inputs.profile == 'emqx' && 'docker.io,public.ecr.aws' || 'docker.io' }}"]
          - ["${{ inputs.profile }}-elixir", "${{ inputs.profile == 'emqx' && 'docker.io,public.ecr.aws' || 'docker.io' }}"]

    env:
      PROFILE: ${{ matrix.profile[0] }}
      DOCKER_REGISTRY: ${{ matrix.profile[1] }}
      DOCKER_ORG: ${{ github.repository_owner }}
      DOCKER_LATEST: ${{ inputs.latest }}
      PKG_VSN: ${{ needs.build.outputs.PKG_VSN }}
      EMQX_SOURCE_TYPE: tgz

    steps:
      - uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0
        with:
          ref: ${{ github.event.inputs.ref }}

      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          pattern: "${{ matrix.profile[0] }}-*.tar.gz"
          path: _packages
          merge-multiple: true

      - name: Move artifacts to root directory
        env:
          PROFILE: ${{ inputs.profile }}
        run: |
          ls -lR _packages/$PROFILE
          mv _packages/$PROFILE/*.tar.gz ./

      - name: Enable containerd image store on Docker Engine
        run: |
          echo "$(sudo cat /etc/docker/daemon.json | jq '. += {"features": {"containerd-snapshotter": true}}')" > daemon.json
          sudo mv daemon.json /etc/docker/daemon.json
          sudo systemctl restart docker

      - uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
      - uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1

      - name: Login to hub.docker.com
        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
        if: inputs.publish && contains(matrix.profile[1], 'docker.io')
        with:
          username: ${{ secrets.DOCKER_HUB_USER }}
          password: ${{ secrets.DOCKER_HUB_TOKEN }}

      - name: Login to AWS ECR
        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
        if: inputs.publish && contains(matrix.profile[1], 'public.ecr.aws')
        with:
          registry: public.ecr.aws
          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          ecr: true

      - name: Build docker image
        env:
          DOCKER_PUSH: false
          DOCKER_BUILD_NOCACHE: true
        run: |
          ./build ${PROFILE} docker
          echo "Built tags:"
          echo "==========="
          cat .emqx_docker_image_tags
          echo "==========="
          echo "_EMQX_DOCKER_IMAGE_TAG=$(head -n 1 .emqx_docker_image_tags)" >> $GITHUB_ENV

      - name: Verify that size of docker image is less than 300 MB
        run: |
          docker image inspect $_EMQX_DOCKER_IMAGE_TAG --format='{{.Size}}' | xargs -I {} test {} -lt 300000000

      - name: smoke test
        timeout-minutes: 1
        run: |
          for tag in $(cat .emqx_docker_image_tags); do
            CID=$(docker run -d -p 18083:18083 $tag)
            HTTP_PORT=$(docker inspect --format='{{(index (index .NetworkSettings.Ports "18083/tcp") 0).HostPort}}' $CID)
            ./scripts/test/emqx-smoke-test.sh localhost $HTTP_PORT
            docker rm -f $CID
          done

      - name: dashboard tests
        working-directory: ./scripts/ui-tests
        timeout-minutes: 5
        run: |
          set -eu
          docker compose up --abort-on-container-exit --exit-code-from selenium
          docker compose rm -fsv

      - name: test node_dump
        run: |
          CID=$(docker run -d -P $_EMQX_DOCKER_IMAGE_TAG)
          docker exec -t -u root -w /root $CID bash -c 'apt-get -y update && apt-get -y install net-tools'
          docker exec -t -u root $CID node_dump
          docker rm -f $CID

      - name: export docker image
        run: |
          docker save "${_EMQX_DOCKER_IMAGE_TAG}" | gzip > $PROFILE-docker-$PKG_VSN.tar.gz

      - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
        with:
          name: "${{ env.PROFILE }}-docker"
          path: "${{ env.PROFILE }}-docker-${{ env.PKG_VSN }}.tar.gz"
          retention-days: 7

      - name: Publish docker image
        if: inputs.publish || github.repository_owner != 'emqx'
        env:
          DOCKER_PUSH: true
          DOCKER_BUILD_NOCACHE: false
          DOCKER_PLATFORMS: linux/amd64,linux/arm64
          DOCKER_LOAD: false
        run: |
          ./build ${PROFILE} docker

      - name: Build and publish docker image with Snowflake ODBC driver
        if: (inputs.publish || github.repository_owner != 'emqx') && matrix.profile[0] == 'emqx-enterprise'
        env:
          DOCKER_PUSH: true
          DOCKER_BUILD_NOCACHE: false
          DOCKER_PLATFORMS: linux/amd64,linux/arm64
          DOCKER_LOAD: false
          EMQX_DOCKERFILE: deploy/docker/Dockerfile.sfodbc
        run: |
          export BUILD_FROM="${_EMQX_DOCKER_IMAGE_TAG}"
          export EMQX_IMAGE_TAG="${_EMQX_DOCKER_IMAGE_TAG##docker.io/}-sf"
          ./build ${PROFILE} docker

      - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
        if: inputs.publish || github.repository_owner != 'emqx'
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_DEFAULT_REGION }}

      - name: upload to aws s3
        if: inputs.publish || github.repository_owner != 'emqx'
        env:
          AWS_S3_BUCKET: ${{ secrets.AWS_S3_BUCKET }}
          AWS_CLOUDFRONT_ID: ${{ secrets.AWS_CLOUDFRONT_ID }}
          ORIG_PROFILE: ${{ inputs.profile }}
        run: |
          set -xeuo pipefail
          if [ $ORIG_PROFILE = 'emqx' ]; then
              s3dir="emqx-ce/v$PKG_VSN"
          elif [ $ORIG_PROFILE = 'emqx-enterprise' ]; then
              s3dir="emqx-ee/e$PKG_VSN"
          else
              echo "unknown profile $ORIG_PROFILE"
              exit 1
          fi
          docker pull --platform linux/amd64 "${_EMQX_DOCKER_IMAGE_TAG}"
          docker save "${_EMQX_DOCKER_IMAGE_TAG}" | gzip > "$PROFILE-$PKG_VSN-docker-amd64.tar.gz"
          docker pull --platform linux/arm64 "${_EMQX_DOCKER_IMAGE_TAG}"
          docker save "${_EMQX_DOCKER_IMAGE_TAG}" | gzip > "$PROFILE-$PKG_VSN-docker-arm64.tar.gz"
          ls -lh
          aws s3 cp "$PROFILE-$PKG_VSN-docker-amd64.tar.gz" "s3://$AWS_S3_BUCKET/$s3dir/"
          aws s3 cp "$PROFILE-$PKG_VSN-docker-arm64.tar.gz" "s3://$AWS_S3_BUCKET/$s3dir/"
          aws cloudfront create-invalidation --distribution-id "$AWS_CLOUDFRONT_ID" --paths "/$s3dir/*docker*"