emqx4.3/build

#!/usr/bin/env bash

# This script helps to build release artifacts.
# arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
# arg2: artifact, e.g. rel | relup | tgz | pkg

if [[ -n "$DEBUG" ]]; then
    set -x
fi
set -euo pipefail

DEBUG="${DEBUG:-0}"
if [ "$DEBUG" -eq 1 ]; then
    set -x
fi

PROFILE="$1"
ARTIFACT="$2"

# ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"

PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
export PKG_VSN

SYSTEM="$(./scripts/get-distro.sh)"

ARCH="$(uname -m)"
case "$ARCH" in
    x86_64)
        ARCH='amd64'
        ;;
    aarch64)
        ARCH='arm64'
        ;;
    arm*)
        ARCH='arm64'
        ;;
esac
export ARCH

##
## Support RPM and Debian based linux systems
##
if [ "$(uname -s)" = 'Linux' ]; then
    case "${SYSTEM:-}" in
        ubuntu*|debian*|raspbian*)
            PKGERDIR='deb'
            ;;
        *)
            PKGERDIR='rpm'
            ;;
    esac
fi

if [ "${SYSTEM}" = 'windows' ]; then
    # windows does not like the find
    FIND="/usr/bin/find"
else
    FIND='find'
fi

log() {
    local msg="$1"
    # rebar3 prints ===>, so we print ===<
    echo "===< $msg"
}

make_doc() {
    local libs_dir1 libs_dir2
    libs_dir1="$("$FIND" "_build/default/lib/" -maxdepth 2 -name ebin -type d)"
    libs_dir2="$("$FIND" "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)"
    # shellcheck disable=SC2086
    erl -noshell -pa $libs_dir1 $libs_dir2 -eval \
        "Dir = filename:join(['_build', '${PROFILE}', lib, emqx_dashboard, priv, www, static]), \
         ok = emqx_conf:dump_schema(Dir), \
         halt(0)."
}

make_rel() {
    ./rebar3 as "$PROFILE" tar
    if [ "$("$FIND" "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then
        echo "gpb should not be included in the release"
        exit 1
    fi
}

make_elixir_rel() {
  export_release_vars "$PROFILE"
  mix release --overwrite
}

## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup
make_relup() {
    local rel_dir="_build/$PROFILE/rel/emqx"
    mkdir -p "${rel_dir}/lib"
    mkdir -p "${rel_dir}/releases"
    local releases=()
    while read -r tgzfile ; do
        local base_vsn
        base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta)\.[0-9])?(-[0-9a-f]{8})?" | head -1)"
        tar -C "$rel_dir" -zxf ---keep-old-files "$tgzfile" emqx/releases emqx/lib
        releases+=( "$base_vsn" )
    done < <("$FIND" _upgrade_base -maxdepth 1 -name "$PROFILE*${SYSTEM}-${ARCH}.tar.gz" -type f)
    if [ ${#releases[@]} -eq 0 ]; then
        log "No upgrade base found, relup ignored"
        return 0
    fi
    RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
    export RELX_BASE_VERSIONS
    ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
}

cp_dyn_libs() {
    local rel_dir="$1"
    local target_dir="${rel_dir}/dynlibs"
    if ! [ "$(uname -s)" = 'Linux' ]; then
        return 0;
    fi
    mkdir -p "$target_dir"
    while read -r so_file; do
        cp -L "$so_file" "$target_dir/"
    done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
        | xargs -0 ldd \
        | grep -E '(libcrypto)|(libtinfo)|(libatomic)' \
        | awk '{print $3}' \
        | sort -u)
}

## Re-pack the relx assembled .tar.gz to EMQX's package naming scheme
## It assumes the .tar.gz has been built -- relies on Makefile dependency
make_tgz() {
    local pkgpath="_packages/${PROFILE}"
    local src_tarball
    local target_name
    local target

    if [ "${IS_ELIXIR:-no}" = "yes" ]
    then
      # ensure src_tarball exists
      ELIXIR_MAKE_TAR=yes make_elixir_rel

      local relpath="_build/${PROFILE}"
      target_name="${PROFILE}-${PKG_VSN}-elixir${ELIXIR_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
    else
      # build the src_tarball again to ensure relup is included
      # elixir does not have relup yet.
      make_rel

      local relpath="_build/${PROFILE}/rel/emqx"
      target_name="${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
    fi
    target="${pkgpath}/${target_name}"

    src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
    tard="tmp/emqx_untar_${PKG_VSN}"
    rm -rf "${tard}"
    mkdir -p "${tard}/emqx"
    mkdir -p "${pkgpath}"
    if [ ! -f "$src_tarball" ]; then
        log "ERROR: $src_tarball is not found"
    fi
    tar zxf "${src_tarball}" -C "${tard}/emqx"
    ## try to be portable for tar.gz packages.
    ## for DEB and RPM packages the dependencies are resoved by yum and apt
    cp_dyn_libs "${tard}/emqx"
    ## create tar after change dir (for windows)
    pushd "${tard}" >/dev/null
    tar -czf "${target_name}" emqx
    popd >/dev/null
    mv "${tard}/${target_name}" "${target}"
    case "$SYSTEM" in
        macos*)
            # sha256sum may not be available on macos
            openssl dgst -sha256 "${target}" | cut -d ' ' -f 2  > "${target}.sha256"
            ;;
        *)
            sha256sum "${target}" | head -c 64 > "${target}.sha256"
            ;;
    esac
    log "Tarball successfully repacked: ${target}"
    log "Tarball sha256sum: $(cat "${target}.sha256")"
}

## This function builds the default docker image based on alpine:3.14 (by default)
make_docker() {
    EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
    EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"

    if [[ "$PROFILE" = *-elixir ]]
    then
      PKG_VSN="$PKG_VSN-elixir"
    fi

    set -x
    docker build --no-cache --pull \
       --build-arg BUILD_FROM="${EMQX_BUILDER}" \
       --build-arg RUN_FROM="${EMQX_RUNNER}" \
       --build-arg EMQX_NAME="$PROFILE" \
       --tag "emqx/${PROFILE%%-elixir}:${PKG_VSN}" \
       -f "${DOCKERFILE}" .
}

## This function accepts any base docker image,
## a emqx tgz-image, and a image tag (for the image to be built),
## to build a docker image which runs EMQX
##
## Export below variables to quickly build an image
##
## Name               Default                         Example
## ---------------------------------------------------------------------
## EMQX_BASE_IMAGE    current os                      centos:7
## EMQX_TGZ_packages/<current-tgz-target>  /tmp/emqx-4.4.0-otp23.3.4.9-3-centos7-amd64.tar.gz
## EMQX_IMAGE_TAG     emqx/emqx:<current-vns-rel>     emqx/emqx:testing-tag
##
make_docker_testing() {
    if [ -z "${EMQX_BASE_IMAGE:-}" ]; then
        case "$SYSTEM" in
            ubuntu20*)
                EMQX_BASE_IMAGE="ubuntu:20.04"
                ;;
            *)
                echo "Unsupported testing base image for $SYSTEM"
                exit 1
                ;;
        esac
    fi
    EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-emqx/$PROFILE:${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}}"
    local default_tgz
    default_tgz="_packages/${PROFILE}/${PROFILE}-${PKG_VSN}-otp${OTP_VSN}-${SYSTEM}-${ARCH}.tar.gz"
    local tgz="${EMQX_TGZ_PACKAGE:-$default_tgz}"
    if [ ! -f "$tgz" ]; then
        log "ERROR: $tgz not built?"
        exit 1
    fi
    set -x
    docker build \
       --build-arg BUILD_FROM="${EMQX_BASE_IMAGE}" \
       --build-arg EMQX_TGZ_PACKAGE="${tgz}" \
       --tag "$EMQX_IMAGE_TAG" \
       -f "${DOCKERFILE_TESTING}" .
}

function join {
  local IFS="$1"
  shift
  echo "$*"
}

# used to control the Elixir Mix Release output
# see docstring in `mix.exs`
export_release_vars() {
  local profile="$1"
  case "$profile" in
    emqx|emqx-edge|emqx-enterprise)
      export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
      ;;
    emqx-pkg|emqx-edge-pkg|emqx-enterprise-pkg)
      export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
      ;;
    *)
      echo Invalid profile "$profile"
      exit 1
  esac
  export MIX_ENV="$profile"

  local erl_opts=()

  if [[ "$profile" = *enterprise* ]]
  then
    erl_opts+=( "{d,'EMQX_ENTERPRISE'}" )
  fi

  # At this time, Mix provides no easy way to pass `erl_opts' to
  # dependencies.  The workaround is to set this variable before
  # compiling the project, so that `emqx_release.erl' picks up
  # `emqx_vsn' as if it was compiled by rebar3.
  erl_opts+=( "{compile_info,[{emqx_vsn,\"${PKG_VSN}\"}]}" )

  ERL_COMPILER_OPTIONS="[$(join , "${erl_opts[@]}")]"
  export ERL_COMPILER_OPTIONS
}

log "building artifact=$ARTIFACT for profile=$PROFILE"

case "$ARTIFACT" in
    doc)
        make_doc
        ;;
    rel)
        make_rel
        ;;
    relup)
        make_relup
        ;;
    tgz)
        make_tgz
        ;;
    pkg)
        if [ -z "${PKGERDIR:-}" ]; then
            log "Skipped making deb/rpm package for $SYSTEM"
            exit 0
        fi
        make -C "deploy/packages/${PKGERDIR}" clean
        env EMQX_REL="$(pwd)" \
            EMQX_BUILD="${PROFILE}" \
            SYSTEM="${SYSTEM}" \
            make -C "deploy/packages/${PKGERDIR}"
        ;;
    docker)
        make_docker
        ;;
    docker-testing)
        make_docker_testing
        ;;
    elixir)
        make_elixir_rel
        ;;
    *)
        log "Unknown artifact $ARTIFACT"
        exit 1
        ;;
esac