emqx4.3/build

#!/usr/bin/env bash

# This script helps to build release artifacts.
# arg1: profile, e.g. emqx | emqx-enterprise
# arg2: artifact, e.g. rel | relup | tgz | pkg

set -euo pipefail

if [ "${DEBUG:-0}" -eq 1 ]; then
    set -x
    # set this for rebar3
    export DIAGNOSTIC=1
fi

PROFILE_ARG="$1"
ARTIFACT="$2"

is_enterprise() {
    case "$1" in
        *enterprise*)
            echo 'yes'
            ;;
        *)
            echo 'no'
            ;;
    esac
}
PROFILE_ENV="${PROFILE:-${PROFILE_ARG}}"
case "$(is_enterprise "$PROFILE_ARG"),$(is_enterprise "$PROFILE_ENV")" in
    'yes,yes')
        true
        ;;
    'no,no')
        true
        ;;
    *)
        echo "PROFILE env var is set to '$PROFILE_ENV', but '$0' arg1 is '$PROFILE_ARG'"
        exit 1
        ;;
esac

# make sure PROFILE is exported, it is needed by rebar.config.erl
PROFILE=$PROFILE_ARG
export PROFILE

# ensure dir
cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"

PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
export PKG_VSN

SYSTEM="$(./scripts/get-distro.sh)"

ARCH="$(uname -m)"
case "$ARCH" in
    x86_64)
        ARCH='amd64'
        ;;
    aarch64)
        ARCH='arm64'
        ;;
    arm*)
        ARCH='arm64'
        ;;
esac
export ARCH

##
## Support RPM and Debian based linux systems
##
if [ "$(uname -s)" = 'Linux' ]; then
    case "${SYSTEM:-}" in
        ubuntu*|debian*|raspbian*)
            PKGERDIR='deb'
            ;;
        *)
            PKGERDIR='rpm'
            ;;
    esac
fi

if [ "${SYSTEM}" = 'windows' ]; then
    # windows does not like the find
    FIND="/usr/bin/find"
    TAR="/usr/bin/tar"
    export BUILD_WITHOUT_ROCKSDB="on"
else
    FIND='find'
    TAR='tar'
fi

log() {
    local msg="$1"
    # rebar3 prints ===>, so we print ===<
    echo "===< $msg"
}

prepare_erl_libs() {
    local libs_dir="$1"
    local erl_libs="${ERL_LIBS:-}"
    local sep
    if [ "${SYSTEM}" = 'windows' ]; then
        sep=';'
    else
        sep=':'
    fi
    for app in "${libs_dir}"/*; do
        if [ -d "${app}/ebin" ]; then
            if [ -n "$erl_libs" ]; then
                erl_libs="${erl_libs}${sep}${app}"
            else
                erl_libs="${app}"
            fi
        fi
    done
    export ERL_LIBS="$erl_libs"
}

make_docs() {
    case "$(is_enterprise "$PROFILE")" in
        'yes')
            SCHEMA_MODULE='emqx_enterprise_schema'
            ;;
        'no')
            SCHEMA_MODULE='emqx_conf_schema'
            ;;
    esac
    prepare_erl_libs "_build/$PROFILE/checkouts"
    prepare_erl_libs "_build/$PROFILE/lib"
    local docdir="_build/docgen/$PROFILE"
    mkdir -p "$docdir"
    # shellcheck disable=SC2086
    erl -noshell -eval \
        "ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE), \
         halt(0)."
}

## arg1 is the profile for which the following args (as app names) should be excluded
assert_no_excluded_deps() {
    local profile="$1"
    shift 1
    if [ "$PROFILE" != "$profile" ]; then
        # not currently building the profile which has apps to be excluded
        return 0
    fi
    local rel_dir="_build/$PROFILE/rel/emqx/lib"
    local excluded_apps=( "$@" )
    local found
    for app in "${excluded_apps[@]}"; do
        found="$($FIND "$rel_dir" -maxdepth 1 -type d -name "$app-*")"
        if [ -n "${found}" ]; then
            echo "ERROR: ${app} should not be included in ${PROFILE}"
            echo "ERROR: found ${app} in ${rel_dir}"
            exit 1
        fi
    done
}

just_compile() {
    ./scripts/pre-compile.sh "$PROFILE"
    # make_elixir_rel always create rebar.lock
    # delete it to make git clone + checkout work because we use shallow close for rebar deps
    rm -f rebar.lock
    # compile all beams
    ./rebar3 as "$PROFILE" compile
    make_docs
}

just_compile_elixir() {
    ./scripts/pre-compile.sh "$PROFILE"
    rm -f rebar.lock
    # shellcheck disable=SC1010
    env MIX_ENV="$PROFILE" mix do local.hex --if-missing --force, \
        local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \
        deps.get
    env MIX_ENV="$PROFILE" mix compile
}

make_rel() {
    local release_or_tar="${1}"
    just_compile
    # now assemble the release tar
    ./rebar3 as "$PROFILE" "$release_or_tar"
    assert_no_excluded_deps emqx-enterprise emqx_telemetry
}

make_elixir_rel() {
    ./scripts/pre-compile.sh "$PROFILE"
    export_elixir_release_vars "$PROFILE"
    # for some reason, this has to be run outside "do"...
    mix local.rebar --if-missing --force
    # shellcheck disable=SC1010
    mix do local.hex --if-missing --force, \
        local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \
        deps.get
    mix release --overwrite
    assert_no_excluded_deps emqx-enterprise emqx_telemetry
}

## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup
make_relup() {
    local rel_dir="_build/$PROFILE/rel/emqx"
    local name_pattern
    name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher --long)"
    local releases=()
    mkdir -p _upgrade_base
    while read -r tgzfile ; do
        local base_vsn
        base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9])?(-[0-9a-f]{8})?" | head -1)"
        ## we have to create tmp dir to untar old tgz, as `tar --skip-old-files` is not supported on all plantforms
        local tmp_dir
        tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
        $TAR -C "$tmp_dir" -zxf "$tgzfile"
        mkdir -p "${rel_dir}/releases/"
        cp -npr "$tmp_dir/releases"/* "${rel_dir}/releases/"
        ## There is for some reason a copy of the '$PROFILE.rel' file to releases dir,
        ## the content is duplicated to releases/5.0.0/$PROFILE.rel.
        ## This file seems to be useless, but yet confusing as it does not change after upgrade/downgrade
        ## Hence we force delete this file.
        rm -f "${rel_dir}/releases/${PROFILE}.rel"
        mkdir -p "${rel_dir}/lib/"
        cp -npr "$tmp_dir/lib"/* "${rel_dir}/lib/"
        rm -rf "$tmp_dir"
        releases+=( "$base_vsn" )
    done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f)
    if [ ${#releases[@]} -eq 0 ]; then
        log "No upgrade base found, relup ignored"
        return 0
    fi
    RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
    export RELX_BASE_VERSIONS
    ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
}

cp_dyn_libs() {
    local rel_dir="$1"
    local target_dir="${rel_dir}/dynlibs"
    if ! [ "$(uname -s)" = 'Linux' ]; then
        return 0;
    fi
    mkdir -p "$target_dir"
    while read -r so_file; do
        cp -L "$so_file" "$target_dir/"
    done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
        | xargs -0 ldd \
        | grep -E '(libcrypto)|(libtinfo)|(libatomic)' \
        | awk '{print $3}' \
        | sort -u)
}

## Re-pack the relx assembled .tar.gz to EMQX's package naming scheme
## It assumes the .tar.gz has been built -- relies on Makefile dependency
make_tgz() {
    local pkgpath="_packages/${PROFILE}"
    local src_tarball
    local target_name
    local target

    if [ "${IS_ELIXIR:-no}" = "yes" ]
    then
      # ensure src_tarball exists
      ELIXIR_MAKE_TAR=yes make_elixir_rel

      local relpath="_build/${PROFILE}"
      full_vsn="$(./pkg-vsn.sh "$PROFILE" --long --elixir)"
    else
      # build the src_tarball again to ensure relup is included
      # elixir does not have relup yet.
      make_rel tar

      local relpath="_build/${PROFILE}/rel/emqx"
      full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)"
    fi

    case "$SYSTEM" in
        macos*)
            target_name="${PROFILE}-${full_vsn}.zip"
            ;;
        windows*)
            target_name="${PROFILE}-${full_vsn}.zip"
            ;;
        *)
            target_name="${PROFILE}-${full_vsn}.tar.gz"
            ;;
    esac

    target="${pkgpath}/${target_name}"

    src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
    tard="$(mktemp -d -t emqx.XXXXXXX)"
    mkdir -p "${tard}/emqx"
    mkdir -p "${pkgpath}"
    if [ ! -f "$src_tarball" ]; then
        log "ERROR: $src_tarball is not found"
    fi
    $TAR zxf "${src_tarball}" -C "${tard}/emqx"
    if [ -f "${tard}/emqx/releases/${PKG_VSN}/relup" ]; then
        ./scripts/relup-build/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup"
    fi
    ## try to be portable for tar.gz packages.
    ## for DEB and RPM packages the dependencies are resoved by yum and apt
    cp_dyn_libs "${tard}/emqx"
    case "$SYSTEM" in
        macos*)
            # if the flag to sign macos binaries is set, but developer certificate
            # or certificate password is not configured, reset the flag
            # could happen, for example, when people submit PR from a fork, in this
            # case they cannot access secrets
            if [[ "${APPLE_SIGN_BINARIES:-0}" == 1 && \
                      ( "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || \
                           "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ) ]]; then
                echo "Apple developer certificate is not configured, skip signing"
                APPLE_SIGN_BINARIES=0
            fi
            if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
                ./scripts/macos-sign-binaries.sh "${tard}/emqx"
            fi
            ## create zip after change dir
            ## to avoid creating an extra level of 'emqx' dir in the .zip file
            pushd "${tard}/emqx" >/dev/null
            zip -r "../${target_name}" -- * >/dev/null
            popd >/dev/null
            mv "${tard}/${target_name}" "${target}"
            if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
                # notarize the package
                # if fails, check what went wrong with this command:
                    # xcrun notarytool log \
                    #   --apple-id <apple id> \
                    #   --password <apple id password>
                    #   --team-id <apple team id> <submission-id>
                echo 'Submitting the package for notarization to Apple (normally takes about a minute)'
                notarytool_output="$(xcrun notarytool submit \
                                           --apple-id "${APPLE_ID}" \
                                           --password "${APPLE_ID_PASSWORD}" \
                                           --team-id "${APPLE_TEAM_ID}" "${target}" \
                                           --no-progress \
                                           --wait)"
                echo "$notarytool_output"
                echo "$notarytool_output" | grep -q 'status: Accepted' || {
                    echo 'Notarization failed';
                    exit 1;
                }
            fi
            # sha256sum may not be available on macos
            openssl dgst -sha256 "${target}" | cut -d ' ' -f 2  > "${target}.sha256"
            ;;
        windows*)
            pushd "${tard}" >/dev/null
            7z a "${target_name}" ./emqx/* >/dev/null
            popd >/dev/null
            mv "${tard}/${target_name}" "${target}"
            sha256sum "${target}" | head -c 64 > "${target}.sha256"
            ;;
        *)
            ## create tar after change dir
            ## to avoid creating an extra level of 'emqx' dir in the .tar.gz file
            pushd "${tard}/emqx" >/dev/null
            $TAR -zcf "../${target_name}" -- *
            popd >/dev/null
            mv "${tard}/${target_name}" "${target}"
            sha256sum "${target}" | head -c 64 > "${target}.sha256"
            ;;
    esac
    log "Archive successfully repacked: ${target}"
    log "Archive sha256sum: $(cat "${target}.sha256")"
}

docker_cleanup() {
    rm -f ./.dockerignore >/dev/null
    # shellcheck disable=SC2015
    [ -f ./.dockerignore.bak ] && mv ./.dockerignore.bak ./.dockerignore >/dev/null || true
}

## Build the default docker image based on debian 11.
make_docker() {
    local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.1-4}"
    local EMQX_BUILDER_PLATFORM="${EMQX_BUILDER_PLATFORM:-debian11}"
    local EMQX_BUILDER_OTP="${EMQX_BUILDER_OTP:-25.3.2-2}"
    local EMQX_BUILDER_ELIXIR="${EMQX_BUILDER_ELIXIR:-1.14.5}"
    local EMQX_BUILDER=${EMQX_BUILDER:-ghcr.io/emqx/emqx-builder/${EMQX_BUILDER_VERSION}:${EMQX_BUILDER_ELIXIR}-${EMQX_BUILDER_OTP}-${EMQX_BUILDER_PLATFORM}}
    local EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
    local EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}"
    local PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
    # shellcheck disable=SC2155
    local VSN_MAJOR="$(scripts/semver.sh "$PKG_VSN" --major)"
    # shellcheck disable=SC2155
    local VSN_MINOR="$(scripts/semver.sh "$PKG_VSN" --minor)"
    # shellcheck disable=SC2155
    local VSN_PATCH="$(scripts/semver.sh "$PKG_VSN" --patch)"
    local SUFFIX=''
    if [[ "$PROFILE" = *-elixir ]]; then
        SUFFIX="-elixir"
    fi
    local DOCKER_REGISTRY="${DOCKER_REGISTRY:-docker.io}"
    local DOCKER_ORG="${DOCKER_ORG:-emqx}"
    local EMQX_BASE_DOCKER_TAG="${DOCKER_REGISTRY}/${DOCKER_ORG}/${PROFILE%%-elixir}"
    local default_tag="${EMQX_BASE_DOCKER_TAG}:${PKG_VSN}${SUFFIX}"
    local EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}"
    local EDITION=Opensource
    local LICENSE='Apache-2.0'
    local PRODUCT_URL='https://www.emqx.io'
    local PRODUCT_DESCRIPTION='Official docker image for EMQX, the most scalable open-source MQTT broker for IoT, IIoT, and connected vehicles.'
    local DOCUMENTATION_URL='https://www.emqx.io/docs/en/latest/'
    ## extra_deps is a comma separated list of debian 11 package names
    local EXTRA_DEPS=''
    if [[ "$PROFILE" = *enterprise* ]]; then
        EXTRA_DEPS='libsasl2-2,libsasl2-modules-gssapi-mit'
        EDITION=Enterprise
        LICENSE='(Apache-2.0 AND BSL-1.1)'
        PRODUCT_URL='https://www.emqx.com/en/products/emqx'
        PRODUCT_DESCRIPTION='Official docker image for EMQX Enterprise, an enterprise MQTT platform at scale. '
        DOCUMENTATION_URL='https://docs.emqx.com/en/enterprise/latest/'
    fi
    local ISO_8601_DATE GIT_REVISION
    ISO_8601_DATE="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
    GIT_REVISION="$(git rev-parse HEAD)"
    export BUILDX_NO_DEFAULT_ATTESTATIONS=1
    local DOCKER_BUILDX_ARGS=(
       --build-arg BUILD_FROM="${EMQX_BUILDER}" \
       --build-arg RUN_FROM="${EMQX_RUNNER}" \
       --build-arg EMQX_NAME="${PROFILE}" \
       --build-arg EXTRA_DEPS="${EXTRA_DEPS}" \
       --build-arg PKG_VSN="${PKG_VSN}" \
       --file "${EMQX_DOCKERFILE}" \
       --label org.opencontainers.image.title="${PROFILE}" \
       --label org.opencontainers.image.edition="${EDITION}" \
       --label org.opencontainers.image.version="${PKG_VSN}" \
       --label org.opencontainers.image.revision="${GIT_REVISION}" \
       --label org.opencontainers.image.created="${ISO_8601_DATE}" \
       --label org.opencontainers.image.source='https://github.com/emqx/emqx' \
       --label org.opencontainers.image.url="${PRODUCT_URL}" \
       --label org.opencontainers.image.description="${PRODUCT_DESCRIPTION}" \
       --label org.opencontainers.image.documentation="${DOCUMENTATION_URL}" \
       --label org.opencontainers.image.licenses="${LICENSE}" \
       --label org.opencontainers.image.otp.version="${EMQX_BUILDER_OTP}" \
       --tag "${EMQX_IMAGE_TAG}" \
       --pull
    )
    if [ "${DOCKER_BUILD_NOCACHE:-false}" = true ]; then
        DOCKER_BUILDX_ARGS+=(--no-cache)
    fi
    if [ "${SUFFIX}" = '-elixir' ]; then
        DOCKER_BUILDX_ARGS+=(--label org.opencontainers.image.elixir.version="${EMQX_BUILDER_ELIXIR}")
    fi
    if [ "${DOCKER_LATEST:-false}" = true ]; then
        DOCKER_BUILDX_ARGS+=(--tag "${EMQX_BASE_DOCKER_TAG}:latest${SUFFIX}")
        DOCKER_BUILDX_ARGS+=(--tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}")
        DOCKER_BUILDX_ARGS+=(--tag "${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}")
    fi
    if [ "${DOCKER_PLATFORMS:-default}" != 'default' ]; then
        DOCKER_BUILDX_ARGS+=(--platform "${DOCKER_PLATFORMS}")
    fi
    if [ "${DOCKER_PUSH:-false}" = true ]; then
        DOCKER_BUILDX_ARGS+=(--push)
    fi
    if [ -d "${REBAR_GIT_CACHE_DIR:-}" ]; then
        cache_tar="$(pwd)/rebar-git-cache.tar"
        if [ ! -f "${cache_tar}" ]; then
            pushd "${REBAR_GIT_CACHE_DIR}" >/dev/null
            tar -cf "${cache_tar}" .
            popd >/dev/null
        fi
    fi
    if [ -n "${DEBUG:-}" ]; then
        DOCKER_BUILDX_ARGS+=(--build-arg DEBUG="${DEBUG}" --progress=plain)
    fi

    # shellcheck disable=SC2015
    [ -f ./.dockerignore ] && mv ./.dockerignore ./.dockerignore.bak || true
    trap docker_cleanup EXIT
    {
        echo '_build/'
        echo 'deps/'
        echo '*.lock'
        echo '_packages/'
        echo '.vs/'
        echo '.vscode/'
        echo 'lux_logs/'
        echo '_upgrade_base/'
    } >> ./.dockerignore
    echo "Docker build args: ${DOCKER_BUILDX_ARGS[*]}"
    docker buildx build "${DOCKER_BUILDX_ARGS[@]}" .
    echo "${EMQX_IMAGE_TAG}" > ./.docker_image_tag
}

function join {
  local IFS="$1"
  shift
  echo "$*"
}

# used to control the Elixir Mix Release output
# see docstring in `mix.exs`
export_elixir_release_vars() {
  local profile="$1"
  case "$profile" in
    emqx|emqx-enterprise)
      export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
      ;;
    emqx-pkg|emqx-enterprise-pkg)
      export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
      ;;
    *)
      echo Invalid profile "$profile"
      exit 1
  esac
  export MIX_ENV="$profile"
}

log "building artifact=$ARTIFACT for profile=$PROFILE"

case "$ARTIFACT" in
    apps)
        if [ "${IS_ELIXIR:-}" = "yes" ]; then
            just_compile_elixir
        else
            just_compile
        fi
        ;;
    doc|docs)
        make_docs
        ;;
    rel)
        make_rel release
        ;;
    relup)
        make_relup
        ;;
    tgz)
        make_tgz
        ;;
    pkg)
        # this only affect build artifacts, such as schema doc
        export EMQX_ETC_DIR='/etc/emqx/'
        if [ -z "${PKGERDIR:-}" ]; then
            log "Skipped making deb/rpm package for $SYSTEM"
            exit 0
        fi
        export EMQX_REL_FORM="$PKGERDIR"
        if [ "${IS_ELIXIR:-}" = 'yes' ]; then
            make_elixir_rel
        else
            make_rel tar
        fi
        env EMQX_REL="$(pwd)" \
            EMQX_BUILD="${PROFILE}" \
            make -C "deploy/packages/${PKGERDIR}" clean
        env EMQX_REL="$(pwd)" \
            EMQX_BUILD="${PROFILE}" \
            make -C "deploy/packages/${PKGERDIR}"
        ;;
    docker)
        make_docker
        ;;
    elixir)
        make_elixir_rel
        ;;
    *)
        log "Unknown artifact $ARTIFACT"
        exit 1
        ;;
esac