chore(build): refine packages builds

This commit merge packages.mk into Makefile
with the complex part moved to shell script ./build.

Makefile

@@ -1,12 +1,12 @@
 REBAR_VERSION = 3.14.3-emqx-3
 REBAR = $(CURDIR)/rebar3
-export PKG_VSN ?= $(shell git describe --tags --match '[0-9]*' 2>/dev/null || git describe --always)
-# comma separated versions
-export RELUP_BASE_VERSIONS ?=
+BUILD = $(CURDIR)/build
+export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 
 PROFILE ?= emqx
-PROFILES := emqx emqx-edge check test
+REL_PROFILES := emqx emqx-edge
 PKG_PROFILES := emqx-pkg emqx-edge-pkg
+PROFILES := $(REL_PROFILES) $(PKG_PROFILES)
 
 export REBAR_GIT_CLONE_OPTIONS += --depth=1
 
@@ -30,8 +30,8 @@ eunit: $(REBAR)
 ct: $(REBAR)
 	$(REBAR) ct
 
-.PHONY: $(PROFILES)
-$(PROFILES:%=%): $(REBAR)
+.PHONY: $(REL_PROFILES)
+$(REL_PROFILES:%=%): $(REBAR)
 ifneq ($(shell echo $(@) |grep edge),)
 	export EMQX_DESC="EMQ X Edge"
 else
@@ -39,10 +39,6 @@ else
 endif
 	$(REBAR) as $(@) release
 
-.PHONY: $(PROFILES:%=build-%)
-$(PROFILES:%=build-%): $(REBAR)
-	$(REBAR) as $(@:build-%=%) compile
-
 # rebar clean
 .PHONY: clean $(PROFILES:%=clean-%)
 clean: $(PROFILES:%=clean-%)
@@ -50,10 +46,10 @@ $(PROFILES:%=clean-%): $(REBAR)
 	$(REBAR) as $(@:clean-%=%) clean
 
 .PHONY: deps-all
-deps-all: $(REBAR) $(PROFILES:%=deps-%) $(PKG_PROFILES:%=deps-%)
+deps-all: $(REBAR) $(PROFILES:%=deps-%)
 
-.PHONY: $(PROFILES:%=deps-%) $(PKG_PROFILES:%=deps-%)
-$(PROFILES:%=deps-%) $(PKG_PROFILES:%=deps-%): $(REBAR)
+.PHONY: $(PROFILES:%=deps-%)
+$(PROFILES:%=deps-%): $(REBAR)
 ifneq ($(shell echo $(@) |grep edge),)
 	export EMQX_DESC="EMQ X Edge"
 else
@@ -69,5 +65,32 @@ xref: $(REBAR)
 dialyzer: $(REBAR)
 	$(REBAR) as check dialyzer
 
-include packages.mk
+.PHONY: $(REL_PROFILES:%=relup-%)
+$(REL_PROFILES:%=relup-%): $(REBAR)
+ifneq ($(OS),Windows_NT)
+	$(BUILD) $(@:relup-%=%) relup
+endif
+
+.PHONY: $(REL_PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar)
+$(REL_PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar): $(REBAR)
+	$(BUILD) $(subst -tar,,$(@)) tar
+
+## zip targets depend on the corresponding relup and tar artifacts
+.PHONY: $(REL_PROFILES:%=%-zip)
+define gen-zip-target
+$1-zip: relup-$1 $1-tar
+	$(BUILD) $1 zip
+endef
+ALL_ZIPS = $(REL_PROFILES) $(PKG_PROFILES)
+$(foreach zt,$(ALL_ZIPS),$(eval $(call gen-zip-target,$(zt))))
+
+## A pkg target depend on a regular release profile zip to include relup,
+## and also a -pkg suffixed profile tar (without relup) for making deb/rpm package
+.PHONY: $(PKG_PROFILES)
+define gen-pkg-target
+$1: $(subst -pkg,,$1)-zip $1-tar
+	$(BUILD) $1 pkg
+endef
+$(foreach pt,$(PKG_PROFILES),$(eval $(call gen-pkg-target,$(pt))))
+
 include docker.mk

build

@@ -0,0 +1,142 @@
+#!/bin/bash
+
+# This script helps to build release artifacts.
+# arg1: profile, e.g. emqx | emqx-edge | emqx-pkg | emqx-edge-pkg
+# arg2: artifact, e.g. tar | relup | zip | pkg
+
+set -euo pipefail
+
+PROFILE="$1"
+ARTIFACT="$2"
+
+# ensure dir
+cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
+
+case "$PROFILE" in
+    emqx-edge*)
+	    export EMQX_DESC="EMQ X Edge"
+        ;;
+    emqx*)
+	    export EMQX_DESC="EMQ X Broker"
+        ;;
+    *)
+        echo "Unknown profile $PROFILE"
+        exit 1
+        ;;
+esac
+
+PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh)}"
+export PKG_VSN
+
+if [ "$(uname -s)" = 'Darwin' ]; then
+	SYSTEM=macos
+elif [ "$(uname -s)" = 'Linux' ]; then
+    if grep -q -i 'centos' /etc/*-release; then
+        DIST='centos'
+        VERSION_ID="$(rpm --eval '%{centos_ver}')"
+    else
+        DIST="$(sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g')"
+        VERSION_ID="$(sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')"
+    fi
+    SYSTEM="$(echo "${DIST}${VERSION_ID}" | sed -r 's/([a-zA-Z]*)-.*/\1/g')"
+fi
+
+##
+## Support RPM and Debian based linux systems
+##
+if [ "$(uname -s)" = 'Linux' ]; then
+    case "${DIST:-}" in
+        ubuntu|debian|raspbian)
+            PKGERDIR='deb'
+            ;;
+        *)
+            PKGERDIR='rpm'
+            ;;
+    esac
+fi
+
+log() {
+    local msg="$1"
+    # rebar3 prints ===>, so we print ===<
+    echo "===< $msg"
+}
+
+make_tar() {
+    ./rebar3 as "$PROFILE" tar
+}
+
+## unzip previous version .zip files to _build/$PROFILE/rel/emqx/releases before making relup
+make_relup() {
+    local releases_dir="_build/$PROFILE/rel/emqx/releases"
+    local releases=()
+    if [ -d "$releases_dir" ]; then
+        while read -r dir; do
+            local version="$(basename "$dir")"
+            # skip current version
+            if [ "$version" != "$PKG_VSN" ]; then
+                releases+=( "$version" )
+            fi
+        done < <(find "_build/$PROFILE/rel/emqx/releases" -maxdepth 1 -name '*.*.*' -type d)
+    fi
+    if [ ${#releases[@]} -eq 0 ]; then
+        log "No previous release found, relup ignored"
+        return 0
+    fi
+    if [ ${#releases[@]} -gt 1 ]; then
+        log "Found more than one previous versions in $releases_dir:"
+        log "${releases[@]}"
+        log "ERROR: So far we can not support multi base-version relup creation"
+        return 1
+    fi
+    local base_version="${releases[0]}"
+    # TODO: comma separate base-versions when supported
+    ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}" --upfrom "$base_version"
+}
+
+## make_zip turns .tar.gz into a .zip with a slightly different name.
+## It assumes the .tar.gz has been built -- relies on Makefile dependency
+make_zip() {
+    tard="/tmp/emqx_untar_${PKG_VSN}"
+    rm -rf "${tard}"
+    mkdir -p "${tard}/emqx"
+    local relpath="_build/${PROFILE}/rel/emqx"
+    local pkgpath="_packages/${PROFILE}"
+    mkdir -p "${pkgpath}"
+    local tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
+    if [ ! -f "$tarball" ]; then
+        log "ERROR: $tarball is not found"
+    fi
+    local zipball
+    zipball="${pkgpath}/${PROFILE}-${SYSTEM}-${PKG_VSN}-$(uname -m).zip"
+    tar zxf "${tarball}" -C "${tard}/emqx"
+    (cd "${tard}" && zip -qr - emqx) > "${zipball}"
+}
+
+log "building artifact=$ARTIFACT for profile=$PROFILE"
+
+case "$ARTIFACT" in
+    tar)
+        make_tar
+        ;;
+    relup)
+        make_relup
+        ;;
+    zip)
+        make_zip
+        ;;
+    pkg)
+        if [ -z "${PKGERDIR:-}" ]; then
+            # zip should have been built as a Makefile dependency
+            log "Skipped making deb/rpm package for $SYSTEM"
+            exit 0
+        fi
+        # build the tar which is going to be used as the base of deb and rpm packages
+        make_tar
+        make -C "deploy/packages/${PKGERDIR}" clean
+        EMQX_REL="$(pwd)" EMQX_BUILD="${PROFILE}" SYSTEM="${SYSTEM}" make -C "deploy/packages/${PKGERDIR}"
+        ;;
+    *)
+        log "Unknown artifact $ARTIFACT"
+        exit 1
+        ;;
+esac

docker.mk

@@ -7,7 +7,7 @@ QEMU_ARCH ?= x86_64
 ARCH ?= amd64
 QEMU_VERSION ?= v5.0.0-2
 OS ?= alpine
-PKG_VSN ?= $(shell git describe --tags --match '[0-9]*')
+export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 
 EMQX_NAME = $(subst emqx/,,$(TARGET))
 ARCH_LIST = amd64 arm64v8 arm32v7 i386 s390x

ensure-rebar3.sh

@@ -5,6 +5,9 @@ set -eu
 
 VERSION="$1"
 
+# ensure dir
+cd -P -- "$(dirname -- "$0")"
+
 DOWNLOAD_URL='https://github.com/emqx/rebar3/releases/download'
 
 download() {

packages.mk

@@ -1,81 +0,0 @@
-#!/usr/bin/make -f
-# -*- makefile -*-
-
-PROFILES := emqx emqx-edge
-PKG_PROFILES := emqx-pkg emqx-edge-pkg
-
-ifeq ($(shell uname -s),Darwin)
-	SYSTEM := macos
-else ifeq ($(shell uname -s),Linux)
-    ifneq ($(shell cat /etc/*-release |grep -o -i centos),)
-    	ID := centos
-    	VERSION_ID := $(shell rpm --eval '%{centos_ver}')
-    else
-    	ID := $(shell sed -n '/^ID=/p' /etc/os-release | sed -r 's/ID=(.*)/\1/g' | sed 's/"//g' )
-    	VERSION_ID := $(shell sed -n '/^VERSION_ID=/p' /etc/os-release | sed -r 's/VERSION_ID=(.*)/\1/g' | sed 's/"//g')
-    endif
-    SYSTEM := $(shell echo $(ID)$(VERSION_ID) | sed -r "s/([a-zA-Z]*)-.*/\1/g")
-    ##
-    ## Support RPM and Debian based linux systems
-    ##
-    ifeq ($(ID),ubuntu)
-    	PKGERDIR := deb
-    else ifeq ($(ID),debian)
-    	PKGERDIR := deb
-    else ifeq ($(ID),raspbian)
-    	PKGERDIR := deb
-    else
-    	PKGERDIR := rpm
-    endif
-endif
-
-.PHONY: $(PROFILES:%=relup-%)
-$(PROFILES:%=relup-%): $(REBAR)
-ifneq ($(RELUP_BASE_VERSIONS),)
-ifneq ($(OS),Windows_NT)
-	@if [ ! -z $$(ls | grep -E "$(@:relup-%=%)-$(SYSTEM)-(.*)-$$(uname -m).zip" | head -1 ) ]; then \
-		mkdir -p tmp/relup_packages/$(@:relup-%=%); \
-		cp $(@:relup-%=%)-$(SYSTEM)-*-$$(uname -m).zip tmp/relup_packages/$(@:relup-%=%); \
-	fi
-	$(REBAR) as $(@:relup-%=%) relup --relname emqx --relvsn $(PKG_VSN) --upfrom $(RELUP_BASE_VERSIONS)
-endif
-endif
-
-.PHONY: $(PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar)
-$(PROFILES:%=%-tar) $(PKG_PROFILES:%=%-tar): $(REBAR)
-ifneq ($(shell echo $(@) |grep edge),)
-	export EMQX_DESC="EMQ X Edge"
-else
-	export EMQX_DESC="EMQ X Broker"
-endif
-	$(REBAR) as $(subst -tar,,$(@)) tar
-
-.PHONY: $(PROFILES:%=%-zip)
-$(PROFILES:%=%-zip): $(REBAR)
-ifneq ($(shell echo $(PKG_VSN) | grep -oE "^[0-9]+\.[0-9]+\.[1-9]+?"),)
-	make relup-$(subst -zip,,$(@))
-endif
-	make $(subst -zip,,$(@))-tar
-
-	@tard="/tmp/emqx_untar_$(PKG_VSN)" \
-	&& rm -rf "$${tard}" && mkdir -p "$${tard}/emqx" \
-	&& prof="$(subst -zip,,$(@))" \
-	&& relpath="$$(pwd)/_build/$${prof}/rel/emqx" \
-	&& pkgpath="$$(pwd)/_packages/$${prof}" \
-	&& mkdir -p $${pkgpath} \
-	&& tarball="$${relpath}/emqx-$(PKG_VSN).tar.gz" \
-	&& zipball="$${pkgpath}/$${prof}-$(SYSTEM)-$(PKG_VSN)-$$(uname -m).zip" \
-	&& tar zxf "$${tarball}" -C "$${tard}/emqx" \
-	&& cd "$${tard}" && zip -q -r "$${zipball}" ./emqx && cd -
-
-.PHONY: $(PKG_PROFILES)
-$(PKG_PROFILES:%=%): $(REBAR)
-ifneq ($(PKGERDIR),)
-	make $(subst -pkg,,$(@))-zip
-	make $(@)-tar
-	make -C deploy/packages/$(PKGERDIR) clean
-	EMQX_REL=$$(pwd) EMQX_BUILD=$(@) PKG_VSN=$(PKG_VSN) SYSTEM=$(SYSTEM) make -C deploy/packages/$(PKGERDIR)
-else
-	make $(subst -pkg,,$(@))-zip
-endif
-

pkg-vsn.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# This script prints the release version for emqx
+
+# ensure dir
+cd -P -- "$(dirname -- "$0")"
+
+# comment SUFFIX out when finalising RELEASE
+RELEASE="4.3.0"
+SUFFIX="-pre-$(git rev-parse HEAD | cut -b1-8)"
+
+echo "${RELEASE}${SUFFIX:-}"

rebar.config.erl

@@ -232,7 +232,7 @@ env(Name, Default) ->
 
 get_vsn() ->
     PkgVsn = case env("PKG_VSN", false) of
-                 false -> os:cmd("git describe --tags --always");
+                 false -> os:cmd("./pkg-vsn.sh");
                  Vsn -> Vsn
              end,
     Vsn2 = re:replace(PkgVsn, "v", "", [{return ,list}]),