build 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426
  1. #!/usr/bin/env bash
  2. # This script helps to build release artifacts.
  3. # arg1: profile, e.g. emqx | emqx-enterprise
  4. # arg2: artifact, e.g. rel | relup | tgz | pkg
  5. set -euo pipefail
  6. [ "${DEBUG:-0}" -eq 1 ] && set -x
  7. PROFILE_ARG="$1"
  8. ARTIFACT="$2"
  9. is_enterprise() {
  10. case "$1" in
  11. *enterprise*)
  12. echo 'yes'
  13. ;;
  14. *)
  15. echo 'no'
  16. ;;
  17. esac
  18. }
  19. PROFILE_ENV="${PROFILE:-${PROFILE_ARG}}"
  20. case "$(is_enterprise "$PROFILE_ARG"),$(is_enterprise "$PROFILE_ENV")" in
  21. 'yes,yes')
  22. true
  23. ;;
  24. 'no,no')
  25. true
  26. ;;
  27. *)
  28. echo "PROFILE env var is set to '$PROFILE_ENV', but '$0' arg1 is '$PROFILE_ARG'"
  29. exit 1
  30. ;;
  31. esac
  32. # make sure PROFILE is exported, it is needed by rebar.config.erl
  33. PROFILE=$PROFILE_ARG
  34. export PROFILE
  35. # ensure dir
  36. cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")"
  37. PKG_VSN="${PKG_VSN:-$(./pkg-vsn.sh "$PROFILE")}"
  38. export PKG_VSN
  39. SYSTEM="$(./scripts/get-distro.sh)"
  40. ARCH="$(uname -m)"
  41. case "$ARCH" in
  42. x86_64)
  43. ARCH='amd64'
  44. ;;
  45. aarch64)
  46. ARCH='arm64'
  47. ;;
  48. arm*)
  49. ARCH='arm64'
  50. ;;
  51. esac
  52. export ARCH
  53. ##
  54. ## Support RPM and Debian based linux systems
  55. ##
  56. if [ "$(uname -s)" = 'Linux' ]; then
  57. case "${SYSTEM:-}" in
  58. ubuntu*|debian*|raspbian*)
  59. PKGERDIR='deb'
  60. ;;
  61. *)
  62. PKGERDIR='rpm'
  63. ;;
  64. esac
  65. fi
  66. if [ "${SYSTEM}" = 'windows' ]; then
  67. # windows does not like the find
  68. FIND="/usr/bin/find"
  69. TAR="/usr/bin/tar"
  70. export BUILD_WITHOUT_ROCKSDB="on"
  71. else
  72. FIND='find'
  73. TAR='tar'
  74. fi
  75. log() {
  76. local msg="$1"
  77. # rebar3 prints ===>, so we print ===<
  78. echo "===< $msg"
  79. }
  80. make_docs() {
  81. local libs_dir1 libs_dir2 libs_dir3 docdir dashboard_www_static
  82. libs_dir1="$("$FIND" "_build/$PROFILE/lib/" -maxdepth 2 -name ebin -type d)"
  83. if [ -d "_build/default/lib/" ]; then
  84. libs_dir2="$("$FIND" "_build/default/lib/" -maxdepth 2 -name ebin -type d)"
  85. else
  86. libs_dir2=''
  87. fi
  88. if [ -d "_build/$PROFILE/checkouts" ]; then
  89. libs_dir3="$("$FIND" "_build/$PROFILE/checkouts/" -maxdepth 2 -name ebin -type d 2>/dev/null || true)"
  90. else
  91. libs_dir3=''
  92. fi
  93. case "$(is_enterprise "$PROFILE")" in
  94. 'yes')
  95. SCHEMA_MODULE='emqx_ee_conf_schema'
  96. ;;
  97. 'no')
  98. SCHEMA_MODULE='emqx_conf_schema'
  99. ;;
  100. esac
  101. docdir="_build/docgen/$PROFILE"
  102. dashboard_www_static='apps/emqx_dashboard/priv/www/static/'
  103. mkdir -p "$docdir" "$dashboard_www_static"
  104. # shellcheck disable=SC2086
  105. erl -noshell -pa $libs_dir1 $libs_dir2 $libs_dir3 -eval \
  106. "I18nFile = filename:join([apps, emqx_dashboard, priv, 'i18n.conf']), \
  107. ok = emqx_conf:dump_schema('$docdir', $SCHEMA_MODULE, I18nFile), \
  108. halt(0)."
  109. cp "$docdir"/bridge-api-*.json "$dashboard_www_static"
  110. cp "$docdir"/hot-config-schema-*.json "$dashboard_www_static"
  111. }
  112. assert_no_compile_time_only_deps() {
  113. if [ "$("$FIND" "_build/$PROFILE/rel/emqx/lib/" -maxdepth 1 -name 'gpb-*' -type d)" != "" ]; then
  114. echo "gpb should not be included in the release"
  115. exit 1
  116. fi
  117. }
  118. make_rel() {
  119. ./scripts/pre-compile.sh "$PROFILE"
  120. # make_elixir_rel always create rebar.lock
  121. # delete it to make git clone + checkout work because we use shallow close for rebar deps
  122. rm -f rebar.lock
  123. # compile all beams
  124. ./rebar3 as "$PROFILE" compile
  125. # generate docs (require beam compiled), generated to etc and priv dirs
  126. make_docs
  127. # now assemble the release tar
  128. ./rebar3 as "$PROFILE" tar
  129. assert_no_compile_time_only_deps
  130. }
  131. make_elixir_rel() {
  132. ./scripts/pre-compile.sh "$PROFILE"
  133. export_elixir_release_vars "$PROFILE"
  134. # for some reason, this has to be run outside "do"...
  135. mix local.rebar --if-missing --force
  136. # shellcheck disable=SC1010
  137. mix do local.hex --if-missing --force, \
  138. local.rebar rebar3 "${PWD}/rebar3" --if-missing --force, \
  139. deps.get
  140. mix release --overwrite
  141. assert_no_compile_time_only_deps
  142. }
  143. ## extract previous version .tar.gz files to _build/$PROFILE/rel/emqx before making relup
  144. make_relup() {
  145. local rel_dir="_build/$PROFILE/rel/emqx"
  146. local name_pattern
  147. name_pattern="${PROFILE}-$(./pkg-vsn.sh "$PROFILE" --vsn_matcher --long)"
  148. local releases=()
  149. mkdir -p _upgrade_base
  150. while read -r tgzfile ; do
  151. local base_vsn
  152. base_vsn="$(echo "$tgzfile" | grep -oE "[0-9]+\.[0-9]+\.[0-9]+(-(alpha|beta|rc)\.[0-9])?(-[0-9a-f]{8})?" | head -1)"
  153. ## we have to create tmp dir to untar old tgz, as `tar --skip-old-files` is not supported on all plantforms
  154. local tmp_dir
  155. tmp_dir="$(mktemp -d -t emqx.XXXXXXX)"
  156. $TAR -C "$tmp_dir" -zxf "$tgzfile"
  157. mkdir -p "${rel_dir}/releases/"
  158. cp -npr "$tmp_dir/releases"/* "${rel_dir}/releases/"
  159. ## There is for some reason a copy of the '$PROFILE.rel' file to releases dir,
  160. ## the content is duplicated to releases/5.0.0/$PROFILE.rel.
  161. ## This file seems to be useless, but yet confusing as it does not change after upgrade/downgrade
  162. ## Hence we force delete this file.
  163. rm -f "${rel_dir}/releases/${PROFILE}.rel"
  164. mkdir -p "${rel_dir}/lib/"
  165. cp -npr "$tmp_dir/lib"/* "${rel_dir}/lib/"
  166. rm -rf "$tmp_dir"
  167. releases+=( "$base_vsn" )
  168. done < <("$FIND" _upgrade_base -maxdepth 1 -name "${name_pattern}.tar.gz" -type f)
  169. if [ ${#releases[@]} -eq 0 ]; then
  170. log "No upgrade base found, relup ignored"
  171. return 0
  172. fi
  173. RELX_BASE_VERSIONS="$(IFS=, ; echo "${releases[*]}")"
  174. export RELX_BASE_VERSIONS
  175. ./rebar3 as "$PROFILE" relup --relname emqx --relvsn "${PKG_VSN}"
  176. }
  177. cp_dyn_libs() {
  178. local rel_dir="$1"
  179. local target_dir="${rel_dir}/dynlibs"
  180. if ! [ "$(uname -s)" = 'Linux' ]; then
  181. return 0;
  182. fi
  183. mkdir -p "$target_dir"
  184. while read -r so_file; do
  185. cp -L "$so_file" "$target_dir/"
  186. done < <("$FIND" "$rel_dir" -type f \( -name "*.so*" -o -name "beam.smp" \) -print0 \
  187. | xargs -0 ldd \
  188. | grep -E '(libcrypto)|(libtinfo)|(libatomic)' \
  189. | awk '{print $3}' \
  190. | sort -u)
  191. }
  192. ## Re-pack the relx assembled .tar.gz to EMQX's package naming scheme
  193. ## It assumes the .tar.gz has been built -- relies on Makefile dependency
  194. make_tgz() {
  195. local pkgpath="_packages/${PROFILE}"
  196. local src_tarball
  197. local target_name
  198. local target
  199. if [ "${IS_ELIXIR:-no}" = "yes" ]
  200. then
  201. # ensure src_tarball exists
  202. ELIXIR_MAKE_TAR=yes make_elixir_rel
  203. local relpath="_build/${PROFILE}"
  204. full_vsn="$(./pkg-vsn.sh "$PROFILE" --long --elixir)"
  205. else
  206. # build the src_tarball again to ensure relup is included
  207. # elixir does not have relup yet.
  208. make_rel
  209. local relpath="_build/${PROFILE}/rel/emqx"
  210. full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)"
  211. fi
  212. case "$SYSTEM" in
  213. macos*)
  214. target_name="${PROFILE}-${full_vsn}.zip"
  215. ;;
  216. windows*)
  217. target_name="${PROFILE}-${full_vsn}.zip"
  218. ;;
  219. *)
  220. target_name="${PROFILE}-${full_vsn}.tar.gz"
  221. ;;
  222. esac
  223. target="${pkgpath}/${target_name}"
  224. src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
  225. tard="$(mktemp -d -t emqx.XXXXXXX)"
  226. mkdir -p "${tard}/emqx"
  227. mkdir -p "${pkgpath}"
  228. if [ ! -f "$src_tarball" ]; then
  229. log "ERROR: $src_tarball is not found"
  230. fi
  231. $TAR zxf "${src_tarball}" -C "${tard}/emqx"
  232. if [ -f "${tard}/emqx/releases/${PKG_VSN}/relup" ]; then
  233. ./scripts/relup-build/inject-relup.escript "${tard}/emqx/releases/${PKG_VSN}/relup"
  234. fi
  235. ## try to be portable for tar.gz packages.
  236. ## for DEB and RPM packages the dependencies are resoved by yum and apt
  237. cp_dyn_libs "${tard}/emqx"
  238. case "$SYSTEM" in
  239. macos*)
  240. # if the flag to sign macos binaries is set, but developer certificate
  241. # or certificate password is not configured, reset the flag
  242. # could happen, for example, when people submit PR from a fork, in this
  243. # case they cannot access secrets
  244. if [[ "${APPLE_SIGN_BINARIES:-0}" == 1 && \
  245. ( "${APPLE_DEVELOPER_ID_BUNDLE:-0}" == 0 || \
  246. "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD:-0}" == 0 ) ]]; then
  247. echo "Apple developer certificate is not configured, skip signing"
  248. APPLE_SIGN_BINARIES=0
  249. fi
  250. if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
  251. ./scripts/macos-sign-binaries.sh "${tard}/emqx"
  252. fi
  253. ## create zip after change dir
  254. ## to avoid creating an extra level of 'emqx' dir in the .zip file
  255. pushd "${tard}/emqx" >/dev/null
  256. zip -r "../${target_name}" -- * >/dev/null
  257. popd >/dev/null
  258. mv "${tard}/${target_name}" "${target}"
  259. if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
  260. # notarize the package
  261. # if fails, check what went wrong with this command:
  262. # xcrun notarytool log --apple-id <apple id> \
  263. # --apple-id <apple id> \
  264. # --password <apple id password>
  265. # --team-id <apple team id> <submission-id>
  266. echo 'Submitting the package for notarization to Apple (normally takes about a minute)'
  267. notarytool_output="$(xcrun notarytool submit \
  268. --apple-id "${APPLE_ID}" \
  269. --password "${APPLE_ID_PASSWORD}" \
  270. --team-id "${APPLE_TEAM_ID}" "${target}" \
  271. --no-progress \
  272. --wait)"
  273. echo "$notarytool_output"
  274. echo "$notarytool_output" | grep -q 'status: Accepted' || {
  275. echo 'Notarization failed';
  276. exit 1;
  277. }
  278. fi
  279. # sha256sum may not be available on macos
  280. openssl dgst -sha256 "${target}" | cut -d ' ' -f 2 > "${target}.sha256"
  281. ;;
  282. windows*)
  283. pushd "${tard}" >/dev/null
  284. 7z a "${target_name}" ./emqx/* >/dev/null
  285. popd >/dev/null
  286. mv "${tard}/${target_name}" "${target}"
  287. sha256sum "${target}" | head -c 64 > "${target}.sha256"
  288. ;;
  289. *)
  290. ## create tar after change dir
  291. ## to avoid creating an extra level of 'emqx' dir in the .tar.gz file
  292. pushd "${tard}/emqx" >/dev/null
  293. $TAR -zcf "../${target_name}" -- *
  294. popd >/dev/null
  295. mv "${tard}/${target_name}" "${target}"
  296. sha256sum "${target}" | head -c 64 > "${target}.sha256"
  297. ;;
  298. esac
  299. log "Archive successfully repacked: ${target}"
  300. log "Archive sha256sum: $(cat "${target}.sha256")"
  301. }
  302. trap docker_cleanup EXIT
  303. docker_cleanup() {
  304. rm -f ./.dockerignore >/dev/null
  305. }
  306. ## This function builds the default docker image based on debian 11
  307. make_docker() {
  308. EMQX_BUILDER="${EMQX_BUILDER:-${EMQX_DEFAULT_BUILDER}}"
  309. EMQX_RUNNER="${EMQX_RUNNER:-${EMQX_DEFAULT_RUNNER}}"
  310. EMQX_DOCKERFILE="${EMQX_DOCKERFILE:-deploy/docker/Dockerfile}"
  311. if [[ "$PROFILE" = *-elixir ]]; then
  312. PKG_VSN="$PKG_VSN-elixir"
  313. fi
  314. local default_tag="emqx/${PROFILE%%-elixir}:${PKG_VSN}"
  315. EMQX_IMAGE_TAG="${EMQX_IMAGE_TAG:-$default_tag}"
  316. echo '_build' >> ./.dockerignore
  317. set -x
  318. docker build --no-cache --pull \
  319. --build-arg BUILD_FROM="${EMQX_BUILDER}" \
  320. --build-arg RUN_FROM="${EMQX_RUNNER}" \
  321. --build-arg EMQX_NAME="$PROFILE" \
  322. --tag "${EMQX_IMAGE_TAG}" \
  323. -f "${EMQX_DOCKERFILE}" .
  324. [[ "${DEBUG:-}" -eq 1 ]] || set +x
  325. }
  326. function join {
  327. local IFS="$1"
  328. shift
  329. echo "$*"
  330. }
  331. # used to control the Elixir Mix Release output
  332. # see docstring in `mix.exs`
  333. export_elixir_release_vars() {
  334. local profile="$1"
  335. case "$profile" in
  336. emqx|emqx-enterprise)
  337. export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-no}
  338. ;;
  339. emqx-pkg|emqx-enterprise-pkg)
  340. export ELIXIR_MAKE_TAR=${ELIXIR_MAKE_TAR:-yes}
  341. ;;
  342. *)
  343. echo Invalid profile "$profile"
  344. exit 1
  345. esac
  346. export MIX_ENV="$profile"
  347. }
  348. log "building artifact=$ARTIFACT for profile=$PROFILE"
  349. case "$ARTIFACT" in
  350. doc|docs)
  351. make_docs
  352. ;;
  353. rel)
  354. make_rel
  355. ;;
  356. relup)
  357. make_relup
  358. ;;
  359. tgz)
  360. make_tgz
  361. ;;
  362. pkg)
  363. # this only affect build artifacts, such as schema doc
  364. export EMQX_ETC_DIR='/etc/emqx/'
  365. if [ -z "${PKGERDIR:-}" ]; then
  366. log "Skipped making deb/rpm package for $SYSTEM"
  367. exit 0
  368. fi
  369. export EMQX_REL_FORM="$PKGERDIR"
  370. if [ "${IS_ELIXIR:-}" = 'yes' ]; then
  371. make_elixir_rel
  372. else
  373. make_rel
  374. fi
  375. env EMQX_REL="$(pwd)" \
  376. EMQX_BUILD="${PROFILE}" \
  377. make -C "deploy/packages/${PKGERDIR}" clean
  378. env EMQX_REL="$(pwd)" \
  379. EMQX_BUILD="${PROFILE}" \
  380. make -C "deploy/packages/${PKGERDIR}"
  381. ;;
  382. docker)
  383. make_docker
  384. ;;
  385. elixir)
  386. make_elixir_rel
  387. ;;
  388. *)
  389. log "Unknown artifact $ARTIFACT"
  390. exit 1
  391. ;;
  392. esac