Merge pull request #8897 from emqx/build-sign-macos-binaries

build: sign binary files in macos package

.github/workflows/build_packages.yaml

@@ -178,6 +178,10 @@ jobs:
       working-directory: source
       env:
         AUTO_INSTALL_BUILD_DEPS: 1
+        APPLE_SIGN_BINARIES: 1
+        APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
+        APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
+        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
       run: |
         . $HOME/.kerl/${{ matrix.otp }}/activate
         make ensure-rebar3

.github/workflows/build_slim_packages.yaml

@@ -143,7 +143,7 @@ jobs:
     - name: prepare
       run: |
         brew update
-        brew install curl zip unzip kerl coreutils
+        brew install curl zip unzip kerl coreutils openssl@1.1
         echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
         echo "/usr/local/bin" >> $GITHUB_PATH
         echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
@@ -152,13 +152,14 @@ jobs:
       id: cache
       with:
         path: ~/.kerl/${{ matrix.otp }}
-        key: otp-install-${{ matrix.otp }}-${{ matrix.macos }}
+        key: otp-install-${{ matrix.otp }}-${{ matrix.macos }}-static-ssl-disable-hipe-disable-jit
     - name: build erlang
       if: steps.cache.outputs.cache-hit != 'true'
       timeout-minutes: 60
       env:
         KERL_BUILD_BACKEND: git
         OTP_GITHUB_URL: https://github.com/emqx/otp
+        KERL_CONFIGURE_OPTIONS: --disable-dynamic-ssl-lib --with-ssl=/usr/local/opt/openssl@1.1 --disable-hipe --disable-jit
       run: |
         kerl update releases
         kerl build ${{ matrix.otp }}
@@ -166,6 +167,13 @@ jobs:
     - name: build ${{ matrix.profile }}
       env:
         AUTO_INSTALL_BUILD_DEPS: 1
+        APPLE_SIGN_BINARIES: 1
+        APPLE_ID: developers@emqx.io
+        APPLE_TEAM_ID: 26N6HYJLZA
+        APPLE_ID_PASSWORD: ${{ secrets.APPLE_ID_PASSWORD }}
+        APPLE_DEVELOPER_IDENTITY: ${{ secrets.APPLE_DEVELOPER_IDENTITY }}
+        APPLE_DEVELOPER_ID_BUNDLE: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE }}
+        APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: ${{ secrets.APPLE_DEVELOPER_ID_BUNDLE_PASSWORD }}
       run: |
         . $HOME/.kerl/${{ matrix.otp }}/activate
         make ensure-rebar3
@@ -173,9 +181,9 @@ jobs:
         make ${{ matrix.profile }}-tgz
     - name: test
       run: |
-        pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.tar.gz)
-        mkdir -p emqx
-        tar -C emqx -zxf $pkg_name
+        pkg_name=$(find _packages/${{ matrix.profile }} -mindepth 1 -maxdepth 1 -iname \*.zip)
+        mkdir emqx
+        unzip -d emqx $pkg_name > /dev/null
         # gsed -i '/emqx_telemetry/d' ./emqx/data/loaded_plugins
         ./emqx/bin/emqx start || cat emqx/log/erlang.log.1
         ready='no'

bin/emqx

@@ -7,7 +7,7 @@ set -euo pipefail
 DEBUG="${DEBUG:-0}"
 [ "$DEBUG" -eq 1 ] && set -x
 
-RUNNER_ROOT_DIR="$(cd "$(dirname "$(realpath "$0" || echo "$0")")"/..; pwd -P)"
+RUNNER_ROOT_DIR="$(cd "$(dirname "$(readlink "$0" || echo "$0")")"/..; pwd -P)"
 
 # shellcheck disable=SC1090,SC1091
 . "$RUNNER_ROOT_DIR"/releases/emqx_vars

build

@@ -225,7 +225,16 @@ make_tgz() {
       local relpath="_build/${PROFILE}/rel/emqx"
       full_vsn="$(./pkg-vsn.sh "$PROFILE" --long)"
     fi
-    target_name="${PROFILE}-${full_vsn}.tar.gz"
+
+    case "$SYSTEM" in
+        macos*)
+            target_name="${PROFILE}-${full_vsn}.zip"
+            ;;
+        *)
+            target_name="${PROFILE}-${full_vsn}.tar.gz"
+            ;;
+    esac
+
     target="${pkgpath}/${target_name}"
 
     src_tarball="${relpath}/emqx-${PKG_VSN}.tar.gz"
@@ -242,23 +251,44 @@ make_tgz() {
     ## try to be portable for tar.gz packages.
     ## for DEB and RPM packages the dependencies are resoved by yum and apt
     cp_dyn_libs "${tard}/emqx"
-    ## create tar after change dir
-    ## to avoid creating an extra level of 'emqx' dir in the .tar.gz file
-    pushd "${tard}/emqx" >/dev/null
-    $TAR -zcf "../${target_name}" -- *
-    popd >/dev/null
-    mv "${tard}/${target_name}" "${target}"
     case "$SYSTEM" in
         macos*)
+            if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
+                ./scripts/macos-sign-binaries.sh "${tard}/emqx"
+            fi
+            ## create zip after change dir
+            ## to avoid creating an extra level of 'emqx' dir in the .zip file
+            pushd "${tard}/emqx" >/dev/null
+            zip -r "../${target_name}" -- *
+            popd >/dev/null
+            mv "${tard}/${target_name}" "${target}"
+            if [ "${APPLE_SIGN_BINARIES:-0}" = 1 ]; then
+                # notarize the package
+                # if fails, check what went wrong with this command:
+                    # xcrun notarytool log --apple-id <apple id> \
+                    #   --apple-id <apple id> \
+                    #   --password <apple id password>
+                    #   --team-id <apple team id> <submission-id>
+                xcrun notarytool submit \
+                      --apple-id "${APPLE_ID}" \
+                      --password "${APPLE_ID_PASSWORD}" \
+                      --team-id "${APPLE_TEAM_ID}" "${target}" --wait
+            fi
             # sha256sum may not be available on macos
             openssl dgst -sha256 "${target}" | cut -d ' ' -f 2  > "${target}.sha256"
             ;;
         *)
+            ## create tar after change dir
+            ## to avoid creating an extra level of 'emqx' dir in the .tar.gz file
+            pushd "${tard}/emqx" >/dev/null
+            $TAR -zcf "../${target_name}" -- *
+            popd >/dev/null
+            mv "${tard}/${target_name}" "${target}"
             sha256sum "${target}" | head -c 64 > "${target}.sha256"
             ;;
     esac
-    log "Tarball successfully repacked: ${target}"
-    log "Tarball sha256sum: $(cat "${target}.sha256")"
+    log "Archive successfully repacked: ${target}"
+    log "Archive sha256sum: $(cat "${target}.sha256")"
 }
 
 ## This function builds the default docker image based on debian 11

scripts/macos-sign-binaries.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# intended to run on MacOS only
+# signs all executable files in a given folder (as $1) with developer certificate
+
+# required variables:
+# APPLE_DEVELOPER_IDENTITY: "Developer ID Application: <company name> (<hex id>)"
+# APPLE_DEVELOPER_ID_BUNDLE: base64-encoded content of apple developer id certificate bundle in pksc12 format
+# APPLE_DEVELOPER_ID_BUNDLE_PASSWORD: password used when exporting the bundle
+
+# note: 'bundle' in apple terminology is 'identity'
+
+set -euo pipefail
+
+REL_DIR="${1}"
+PKSC12_FILE="$HOME/developer-id-application.p12"
+base64 --decode > "${PKSC12_FILE}" <<<"${APPLE_DEVELOPER_ID_BUNDLE}"
+
+KEYCHAIN='emqx.keychain-db'
+KEYCHAIN_PASSWORD="$(openssl rand -base64 32)"
+
+security create-keychain -p "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}"
+security set-keychain-settings -lut 21600 "${KEYCHAIN}"
+security unlock-keychain -p "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}"
+security import "${PKSC12_FILE}" -P "${APPLE_DEVELOPER_ID_BUNDLE_PASSWORD}" -t cert -f pkcs12 -k "${KEYCHAIN}" -T /usr/bin/codesign
+security set-key-partition-list -S "apple-tool:,apple:,codesign:" -s -k "${KEYCHAIN_PASSWORD}" "${KEYCHAIN}"
+security verify-cert -k "${KEYCHAIN}" -c "${PKSC12_FILE}"
+security find-identity -p codesigning "${KEYCHAIN}"
+
+# add new keychain into the search path for codesign, otherwise the stuff does not work
+keychains=$(security list-keychains -d user)
+keychain_names=();
+for keychain in ${keychains}; do
+    basename=$(basename "${keychain}")
+    keychain_name=${basename::${#basename}-4}
+    keychain_names+=("${keychain_name}")
+done
+security -v list-keychains -s "${keychain_names[@]}" "${KEYCHAIN}"
+
+# sign
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/erts-*/bin/{beam.smp,dyn_erl,epmd,erl,erl_call,erl_child_setup,erlexec,escript,heart,inet_gethost,run_erl,to_erl}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/asn1-*/priv/lib/asn1rt_nif.so
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/bcrypt-*/priv/bcrypt_nif.so
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/crypto-*/priv/lib/{crypto.so,otp_test_engine.so}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/jiffy-*/priv/jiffy.so
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/jq-*/priv/{jq_nif1.so,libjq.1.dylib,libonig.4.dylib,erlang_jq_port}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/os_mon-*/priv/bin/{cpu_sup,memsup}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/rocksdb-*/priv/liberocksdb.so
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/runtime_tools-*/priv/lib/{dyntrace.so,trace_ip_drv.so,trace_file_drv.so}
+codesign -s "${APPLE_DEVELOPER_IDENTITY}" -f --verbose=4 --timestamp --options=runtime "${REL_DIR}"/lib/quicer-*/priv/libquicer_nif.so