feat(rule_engine): add jq function to the rule engine

This commit adds a function to the rule engine that alows users
to transform text or JSON objects using [jq filter programs][1].

[jq][1] is a command line tool that can be used to transform
and filter JSON text using jq's built-in language. The rule engine
function that is added with this commit uses the
[Erlang jq NIF library][2] that wraps the jq C library in an
Erlang NIF function.

[1]: https://stedolan.github.io/jq/
[2]: https://github.com/emqx/jq

.ci/docker-compose-file/docker-compose.yaml

@@ -3,7 +3,7 @@ version: '3.9'
 services:
   erlang23:
     container_name: erlang23
-    image: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-23.3.4.9-4-ubuntu20.04
+    image: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-23.3.4.9-4-ubuntu20.04
     env_file:
       - conf.env
     environment:
@@ -23,7 +23,7 @@ services:
 
   erlang24:
     container_name: erlang24
-    image: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
+    image: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
     env_file:
       - conf.env
     environment:

.github/workflows/apps_version_check.yaml

@@ -7,6 +7,8 @@ jobs:
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
       - name: Check apps version
         run: ./scripts/apps-version-check.sh

.github/workflows/build_packages.yaml

@@ -22,7 +22,7 @@ jobs:
   prepare:
     runs-on: ubuntu-20.04
     # prepare source with any OTP version, no need for a matrix
-    container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04"
 
     outputs:
       BUILD_PROFILES: ${{ steps.get_profiles.outputs.BUILD_PROFILES }}
@@ -139,7 +139,8 @@ jobs:
     - name: prepare
       run: |
         brew update
-        brew install curl zip unzip gnu-sed kerl unixodbc freetds
+        brew install curl zip unzip gnu-sed kerl unixodbc freetds automake bison
+        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
         echo "/usr/local/bin" >> $GITHUB_PATH
         git config --global credential.helper store
     - uses: actions/cache@v2
@@ -205,7 +206,7 @@ jobs:
     needs: prepare
     runs-on: ${{ matrix.build_machine }}
     container:
-      image: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+      image: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
     strategy:
       fail-fast: false
@@ -234,8 +235,8 @@ jobs:
           - debian11
           - debian10
           - debian9
-          - rockylinux8
-          - centos7
+          - el8
+          - el7
           - raspbian10
         build_machine:
           - aws-arm64
@@ -272,7 +273,7 @@ jobs:
             elixir: 1.13.3
             build_elixir: with_elixir
             arch: amd64
-            os: centos7
+            os: el7
             build_machine: ubuntu-20.04
 
     defaults:
@@ -325,7 +326,7 @@ jobs:
             --pkgtype "${PKGTYPE}" \
             --arch "${ARCH}" \
             --elixir "${IsElixir}" \
-            --builder "ghcr.io/emqx/emqx-builder/5.0-10:${ELIXIR}-${OTP}-${SYSTEM}"
+            --builder "ghcr.io/emqx/emqx-builder/5.0-14:${ELIXIR}-${OTP}-${SYSTEM}"
         done
     - uses: actions/upload-artifact@v1
       if: startsWith(github.ref, 'refs/tags/')
@@ -462,7 +463,7 @@ jobs:
         tags: ${{ steps.meta.outputs.tags }}
         labels: ${{ steps.meta.outputs.labels }}
         build-args: |
-          BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
+          BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
           RUN_FROM=${{ steps.pre-meta.outputs.img }}
           EMQX_NAME=${{ steps.pre-meta.outputs.emqx_name }}
         file: source/deploy/docker/Dockerfile

.github/workflows/build_slim_packages.yaml

@@ -38,15 +38,15 @@ jobs:
         - 1.13.3
         os:
         - ubuntu20.04
-        - rockylinux8
+        - el8
 
-    container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+    container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
     steps:
-    - name: cleanup
-      run: |
-        rm -rf "${GITHUB_WORKSPACE}/"
-    - uses: actions/checkout@v1
+    - uses: AutoModality/action-clean@v1
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
     - name: prepare
       run: |
         echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
@@ -63,6 +63,9 @@ jobs:
           _build/default/lib/quicer/
           deps/quicer/
         key: ${{ matrix.os }}-${{ matrix.elixir }}-${{ matrix.otp }}-amd64-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
+    - name: Work around https://github.com/actions/checkout/issues/766
+      run: |
+        git config --global --add safe.directory "$GITHUB_WORKSPACE"
     - name: build and test tgz package
       run: |
         make ${EMQX_NAME}-tgz
@@ -148,7 +151,8 @@ jobs:
     - name: prepare
       run: |
         brew update
-        brew install curl zip unzip gnu-sed kerl unixodbc freetds
+        brew install curl zip unzip gnu-sed kerl unixodbc freetds automake bison
+        echo "/usr/local/opt/bison/bin" >> $GITHUB_PATH
         echo "/usr/local/bin" >> $GITHUB_PATH
         echo "EMQX_NAME=${{ matrix.profile }}" >> $GITHUB_ENV
     - uses: actions/cache@v2

.github/workflows/check_deps_integrity.yaml

@@ -5,7 +5,7 @@ on: [pull_request, push]
 jobs:
   check_deps_integrity:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
 
     steps:
       - uses: actions/checkout@v2

.github/workflows/code_style_check.yaml

@@ -5,7 +5,7 @@ on: [pull_request]
 jobs:
   code_style_check:
     runs-on: ubuntu-20.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04"
     steps:
       - uses: actions/checkout@v2
         with:
@@ -13,6 +13,9 @@ jobs:
       - name: Check line-break at EOF
         run: |
           ./scripts/check-nl-at-eof.sh
+      - name: Work around https://github.com/actions/checkout/issues/766
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - name: Check Elixir code formatting
         run: |
           mix format --check-formatted

.github/workflows/elixir_apps_check.yaml

@@ -6,9 +6,9 @@ on: [pull_request, push]
 
 jobs:
   elixir_apps_check:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     # just use the latest builder
-    container: "ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04"
+    container: "ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04"
 
     strategy:
       fail-fast: false
@@ -31,12 +31,17 @@ jobs:
             edition_type: enterprise
 
     steps:
+      - name: fix_git_permission
+        run: git config --global --add safe.directory '/__w/emqx/emqx'
       - name: Checkout
-        uses: actions/checkout@v2.4.0
+        uses: actions/checkout@v2
         with:
           fetch-depth: 0
       - name: ensure rebar
         run: ./scripts/ensure-rebar3.sh
+      - name: Work around https://github.com/actions/checkout/issues/766
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - name: check applications
         run: ./scripts/check-elixir-applications.exs
       - name: check applications started with emqx_machine

.github/workflows/elixir_deps_check.yaml

@@ -7,13 +7,16 @@ on: [pull_request, push]
 jobs:
   elixir_deps_check:
     runs-on: ubuntu-20.04
-    container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2.4.0
+        uses: actions/checkout@v2
       - name: ensure rebar
         run: ./scripts/ensure-rebar3.sh
+      - name: Work around https://github.com/actions/checkout/issues/766
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - name: setup mix
         run: |
           mix local.hex --force

.github/workflows/elixir_release.yml

@@ -12,13 +12,16 @@ on:
 jobs:
   elixir_release_build:
     runs-on: ubuntu-latest
-    container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
+    container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2.4.0
+        uses: actions/checkout@v2
       - name: install tools
         run: apt update && apt install netcat-openbsd
+      - name: Work around https://github.com/actions/checkout/issues/766
+        run: |
+          git config --global --add safe.directory "$GITHUB_WORKSPACE"
       - name: elixir release
         run: make emqx-elixir
       - name: start release

.github/workflows/run_emqx_app_tests.yaml

@@ -25,7 +25,7 @@ jobs:
           - amd64
 
     runs-on: aws-amd64
-    container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
+    container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir}}-${{ matrix.otp }}-${{ matrix.os }}"
 
     defaults:
       run:
@@ -47,14 +47,19 @@ jobs:
         key: ${{ matrix.os }}-${{ matrix.otp }}-${{ matrix.arch }}-${{ steps.deps-refs.outputs.DEP_QUICER_REF }}
     - name: run
       run: |
+        git config --global --add safe.directory "$GITHUB_WORKSPACE"
         echo "git diff base: $GITHUB_BASE_REF"
         if [[ "$GITHUB_BASE_REF" =~ [0-9a-f]{8,40} ]]; then
           # base is a commit sha1
           compare_base="$GITHUB_BASE_REF"
         else
-          compare_base="origin/$GITHUB_BASE_REF"
+          repo="${GITHUB_REPOSITORY}"
+          git remote -v
+          remote="$(git remote -v | grep -E "github\.com(:|/)$repo((\.git)|(\s))" | grep fetch | awk '{print $1}')"
+          git fetch "$remote" "$GITHUB_BASE_REF"
+          compare_base="$remote/$GITHUB_BASE_REF"
         fi
-        changed_files="$(git diff --name-only ${compare_base}...HEAD apps/emqx)"
+        changed_files="$(git diff --name-only ${compare_base} HEAD apps/emqx)"
         if [ "$changed_files" = '' ]; then
           echo "nothing changed in apps/emqx, ignored."
           exit 0

.github/workflows/run_fvt_tests.yaml

@@ -16,7 +16,7 @@ jobs:
   prepare:
     runs-on: ubuntu-20.04
     # prepare source with any OTP version, no need for a matrix
-    container: ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1
+    container: ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1
 
     steps:
       - uses: actions/checkout@v2
@@ -84,7 +84,7 @@ jobs:
     - name: make docker image
       working-directory: source
       env:
-        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
+        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
       run: |
         make ${{ matrix.profile }}-docker
     - name: run emqx
@@ -162,7 +162,7 @@ jobs:
     - name: make docker image
       working-directory: source
       env:
-        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
+        EMQX_BUILDER: ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}
       run: |
         make ${{ matrix.profile }}-docker
         echo "TARGET=emqx/${{ matrix.profile }}" >> $GITHUB_ENV

.github/workflows/run_relup_tests.yaml

@@ -33,7 +33,7 @@ jobs:
           - amd64
 
     runs-on: ubuntu-20.04
-    container: "ghcr.io/emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+    container: "ghcr.io/emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
     defaults:
       run:

.github/workflows/run_test_cases.yaml

@@ -27,7 +27,7 @@ jobs:
               - amd64
 
         runs-on: aws-amd64
-        container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+        container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
 
         steps:
         - uses: actions/checkout@v2
@@ -58,7 +58,7 @@ jobs:
 
     find_apps:
         runs-on: aws-amd64
-        container: emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-ubuntu20.04
+        container: emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-ubuntu20.04
         outputs:
           fast_ct_apps: ${{ steps.run_find_apps.outputs.fast_ct_apps }}
           docker_ct_apps: ${{ steps.run_find_apps.outputs.docker_ct_apps }}
@@ -114,7 +114,7 @@ jobs:
           # produces <app-name>.coverdata
         - name: run common test
           run: |
-            docker exec -i ${{ matrix.otp_release }} bash -c "make ${{ matrix.app_name }}-ct"
+            docker exec -i ${{ matrix.otp_release }} bash -c "git config --global --add safe.directory \"$GITHUB_WORKSPACE\" && make ${{ matrix.app_name }}-ct"
         - uses: actions/upload-artifact@v1
           if: matrix.otp_release == 'erlang24'
           with:
@@ -141,7 +141,7 @@ jobs:
             arch:
               - amd64
         runs-on: aws-amd64
-        container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+        container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
         defaults:
           run:
             shell: bash
@@ -180,7 +180,7 @@ jobs:
             - amd64
 
       runs-on: aws-amd64
-      container: "emqx/emqx-builder/5.0-10:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
+      container: "emqx/emqx-builder/5.0-14:${{ matrix.elixir }}-${{ matrix.otp }}-${{ matrix.os }}"
       steps:
       - uses: actions/checkout@v2
 

Makefile

@@ -3,7 +3,7 @@ REBAR = $(CURDIR)/rebar3
 BUILD = $(CURDIR)/build
 SCRIPTS = $(CURDIR)/scripts
 export EMQX_RELUP ?= true
-export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1
+export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1
 export EMQX_DEFAULT_RUNNER = alpine:3.15.1
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)

apps/emqx_rule_engine/rebar.config

@@ -1,6 +1,8 @@
 %% -*- mode: erlang -*-
 
-{deps, [{emqx, {path, "../emqx"}}]}.
+{deps, [
+    {emqx, {path, "../emqx"}}
+]}.
 
 {erl_opts, [
     warn_unused_vars,

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -147,7 +147,8 @@
     regex_replace/3,
     ascii/1,
     find/2,
-    find/3
+    find/3,
+    jq/2
 ]).
 
 %% Map Funcs
@@ -779,6 +780,23 @@ find_s(S, P, Dir) ->
         SubStr -> SubStr
     end.
 
+-spec jq(FilterProgram, JSON) -> Result when
+    FilterProgram :: binary(),
+    JSON :: binary() | term(),
+    Result :: [term()].
+jq(FilterProgram, JSONBin) when
+    is_binary(FilterProgram), is_binary(JSONBin)
+->
+    case jq:parse(FilterProgram, JSONBin) of
+        {ok, Result} ->
+            [json_decode(JSONString) || JSONString <- Result];
+        {error, ErrorReason} ->
+            erlang:throw({jq_exception, ErrorReason})
+    end;
+jq(FilterProgram, JSONTerm) when is_binary(FilterProgram) ->
+    JSONBin = json_encode(JSONTerm),
+    jq(FilterProgram, JSONBin).
+
 %%------------------------------------------------------------------------------
 %% Array Funcs
 %%------------------------------------------------------------------------------

apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl

@@ -57,6 +57,7 @@ groups() ->
             t_match_atom_and_binary,
             t_sqlselect_0,
             t_sqlselect_00,
+            t_sqlselect_001,
             t_sqlselect_01,
             t_sqlselect_02,
             t_sqlselect_1,
@@ -728,6 +729,43 @@ t_sqlselect_00(_Config) ->
         )
     ).
 
+t_sqlselect_001(_Config) ->
+    %% Verify that the jq function can be called from SQL
+    Sql =
+        "select jq('.what + .what', payload) as ans "
+        "from \"t/#\" ",
+    ?assertMatch(
+        {ok, #{<<"ans">> := [8]}},
+        emqx_rule_sqltester:test(
+            #{
+                sql => Sql,
+                context =>
+                    #{
+                        payload => #{<<"what">> => 4},
+                        topic => <<"t/a">>
+                    }
+            }
+        )
+    ),
+    Sql2 =
+        "SELECT jq('.a|.[]', "
+        "'{\"a\": [{\"b\": 1}, {\"b\": 2}, {\"b\": 3}]}') "
+        "as jq_output, "
+        "   jq_output[1].b as first_b from \"t/#\" ",
+    ?assertMatch(
+        {ok, #{<<"first_b">> := 1}},
+        emqx_rule_sqltester:test(
+            #{
+                sql => Sql2,
+                context =>
+                    #{
+                        payload => #{<<"what">> => 4},
+                        topic => <<"t/a">>
+                    }
+            }
+        )
+    ).
+
 t_sqlselect_01(_Config) ->
     SQL =
         "SELECT json_decode(payload) as p, payload "

apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl

@@ -629,6 +629,24 @@ t_regex_replace(_) ->
     ?assertEqual(<<"aebed">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"e">>])),
     ?assertEqual(<<"a[cc]b[c]d">>, apply_func(regex_replace, [<<"accbcd">>, <<"c+">>, <<"[&]">>])).
 
+jq_1_elm_res(JSONString) ->
+    Bin = list_to_binary(JSONString),
+    [apply_func(json_decode, [Bin])].
+
+t_jq(_) ->
+    ?assertEqual(
+        jq_1_elm_res("{\"b\":2}"),
+        apply_func(jq, [<<".">>, apply_func(json_decode, [<<"{\"b\": 2}">>])])
+    ),
+    ?assertEqual(
+        jq_1_elm_res("6"),
+        apply_func(jq, [<<".+1">>, apply_func(json_decode, [<<"5">>])])
+    ),
+    ?assertEqual(
+        jq_1_elm_res("{\"b\":2}"),
+        apply_func(jq, [<<".">>, <<"{\"b\": 2}">>])
+    ).
+
 ascii_string() -> list(range(0, 127)).
 
 bin(S) -> iolist_to_binary(S).

deploy/docker/Dockerfile

@@ -1,22 +1,27 @@
-ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-alpine3.15.1
+ARG BUILD_FROM=ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-alpine3.15.1
 ARG RUN_FROM=alpine:3.15.1
 FROM ${BUILD_FROM} AS builder
 
 RUN apk add --no-cache \
-    git \
+    autoconf \
+    automake \
+    bash \
+    bison \
+    bsd-compat-headers \
+    coreutils \
     curl \
-    gcc \
+    flex \
     g++ \
+    gcc \
+    git \
+    jq \
+    libc-dev \
+    libstdc++ \
+    libtool \
     make \
-    perl \
     ncurses-dev \
     openssl-dev \
-    coreutils \
-    bsd-compat-headers \
-    libc-dev \
-    libstdc++ \
-    bash \
-    jq
+    perl
 
 COPY . /emqx
 

elvis.config

@@ -37,6 +37,12 @@
            {elvis_style, nesting_level, #{ level => 6 }}
           ]
         },
+      #{dirs => ["apps/emqx_rule_engine/src"],
+        filter => "*_rule_funcs.erl",
+        rules => [
+            {elvis_style, god_modules, disable}
+           ]
+       },
       #{dirs => ["."],
         filter => "Makefile",
         ruleset => makefiles

mix.exs

@@ -91,7 +91,7 @@ defmodule EMQXUmbrella.MixProject do
        github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
       # in conflict by grpc and eetcd
       {:gpb, "4.11.2", override: true, runtime: false}
-    ] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep()
+    ] ++ umbrella_apps() ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
   end
 
   defp umbrella_apps() do
@@ -202,6 +202,7 @@ defmodule EMQXUmbrella.MixProject do
     ] ++
       if(enable_quicer?(), do: [quicer: :permanent], else: []) ++
       if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++
+      if(enable_jq?(), do: [jq: :permanent], else: []) ++
       if(edition_type == :enterprise,
         do: [
           emqx_enterprise_conf: :load,
@@ -608,6 +609,12 @@ defmodule EMQXUmbrella.MixProject do
       else: []
   end
 
+  defp jq_dep() do
+    if enable_jq?(),
+      do: [{:jq, github: "emqx/jq", tag: "v0.1.0", override: true}],
+      else: []
+  end
+
   defp quicer_dep() do
     if enable_quicer?(),
       # in conflict with emqx and emqtt
@@ -619,6 +626,10 @@ defmodule EMQXUmbrella.MixProject do
     not win32?()
   end
 
+  defp enable_jq?() do
+    not win32?()
+  end
+
   defp enable_quicer?() do
     not Enum.any?([
       build_without_quic?(),

pkg-vsn.sh

@@ -84,6 +84,8 @@ RELEASE="$(grep -E "define.+${RELEASE_EDITION}" apps/emqx/include/emqx_release.h
 
 git_exact_vsn() {
     local tag
+    ## Needed to avoid error in github action
+    git config --global --add safe.directory "/__w/emqx/emqx"
     tag="$(git describe --tags --match "${GIT_TAG_PREFIX}*" --exact 2>/dev/null)"
     echo "${tag#[v|e]}"
 }

rebar.config.erl

@@ -19,12 +19,17 @@ assert_otp() ->
     OtpRelease = list_to_integer(erlang:system_info(otp_release)),
     case OtpRelease < Oldest orelse OtpRelease > Latest of
         true ->
-            io:format(standard_error, "ERROR: Erlang/OTP version ~p found. min=~p, recommended=~p~n",
-                      [OtpRelease, Oldest, Latest]),
+            io:format(
+                standard_error,
+                "ERROR: Erlang/OTP version ~p found. min=~p, recommended=~p~n",
+                [OtpRelease, Oldest, Latest]
+            ),
             halt(1);
         false when OtpRelease =/= Latest ->
-            io:format("WARNING: Erlang/OTP version ~p found, recommended==~p~n",
-                      [OtpRelease, Latest]);
+            io:format(
+                "WARNING: Erlang/OTP version ~p found, recommended==~p~n",
+                [OtpRelease, Latest]
+            );
         false ->
             ok
     end.
@@ -35,16 +40,19 @@ bcrypt() ->
 quicer() ->
     {quicer, {git, "https://github.com/emqx/quic.git", {tag, "0.0.9"}}}.
 
+jq() ->
+    {jq, {git, "https://github.com/emqx/jq", {tag, "v0.1.0"}}}.
+
 deps(Config) ->
     {deps, OldDeps} = lists:keyfind(deps, 1, Config),
-    MoreDeps = [bcrypt() || provide_bcrypt_dep()] ++
-        [quicer() || is_quicer_supported()],
+    MoreDeps =
+        [bcrypt() || provide_bcrypt_dep()] ++
+            [jq() || provide_jq()] ++
+            [quicer() || is_quicer_supported()],
     lists:keystore(deps, 1, Config, {deps, OldDeps ++ MoreDeps}).
 
 overrides() ->
-    [ {add, [ {extra_src_dirs, [{"etc", [{recursive, true}]}]}
-            ]}
-    ] ++ snabbkaffe_overrides().
+    [{add, [{extra_src_dirs, [{"etc", [{recursive, true}]}]}]}] ++ snabbkaffe_overrides().
 
 %% Temporary workaround for a rebar3 erl_opts duplication
 %% bug. Ideally, we want to set this define globally
@@ -53,14 +61,15 @@ snabbkaffe_overrides() ->
     [{add, App, [{erl_opts, [{d, snk_kind, msg}]}]} || App <- Apps].
 
 config() ->
-    [ {cover_enabled, is_cover_enabled()}
-    , {profiles, profiles()}
-    , {plugins, plugins()}
+    [
+        {cover_enabled, is_cover_enabled()},
+        {profiles, profiles()},
+        {plugins, plugins()}
     ].
 
 is_cover_enabled() ->
     case os:getenv("ENABLE_COVER_COMPILE") of
-        "1"-> true;
+        "1" -> true;
         "true" -> true;
         _ -> false
     end.
@@ -70,14 +79,13 @@ is_enterprise(ee) -> true.
 
 is_quicer_supported() ->
     not (false =/= os:getenv("BUILD_WITHOUT_QUIC") orelse
-         is_win32() orelse is_centos_6()
-        ).
+        is_win32() orelse is_centos_6()).
 
 is_centos_6() ->
     %% reason:
     %% glibc is too old
     case file:read_file("/etc/centos-release") of
-        {ok, <<"CentOS release 6", _/binary >>} ->
+        {ok, <<"CentOS release 6", _/binary>>} ->
             true;
         _ ->
             false
@@ -88,145 +96,158 @@ is_win32() ->
 
 project_app_dirs(Edition) ->
     ["apps/*"] ++
-    case is_enterprise(Edition) of
-        true -> ["lib-ee/*"];
-        false -> []
-    end.
+        case is_enterprise(Edition) of
+            true -> ["lib-ee/*"];
+            false -> []
+        end.
 
 plugins() ->
-    [ {relup_helper,{git,"https://github.com/emqx/relup_helper", {tag, "2.0.0"}}}
-    , {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}}
-      %% emqx main project does not require port-compiler
-      %% pin at root level for deterministic
-    , {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}}
-    ]
-    %% test plugins are concatenated to default profile plugins
-    %% otherwise rebar3 test profile runs are super slow
-    ++ test_plugins().
+    [
+        {relup_helper, {git, "https://github.com/emqx/relup_helper", {tag, "2.0.0"}}},
+        {er_coap_client, {git, "https://github.com/emqx/er_coap_client", {tag, "v1.0.5"}}},
+        %% emqx main project does not require port-compiler
+        %% pin at root level for deterministic
+        {pc, {git, "https://github.com/emqx/port_compiler.git", {tag, "v1.11.1"}}}
+    ] ++
+        %% test plugins are concatenated to default profile plugins
+        %% otherwise rebar3 test profile runs are super slow
+        test_plugins().
 
 test_plugins() ->
-    [ {rebar3_proper, "0.12.1"}
-    , {coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}}
+    [
+        {rebar3_proper, "0.12.1"},
+        {coveralls, {git, "https://github.com/emqx/coveralls-erl", {tag, "v2.2.0-emqx-1"}}}
     ].
 
 test_deps() ->
-    [ {bbmustache, "1.10.0"}
-    , {meck, "0.9.2"}
-    , {proper, "1.4.0"}
+    [
+        {bbmustache, "1.10.0"},
+        {meck, "0.9.2"},
+        {proper, "1.4.0"}
     ].
 
 common_compile_opts(Edition, Vsn) ->
-    [ debug_info % always include debug_info
-    , {compile_info, [{emqx_vsn, Vsn}]}
-    , {d, 'EMQX_RELEASE_EDITION', Edition}
+    % always include debug_info
+    [
+        debug_info,
+        {compile_info, [{emqx_vsn, Vsn}]},
+        {d, 'EMQX_RELEASE_EDITION', Edition}
     ] ++
-    [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1" ].
+        [{d, 'EMQX_BENCHMARK'} || os:getenv("EMQX_BENCHMARK") =:= "1"].
 
 prod_compile_opts(Edition, Vsn) ->
-    [ compressed
-    , deterministic
-    , warnings_as_errors
-    | common_compile_opts(Edition, Vsn)
+    [
+        compressed,
+        deterministic,
+        warnings_as_errors
+        | common_compile_opts(Edition, Vsn)
     ].
 
 prod_overrides() ->
-    [{add, [ {erl_opts, [deterministic]}]}].
+    [{add, [{erl_opts, [deterministic]}]}].
 
 profiles() ->
     profiles_ce() ++ profiles_ee() ++ profiles_dev().
 
 profiles_ce() ->
     Vsn = get_vsn(emqx),
-    [ {'emqx',
-       [ {erl_opts, prod_compile_opts(ce, Vsn)}
-       , {relx, relx(Vsn, cloud, bin, ce)}
-       , {overrides, prod_overrides()}
-       , {project_app_dirs, project_app_dirs(ce)}
-       , {post_hooks, [{compile, "bash build emqx doc"}]}
-       ]}
-    , {'emqx-pkg',
-       [ {erl_opts, prod_compile_opts(ce, Vsn)}
-       , {relx, relx(Vsn, cloud, pkg, ce)}
-       , {overrides, prod_overrides()}
-       , {project_app_dirs, project_app_dirs(ce)}
-       , {post_hooks, [{compile, "bash build emqx-pkg doc"}]}
-       ]}
-    , {'emqx-edge',
-       [ {erl_opts, prod_compile_opts(edge, Vsn)}
-       , {relx, relx(Vsn, edge, bin, ce)}
-       , {overrides, prod_overrides()}
-       , {project_app_dirs, project_app_dirs(ce)}
-       , {post_hooks, [{compile, "bash build emqx-edge doc"}]}
-       ]}
-    , {'emqx-edge-pkg',
-       [ {erl_opts, prod_compile_opts(edge, Vsn)}
-       , {relx, relx(Vsn, edge, pkg, ce)}
-       , {overrides, prod_overrides()}
-       , {project_app_dirs, project_app_dirs(ce)}
-       , {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]}
-       ]}
+    [
+        {'emqx', [
+            {erl_opts, prod_compile_opts(ce, Vsn)},
+            {relx, relx(Vsn, cloud, bin, ce)},
+            {overrides, prod_overrides()},
+            {project_app_dirs, project_app_dirs(ce)},
+            {post_hooks, [{compile, "bash build emqx doc"}]}
+        ]},
+        {'emqx-pkg', [
+            {erl_opts, prod_compile_opts(ce, Vsn)},
+            {relx, relx(Vsn, cloud, pkg, ce)},
+            {overrides, prod_overrides()},
+            {project_app_dirs, project_app_dirs(ce)},
+            {post_hooks, [{compile, "bash build emqx-pkg doc"}]}
+        ]},
+        {'emqx-edge', [
+            {erl_opts, prod_compile_opts(edge, Vsn)},
+            {relx, relx(Vsn, edge, bin, ce)},
+            {overrides, prod_overrides()},
+            {project_app_dirs, project_app_dirs(ce)},
+            {post_hooks, [{compile, "bash build emqx-edge doc"}]}
+        ]},
+        {'emqx-edge-pkg', [
+            {erl_opts, prod_compile_opts(edge, Vsn)},
+            {relx, relx(Vsn, edge, pkg, ce)},
+            {overrides, prod_overrides()},
+            {project_app_dirs, project_app_dirs(ce)},
+            {post_hooks, [{compile, "bash build emqx-edge-pkg doc"}]}
+        ]}
     ].
 
 profiles_ee() ->
     Vsn = get_vsn('emqx-enterprise'),
-    [ {'emqx-enterprise',
-       [ {erl_opts, prod_compile_opts(ee, Vsn)}
-       , {relx, relx(Vsn, cloud, bin, ee)}
-       , {overrides, prod_overrides()}
-       , {project_app_dirs, project_app_dirs(ee)}
-       , {post_hooks, [{compile, "bash build emqx-enterprise doc"}]}
-       ]}
-    , {'emqx-enterprise-pkg',
-       [ {erl_opts, prod_compile_opts(ee, Vsn)}
-       , {relx, relx(Vsn, cloud, pkg, ee)}
-       , {overrides, prod_overrides()}
-       , {project_app_dirs, project_app_dirs(ee)}
-       , {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]}
-       ]}
+    [
+        {'emqx-enterprise', [
+            {erl_opts, prod_compile_opts(ee, Vsn)},
+            {relx, relx(Vsn, cloud, bin, ee)},
+            {overrides, prod_overrides()},
+            {project_app_dirs, project_app_dirs(ee)},
+            {post_hooks, [{compile, "bash build emqx-enterprise doc"}]}
+        ]},
+        {'emqx-enterprise-pkg', [
+            {erl_opts, prod_compile_opts(ee, Vsn)},
+            {relx, relx(Vsn, cloud, pkg, ee)},
+            {overrides, prod_overrides()},
+            {project_app_dirs, project_app_dirs(ee)},
+            {post_hooks, [{compile, "bash build emqx-enterprise-pkg doc"}]}
+        ]}
     ].
 
 %% EE has more files than CE, always test/check with EE options.
 profiles_dev() ->
     Vsn = get_vsn('emqx-enterprise'),
-    [ {check,
-       [ {erl_opts, common_compile_opts(ee, Vsn)}
-       , {project_app_dirs, project_app_dirs(ee)}
-       ]}
-    , {test,
-       [ {deps, test_deps()}
-       , {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()}
-       , {extra_src_dirs, [{"test", [{recursive, true}]}]}
-       , {project_app_dirs, project_app_dirs(ee)}
-       ]}
+    [
+        {check, [
+            {erl_opts, common_compile_opts(ee, Vsn)},
+            {project_app_dirs, project_app_dirs(ee)}
+        ]},
+        {test, [
+            {deps, test_deps()},
+            {erl_opts, common_compile_opts(ee, Vsn) ++ erl_opts_i()},
+            {extra_src_dirs, [{"test", [{recursive, true}]}]},
+            {project_app_dirs, project_app_dirs(ee)}
+        ]}
     ].
 
 %% RelType: cloud (full size) | edge (slim size)
 %% PkgType: bin | pkg
 %% Edition: ce (community) | ee (enterprise)
 relx(Vsn, RelType, PkgType, Edition) ->
-    [ {include_src,false}
-    , {include_erts, true}
-    , {extended_start_script,false}
-    , {generate_start_script,false}
-    , {sys_config,false}
-    , {vm_args,false}
-    , {release, {emqx, Vsn}, relx_apps(RelType, Edition)}
-    , {overlay, relx_overlay(RelType, Edition)}
-    , {overlay_vars, build_info() ++
-                     [ {emqx_description, emqx_description(RelType, Edition)}
-                     | overlay_vars(RelType, PkgType, Edition)
-                     ]}
+    [
+        {include_src, false},
+        {include_erts, true},
+        {extended_start_script, false},
+        {generate_start_script, false},
+        {sys_config, false},
+        {vm_args, false},
+        {release, {emqx, Vsn}, relx_apps(RelType, Edition)},
+        {overlay, relx_overlay(RelType, Edition)},
+        {overlay_vars,
+            build_info() ++
+                [
+                    {emqx_description, emqx_description(RelType, Edition)}
+                    | overlay_vars(RelType, PkgType, Edition)
+                ]}
     ].
 
 %% Make a HOCON compatible format
 build_info() ->
     Os = os_cmd("./scripts/get-distro.sh"),
-    [ {build_info_arch, erlang:system_info(system_architecture)}
-    , {build_info_wordsize, rebar_utils:wordsize()}
-    , {build_info_os, Os}
-    , {build_info_erlang, rebar_utils:otp_release()}
-    , {build_info_elixir, none}
-    , {build_info_relform, relform()}
+    [
+        {build_info_arch, erlang:system_info(system_architecture)},
+        {build_info_wordsize, rebar_utils:wordsize()},
+        {build_info_os, Os},
+        {build_info_erlang, rebar_utils:otp_release()},
+        {build_info_elixir, none},
+        {build_info_relform, relform()}
     ].
 
 relform() ->
@@ -237,111 +258,120 @@ relform() ->
 
 emqx_description(cloud, ee) -> "EMQX Enterprise";
 emqx_description(cloud, ce) -> "EMQX";
-emqx_description(edge, ce)  -> "EMQX Edge".
+emqx_description(edge, ce) -> "EMQX Edge".
 
 overlay_vars(RelType, PkgType, Edition) ->
-    overlay_vars_rel(RelType)
-    ++ overlay_vars_pkg(PkgType)
-    ++ overlay_vars_edition(Edition).
+    overlay_vars_rel(RelType) ++
+        overlay_vars_pkg(PkgType) ++
+        overlay_vars_edition(Edition).
 
 %% vars per release type, cloud or edge
 overlay_vars_rel(RelType) ->
-    VmArgs = case RelType of
-                 cloud -> "vm.args";
-                 edge -> "vm.args.edge"
-             end,
+    VmArgs =
+        case RelType of
+            cloud -> "vm.args";
+            edge -> "vm.args.edge"
+        end,
 
-    [ {vm_args_file, VmArgs}
-    ].
+    [{vm_args_file, VmArgs}].
 
 overlay_vars_edition(ce) ->
-    [ {emqx_schema_mod, emqx_conf_schema}
-    , {is_enterprise, "no"}
-    , {emqx_machine_boot_apps, emqx_machine_boot_app_list(ce)}
+    [
+        {emqx_schema_mod, emqx_conf_schema},
+        {is_enterprise, "no"},
+        {emqx_machine_boot_apps, emqx_machine_boot_app_list(ce)}
     ];
 overlay_vars_edition(ee) ->
-    [ {emqx_schema_mod, emqx_enterprise_conf_schema}
-    , {is_enterprise, "yes"}
-    , {emqx_machine_boot_apps, emqx_machine_boot_app_list(ee)}
+    [
+        {emqx_schema_mod, emqx_enterprise_conf_schema},
+        {is_enterprise, "yes"},
+        {emqx_machine_boot_apps, emqx_machine_boot_app_list(ee)}
     ].
 
 %% vars per packaging type, bin(zip/tar.gz/docker) or pkg(rpm/deb)
 overlay_vars_pkg(bin) ->
-    [ {platform_data_dir, "data"}
-    , {platform_etc_dir, "etc"}
-    , {platform_log_dir, "log"}
-    , {platform_plugins_dir, "plugins"}
-    , {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"}
-    , {emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"}
-    , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}
-    , {runner_log_dir, "$RUNNER_ROOT_DIR/log"}
-    , {runner_user, ""}
-    , {is_elixir, "no"}
+    [
+        {platform_data_dir, "data"},
+        {platform_etc_dir, "etc"},
+        {platform_log_dir, "log"},
+        {platform_plugins_dir, "plugins"},
+        {runner_bin_dir, "$RUNNER_ROOT_DIR/bin"},
+        {emqx_etc_dir, "$RUNNER_ROOT_DIR/etc"},
+        {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"},
+        {runner_log_dir, "$RUNNER_ROOT_DIR/log"},
+        {runner_user, ""},
+        {is_elixir, "no"}
     ];
 overlay_vars_pkg(pkg) ->
-    [ {platform_data_dir, "/var/lib/emqx"}
-    , {platform_etc_dir, "/etc/emqx"}
-    , {platform_log_dir, "/var/log/emqx"}
-    , {platform_plugins_dir, "/var/lib/emqx/plugins"}
-    , {runner_bin_dir, "/usr/bin"}
-    , {emqx_etc_dir, "/etc/emqx"}
-    , {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"}
-    , {runner_log_dir, "/var/log/emqx"}
-    , {runner_user, "emqx"}
-    , {is_elixir, "no"}
+    [
+        {platform_data_dir, "/var/lib/emqx"},
+        {platform_etc_dir, "/etc/emqx"},
+        {platform_log_dir, "/var/log/emqx"},
+        {platform_plugins_dir, "/var/lib/emqx/plugins"},
+        {runner_bin_dir, "/usr/bin"},
+        {emqx_etc_dir, "/etc/emqx"},
+        {runner_lib_dir, "$RUNNER_ROOT_DIR/lib"},
+        {runner_log_dir, "/var/log/emqx"},
+        {runner_user, "emqx"},
+        {is_elixir, "no"}
     ].
 
 relx_apps(ReleaseType, Edition) ->
-    [ kernel
-    , sasl
-    , crypto
-    , public_key
-    , asn1
-    , syntax_tools
-    , ssl
-    , os_mon
-    , inets
-    , compiler
-    , runtime_tools
-    , redbug
-    , {hocon, load}
-    , {emqx, load} % started by emqx_machine
-    , {emqx_conf, load}
-    , emqx_machine
-    , {mnesia, load}
-    , {ekka, load}
-    , {emqx_plugin_libs, load}
-    , {esasl, load}
-    , observer_cli
-    , {system_monitor, load} % started by emqx_machine
-    , emqx_http_lib
-    , emqx_resource
-    , emqx_connector
-    , emqx_authn
-    , emqx_authz
-    , emqx_auto_subscribe
-    , emqx_gateway
-    , emqx_exhook
-    , emqx_bridge
-    , emqx_rule_engine
-    , emqx_modules
-    , emqx_management
-    , emqx_dashboard
-    , emqx_retainer
-    , emqx_statsd
-    , emqx_prometheus
-    , emqx_psk
-    , emqx_slow_subs
-    , emqx_plugins
-    ]
-    ++ [quicer || is_quicer_supported()]
-    ++ [bcrypt || provide_bcrypt_release(ReleaseType)]
-    ++ relx_apps_per_rel(ReleaseType)
-    ++ relx_additional_apps(ReleaseType, Edition).
+    [
+        kernel,
+        sasl,
+        crypto,
+        public_key,
+        asn1,
+        syntax_tools,
+        ssl,
+        os_mon,
+        inets,
+        compiler,
+        runtime_tools,
+        redbug,
+        {hocon, load},
+        % started by emqx_machine
+        {emqx, load},
+        {emqx_conf, load},
+        emqx_machine,
+        {mnesia, load},
+        {ekka, load},
+        {emqx_plugin_libs, load},
+        {esasl, load},
+        observer_cli,
+        % started by emqx_machine
+        {system_monitor, load},
+        emqx_http_lib,
+        emqx_resource,
+        emqx_connector,
+        emqx_authn,
+        emqx_authz,
+        emqx_auto_subscribe,
+        emqx_gateway,
+        emqx_exhook,
+        emqx_bridge,
+        emqx_rule_engine,
+        emqx_modules,
+        emqx_management,
+        emqx_dashboard,
+        emqx_retainer,
+        emqx_statsd,
+        emqx_prometheus,
+        emqx_psk,
+        emqx_slow_subs,
+        emqx_plugins
+    ] ++
+        [quicer || is_quicer_supported()] ++
+        [bcrypt || provide_bcrypt_release(ReleaseType)] ++
+        [jq || provide_jq()] ++
+        relx_apps_per_rel(ReleaseType) ++
+        relx_additional_apps(ReleaseType, Edition).
 
 relx_apps_per_rel(cloud) ->
-    [ xmerl
-    | [{observer, load} || is_app(observer)]
+    [
+        xmerl
+        | [{observer, load} || is_app(observer)]
     ];
 relx_apps_per_rel(edge) ->
     [].
@@ -349,13 +379,13 @@ relx_apps_per_rel(edge) ->
 is_app(Name) ->
     case application:load(Name) of
         ok -> true;
-        {error,{already_loaded, _}} -> true;
+        {error, {already_loaded, _}} -> true;
         _ -> false
     end.
 
 relx_additional_apps(ReleaseType, Edition) ->
-    relx_plugin_apps_per_rel(ReleaseType)
-    ++ relx_apps_per_edition(Edition).
+    relx_plugin_apps_per_rel(ReleaseType) ++
+        relx_apps_per_edition(Edition).
 
 relx_plugin_apps_per_rel(cloud) ->
     [];
@@ -363,101 +393,110 @@ relx_plugin_apps_per_rel(edge) ->
     [].
 
 relx_apps_per_edition(ee) ->
-    [ emqx_license
-    , {emqx_enterprise_conf, load}
+    [
+        emqx_license,
+        {emqx_enterprise_conf, load}
     ];
-
-relx_apps_per_edition(ce) -> [].
+relx_apps_per_edition(ce) ->
+    [].
 
 emqx_machine_boot_apps(ce) ->
-    [ emqx_prometheus
-    , emqx_modules
-    , emqx_dashboard
-    , emqx_connector
-    , emqx_gateway
-    , emqx_statsd
-    , emqx_resource
-    , emqx_rule_engine
-    , emqx_bridge
-    , emqx_plugin_libs
-    , emqx_management
-    , emqx_retainer
-    , emqx_exhook
-    , emqx_authn
-    , emqx_authz
-    , emqx_slow_subs
-    , emqx_auto_subscribe
-    , emqx_plugins
+    [
+        emqx_prometheus,
+        emqx_modules,
+        emqx_dashboard,
+        emqx_connector,
+        emqx_gateway,
+        emqx_statsd,
+        emqx_resource,
+        emqx_rule_engine,
+        emqx_bridge,
+        emqx_plugin_libs,
+        emqx_management,
+        emqx_retainer,
+        emqx_exhook,
+        emqx_authn,
+        emqx_authz,
+        emqx_slow_subs,
+        emqx_auto_subscribe,
+        emqx_plugins
     ];
-
 emqx_machine_boot_apps(ee) ->
     emqx_machine_boot_apps(ce) ++
-    [].
+        [].
 
 emqx_machine_boot_app_list(Edition) ->
     string:join(
-      [atom_to_list(AppName) || AppName <- emqx_machine_boot_apps(Edition)],
-      ", ").
+        [atom_to_list(AppName) || AppName <- emqx_machine_boot_apps(Edition)],
+        ", "
+    ).
 
 relx_overlay(ReleaseType, Edition) ->
-    [ {mkdir, "log/"}
-    , {mkdir, "data/"}
-    , {mkdir, "plugins"}
-    , {mkdir, "data/mnesia"}
-    , {mkdir, "data/configs"}
-    , {mkdir, "data/patches"}
-    , {mkdir, "data/scripts"}
-    , {template, "rel/emqx_vars", "releases/emqx_vars"}
-    , {template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"}
-    , {copy, "bin/emqx", "bin/emqx"}
-    , {copy, "bin/emqx_ctl", "bin/emqx_ctl"}
-    , {copy, "bin/node_dump", "bin/node_dump"}
-    , {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"}
-    , {copy, "bin/emqx", "bin/emqx-{{release_version}}"} %% for relup
-    , {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"} %% for relup
-    , {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"} %% for relup
-    , {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"}
-    , {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"}
-    , {template, "bin/emqx.cmd", "bin/emqx.cmd"}
-    , {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"}
-    , {copy, "bin/nodetool", "bin/nodetool"}
-    , {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
+    [
+        {mkdir, "log/"},
+        {mkdir, "data/"},
+        {mkdir, "plugins"},
+        {mkdir, "data/mnesia"},
+        {mkdir, "data/configs"},
+        {mkdir, "data/patches"},
+        {mkdir, "data/scripts"},
+        {template, "rel/emqx_vars", "releases/emqx_vars"},
+        {template, "rel/BUILD_INFO", "releases/{{release_version}}/BUILD_INFO"},
+        {copy, "bin/emqx", "bin/emqx"},
+        {copy, "bin/emqx_ctl", "bin/emqx_ctl"},
+        {copy, "bin/node_dump", "bin/node_dump"},
+        {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript"},
+        %% for relup
+        {copy, "bin/emqx", "bin/emqx-{{release_version}}"},
+        %% for relup
+        {copy, "bin/emqx_ctl", "bin/emqx_ctl-{{release_version}}"},
+        %% for relup
+        {copy, "bin/install_upgrade.escript", "bin/install_upgrade.escript-{{release_version}}"},
+        {copy, "apps/emqx_gateway/src/lwm2m/lwm2m_xml", "etc/lwm2m_xml"},
+        {copy, "apps/emqx_authz/etc/acl.conf", "etc/acl.conf"},
+        {template, "bin/emqx.cmd", "bin/emqx.cmd"},
+        {template, "bin/emqx_ctl.cmd", "bin/emqx_ctl.cmd"},
+        {copy, "bin/nodetool", "bin/nodetool"},
+        {copy, "bin/nodetool", "bin/nodetool-{{release_version}}"}
     ] ++ etc_overlay(ReleaseType, Edition).
 
 etc_overlay(ReleaseType, Edition) ->
     Templates = emqx_etc_overlay(ReleaseType, Edition),
-    [ {mkdir, "etc/"}
-    , {copy, "{{base_dir}}/lib/emqx/etc/certs","etc/"}
+    [
+        {mkdir, "etc/"},
+        {copy, "{{base_dir}}/lib/emqx/etc/certs", "etc/"}
     ] ++
-    lists:map(
-      fun({From, To}) -> {template, From, To};
-         (FromTo)     -> {template, FromTo, FromTo}
-      end, Templates).
+        lists:map(
+            fun
+                ({From, To}) -> {template, From, To};
+                (FromTo) -> {template, FromTo, FromTo}
+            end,
+            Templates
+        ).
 
 emqx_etc_overlay(ReleaseType, Edition) ->
-    emqx_etc_overlay_per_rel(ReleaseType)
-    ++ emqx_etc_overlay_per_edition(Edition)
-    ++ emqx_etc_overlay_common().
+    emqx_etc_overlay_per_rel(ReleaseType) ++
+        emqx_etc_overlay_per_edition(Edition) ++
+        emqx_etc_overlay_common().
 
 emqx_etc_overlay_per_rel(cloud) ->
-    [ {"{{base_dir}}/lib/emqx/etc/emqx_cloud/vm.args","etc/vm.args"}
-    ];
+    [{"{{base_dir}}/lib/emqx/etc/emqx_cloud/vm.args", "etc/vm.args"}];
 emqx_etc_overlay_per_rel(edge) ->
-    [ {"{{base_dir}}/lib/emqx/etc/emqx_edge/vm.args","etc/vm.args"}
-    ].
+    [{"{{base_dir}}/lib/emqx/etc/emqx_edge/vm.args", "etc/vm.args"}].
 
 emqx_etc_overlay_common() ->
-    [ {"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}
-    ].
+    [{"{{base_dir}}/lib/emqx/etc/ssl_dist.conf", "etc/ssl_dist.conf"}].
 
 emqx_etc_overlay_per_edition(ce) ->
-    [ {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}
-    , {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
+    [
+        {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"},
+        {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
     ];
 emqx_etc_overlay_per_edition(ee) ->
-    [ {"{{base_dir}}/lib/emqx_conf/etc/emqx_enterprise.conf.all", "etc/emqx_enterprise.conf"}
-    , {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"}
-    , {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
+    [
+        {"{{base_dir}}/lib/emqx_conf/etc/emqx_enterprise.conf.all", "etc/emqx_enterprise.conf"},
+        {"{{base_dir}}/lib/emqx_conf/etc/emqx.conf.all", "etc/emqx.conf"},
+        {"{{base_dir}}/lib/emqx_dashboard/etc/i18n.conf.all", "etc/i18n.conf"}
     ].
 
 get_vsn(Profile) ->
@@ -468,10 +507,11 @@ get_vsn(Profile) ->
 
 os_cmd(Cmd) ->
     Output = os:cmd("bash " ++ Cmd),
-    re:replace(Output, "\n", "", [{return ,list}]).
+    re:replace(Output, "\n", "", [{return, list}]).
 
 maybe_dump(Config) ->
-    is_debug() andalso file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]),
+    is_debug() andalso
+        file:write_file("rebar.config.rendered", [io_lib:format("~p.\n", [I]) || I <- Config]),
     Config.
 
 is_debug() -> is_debug("DEBUG") orelse is_debug("DIAGNOSTIC").
@@ -486,54 +526,67 @@ is_debug(VarName) ->
 provide_bcrypt_dep() ->
     not is_win32().
 
+provide_jq() ->
+    not is_win32().
+
 provide_bcrypt_release(ReleaseType) ->
     provide_bcrypt_dep() andalso ReleaseType =:= cloud.
 
 erl_opts_i() ->
     [{i, "apps"}] ++
-    [{i, Dir}  || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++
-    [{i, Dir}  || Dir <- filelib:wildcard(filename:join(["lib-ee", "*", "include"]))].
+        [{i, Dir} || Dir <- filelib:wildcard(filename:join(["apps", "*", "include"]))] ++
+        [{i, Dir} || Dir <- filelib:wildcard(filename:join(["lib-ee", "*", "include"]))].
 
 dialyzer(Config) ->
     {dialyzer, OldDialyzerConfig} = lists:keyfind(dialyzer, 1, Config),
 
-    AppsToAnalyse = case os:getenv("DIALYZER_ANALYSE_APP") of
-        false ->
-            [];
-        Value ->
-            [ list_to_atom(App) || App <- string:tokens(Value, ",")]
-    end,
+    AppsToAnalyse =
+        case os:getenv("DIALYZER_ANALYSE_APP") of
+            false ->
+                [];
+            Value ->
+                [list_to_atom(App) || App <- string:tokens(Value, ",")]
+        end,
 
     AppNames = app_names(),
 
-    KnownApps = [Name ||  Name <- AppsToAnalyse, lists:member(Name, AppNames)],
+    KnownApps = [Name || Name <- AppsToAnalyse, lists:member(Name, AppNames)],
 
     AppsToExclude = AppNames -- KnownApps,
 
     case length(AppsToAnalyse) > 0 of
         true ->
-            lists:keystore(dialyzer, 1, Config, {dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]});
+            lists:keystore(
+                dialyzer,
+                1,
+                Config,
+                {dialyzer, OldDialyzerConfig ++ [{exclude_apps, AppsToExclude}]}
+            );
         false ->
             Config
     end.
 
 coveralls() ->
     case {os:getenv("GITHUB_ACTIONS"), os:getenv("GITHUB_TOKEN")} of
-      {"true", Token} when is_list(Token) ->
-        Cfgs = [{coveralls_repo_token, Token},
+        {"true", Token} when is_list(Token) ->
+            Cfgs = [
+                {coveralls_repo_token, Token},
                 {coveralls_service_job_id, os:getenv("GITHUB_RUN_ID")},
                 {coveralls_commit_sha, os:getenv("GITHUB_SHA")},
                 {coveralls_coverdata, "_build/test/cover/*.coverdata"},
-                {coveralls_service_name, "github"}],
-        case os:getenv("GITHUB_EVENT_NAME") =:= "pull_request"
-            andalso string:tokens(os:getenv("GITHUB_REF"), "/") of
-            [_, "pull", PRNO, _] ->
-                [{coveralls_service_pull_request, PRNO} | Cfgs];
-            _ ->
-                Cfgs
-        end;
-      _ ->
-        []
+                {coveralls_service_name, "github"}
+            ],
+            case
+                os:getenv("GITHUB_EVENT_NAME") =:= "pull_request" andalso
+                    string:tokens(os:getenv("GITHUB_REF"), "/")
+            of
+                [_, "pull", PRNO, _] ->
+                    [{coveralls_service_pull_request, PRNO} | Cfgs];
+                _ ->
+                    Cfgs
+            end;
+        _ ->
+            []
     end.
 
 app_names() -> list_dir("apps") ++ list_dir("lib-ee").

scripts/buildx.sh

@@ -9,7 +9,7 @@
 
 ## example:
 ## ./scripts/buildx.sh --profile emqx --pkgtype tgz --arch arm64 \
-##     --builder ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-debian10
+##     --builder ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-debian10
 
 set -euo pipefail
 
@@ -24,7 +24,7 @@ help() {
     echo "--arch amd64|arm64:        Target arch to build the EMQX package for"
     echo "--src_dir <SRC_DIR>:       EMQX source ode in this dir, default to PWD"
     echo "--builder <BUILDER>:       Builder image to pull"
-    echo "                           E.g. ghcr.io/emqx/emqx-builder/5.0-10:1.13.3-24.2.1-1-debian10"
+    echo "                           E.g. ghcr.io/emqx/emqx-builder/5.0-14:1.13.3-24.2.1-1-debian10"
 }
 
 while [ "$#" -gt 0 ]; do
@@ -120,13 +120,6 @@ fi
 HOST_SYSTEM="$(./scripts/get-distro.sh)"
 BUILDER_SYSTEM="$(echo "$BUILDER" | awk -F'-' '{print $NF}')"
 
-# quick workaround before builder image is renamed
-if [ "$BUILDER_SYSTEM" = 'centos7' ]; then
-    BUILDER_SYSTEM='el7'
-elif [ "$BUILDER_SYSTEM" = 'rockylinux8' ]; then
-    BUILDER_SYSTEM='el8'
-fi
-
 CMD_RUN="make ${MAKE_TARGET} && ./scripts/pkg-tests.sh ${MAKE_TARGET}"
 
 IS_NATIVE_SYSTEM='no'