Merge branch 'master' into support-listen-ipv6

.github/workflows/run_fvt_tests.yaml

@@ -157,6 +157,10 @@ jobs:
       if: matrix.discovery == 'k8s'
       run: |
         helm install emqx \
+            --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="k8s" \
+            --set emqxConfig.EMQX_CLUSTER__K8S__APISERVER="https://kubernetes.default.svc:443" \
+            --set emqxConfig.EMQX_CLUSTER__K8S__SERVICE_NAME="emqx-headless" \
+            --set emqxConfig.EMQX_CLUSTER__K8S__NAMESPACE="default" \
             --set image.repository=$TARGET \
             --set image.pullPolicy=Never \
             --set emqxAclConfig="" \
@@ -173,8 +177,8 @@ jobs:
       run: |
         helm install emqx \
             --set emqxConfig.EMQX_CLUSTER__DISCOVERY_STRATEGY="dns" \
-            --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \
             --set emqxConfig.EMQX_CLUSTER__DNS__RECORD_TYPE="srv" \
+            --set emqxConfig.EMQX_CLUSTER__DNS__NAME="emqx-headless.default.svc.cluster.local" \
             --set image.repository=$TARGET \
             --set image.pullPolicy=Never \
             --set emqxAclConfig="" \

CHANGES-5.0.md

@@ -18,6 +18,10 @@
 * Ensuring that exhook dispatches the client events are sequential. [#8530](https://github.com/emqx/emqx/pull/8530)
 * Avoid using RocksDB backend for persistent sessions when such backend is unavailable. [#8528](https://github.com/emqx/emqx/pull/8528)
 * Support listen on an IPv6 address. [#8547](https://github.com/emqx/emqx/pull/8547)
+* GET '/rules' support for pagination and fuzzy search. [#8472](https://github.com/emqx/emqx/pull/8472)
+  **‼️ Note** : The previous API only returns array: `[RuleObj1,RuleObj2]`, after updating, it will become
+  `{"data": [RuleObj1,RuleObj2], "meta":{"count":2, "limit":100, "page":1}`,
+  which will carry the paging meta information.
 
 ## Enhancements
 

Makefile

@@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = ghcr.io/emqx/emqx-builder/5.0-17:1.13.4-24.2.1-1-d
 export EMQX_DEFAULT_RUNNER = debian:11-slim
 export OTP_VSN ?= $(shell $(CURDIR)/scripts/get-otp-vsn.sh)
 export ELIXIR_VSN ?= $(shell $(CURDIR)/scripts/get-elixir-vsn.sh)
-export EMQX_DASHBOARD_VERSION ?= v1.0.3
+export EMQX_DASHBOARD_VERSION ?= v1.0.5-beta.1
 export EMQX_REL_FORM ?= tgz
 export QUICER_DOWNLOAD_FROM_RELEASE = 1
 ifeq ($(OS),Windows_NT)

apps/emqx_rule_engine/i18n/emqx_rule_engine_api.conf

@@ -10,6 +10,46 @@ emqx_rule_engine_api {
                            zh: "列出所有规则"
                           }
                   }
+   api1_enable {
+                   desc {
+                         en: "Filter enable/disable rules"
+                         zh: "根据规则是否开启条件过滤"
+                        }
+                  }
+
+   api1_from {
+                   desc {
+                         en: "Filter rules by from(topic), exact match"
+                         zh: "根据规则来源 Topic 过滤, 需要完全匹配"
+                        }
+                  }
+
+   api1_like_id {
+                   desc {
+                         en: "Filter rules by id, Substring matching"
+                         zh: "根据规则 id 过滤, 使用子串模糊匹配"
+                        }
+                  }
+
+   api1_like_from {
+                   desc {
+                         en: "Filter rules by from(topic), Substring matching"
+                         zh: "根据规则来源 Topic 过滤, 使用子串模糊匹配"
+                        }
+                  }
+
+   api1_like_description {
+                   desc {
+                         en: "Filter rules by description, Substring matching"
+                         zh: "根据规则描述过滤, 使用子串模糊匹配"
+                        }
+                  }
+   api1_match_from {
+                   desc {
+                         en: "Filter rules by from(topic), Mqtt topic matching"
+                         zh: "根据规则来源 Topic 过滤, 使用 MQTT Topic 匹配"
+                        }
+                  }
 
     api2 {
                    desc {

apps/emqx_rule_engine/src/emqx_rule_engine_api.erl

@@ -33,6 +33,9 @@
 %% API callbacks
 -export(['/rule_events'/2, '/rule_test'/2, '/rules'/2, '/rules/:id'/2, '/rules/:id/reset_metrics'/2]).
 
+%% query callback
+-export([query/4]).
+
 -define(ERR_NO_RULE(ID), list_to_binary(io_lib:format("Rule ~ts Not Found", [(ID)]))).
 -define(ERR_BADARGS(REASON), begin
     R0 = err_msg(REASON),
@@ -109,6 +112,15 @@ end).
     }
 ).
 
+-define(RULE_QS_SCHEMA, [
+    {<<"enable">>, atom},
+    {<<"from">>, binary},
+    {<<"like_id">>, binary},
+    {<<"like_from">>, binary},
+    {<<"match_from">>, binary},
+    {<<"like_description">>, binary}
+]).
+
 namespace() -> "rule".
 
 api_spec() ->
@@ -134,9 +146,31 @@ schema("/rules") ->
         get => #{
             tags => [<<"rules">>],
             description => ?DESC("api1"),
+            parameters => [
+                {enable,
+                    mk(boolean(), #{desc => ?DESC("api1_enable"), in => query, required => false})},
+                {from, mk(binary(), #{desc => ?DESC("api1_from"), in => query, required => false})},
+                {like_id,
+                    mk(binary(), #{desc => ?DESC("api1_like_id"), in => query, required => false})},
+                {like_from,
+                    mk(binary(), #{desc => ?DESC("api1_like_from"), in => query, required => false})},
+                {like_description,
+                    mk(binary(), #{
+                        desc => ?DESC("api1_like_description"), in => query, required => false
+                    })},
+                {match_from,
+                    mk(binary(), #{desc => ?DESC("api1_match_from"), in => query, required => false})},
+                ref(emqx_dashboard_swagger, page),
+                ref(emqx_dashboard_swagger, limit)
+            ],
             summary => <<"List Rules">>,
             responses => #{
-                200 => mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})
+                200 =>
+                    [
+                        {data, mk(array(rule_info_schema()), #{desc => ?DESC("desc9")})},
+                        {meta, mk(ref(emqx_dashboard_swagger, meta), #{})}
+                    ],
+                400 => error_schema('BAD_REQUEST', "Invalid Parameters")
             }
         },
         post => #{
@@ -236,9 +270,21 @@ param_path_id() ->
 '/rule_events'(get, _Params) ->
     {200, emqx_rule_events:event_info()}.
 
-'/rules'(get, _Params) ->
-    Records = emqx_rule_engine:get_rules_ordered_by_ts(),
-    {200, format_rule_resp(Records)};
+'/rules'(get, #{query_string := QueryString}) ->
+    case
+        emqx_mgmt_api:node_query(
+            node(),
+            QueryString,
+            ?RULE_TAB,
+            ?RULE_QS_SCHEMA,
+            {?MODULE, query}
+        )
+    of
+        {error, page_limit_invalid} ->
+            {400, #{code => 'BAD_REQUEST', message => <<"page_limit_invalid">>}};
+        Result ->
+            {200, Result}
+    end;
 '/rules'(post, #{body := Params0}) ->
     case maps:get(<<"id">>, Params0, list_to_binary(emqx_misc:gen_id(8))) of
         <<>> ->
@@ -335,6 +381,8 @@ err_msg(Msg) -> emqx_misc:readable_error_msg(Msg).
 
 format_rule_resp(Rules) when is_list(Rules) ->
     [format_rule_resp(R) || R <- Rules];
+format_rule_resp({Id, Rule}) ->
+    format_rule_resp(Rule#{id => Id});
 format_rule_resp(#{
     id := Id,
     name := Name,
@@ -503,3 +551,51 @@ filter_out_request_body(Conf) ->
         <<"node">>
     ],
     maps:without(ExtraConfs, Conf).
+
+query(Tab, {Qs, Fuzzy}, Start, Limit) ->
+    Ms = qs2ms(),
+    FuzzyFun = fuzzy_match_fun(Qs, Ms, Fuzzy),
+    emqx_mgmt_api:select_table_with_count(
+        Tab, {Ms, FuzzyFun}, Start, Limit, fun format_rule_resp/1
+    ).
+
+%% rule is not a record, so everything is fuzzy filter.
+qs2ms() ->
+    [{'_', [], ['$_']}].
+
+fuzzy_match_fun(Qs, Ms, Fuzzy) ->
+    MsC = ets:match_spec_compile(Ms),
+    fun(Rows) ->
+        Ls = ets:match_spec_run(Rows, MsC),
+        lists:filter(
+            fun(E) ->
+                run_qs_match(E, Qs) andalso
+                    run_fuzzy_match(E, Fuzzy)
+            end,
+            Ls
+        )
+    end.
+
+run_qs_match(_, []) ->
+    true;
+run_qs_match(E = {_Id, #{enable := Enable}}, [{enable, '=:=', Pattern} | Qs]) ->
+    Enable =:= Pattern andalso run_qs_match(E, Qs);
+run_qs_match(E = {_Id, #{from := From}}, [{from, '=:=', Pattern} | Qs]) ->
+    lists:member(Pattern, From) andalso run_qs_match(E, Qs);
+run_qs_match(E, [_ | Qs]) ->
+    run_qs_match(E, Qs).
+
+run_fuzzy_match(_, []) ->
+    true;
+run_fuzzy_match(E = {Id, _}, [{id, like, Pattern} | Fuzzy]) ->
+    binary:match(Id, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = {_Id, #{description := Desc}}, [{description, like, Pattern} | Fuzzy]) ->
+    binary:match(Desc, Pattern) /= nomatch andalso run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy]) ->
+    lists:any(fun(For) -> emqx_topic:match(For, Pattern) end, Topics) andalso
+        run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) ->
+    lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) andalso
+        run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E, [_ | Fuzzy]) ->
+    run_fuzzy_match(E, Fuzzy).

apps/emqx_rule_engine/test/emqx_rule_engine_api_SUITE.erl

@@ -45,7 +45,7 @@ t_crud_rule_api(_Config) ->
     ),
 
     ?assertEqual(RuleID, maps:get(id, Rule)),
-    {200, Rules} = emqx_rule_engine_api:'/rules'(get, #{}),
+    {200, #{data := Rules}} = emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}),
     ct:pal("RList : ~p", [Rules]),
     ?assert(length(Rules) > 0),
 
@@ -91,6 +91,81 @@ t_crud_rule_api(_Config) ->
     ),
     ok.
 
+t_list_rule_api(_Config) ->
+    AddIds =
+        lists:map(
+            fun(Seq0) ->
+                Seq = integer_to_binary(Seq0),
+                Params = #{
+                    <<"description">> => <<"A simple rule">>,
+                    <<"enable">> => true,
+                    <<"actions">> => [#{<<"function">> => <<"console">>}],
+                    <<"sql">> => <<"SELECT * from \"t/1\"">>,
+                    <<"name">> => <<"test_rule", Seq/binary>>
+                },
+                {201, #{id := Id}} = emqx_rule_engine_api:'/rules'(post, #{body => Params}),
+                Id
+            end,
+            lists:seq(1, 20)
+        ),
+
+    {200, #{data := Rules, meta := #{count := Count}}} =
+        emqx_rule_engine_api:'/rules'(get, #{query_string => #{}}),
+    ?assertEqual(20, length(AddIds)),
+    ?assertEqual(20, length(Rules)),
+    ?assertEqual(20, Count),
+
+    [RuleID | _] = AddIds,
+    UpdateParams = #{
+        <<"description">> => <<"中文的描述也能搜索"/utf8>>,
+        <<"enable">> => false,
+        <<"actions">> => [#{<<"function">> => <<"console">>}],
+        <<"sql">> => <<"SELECT * from \"t/1/+\"">>,
+        <<"name">> => <<"test_rule_update1">>
+    },
+    {200, _Rule2} = emqx_rule_engine_api:'/rules/:id'(put, #{
+        bindings => #{id => RuleID},
+        body => UpdateParams
+    }),
+    QueryStr1 = #{query_string => #{<<"enable">> => false}},
+    {200, Result1 = #{meta := #{count := Count1}}} = emqx_rule_engine_api:'/rules'(get, QueryStr1),
+    ?assertEqual(1, Count1),
+
+    QueryStr2 = #{query_string => #{<<"like_description">> => <<"也能"/utf8>>}},
+    {200, Result2} = emqx_rule_engine_api:'/rules'(get, QueryStr2),
+    ?assertEqual(Result1, Result2),
+
+    QueryStr3 = #{query_string => #{<<"from">> => <<"t/1">>}},
+    {200, #{meta := #{count := Count3}}} = emqx_rule_engine_api:'/rules'(get, QueryStr3),
+    ?assertEqual(19, Count3),
+
+    QueryStr4 = #{query_string => #{<<"like_from">> => <<"t/1/+">>}},
+    {200, Result4} = emqx_rule_engine_api:'/rules'(get, QueryStr4),
+    ?assertEqual(Result1, Result4),
+
+    QueryStr5 = #{query_string => #{<<"match_from">> => <<"t/+/+">>}},
+    {200, Result5} = emqx_rule_engine_api:'/rules'(get, QueryStr5),
+    ?assertEqual(Result1, Result5),
+
+    QueryStr6 = #{query_string => #{<<"like_id">> => RuleID}},
+    {200, Result6} = emqx_rule_engine_api:'/rules'(get, QueryStr6),
+    ?assertEqual(Result1, Result6),
+
+    %% clean up
+    lists:foreach(
+        fun(Id) ->
+            ?assertMatch(
+                {204},
+                emqx_rule_engine_api:'/rules/:id'(
+                    delete,
+                    #{bindings => #{id => Id}}
+                )
+            )
+        end,
+        AddIds
+    ),
+    ok.
+
 test_rule_params() ->
     #{
         body => #{

deploy/charts/emqx/Chart.yaml

@@ -14,7 +14,7 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 4.4.1
+version: 5
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.

deploy/charts/emqx/values.yaml

@@ -91,13 +91,13 @@ initContainers: {}
 
 ## EMQX configuration item, see the documentation (https://hub.docker.com/r/emqx/emqx)
 emqxConfig:
-  EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s"
-  # EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns"
-  # EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
-  # EMQX_CLUSTER__DNS__RECORD_TYPE: "srv"
-  EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
-  EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless"
-  EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}"
+  EMQX_CLUSTER__DISCOVERY_STRATEGY: "dns"
+  EMQX_CLUSTER__DNS__NAME: "{{ .Release.Name }}-headless.{{ .Release.Namespace }}.svc.cluster.local"
+  EMQX_CLUSTER__DNS__RECORD_TYPE: "srv"
+  # EMQX_CLUSTER__DISCOVERY_STRATEGY: "k8s"
+  # EMQX_CLUSTER__K8S__APISERVER: "https://kubernetes.default.svc:443"
+  # EMQX_CLUSTER__K8S__SERVICE_NAME: "{{ .Release.Name }}-headless"
+  # EMQX_CLUSTER__K8S__NAMESPACE: "{{ .Release.Namespace }}"
   ## The address type is used to extract host from k8s service.
   ## Value: ip | dns | hostname
   ## Note：Hostname is only supported after v4.0-rc.2

deploy/docker/docker-entrypoint.sh

@@ -16,22 +16,31 @@ shopt -s nullglob
 
 LOCAL_IP=$(hostname -i | grep -oE '((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])\.){3}(25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])' | head -n 1)
 
-if [[ -z "$EMQX_NODE_NAME" ]]; then
-    EMQX_NAME="${EMQX_NAME:-emqx}"
-    if [[ -z "$EMQX_HOST" ]]; then
-        if [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
+export EMQX_NAME="${EMQX_NAME:-emqx}"
+
+if [[ -z "$EMQX_HOST" ]]; then
+    if [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "dns" ]] && \
+        [[ "$EMQX_CLUSTER__DNS__RECORD_TYPE" == "srv" ]] && \
+        grep -q "$(hostname).$EMQX_CLUSTER__DNS__NAME" /etc/hosts; then
+            EMQX_HOST="$(hostname).$EMQX_CLUSTER__DNS__NAME"
+    elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \
+        [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == "dns" ]] && \
+        [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
             EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-"pod.cluster.local"}
             EMQX_HOST="${LOCAL_IP//./-}.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX"
-        elif [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
+    elif [[ "$EMQX_CLUSTER__DISCOVERY_STRATEGY" == "k8s" ]] && \
+        [[ "$EMQX_CLUSTER__K8S__ADDRESS_TYPE" == 'hostname' ]] && \
+        [[ -n "$EMQX_CLUSTER__K8S__NAMESPACE" ]]; then
             EMQX_CLUSTER__K8S__SUFFIX=${EMQX_CLUSTER__K8S__SUFFIX:-'svc.cluster.local'}
             EMQX_HOST=$(grep -h "^$LOCAL_IP" /etc/hosts | grep -o "$(hostname).*.$EMQX_CLUSTER__K8S__NAMESPACE.$EMQX_CLUSTER__K8S__SUFFIX")
-        else
-            EMQX_HOST="$LOCAL_IP"
-        fi
+    else
+        EMQX_HOST="$LOCAL_IP"
     fi
+    export EMQX_HOST
+fi
+
+if [[ -z "$EMQX_NODE_NAME" ]]; then
     export EMQX_NODE_NAME="$EMQX_NAME@$EMQX_HOST"
-    unset EMQX_NAME
-    unset EMQX_HOST
 fi
 
 # The default rpc port discovery 'stateless' is mostly for clusters