Merge pull request #13505 from thalesmg/20240722-m-rule-conn-deps-part-2

feat(rule engine api): add filters options for action and source ids

apps/emqx_rule_engine/src/emqx_rule_engine_api.erl

@@ -131,6 +131,8 @@ end).
     {<<"like_id">>, binary},
     {<<"like_from">>, binary},
     {<<"match_from">>, binary},
+    {<<"action">>, binary},
+    {<<"source">>, binary},
     {<<"like_description">>, binary}
 ]).
 
@@ -194,6 +196,10 @@ schema("/rules") ->
                     })},
                 {match_from,
                     mk(binary(), #{desc => ?DESC("api1_match_from"), in => query, required => false})},
+                {action,
+                    mk(hoconsc:array(binary()), #{in => query, desc => ?DESC("api1_qs_action")})},
+                {source,
+                    mk(hoconsc:array(binary()), #{in => query, desc => ?DESC("api1_qs_source")})},
                 ref(emqx_dashboard_swagger, page),
                 ref(emqx_dashboard_swagger, limit)
             ],
@@ -748,7 +754,8 @@ filter_out_request_body(Conf) ->
     maps:without(ExtraConfs, Conf).
 
 -spec qs2ms(atom(), {list(), list()}) -> emqx_mgmt_api:match_spec_and_filter().
-qs2ms(_Tab, {Qs, Fuzzy}) ->
+qs2ms(_Tab, {Qs0, Fuzzy0}) ->
+    {Qs, Fuzzy} = adapt_custom_filters(Qs0, Fuzzy0),
     case lists:keytake(from, 1, Qs) of
         false ->
             #{match_spec => generate_match_spec(Qs), fuzzy_fun => fuzzy_match_fun(Fuzzy)};
@@ -759,6 +766,38 @@ qs2ms(_Tab, {Qs, Fuzzy}) ->
             }
     end.
 
+%% Some filters are run as fuzzy filters because they cannot be expressed as simple ETS
+%% match specs.
+-spec adapt_custom_filters(Qs, Fuzzy) -> {Qs, Fuzzy}.
+adapt_custom_filters(Qs, Fuzzy) ->
+    lists:foldl(
+        fun
+            ({action, '=:=', X}, {QsAcc, FuzzyAcc}) ->
+                ActionIds = wrap(X),
+                Parsed = lists:map(fun emqx_rule_actions:parse_action/1, ActionIds),
+                {QsAcc, [{action, in, Parsed} | FuzzyAcc]};
+            ({source, '=:=', X}, {QsAcc, FuzzyAcc}) ->
+                SourceIds = wrap(X),
+                Parsed = lists:flatmap(
+                    fun(SourceId) ->
+                        [
+                            emqx_bridge_resource:bridge_hookpoint(SourceId),
+                            emqx_bridge_v2:source_hookpoint(SourceId)
+                        ]
+                    end,
+                    SourceIds
+                ),
+                {QsAcc, [{source, in, Parsed} | FuzzyAcc]};
+            (Clause, {QsAcc, FuzzyAcc}) ->
+                {[Clause | QsAcc], FuzzyAcc}
+        end,
+        {[], Fuzzy},
+        Qs
+    ).
+
+wrap(Xs) when is_list(Xs) -> Xs;
+wrap(X) -> [X].
+
 generate_match_spec(Qs) ->
     {MtchHead, Conds} = generate_match_spec(Qs, 2, {#{}, []}),
     [{{'_', MtchHead}, Conds, ['$_']}].
@@ -796,6 +835,12 @@ run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, match, Pattern} | Fuzzy])
 run_fuzzy_match(E = {_Id, #{from := Topics}}, [{from, like, Pattern} | Fuzzy]) ->
     lists:any(fun(For) -> binary:match(For, Pattern) /= nomatch end, Topics) andalso
         run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = {_Id, #{actions := Actions}}, [{action, in, ActionIds} | Fuzzy]) ->
+    lists:any(fun(AId) -> lists:member(AId, Actions) end, ActionIds) andalso
+        run_fuzzy_match(E, Fuzzy);
+run_fuzzy_match(E = {_Id, #{from := Froms}}, [{source, in, SourceIds} | Fuzzy]) ->
+    lists:any(fun(SId) -> lists:member(SId, Froms) end, SourceIds) andalso
+        run_fuzzy_match(E, Fuzzy);
 run_fuzzy_match(E, [_ | Fuzzy]) ->
     run_fuzzy_match(E, Fuzzy).
 

apps/emqx_rule_engine/test/emqx_rule_engine_api_2_SUITE.erl

@@ -33,7 +33,6 @@ init_per_suite(Config) ->
         app_specs(),
         #{work_dir => emqx_cth_suite:work_dir(Config)}
     ),
-    emqx_common_test_http:create_default_app(),
     [{apps, Apps} | Config].
 
 end_per_suite(Config) ->
@@ -46,7 +45,7 @@ app_specs() ->
         emqx_conf,
         emqx_rule_engine,
         emqx_management,
-        {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"}
+        emqx_mgmt_api_test_util:emqx_dashboard()
     ].
 
 %%------------------------------------------------------------------------------
@@ -64,8 +63,12 @@ request(Method, Path, Params) ->
     request(Method, Path, Params, Opts).
 
 request(Method, Path, Params, Opts) ->
+    request(Method, Path, Params, _QueryParams = [], Opts).
+
+request(Method, Path, Params, QueryParams0, Opts) when is_list(QueryParams0) ->
     AuthHeader = emqx_mgmt_api_test_util:auth_header_(),
-    case emqx_mgmt_api_test_util:request_api(Method, Path, "", AuthHeader, Params, Opts) of
+    QueryParams = uri_string:compose_query(QueryParams0, [{encoding, utf8}]),
+    case emqx_mgmt_api_test_util:request_api(Method, Path, QueryParams, AuthHeader, Params, Opts) of
         {ok, {Status, Headers, Body0}} ->
             Body = maybe_json_decode(Body0),
             {ok, {Status, Headers, Body}};
@@ -93,6 +96,45 @@ sql_test_api(Params) ->
     ct:pal("sql test (http) result:\n  ~p", [Res]),
     Res.
 
+list_rules(QueryParams) when is_list(QueryParams) ->
+    Method = get,
+    Path = emqx_mgmt_api_test_util:api_path(["rules"]),
+    Opts = #{return_all => true},
+    Res = request(Method, Path, _Body = [], QueryParams, Opts),
+    emqx_mgmt_api_test_util:simplify_result(Res).
+
+list_rules_just_ids(QueryParams) when is_list(QueryParams) ->
+    case list_rules(QueryParams) of
+        {200, #{<<"data">> := Results0}} ->
+            Results = lists:sort([Id || #{<<"id">> := Id} <- Results0]),
+            {200, Results};
+        Res ->
+            Res
+    end.
+
+create_rule() ->
+    create_rule(_Overrides = #{}).
+
+create_rule(Overrides) ->
+    Params0 = #{
+        <<"enable">> => true,
+        <<"sql">> => <<"select true from t">>
+    },
+    Params = emqx_utils_maps:deep_merge(Params0, Overrides),
+    Method = post,
+    Path = emqx_mgmt_api_test_util:api_path(["rules"]),
+    Res = request(Method, Path, Params),
+    emqx_mgmt_api_test_util:simplify_result(Res).
+
+sources_sql(Sources) ->
+    Froms = iolist_to_binary(lists:join(<<", ">>, lists:map(fun source_from/1, Sources))),
+    <<"select * from ", Froms/binary>>.
+
+source_from({v1, Id}) ->
+    <<"\"$bridges/", Id/binary, "\" ">>;
+source_from({v2, Id}) ->
+    <<"\"$sources/", Id/binary, "\" ">>.
+
 %%------------------------------------------------------------------------------
 %% Test cases
 %%------------------------------------------------------------------------------
@@ -450,3 +492,64 @@ do_t_rule_test_smoke(#{input := Input, expected := #{code := ExpectedCode}} = Ca
                 resp_body => Body
             }}
     end.
+
+%% Tests filtering the rule list by used actions and/or sources.
+t_filter_by_source_and_action(_Config) ->
+    ?assertMatch(
+        {200, #{<<"data">> := []}},
+        list_rules([])
+    ),
+
+    ActionId1 = <<"mqtt:a1">>,
+    ActionId2 = <<"mqtt:a2">>,
+    SourceId1 = <<"mqtt:s1">>,
+    SourceId2 = <<"mqtt:s2">>,
+    {201, #{<<"id">> := Id1}} = create_rule(#{<<"actions">> => [ActionId1]}),
+    {201, #{<<"id">> := Id2}} = create_rule(#{<<"actions">> => [ActionId2]}),
+    {201, #{<<"id">> := Id3}} = create_rule(#{<<"actions">> => [ActionId2, ActionId1]}),
+    {201, #{<<"id">> := Id4}} = create_rule(#{<<"sql">> => sources_sql([{v1, SourceId1}])}),
+    {201, #{<<"id">> := Id5}} = create_rule(#{<<"sql">> => sources_sql([{v2, SourceId2}])}),
+    {201, #{<<"id">> := Id6}} = create_rule(#{
+        <<"sql">> => sources_sql([{v2, SourceId1}, {v2, SourceId1}])
+    }),
+    {201, #{<<"id">> := Id7}} = create_rule(#{
+        <<"sql">> => sources_sql([{v2, SourceId1}]),
+        <<"actions">> => [ActionId1]
+    }),
+
+    ?assertMatch(
+        {200, [_, _, _, _, _, _, _]},
+        list_rules_just_ids([])
+    ),
+
+    ?assertEqual(
+        {200, lists:sort([Id1, Id3, Id7])},
+        list_rules_just_ids([{<<"action">>, ActionId1}])
+    ),
+
+    ?assertEqual(
+        {200, lists:sort([Id1, Id2, Id3, Id7])},
+        list_rules_just_ids([{<<"action">>, ActionId1}, {<<"action">>, ActionId2}])
+    ),
+
+    ?assertEqual(
+        {200, lists:sort([Id4, Id6, Id7])},
+        list_rules_just_ids([{<<"source">>, SourceId1}])
+    ),
+
+    ?assertEqual(
+        {200, lists:sort([Id4, Id5, Id6, Id7])},
+        list_rules_just_ids([{<<"source">>, SourceId1}, {<<"source">>, SourceId2}])
+    ),
+
+    %% When mixing source and action id filters, we use AND.
+    ?assertEqual(
+        {200, lists:sort([])},
+        list_rules_just_ids([{<<"source">>, SourceId2}, {<<"action">>, ActionId2}])
+    ),
+    ?assertEqual(
+        {200, lists:sort([Id7])},
+        list_rules_just_ids([{<<"source">>, SourceId1}, {<<"action">>, ActionId1}])
+    ),
+
+    ok.

changes/ce/feat-13505.en.md

@@ -0,0 +1 @@
+Now, it's possible to filter rules in the HTTP API by the IDs of used data integration actions/sources.

rel/i18n/emqx_rule_engine_api.hocon

@@ -96,4 +96,10 @@ api11.desc:
 api11.label:
 """Apply Rule"""
 
+api1_qs_action.desc:
+"""Filters rules that contain any of the given action id(s).  When used in conjunction with source id filtering, the rules must contain sources *and* actions that match some of the criteria."""
+
+api1_qs_source.desc:
+"""Filters rules that contain any of the given source id(s).  When used in conjunction with action id filtering, the rules must contain sources *and* actions that match some of the criteria."""
+
 }