Merge pull request #10728 from lafirest/fix/port_ruleengine_export_colums

fix: cannot access columns exported by FOREACH in DO clause

apps/emqx_rule_engine/src/emqx_rule_engine.app.src

@@ -2,7 +2,7 @@
 {application, emqx_rule_engine, [
     {description, "EMQX Rule Engine"},
     % strict semver, bump manually!
-    {vsn, "5.0.16"},
+    {vsn, "5.0.17"},
     {modules, []},
     {registered, [emqx_rule_engine_sup, emqx_rule_engine]},
     {applications, [kernel, stdlib, rulesql, getopt, emqx_ctl]},

apps/emqx_rule_engine/src/emqx_rule_runtime.erl

@@ -144,14 +144,14 @@ do_apply_rule(
         )
     of
         true ->
-            Collection2 = filter_collection(Columns, InCase, DoEach, Collection),
+            Collection2 = filter_collection(ColumnsAndSelected, InCase, DoEach, Collection),
             case Collection2 of
                 [] ->
                     ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'failed.no_result');
                 _ ->
                     ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'passed')
             end,
-            NewEnvs = maps:merge(Columns, Envs),
+            NewEnvs = maps:merge(ColumnsAndSelected, Envs),
             {ok, [handle_action_list(RuleId, Actions, Coll, NewEnvs) || Coll <- Collection2]};
         false ->
             ok = emqx_metrics_worker:inc(rule_metrics, RuleId, 'failed.no_result'),

apps/emqx_rule_engine/test/emqx_rule_engine_SUITE.erl

@@ -1735,11 +1735,12 @@ t_sqlparse_foreach_7(_Config) ->
         )
     ).
 
+-define(COLL, #{<<"info">> := [<<"haha">>, #{<<"name">> := <<"cmd1">>, <<"cmd">> := <<"1">>}]}).
 t_sqlparse_foreach_8(_Config) ->
     %% Verify foreach-do-incase and cascaded AS
     Sql =
         "foreach json_decode(payload) as p, p.sensors as s, s.collection as c, c.info as info "
-        "do info.cmd as msg_type, info.name as name "
+        "do info.cmd as msg_type, info.name as name, s, c "
         "incase is_map(info) "
         "from \"t/#\" "
         "where s.page = '2' ",
@@ -1748,7 +1749,14 @@ t_sqlparse_foreach_8(_Config) ->
         "{\"info\":[\"haha\", {\"name\":\"cmd1\", \"cmd\":\"1\"}]} } }"
     >>,
     ?assertMatch(
-        {ok, [#{<<"name">> := <<"cmd1">>, <<"msg_type">> := <<"1">>}]},
+        {ok, [
+            #{
+                <<"name">> := <<"cmd1">>,
+                <<"msg_type">> := <<"1">>,
+                <<"s">> := #{<<"page">> := 2, <<"collection">> := ?COLL},
+                <<"c">> := ?COLL
+            }
+        ]},
         emqx_rule_sqltester:test(
             #{
                 sql => Sql,

changes/ce/fix-10728.en.md

@@ -0,0 +1,11 @@
+Fixed an issue where the rule engine was unable to access variables exported by `FOREACH` in the `DO` clause. 
+
+  Given a payload: `{"date": "2023-05-06", "array": ["a"]}`, as well as the following SQL statement:
+  ```
+  FOREACH payload.date as date, payload.array as elem
+  DO date, elem
+  FROM "t/#"
+  ```
+  Prior to the fix, the `date` variable exported by `FOREACH` could not be accessed in the `DO` clause of the above SQL, resulting in the following output for the SQL statement:
+  `[{"elem": "a","date": "undefined"}]`.
+  After the fix, the output of the SQL statement is: `[{"elem": "a","date": "2023-05-06"}]`