Merge pull request #11401 from kjellwinblad/kjell/fix/mongo_date_format/EMQX-10727

fix: rule SQL mongo_date function should return a string in test mode

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -1174,15 +1174,30 @@ function_literal(Fun, Args) ->
     {invalid_func, {Fun, Args}}.
 
 mongo_date() ->
-    erlang:timestamp().
+    maybe_isodate_format(erlang:timestamp()).
 
 mongo_date(MillisecondsTimestamp) ->
-    convert_timestamp(MillisecondsTimestamp).
+    maybe_isodate_format(convert_timestamp(MillisecondsTimestamp)).
 
 mongo_date(Timestamp, Unit) ->
     InsertedTimeUnit = time_unit(Unit),
     ScaledEpoch = erlang:convert_time_unit(Timestamp, InsertedTimeUnit, millisecond),
-    convert_timestamp(ScaledEpoch).
+    mongo_date(ScaledEpoch).
+
+maybe_isodate_format(ErlTimestamp) ->
+    case emqx_rule_sqltester:is_test_runtime_env() of
+        false ->
+            ErlTimestamp;
+        true ->
+            %% if this is called from sqltest, we need to convert it to the ISODate() format,
+            %% so that it can be correctly converted into a JSON string.
+            isodate_format(ErlTimestamp)
+    end.
+
+isodate_format({MegaSecs, Secs, MicroSecs}) ->
+    SystemTimeMs = (MegaSecs * 1000_000_000_000 + Secs * 1000_000 + MicroSecs) div 1000,
+    Ts3339Str = calendar:system_time_to_rfc3339(SystemTimeMs, [{unit, millisecond}, {offset, "Z"}]),
+    iolist_to_binary(["ISODate(", Ts3339Str, ")"]).
 
 convert_timestamp(MillisecondsTimestamp) ->
     MicroTimestamp = MillisecondsTimestamp * 1000,

apps/emqx_rule_engine/src/emqx_rule_sqltester.erl

@@ -18,7 +18,9 @@
 
 -export([
     test/1,
-    get_selected_data/3
+    get_selected_data/3,
+    %% Some SQL functions return different results in the test environment
+    is_test_runtime_env/0
 ]).
 
 -spec test(#{sql := binary(), context := map()}) -> {ok, map() | list()} | {error, term()}.
@@ -63,12 +65,14 @@ test_rule(Sql, Select, Context, EventTopics) ->
         created_at => erlang:system_time(millisecond)
     },
     FullContext = fill_default_values(hd(EventTopics), emqx_rule_maps:atom_key_map(Context)),
+    set_is_test_runtime_env(),
     try emqx_rule_runtime:apply_rule(Rule, FullContext, #{}) of
         {ok, Data} ->
             {ok, flatten(Data)};
         {error, Reason} ->
             {error, Reason}
     after
+        unset_is_test_runtime_env(),
         ok = emqx_rule_engine:clear_metrics_for_rule(RuleId)
     end.
 
@@ -97,3 +101,20 @@ envs_examp(EventTopic) ->
             emqx_rule_events:columns_with_exam(EventName)
         )
     ).
+
+is_test_runtime_env_atom() ->
+    'emqx_rule_sqltester:is_test_runtime_env'.
+
+set_is_test_runtime_env() ->
+    erlang:put(is_test_runtime_env_atom(), true),
+    ok.
+
+unset_is_test_runtime_env() ->
+    erlang:erase(is_test_runtime_env_atom()),
+    ok.
+
+is_test_runtime_env() ->
+    case erlang:get(is_test_runtime_env_atom()) of
+        true -> true;
+        _ -> false
+    end.

apps/emqx_rule_engine/test/emqx_rule_engine_api_rule_test_SUITE.erl

@@ -214,6 +214,27 @@ t_ctx_delivery_dropped(_) ->
     Expected = check_result([from_clientid, from_username, reason, qos, topic], [], Context),
     do_test(SQL, Context, Expected).
 
+t_mongo_date_function_should_return_string_in_test_env(_) ->
+    SQL =
+        <<"SELECT mongo_date() as mongo_date FROM \"t/1\"">>,
+    Context =
+        #{
+            action => <<"publish">>,
+            clientid => <<"c_emqx">>,
+            event_type => client_check_authz_complete,
+            result => <<"allow">>,
+            topic => <<"t/1">>,
+            username => <<"u_emqx">>
+        },
+    CheckFunction = fun(Result) ->
+        MongoDate = maps:get(mongo_date, Result),
+        %% Use regex to match the expected string
+        MatchResult = re:run(MongoDate, <<"ISODate\\([0-9]{4}-[0-9]{2}-[0-9]{2}T.*\\)">>),
+        ?assertMatch({match, _}, MatchResult),
+        ok
+    end,
+    do_test(SQL, Context, CheckFunction).
+
 do_test(SQL, Context, Expected0) ->
     Res = emqx_rule_engine_api:'/rule_test'(
         post,

changes/ee/fix-11401.en.md

@@ -0,0 +1 @@
+When running one of the rule engine SQL `mongo_date` functions in the EMQX dashboard test interface, the resulting date is formatted as `ISODate(*)`, where * is the date in ISO date format instead of only the ISO date string. This is the format used by MongoDB to store dates.