Merge pull request #14423 from JimMoen/fix-rule-str-convert-func

fix: str_utf8/1 and str_utf16_le/1 should only convert str as binary

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -712,12 +712,12 @@ do_get_subbits(Bits, Sz, Len, <<"bits">>, <<"signed">>, <<"little">>) ->
 str(Data) ->
     emqx_utils_conv:bin(Data).
 
-str_utf8(Data) when is_binary(Data); is_list(Data) ->
+str_utf8(Data) when is_binary(Data) ->
     unicode:characters_to_binary(Data);
 str_utf8(Data) ->
     unicode:characters_to_binary(str(Data)).
 
-str_utf16_le(Data) when is_binary(Data); is_list(Data) ->
+str_utf16_le(Data) when is_binary(Data) ->
     unicode:characters_to_binary(Data, utf8, {utf16, little});
 str_utf16_le(Data) ->
     unicode:characters_to_binary(str(Data), utf8, {utf16, little}).

apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl

@@ -106,19 +106,29 @@ t_payload(_) ->
 t_str(_) ->
     ?assertEqual(<<"abc">>, emqx_rule_funcs:str("abc")),
     ?assertEqual(<<"abc">>, emqx_rule_funcs:str(abc)),
+    ?assertEqual(<<"👋"/utf8>>, emqx_rule_funcs:str("👋")),
+    ?assertEqual(<<"你好🐸"/utf8>>, emqx_rule_funcs:str("你好🐸")),
     ?assertEqual(<<"{\"a\":1}">>, emqx_rule_funcs:str(#{a => 1})),
     ?assertEqual(<<"[{\"a\":1},{\"b\":1}]">>, emqx_rule_funcs:str([#{a => 1}, #{b => 1}])),
     ?assertEqual(<<"1">>, emqx_rule_funcs:str(1)),
     ?assertEqual(<<"2.0">>, emqx_rule_funcs:str(2.0)),
     ?assertEqual(<<"true">>, emqx_rule_funcs:str(true)),
-    ?assertError(_, emqx_rule_funcs:str({a, v})),
+    ?assertError(_, emqx_rule_funcs:str({a, v})).
 
+t_str_utf8(_) ->
     ?assertEqual(<<"abc">>, emqx_rule_funcs:str_utf8("abc")),
     ?assertEqual(<<"abc 你好"/utf8>>, emqx_rule_funcs:str_utf8("abc 你好")),
+    ?assertEqual(<<"👋"/utf8>>, emqx_rule_funcs:str_utf8("👋")),
+    ?assertEqual(<<"你好🐸"/utf8>>, emqx_rule_funcs:str_utf8("你好🐸")),
     ?assertEqual(<<"abc 你好"/utf8>>, emqx_rule_funcs:str_utf8(<<"abc 你好"/utf8>>)),
     ?assertEqual(<<"abc">>, emqx_rule_funcs:str_utf8(abc)),
     ?assertEqual(
-        <<"{\"a\":\"abc 你好\"}"/utf8>>, emqx_rule_funcs:str_utf8(#{a => <<"abc 你好"/utf8>>})
+        <<"{\"a\":\"abc 你好\"}"/utf8>>,
+        emqx_rule_funcs:str_utf8(#{a => <<"abc 你好"/utf8>>})
+    ),
+    ?assertEqual(
+        <<"[{\"a\":1},{\"你好👋\":1}]"/utf8>>,
+        emqx_rule_funcs:str_utf8([#{a => 1}, #{<<"你好👋"/utf8>> => 1}])
     ),
     ?assertEqual(<<"1">>, emqx_rule_funcs:str_utf8(1)),
     ?assertEqual(<<"2.0">>, emqx_rule_funcs:str_utf8(2.0)),
@@ -136,12 +146,18 @@ t_str_utf16_le(_) ->
 
     ?assertEqual(<<"abc"/utf16-little>>, emqx_rule_funcs:str_utf16_le("abc")),
     ?assertEqual(<<"abc 你好"/utf16-little>>, emqx_rule_funcs:str_utf16_le("abc 你好")),
+    ?assertEqual(<<"👋"/utf16-little>>, emqx_rule_funcs:str_utf16_le("👋")),
+    ?assertEqual(<<"你好🐸"/utf16-little>>, emqx_rule_funcs:str_utf16_le("你好🐸")),
     ?assertEqual(<<"abc 你好"/utf16-little>>, emqx_rule_funcs:str_utf16_le(<<"abc 你好"/utf8>>)),
     ?assertEqual(<<"abc"/utf16-little>>, emqx_rule_funcs:str_utf16_le(abc)),
     ?assertEqual(
         <<"{\"a\":\"abc 你好\"}"/utf16-little>>,
         emqx_rule_funcs:str_utf16_le(#{a => <<"abc 你好"/utf8>>})
     ),
+    ?assertEqual(
+        <<"[{\"a\":1},{\"你好👋\":1}]"/utf16-little>>,
+        emqx_rule_funcs:str_utf16_le([#{a => 1}, #{<<"你好👋"/utf8>> => 1}])
+    ),
     ?assertEqual(<<"1"/utf16-little>>, emqx_rule_funcs:str_utf16_le(1)),
     ?assertEqual(<<"2.0"/utf16-little>>, emqx_rule_funcs:str_utf16_le(2.0)),
     ?assertEqual(<<"true"/utf16-little>>, emqx_rule_funcs:str_utf16_le(true)),

apps/emqx_utils/src/emqx_utils_conv.erl

@@ -34,7 +34,7 @@ bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
 bin(Map) when is_map(Map) -> emqx_utils_json:encode(Map);
 bin(List) when is_list(List) ->
     case io_lib:printable_list(List) of
-        true -> list_to_binary(List);
+        true -> unicode:characters_to_binary(List);
         false -> emqx_utils_json:encode(List)
     end;
 bin(Data) ->

changes/fix-14423.en.md

@@ -0,0 +1 @@
+Fixed the issue that the built-in SQL function `str_utf8(Data: Term) -> string` cannot correctly handle array type json objects.