Merge pull request #12381 from emqx/port-sql-funcs-from-emqx4

feat: port emqx/emqx-enterprise#1892, add some SQL functions

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -116,7 +116,9 @@
 %% Data Type Validation Funcs
 -export([
     is_null/1,
+    is_null_var/1,
     is_not_null/1,
+    is_not_null_var/1,
     is_str/1,
     is_bool/1,
     is_int/1,
@@ -153,6 +155,9 @@
     ascii/1,
     find/2,
     find/3,
+    join_to_string/1,
+    join_to_string/2,
+    join_to_sql_values_string/1,
     jq/2,
     jq/3
 ]).
@@ -163,7 +168,10 @@
 -export([
     map_get/2,
     map_get/3,
-    map_put/3
+    map_put/3,
+    map_keys/1,
+    map_values/1,
+    map_to_entries/1
 ]).
 
 %% For backward compatibility
@@ -699,9 +707,16 @@ hexstr2bin(Str) when is_binary(Str) ->
 is_null(undefined) -> true;
 is_null(_Data) -> false.
 
+%% Similar to is_null/1, but also works for the JSON value 'null'
+is_null_var(null) -> true;
+is_null_var(Data) -> is_null(Data).
+
 is_not_null(Data) ->
     not is_null(Data).
 
+is_not_null_var(Data) ->
+    not is_null_var(Data).
+
 is_str(T) when is_binary(T) -> true;
 is_str(_) -> false.
 
@@ -847,6 +862,23 @@ find_s(S, P, Dir) ->
         SubStr -> SubStr
     end.
 
+join_to_string(List) when is_list(List) ->
+    join_to_string(<<", ">>, List).
+join_to_string(Sep, List) when is_list(List), is_binary(Sep) ->
+    iolist_to_binary(lists:join(Sep, [str(Item) || Item <- List])).
+join_to_sql_values_string(List) ->
+    QuotedList =
+        [
+            case is_list(Item) of
+                true ->
+                    emqx_placeholder:quote_sql(emqx_utils_json:encode(Item));
+                false ->
+                    emqx_placeholder:quote_sql(Item)
+            end
+         || Item <- List
+        ],
+    iolist_to_binary(lists:join(<<", ">>, QuotedList)).
+
 -spec jq(FilterProgram, JSON, TimeoutMS) -> Result when
     FilterProgram :: binary(),
     JSON :: binary() | term(),
@@ -920,7 +952,8 @@ map_put(Key, Val, Map) ->
 mget(Key, Map) ->
     mget(Key, Map, undefined).
 
-mget(Key, Map, Default) ->
+mget(Key, Map0, Default) ->
+    Map = map(Map0),
     case maps:find(Key, Map) of
         {ok, Val} ->
             Val;
@@ -947,7 +980,8 @@ mget(Key, Map, Default) ->
             Default
     end.
 
-mput(Key, Val, Map) ->
+mput(Key, Val, Map0) ->
+    Map = map(Map0),
     case maps:find(Key, Map) of
         {ok, _} ->
             maps:put(Key, Val, Map);
@@ -974,6 +1008,13 @@ mput(Key, Val, Map) ->
             maps:put(Key, Val, Map)
     end.
 
+map_keys(Map) ->
+    maps:keys(map(Map)).
+map_values(Map) ->
+    maps:values(map(Map)).
+map_to_entries(Map) ->
+    [#{key => K, value => V} || {K, V} <- maps:to_list(map(Map))].
+
 %%------------------------------------------------------------------------------
 %% Hash Funcs
 %%------------------------------------------------------------------------------
@@ -1168,16 +1209,18 @@ map_path(Key) ->
     {path, [{key, P} || P <- string:split(Key, ".", all)]}.
 
 function_literal(Fun, []) when is_atom(Fun) ->
-    atom_to_list(Fun) ++ "()";
+    iolist_to_binary(atom_to_list(Fun) ++ "()");
 function_literal(Fun, [FArg | Args]) when is_atom(Fun), is_list(Args) ->
     WithFirstArg = io_lib:format("~ts(~0p", [atom_to_list(Fun), FArg]),
-    lists:foldl(
-        fun(Arg, Literal) ->
-            io_lib:format("~ts, ~0p", [Literal, Arg])
-        end,
-        WithFirstArg,
-        Args
-    ) ++ ")";
+    FuncLiteral =
+        lists:foldl(
+            fun(Arg, Literal) ->
+                io_lib:format("~ts, ~0p", [Literal, Arg])
+            end,
+            WithFirstArg,
+            Args
+        ) ++ ")",
+    iolist_to_binary(FuncLiteral);
 function_literal(Fun, Args) ->
     {invalid_func, {Fun, Args}}.
 

apps/emqx_rule_engine/test/emqx_rule_funcs_SUITE.erl

@@ -215,15 +215,32 @@ hex_convert() ->
     ).
 
 t_is_null(_) ->
+    ?assertEqual(false, emqx_rule_funcs:is_null(null)),
     ?assertEqual(true, emqx_rule_funcs:is_null(undefined)),
+    ?assertEqual(false, emqx_rule_funcs:is_null(<<"undefined">>)),
     ?assertEqual(false, emqx_rule_funcs:is_null(a)),
     ?assertEqual(false, emqx_rule_funcs:is_null(<<>>)),
     ?assertEqual(false, emqx_rule_funcs:is_null(<<"a">>)).
 
+t_is_null_var(_) ->
+    ?assertEqual(true, emqx_rule_funcs:is_null_var(null)),
+    ?assertEqual(false, emqx_rule_funcs:is_null_var(<<"null">>)),
+    ?assertEqual(true, emqx_rule_funcs:is_null_var(undefined)),
+    ?assertEqual(false, emqx_rule_funcs:is_null_var(<<"undefined">>)),
+    ?assertEqual(false, emqx_rule_funcs:is_null_var(a)),
+    ?assertEqual(false, emqx_rule_funcs:is_null_var(<<>>)),
+    ?assertEqual(false, emqx_rule_funcs:is_null_var(<<"a">>)).
+
 t_is_not_null(_) ->
     [
         ?assertEqual(emqx_rule_funcs:is_not_null(T), not emqx_rule_funcs:is_null(T))
-     || T <- [undefined, a, <<"a">>, <<>>]
+     || T <- [undefined, <<"undefined">>, null, <<"null">>, a, <<"a">>, <<>>]
+    ].
+
+t_is_not_null_var(_) ->
+    [
+        ?assertEqual(emqx_rule_funcs:is_not_null_var(T), not emqx_rule_funcs:is_null_var(T))
+     || T <- [undefined, <<"undefined">>, null, <<"null">>, a, <<"a">>, <<>>]
     ].
 
 t_is_str(_) ->
@@ -622,6 +639,63 @@ t_ascii(_) ->
     ?assertEqual(97, apply_func(ascii, [<<"a">>])),
     ?assertEqual(97, apply_func(ascii, [<<"ab">>])).
 
+t_join_to_string(_) ->
+    A = 1,
+    B = a,
+    C = <<"c">>,
+    D = #{a => 1},
+    E = [1, 2, 3],
+    F = [#{<<"key">> => 1, <<"value">> => 2}],
+    M = #{<<"a">> => a, <<"b">> => 1, <<"c">> => <<"c">>},
+    J = <<"{\"a\":\"a\",\"b\":1,\"c\":\"c\"}">>,
+    ?assertEqual(<<"a,b,c">>, apply_func(join_to_string, [<<",">>, [<<"a">>, <<"b">>, <<"c">>]])),
+    ?assertEqual(<<"a b c">>, apply_func(join_to_string, [<<" ">>, [<<"a">>, <<"b">>, <<"c">>]])),
+    ?assertEqual(
+        <<"a, b, c">>, apply_func(join_to_string, [<<", ">>, [<<"a">>, <<"b">>, <<"c">>]])
+    ),
+    ?assertEqual(
+        <<"1, a, c, {\"a\":1}, [1,2,3], [{\"value\":2,\"key\":1}]">>,
+        apply_func(join_to_string, [<<", ">>, [A, B, C, D, E, F]])
+    ),
+    ?assertEqual(<<"a">>, apply_func(join_to_string, [<<",">>, [<<"a">>]])),
+    ?assertEqual(<<"">>, apply_func(join_to_string, [<<",">>, []])),
+    ?assertEqual(<<"a, b, c">>, apply_func(join_to_string, [emqx_rule_funcs:map_keys(M)])),
+    ?assertEqual(<<"a, b, c">>, apply_func(join_to_string, [emqx_rule_funcs:map_keys(J)])),
+    ?assertEqual(<<"a, 1, c">>, apply_func(join_to_string, [emqx_rule_funcs:map_values(M)])),
+    ?assertEqual(<<"a, 1, c">>, apply_func(join_to_string, [emqx_rule_funcs:map_values(J)])).
+
+t_join_to_sql_values_string(_) ->
+    A = 1,
+    B = a,
+    C = <<"c">>,
+    D = #{a => 1},
+    E = [1, 2, 3],
+    E1 = [97, 98],
+    F = [#{<<"key">> => 1, <<"value">> => 2}],
+    M = #{<<"a">> => a, <<"b">> => 1, <<"c">> => <<"c">>},
+    J = <<"{\"a\":\"a\",\"b\":1,\"c\":\"c\"}">>,
+    ?assertEqual(
+        <<"'a', 'b', 'c'">>, apply_func(join_to_sql_values_string, [[<<"a">>, <<"b">>, <<"c">>]])
+    ),
+    ?assertEqual(
+        <<"1, 'a', 'c', '{\"a\":1}', '[1,2,3]', '[97,98]', '[{\"value\":2,\"key\":1}]'">>,
+        apply_func(join_to_sql_values_string, [[A, B, C, D, E, E1, F]])
+    ),
+    ?assertEqual(<<"'a'">>, apply_func(join_to_sql_values_string, [[<<"a">>]])),
+    ?assertEqual(<<"">>, apply_func(join_to_sql_values_string, [[]])),
+    ?assertEqual(
+        <<"'a', 'b', 'c'">>, apply_func(join_to_sql_values_string, [emqx_rule_funcs:map_keys(M)])
+    ),
+    ?assertEqual(
+        <<"'a', 'b', 'c'">>, apply_func(join_to_sql_values_string, [emqx_rule_funcs:map_keys(J)])
+    ),
+    ?assertEqual(
+        <<"'a', 1, 'c'">>, apply_func(join_to_sql_values_string, [emqx_rule_funcs:map_values(M)])
+    ),
+    ?assertEqual(
+        <<"'a', 1, 'c'">>, apply_func(join_to_sql_values_string, [emqx_rule_funcs:map_values(J)])
+    ).
+
 t_find(_) ->
     ?assertEqual(<<"cbcd">>, apply_func(find, [<<"acbcd">>, <<"c">>])),
     ?assertEqual(<<"cbcd">>, apply_func(find, [<<"acbcd">>, <<"c">>, <<"leading">>])),
@@ -746,14 +820,37 @@ t_map_put(_) ->
     ?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])).
 
 t_mget(_) ->
-    ?assertEqual(1, apply_func(map_get, [<<"a">>, #{a => 1}])),
-    ?assertEqual(1, apply_func(map_get, [<<"a">>, #{<<"a">> => 1}])),
-    ?assertEqual(undefined, apply_func(map_get, [<<"a">>, #{}])).
+    ?assertEqual(1, apply_func(mget, [<<"a">>, #{a => 1}])),
+    ?assertEqual(1, apply_func(mget, [<<"a">>, <<"{\"a\" : 1}">>])),
+    ?assertEqual(1, apply_func(mget, [<<"a">>, #{<<"a">> => 1}])),
+    ?assertEqual(1, apply_func(mget, [<<"a.b">>, #{<<"a.b">> => 1}])),
+    ?assertEqual(undefined, apply_func(mget, [<<"a">>, #{}])).
 
 t_mput(_) ->
-    ?assertEqual(#{<<"a">> => 1}, apply_func(map_put, [<<"a">>, 1, #{}])),
-    ?assertEqual(#{<<"a">> => 2}, apply_func(map_put, [<<"a">>, 2, #{<<"a">> => 1}])),
-    ?assertEqual(#{a => 2}, apply_func(map_put, [<<"a">>, 2, #{a => 1}])).
+    ?assertEqual(#{<<"a">> => 1}, apply_func(mput, [<<"a">>, 1, #{}])),
+    ?assertEqual(#{<<"a">> => 2}, apply_func(mput, [<<"a">>, 2, #{<<"a">> => 1}])),
+    ?assertEqual(#{<<"a">> => 2}, apply_func(mput, [<<"a">>, 2, <<"{\"a\" : 1}">>])),
+    ?assertEqual(#{<<"a.b">> => 2}, apply_func(mput, [<<"a.b">>, 2, #{<<"a.b">> => 1}])),
+    ?assertEqual(#{a => 2}, apply_func(mput, [<<"a">>, 2, #{a => 1}])).
+
+t_map_to_entries(_) ->
+    ?assertEqual([], apply_func(map_to_entries, [#{}])),
+    M = #{a => 1, b => <<"b">>},
+    J = <<"{\"a\":1,\"b\":\"b\"}">>,
+    ?assertEqual(
+        [
+            #{key => a, value => 1},
+            #{key => b, value => <<"b">>}
+        ],
+        apply_func(map_to_entries, [M])
+    ),
+    ?assertEqual(
+        [
+            #{key => <<"a">>, value => 1},
+            #{key => <<"b">>, value => <<"b">>}
+        ],
+        apply_func(map_to_entries, [J])
+    ).
 
 t_bitsize(_) ->
     ?assertEqual(8, apply_func(bitsize, [<<"a">>])),

changes/ee/feat-12381.en.md

@@ -0,0 +1,3 @@
+Added new SQL functions: map_keys(), map_values(), map_to_entries(), join_to_string(), join_to_string(), join_to_sql_values_string(), is_null_var(), is_not_null_var().
+
+For more information on the functions and their usage, refer to the documentation.