feat(variform): implement variform engine

apps/emqx/src/variform/emqx_variform.erl

@@ -14,17 +14,46 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
-%% Predefined functions for templating
+%% @doc This module provides a single-line expression string rendering engine.
+%% A predefined set of functions are allowed to be called in the expressions.
+%% Only simple string expressions are supported, and no control flow is allowed.
+%% However, with the help from the functions, some control flow can be achieved.
+%% For example, the `coalesce` function can be used to provide a default value,
+%% or used to choose the first non-empty value from a list of variables.
 -module(emqx_variform).
 
--export([render/2]).
+-export([inject_allowed_modules/1]).
+-export([render/2, render/3]).
 
-render(Expression, Context) ->
+%% @doc Render a variform expression with bindings.
+%% A variform expression is a template string which supports variable substitution
+%% and function calls.
+%%
+%% The function calls are in the form of `module.function(arg1, arg2, ...)` where `module`
+%% is optional, and if not provided, the function is assumed to be in the `emqx_variform_str` module.
+%% Both module and function must be existing atoms, and only whitelisted functions are allowed.
+%%
+%% A function arg can be a constant string or a number.
+%% Strings can be quoted with single quotes or double quotes, without support of escape characters.
+%% If some special characters are needed, the function `unescape' can be used convert a escaped string
+%% to raw bytes.
+%% For example, to get the first line of a multi-line string, the expression can be
+%% `coalesce(tokens(variable_name, unescape("\n")))'.
+%%
+%% The bindings is a map of variables to their values.
+%%
+%% For unresolved variables, empty string (but not "undefined") is used.
+%% In case of runtime exeption, an error is returned.
+-spec render(string(), map()) -> {ok, binary()} | {error, term()}.
+render(Expression, Bindings) ->
+    render(Expression, Bindings, #{}).
+
+render(Expression, Bindings, Opts) ->
     case emqx_variform_scan:string(Expression) of
         {ok, Tokens, _Line} ->
             case emqx_variform_parser:parse(Tokens) of
                 {ok, Expr} ->
-                    eval(Expr, Context);
+                    eval_as_string(Expr, Bindings, Opts);
                 {error, {_, emqx_variform_parser, Msg}} ->
                     %% syntax error
                     {error, lists:flatten(Msg)};
@@ -35,5 +64,122 @@ render(Expression, Context) ->
             {error, Reason}
     end.
 
-eval(Expr, _Context) ->
-    io:format(user, "~p~n", [Expr]).
+eval_as_string(Expr, Bindings, _Opts) ->
+    try
+        {ok, iolist_to_binary(eval(Expr, Bindings))}
+    catch
+        throw:Reason ->
+            {error, Reason};
+        C:E:S ->
+            {error, #{exception => C, reason => E, stack_trace => S}}
+    end.
+
+eval({str, Str}, _Bindings) ->
+    str(Str);
+eval({num, Num}, _Bindings) ->
+    str(Num);
+eval({call, FuncNameStr, Args}, Bindings) ->
+    {Mod, Fun} = resolve_func_name(FuncNameStr),
+    ok = assert_func_exported(Mod, Fun, length(Args)),
+    call(Mod, Fun, eval(Args, Bindings));
+eval({var, VarName}, Bindings) ->
+    resolve_var_value(VarName, Bindings);
+eval([Arg | Args], Bindings) ->
+    [eval(Arg, Bindings) | eval(Args, Bindings)];
+eval([], _Bindings) ->
+    [].
+
+%% Some functions accept arbitrary number of arguments but implemented as /1.
+call(emqx_variform_str, concat, Args) ->
+    str(emqx_variform_str:concat(Args));
+call(emqx_variform_str, coalesce, Args) ->
+    str(emqx_variform_str:coalesce(Args));
+call(Mod, Fun, Args) ->
+    str(erlang:apply(Mod, Fun, Args)).
+
+resolve_func_name(FuncNameStr) ->
+    case string:tokens(FuncNameStr, ".") of
+        [Mod0, Fun0] ->
+            Mod =
+                try
+                    list_to_existing_atom(Mod0)
+                catch
+                    error:badarg ->
+                        throw(#{unknown_module => Mod0})
+                end,
+            ok = assert_module_allowed(Mod),
+            Fun =
+                try
+                    list_to_existing_atom(Fun0)
+                catch
+                    error:badarg ->
+                        throw(#{unknown_function => Fun0})
+                end,
+            {Mod, Fun};
+        [Fun] ->
+            FuncName =
+                try
+                    list_to_existing_atom(Fun)
+                catch
+                    error:badarg ->
+                        throw(#{
+                            reason => "unknown_variform_function",
+                            function => Fun
+                        })
+                end,
+            {emqx_variform_str, FuncName}
+    end.
+
+resolve_var_value(VarName, Bindings) ->
+    case emqx_template:lookup_var(split(VarName), Bindings) of
+        {ok, Value} ->
+            str(Value);
+        {error, _Reason} ->
+            <<>>
+    end.
+
+assert_func_exported(emqx_variform_str, concat, _Arity) ->
+    ok;
+assert_func_exported(emqx_variform_str, coalesce, _Arity) ->
+    ok;
+assert_func_exported(Mod, Fun, Arity) ->
+    _ = Mod:module_info(md5),
+    case erlang:function_exported(Mod, Fun, Arity) of
+        true ->
+            ok;
+        false ->
+            throw(#{
+                reason => "unknown_variform_function",
+                module => Mod,
+                function => Fun,
+                arity => Arity
+            })
+    end.
+
+assert_module_allowed(emqx_variform_str) ->
+    ok;
+assert_module_allowed(Mod) ->
+    Allowed = get_allowed_modules(),
+    case lists:member(Mod, Allowed) of
+        true ->
+            ok;
+        false ->
+            throw(#{
+                reason => "unallowed_veriform_module",
+                module => Mod
+            })
+    end.
+
+inject_allowed_modules(Modules) ->
+    Allowed0 = get_allowed_modules(),
+    Allowed = lists:usort(Allowed0 ++ Modules),
+    persistent_term:put({emqx_variform, allowed_modules}, Allowed).
+
+get_allowed_modules() ->
+    persistent_term:get({emqx_variform, allowed_modules}, []).
+
+str(Value) ->
+    emqx_utils_conv:bin(Value).
+
+split(VarName) ->
+    lists:map(fun erlang:iolist_to_binary/1, string:tokens(VarName, ".")).

apps/emqx/src/variform/emqx_variform_str.erl

@@ -64,10 +64,10 @@
 %% @doc Return the first non-empty string
 coalesce(A, B) when ?IS_EMPTY(A) andalso ?IS_EMPTY(B) ->
     <<>>;
-coalesce(A, _) when is_binary(A) ->
-    A;
-coalesce(_, B) ->
-    B.
+coalesce(A, B) when ?IS_EMPTY(A) ->
+    B;
+coalesce(A, _B) ->
+    A.
 
 %% @doc Return the first non-empty string
 coalesce([]) ->
@@ -140,7 +140,7 @@ tokens(S, Separators, <<"nocrlf">>) ->
 
 %% implicit convert args to strings, and then do concatenation
 concat(S1, S2) ->
-    concat([S1, S2], unicode).
+    concat([S1, S2]).
 
 concat(List) ->
     unicode:characters_to_binary(lists:map(fun str/1, List), unicode).

apps/emqx_rule_engine/src/emqx_rule_funcs.erl

@@ -145,6 +145,7 @@
     upper/1,
     split/2,
     split/3,
+    concat/1,
     concat/2,
     tokens/2,
     tokens/3,