fix(ruleeng): ensure full backward compatibility

apps/emqx_rule_engine/src/emqx_rule_actions.erl

@@ -72,8 +72,8 @@ pre_process_action_args(
     Args#{
         preprocessed_tmpl => #{
             topic => emqx_template:parse(Topic),
-            qos => parse_vars(QoS),
-            retain => parse_vars(Retain),
+            qos => parse_simple_var(QoS),
+            retain => parse_simple_var(Retain),
             payload => parse_payload(Payload),
             mqtt_properties => parse_mqtt_properties(MQTTProperties),
             user_properties => parse_user_properties(UserProperties)
@@ -119,8 +119,8 @@ republish(
     }
 ) ->
     % NOTE: rendering missing bindings as string "undefined"
-    {TopicString, _Errors1} = emqx_template:render(TopicTemplate, Selected),
-    {PayloadString, _Errors2} = emqx_template:render(PayloadTemplate, Selected),
+    {TopicString, _Errors1} = render_template(TopicTemplate, Selected),
+    {PayloadString, _Errors2} = render_template(PayloadTemplate, Selected),
     Topic = iolist_to_binary(TopicString),
     Payload = iolist_to_binary(PayloadString),
     QoS = render_simple_var(QoSTemplate, Selected, 0),
@@ -201,11 +201,17 @@ safe_publish(RuleId, Topic, QoS, Flags, Payload, PubProps) ->
     _ = emqx_broker:safe_publish(Msg),
     emqx_metrics:inc_msg(Msg).
 
-parse_vars(Data) when is_binary(Data) ->
+parse_simple_var(Data) when is_binary(Data) ->
     emqx_template:parse(Data);
-parse_vars(Data) ->
+parse_simple_var(Data) ->
     {const, Data}.
 
+parse_payload(Payload) ->
+    case string:is_empty(Payload) of
+        false -> emqx_template:parse(Payload);
+        true -> emqx_template:parse("${.}")
+    end.
+
 parse_mqtt_properties(MQTTPropertiesTemplate) ->
     maps:map(
         fun(_Key, V) -> emqx_template:parse(V) end,
@@ -225,8 +231,12 @@ parse_user_properties(_) ->
     %% invalid, discard
     undefined.
 
+render_template(Template, Bindings) ->
+    Opts = #{var_lookup => fun emqx_template:lookup_loose_json/2},
+    emqx_template:render(Template, Bindings, Opts).
+
 render_simple_var([{var, _Name, Accessor}], Data, Default) ->
-    case emqx_template:lookup_var(Accessor, Data) of
+    case emqx_template:lookup_loose_json(Accessor, Data) of
         {ok, Var} -> Var;
         %% cannot find the variable from Data
         {error, _} -> Default
@@ -234,12 +244,6 @@ render_simple_var([{var, _Name, Accessor}], Data, Default) ->
 render_simple_var({const, Val}, _Data, _Default) ->
     Val.
 
-parse_payload(Payload) ->
-    case string:is_empty(Payload) of
-        false -> emqx_template:parse(Payload);
-        true -> emqx_template:parse("${.}")
-    end.
-
 render_pub_props(UserPropertiesTemplate, Selected, Env) ->
     UserProperties =
         case UserPropertiesTemplate of
@@ -257,26 +261,24 @@ render_mqtt_properties(MQTTPropertiesTemplate, Selected, Env) ->
     MQTTProperties =
         maps:fold(
             fun(K, Template, Acc) ->
-                try
-                    V = unicode:characters_to_binary(
-                        emqx_template:render_strict(Template, Selected)
-                    ),
-                    Acc#{K => V}
-                catch
-                    Kind:Error ->
+                {V, Errors} = render_template(Template, Selected),
+                NAcc = Acc#{K => iolist_to_binary(V)},
+                case Errors of
+                    [] ->
+                        ok;
+                    Errors ->
                         ?SLOG(
                             debug,
                             #{
                                 msg => "bad_mqtt_property_value_ignored",
                                 rule_id => RuleId,
-                                exception => Kind,
-                                reason => Error,
+                                reason => Errors,
                                 property => K,
                                 selected => Selected
                             }
-                        ),
-                        Acc
-                end
+                        )
+                end,
+                NAcc
             end,
             #{},
             MQTTPropertiesTemplate

apps/emqx_utils/src/emqx_template.erl

@@ -29,6 +29,7 @@
 -export([render_strict/3]).
 
 -export([lookup_var/2]).
+-export([lookup_loose_json/2]).
 -export([to_string/1]).
 
 -export_type([t/0]).
@@ -62,16 +63,23 @@
 -type binding() :: scalar() | list(scalar()) | bindings().
 -type bindings() :: #{atom() | binary() => binding()}.
 
+-type reason() :: undefined | {location(), _InvalidType :: atom()}.
+-type location() :: non_neg_integer().
+
 -type var_trans() ::
     fun((Value :: term()) -> unicode:chardata())
     | fun((varname(), Value :: term()) -> unicode:chardata()).
 
+-type var_lookup() ::
+    fun((accessor(), bindings()) -> {ok, binding()} | {error, reason()}).
+
 -type parse_opts() :: #{
     strip_double_quote => boolean()
 }.
 
 -type render_opts() :: #{
-    var_trans => var_trans()
+    var_trans => var_trans(),
+    var_lookup => var_lookup()
 }.
 
 -define(PH_VAR_THIS, '$this').
@@ -173,7 +181,7 @@ render_placeholder(Name) ->
 %% By default, all binding values are converted to strings using `to_string/1`
 %% function. Option `var_trans` can be used to override this behaviour.
 -spec render(t(), bindings()) ->
-    {term(), [_Error :: {varname(), undefined}]}.
+    {term(), [_Error :: {varname(), reason()}]}.
 render(Template, Bindings) ->
     render(Template, Bindings, #{}).
 
@@ -195,7 +203,7 @@ render({'$tpl', Template}, Bindings, Opts) ->
     render_deep(Template, Bindings, Opts).
 
 render_binding(Name, Accessor, Bindings, Opts) ->
-    case lookup_var(Accessor, Bindings) of
+    case lookup_value(Accessor, Bindings, Opts) of
         {ok, Value} ->
             {render_value(Name, Value, Opts), []};
         {error, Reason} ->
@@ -205,6 +213,11 @@ render_binding(Name, Accessor, Bindings, Opts) ->
             {render_value(Name, undefined, Opts), [{Name, Reason}]}
     end.
 
+lookup_value(Accessor, Bindings, #{var_lookup := LookupFun}) ->
+    LookupFun(Accessor, Bindings);
+lookup_value(Accessor, Bindings, #{}) ->
+    lookup_var(Accessor, Bindings).
+
 render_value(_Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 1) ->
     TransFun(Value);
 render_value(Name, Value, #{var_trans := TransFun}) when is_function(TransFun, 2) ->
@@ -309,17 +322,60 @@ unparse_deep(Term) ->
 
 %%
 
+%% @doc Lookup a variable in the bindings accessible through the accessor.
+%% Lookup is "loose" in the sense that atom and binary keys in the bindings are
+%% treated equally. This is useful for both hand-crafted and JSON-like bindings.
+%% This is the default lookup function used by rendering functions.
 -spec lookup_var(accessor(), bindings()) ->
-    {ok, binding()} | {error, undefined}.
-lookup_var(Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] ->
+    {ok, binding()} | {error, reason()}.
+lookup_var(Var, Bindings) ->
+    lookup_var(0, Var, Bindings).
+
+lookup_var(_, Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] ->
     {ok, Value};
-lookup_var([Prop | Rest], Bindings) ->
+lookup_var(Loc, [Prop | Rest], Bindings) when is_map(Bindings) ->
     case lookup(Prop, Bindings) of
         {ok, Value} ->
-            lookup_var(Rest, Value);
+            lookup_var(Loc + 1, Rest, Value);
         {error, Reason} ->
             {error, Reason}
-    end.
+    end;
+lookup_var(Loc, _, Invalid) ->
+    {error, {Loc, type_name(Invalid)}}.
+
+%% @doc Lookup a variable in the bindings accessible through the accessor.
+%% Additionally to `lookup_var/2` behavior, this function also tries to parse any
+%% binary as JSON to a map if accessor needs to go deeper into it.
+-spec lookup_loose_json(accessor(), bindings() | binary()) ->
+    {ok, binding()} | {error, reason()}.
+lookup_loose_json(Var, Bindings) ->
+    lookup_loose_json(0, Var, Bindings).
+
+lookup_loose_json(_, Var, Value) when Var == ?PH_VAR_THIS orelse Var == [] ->
+    {ok, Value};
+lookup_loose_json(Loc, [Prop | Rest], Bindings) when is_map(Bindings) ->
+    case lookup(Prop, Bindings) of
+        {ok, Value} ->
+            lookup_loose_json(Loc + 1, Rest, Value);
+        {error, Reason} ->
+            {error, Reason}
+    end;
+lookup_loose_json(Loc, Rest, Json) when is_binary(Json) ->
+    try emqx_utils_json:decode(Json) of
+        Bindings ->
+            % NOTE: This is intentional, we don't want to parse nested JSON.
+            lookup_var(Loc, Rest, Bindings)
+    catch
+        error:_ ->
+            {error, {Loc, binary}}
+    end;
+lookup_loose_json(Loc, _, Invalid) ->
+    {error, {Loc, type_name(Invalid)}}.
+
+type_name(Term) when is_atom(Term) -> atom;
+type_name(Term) when is_number(Term) -> number;
+type_name(Term) when is_binary(Term) -> binary;
+type_name(Term) when is_list(Term) -> list.
 
 -spec lookup(Prop :: binary(), bindings()) ->
     {ok, binding()} | {error, undefined}.

apps/emqx_utils/test/emqx_template_SUITE.erl

@@ -89,15 +89,15 @@ t_render_this(_) ->
     ).
 
 t_render_missing_bindings(_) ->
-    Bindings = #{no => #{}},
+    Bindings = #{no => #{}, c => #{<<"c1">> => 42}},
     Template = emqx_template:parse(
-        <<"a:${a},b:${b},c:${c},d:${d.d1},e:${no.such_atom_i_swear}">>
+        <<"a:${a},b:${b},c:${c.c1.c2},d:${d.d1},e:${no.such_atom_i_swear}">>
     ),
     ?assertEqual(
         {<<"a:undefined,b:undefined,c:undefined,d:undefined,e:undefined">>, [
             {"no.such_atom_i_swear", undefined},
             {"d.d1", undefined},
-            {"c", undefined},
+            {"c.c1.c2", {2, number}},
             {"b", undefined},
             {"a", undefined}
         ]},
@@ -107,7 +107,7 @@ t_render_missing_bindings(_) ->
         [
             {"no.such_atom_i_swear", undefined},
             {"d.d1", undefined},
-            {"c", undefined},
+            {"c.c1.c2", {2, number}},
             {"b", undefined},
             {"a", undefined}
         ],