feat: add tns to log metadata

apps/emqx/include/emqx.hrl

@@ -112,4 +112,9 @@
 -define(KIND_REPLICATE, replicate).
 -define(KIND_INITIATE, initiate).
 
+%%--------------------------------------------------------------------
+%% Client Attributes
+%%--------------------------------------------------------------------
+-define(CLIENT_ATTR_NAME_TNS, <<"tns">>).
+
 -endif.

apps/emqx/src/emqx_channel.erl

@@ -71,8 +71,14 @@
     prepare_will_message_for_publishing/2
 ]).
 
-%% Exports for CT
--export([set_field/3]).
+%% Exports for tests
+-ifdef(TEST).
+-export([
+    dummy/0,
+    set_field/3,
+    set_log_meta/2
+]).
+-endif.
 
 -import(
     emqx_utils,
@@ -1825,8 +1831,33 @@ fix_mountpoint(ClientInfo = #{mountpoint := MountPoint}) ->
 
 set_log_meta(_ConnPkt, #channel{clientinfo = #{clientid := ClientId} = ClientInfo}) ->
     Username = maps:get(username, ClientInfo, undefined),
-    emqx_logger:set_metadata_clientid(ClientId),
-    emqx_logger:set_metadata_username(Username).
+    Attrs = maps:get(client_attrs, ClientInfo, #{}),
+    Tns0 = maps:get(?CLIENT_ATTR_NAME_TNS, Attrs, undefined),
+    %% No need to add Tns to log metadata if it's aready a prefix is client ID
+    %% Or if it's the username.
+    Tns =
+        case is_clientid_namespaced(ClientId, Tns0) orelse Username =:= Tns0 of
+            true ->
+                undefined;
+            false ->
+                Tns0
+        end,
+    Meta0 = [{clientid, ClientId}, {username, Username}, {tns, Tns}],
+    %% Drop undefined or <<>>
+    Meta = lists:filter(fun({_, V}) -> V =/= undefined andalso V =/= <<>> end, Meta0),
+    emqx_logger:set_proc_metadata(maps:from_list(Meta)).
+
+%% clientid_override is an expression which is free to set tns as a prefix, suffix or whatsoever,
+%% but as a best-effort log metadata optimization, we only check for prefix
+is_clientid_namespaced(ClientId, Tns) when is_binary(Tns) andalso Tns =/= <<>> ->
+    case ClientId of
+        <<Tns:(size(Tns))/binary, _/binary>> ->
+            true;
+        _ ->
+            false
+    end;
+is_clientid_namespaced(_ClientId, _Tns) ->
+    false.
 
 %%--------------------------------------------------------------------
 %% Check banned
@@ -2858,6 +2889,10 @@ proto_ver(_, _) ->
 %% For CT tests
 %%--------------------------------------------------------------------
 
+-ifdef(TEST).
+dummy() -> #channel{}.
+
 set_field(Name, Value, Channel) ->
     Pos = emqx_utils:index_of(Name, record_info(fields, channel)),
     setelement(Pos + 1, Channel, Value).
+-endif.

apps/emqx/src/emqx_logger_textfmt.erl

@@ -122,6 +122,7 @@ enrich_report(ReportRaw0, Meta, Config) ->
             undefined -> maps:get(username, ReportRaw, undefined);
             Username0 -> Username0
         end,
+    Tns = maps:get(tns, Meta, undefined),
     ClientId = maps:get(clientid, Meta, undefined),
     Peer = maps:get(peername, Meta, undefined),
     Msg = maps:get(msg, ReportRaw, undefined),
@@ -135,7 +136,7 @@ enrich_report(ReportRaw0, Meta, Config) ->
             ({_, undefined}, Acc) -> Acc;
             (Item, Acc) -> [Item | Acc]
         end,
-        maps:to_list(maps:without([topic, msg, clientid, username, tag], ReportRaw)),
+        maps:to_list(maps:without([topic, msg, tns, clientid, username, tag], ReportRaw)),
         [
             {topic, try_format_unicode(Topic)},
             {username, try_format_unicode(Username)},
@@ -143,6 +144,7 @@ enrich_report(ReportRaw0, Meta, Config) ->
             {mfa, try_format_unicode(MFA)},
             {msg, Msg},
             {clientid, try_format_unicode(ClientId)},
+            {tns, try_format_unicode(Tns)},
             {tag, Tag}
         ]
     ).

apps/emqx/src/emqx_trace/emqx_trace_formatter.erl

@@ -33,12 +33,17 @@ format(
     #{level := debug, meta := Meta = #{trace_tag := Tag}, msg := Msg} =
         emqx_logger_textfmt:evaluate_lazy_values(Entry),
     Time = emqx_utils_calendar:now_to_rfc3339(microsecond),
+    Tns =
+        case to_iolist(maps:get(tns, Meta, "")) of
+            "" -> "";
+            X -> [" tns: ", X]
+        end,
     ClientId = to_iolist(maps:get(clientid, Meta, "")),
     Peername = maps:get(peername, Meta, ""),
     MetaBin = format_meta(Meta, PEncode),
     Msg1 = to_iolist(Msg),
     Tag1 = to_iolist(Tag),
-    [Time, " [", Tag1, "] ", ClientId, "@", Peername, " msg: ", Msg1, ", ", MetaBin, "\n"];
+    [Time, " [", Tag1, "] ", ClientId, "@", Peername, Tns, " msg: ", Msg1, ", ", MetaBin, "\n"];
 format(Event, Config) ->
     emqx_logger_textfmt:format(Event, Config).
 
@@ -79,7 +84,7 @@ format_meta_data(Meta, _Encode) ->
     Meta.
 
 format_meta(Meta0, Encode) ->
-    Meta1 = maps:without([msg, clientid, peername, trace_tag], Meta0),
+    Meta1 = maps:without([msg, tns, clientid, peername, trace_tag], Meta0),
     Meta2 = format_meta_data(Meta1, Encode),
     kvs_to_iolist(lists:sort(fun compare_meta_kvs/2, maps:to_list(Meta2))).
 

apps/emqx/src/emqx_types.erl

@@ -49,6 +49,7 @@
     sockstate/0,
     conninfo/0,
     clientinfo/0,
+    tns/0,
     clientid/0,
     username/0,
     password/0,
@@ -195,6 +196,7 @@
     atom() => term()
 }.
 -type client_attrs() :: #{binary() => binary()}.
+-type tns() :: binary().
 -type clientid() :: binary() | atom().
 -type username() :: option(binary()).
 -type password() :: option(binary()).

apps/emqx/test/emqx_channel_tests.erl

@@ -0,0 +1,110 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_channel_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+set_tns_in_log_meta_test_() ->
+    PdKey = '$logger_metadata$',
+    Original = get(PdKey),
+    Set = fun(Cinfo) ->
+        Ch = emqx_channel:dummy(),
+        Ch1 = emqx_channel:set_field(clientinfo, Cinfo, Ch),
+        emqx_channel:set_log_meta(dummy, Ch1)
+    end,
+    Restore = fun() -> put(PdKey, Original) end,
+    NoTns = #{
+        clientid => <<"id1">>,
+        client_attrs => #{<<"not_tns">> => <<"tns1">>},
+        username => <<"user1">>
+    },
+    NoTnsFn = fun(M) ->
+        ?assertMatch(
+            #{
+                clientid := <<"id1">>,
+                username := <<"user1">>
+            },
+            M
+        ),
+        ?assertNot(maps:is_key(tns, M))
+    end,
+    Prefixed = #{
+        clientid => <<"tns1-id1">>,
+        client_attrs => #{<<"tns">> => <<"tns1">>},
+        username => <<"user2">>
+    },
+    PrefixedFn = fun(M) ->
+        ?assertMatch(
+            #{
+                clientid := <<"tns1-id1">>,
+                username := <<"user2">>
+            },
+            M
+        ),
+        ?assertNot(maps:is_key(tns, M))
+    end,
+
+    Username = #{
+        clientid => <<"id1">>,
+        client_attrs => #{<<"tns">> => <<"user3">>},
+        username => <<"user3">>
+    },
+    UsernameFn =
+        fun(M) ->
+            ?assertMatch(
+                #{
+                    clientid := <<"id1">>,
+                    username := <<"user3">>
+                },
+                M
+            ),
+            ?assertNot(maps:is_key(tns, M))
+        end,
+    TnsAdded = #{
+        clientid => <<"id4">>,
+        client_attrs => #{<<"tns">> => <<"tns1">>},
+        username => <<"user4">>
+    },
+    TnsAddedFn = fun(M) ->
+        ?assertMatch(
+            #{
+                clientid := <<"id4">>,
+                username := <<"user4">>,
+                tns := <<"tns1">>
+            },
+            M
+        )
+    end,
+    Run = fun(Cinfo, CheckFn) ->
+        Set(Cinfo),
+        try
+            CheckFn(get(PdKey))
+        after
+            Restore()
+        end
+    end,
+    MakeTestFn = fun(Cinfo, CheckFn) ->
+        fun() ->
+            Run(Cinfo, CheckFn)
+        end
+    end,
+    [
+        {"tns-added", MakeTestFn(TnsAdded, TnsAddedFn)},
+        {"username as tns", MakeTestFn(Username, UsernameFn)},
+        {"tns prefixed clientid", MakeTestFn(Prefixed, PrefixedFn)},
+        {"no tns", MakeTestFn(NoTns, NoTnsFn)}
+    ].

apps/emqx/test/emqx_trace_formatter_tests.erl

@@ -0,0 +1,52 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+-module(emqx_trace_formatter_tests).
+
+-include_lib("eunit/include/eunit.hrl").
+
+format_no_tns_in_meta_test() ->
+    Meta = #{
+        clientid => <<"c">>,
+        trace_tag => tag
+    },
+    Event = #{
+        level => debug,
+        meta => Meta,
+        msg => <<"test_msg">>
+    },
+    Config = #{payload_encode => hidden},
+    Formatted = format(Event, Config),
+    ?assertMatch(nomatch, re:run(Formatted, "tns:")),
+    ok.
+
+format_tns_in_meta_test() ->
+    Meta = #{
+        tns => <<"a">>,
+        clientid => <<"c">>,
+        trace_tag => tag
+    },
+    Event = #{
+        level => debug,
+        meta => Meta,
+        msg => <<"test_msg">>
+    },
+    Config = #{payload_encode => hidden},
+    Formatted = format(Event, Config),
+    ?assertMatch({match, _}, re:run(Formatted, "\stns:\sa\s")),
+    ok.
+
+format(Event, Config) ->
+    unicode:characters_to_binary(emqx_trace_formatter:format(Event, Config)).

changes/ce/feat-14247.en.md

@@ -0,0 +1 @@
+Write client attribute named `tns` to log messages if such client attribute exists.