feat(tlslib): add separate managed certfiles GC process

Which periodically inpects managed certificates directory and tries
to collect "orphans" here, in other words files that aren't
referenced anywhere in the current emqx config.

apps/emqx/src/emqx_kernel_sup.erl

@@ -37,7 +37,8 @@ init([]) ->
             child_spec(emqx_metrics, worker),
             child_spec(emqx_authn_authz_metrics_sup, supervisor),
             child_spec(emqx_ocsp_cache, worker),
-            child_spec(emqx_crl_cache, worker)
+            child_spec(emqx_crl_cache, worker),
+            child_spec(emqx_tls_lib_sup, supervisor)
         ]
     }}.
 

apps/emqx/src/emqx_tls_certfile_gc.erl

@@ -0,0 +1,335 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+%% @doc Orphaned TLS certificates / keyfiles garbage collector
+%%
+%% This module is a worker process that periodically scans the mutable
+%% certificates directory and removes any files that are not referenced
+%% by _any_ TLS configuration in _any_ of the config roots. Such files
+%% are called "orphans".
+%%
+%% In order to ensure safety, GC considers a file to be a candidate for
+%% deletion (a "convict") only if it was considered an orphan twice in
+%% a row. This should help avoid deleting files that are not yet in the
+%% config but was already materialized on disk (e.g. during
+%% `pre_config_update/3`).
+
+-module(emqx_tls_certfile_gc).
+
+-include_lib("kernel/include/file.hrl").
+-include_lib("emqx/include/logger.hrl").
+-include_lib("snabbkaffe/include/trace.hrl").
+
+%% API
+-export([
+    start_link/0,
+    start_link/1,
+    run/0,
+    force/0
+]).
+
+-behaviour(gen_server).
+-export([
+    init/1,
+    handle_call/3,
+    handle_cast/2,
+    handle_info/2
+]).
+
+%% Testing & maintenance
+-export([
+    orphans/0,
+    orphans/1,
+    convicts/2,
+    collect_files/2
+]).
+
+-define(GC_INVERVAL, 5 * 60 * 1000).
+
+-define(HAS_OWNER_READ(Mode), ((Mode band 8#00400) > 0)).
+-define(HAS_OWNER_WRITE(Mode), ((Mode band 8#00200) > 0)).
+
+-type filename() :: string().
+-type fileinfo() :: #file_info{}.
+
+-type st() :: #{
+    orphans => #{filename() => fileinfo()},
+    gc_interval => pos_integer(),
+    next_gc_timer => reference()
+}.
+
+-type event() ::
+    {collect, filename(), ok | {error, file:posix()}}.
+
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
+
+-spec start_link() ->
+    {ok, pid()}.
+start_link() ->
+    start_link(?GC_INVERVAL).
+
+-spec start_link(_Interval :: pos_integer()) ->
+    {ok, pid()}.
+start_link(Interval) ->
+    gen_server:start_link({local, ?MODULE}, ?MODULE, Interval, []).
+
+-spec run() ->
+    {ok, _Events :: [event()]}.
+run() ->
+    gen_server:call(?MODULE, collect).
+
+-spec force() ->
+    {ok, _Events :: [event()]}.
+force() ->
+    % NOTE
+    % Simulate a complete GC cycle by running it twice. Mostly useful in tests.
+    {ok, Events1} = run(),
+    {ok, Events2} = run(),
+    {ok, Events1 ++ Events2}.
+
+%%--------------------------------------------------------------------
+%% Supervisor callbacks
+%%--------------------------------------------------------------------
+
+-spec init(_) ->
+    {ok, st()}.
+init(Interval) ->
+    {ok, start_timer(#{gc_interval => Interval})}.
+
+-spec handle_call(collect | _Call, gen_server:from(), st()) ->
+    {reply, {ok, [event()]}, st()} | {noreply, st()}.
+handle_call(collect, From, St) ->
+    {ok, Events, StNext} = ?tp_span(
+        tls_certfile_gc_manual,
+        #{caller => From},
+        collect(St, #{evhandler => {fun emqx_utils:cons/2, []}})
+    ),
+    {reply, {ok, Events}, restart_timer(StNext)};
+handle_call(Call, From, St) ->
+    ?SLOG(error, #{msg => "unexpected_call", call => Call, from => From}),
+    {noreply, St}.
+
+-spec handle_cast(_Cast, st()) ->
+    {noreply, st()}.
+handle_cast(Cast, State) ->
+    ?SLOG(error, #{msg => "unexpected_cast", cast => Cast}),
+    {noreply, State}.
+
+-spec handle_info({timeout, reference(), collect}, st()) ->
+    {noreply, st()}.
+handle_info({timeout, TRef, collect}, St = #{next_gc_timer := TRef}) ->
+    {ok, _, StNext} = ?tp_span(
+        tls_certfile_gc_periodic,
+        #{},
+        collect(St, #{evhandler => {fun log_event/2, []}})
+    ),
+    {noreply, start_timer(StNext)}.
+
+start_timer(St = #{gc_interval := Interval}) ->
+    TRef = erlang:start_timer(Interval, self(), collect),
+    St#{next_gc_timer => TRef}.
+
+restart_timer(St = #{next_gc_timer := TRef}) ->
+    ok = emqx_utils:cancel_timer(TRef),
+    start_timer(St).
+
+log_event({collect, Filename, ok}, _) ->
+    ?tp(debug, "tls_certfile_gc_collected", #{filename => Filename});
+log_event({collect, Filename, {error, Reason}}, _) ->
+    ?tp(warning, "tls_certfile_gc_collect_error", #{filename => Filename, reason => Reason}).
+
+%%--------------------------------------------------------------------
+%% Internal functions
+%% -------------------------------------------------------------------
+
+collect(St, Opts) ->
+    RootDir = emqx_utils_fs:canonicalize(emqx:mutable_certs_dir()),
+    Orphans = orphans(RootDir),
+    OrphansLast = maps:get(orphans, St, #{}),
+    Convicts = convicts(Orphans, OrphansLast),
+    Result = collect_files(Convicts, RootDir, Opts),
+    {ok, Result, St#{orphans => maps:without(Convicts, Orphans)}}.
+
+orphans() ->
+    Dir = emqx_utils_fs:canonicalize(emqx:mutable_certs_dir()),
+    orphans(Dir).
+
+orphans(Dir) ->
+    Certfiles = find_managed_files(fun is_managed_file/2, Dir),
+    lists:foldl(
+        fun(Root, Acc) ->
+            References = find_references(Root),
+            maps:without(References, Acc)
+        end,
+        Certfiles,
+        emqx_config:get_root_names()
+    ).
+
+convicts(Orphans, OrphansLast) ->
+    maps:fold(
+        fun(AbsPath, #file_info{mtime = MTime, inode = Inode}, Acc) ->
+            case maps:get(AbsPath, Orphans, undefined) of
+                #file_info{mtime = MTime, inode = Inode} ->
+                    % Certfile was not changed / recreated in the meantime
+                    [AbsPath | Acc];
+                _ ->
+                    % Certfile was changed / created / recreated in the meantime
+                    Acc
+            end
+        end,
+        [],
+        OrphansLast
+    ).
+
+find_managed_files(Filter, Dir) ->
+    emqx_utils_fs:traverse_dir(
+        fun
+            (AbsPath, Info = #file_info{}, Acc) ->
+                case Filter(AbsPath, Info) of
+                    true -> Acc#{AbsPath => Info};
+                    false -> Acc
+                end;
+            (AbsPath, {error, Reason}, Acc) ->
+                ?SLOG(notice, "filesystem_object_inaccessible", #{
+                    abspath => AbsPath,
+                    reason => Reason
+                }),
+                Acc
+        end,
+        #{},
+        Dir
+    ).
+
+is_managed_file(AbsPath, #file_info{type = Type, mode = Mode}) ->
+    % NOTE
+    % We consider a certfile is managed if: this is a regular file, owner has RW permission
+    % and the filename looks like a managed filename.
+    Type == regular andalso
+        ?HAS_OWNER_READ(Mode) andalso
+        ?HAS_OWNER_WRITE(Mode) andalso
+        emqx_tls_lib:is_managed_ssl_file(AbsPath).
+
+find_references(Root) ->
+    Config = emqx_config:get_raw([Root]),
+    fold_config(
+        fun(Stack, Value, Acc) ->
+            case is_file_reference(Stack) andalso is_string(Value) of
+                true ->
+                    Filename = emqx_schema:naive_env_interpolation(Value),
+                    {stop, [emqx_utils_fs:canonicalize(Filename) | Acc]};
+                false ->
+                    {cont, Acc}
+            end
+        end,
+        [],
+        Config
+    ).
+
+is_file_reference([<<"keyfile">> | _]) -> true;
+is_file_reference([<<"certfile">> | _]) -> true;
+is_file_reference([<<"cacertfile">> | _]) -> true;
+is_file_reference([<<"issuer_pem">>, <<"ocsp">> | _]) -> true;
+is_file_reference(_) -> false.
+
+is_string(Value) ->
+    is_list(Value) orelse is_binary(Value).
+
+%%
+
+fold_config(FoldFun, AccIn, Config) ->
+    fold_config(FoldFun, AccIn, [], Config).
+
+fold_config(FoldFun, AccIn, Stack, Config) when is_map(Config) ->
+    maps:fold(
+        fun(K, SubConfig, Acc) ->
+            fold_subconf(FoldFun, Acc, [K | Stack], SubConfig)
+        end,
+        AccIn,
+        Config
+    );
+fold_config(FoldFun, Acc, Stack, []) ->
+    fold_confval(FoldFun, Acc, Stack, []);
+fold_config(FoldFun, Acc, Stack, String = [C | _]) when is_integer(C), C >= 0, C < 16#10FFFF ->
+    fold_confval(FoldFun, Acc, Stack, String);
+fold_config(FoldFun, Acc, Stack, Config) when is_list(Config) ->
+    fold_confarray(FoldFun, Acc, Stack, 1, Config);
+fold_config(FoldFun, Acc, Stack, Config) ->
+    fold_confval(FoldFun, Acc, Stack, Config).
+
+fold_confarray(FoldFun, AccIn, StackIn, I, [H | T]) ->
+    Stack = [I | StackIn],
+    case FoldFun(Stack, H, AccIn) of
+        {cont, Acc} ->
+            AccOut = fold_config(FoldFun, Acc, Stack, H),
+            fold_confarray(FoldFun, AccOut, StackIn, I + 1, T);
+        {stop, Acc} ->
+            fold_confarray(FoldFun, Acc, StackIn, I + 1, T)
+    end;
+fold_confarray(_FoldFun, Acc, _Stack, _, []) ->
+    Acc.
+
+fold_subconf(FoldFun, AccIn, Stack, SubConfig) ->
+    case FoldFun(Stack, SubConfig, AccIn) of
+        {cont, Acc} ->
+            fold_config(FoldFun, Acc, Stack, SubConfig);
+        {stop, Acc} ->
+            Acc
+    end.
+
+fold_confval(FoldFun, AccIn, Stack, ConfVal) ->
+    case FoldFun(Stack, ConfVal, AccIn) of
+        {_, Acc} ->
+            Acc
+    end.
+
+%%
+
+-spec collect_files([filename()], filename()) ->
+    [event()].
+collect_files(Filenames, RootDir) ->
+    collect_files(Filenames, RootDir, #{evhandler => {fun emqx_utils:cons/2, []}}).
+
+collect_files(Filenames, RootDir, Opts) ->
+    {Handler, AccIn} = maps:get(evhandler, Opts),
+    lists:foldl(
+        fun(Filename, Acc) -> collect_file(Filename, RootDir, Handler, Acc) end,
+        AccIn,
+        Filenames
+    ).
+
+collect_file(Filename, RootDir, Handler, AccIn) ->
+    case file:delete(Filename) of
+        ok ->
+            Acc = Handler({collect, Filename, ok}, AccIn),
+            collect_parents(filename:dirname(Filename), RootDir, Handler, Acc);
+        {error, _} = Error ->
+            Handler({collect, Filename, Error}, AccIn)
+    end.
+
+collect_parents(RootDir, RootDir, _Handler, Acc) ->
+    Acc;
+collect_parents(ParentDir, RootDir, Handler, AccIn) ->
+    case file:del_dir(ParentDir) of
+        ok ->
+            Acc = Handler({collect, ParentDir, ok}, AccIn),
+            collect_parents(filename:dirname(ParentDir), RootDir, Handler, Acc);
+        {error, eexist} ->
+            AccIn;
+        {error, _} = Error ->
+            Handler({collect, ParentDir, Error}, AccIn)
+    end.

apps/emqx/src/emqx_tls_lib.erl

@@ -32,6 +32,7 @@
     ensure_ssl_files/3,
     drop_invalid_certs/1,
     pem_dir/1,
+    is_managed_ssl_file/1,
     is_valid_pem_file/1,
     is_pem/1
 ]).
@@ -400,7 +401,7 @@ save_pem_file(Dir, KeyPath, Pem, DryRun) ->
 %% the filename is prefixed by the option name without the 'file' part
 %% and suffixed with the first 8 byets the PEM content's md5 checksum.
 %% e.g. key-1234567890abcdef, cert-1234567890abcdef, and cacert-1234567890abcdef
-is_generated_file(Filename) ->
+is_managed_ssl_file(Filename) ->
     case string:split(filename:basename(Filename), "-") of
         [_Name, Suffix] -> is_hex_str(Suffix);
         _ -> false

apps/emqx/src/emqx_tls_lib_sup.erl

@@ -0,0 +1,49 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_tls_lib_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/0]).
+
+-export([init/1]).
+
+%%--------------------------------------------------------------------
+%% API
+%%--------------------------------------------------------------------
+
+start_link() ->
+    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+%%--------------------------------------------------------------------
+%% Supervisor callbacks
+%%--------------------------------------------------------------------
+
+init([]) ->
+    SupFlags = #{
+        strategy => one_for_one,
+        intensity => 100,
+        period => 10
+    },
+    GC = #{
+        id => emqx_tls_certfile_gc,
+        start => {emqx_tls_certfile_gc, start_link, []},
+        restart => permanent,
+        shutdown => brutal_kill,
+        type => worker
+    },
+    {ok, {SupFlags, [GC]}}.

apps/emqx/test/emqx_tls_certfile_gc_SUITE.erl

@@ -0,0 +1,430 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_tls_certfile_gc_SUITE).
+
+-compile([export_all, nowarn_export_all]).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("common_test/include/ct.hrl").
+
+-include_lib("snabbkaffe/include/test_macros.hrl").
+
+-define(of_pid(PID, EVENTS), [E || #{?snk_meta := #{pid := __Pid}} = E <- EVENTS, __Pid == PID]).
+
+-import(hoconsc, [mk/1, mk/2, ref/2]).
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+init_per_suite(Config) ->
+    _ = application:load(emqx),
+    ok = application:set_env(emqx, data_dir, ?config(priv_dir, Config)),
+    ok = emqx_config:save_schema_mod_and_names(?MODULE),
+    Config.
+
+end_per_suite(_Config) ->
+    emqx_config:erase_all().
+
+init_per_testcase(TC, Config) ->
+    TCAbsDir = filename:join(?config(priv_dir, Config), TC),
+    ok = application:set_env(emqx, data_dir, TCAbsDir),
+    ok = snabbkaffe:start_trace(),
+    [{tc_name, TC}, {tc_absdir, TCAbsDir} | Config].
+
+end_per_testcase(_TC, Config) ->
+    ok = snabbkaffe:stop(),
+    ok = application:set_env(emqx, data_dir, ?config(priv_dir, Config)),
+    ok.
+
+t_no_orphans(Config) ->
+    SSL0 = #{
+        <<"keyfile">> => key(),
+        <<"certfile">> => cert(),
+        <<"cacertfile">> => cert()
+    },
+    {ok, SSL} = emqx_tls_lib:ensure_ssl_files("ssl", SSL0),
+    {ok, SSLUnused} = emqx_tls_lib:ensure_ssl_files("unused", SSL0),
+    SSLKeyfile = maps:get(<<"keyfile">>, SSL),
+    ok = load_config(#{
+        <<"clients">> => [
+            #{<<"transport">> => #{<<"ssl">> => SSL}}
+        ],
+        <<"servers">> => #{
+            <<"noname">> => #{<<"ssl">> => SSL}
+        }
+    }),
+    % Should not be considered an orphan: it's a symlink.
+    ok = file:make_symlink(
+        SSLKeyfile,
+        filename:join(?config(tc_absdir, Config), filename:basename(SSLKeyfile))
+    ),
+    % Should not be considered orphans: files are now read-only.
+    ok = file:change_mode(maps:get(<<"keyfile">>, SSLUnused), 8#400),
+    ok = file:change_mode(maps:get(<<"certfile">>, SSLUnused), 8#400),
+    ok = file:change_mode(maps:get(<<"cacertfile">>, SSLUnused), 8#400),
+    % Verify there are no orphans
+    ?assertEqual(
+        #{},
+        orphans()
+    ),
+    % Verify there are no orphans, since SSL config is still in use
+    ok = put_config([<<"servers">>, <<"noname">>, <<"ssl">>], #{<<"enable">> => false}),
+    ?assertEqual(
+        #{},
+        orphans()
+    ).
+
+t_collect_orphans(_Config) ->
+    % 0. Set up a client and two servers (each with the same set of certfiles).
+    SSL0 = #{
+        <<"keyfile">> => key(),
+        <<"certfile">> => cert(),
+        <<"cacertfile">> => cert()
+    },
+    SSL1 = SSL0#{
+        <<"ocsp">> => #{<<"issuer_pem">> => cert()}
+    },
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files("client", SSL0),
+    {ok, SSL3} = emqx_tls_lib:ensure_ssl_files("server", SSL1),
+    ok = load_config(#{
+        <<"clients">> => [
+            #{<<"transport">> => #{<<"ssl">> => SSL2}}
+        ],
+        <<"servers">> => #{
+            <<"name">> => #{<<"ssl">> => SSL3},
+            <<"noname">> => #{<<"ssl">> => SSL3}
+        }
+    }),
+    Orphans1 = orphans(),
+    ?assertEqual(
+        #{},
+        Orphans1
+    ),
+    % 1. Remove clients from the config
+    ok = put_config([<<"clients">>], []),
+    Orphans2 = orphans(),
+    ?assertMatch(
+        M = #{} when map_size(M) == 3,
+        Orphans2
+    ),
+    % All orphans are newly observed, nothing to collect
+    ?assertEqual(
+        [],
+        collect(convicts(Orphans2, Orphans1))
+    ),
+    % Same orphans are "observed", should be collected
+    ?assertMatch(
+        [
+            {collect, _DirClient, ok},
+            {collect, _CACert, ok},
+            {collect, _Cert, ok},
+            {collect, _Key, ok}
+        ],
+        collect(convicts(Orphans2, Orphans2))
+    ),
+    % 2. Remove server from the config
+    ok = put_config([<<"servers">>, <<"name">>, <<"ssl">>], #{}),
+    Orphans3 = orphans(),
+    % Files are still referenced by the "noname" server
+    ?assertEqual(
+        #{},
+        Orphans3
+    ),
+    % 3. Remove another server from the config
+    ok = put_config([<<"servers">>, <<"noname">>, <<"ssl">>], #{}),
+    Orphans4 = orphans(),
+    ?assertMatch(
+        M = #{} when map_size(M) == 4,
+        Orphans4
+    ),
+    ?assertMatch(
+        [
+            {collect, _DirServer, ok},
+            {collect, _IssuerPEM, ok},
+            {collect, _CACert, ok},
+            {collect, _Cert, ok},
+            {collect, _Key, ok}
+        ],
+        collect(convicts(Orphans4, Orphans4))
+    ),
+    % No more orphans left
+    ?assertEqual(
+        #{},
+        orphans()
+    ).
+
+t_gc_runs_periodically(_Config) ->
+    {ok, Pid} = emqx_tls_certfile_gc:start_link(500),
+
+    % Set up two servers in the config, each with its own set of certfiles
+    SSL = #{
+        <<"keyfile">> => key(),
+        <<"certfile">> => cert()
+    },
+    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files("s1", SSL),
+    SSL1Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL1)),
+    SSL1Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL1)),
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files("s2", SSL#{
+        <<"ocsp">> => #{<<"issuer_pem">> => cert()}
+    }),
+    SSL2Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL2)),
+    SSL2Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL2)),
+    SSL2IssPEM = emqx_utils_fs:canonicalize(
+        emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2)
+    ),
+    ok = load_config(#{
+        <<"servers">> => #{
+            <<"name">> => #{<<"ssl">> => SSL1},
+            <<"noname">> => #{<<"ssl">> => SSL2}
+        }
+    }),
+
+    % Wait for a periodic collection event
+    ?check_trace(
+        ?block_until(#{?snk_kind := tls_certfile_gc_periodic, ?snk_span := {complete, _}}),
+        fun(Events) ->
+            % Nothing should have been collected yet
+            ?assertMatch(
+                [
+                    #{?snk_kind := tls_certfile_gc_periodic, ?snk_span := start},
+                    #{?snk_kind := tls_certfile_gc_periodic, ?snk_span := {complete, _}}
+                ],
+                ?of_pid(Pid, Events)
+            )
+        end
+    ),
+
+    % Delete the server from the config
+    ok = put_config([<<"servers">>, <<"noname">>, <<"ssl">>], #{}),
+
+    % Wait for a periodic collection event
+    ?check_trace(
+        ?block_until(#{?snk_kind := tls_certfile_gc_periodic, ?snk_span := {complete, _}}),
+        fun(Events) ->
+            % Nothing should have been collected yet, certfiles considered orphans for the first time
+            ?assertMatch(
+                [
+                    #{?snk_kind := tls_certfile_gc_periodic, ?snk_span := start},
+                    #{?snk_kind := tls_certfile_gc_periodic, ?snk_span := {complete, _}}
+                ],
+                ?of_pid(Pid, Events)
+            )
+        end
+    ),
+
+    % Delete another server from the config
+    ok = put_config([<<"servers">>, <<"name">>, <<"ssl">>], #{}),
+
+    % Wait for next periodic collection event
+    ?check_trace(
+        ?block_until(#{?snk_kind := tls_certfile_gc_periodic, ?snk_span := {complete, _}}),
+        fun(Events) ->
+            % SSL2 certfiles should have been collected now, but not SSL1
+            ?assertMatch(
+                [
+                    #{?snk_kind := tls_certfile_gc_periodic, ?snk_span := start},
+                    #{?snk_kind := "tls_certfile_gc_collected", filename := SSL2IssPEM},
+                    #{?snk_kind := "tls_certfile_gc_collected", filename := SSL2Keyfile},
+                    #{?snk_kind := "tls_certfile_gc_collected", filename := SSL2Certfile},
+                    #{?snk_kind := "tls_certfile_gc_collected", filename := _SSL2Dir},
+                    #{?snk_kind := tls_certfile_gc_periodic, ?snk_span := {complete, _}}
+                ],
+                ?of_pid(Pid, Events)
+            )
+        end
+    ),
+
+    % Wait for next periodic collection event
+    ?check_trace(
+        ?block_until(#{?snk_kind := tls_certfile_gc_periodic, ?snk_span := {complete, _}}),
+        fun(Events) ->
+            % SSL1 certfiles should have been collected finally, they were considered orphans before
+            ?assertMatch(
+                [
+                    #{?snk_kind := tls_certfile_gc_periodic, ?snk_span := start},
+                    #{?snk_kind := "tls_certfile_gc_collected", filename := SSL1Keyfile},
+                    #{?snk_kind := "tls_certfile_gc_collected", filename := SSL1Certfile},
+                    #{?snk_kind := "tls_certfile_gc_collected", filename := _SSL1Dir},
+                    #{?snk_kind := tls_certfile_gc_periodic, ?snk_span := {complete, _}}
+                ],
+                ?of_pid(Pid, Events)
+            )
+        end
+    ),
+
+    ok = proc_lib:stop(Pid).
+
+t_gc_spares_recreated_certfiles(_Config) ->
+    {ok, Pid} = emqx_tls_certfile_gc:start_link(),
+
+    % Create two sets of certfiles, with no references to them
+    SSL = #{
+        <<"keyfile">> => key(),
+        <<"certfile">> => cert()
+    },
+    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files("s1", SSL),
+    SSL1Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL1)),
+    SSL1Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL1)),
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files("s2", SSL),
+    SSL2Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL2)),
+    SSL2Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL2)),
+    ok = load_config(#{}),
+
+    % Nothing should have been collected yet
+    ?assertMatch(
+        {ok, []},
+        emqx_tls_certfile_gc:run()
+    ),
+
+    % At least one second should pass, since mtime has second resolution
+    ok = timer:sleep(1000),
+    ok = file:change_time(SSL2Keyfile, erlang:localtime()),
+    ok = file:change_time(SSL2Certfile, erlang:localtime()),
+    % Only SSL1 certfiles should have been collected
+    ?assertMatch(
+        {ok, [
+            {collect, _SSL1Dir, ok},
+            {collect, SSL1Certfile, ok},
+            {collect, SSL1Keyfile, ok}
+        ]},
+        emqx_tls_certfile_gc:run()
+    ),
+
+    % Recreate the SSL2 certfiles
+    ok = file:delete(SSL2Keyfile),
+    ok = file:delete(SSL2Certfile),
+    {ok, _} = emqx_tls_lib:ensure_ssl_files("s2", SSL),
+    % Nothing should have been collected
+    ?assertMatch(
+        {ok, []},
+        emqx_tls_certfile_gc:run()
+    ),
+
+    ok = proc_lib:stop(Pid).
+
+t_gc_active(_Config) ->
+    ok = emqx_common_test_helpers:boot_modules([]),
+    ok = emqx_common_test_helpers:start_apps([]),
+    try
+        ?assertEqual(
+            {ok, []},
+            emqx_tls_certfile_gc:run()
+        )
+    after
+        emqx_common_test_helpers:stop_apps([]),
+        emqx_common_test_helpers:boot_modules(all)
+    end.
+
+orphans() ->
+    emqx_tls_certfile_gc:orphans(emqx:mutable_certs_dir()).
+
+convicts(Orphans, OrphansLast) ->
+    emqx_tls_certfile_gc:convicts(Orphans, OrphansLast).
+
+collect(Convicts) ->
+    emqx_tls_certfile_gc:collect_files(Convicts, emqx:mutable_certs_dir()).
+
+load_config(Config) ->
+    emqx_config:init_load(
+        ?MODULE,
+        emqx_utils_json:encode(#{<<?MODULE_STRING>> => Config})
+    ).
+
+put_config(Path, SubConfig) ->
+    emqx_config:put_raw([<<?MODULE_STRING>> | Path], SubConfig).
+
+cert() ->
+    <<
+        "-----BEGIN CERTIFICATE-----\n"
+        "MIIFljCCA36gAwIBAgICEAEwDQYJKoZIhvcNAQELBQAwazELMAkGA1UEBhMCU0Ux\n"
+        "EjAQBgNVBAgMCVN0b2NraG9sbTESMBAGA1UECgwJTXlPcmdOYW1lMRkwFwYDVQQL\n"
+        "DBBNeUludGVybWVkaWF0ZUNBMRkwFwYDVQQDDBBNeUludGVybWVkaWF0ZUNBMB4X\n"
+        "DTIzMDExMjEzMDgxNloXDTMzMDQxOTEzMDgxNlowdzELMAkGA1UEBhMCU0UxEjAQ\n"
+        "BgNVBAgMCVN0b2NraG9sbTESMBAGA1UEBwwJU3RvY2tob2xtMRIwEAYDVQQKDAlN\n"
+        "eU9yZ05hbWUxGTAXBgNVBAsMEE15SW50ZXJtZWRpYXRlQ0ExETAPBgNVBAMMCE15\n"
+        "Q2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAvGuAShewEo8V\n"
+        "/+aWVO/MuUt92m8K0Ut4nC2gOvpjMjf8mhSSf6KfnxPklsFwP4fdyPOjOiXwCsf3\n"
+        "1QO5fjVr8to3iGTHhEyZpzRcRqmw1eYJC7iDh3BqtYLAT30R+Kq6Mk+f4tXB5Lp/\n"
+        "2jXgdi0wshWagCPgJO3CtiwGyE8XSa+Q6EBYwzgh3NFbgYdJma4x+S86Y/5WfmXP\n"
+        "zF//UipsFp4gFUqwGuj6kJrN9NnA1xCiuOxCyN4JuFNMfM/tkeh26jAp0OHhJGsT\n"
+        "s3YiUm9Dpt7Rs7o0so9ov9K+hgDFuQw9HZW3WIJI99M5a9QZ4ZEQqKpABtYBl/Nb\n"
+        "VPXcr+T3fQIDAQABo4IBNjCCATIwCQYDVR0TBAIwADARBglghkgBhvhCAQEEBAMC\n"
+        "BaAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2VuZXJhdGVkIENsaWVudCBDZXJ0\n"
+        "aWZpY2F0ZTAdBgNVHQ4EFgQUOIChBA5aZB0dPWEtALfMIfSopIIwHwYDVR0jBBgw\n"
+        "FoAUTHCGOxVSibq1D5KqrrExQRO+c/MwDgYDVR0PAQH/BAQDAgXgMB0GA1UdJQQW\n"
+        "MBQGCCsGAQUFBwMCBggrBgEFBQcDBDA7BgNVHR8ENDAyMDCgLqAshipodHRwOi8v\n"
+        "bG9jYWxob3N0Ojk4NzgvaW50ZXJtZWRpYXRlLmNybC5wZW0wMQYIKwYBBQUHAQEE\n"
+        "JTAjMCEGCCsGAQUFBzABhhVodHRwOi8vbG9jYWxob3N0Ojk4NzcwDQYJKoZIhvcN\n"
+        "AQELBQADggIBAE0qTL5WIWcxRPU9oTrzJ+oxMTp1JZ7oQdS+ZekLkQ8mP7T6C/Ew\n"
+        "6YftjvkopnHUvn842+PTRXSoEtlFiTccmA60eMAai2tn5asxWBsLIRC9FH3LzOgV\n"
+        "/jgyY7HXuh8XyDBCDD+Sj9QityO+accTHijYAbHPAVBwmZU8nO5D/HsxLjRrCfQf\n"
+        "qf4OQpX3l1ryOi19lqoRXRGwcoZ95dqq3YgTMlLiEqmerQZSR6iSPELw3bcwnAV1\n"
+        "hoYYzeKps3xhwszCTz2+WaSsUO2sQlcFEsZ9oHex/02UiM4a8W6hGFJl5eojErxH\n"
+        "7MqaSyhwwyX6yt8c75RlNcUThv+4+TLkUTbTnWgC9sFjYfd5KSfAdIMp3jYzw3zw\n"
+        "XEMTX5FaLaOCAfUDttPzn+oNezWZ2UyFTQXQE2CazpRdJoDd04qVg9WLpQxLYRP7\n"
+        "xSFEHulOPccdAYF2C45yNtJAZyWKfGaAZIxrgEXbMkcdDMlYphpRwpjS8SIBNZ31\n"
+        "KFE8BczKrg2qO0ywIjanPaRgrFVmeSvBKeU/YLQVx6fZMgOk6vtidLGZLyDXy0Ff\n"
+        "yaZSoj+on++RDz1IXb96Y8scuNlfcYI8QeoNjwiLtf80BV8SRJiG4e/jTvMf/z9L\n"
+        "kWrnDWvx4xkUmxFg4TK42dkNp7sEYBTlVVq9fjKE92ha7FGZRqsxOLNQ\n"
+        "-----END CERTIFICATE-----\n"
+    >>.
+
+key() ->
+    <<
+        "-----BEGIN EC PRIVATE KEY-----\n"
+        "MHQCAQEEICKTbbathzvD8zvgjL7qRHhW4alS0+j0Loo7WeYX9AxaoAcGBSuBBAAK\n"
+        "oUQDQgAEJBdF7MIdam5T4YF3JkEyaPKdG64TVWCHwr/plC0QzNVJ67efXwxlVGTo\n"
+        "ju0VBj6tOX1y6C0U+85VOM0UU5xqvw==\n"
+        "-----END EC PRIVATE KEY-----\n"
+    >>.
+
+%%--------------------------------------------------------------------
+%% Schema
+%% -------------------------------------------------------------------
+
+roots() ->
+    [?MODULE].
+
+namespace() ->
+    "ct".
+
+fields(?MODULE) ->
+    [
+        {servers, mk(hoconsc:map(string(), ref(?MODULE, server)))},
+        {clients, mk(hoconsc:array(ref(?MODULE, client)))}
+    ];
+fields(server) ->
+    [
+        {ssl, mk(ref(emqx_schema, "listener_ssl_opts"))}
+    ];
+fields(client) ->
+    [
+        {transport,
+            mk(
+                ref(?MODULE, transport),
+                #{required => false}
+            )}
+    ];
+fields(transport) ->
+    [
+        {ssl,
+            mk(
+                ref(emqx_schema, "ssl_client_opts"),
+                #{default => #{<<"enable">> => false}}
+            )}
+    ].

apps/emqx_bridge/src/emqx_bridge_resource.erl

@@ -247,18 +247,16 @@ recreate(Type, Name, Conf, Opts) ->
     ).
 
 create_dry_run(Type, Conf0) ->
-    TmpPath = emqx_utils:safe_filename(?TEST_ID_PREFIX ++ emqx_utils:gen_id(8)),
+    TmpName = iolist_to_binary([?TEST_ID_PREFIX, emqx_utils:gen_id(8)]),
+    TmpPath = emqx_utils:safe_filename(TmpName),
     Conf = emqx_utils_maps:safe_atom_key_map(Conf0),
     case emqx_connector_ssl:convert_certs(TmpPath, Conf) of
         {error, Reason} ->
             {error, Reason};
         {ok, ConfNew} ->
             try
-                ParseConf = parse_confs(bin(Type), TmpPath, ConfNew),
-                Res = emqx_resource:create_dry_run_local(
-                    bridge_to_resource_type(Type), ParseConf
-                ),
-                Res
+                ParseConf = parse_confs(bin(Type), TmpName, ConfNew),
+                emqx_resource:create_dry_run_local(bridge_to_resource_type(Type), ParseConf)
             catch
                 %% validation errors
                 throw:Reason ->

apps/emqx_bridge/test/emqx_bridge_SUITE.erl

@@ -156,6 +156,7 @@ setup_fake_telemetry_data() ->
 
 t_update_ssl_conf(Config) ->
     Path = proplists:get_value(config_path, Config),
+    CertDir = filename:join([emqx:mutable_certs_dir() | Path]),
     EnableSSLConf = #{
         <<"bridge_mode">> => false,
         <<"clean_start">> => true,
@@ -172,22 +173,13 @@ t_update_ssl_conf(Config) ->
             }
     },
     {ok, _} = emqx:update_config(Path, EnableSSLConf),
-    {ok, Certs} = list_pem_dir(Path),
-    ?assertMatch([_, _, _], Certs),
+    ?assertMatch({ok, [_, _, _]}, file:list_dir(CertDir)),
     NoSSLConf = EnableSSLConf#{<<"ssl">> := #{<<"enable">> => false}},
     {ok, _} = emqx:update_config(Path, NoSSLConf),
-    ?assertMatch({error, not_dir}, list_pem_dir(Path)),
+    {ok, _} = emqx_tls_certfile_gc:force(),
+    ?assertMatch({error, enoent}, file:list_dir(CertDir)),
     ok.
 
-list_pem_dir(Path) ->
-    Dir = filename:join([emqx:mutable_certs_dir() | Path]),
-    case filelib:is_dir(Dir) of
-        true ->
-            file:list_dir(Dir);
-        _ ->
-            {error, not_dir}
-    end.
-
 data_file(Name) ->
     Dir = code:lib_dir(emqx_bridge, test),
     {ok, Bin} = file:read_file(filename:join([Dir, "data", Name])),

apps/emqx_exhook/test/emqx_exhook_SUITE.erl

@@ -360,13 +360,10 @@ t_stop_timeout(_) ->
 t_ssl_clear(_) ->
     SvrName = <<"ssl_test">>,
     SSLConf = #{
+        <<"enable">> => true,
         <<"cacertfile">> => cert_file("cafile"),
-
         <<"certfile">> => cert_file("certfile"),
-
-        <<"enable">> => true,
         <<"keyfile">> => cert_file("keyfile"),
-
         <<"verify">> => <<"verify_peer">>
     },
     AddConf = #{
@@ -377,7 +374,6 @@ t_ssl_clear(_) ->
         <<"pool_size">> => 16,
         <<"request_timeout">> => <<"5s">>,
         <<"ssl">> => SSLConf,
-
         <<"url">> => <<"http://127.0.0.1:9000">>
     },
     emqx_exhook_mgr:update_config([exhook, servers], {add, AddConf}),
@@ -387,6 +383,7 @@ t_ssl_clear(_) ->
 
     UpdateConf = AddConf#{<<"ssl">> => SSLConf#{<<"keyfile">> => cert_file("keyfile2")}},
     emqx_exhook_mgr:update_config([exhook, servers], {update, SvrName, UpdateConf}),
+    {ok, _} = emqx_tls_certfile_gc:force(),
     ListResult2 = list_pem_dir(SvrName),
     ?assertMatch({ok, [_, _, _]}, ListResult2),
     {ok, ResultList2} = ListResult2,
@@ -403,7 +400,8 @@ t_ssl_clear(_) ->
     ?assertNotEqual(FindKeyFile(ResultList1), FindKeyFile(ResultList2)),
 
     emqx_exhook_mgr:update_config([exhook, servers], {delete, SvrName}),
-    ?assertMatch({error, not_dir}, list_pem_dir(SvrName)),
+    {ok, _} = emqx_tls_certfile_gc:force(),
+    ?assertMatch({error, enoent}, list_pem_dir(SvrName)),
     ok.
 
 %%--------------------------------------------------------------------
@@ -475,12 +473,7 @@ is_exhook_callback(Cb) ->
 
 list_pem_dir(Name) ->
     Dir = filename:join([emqx:mutable_certs_dir(), "exhook", Name]),
-    case filelib:is_dir(Dir) of
-        true ->
-            file:list_dir(Dir);
-        _ ->
-            {error, not_dir}
-    end.
+    file:list_dir(Dir).
 
 data_file(Name) ->
     Dir = code:lib_dir(emqx_exhook, test),

apps/emqx_gateway/test/emqx_gateway_conf_SUITE.erl

@@ -495,6 +495,7 @@ t_add_listener_with_certs_content(_) ->
     ok.
 
 assert_ssl_confs_files_deleted(SslConf) when is_map(SslConf) ->
+    {ok, _} = emqx_tls_certfile_gc:force(),
     Ks = [<<"cacertfile">>, <<"certfile">>, <<"keyfile">>],
     lists:foreach(
         fun(K) ->

apps/emqx_management/test/emqx_mgmt_api_listeners_SUITE.erl

@@ -237,6 +237,7 @@ t_clear_certs(Config) when is_list(Config) ->
         [<<"ssl_options">>, <<"keyfile">>], NewConf2, cert_file("keyfile2")
     ),
     _ = request(put, NewPath, [], UpdateConf),
+    _ = emqx_tls_certfile_gc:force(),
     ListResult2 = list_pem_dir("ssl", "clear"),
 
     %% make sure the old cret file is deleted
@@ -259,7 +260,8 @@ t_clear_certs(Config) when is_list(Config) ->
 
     %% remove, check all cert files are deleted
     _ = delete(NewPath),
-    ?assertMatch({error, not_dir}, list_pem_dir("ssl", "clear")),
+    _ = emqx_tls_certfile_gc:force(),
+    ?assertMatch({error, enoent}, list_pem_dir("ssl", "clear")),
     ok.
 
 get_tcp_listeners(Node) ->
@@ -431,12 +433,7 @@ is_running(Id) ->
 list_pem_dir(Type, Name) ->
     ListenerDir = emqx_listeners:certs_dir(Type, Name),
     Dir = filename:join([emqx:mutable_certs_dir(), ListenerDir]),
-    case filelib:is_dir(Dir) of
-        true ->
-            file:list_dir(Dir);
-        _ ->
-            {error, not_dir}
-    end.
+    file:list_dir(Dir).
 
 data_file(Name) ->
     Dir = code:lib_dir(emqx, test),

apps/emqx_utils/src/emqx_utils.erl

@@ -25,6 +25,7 @@
     maybe_apply/2,
     compose/1,
     compose/2,
+    cons/2,
     run_fold/3,
     pipeline/3,
     start_timer/2,
@@ -135,6 +136,10 @@ compose(F, G) when is_function(G) -> fun(X) -> G(F(X)) end;
 compose(F, [G]) -> compose(F, G);
 compose(F, [G | More]) -> compose(compose(F, G), More).
 
+-spec cons(X, [X]) -> [X, ...].
+cons(Head, Tail) ->
+    [Head | Tail].
+
 %% @doc RunFold
 run_fold([], Acc, _State) ->
     Acc;

apps/emqx_utils/src/emqx_utils_fs.erl

@@ -0,0 +1,80 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_utils_fs).
+
+-include_lib("kernel/include/file.hrl").
+
+-export([traverse_dir/3]).
+-export([canonicalize/1]).
+
+-type fileinfo() :: #file_info{}.
+-type foldfun(Acc) ::
+    fun((_Filepath :: file:name(), fileinfo() | {error, file:posix()}, Acc) -> Acc).
+
+%% @doc Traverse a directory recursively and apply a fold function to each file.
+%%
+%% This is a safer version of `filelib:fold_files/5` which does not follow symlinks
+%% and reports errors to the fold function, giving the user more control over the
+%% traversal.
+%% It's not an error if `Dirpath` is not a directory, in which case the fold function
+%% will be called once with the file info of `Dirpath`.
+-spec traverse_dir(foldfun(Acc), Acc, _Dirpath :: file:name()) ->
+    Acc.
+traverse_dir(FoldFun, Acc, Dirpath) ->
+    traverse_dir(FoldFun, Acc, Dirpath, read_info(Dirpath)).
+
+traverse_dir(FoldFun, AccIn, DirPath, {ok, #file_info{type = directory}}) ->
+    case file:list_dir(DirPath) of
+        {ok, Filenames} ->
+            lists:foldl(
+                fun(Filename, Acc) ->
+                    AbsPath = filename:join(DirPath, Filename),
+                    traverse_dir(FoldFun, Acc, AbsPath)
+                end,
+                AccIn,
+                Filenames
+            );
+        {error, Reason} ->
+            FoldFun(DirPath, {error, Reason}, AccIn)
+    end;
+traverse_dir(FoldFun, Acc, AbsPath, {ok, Info}) ->
+    FoldFun(AbsPath, Info, Acc);
+traverse_dir(FoldFun, Acc, AbsPath, {error, Reason}) ->
+    FoldFun(AbsPath, {error, Reason}, Acc).
+
+read_info(AbsPath) ->
+    file:read_link_info(AbsPath, [{time, posix}, raw]).
+
+%% @doc Canonicalize a file path.
+%% Removes stray slashes and converts to a string.
+-spec canonicalize(file:name()) ->
+    string().
+canonicalize(Filename) ->
+    case filename:split(str(Filename)) of
+        Components = [_ | _] ->
+            filename:join(Components);
+        [] ->
+            ""
+    end.
+
+str(Value) ->
+    case unicode:characters_to_list(Value, unicode) of
+        Str when is_list(Str) ->
+            Str;
+        {error, _, _} ->
+            erlang:error(badarg, [Value])
+    end.

apps/emqx_utils/test/emqx_utils_fs_SUITE.erl

@@ -0,0 +1,98 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_utils_fs_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("eunit/include/eunit.hrl").
+-include_lib("kernel/include/file.hrl").
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+%%
+
+t_traverse_dir(Config) ->
+    Dir = ?config(data_dir, Config),
+    Traversal = lists:sort(emqx_utils_fs:traverse_dir(fun cons_fileinfo/3, [], Dir)),
+    ?assertMatch(
+        [
+            {"nonempty/d1/1", #file_info{type = regular, mode = ORWAR}},
+            {"nonempty/d1/2", #file_info{type = regular, mode = ORWAR}},
+            {"nonempty/d1/mutrec", #file_info{type = symlink, mode = ARWX}},
+            {"nonempty/d1/файл", #file_info{type = regular}},
+            {"nonempty/d2/deep/down/here", #file_info{type = regular, mode = ORW}},
+            {"nonempty/d2/deep/mutrec", #file_info{type = symlink, mode = ARWX}},
+            {"nonempty/д3", #file_info{type = symlink, mode = ARWX}}
+        ] when
+            ((ORWAR band 8#00644 =:= 8#00644) and
+                (ORW band 8#00600 =:= 8#00600) and
+                (ARWX band 8#00777 =:= 8#00777)),
+
+        [{string:prefix(Filename, Dir), Info} || {Filename, Info} <- Traversal]
+    ).
+
+t_traverse_empty(Config) ->
+    Dir = filename:join(?config(data_dir, Config), "empty"),
+    ok = file:make_dir(Dir),
+    ?assertEqual(
+        [],
+        emqx_utils_fs:traverse_dir(fun cons_fileinfo/3, [], Dir)
+    ).
+
+t_traverse_nonexisting(_) ->
+    ?assertEqual(
+        [{"this/should/not/exist", {error, enoent}}],
+        emqx_utils_fs:traverse_dir(fun cons_fileinfo/3, [], "this/should/not/exist")
+    ).
+
+cons_fileinfo(Filename, Info, Acc) ->
+    [{Filename, Info} | Acc].
+
+%%
+
+t_canonicalize_empty(_) ->
+    ?assertEqual(
+        "",
+        emqx_utils_fs:canonicalize(<<>>)
+    ).
+
+t_canonicalize_relative(_) ->
+    ?assertEqual(
+        "rel",
+        emqx_utils_fs:canonicalize(<<"rel/">>)
+    ).
+
+t_canonicalize_trailing_slash(_) ->
+    ?assertEqual(
+        "/usr/local",
+        emqx_utils_fs:canonicalize("/usr/local/")
+    ).
+
+t_canonicalize_double_slashes(_) ->
+    ?assertEqual(
+        "/usr/local/.",
+        emqx_utils_fs:canonicalize("//usr//local//.//")
+    ).
+
+t_canonicalize_non_utf8(_) ->
+    ?assertError(
+        badarg,
+        emqx_utils_fs:canonicalize(<<128, 128, 128>>)
+    ).

apps/emqx_utils/test/emqx_utils_fs_SUITE_data/nonempty/d1/mutrec

@@ -0,0 +1 @@
+../../empty

apps/emqx_utils/test/emqx_utils_fs_SUITE_data/nonempty/d2/deep/mutrec

@@ -0,0 +1 @@
+../../d1

apps/emqx_utils/test/emqx_utils_fs_SUITE_data/nonempty/д3

@@ -0,0 +1 @@
+../dx