fix(fs-fold): avoid folding through symlinked directories

Also a testsuite that verifies multilevel fold behaviour.

apps/emqx_ft/src/emqx_ft_fs_util.erl

@@ -17,12 +17,14 @@
 -module(emqx_ft_fs_util).
 
 -include_lib("snabbkaffe/include/trace.hrl").
+-include_lib("kernel/include/file.hrl").
 
 -export([is_filename_safe/1]).
 -export([escape_filename/1]).
 -export([unescape_filename/1]).
 
 -export([read_decode_file/2]).
+-export([read_info/1]).
 
 -export([fold/4]).
 
@@ -144,13 +146,20 @@ safe_decode(Content, DecodeFun) ->
             {error, corrupted}
     end.
 
+-spec read_info(file:name_all()) ->
+    {ok, file:file_info()} | {error, file:posix() | badarg}.
+read_info(AbsPath) ->
+    % NOTE
+    % Be aware that this function is occasionally mocked in `emqx_ft_fs_util_SUITE`.
+    file:read_link_info(AbsPath, [{time, posix}, raw]).
+
 -spec fold(foldfun(Acc), Acc, _Root :: file:name(), glob()) ->
     Acc.
 fold(Fun, Acc, Root, Glob) ->
     fold(Fun, Acc, [], Root, Glob, []).
 
 fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_function(Glob) ->
-    case file:list_dir(filename:join(Root, Path)) of
+    case list_dir(filename:join(Root, Path)) of
         {ok, Filenames} ->
             lists:foldl(
                 fun(FN, Acc) ->
@@ -172,7 +181,7 @@ fold(Fun, AccIn, Path, Root, [Glob | Rest], Stack) when Glob == '*' orelse is_fu
             Fun(Path, {error, Reason}, Stack, AccIn)
     end;
 fold(Fun, AccIn, Filepath, Root, [], Stack) ->
-    case file:read_link_info(filename:join(Root, Filepath), [{time, posix}, raw]) of
+    case ?MODULE:read_info(filename:join(Root, Filepath)) of
         {ok, Info} ->
             Fun(Filepath, Info, Stack, AccIn);
         {error, Reason} ->
@@ -183,3 +192,13 @@ matches_glob('*', _) ->
     true;
 matches_glob(FilterFun, Filename) when is_function(FilterFun) ->
     FilterFun(Filename).
+
+list_dir(AbsPath) ->
+    case ?MODULE:read_info(AbsPath) of
+        {ok, #file_info{type = directory}} ->
+            file:list_dir(AbsPath);
+        {ok, #file_info{}} ->
+            {error, enotdir};
+        {error, Reason} ->
+            {error, Reason}
+    end.

apps/emqx_ft/test/emqx_ft_fs_util_SUITE.erl

@@ -0,0 +1,159 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2020-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_ft_fs_util_SUITE).
+
+-compile(export_all).
+-compile(nowarn_export_all).
+
+-include_lib("common_test/include/ct.hrl").
+-include_lib("stdlib/include/assert.hrl").
+-include_lib("kernel/include/file.hrl").
+
+all() ->
+    emqx_common_test_helpers:all(?MODULE).
+
+t_fold_single_level(Config) ->
+    Root = ?config(data_dir, Config),
+    ?assertMatch(
+        [
+            {"a", #file_info{type = directory}, ["a"]},
+            {"c", #file_info{type = directory}, ["c"]},
+            {"d", #file_info{type = directory}, ["d"]}
+        ],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*']))
+    ).
+
+t_fold_multi_level(Config) ->
+    Root = ?config(data_dir, Config),
+    ?assertMatch(
+        [
+            {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]},
+            {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]},
+            {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]}
+        ],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*']))
+    ),
+    ?assertMatch(
+        [
+            {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]},
+            {"c/bar/中文", #file_info{type = regular}, ["中文", "bar", "c"]},
+            {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]}
+        ],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*']))
+    ).
+
+t_fold_no_glob(Config) ->
+    Root = ?config(data_dir, Config),
+    ?assertMatch(
+        [{"", #file_info{type = directory}, []}],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, []))
+    ).
+
+t_fold_glob_too_deep(Config) ->
+    Root = ?config(data_dir, Config),
+    ?assertMatch(
+        [],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*', '*']))
+    ).
+
+t_fold_invalid_root(Config) ->
+    Root = ?config(data_dir, Config),
+    ?assertMatch(
+        [],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "a", "link"]), ['*']))
+    ),
+    ?assertMatch(
+        [],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], filename:join([Root, "d", "haystack"]), ['*']))
+    ).
+
+t_fold_filter_unicode(Config) ->
+    Root = ?config(data_dir, Config),
+    ?assertMatch(
+        [
+            {"a/b/foo/42", #file_info{type = regular}, ["42", "foo", "b", "a"]},
+            {"d/e/baz/needle", #file_info{type = regular}, ["needle", "baz", "e", "d"]}
+        ],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', fun is_latin1/1]))
+    ),
+    ?assertMatch(
+        [
+            {"a/b/foo/Я", #file_info{type = regular}, ["Я", "foo", "b", "a"]}
+        ],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', is_not(fun is_latin1/1)]))
+    ).
+
+t_fold_filter_levels(Config) ->
+    Root = ?config(data_dir, Config),
+    ?assertMatch(
+        [
+            {"a/b/foo", #file_info{type = directory}, ["foo", "b", "a"]},
+            {"d/e/baz", #file_info{type = directory}, ["baz", "e", "d"]}
+        ],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, [fun is_letter/1, fun is_letter/1, '*']))
+    ).
+
+t_fold_errors(Config) ->
+    Root = ?config(data_dir, Config),
+    ok = meck:new(emqx_ft_fs_util, [passthrough]),
+    ok = meck:expect(emqx_ft_fs_util, read_info, fun(AbsFilepath) ->
+        ct:pal("read_info(~p)", [AbsFilepath]),
+        Filename = filename:basename(AbsFilepath),
+        case Filename of
+            "b" -> {error, eacces};
+            "link" -> {error, enotsup};
+            "bar" -> {error, enotdir};
+            "needle" -> {error, ebusy};
+            _ -> meck:passthrough([AbsFilepath])
+        end
+    end),
+    ?assertMatch(
+        [
+            {"a/b", {error, eacces}, ["b", "a"]},
+            {"a/link", {error, enotsup}, ["link", "a"]},
+            {"c/link", {error, enotsup}, ["link", "c"]},
+            {"d/e/baz/needle", {error, ebusy}, ["needle", "baz", "e", "d"]}
+        ],
+        sort(emqx_ft_fs_util:fold(fun cons/4, [], Root, ['*', '*', '*', '*']))
+    ).
+
+%%
+
+is_not(F) ->
+    fun(X) -> not F(X) end.
+
+is_latin1(Filename) ->
+    case unicode:characters_to_binary(Filename, unicode, latin1) of
+        {error, _, _} ->
+            false;
+        _ ->
+            true
+    end.
+
+is_letter(Filename) ->
+    case Filename of
+        [_] ->
+            true;
+        _ ->
+            false
+    end.
+
+cons(Path, Info, Stack, Acc) ->
+    [{Path, Info, Stack} | Acc].
+
+sort(L) when is_list(L) ->
+    lists:sort(L).

apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/b/foo/Я

@@ -0,0 +1 @@
+Ты

apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/a/link

@@ -0,0 +1 @@
+../c

apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/bar/中文

@@ -0,0 +1 @@
+Zhōngwén

apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/c/link

@@ -0,0 +1 @@
+../a

apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/e/baz/needle

@@ -0,0 +1 @@
+haystack

apps/emqx_ft/test/emqx_ft_fs_util_SUITE_data/d/haystack

@@ -0,0 +1 @@
+needle