emqx_utils.erl 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738
  1. %%--------------------------------------------------------------------
  2. %% Copyright (c) 2017-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
  3. %%
  4. %% Licensed under the Apache License, Version 2.0 (the "License");
  5. %% you may not use this file except in compliance with the License.
  6. %% You may obtain a copy of the License at
  7. %%
  8. %% http://www.apache.org/licenses/LICENSE-2.0
  9. %%
  10. %% Unless required by applicable law or agreed to in writing, software
  11. %% distributed under the License is distributed on an "AS IS" BASIS,
  12. %% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13. %% See the License for the specific language governing permissions and
  14. %% limitations under the License.
  15. %%--------------------------------------------------------------------
  16. -module(emqx_utils).
  17. -compile(inline).
  18. %% [TODO] Cleanup so the instruction below is not necessary.
  19. -elvis([{elvis_style, god_modules, disable}]).
  20. -export([
  21. merge_opts/2,
  22. maybe_apply/2,
  23. compose/1,
  24. compose/2,
  25. run_fold/3,
  26. pipeline/3,
  27. start_timer/2,
  28. start_timer/3,
  29. cancel_timer/1,
  30. drain_deliver/0,
  31. drain_deliver/1,
  32. drain_down/1,
  33. check_oom/1,
  34. check_oom/2,
  35. tune_heap_size/1,
  36. proc_name/2,
  37. proc_stats/0,
  38. proc_stats/1,
  39. rand_seed/0,
  40. now_to_secs/1,
  41. now_to_ms/1,
  42. index_of/2,
  43. maybe_parse_ip/1,
  44. ipv6_probe/1,
  45. gen_id/0,
  46. gen_id/1,
  47. explain_posix/1,
  48. pmap/2,
  49. pmap/3,
  50. readable_error_msg/1,
  51. safe_to_existing_atom/1,
  52. safe_to_existing_atom/2,
  53. pub_props_to_packet/1,
  54. safe_filename/1
  55. ]).
  56. -export([
  57. bin_to_hexstr/2,
  58. hexstr_to_bin/1
  59. ]).
  60. -export([
  61. nolink_apply/1,
  62. nolink_apply/2
  63. ]).
  64. -export([clamp/3, redact/1, redact/2, is_redacted/2, is_redacted/3]).
  65. -type maybe(T) :: undefined | T.
  66. -dialyzer({nowarn_function, [nolink_apply/2]}).
  67. -define(SHORT, 8).
  68. -define(DEFAULT_PMAP_TIMEOUT, 5000).
  69. %% @doc Parse v4 or v6 string format address to tuple.
  70. %% `Host' itself is returned if it's not an ip string.
  71. maybe_parse_ip(Host) ->
  72. case inet:parse_address(Host) of
  73. {ok, Addr} when is_tuple(Addr) -> Addr;
  74. {error, einval} -> Host
  75. end.
  76. %% @doc Add `ipv6_probe' socket option if it's supported.
  77. %% gen_tcp:ipv6_probe() -> true. is added to EMQ's OTP forks
  78. ipv6_probe(Opts) ->
  79. case erlang:function_exported(gen_tcp, ipv6_probe, 0) of
  80. true -> [{ipv6_probe, true} | Opts];
  81. false -> Opts
  82. end.
  83. %% @doc Merge options
  84. -spec merge_opts(Opts, Opts) -> Opts when Opts :: proplists:proplist().
  85. merge_opts(Defaults, Options) ->
  86. lists:foldl(
  87. fun
  88. ({Opt, Val}, Acc) ->
  89. lists:keystore(Opt, 1, Acc, {Opt, Val});
  90. (Opt, Acc) ->
  91. lists:usort([Opt | Acc])
  92. end,
  93. Defaults,
  94. Options
  95. ).
  96. %% @doc Apply a function to a maybe argument.
  97. -spec maybe_apply(fun((maybe(A)) -> maybe(A)), maybe(A)) ->
  98. maybe(A)
  99. when
  100. A :: any().
  101. maybe_apply(_Fun, undefined) ->
  102. undefined;
  103. maybe_apply(Fun, Arg) when is_function(Fun) ->
  104. erlang:apply(Fun, [Arg]).
  105. -spec compose(list(F)) -> G when
  106. F :: fun((any()) -> any()),
  107. G :: fun((any()) -> any()).
  108. compose([F | More]) -> compose(F, More).
  109. -spec compose(F, G | [Gs]) -> C when
  110. F :: fun((X1) -> X2),
  111. G :: fun((X2) -> X3),
  112. Gs :: [fun((Xn) -> Xn1)],
  113. C :: fun((X1) -> Xm),
  114. X3 :: any(),
  115. Xn :: any(),
  116. Xn1 :: any(),
  117. Xm :: any().
  118. compose(F, G) when is_function(G) -> fun(X) -> G(F(X)) end;
  119. compose(F, [G]) -> compose(F, G);
  120. compose(F, [G | More]) -> compose(compose(F, G), More).
  121. %% @doc RunFold
  122. run_fold([], Acc, _State) ->
  123. Acc;
  124. run_fold([Fun | More], Acc, State) ->
  125. run_fold(More, Fun(Acc, State), State).
  126. %% @doc Pipeline
  127. pipeline([], Input, State) ->
  128. {ok, Input, State};
  129. pipeline([Fun | More], Input, State) ->
  130. case apply_fun(Fun, Input, State) of
  131. ok -> pipeline(More, Input, State);
  132. {ok, NState} -> pipeline(More, Input, NState);
  133. {ok, Output, NState} -> pipeline(More, Output, NState);
  134. {error, Reason} -> {error, Reason, State};
  135. {error, Reason, NState} -> {error, Reason, NState}
  136. end.
  137. -compile({inline, [apply_fun/3]}).
  138. apply_fun(Fun, Input, State) ->
  139. case erlang:fun_info(Fun, arity) of
  140. {arity, 1} -> Fun(Input);
  141. {arity, 2} -> Fun(Input, State)
  142. end.
  143. -spec start_timer(integer() | atom(), term()) -> maybe(reference()).
  144. start_timer(Interval, Msg) ->
  145. start_timer(Interval, self(), Msg).
  146. -spec start_timer(integer() | atom(), pid() | atom(), term()) -> maybe(reference()).
  147. start_timer(Interval, Dest, Msg) when is_number(Interval) ->
  148. erlang:start_timer(erlang:ceil(Interval), Dest, Msg);
  149. start_timer(_Atom, _Dest, _Msg) ->
  150. undefined.
  151. -spec cancel_timer(maybe(reference())) -> ok.
  152. cancel_timer(Timer) when is_reference(Timer) ->
  153. case erlang:cancel_timer(Timer) of
  154. false ->
  155. receive
  156. {timeout, Timer, _} -> ok
  157. after 0 -> ok
  158. end;
  159. _ ->
  160. ok
  161. end;
  162. cancel_timer(_) ->
  163. ok.
  164. %% @doc Drain delivers
  165. drain_deliver() ->
  166. drain_deliver(-1).
  167. drain_deliver(N) when is_integer(N) ->
  168. drain_deliver(N, []).
  169. drain_deliver(0, Acc) ->
  170. lists:reverse(Acc);
  171. drain_deliver(N, Acc) ->
  172. receive
  173. Deliver = {deliver, _Topic, _Msg} ->
  174. drain_deliver(N - 1, [Deliver | Acc])
  175. after 0 ->
  176. lists:reverse(Acc)
  177. end.
  178. %% @doc Drain process 'DOWN' events.
  179. -spec drain_down(pos_integer()) -> list(pid()).
  180. drain_down(Cnt) when Cnt > 0 ->
  181. drain_down(Cnt, []).
  182. drain_down(0, Acc) ->
  183. lists:reverse(Acc);
  184. drain_down(Cnt, Acc) ->
  185. receive
  186. {'DOWN', _MRef, process, Pid, _Reason} ->
  187. drain_down(Cnt - 1, [Pid | Acc])
  188. after 0 ->
  189. lists:reverse(Acc)
  190. end.
  191. %% @doc Check process's mailbox and heapsize against OOM policy,
  192. %% return `ok | {shutdown, Reason}' accordingly.
  193. %% `ok': There is nothing out of the ordinary.
  194. %% `shutdown': Some numbers (message queue length hit the limit),
  195. %% hence shutdown for greater good (system stability).
  196. %% [FIXME] cross-dependency on `emqx_types`.
  197. -spec check_oom(emqx_types:oom_policy()) -> ok | {shutdown, term()}.
  198. check_oom(Policy) ->
  199. check_oom(self(), Policy).
  200. -spec check_oom(pid(), emqx_types:oom_policy()) -> ok | {shutdown, term()}.
  201. check_oom(_Pid, #{enable := false}) ->
  202. ok;
  203. check_oom(Pid, #{
  204. max_message_queue_len := MaxQLen,
  205. max_heap_size := MaxHeapSize
  206. }) ->
  207. case process_info(Pid, [message_queue_len, total_heap_size]) of
  208. undefined ->
  209. ok;
  210. [{message_queue_len, QLen}, {total_heap_size, HeapSize}] ->
  211. do_check_oom([
  212. {QLen, MaxQLen, message_queue_too_long},
  213. {HeapSize, MaxHeapSize, proc_heap_too_large}
  214. ])
  215. end.
  216. do_check_oom([]) ->
  217. ok;
  218. do_check_oom([{Val, Max, Reason} | Rest]) ->
  219. case is_integer(Max) andalso (0 < Max) andalso (Max < Val) of
  220. true -> {shutdown, #{reason => Reason, value => Val, max => Max}};
  221. false -> do_check_oom(Rest)
  222. end.
  223. tune_heap_size(#{enable := false}) ->
  224. ok;
  225. %% If the max_heap_size is set to zero, the limit is disabled.
  226. tune_heap_size(#{max_heap_size := MaxHeapSize}) when MaxHeapSize > 0 ->
  227. MaxSize =
  228. case erlang:system_info(wordsize) of
  229. % arch_64
  230. 8 ->
  231. (1 bsl 59) - 1;
  232. % arch_32
  233. 4 ->
  234. (1 bsl 27) - 1
  235. end,
  236. OverflowedSize =
  237. case erlang:trunc(MaxHeapSize * 1.5) of
  238. SZ when SZ > MaxSize -> MaxSize;
  239. SZ -> SZ
  240. end,
  241. erlang:process_flag(max_heap_size, #{
  242. size => OverflowedSize,
  243. kill => true,
  244. error_logger => true
  245. }).
  246. -spec proc_name(atom(), pos_integer()) -> atom().
  247. proc_name(Mod, Id) ->
  248. list_to_atom(lists:concat([Mod, "_", Id])).
  249. %% Get Proc's Stats.
  250. %% [FIXME] cross-dependency on `emqx_types`.
  251. -spec proc_stats() -> emqx_types:stats().
  252. proc_stats() -> proc_stats(self()).
  253. -spec proc_stats(pid()) -> emqx_types:stats().
  254. proc_stats(Pid) ->
  255. case
  256. process_info(Pid, [
  257. message_queue_len,
  258. heap_size,
  259. total_heap_size,
  260. reductions,
  261. memory
  262. ])
  263. of
  264. undefined -> [];
  265. [{message_queue_len, Len} | ProcStats] -> [{mailbox_len, Len} | ProcStats]
  266. end.
  267. rand_seed() ->
  268. rand:seed(exsplus, erlang:timestamp()).
  269. -spec now_to_secs(erlang:timestamp()) -> pos_integer().
  270. now_to_secs({MegaSecs, Secs, _MicroSecs}) ->
  271. MegaSecs * 1000000 + Secs.
  272. -spec now_to_ms(erlang:timestamp()) -> pos_integer().
  273. now_to_ms({MegaSecs, Secs, MicroSecs}) ->
  274. (MegaSecs * 1000000 + Secs) * 1000 + round(MicroSecs / 1000).
  275. %% lists:index_of/2
  276. index_of(E, L) ->
  277. index_of(E, 1, L).
  278. index_of(_E, _I, []) ->
  279. error(badarg);
  280. index_of(E, I, [E | _]) ->
  281. I;
  282. index_of(E, I, [_ | L]) ->
  283. index_of(E, I + 1, L).
  284. -spec bin_to_hexstr(binary(), lower | upper) -> binary().
  285. bin_to_hexstr(B, upper) when is_binary(B) ->
  286. <<<<(int2hexchar(H, upper)), (int2hexchar(L, upper))>> || <<H:4, L:4>> <= B>>;
  287. bin_to_hexstr(B, lower) when is_binary(B) ->
  288. <<<<(int2hexchar(H, lower)), (int2hexchar(L, lower))>> || <<H:4, L:4>> <= B>>.
  289. int2hexchar(I, _) when I >= 0 andalso I < 10 -> I + $0;
  290. int2hexchar(I, upper) -> I - 10 + $A;
  291. int2hexchar(I, lower) -> I - 10 + $a.
  292. -spec hexstr_to_bin(binary()) -> binary().
  293. hexstr_to_bin(B) when is_binary(B) ->
  294. hexstr_to_bin(B, erlang:bit_size(B)).
  295. hexstr_to_bin(B, Size) when is_binary(B) ->
  296. case Size rem 16 of
  297. 0 ->
  298. make_binary(B);
  299. 8 ->
  300. make_binary(<<"0", B/binary>>);
  301. _ ->
  302. throw({unsupport_hex_string, B, Size})
  303. end.
  304. make_binary(B) -> <<<<(hexchar2int(H) * 16 + hexchar2int(L))>> || <<H:8, L:8>> <= B>>.
  305. hexchar2int(I) when I >= $0 andalso I =< $9 -> I - $0;
  306. hexchar2int(I) when I >= $A andalso I =< $F -> I - $A + 10;
  307. hexchar2int(I) when I >= $a andalso I =< $f -> I - $a + 10.
  308. -spec gen_id() -> list().
  309. gen_id() ->
  310. gen_id(?SHORT).
  311. -spec gen_id(integer()) -> list().
  312. gen_id(Len) ->
  313. BitLen = Len * 4,
  314. <<R:BitLen>> = crypto:strong_rand_bytes(Len div 2),
  315. int_to_hex(R, Len).
  316. -spec clamp(number(), number(), number()) -> number().
  317. clamp(Val, Min, _Max) when Val < Min -> Min;
  318. clamp(Val, _Min, Max) when Val > Max -> Max;
  319. clamp(Val, _Min, _Max) -> Val.
  320. %% @doc https://www.erlang.org/doc/man/file.html#posix-error-codes
  321. explain_posix(eacces) -> "Permission denied";
  322. explain_posix(eagain) -> "Resource temporarily unavailable";
  323. explain_posix(ebadf) -> "Bad file number";
  324. explain_posix(ebusy) -> "File busy";
  325. explain_posix(edquot) -> "Disk quota exceeded";
  326. explain_posix(eexist) -> "File already exists";
  327. explain_posix(efault) -> "Bad address in system call argument";
  328. explain_posix(efbig) -> "File too large";
  329. explain_posix(eintr) -> "Interrupted system call";
  330. explain_posix(einval) -> "Invalid argument argument file/socket";
  331. explain_posix(eio) -> "I/O error";
  332. explain_posix(eisdir) -> "Illegal operation on a directory";
  333. explain_posix(eloop) -> "Too many levels of symbolic links";
  334. explain_posix(emfile) -> "Too many open files";
  335. explain_posix(emlink) -> "Too many links";
  336. explain_posix(enametoolong) -> "Filename too long";
  337. explain_posix(enfile) -> "File table overflow";
  338. explain_posix(enodev) -> "No such device";
  339. explain_posix(enoent) -> "No such file or directory";
  340. explain_posix(enomem) -> "Not enough memory";
  341. explain_posix(enospc) -> "No space left on device";
  342. explain_posix(enotblk) -> "Block device required";
  343. explain_posix(enotdir) -> "Not a directory";
  344. explain_posix(enotsup) -> "Operation not supported";
  345. explain_posix(enxio) -> "No such device or address";
  346. explain_posix(eperm) -> "Not owner";
  347. explain_posix(epipe) -> "Broken pipe";
  348. explain_posix(erofs) -> "Read-only file system";
  349. explain_posix(espipe) -> "Invalid seek";
  350. explain_posix(esrch) -> "No such process";
  351. explain_posix(estale) -> "Stale remote file handle";
  352. explain_posix(exdev) -> "Cross-domain link";
  353. explain_posix(NotPosix) -> NotPosix.
  354. %% @doc Like lists:map/2, only the callback function is evaluated
  355. %% concurrently.
  356. -spec pmap(fun((A) -> B), list(A)) -> list(B).
  357. pmap(Fun, List) when is_function(Fun, 1), is_list(List) ->
  358. pmap(Fun, List, ?DEFAULT_PMAP_TIMEOUT).
  359. -spec pmap(fun((A) -> B), list(A), timeout()) -> list(B).
  360. pmap(Fun, List, Timeout) when
  361. is_function(Fun, 1), is_list(List), is_integer(Timeout), Timeout >= 0
  362. ->
  363. nolink_apply(fun() -> do_parallel_map(Fun, List) end, Timeout).
  364. %% @doc Delegate a function to a worker process.
  365. %% The function may spawn_link other processes but we do not
  366. %% want the caller process to be linked.
  367. %% This is done by isolating the possible link with a not-linked
  368. %% middleman process.
  369. nolink_apply(Fun) -> nolink_apply(Fun, infinity).
  370. %% @doc Same as `nolink_apply/1', with a timeout.
  371. -spec nolink_apply(function(), timer:timeout()) -> term().
  372. nolink_apply(Fun, Timeout) when is_function(Fun, 0) ->
  373. Caller = self(),
  374. ResRef = make_ref(),
  375. Middleman = erlang:spawn(
  376. fun() ->
  377. process_flag(trap_exit, true),
  378. CallerMRef = erlang:monitor(process, Caller),
  379. Worker = erlang:spawn_link(
  380. fun() ->
  381. Res =
  382. try
  383. {normal, Fun()}
  384. catch
  385. C:E:S ->
  386. {exception, {C, E, S}}
  387. end,
  388. _ = erlang:send(Caller, {ResRef, Res}),
  389. exit(normal)
  390. end
  391. ),
  392. receive
  393. {'DOWN', CallerMRef, process, _, _} ->
  394. %% For whatever reason, if the caller is dead,
  395. %% there is no reason to continue
  396. exit(Worker, kill),
  397. exit(normal);
  398. {'EXIT', Worker, normal} ->
  399. exit(normal);
  400. {'EXIT', Worker, Reason} ->
  401. %% worker exited with some reason other than 'normal'
  402. _ = erlang:send(Caller, {ResRef, {'EXIT', Reason}}),
  403. exit(normal)
  404. end
  405. end
  406. ),
  407. receive
  408. {ResRef, {normal, Result}} ->
  409. Result;
  410. {ResRef, {exception, {C, E, S}}} ->
  411. erlang:raise(C, E, S);
  412. {ResRef, {'EXIT', Reason}} ->
  413. exit(Reason)
  414. after Timeout ->
  415. exit(Middleman, kill),
  416. exit(timeout)
  417. end.
  418. safe_to_existing_atom(In) ->
  419. safe_to_existing_atom(In, utf8).
  420. safe_to_existing_atom(Bin, Encoding) when is_binary(Bin) ->
  421. try_to_existing_atom(fun erlang:binary_to_existing_atom/2, Bin, Encoding);
  422. safe_to_existing_atom(List, Encoding) when is_list(List) ->
  423. try_to_existing_atom(fun(In, _) -> erlang:list_to_existing_atom(In) end, List, Encoding);
  424. safe_to_existing_atom(Atom, _Encoding) when is_atom(Atom) ->
  425. {ok, Atom};
  426. safe_to_existing_atom(_Any, _Encoding) ->
  427. {error, invalid_type}.
  428. %%------------------------------------------------------------------------------
  429. %% Internal Functions
  430. %%------------------------------------------------------------------------------
  431. do_parallel_map(Fun, List) ->
  432. Parent = self(),
  433. PidList = lists:map(
  434. fun(Item) ->
  435. erlang:spawn_link(
  436. fun() ->
  437. Res =
  438. try
  439. {normal, Fun(Item)}
  440. catch
  441. C:E:St ->
  442. {exception, {C, E, St}}
  443. end,
  444. Parent ! {self(), Res}
  445. end
  446. )
  447. end,
  448. List
  449. ),
  450. lists:foldr(
  451. fun(Pid, Acc) ->
  452. receive
  453. {Pid, {normal, Result}} ->
  454. [Result | Acc];
  455. {Pid, {exception, {C, E, St}}} ->
  456. erlang:raise(C, E, St)
  457. end
  458. end,
  459. [],
  460. PidList
  461. ).
  462. int_to_hex(I, N) when is_integer(I), I >= 0 ->
  463. int_to_hex([], I, 1, N).
  464. int_to_hex(L, I, Count, N) when
  465. I < 16
  466. ->
  467. pad([int_to_hex(I) | L], N - Count);
  468. int_to_hex(L, I, Count, N) ->
  469. int_to_hex([int_to_hex(I rem 16) | L], I div 16, Count + 1, N).
  470. int_to_hex(I) when 0 =< I, I =< 9 ->
  471. I + $0;
  472. int_to_hex(I) when 10 =< I, I =< 15 ->
  473. (I - 10) + $a.
  474. pad(L, 0) ->
  475. L;
  476. pad(L, Count) ->
  477. pad([$0 | L], Count - 1).
  478. readable_error_msg(Msg) when is_binary(Msg) -> Msg;
  479. readable_error_msg(Error) ->
  480. case io_lib:printable_unicode_list(Error) of
  481. true ->
  482. unicode:characters_to_binary(Error, utf8);
  483. false ->
  484. case emqx_hocon:format_error(Error, #{no_stacktrace => true}) of
  485. {ok, Msg} ->
  486. Msg;
  487. false ->
  488. to_hr_error(Error)
  489. end
  490. end.
  491. to_hr_error(nxdomain) ->
  492. <<"Could not resolve host">>;
  493. to_hr_error(econnrefused) ->
  494. <<"Connection refused">>;
  495. to_hr_error({unauthorized_client, _}) ->
  496. <<"Unauthorized client">>;
  497. to_hr_error({not_authorized, _}) ->
  498. <<"Not authorized">>;
  499. to_hr_error({malformed_username_or_password, _}) ->
  500. <<"Bad username or password">>;
  501. to_hr_error(Error) ->
  502. iolist_to_binary(io_lib:format("~0p", [Error])).
  503. try_to_existing_atom(Convert, Data, Encoding) ->
  504. try Convert(Data, Encoding) of
  505. Atom ->
  506. {ok, Atom}
  507. catch
  508. _:Reason -> {error, Reason}
  509. end.
  510. is_sensitive_key(token) -> true;
  511. is_sensitive_key("token") -> true;
  512. is_sensitive_key(<<"token">>) -> true;
  513. is_sensitive_key(password) -> true;
  514. is_sensitive_key("password") -> true;
  515. is_sensitive_key(<<"password">>) -> true;
  516. is_sensitive_key(secret) -> true;
  517. is_sensitive_key("secret") -> true;
  518. is_sensitive_key(<<"secret">>) -> true;
  519. is_sensitive_key(_) -> false.
  520. redact(Term) ->
  521. do_redact(Term, fun is_sensitive_key/1).
  522. redact(Term, Checker) ->
  523. do_redact(Term, fun(V) ->
  524. is_sensitive_key(V) orelse Checker(V)
  525. end).
  526. do_redact(L, Checker) when is_list(L) ->
  527. lists:map(fun(E) -> do_redact(E, Checker) end, L);
  528. do_redact(M, Checker) when is_map(M) ->
  529. maps:map(
  530. fun(K, V) ->
  531. do_redact(K, V, Checker)
  532. end,
  533. M
  534. );
  535. do_redact({Key, Value}, Checker) ->
  536. case Checker(Key) of
  537. true ->
  538. {Key, redact_v(Value)};
  539. false ->
  540. {do_redact(Key, Checker), do_redact(Value, Checker)}
  541. end;
  542. do_redact(T, Checker) when is_tuple(T) ->
  543. Elements = erlang:tuple_to_list(T),
  544. Redact = do_redact(Elements, Checker),
  545. erlang:list_to_tuple(Redact);
  546. do_redact(Any, _Checker) ->
  547. Any.
  548. do_redact(K, V, Checker) ->
  549. case Checker(K) of
  550. true ->
  551. redact_v(V);
  552. false ->
  553. do_redact(V, Checker)
  554. end.
  555. -define(REDACT_VAL, "******").
  556. redact_v(V) when is_binary(V) -> <<?REDACT_VAL>>;
  557. %% The HOCON schema system may generate sensitive values with this format
  558. redact_v([{str, Bin}]) when is_binary(Bin) ->
  559. [{str, <<?REDACT_VAL>>}];
  560. redact_v(_V) ->
  561. ?REDACT_VAL.
  562. is_redacted(K, V) ->
  563. do_is_redacted(K, V, fun is_sensitive_key/1).
  564. is_redacted(K, V, Fun) ->
  565. do_is_redacted(K, V, fun(E) ->
  566. is_sensitive_key(E) orelse Fun(E)
  567. end).
  568. do_is_redacted(K, ?REDACT_VAL, Fun) ->
  569. Fun(K);
  570. do_is_redacted(K, <<?REDACT_VAL>>, Fun) ->
  571. Fun(K);
  572. do_is_redacted(_K, _V, _Fun) ->
  573. false.
  574. -ifdef(TEST).
  575. -include_lib("eunit/include/eunit.hrl").
  576. ipv6_probe_test() ->
  577. try gen_tcp:ipv6_probe() of
  578. true ->
  579. ?assertEqual([{ipv6_probe, true}], ipv6_probe([]))
  580. catch
  581. _:_ ->
  582. ok
  583. end.
  584. redact_test_() ->
  585. Case = fun(Type, KeyT) ->
  586. Key =
  587. case Type of
  588. atom -> KeyT;
  589. string -> erlang:atom_to_list(KeyT);
  590. binary -> erlang:atom_to_binary(KeyT)
  591. end,
  592. ?assert(is_sensitive_key(Key)),
  593. %% direct
  594. ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo})),
  595. ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo})),
  596. ?assertEqual({Key, Key, Key}, redact({Key, Key, Key})),
  597. ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar})),
  598. %% 1 level nested
  599. ?assertEqual([{Key, ?REDACT_VAL}], redact([{Key, foo}])),
  600. ?assertEqual([#{Key => ?REDACT_VAL}], redact([#{Key => foo}])),
  601. %% 2 level nested
  602. ?assertEqual(#{opts => [{Key, ?REDACT_VAL}]}, redact(#{opts => [{Key, foo}]})),
  603. ?assertEqual(#{opts => #{Key => ?REDACT_VAL}}, redact(#{opts => #{Key => foo}})),
  604. ?assertEqual({opts, [{Key, ?REDACT_VAL}]}, redact({opts, [{Key, foo}]})),
  605. %% 3 level nested
  606. ?assertEqual([#{opts => [{Key, ?REDACT_VAL}]}], redact([#{opts => [{Key, foo}]}])),
  607. ?assertEqual([{opts, [{Key, ?REDACT_VAL}]}], redact([{opts, [{Key, foo}]}])),
  608. ?assertEqual([{opts, [#{Key => ?REDACT_VAL}]}], redact([{opts, [#{Key => foo}]}]))
  609. end,
  610. Types = [atom, string, binary],
  611. Keys = [
  612. token,
  613. password,
  614. secret
  615. ],
  616. [{case_name(Type, Key), fun() -> Case(Type, Key) end} || Key <- Keys, Type <- Types].
  617. redact2_test_() ->
  618. Case = fun(Key, Checker) ->
  619. ?assertEqual({Key, ?REDACT_VAL}, redact({Key, foo}, Checker)),
  620. ?assertEqual(#{Key => ?REDACT_VAL}, redact(#{Key => foo}, Checker)),
  621. ?assertEqual({Key, Key, Key}, redact({Key, Key, Key}, Checker)),
  622. ?assertEqual({[{Key, ?REDACT_VAL}], bar}, redact({[{Key, foo}], bar}, Checker))
  623. end,
  624. Checker = fun(E) -> E =:= passcode end,
  625. Keys = [secret, passcode],
  626. [{case_name(atom, Key), fun() -> Case(Key, Checker) end} || Key <- Keys].
  627. case_name(Type, Key) ->
  628. lists:concat([Type, "-", Key]).
  629. -endif.
  630. pub_props_to_packet(Properties) ->
  631. F = fun
  632. ('User-Property', M) ->
  633. case is_map(M) andalso map_size(M) > 0 of
  634. true -> {true, maps:to_list(M)};
  635. false -> false
  636. end;
  637. ('User-Property-Pairs', _) ->
  638. false;
  639. (_, _) ->
  640. true
  641. end,
  642. maps:filtermap(F, Properties).
  643. %% fix filename by replacing characters which could be invalid on some filesystems
  644. %% with safe ones
  645. -spec safe_filename(binary() | unicode:chardata()) -> binary() | [unicode:chardata()].
  646. safe_filename(Filename) when is_binary(Filename) ->
  647. binary:replace(Filename, <<":">>, <<"-">>, [global]);
  648. safe_filename(Filename) when is_list(Filename) ->
  649. lists:flatten(string:replace(Filename, ":", "-", all)).