refactor(tls): abstract lib for tls options parsing

.gitignore

@@ -1,4 +1,5 @@
 .eunit
+test-data/
 deps
 !deps/.placeholder
 *.o

apps/emqx_bridge_mqtt/src/emqx_bridge_mqtt_actions.erl

@@ -739,8 +739,6 @@ options(Options, PoolName, ResId) ->
                      Topic ->
                          [{subscriptions, [{Topic, Get(<<"qos">>)}]} | Subscriptions]
                  end,
-                 %% TODO check why only ciphers are configurable but not versions
-                 TlsVersions = emqx_tls_lib:default_versions(),
                  [{address, binary_to_list(Address)},
                   {bridge_mode, GetD(<<"bridge_mode">>, true)},
                   {clean_start, true},
@@ -751,15 +749,17 @@ options(Options, PoolName, ResId) ->
                   {username, str(Get(<<"username">>))},
                   {password, str(Get(<<"password">>))},
                   {proto_ver, mqtt_ver(Get(<<"proto_ver">>))},
-                  {retry_interval, cuttlefish_duration:parse(str(GetD(<<"retry_interval">>, "30s")), s)},
-                  {ssl, cuttlefish_flag:parse(str(Get(<<"ssl">>)))},
-                  {ssl_opts, [ {versions, TlsVersions}
-                             , {ciphers, emqx_tls_lib:integral_ciphers(TlsVersions, Get(<<"ciphers">>))}
-                             | get_ssl_opts(Options, ResId)
-                             ]}
+                  {retry_interval, cuttlefish_duration:parse(str(GetD(<<"retry_interval">>, "30s")), s)}
+                  | maybe_ssl(Options, cuttlefish_flag:parse(str(Get(<<"ssl">>))), ResId)
                  ] ++ Subscriptions1
          end.
 
+maybe_ssl(_Options, false, _ResId) ->
+    [{ssl, false}];
+maybe_ssl(Options, true, ResId) ->
+    Dir = filename:join([emqx:get_env(data_dir), "rule", ResId]),
+    [{ssl, true}, {ssl_opts, emqx_plugin_libs_ssl:save_files_return_opts(Options, Dir)}].
+
 mqtt_ver(ProtoVer) ->
     case ProtoVer of
        <<"mqttv3">> -> v3;
@@ -772,43 +772,3 @@ format_subscriptions(SubOpts) ->
     lists:map(fun(Sub) ->
         {maps:get(<<"topic">>, Sub), maps:get(<<"qos">>, Sub)}
     end, SubOpts).
-
-get_ssl_opts(Opts, ResId) ->
-    KeyFile = maps:get(<<"keyfile">>, Opts, undefined),
-    CertFile = maps:get(<<"certfile">>, Opts, undefined),
-    CAFile = case maps:get(<<"cacertfile">>, Opts, undefined) of
-        undefined -> maps:get(<<"cafile">>, Opts, undefined);
-        CAFile0 -> CAFile0
-    end,
-    Filter = fun(Opts1) ->
-                     [{K, V} || {K, V} <- Opts1,
-                                    V =/= undefined,
-                                    V =/= <<>>,
-                                    V =/= "" ]
-             end,
-    Key = save_upload_file(KeyFile, ResId),
-    Cert = save_upload_file(CertFile, ResId),
-    CA = save_upload_file(CAFile, ResId),
-    Verify = case maps:get(<<"verify">>, Opts, false) of
-        false -> verify_none;
-        true -> verify_peer
-    end,
-    case Filter([{keyfile, Key}, {certfile, Cert}, {cacertfile, CA}]) of
-        [] -> [{verify, Verify}];
-        SslOpts ->
-            [{verify, Verify} | SslOpts]
-    end.
-
-save_upload_file(#{<<"file">> := <<>>, <<"filename">> := <<>>}, _ResId) -> "";
-save_upload_file(FilePath, _) when is_binary(FilePath) -> binary_to_list(FilePath);
-save_upload_file(#{<<"file">> := File, <<"filename">> := FileName}, ResId) ->
-     FullFilename = filename:join([emqx:get_env(data_dir), rules, ResId, FileName]),
-     ok = filelib:ensure_dir(FullFilename),
-     case file:write_file(FullFilename, File) of
-          ok ->
-               binary_to_list(FullFilename);
-          {error, Reason} ->
-               logger:error("Store file failed, ResId: ~p, ~0p", [ResId, Reason]),
-               error({ResId, store_file_fail})
-     end;
-save_upload_file(_, _) -> "".

apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl

@@ -0,0 +1,89 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_plugin_libs_ssl).
+
+-export([save_files_return_opts/2]).
+
+-type file_input_key() :: binary(). %% <<"file">> | <<"filename">>
+-type file_input() :: #{file_input_key() => binary()}.
+
+%% options are below paris
+%% <<"keyfile">> => file_input()
+%% <<"certfile">> => file_input()
+%% <<"cafile">> => file_input() %% backward compatible
+%% <<"cacertfile">> => file_input()
+%% <<"verify">> => boolean()
+%% <<"tls_versions">> => binary()
+%% <<"ciphers">> => binary()
+-type opts_key() :: binary().
+-type opts_input() :: #{opts_key() => file_input() | boolean() | binary()}.
+
+-type opt_key() :: keyfile | certfile | cacertfile | verify | versions | ciphers.
+-type opt_value() :: term().
+-type opts() :: [{opt_key(), opt_value()}].
+
+%% @doc Parse ssl options input.
+%% If the input contains file content, save the files in the given dir.
+%% Returns ssl options for Erlang's ssl application.
+-spec save_files_return_opts(opts_input(), file:name_all()) -> opts().
+save_files_return_opts(Options, Dir) ->
+    GetD = fun(Key, Default) -> maps:get(Key, Options, Default) end,
+    Get = fun(Key) -> GetD(Key, undefined) end,
+    KeyFile = Get(<<"keyfile">>),
+    CertFile = Get(<<"certfile">>),
+    CAFile = GetD(<<"cacertfile">>, Get(<<"cafile">>)),
+    Key = save_file(KeyFile, Dir),
+    Cert = save_file(CertFile, Dir),
+    CA = save_file(CAFile, Dir),
+    Verify = case GetD(<<"verify">>, false) of
+                  false -> verify_none;
+                  _ -> verify_peer
+             end,
+    Versions = emqx_tls_lib:integral_versions(Get(<<"tls_versions">>)),
+    Ciphers = emqx_tls_lib:integral_ciphers(Versions, Get(<<"ciphers">>)),
+    filter([{keyfile, Key}, {certfile, Cert}, {cacertfile, CA},
+            {verify, Verify}, {versions, Versions}, {ciphers, Ciphers}]).
+
+filter([]) -> [];
+filter([{_, ""} | T]) -> filter(T);
+filter([H | T]) -> [H | filter(T)].
+
+save_file(#{<<"filename">> := FileName, <<"file">> := Content}, Dir)
+  when FileName =/= undefined andalso Content =/= undefined ->
+    save_file(ensure_str(FileName), iolist_to_binary(Content), Dir);
+save_file(FilePath, _) when is_binary(FilePath) ->
+    ensure_str(FilePath);
+save_file(FilePath, _) when is_list(FilePath) ->
+    FilePath;
+save_file(_, _) -> "".
+
+save_file("", _, _Dir) -> ""; %% ignore
+save_file(_, <<>>, _Dir) -> ""; %% ignore
+save_file(FileName, Content, Dir) ->
+     FullFilename = filename:join([Dir, FileName]),
+     ok = filelib:ensure_dir(FullFilename),
+     case file:write_file(FullFilename, Content) of
+          ok ->
+               ensure_str(FullFilename);
+          {error, Reason} ->
+               logger:error("failed_to_save_ssl_file ~s: ~0p", [FullFilename, Reason]),
+               error({"failed_to_save_ssl_file", FullFilename, Reason})
+     end.
+
+ensure_str(L) when is_list(L) -> L;
+ensure_str(B) when is_binary(B) -> unicode:characters_to_list(B, utf8).
+

apps/emqx_plugin_libs/test/emqx_plugin_libs_ssl_tests.erl

@@ -0,0 +1,78 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_plugin_libs_ssl_tests).
+
+-include_lib("proper/include/proper.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
+no_crash_test_() ->
+    Opts = [{numtests, 1000}, {to_file, user}],
+    {timeout, 60,
+     fun() -> ?assert(proper:quickcheck(prop_run(), Opts)) end}.
+
+prop_run() ->
+    ?FORALL(Generated, prop_opts_input(), test_opts_input(Generated)).
+
+%% proper type to generate input value.
+prop_opts_input() ->
+    [{keyfile, prop_file_or_content()},
+     {certfile, prop_file_or_content()},
+     {cacertfile, prop_file_or_content()},
+     {verify, proper_types:boolean()},
+     {versions, prop_tls_versions()},
+     {ciphers, prop_tls_ciphers()},
+     {other, proper_types:binary()}].
+
+prop_file_or_content() ->
+    proper_types:oneof([prop_cert_file_name(),
+                        {prop_cert_file_name(), proper_types:binary()}]).
+
+prop_cert_file_name() ->
+    proper_types:oneof(["certname1", <<"certname2">>, "", <<>>, undefined]).
+
+prop_tls_versions() ->
+    proper_types:oneof(["tlsv1.3",
+                        <<"tlsv1.3,tlsv1.2">>,
+                        "tlsv1.2 , tlsv1.1",
+                        "1.2",
+                        "v1.3",
+                        "",
+                        <<>>,
+                        undefined]).
+
+prop_tls_ciphers() ->
+    proper_types:oneof(["TLS_AES_256_GCM_SHA384,TLS_AES_128_GCM_SHA256",
+                        <<>>,
+                        "",
+                        undefined]).
+
+test_opts_input(Inputs) ->
+    KF = fun(K) -> {_, V} = lists:keyfind(K, 1, Inputs), V end,
+    Generated = #{<<"keyfile">> => file_or_content(KF(keyfile)),
+                  <<"certfile">> => file_or_content(KF(certfile)),
+                  <<"cafile">> => file_or_content(KF(cacertfile)),
+                  <<"verify">> => file_or_content(KF(verify)),
+                  <<"tls_versions">> => KF(versions),
+                  <<"ciphers">> => KF(ciphers),
+                  <<"other">> => KF(other)},
+    _ = emqx_plugin_libs_ssl:save_files_return_opts(Generated, "test-data"),
+    true.
+
+file_or_content({Name, Content}) ->
+    #{<<"file">> => Content, <<"filename">> => Name};
+file_or_content(Name) ->
+    Name.

src/emqx_tls_lib.erl

@@ -23,7 +23,8 @@
         , integral_ciphers/2
         ]).
 
--define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso is_list(hd(L)))).
+-define(IS_STRING(L), (is_list(L) andalso L =/= [] andalso is_integer(hd(L)))).
+-define(IS_STRING_LIST(L), (is_list(L) andalso L =/= [] andalso ?IS_STRING(hd(L)))).
 
 %% @doc Returns the default supported tls versions.
 -spec default_versions() -> [atom()].
@@ -33,7 +34,18 @@ default_versions() ->
 
 %% @doc Validate a given list of desired tls versions.
 %% raise an error exception if non of them are available.
--spec integral_versions([ssl:tls_version()]) -> [ssl:tls_version()].
+%% The input list can be a string/binary of comma separated versions.
+-spec integral_versions(undefined | string() | binary() | [ssl:tls_version()]) -> [ssl:tls_version()].
+integral_versions(undefined) ->
+    integral_versions(default_versions());
+integral_versions([]) ->
+    integral_versions(default_versions());
+integral_versions(<<>>) ->
+    integral_versions(default_versions());
+integral_versions(Desired) when is_binary(Desired) ->
+    integral_versions(parse_versions(Desired));
+integral_versions(Desired) when ?IS_STRING(Desired) ->
+    integral_versions(iolist_to_binary(Desired));
 integral_versions(Desired) ->
     {_, Available} = lists:keyfind(available, 1, ssl:versions()),
     case lists:filter(fun(V) -> lists:member(V, Available) end, Desired) of
@@ -96,3 +108,32 @@ default_versions(_) ->
 %% Deduplicate a list without re-ordering the elements.
 dedup([]) -> [];
 dedup([H | T]) -> [H | dedup([I || I <- T, I =/= H])].
+
+%% parse comma separated tls version strings
+parse_versions(Versions) ->
+    do_parse_versions(split_by_comma(Versions), []).
+
+do_parse_versions([], Acc) -> lists:reverse(Acc);
+do_parse_versions([V | More], Acc) ->
+    case parse_version(V) of
+        unknown ->
+            emqx_logger:warning("unknown_tls_version_discarded: ~p", [V]),
+            do_parse_versions(More, Acc);
+        Parsed ->
+            do_parse_versions(More, [Parsed | Acc])
+    end.
+
+parse_version(<<"tlsv", Vsn/binary>>) -> parse_version(Vsn);
+parse_version(<<"v", Vsn/binary>>) -> parse_version(Vsn);
+parse_version(<<"1.3">>) -> 'tlsv1.3';
+parse_version(<<"1.2">>) -> 'tlsv1.2';
+parse_version(<<"1.1">>) -> 'tlsv1.1';
+parse_version(<<"1">>) -> 'tlsv1';
+parse_version(_) -> unknown.
+
+split_by_comma(Bin) ->
+    [trim_space(I) || I <- binary:split(Bin, <<",">>, [global])].
+
+%% trim spaces
+trim_space(Bin) ->
+    hd([I || I <- binary:split(Bin, <<" ">>), I =/= <<>>]).