refactor(authz): call emqx_tls_lib to save & read SSL files

apps/emqx/src/emqx_tls_lib.erl

@@ -26,9 +26,10 @@
         , all_ciphers/0
         ]).
 
-%% files
+%% SSL files
 -export([ ensure_ssl_files/2
         , delete_ssl_files/3
+        , file_content_as_options/1
         ]).
 
 -include("logger.hrl").
@@ -248,7 +249,7 @@ ensure_ssl_files(Dir, Opts, [Key | Keys], DryRun) ->
     end.
 
 %% @doc Compare old and new config, delete the ones in old but not in new.
--spec delete_ssl_files(file:name_all(), undefiend | map(), undefined | map()) -> ok.
+-spec delete_ssl_files(file:name_all(), undefined | map(), undefined | map()) -> ok.
 delete_ssl_files(Dir, NewOpts0, OldOpts0) ->
     DryRun = true,
     {ok, NewOpts} = ensure_ssl_files(Dir, NewOpts0, DryRun),
@@ -345,6 +346,28 @@ is_valid_pem_file(Path) ->
         {error, Reason} -> {error, Reason}
     end.
 
+%% @doc This is to return SSL file content in management APIs.
+file_content_as_options(undefined) -> undefined;
+file_content_as_options(#{<<"enable">> := false} = SSL) ->
+    maps:without(?SSL_FILE_OPT_NAMES, SSL);
+file_content_as_options(#{<<"enable">> := true} = SSL) ->
+    file_content_as_options(?SSL_FILE_OPT_NAMES, SSL).
+
+file_content_as_options([], SSL) -> {ok, SSL};
+file_content_as_options([Key | Keys], SSL) ->
+    case maps:get(Key, SSL, undefined) of
+        undefined -> file_content_as_options(Keys, SSL);
+        Path ->
+            case file:read_file(Path) of
+                {ok, Bin} ->
+                    file_content_as_options(Keys, SSL#{Key => Bin});
+                {error, Reason} ->
+                    {error, #{file_path => Path,
+                              reason => Reason
+                             }}
+            end
+    end.
+
 -if(?OTP_RELEASE > 22).
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").

apps/emqx_authn/test/emqx_authn_api_SUITE.erl

@@ -51,7 +51,6 @@ set_special_configs(emqx_dashboard) ->
         }]
     },
     emqx_config:put([emqx_dashboard], Config),
-    emqx_config:put([node, data_dir], "data"),
     ok;
 set_special_configs(_App) ->
     ok.

apps/emqx_authz/src/emqx_authz.erl

@@ -38,6 +38,7 @@
 
 -export([post_config_update/4, pre_config_update/2]).
 
+-export([acl_conf_file/0]).
 
 -spec(register_metrics() -> ok).
 register_metrics() ->
@@ -361,3 +362,7 @@ type(<<"postgresql">>) -> postgresql;
 type('built-in-database') -> 'built-in-database';
 type(<<"built-in-database">>) -> 'built-in-database';
 type(Unknown) -> error({unknown_authz_source_type, Unknown}). % should never happend if the input is type-checked by hocon schema
+
+%% @doc where the acl.conf file is stored.
+acl_conf_file() ->
+    filename:join([emqx:data_dir(), "ahtz", "acl.conf"]).

apps/emqx_authz/src/emqx_authz_api_sources.erl

@@ -340,11 +340,11 @@ sources(get, _) ->
                                                                 }])
                                   end;
                               (Source, AccIn) ->
-                                  lists:append(AccIn, [read_cert(Source)])
+                                  lists:append(AccIn, [read_certs(Source)])
                           end, [], get_raw_sources()),
     {200, #{sources => Sources}};
 sources(post, #{body := #{<<"type">> := <<"file">>, <<"rules">> := Rules}}) ->
-    {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), Rules),
+    {ok, Filename} = write_file(acl_conf_file(), Rules),
     update_config(?CMD_PREPEND, [#{<<"type">> => <<"file">>, <<"enable">> => true, <<"path">> => Filename}]);
 sources(post, #{body := Body}) when is_map(Body) ->
     update_config(?CMD_PREPEND, [maybe_write_certs(Body)]);
@@ -352,7 +352,7 @@ sources(put, #{body := Body}) when is_list(Body) ->
     NBody = [ begin
                 case Source of
                     #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable} ->
-                        {ok, Filename} = write_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]), Rules),
+                        {ok, Filename} = write_file(acl_conf_file(), Rules),
                         #{<<"type">> => <<"file">>, <<"enable">> => Enable, <<"path">> => Filename};
                     _ -> maybe_write_certs(Source)
                 end
@@ -375,7 +375,7 @@ source(get, #{bindings := #{type := Type}}) ->
                             message => bin(Reason)}}
             end;
         [Source] ->
-            {200, read_cert(Source)}
+            {200, read_certs(Source)}
     end;
 source(put, #{bindings := #{type := <<"file">>}, body := #{<<"type">> := <<"file">>, <<"rules">> := Rules, <<"enable">> := Enable}}) ->
     {ok, Filename} = write_file(maps:get(path, emqx_authz:lookup(file), ""), Rules),
@@ -427,54 +427,18 @@ update_config(Cmd, Sources) ->
                     message => bin(Reason)}}
     end.
 
-read_cert(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) ->
-    CaCert = case file:read_file(maps:get(<<"cacertfile">>, SSL, "")) of
-                 {ok, CaCert0} -> CaCert0;
-                 _ -> ""
-             end,
-    Cert =   case file:read_file(maps:get(<<"certfile">>, SSL, "")) of
-                 {ok, Cert0} -> Cert0;
-                 _ -> ""
-             end,
-    Key =   case file:read_file(maps:get(<<"keyfile">>, SSL, "")) of
-                 {ok, Key0} -> Key0;
-                 _ -> ""
-             end,
-    Source#{<<"ssl">> => SSL#{<<"cacertfile">> => CaCert,
-                              <<"certfile">> => Cert,
-                              <<"keyfile">> => Key
-                             }
-           };
-read_cert(Source) -> Source.
+read_certs(#{<<"ssl">> := SSL} = Source) ->
+    case emqx_tls_lib:file_content_as_options(SSL) of
+        {ok, NewSSL} -> Source#{<<"ssl">> => NewSSL};
+        {error, Reason} ->
+            ?SLOG(error, Reason#{msg => failed_to_readd_ssl_file}),
+            throw(failed_to_readd_ssl_file)
+    end;
+read_certs(Source) -> Source.
 
 maybe_write_certs(#{<<"ssl">> := #{<<"enable">> := true} = SSL} = Source) ->
-    CertPath = filename:join([emqx:get_config([node, data_dir]), "certs"]),
-    CaCert = case maps:is_key(<<"cacertfile">>, SSL) of
-                 true ->
-                     {ok, CaCertFile} = write_file(filename:join([CertPath, "cacert-" ++ emqx_misc:gen_id() ++".pem"]),
-                                                 maps:get(<<"cacertfile">>, SSL)),
-                     CaCertFile;
-                 false -> ""
-             end,
-    Cert =   case maps:is_key(<<"certfile">>, SSL) of
-                 true ->
-                     {ok, CertFile} = write_file(filename:join([CertPath, "cert-" ++ emqx_misc:gen_id() ++".pem"]),
-                                                 maps:get(<<"certfile">>, SSL)),
-                     CertFile;
-                 false -> ""
-             end,
-    Key =    case maps:is_key(<<"keyfile">>, SSL) of
-                 true ->
-                     {ok, KeyFile}  = write_file(filename:join([CertPath, "key-" ++ emqx_misc:gen_id() ++".pem"]),
-                                                 maps:get(<<"keyfile">>, SSL)),
-                     KeyFile;
-                 false -> ""
-             end,
-    Source#{<<"ssl">> => SSL#{<<"cacertfile">> => CaCert,
-                              <<"certfile">> => Cert,
-                              <<"keyfile">> => Key
-                             }
-           };
+    Type = maps:get(<<"type">>, Source),
+    emqx_tls_lib:ensure_ssl_files(filename:join(["authz", "certs", Type]), SSL);
 maybe_write_certs(Source) -> Source.
 
 write_file(Filename, Bytes0) ->
@@ -482,7 +446,7 @@ write_file(Filename, Bytes0) ->
     case file:read_file(Filename) of
         {ok, Bytes1} ->
             case crypto:hash(md5, Bytes1) =:= crypto:hash(md5, Bytes0) of
-                true -> {ok,iolist_to_binary(Filename)};
+                true -> {ok, iolist_to_binary(Filename)};
                 false -> do_write_file(Filename, Bytes0)
             end;
         _ -> do_write_file(Filename, Bytes0)
@@ -498,3 +462,6 @@ do_write_file(Filename, Bytes) ->
 
 bin(Term) ->
    erlang:iolist_to_binary(io_lib:format("~p", [Term])).
+
+acl_conf_file() ->
+    emqx_authz:acl_conf_file().

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -71,7 +71,14 @@ fields(file) ->
     , {enable, #{type => boolean(),
                  default => true}}
     , {path, #{type => string(),
-               desc => "Path to the file which contains the ACL rules."
+               desc => """
+Path to the file which contains the ACL rules.<br>
+If the file provisioned before starting EMQ X node, it can be placed anywhere
+as long as EMQ X has read access to it.
+In case rule set is created from EMQ X dashboard or management HTTP API,
+the file will be placed in `authz` sub directory inside EMQ X's `data_dir`,
+and the new rules will override all rules from the old config file.
+"""
               }}
     ];
 fields(http_get) ->

apps/emqx_authz/test/emqx_authz_api_sources_SUITE.erl

@@ -144,12 +144,10 @@ init_per_testcase(t_api, Config) ->
     meck:expect(emqx_misc, gen_id, fun() -> "fake" end),
 
     meck:new(emqx, [non_strict, passthrough, no_history, no_link]),
-    meck:expect(emqx, get_config, fun([node, data_dir]) ->
-                                          % emqx_common_test_helpers:deps_path(emqx_authz, "test");
-                                          {data_dir, Data} = lists:keyfind(data_dir, 1, Config),
-                                          Data;
-                                     (C) -> meck:passthrough([C])
-                                  end),
+    meck:expect(emqx, data_dir, fun() ->
+                                        {data_dir, Data} = lists:keyfind(data_dir, 1, Config),
+                                        Data
+                                end),
     Config;
 init_per_testcase(_, Config) -> Config.
 
@@ -179,7 +177,7 @@ t_api(_) ->
                  , #{<<"type">> := <<"redis">>}
                  , #{<<"type">> := <<"file">>}
                  ], Sources),
-    ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "acl.conf"]))),
+    ?assert(filelib:is_file(emqx_authz:acl_conf_file())),
 
     {ok, 204, _} = request(put, uri(["authorization", "sources", "http"]),  ?SOURCE1#{<<"enable">> := false}),
     {ok, 200, Result3} = request(get, uri(["authorization", "sources", "http"]), []),
@@ -202,9 +200,9 @@ t_api(_) ->
                                   <<"verify">> := false
                                  }
                   }, jsx:decode(Result4)),
-    ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "certs", "cacert-fake.pem"]))),
-    ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "certs", "cert-fake.pem"]))),
-    ?assert(filelib:is_file(filename:join([emqx:get_config([node, data_dir]), "certs", "key-fake.pem"]))),
+    ?assert(filelib:is_file(filename:join([data_dir(), "certs", "cacert-fake.pem"]))),
+    ?assert(filelib:is_file(filename:join([data_dir(), "certs", "cert-fake.pem"]))),
+    ?assert(filelib:is_file(filename:join([data_dir(), "certs", "key-fake.pem"]))),
 
     lists:foreach(fun(#{<<"type">> := Type}) ->
                     {ok, 204, _} = request(delete, uri(["authorization", "sources", binary_to_list(Type)]), [])
@@ -293,3 +291,5 @@ auth_header_() ->
     Password = <<"public">>,
     {ok, Token} = emqx_dashboard_admin:sign_token(Username, Password),
     {"Authorization", "Bearer " ++ binary_to_list(Token)}.
+
+data_dir() -> emqx:data_dir().

apps/emqx_plugin_libs/src/emqx_plugin_libs_ssl.erl

@@ -42,15 +42,27 @@
 %% @doc Parse ssl options input.
 %% If the input contains file content, save the files in the given dir.
 %% Returns ssl options for Erlang's ssl application.
+%%
+%% For SSL files in the input Option, it can either be a file path
+%% or a map like `#{filename := FileName, file := Content}`.
+%% In case it's a map, the file is saved in EMQ X's `data_dir'
+%% (unless `SubDir' is an absolute path).
+%% NOTE: This function is now deprecated, use emqx_tls_lib:ensure_ssl_files/2 instead.
 -spec save_files_return_opts(opts_input(), atom() | string() | binary(),
                              string() | binary()) -> opts().
 save_files_return_opts(Options, SubDir, ResId) ->
-    Dir = filename:join([emqx:get_config([node, data_dir]), SubDir, ResId]),
+    Dir = filename:join([emqx:data_dir(), SubDir, ResId]),
     save_files_return_opts(Options, Dir).
 
 %% @doc Parse ssl options input.
 %% If the input contains file content, save the files in the given dir.
 %% Returns ssl options for Erlang's ssl application.
+%%
+%% For SSL files in the input Option, it can either be a file path
+%% or a map like `#{filename := FileName, file := Content}`.
+%% In case it's a map, the file is saved in EMQ X's `data_dir'
+%% (unless `SubDir' is an absolute path).
+%% NOTE: This function is now deprecated, use emqx_tls_lib:ensure_ssl_files/2 instead.
 -spec save_files_return_opts(opts_input(), file:name_all()) -> opts().
 save_files_return_opts(Options, Dir) ->
     GetD = fun(Key, Default) -> fuzzy_map_get(Key, Options, Default) end,
@@ -76,7 +88,7 @@ save_files_return_opts(Options, Dir) ->
 %% empty string is returned if the input is empty.
 -spec save_file(file_input(), atom() | string() | binary()) -> string().
 save_file(Param, SubDir) ->
-   Dir = filename:join([emqx:get_config([node, data_dir]), SubDir]),
+   Dir = filename:join([emqx:data_dir(), SubDir]),
    do_save_file(Param, Dir).
 
 filter([]) -> [];