Merge pull request #13597 from JimMoen/feat-certs-not-in-mutable-dir

feat: store certs not in mutable dir

apps/emqx/src/emqx_listeners.erl

@@ -895,7 +895,7 @@ convert_certs(ListenerConf) ->
 
 convert_certs(Type, Name, Conf) ->
     CertsDir = certs_dir(Type, Name),
-    case emqx_tls_lib:ensure_ssl_files(CertsDir, get_ssl_options(Conf)) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(CertsDir, get_ssl_options(Conf)) of
         {ok, undefined} ->
             Conf;
         {ok, SSL} ->

apps/emqx/src/emqx_tls_lib.erl

@@ -29,6 +29,8 @@
 
 %% SSL files
 -export([
+    ensure_ssl_files_in_mutable_certs_dir/2,
+    ensure_ssl_files_in_mutable_certs_dir/3,
     ensure_ssl_files/2,
     ensure_ssl_files/3,
     drop_invalid_certs/1,
@@ -310,17 +312,28 @@ trim_space(Bin) ->
 %% or file path.
 %% When PEM format key or certificate is given, it tries to to save them in the given
 %% sub-dir in emqx's data_dir, and replace saved file paths for SSL options.
--spec ensure_ssl_files(file:name_all(), undefined | map()) ->
+-spec ensure_ssl_files_in_mutable_certs_dir(file:name_all(), undefined | map()) ->
     {ok, undefined | map()} | {error, map()}.
-ensure_ssl_files(Dir, SSL) ->
-    ensure_ssl_files(Dir, SSL, #{dry_run => false, required_keys => []}).
+ensure_ssl_files_in_mutable_certs_dir(Dir, SSL) ->
+    ensure_ssl_files_in_mutable_certs_dir(Dir, SSL, #{dry_run => false, required_keys => []}).
 
-ensure_ssl_files(_Dir, undefined, _Opts) ->
+ensure_ssl_files_in_mutable_certs_dir(_Dir, undefined, _Opts) ->
     {ok, undefined};
-ensure_ssl_files(_Dir, #{<<"enable">> := False} = SSL, _Opts) when ?IS_FALSE(False) ->
+ensure_ssl_files_in_mutable_certs_dir(_Dir, #{<<"enable">> := False} = SSL, _Opts) when
+    ?IS_FALSE(False)
+->
     {ok, SSL};
-ensure_ssl_files(_Dir, #{enable := False} = SSL, _Opts) when ?IS_FALSE(False) ->
+ensure_ssl_files_in_mutable_certs_dir(_Dir, #{enable := False} = SSL, _Opts) when
+    ?IS_FALSE(False)
+->
     {ok, SSL};
+ensure_ssl_files_in_mutable_certs_dir(Dir, SSL, Opts) ->
+    %% NOTE:
+    %% Pass Raw Dir to keep the file name hash consistent with the previous version
+    ensure_ssl_files(pem_dir(Dir), SSL, Opts#{raw_dir => Dir}).
+
+ensure_ssl_files(Dir, SSL) ->
+    ensure_ssl_files(Dir, SSL, #{dry_run => false, required_keys => [], raw_dir => Dir}).
 ensure_ssl_files(Dir, SSL, Opts) ->
     RequiredKeys = maps:get(required_keys, Opts, []),
     case ensure_ssl_file_key(SSL, RequiredKeys) of
@@ -356,15 +369,19 @@ ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, Opts) ->
     case is_valid_string(MaybePem) of
         true ->
             DryRun = maps:get(dry_run, Opts, false),
-            do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun);
+            RawDir = maps:get(raw_dir, Opts, Dir),
+            %% RawDir for backward compatibility
+            %% when RawDir is not given, it is the same as Dir
+            %% to keep the file name hash consistent with the previous version (Depends on RawDir)
+            do_ensure_ssl_file(Dir, RawDir, KeyPath, SSL, MaybePem, DryRun);
         false ->
             {error, #{reason => invalid_file_path_or_pem_string}}
     end.
 
-do_ensure_ssl_file(Dir, KeyPath, SSL, MaybePem, DryRun) ->
+do_ensure_ssl_file(Dir, RawDir, KeyPath, SSL, MaybePem, DryRun) ->
     case is_pem(MaybePem) of
         true ->
-            case save_pem_file(Dir, KeyPath, MaybePem, DryRun) of
+            case save_pem_file(Dir, RawDir, KeyPath, MaybePem, DryRun) of
                 {ok, Path} ->
                     NewSSL = emqx_utils_maps:deep_put(KeyPath, SSL, Path),
                     {ok, NewSSL};
@@ -410,8 +427,8 @@ is_pem(MaybePem) ->
 %% To make it simple, the file is always overwritten.
 %% Also a potentially half-written PEM file (e.g. due to power outage)
 %% can be corrected with an overwrite.
-save_pem_file(Dir, KeyPath, Pem, DryRun) ->
-    Path = pem_file_name(Dir, KeyPath, Pem),
+save_pem_file(Dir, RawDir, KeyPath, Pem, DryRun) ->
+    Path = pem_file_path(Dir, RawDir, KeyPath, Pem),
     case filelib:ensure_dir(Path) of
         ok when DryRun ->
             {ok, Path};
@@ -434,16 +451,16 @@ is_managed_ssl_file(Filename) ->
         _ -> false
     end.
 
-pem_file_name(Dir, KeyPath, Pem) ->
+pem_file_path(Dir, RawDir, KeyPath, Pem) ->
     % NOTE
     % Wee need to have the same filename on every cluster node.
     Segments = lists:map(fun ensure_bin/1, KeyPath),
     Filename0 = iolist_to_binary(lists:join(<<"_">>, Segments)),
     Filename1 = binary:replace(Filename0, <<"file">>, <<>>),
-    Fingerprint = crypto:hash(md5, [Dir, Filename1, Pem]),
+    Fingerprint = crypto:hash(md5, [RawDir, Filename1, Pem]),
     Suffix = binary:encode_hex(binary:part(Fingerprint, 0, 8)),
     Filename = <<Filename1/binary, "-", Suffix/binary>>,
-    filename:join([pem_dir(Dir), Filename]).
+    filename:join([Dir, Filename]).
 
 pem_dir(Dir) ->
     filename:join([emqx:mutable_certs_dir(), Dir]).

apps/emqx/test/emqx_tls_certfile_gc_SUITE.erl

@@ -55,8 +55,8 @@ t_no_orphans(Config) ->
         <<"certfile">> => cert(),
         <<"cacertfile">> => cert()
     },
-    {ok, SSL} = emqx_tls_lib:ensure_ssl_files("ssl", SSL0),
-    {ok, SSLUnused} = emqx_tls_lib:ensure_ssl_files("unused", SSL0),
+    {ok, SSL} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("ssl", SSL0),
+    {ok, SSLUnused} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("unused", SSL0),
     SSLKeyfile = maps:get(<<"keyfile">>, SSL),
     ok = load_config(#{
         <<"clients">> => [
@@ -97,8 +97,8 @@ t_collect_orphans(_Config) ->
     SSL1 = SSL0#{
         <<"ocsp">> => #{<<"issuer_pem">> => cert()}
     },
-    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files("client", SSL0),
-    {ok, SSL3} = emqx_tls_lib:ensure_ssl_files("server", SSL1),
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("client", SSL0),
+    {ok, SSL3} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("server", SSL1),
     ok = load_config(#{
         <<"clients">> => [
             #{<<"transport">> => #{<<"ssl">> => SSL2}}
@@ -174,10 +174,10 @@ t_gc_runs_periodically(_Config) ->
         <<"keyfile">> => key(),
         <<"certfile">> => cert()
     },
-    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files("s1", SSL),
+    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s1", SSL),
     SSL1Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL1)),
     SSL1Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL1)),
-    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files("s2", SSL#{
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s2", SSL#{
         <<"ocsp">> => #{<<"issuer_pem">> => cert()}
     }),
     SSL2Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL2)),
@@ -275,10 +275,10 @@ t_gc_spares_recreated_certfiles(_Config) ->
         <<"keyfile">> => key(),
         <<"certfile">> => cert()
     },
-    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files("s1", SSL),
+    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s1", SSL),
     SSL1Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL1)),
     SSL1Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL1)),
-    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files("s2", SSL),
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s2", SSL),
     SSL2Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL2)),
     SSL2Certfile = emqx_utils_fs:canonicalize(maps:get(<<"certfile">>, SSL2)),
     ok = load_config(#{}),
@@ -306,7 +306,7 @@ t_gc_spares_recreated_certfiles(_Config) ->
     % Recreate the SSL2 certfiles
     ok = file:delete(SSL2Keyfile),
     ok = file:delete(SSL2Certfile),
-    {ok, _} = emqx_tls_lib:ensure_ssl_files("s2", SSL),
+    {ok, _} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("s2", SSL),
     % Nothing should have been collected
     ?assertMatch(
         {ok, []},
@@ -324,7 +324,7 @@ t_gc_spares_symlinked_datadir(Config) ->
         <<"certfile">> => cert(),
         <<"ocsp">> => #{<<"issuer_pem">> => cert()}
     },
-    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files("srv", SSL),
+    {ok, SSL1} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("srv", SSL),
     SSL1Keyfile = emqx_utils_fs:canonicalize(maps:get(<<"keyfile">>, SSL1)),
 
     ok = load_config(#{

apps/emqx/test/emqx_tls_lib_tests.erl

@@ -90,14 +90,14 @@ ssl_files_failure_test_() ->
         {"undefined_is_undefined", fun() ->
             ?assertEqual(
                 {ok, undefined},
-                emqx_tls_lib:ensure_ssl_files("dir", undefined)
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("dir", undefined)
             )
         end},
         {"no_op_if_disabled", fun() ->
             Disabled = #{<<"enable">> => false, foo => bar},
             ?assertEqual(
                 {ok, Disabled},
-                emqx_tls_lib:ensure_ssl_files("dir", Disabled)
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("dir", Disabled)
             )
         end},
         {"enoent_key_file", fun() ->
@@ -106,7 +106,7 @@ ssl_files_failure_test_() ->
             ),
             ?assertMatch(
                 {error, #{file_read := enoent, pem_check := invalid_pem}},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => NonExistingFile,
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key()
@@ -116,7 +116,7 @@ ssl_files_failure_test_() ->
         {"empty_cacertfile", fun() ->
             ?assertMatch(
                 {ok, _},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => test_key(),
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => <<"">>
@@ -130,7 +130,7 @@ ssl_files_failure_test_() ->
                     reason := pem_file_path_or_string_is_required,
                     which_options := [[<<"keyfile">>]]
                 }},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => <<>>,
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key()
@@ -141,7 +141,7 @@ ssl_files_failure_test_() ->
                 {error, #{
                     reason := invalid_file_path_or_pem_string, which_options := [[<<"keyfile">>]]
                 }},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => <<255, 255>>,
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key()
@@ -152,7 +152,7 @@ ssl_files_failure_test_() ->
                     reason := invalid_file_path_or_pem_string,
                     which_options := [[<<"ocsp">>, <<"issuer_pem">>]]
                 }},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => test_key(),
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key(),
@@ -162,7 +162,7 @@ ssl_files_failure_test_() ->
             %% not printable
             ?assertMatch(
                 {error, #{reason := invalid_file_path_or_pem_string}},
-                emqx_tls_lib:ensure_ssl_files("/tmp", #{
+                emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir("/tmp", #{
                     <<"keyfile">> => <<33, 22>>,
                     <<"certfile">> => test_key(),
                     <<"cacertfile">> => test_key()
@@ -173,7 +173,7 @@ ssl_files_failure_test_() ->
                 ok = file:write_file(TmpFile, <<"not a valid pem">>),
                 ?assertMatch(
                     {error, #{file_read := not_pem}},
-                    emqx_tls_lib:ensure_ssl_files(
+                    emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
                         "/tmp",
                         #{
                             <<"cacertfile">> => bin(TmpFile),
@@ -205,8 +205,8 @@ ssl_file_replace_test() ->
         <<"ocsp">> => #{<<"issuer_pem">> => Key2}
     },
     Dir = filename:join(["/tmp", "ssl-test-dir2"]),
-    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files(Dir, SSL0),
-    {ok, SSL3} = emqx_tls_lib:ensure_ssl_files(Dir, SSL1),
+    {ok, SSL2} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir, SSL0),
+    {ok, SSL3} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir, SSL1),
     File1 = maps:get(<<"keyfile">>, SSL2),
     File2 = maps:get(<<"keyfile">>, SSL3),
     IssuerPem1 = emqx_utils_maps:deep_get([<<"ocsp">>, <<"issuer_pem">>], SSL2),
@@ -224,17 +224,39 @@ ssl_file_deterministic_names_test() ->
     },
     Dir0 = filename:join(["/tmp", ?FUNCTION_NAME, "ssl0"]),
     Dir1 = filename:join(["/tmp", ?FUNCTION_NAME, "ssl1"]),
-    {ok, SSLFiles0} = emqx_tls_lib:ensure_ssl_files(Dir0, SSL0),
+    {ok, SSLFiles0} = emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir0, SSL0),
     ?assertEqual(
         {ok, SSLFiles0},
-        emqx_tls_lib:ensure_ssl_files(Dir0, SSL0)
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir0, SSL0)
     ),
     ?assertNotEqual(
         {ok, SSLFiles0},
-        emqx_tls_lib:ensure_ssl_files(Dir1, SSL0)
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(Dir1, SSL0)
     ),
     _ = file:del_dir_r(filename:join(["/tmp", ?FUNCTION_NAME])).
 
+ssl_file_name_hash_test() ->
+    ?assertMatch(
+        {ok, #{
+            <<"cacertfile">> := <<"data/certs/authz/http/cacert-C62234D748AB82B0">>,
+            <<"certfile">> := <<"data/certs/authz/http/cert-0D6E53DBDEF594A4">>,
+            <<"enable">> := true,
+            <<"keyfile">> := <<"data/certs/authz/http/key-D5BB7F027841FA62">>,
+            <<"verify">> := <<"verify_peer">>
+        }},
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
+            <<"authz/http">>,
+            #{
+                <<"cacertfile">> => test_name_hash_cacert(),
+                <<"certfile">> => test_name_hash_cert(),
+                <<"keyfile">> => test_name_hash_key(),
+                <<"enable">> => true,
+                <<"verify">> => <<"verify_peer">>
+            }
+        )
+    ),
+    ok.
+
 to_client_opts_test() ->
     VersionsAll = [tlsv1, 'tlsv1.1', 'tlsv1.2', 'tlsv1.3'],
     Versions13Only = ['tlsv1.3'],
@@ -329,3 +351,81 @@ test_key2() ->
         "AbToUD4JmV9m/XwcSVH06ZaWqNuC5w==\n"
         "-----END EC PRIVATE KEY-----\n"
     >>.
+
+test_name_hash_cacert() ->
+    <<
+        "-----BEGIN CERTIFICATE-----\n"
+        "MIIDUTCCAjmgAwIBAgIJAPPYCjTmxdt/MA0GCSqGSIb3DQEBCwUAMD8xCzAJBgNV\n"
+        "BAYTAkNOMREwDwYDVQQIDAhoYW5nemhvdTEMMAoGA1UECgwDRU1RMQ8wDQYDVQQD\n"
+        "DAZSb290Q0EwHhcNMjAwNTA4MDgwNjUyWhcNMzAwNTA2MDgwNjUyWjA/MQswCQYD\n"
+        "VQQGEwJDTjERMA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UE\n"
+        "AwwGUm9vdENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzcgVLex1\n"
+        "EZ9ON64EX8v+wcSjzOZpiEOsAOuSXOEN3wb8FKUxCdsGrsJYB7a5VM/Jot25Mod2\n"
+        "juS3OBMg6r85k2TWjdxUoUs+HiUB/pP/ARaaW6VntpAEokpij/przWMPgJnBF3Ur\n"
+        "MjtbLayH9hGmpQrI5c2vmHQ2reRZnSFbY+2b8SXZ+3lZZgz9+BaQYWdQWfaUWEHZ\n"
+        "uDaNiViVO0OT8DRjCuiDp3yYDj3iLWbTA/gDL6Tf5XuHuEwcOQUrd+h0hyIphO8D\n"
+        "tsrsHZ14j4AWYLk1CPA6pq1HIUvEl2rANx2lVUNv+nt64K/Mr3RnVQd9s8bK+TXQ\n"
+        "KGHd2Lv/PALYuwIDAQABo1AwTjAdBgNVHQ4EFgQUGBmW+iDzxctWAWxmhgdlE8Pj\n"
+        "EbQwHwYDVR0jBBgwFoAUGBmW+iDzxctWAWxmhgdlE8PjEbQwDAYDVR0TBAUwAwEB\n"
+        "/zANBgkqhkiG9w0BAQsFAAOCAQEAGbhRUjpIred4cFAFJ7bbYD9hKu/yzWPWkMRa\n"
+        "ErlCKHmuYsYk+5d16JQhJaFy6MGXfLgo3KV2itl0d+OWNH0U9ULXcglTxy6+njo5\n"
+        "CFqdUBPwN1jxhzo9yteDMKF4+AHIxbvCAJa17qcwUKR5MKNvv09C6pvQDJLzid7y\n"
+        "E2dkgSuggik3oa0427KvctFf8uhOV94RvEDyqvT5+pgNYZ2Yfga9pD/jjpoHEUlo\n"
+        "88IGU8/wJCx3Ds2yc8+oBg/ynxG8f/HmCC1ET6EHHoe2jlo8FpU/SgGtghS1YL30\n"
+        "IWxNsPrUP+XsZpBJy/mvOhE5QXo6Y35zDqqj8tI7AGmAWu22jg==\n"
+        "-----END CERTIFICATE-----\n"
+    >>.
+
+test_name_hash_cert() ->
+    <<
+        "-----BEGIN CERTIFICATE-----\n"
+        "MIIDEzCCAfugAwIBAgIBAjANBgkqhkiG9w0BAQsFADA/MQswCQYDVQQGEwJDTjER\n"
+        "MA8GA1UECAwIaGFuZ3pob3UxDDAKBgNVBAoMA0VNUTEPMA0GA1UEAwwGUm9vdENB\n"
+        "MB4XDTIwMDUwODA4MDcwNVoXDTMwMDUwNjA4MDcwNVowPzELMAkGA1UEBhMCQ04x\n"
+        "ETAPBgNVBAgMCGhhbmd6aG91MQwwCgYDVQQKDANFTVExDzANBgNVBAMMBlNlcnZl\n"
+        "cjCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALNeWT3pE+QFfiRJzKmn\n"
+        "AMUrWo3K2j/Tm3+Xnl6WLz67/0rcYrJbbKvS3uyRP/stXyXEKw9CepyQ1ViBVFkW\n"
+        "Aoy8qQEOWFDsZc/5UzhXUnb6LXr3qTkFEjNmhj+7uzv/lbBxlUG1NlYzSeOB6/RT\n"
+        "8zH/lhOeKhLnWYPXdXKsa1FL6ij4X8DeDO1kY7fvAGmBn/THh1uTpDizM4YmeI+7\n"
+        "4dmayA5xXvARte5h4Vu5SIze7iC057N+vymToMk2Jgk+ZZFpyXrnq+yo6RaD3ANc\n"
+        "lrc4FbeUQZ5a5s5Sxgs9a0Y3WMG+7c5VnVXcbjBRz/aq2NtOnQQjikKKQA8GF080\n"
+        "BQkCAwEAAaMaMBgwCQYDVR0TBAIwADALBgNVHQ8EBAMCBeAwDQYJKoZIhvcNAQEL\n"
+        "BQADggEBAJefnMZpaRDHQSNUIEL3iwGXE9c6PmIsQVE2ustr+CakBp3TZ4l0enLt\n"
+        "iGMfEVFju69cO4oyokWv+hl5eCMkHBf14Kv51vj448jowYnF1zmzn7SEzm5Uzlsa\n"
+        "sqjtAprnLyof69WtLU1j5rYWBuFX86yOTwRAFNjm9fvhAcrEONBsQtqipBWkMROp\n"
+        "iUYMkRqbKcQMdwxov+lHBYKq9zbWRoqLROAn54SRqgQk6c15JdEfgOOjShbsOkIH\n"
+        "UhqcwRkQic7n1zwHVGVDgNIZVgmJ2IdIWBlPEC7oLrRrBD/X1iEEXtKab6p5o22n\n"
+        "KB5mN+iQaE+Oe2cpGKZJiJRdM+IqDDQ=\n"
+        "-----END CERTIFICATE-----\n"
+    >>.
+
+test_name_hash_key() ->
+    <<
+        "-----BEGIN RSA PRIVATE KEY-----\n"
+        "MIIEowIBAAKCAQEAs15ZPekT5AV+JEnMqacAxStajcraP9Obf5eeXpYvPrv/Stxi\n"
+        "sltsq9Le7JE/+y1fJcQrD0J6nJDVWIFUWRYCjLypAQ5YUOxlz/lTOFdSdvotevep\n"
+        "OQUSM2aGP7u7O/+VsHGVQbU2VjNJ44Hr9FPzMf+WE54qEudZg9d1cqxrUUvqKPhf\n"
+        "wN4M7WRjt+8AaYGf9MeHW5OkOLMzhiZ4j7vh2ZrIDnFe8BG17mHhW7lIjN7uILTn\n"
+        "s36/KZOgyTYmCT5lkWnJeuer7KjpFoPcA1yWtzgVt5RBnlrmzlLGCz1rRjdYwb7t\n"
+        "zlWdVdxuMFHP9qrY206dBCOKQopADwYXTzQFCQIDAQABAoIBAQCuvCbr7Pd3lvI/\n"
+        "n7VFQG+7pHRe1VKwAxDkx2t8cYos7y/QWcm8Ptwqtw58HzPZGWYrgGMCRpzzkRSF\n"
+        "V9g3wP1S5Scu5C6dBu5YIGc157tqNGXB+SpdZddJQ4Nc6yGHXYERllT04ffBGc3N\n"
+        "WG/oYS/1cSteiSIrsDy/91FvGRCi7FPxH3wIgHssY/tw69s1Cfvaq5lr2NTFzxIG\n"
+        "xCvpJKEdSfVfS9I7LYiymVjst3IOR/w76/ZFY9cRa8ZtmQSWWsm0TUpRC1jdcbkm\n"
+        "ZoJptYWlP+gSwx/fpMYftrkJFGOJhHJHQhwxT5X/ajAISeqjjwkWSEJLwnHQd11C\n"
+        "Zy2+29lBAoGBANlEAIK4VxCqyPXNKfoOOi5dS64NfvyH4A1v2+KaHWc7lqaqPN49\n"
+        "ezfN2n3X+KWx4cviDD914Yc2JQ1vVJjSaHci7yivocDo2OfZDmjBqzaMp/y+rX1R\n"
+        "/f3MmiTqMa468rjaxI9RRZu7vDgpTR+za1+OBCgMzjvAng8dJuN/5gjlAoGBANNY\n"
+        "uYPKtearBmkqdrSV7eTUe49Nhr0XotLaVBH37TCW0Xv9wjO2xmbm5Ga/DCtPIsBb\n"
+        "yPeYwX9FjoasuadUD7hRvbFu6dBa0HGLmkXRJZTcD7MEX2Lhu4BuC72yDLLFd0r+\n"
+        "Ep9WP7F5iJyagYqIZtz+4uf7gBvUDdmvXz3sGr1VAoGAdXTD6eeKeiI6PlhKBztF\n"
+        "zOb3EQOO0SsLv3fnodu7ZaHbUgLaoTMPuB17r2jgrYM7FKQCBxTNdfGZmmfDjlLB\n"
+        "0xZ5wL8ibU30ZXL8zTlWPElST9sto4B+FYVVF/vcG9sWeUUb2ncPcJ/Po3UAktDG\n"
+        "jYQTTyuNGtSJHpad/YOZctkCgYBtWRaC7bq3of0rJGFOhdQT9SwItN/lrfj8hyHA\n"
+        "OjpqTV4NfPmhsAtu6j96OZaeQc+FHvgXwt06cE6Rt4RG4uNPRluTFgO7XYFDfitP\n"
+        "vCppnoIw6S5BBvHwPP+uIhUX2bsi/dm8vu8tb+gSvo4PkwtFhEr6I9HglBKmcmog\n"
+        "q6waEQKBgHyecFBeM6Ls11Cd64vborwJPAuxIW7HBAFj/BS99oeG4TjBx4Sz2dFd\n"
+        "rzUibJt4ndnHIvCN8JQkjNG14i9hJln+H3mRss8fbZ9vQdqG+2vOWADYSzzsNI55\n"
+        "RFY7JjluKcVkp/zCDeUxTU3O6sS+v6/3VE11Cob6OYQx3lN5wrZ3\n"
+        "-----END RSA PRIVATE KEY-----\n"
+    >>.

apps/emqx_auth/src/emqx_authn/emqx_authn_config.erl

@@ -280,7 +280,7 @@ convert_certs_for_conf_path(ConfPath, NewConfig) ->
 
 convert_certs(CertsDir, NewConfig) ->
     NewSSL = maps:get(<<"ssl">>, NewConfig, undefined),
-    case emqx_tls_lib:ensure_ssl_files(CertsDir, NewSSL) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(CertsDir, NewSSL) of
         {ok, NewSSL1} ->
             new_ssl_config(NewConfig, NewSSL1);
         {error, Reason} ->

apps/emqx_auth/src/emqx_authz/emqx_authz.erl

@@ -745,7 +745,7 @@ maybe_read_source_files_safe(Source0) ->
     end.
 
 maybe_write_certs(#{<<"type">> := Type, <<"ssl">> := SSL = #{}} = Source) ->
-    case emqx_tls_lib:ensure_ssl_files(ssl_file_path(Type), SSL) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(ssl_file_path(Type), SSL) of
         {ok, NSSL} ->
             Source#{<<"ssl">> => NSSL};
         {error, Reason} ->

apps/emqx_cluster_link/src/emqx_cluster_link_config.erl

@@ -397,7 +397,7 @@ convert_certs(LinksConf) ->
     ).
 
 do_convert_certs(LinkName, SSLOpts) ->
-    case emqx_tls_lib:ensure_ssl_files(?CERTS_PATH(LinkName), SSLOpts) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(?CERTS_PATH(LinkName), SSLOpts) of
         {ok, undefined} ->
             SSLOpts;
         {ok, SSLOpts1} ->

apps/emqx_connector/src/emqx_connector_ssl.erl

@@ -31,7 +31,7 @@ convert_certs(_RltvDir, Config) ->
     {ok, Config}.
 
 new_ssl_config(RltvDir, Config, SSL) ->
-    case emqx_tls_lib:ensure_ssl_files(RltvDir, SSL) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(RltvDir, SSL) of
         {ok, NewSSL} ->
             {ok, new_ssl_config(Config, NewSSL)};
         {error, Reason} ->

apps/emqx_dashboard/src/emqx_dashboard_listener.erl

@@ -189,7 +189,7 @@ ensure_ssl_cert(#{<<"listeners">> := #{<<"https">> := #{<<"bind">> := Bind} = Ht
     Conf1 = emqx_utils_maps:deep_put([<<"listeners">>, <<"https">>], Conf0, Https1),
     Ssl = maps:get(<<"ssl_options">>, Https1, undefined),
     Opts = #{required_keys => [[<<"keyfile">>], [<<"certfile">>], [<<"cacertfile">>]]},
-    case emqx_tls_lib:ensure_ssl_files(?DIR, Ssl, Opts) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(?DIR, Ssl, Opts) of
         {ok, undefined} ->
             {error, <<"ssl_cert_not_found">>};
         {ok, NewSsl} ->

apps/emqx_dashboard_sso/src/emqx_dashboard_sso_ldap.erl

@@ -163,7 +163,7 @@ ensure_user_exists(Username) ->
 
 convert_certs(Dir, Conf) ->
     case
-        emqx_tls_lib:ensure_ssl_files(
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
             Dir, maps:get(<<"ssl">>, Conf, undefined)
         )
     of

apps/emqx_dashboard_sso/src/emqx_dashboard_sso_saml.erl

@@ -162,7 +162,7 @@ convert_certs(
         Conf
 ) ->
     case
-        emqx_tls_lib:ensure_ssl_files(
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
             Dir, #{enable => true, certfile => Cert, keyfile => Key}, #{}
         )
     of

apps/emqx_exhook/src/emqx_exhook_mgr.erl

@@ -658,7 +658,7 @@ hooks(Name) ->
 
 maybe_write_certs(#{<<"name">> := Name} = Conf) ->
     case
-        emqx_tls_lib:ensure_ssl_files(
+        emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(
             ssl_file_path(Name), maps:get(<<"ssl">>, Conf, undefined)
         )
     of

apps/emqx_gateway/src/emqx_gateway_conf.erl

@@ -887,7 +887,7 @@ convert_certs(SubDir, Conf) ->
 
 convert_certs(Type, SubDir, Conf) ->
     SSL = maps:get(Type, Conf, undefined),
-    case is_map(SSL) andalso emqx_tls_lib:ensure_ssl_files(SubDir, SSL) of
+    case is_map(SSL) andalso emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(SubDir, SSL) of
         false ->
             Conf;
         {ok, NSSL = #{}} ->

apps/emqx_opentelemetry/src/emqx_otel_config.erl

@@ -108,7 +108,7 @@ convert_exporter_certs(ExporterConf) ->
     ExporterConf.
 
 do_convert_certs(SSLOpts) ->
-    case emqx_tls_lib:ensure_ssl_files(?CERTS_PATH, SSLOpts) of
+    case emqx_tls_lib:ensure_ssl_files_in_mutable_certs_dir(?CERTS_PATH, SSLOpts) of
         {ok, undefined} ->
             SSLOpts;
         {ok, SSLOpts1} ->

apps/emqx_plugins/src/emqx_plugins.erl

@@ -75,7 +75,9 @@
     install_dir/0,
     avsc_file_path/1,
     md5sum_file/1,
-    with_plugin_avsc/1
+    with_plugin_avsc/1,
+    ensure_ssl_files/2,
+    ensure_ssl_files/3
 ]).
 
 %% `emqx_config_handler' API
@@ -514,6 +516,12 @@ get_tar(NameVsn) ->
             end
     end.
 
+ensure_ssl_files(NameVsn, SSL) ->
+    emqx_tls_lib:ensure_ssl_files(plugin_certs_dir(NameVsn), SSL).
+
+ensure_ssl_files(NameVsn, SSL, Opts) ->
+    emqx_tls_lib:ensure_ssl_files(plugin_certs_dir(NameVsn), SSL, Opts).
+
 %%--------------------------------------------------------------------
 %% Internal
 %%--------------------------------------------------------------------
@@ -1049,19 +1057,22 @@ do_load_plugin_app(AppName, Ebin) ->
     end.
 
 start_app(App) ->
-    case application:ensure_all_started(App) of
-        {ok, Started} ->
+    case run_with_timeout(application, ensure_all_started, [App], 10_000) of
+        {ok, {ok, Started}} ->
             case Started =/= [] of
                 true -> ?SLOG(debug, #{msg => "started_plugin_apps", apps => Started});
                 false -> ok
-            end,
-            ?SLOG(debug, #{msg => "started_plugin_app", app => App}),
-            ok;
-        {error, {ErrApp, Reason}} ->
+            end;
+        {ok, {error, Reason}} ->
+            throw(#{
+                msg => "failed_to_start_app",
+                app => App,
+                reason => Reason
+            });
+        {error, Reason} ->
             throw(#{
                 msg => "failed_to_start_plugin_app",
                 app => App,
-                err_app => ErrApp,
                 reason => Reason
             })
     end.
@@ -1287,7 +1298,7 @@ maybe_create_config_dir(NameVsn, Mode) ->
         do_create_config_dir(NameVsn, Mode).
 
 do_create_config_dir(NameVsn, Mode) ->
-    case plugin_config_dir(NameVsn) of
+    case plugin_data_dir(NameVsn) of
         {error, Reason} ->
             {error, {gen_config_dir_failed, Reason}};
         ConfigDir ->
@@ -1329,7 +1340,7 @@ ensure_plugin_config({NameVsn, ?fresh_install}) ->
 -spec ensure_plugin_config(name_vsn(), list()) -> ok.
 ensure_plugin_config(NameVsn, []) ->
     ?SLOG(debug, #{
-        msg => "default_plugin_config_used",
+        msg => "local_plugin_config_used",
         name_vsn => NameVsn,
         reason => "no_other_running_nodes"
     }),
@@ -1355,7 +1366,13 @@ cp_default_config_file(NameVsn) ->
     maybe
         true ?= filelib:is_regular(Source),
         %% destination path not existed (not configured)
-        true ?= (not filelib:is_regular(Destination)),
+        false ?=
+            case filelib:is_regular(Destination) of
+                true ->
+                    ?SLOG(debug, #{msg => "plugin_config_file_already_existed", name_vsn => NameVsn});
+                false ->
+                    false
+            end,
         ok = filelib:ensure_dir(Destination),
         case file:copy(Source, Destination) of
             {ok, _} ->
@@ -1506,8 +1523,8 @@ plugin_priv_dir(NameVsn) ->
         _ -> wrap_to_list(filename:join([install_dir(), NameVsn, "priv"]))
     end.
 
--spec plugin_config_dir(name_vsn()) -> string() | {error, Reason :: string()}.
-plugin_config_dir(NameVsn) ->
+-spec plugin_data_dir(name_vsn()) -> string() | {error, Reason :: string()}.
+plugin_data_dir(NameVsn) ->
     case parse_name_vsn(NameVsn) of
         {ok, NameAtom, _Vsn} ->
             wrap_to_list(filename:join([emqx:data_dir(), "plugins", atom_to_list(NameAtom)]));
@@ -1520,6 +1537,9 @@ plugin_config_dir(NameVsn) ->
             {error, Reason}
     end.
 
+plugin_certs_dir(NameVsn) ->
+    wrap_to_list(filename:join([plugin_data_dir(NameVsn), "certs"])).
+
 %% Files
 -spec pkg_file_path(name_vsn()) -> string().
 pkg_file_path(NameVsn) ->
@@ -1535,7 +1555,7 @@ avsc_file_path(NameVsn) ->
 
 -spec plugin_config_file(name_vsn()) -> string().
 plugin_config_file(NameVsn) ->
-    wrap_to_list(filename:join([plugin_config_dir(NameVsn), "config.hocon"])).
+    wrap_to_list(filename:join([plugin_data_dir(NameVsn), "config.hocon"])).
 
 %% should only used when plugin installing
 -spec default_plugin_config_file(name_vsn()) -> string().
@@ -1586,3 +1606,20 @@ bin(B) when is_binary(B) -> B.
 
 wrap_to_list(Path) ->
     binary_to_list(iolist_to_binary(Path)).
+
+run_with_timeout(Module, Function, Args, Timeout) ->
+    Self = self(),
+    Fun = fun() ->
+        Result = apply(Module, Function, Args),
+        Self ! {self(), Result}
+    end,
+    Pid = spawn(Fun),
+    TimerRef = erlang:send_after(Timeout, self(), {timeout, Pid}),
+    receive
+        {Pid, Result} ->
+            _ = erlang:cancel_timer(TimerRef),
+            {ok, Result};
+        {timeout, Pid} ->
+            exit(Pid, kill),
+            {error, timeout}
+    end.

changes/ce/feat-13597.en.md

@@ -0,0 +1,2 @@
+Add thin wrapper functions for plugins to let them store and manage the certificate files used by the plugins themselves.
+This can prevent the certificates used by the plugins from being accidentally deleted by the certificate gc function.

changes/ce/fix-13552.en.md

@@ -0,0 +1,8 @@
+Add a startup timeout limit for the plug-in application. Currently the timeout is 10 seconds.
+
+Starting a bad plugin while EMQX is running will result in a thrown runtime error.
+When EMQX is closed and restarted, the main starting process may hang due to the the plugin application to start failures.
+
+This may happen in the following cases:
+- After closing EMQX, the EMQX's configuration file is changed, and make the bad plugin is configured to enable.
+- After closing EMQX, the Plugin's configuration file is changed, so that the plugin cannot be started with the bad configuration file.