Merge pull request #4151 from emqx/v4.3.0-to-v5.0-resolve-tls-config-conflicts

Auto-pull-request-on-2021-02-05

apps/emqx_exproto/test/emqx_exproto_SUITE.erl

@@ -424,30 +424,31 @@ udp_opts() ->
      {reuseaddr, true}].
 
 ssl_opts() ->
-    Path = emqx_ct_helpers:deps_path(emqx, "etc/certs"),
+    Certs = certs("key.pem", "cert.pem", "cacert.pem"),
     [{versions, ['tlsv1.2','tlsv1.1',tlsv1]},
-     {ciphers, ciphers()},
-     {keyfile, Path ++ "/key.pem"},
-     {certfile, Path ++ "/cert.pem"},
-     {cacertfile, Path ++ "/cacert.pem"},
+     {ciphers, ciphers('tlsv1.2')},
      {verify, verify_peer},
      {fail_if_no_peer_cert, true},
      {secure_renegotiate, false},
      {reuse_sessions, true},
-     {honor_cipher_order, true}].
+     {honor_cipher_order, true}]++Certs.
 
 dtls_opts() ->
     Opts = ssl_opts(),
     lists:keyreplace(versions, 1, Opts, {versions, ['dtlsv1.2', 'dtlsv1']}).
 
-ciphers() ->
-    proplists:get_value(ciphers, emqx_ct_helpers:client_ssl()).
+ciphers(Version) ->
+    proplists:get_value(ciphers, emqx_ct_helpers:client_ssl(Version)).
 
 %%--------------------------------------------------------------------
 %% Client-Opts
 
 client_ssl_opts() ->
-    Path = emqx_ct_helpers:deps_path(emqx, "etc/certs"),
-    [{keyfile, Path ++ "/client-key.pem"},
-     {certfile, Path ++ "/client-cert.pem"},
-     {cacertfile, Path ++ "/cacert.pem"}].
+    certs( "client-key.pem", "client-cert.pem", "cacert.pem" ).
+
+certs( Key, Cert, CACert ) ->
+    CertsPath = emqx_ct_helpers:deps_path(emqx, "etc/certs"),
+    [ { keyfile,    filename:join([ CertsPath, Key    ]) },
+      { certfile,   filename:join([ CertsPath, Cert   ]) },
+      { cacertfile, filename:join([ CertsPath, CACert ]) } ].
+

apps/emqx_rule_engine/src/emqx_rule_utils.erl

@@ -27,6 +27,7 @@
         , preproc_sql/2
         , proc_sql/2
         , proc_sql_param_str/2
+        , proc_cql_param_str/2
         ]).
 
 %% type converting
@@ -145,8 +146,15 @@ proc_sql(Tokens, Data) ->
 
 -spec(proc_sql_param_str(tmpl_token(), map()) -> binary()).
 proc_sql_param_str(Tokens, Data) ->
+    proc_param_str(Tokens, Data, fun quote_sql/1).
+
+-spec(proc_cql_param_str(tmpl_token(), map()) -> binary()).
+proc_cql_param_str(Tokens, Data) ->
+    proc_param_str(Tokens, Data, fun quote_cql/1).
+
+proc_param_str(Tokens, Data, Quote) ->
     iolist_to_binary(
-      proc_tmpl(Tokens, Data, #{return => rawlist, var_trans => fun quote/1})).
+      proc_tmpl(Tokens, Data, #{return => rawlist, var_trans => Quote})).
 
 %% backward compatibility for hot upgrading from =< e4.2.1
 get_phld_var(Fun, Data) when is_function(Fun) ->
@@ -238,12 +246,23 @@ sql_data(Bool) when is_boolean(Bool) -> Bool;
 sql_data(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);
 sql_data(Map) when is_map(Map) -> emqx_json:encode(Map).
 
-quote(List) when is_list(List) -> [$', List, $'];
-quote(Bin) when is_binary(Bin) -> [$', Bin, $'];
-quote(Num) when is_number(Num) -> bin(Num);
-quote(Bool) when is_boolean(Bool) -> bin(Bool);
-quote(Atom) when is_atom(Atom) -> [$', atom_to_binary(Atom, utf8), $'];
-quote(Map) when is_map(Map) -> [$', emqx_json:encode(Map), $'].
+quote_sql(Str) ->
+    quote(Str, <<"\\\\'">>).
+
+quote_cql(Str) ->
+    quote(Str, <<"''">>).
+
+quote(Str, ReplaceWith) when
+        is_list(Str);
+        is_binary(Str);
+        is_atom(Str);
+        is_map(Str) ->
+    [$', escape_apo(bin(Str), ReplaceWith), $'];
+quote(Val, _) ->
+    bin(Val).
+
+escape_apo(Str, ReplaceWith) ->
+    re:replace(Str, <<"'">>, ReplaceWith, [{return, binary}, global]).
 
 str(Bin) when is_binary(Bin) -> binary_to_list(Bin);
 str(Num) when is_number(Num) -> number_to_list(Num);

apps/emqx_rule_engine/test/emqx_rule_utils_SUITE.erl

@@ -116,3 +116,21 @@ t_preproc_sql3(_) ->
     ParamsTokens = emqx_rule_utils:preproc_tmpl(<<"a:${a},b:${b},c:${c},d:${d}">>),
     ?assertEqual(<<"a:'1',b:1,c:1.0,d:'{\"d1\":\"hi\"}'">>,
                  emqx_rule_utils:proc_sql_param_str(ParamsTokens, Selected)).
+
+t_preproc_sql4(_) ->
+    %% with apostrophes
+    %% https://github.com/emqx/emqx/issues/4135
+    Selected = #{a => <<"1''2">>, b => 1, c => 1.0,
+                 d => #{d1 => <<"someone's phone">>}},
+    ParamsTokens = emqx_rule_utils:preproc_tmpl(<<"a:${a},b:${b},c:${c},d:${d}">>),
+    ?assertEqual(<<"a:'1\\'\\'2',b:1,c:1.0,d:'{\"d1\":\"someone\\'s phone\"}'">>,
+                 emqx_rule_utils:proc_sql_param_str(ParamsTokens, Selected)).
+
+t_preproc_sql5(_) ->
+    %% with apostrophes for cassandra
+    %% https://github.com/emqx/emqx/issues/4148
+    Selected = #{a => <<"1''2">>, b => 1, c => 1.0,
+                 d => #{d1 => <<"someone's phone">>}},
+    ParamsTokens = emqx_rule_utils:preproc_tmpl(<<"a:${a},b:${b},c:${c},d:${d}">>),
+    ?assertEqual(<<"a:'1''''2',b:1,c:1.0,d:'{\"d1\":\"someone''s phone\"}'">>,
+                 emqx_rule_utils:proc_cql_param_str(ParamsTokens, Selected)).

etc/emqx.d/acl.conf

@@ -1,26 +0,0 @@
-%%--------------------------------------------------------------------
-%% [ACL](https://docs.emqx.io/broker/v3/en/config.html)
-%%
-%% -type(who() :: all | binary() |
-%%                {ipaddr, esockd_access:cidr()} |
-%%                {client, binary()} |
-%%                {user, binary()}).
-%%
-%% -type(access() :: subscribe | publish | pubsub).
-%%
-%% -type(topic() :: binary()).
-%%
-%% -type(rule() :: {allow, all} |
-%%                 {allow, who(), access(), list(topic())} |
-%%                 {deny, all} |
-%%                 {deny, who(), access(), list(topic())}).
-%%--------------------------------------------------------------------
-
-{allow, {user, "dashboard"}, subscribe, ["$SYS/#"]}.
-
-{allow, {ipaddr, "127.0.0.1"}, pubsub, ["$SYS/#", "#"]}.
-
-{deny, all, subscribe, ["$SYS/#", {eq, "#"}]}.
-
-{allow, all}.
-

etc/emqx.d/ssl_dist.conf

@@ -1,11 +0,0 @@
-%% The options in the {server, Opts} tuple are used when calling ssl:ssl_accept/3,
-%% and the options in the {client, Opts} tuple are used when calling ssl:connect/4.
-%%
-%% More information at: http://erlang.org/doc/apps/ssl/ssl_distribution.html
-[{server,
-  [{certfile, "{{ platform_etc_dir }}/certs/cert.pem"},
-   {keyfile, "{{ platform_etc_dir }}/certs/key.pem"},
-   {secure_renegotiate, true},
-   {depth, 0}]},
- {client,
-  [{secure_renegotiate, true}]}].

test/emqx_client_SUITE.erl

@@ -70,7 +70,9 @@ groups() ->
       ]},
      {others, [non_parallel_tests],
       [t_username_as_clientid,
-       t_certcn_as_clientid
+       t_certcn_as_clientid_default_config_tls,
+       t_certcn_as_clientid_tlsv1_3,
+       t_certcn_as_clientid_tlsv1_2
       ]}
     ].
 
@@ -278,14 +280,18 @@ t_username_as_clientid(_) ->
     #{clientinfo := #{clientid := Username}} = emqx_cm:get_chan_info(Username),
     emqtt:disconnect(C).
 
-t_certcn_as_clientid(_) ->
-    CN = <<"Client">>,
-    emqx_zone:set_env(external, use_username_as_clientid, true),
-    SslConf = emqx_ct_helpers:client_ssl_twoway(),
-    {ok, C} = emqtt:start_link([{port, 8883}, {ssl, true}, {ssl_opts, SslConf}]),
-    {ok, _} = emqtt:connect(C),
-    #{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN),
-    emqtt:disconnect(C).
+
+
+t_certcn_as_clientid_default_config_tls(_) ->
+    tls_certcn_as_clientid(default).
+
+t_certcn_as_clientid_tlsv1_3(_) ->
+    tls_certcn_as_clientid('tlsv1.3').
+
+t_certcn_as_clientid_tlsv1_2(_) ->
+    tls_certcn_as_clientid('tlsv1.2').
+
+
 
 %%--------------------------------------------------------------------
 %% Helper functions
@@ -304,3 +310,29 @@ recv_msgs(Count, Msgs) ->
     after 100 ->
         Msgs
     end.
+
+
+confirm_tls_version( Client, RequiredProtocol ) ->
+    Info = emqtt:info(Client),
+    SocketInfo = proplists:get_value( socket, Info ),
+    %% emqtt_sock has #ssl_socket.ssl
+    SSLSocket = element( 3, SocketInfo ),
+    { ok, SSLInfo } = ssl:connection_information(SSLSocket),
+    Protocol = proplists:get_value( protocol, SSLInfo ),
+    RequiredProtocol = Protocol.
+
+
+tls_certcn_as_clientid(default = TLSVsn) ->
+    tls_certcn_as_clientid(TLSVsn, 'tlsv1.3');
+tls_certcn_as_clientid(TLSVsn) ->
+    tls_certcn_as_clientid(TLSVsn, TLSVsn).
+
+tls_certcn_as_clientid(TLSVsn, RequiredTLSVsn) ->
+    CN = <<"Client">>,
+    emqx_zone:set_env(external, use_username_as_clientid, true),
+    SslConf = emqx_ct_helpers:client_ssl_twoway(TLSVsn),
+    {ok, Client} = emqtt:start_link([{port, 8883}, {ssl, true}, {ssl_opts, SslConf}]),
+    {ok, _} = emqtt:connect(Client),
+    #{clientinfo := #{clientid := CN}} = emqx_cm:get_chan_info(CN),
+    confirm_tls_version( Client, RequiredTLSVsn ),
+    emqtt:disconnect(Client).