test(s3): verify vhost-style access method works

.ci/docker-compose-file/docker-compose-minio-tcp.yaml

@@ -5,6 +5,8 @@ services:
     hostname: minio
     image: quay.io/minio/minio:${MINIO_TAG}
     command: server --address ":9000" --console-address ":9001" /minio-data
+    environment:
+      MINIO_DOMAIN: "minio.net"
     expose:
       - "9000"
       - "9001"
@@ -18,4 +20,8 @@ services:
       retries: 3
     networks:
       emqx_bridge:
-
+        aliases: 
+          - minio.net
+          - test1.minio.net
+          - test2.minio.net
+          - test3.minio.net

apps/emqx_s3/test/emqx_s3_test_helpers.erl

@@ -13,6 +13,9 @@
 -define(TOXIPROXY_HOST, "toxiproxy").
 -define(TOXIPROXY_PORT, 8474).
 
+-define(MINIO_HOST, "minio.net").
+-define(MINIO_PORT, 9000).
+
 -define(TCP_HOST, ?TOXIPROXY_HOST).
 -define(TCP_PORT, 19000).
 -define(TLS_HOST, ?TOXIPROXY_HOST).
@@ -38,7 +41,9 @@
 aws_config(tcp) ->
     aws_config(tcp, ?TCP_HOST, ?TCP_PORT);
 aws_config(tls) ->
-    aws_config(tls, ?TLS_HOST, ?TLS_PORT).
+    aws_config(tls, ?TLS_HOST, ?TLS_PORT);
+aws_config({tcp, direct}) ->
+    aws_config(tcp, ?MINIO_HOST, ?MINIO_PORT).
 
 aws_config(tcp, Host, Port) ->
     erlcloud_s3_new(
@@ -88,7 +93,15 @@ base_raw_config(tls) ->
                     <<"verify">> => <<"verify_peer">>
                 }
             }
-    }.
+    };
+base_raw_config({tcp, direct}) ->
+    maps:merge(
+        base_raw_config(tcp),
+        #{
+            <<"host">> => ?MINIO_HOST,
+            <<"port">> => ?MINIO_PORT
+        }
+    ).
 
 base_config(ConnType) ->
     emqx_s3_schema:translate(base_raw_config(ConnType)).
@@ -101,13 +114,13 @@ unique_bucket() ->
     "bucket-" ++ integer_to_list(erlang:system_time(millisecond)) ++ "-" ++
         integer_to_list(erlang:unique_integer([positive])).
 
-with_failure(_ConnType, ehttpc_500, Fun) ->
+with_failure(_ConnType, httpc_500, Fun) ->
     try
-        meck:new(ehttpc, [passthrough, no_history]),
-        meck:expect(ehttpc, request, fun(_, _, _, _, _) -> {ok, 500, []} end),
+        meck:new(hackney, [passthrough, no_history]),
+        meck:expect(hackney, request, fun(_, _, _, _, _) -> {ok, 500, []} end),
         Fun()
     after
-        meck:unload(ehttpc)
+        meck:unload(hackney)
     end;
 with_failure(ConnType, FailureType, Fun) ->
     emqx_common_test_helpers:with_failure(
@@ -118,6 +131,29 @@ with_failure(ConnType, FailureType, Fun) ->
         Fun
     ).
 
+%%--------------------------------------------------------------------
+
+recreate_bucket(Bucket, AwsConfig) ->
+    ok = delete_bucket(Bucket, AwsConfig),
+    ok = erlcloud_s3:create_bucket(Bucket, AwsConfig).
+
+delete_bucket(Bucket, AwsConfig) ->
+    try erlcloud_s3:list_objects(Bucket, AwsConfig) of
+        List ->
+            Contents = proplists:get_value(contents, List),
+            ok = lists:foreach(
+                fun(Object) ->
+                    Key = emqx_s3_client:erlcloud_key(proplists:get_value(key, Object)),
+                    _Ok = erlcloud_s3:delete_object(Bucket, Key, AwsConfig)
+                end,
+                Contents
+            ),
+            ok = erlcloud_s3:delete_bucket(Bucket, AwsConfig)
+    catch
+        error:{aws_error, _NotFound} ->
+            ok
+    end.
+
 %%--------------------------------------------------------------------
 %% Internal functions
 %%--------------------------------------------------------------------

apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl

@@ -33,10 +33,13 @@
     )
 ).
 
+-define(VHOST_BUCKET, "test1").
+
 all() ->
     [
         {group, tcp},
-        {group, tls}
+        {group, tls},
+        {group, vhost}
     ].
 
 groups() ->
@@ -49,17 +52,24 @@ groups() ->
             {group, common_cases},
             {group, tls_cases}
         ]},
+        {vhost, [], [
+            {group, happy_cases}
+        ]},
+
         {common_cases, [], [
+            {group, happy_cases},
+            {group, noconn_errors},
+            {group, timeout_errors},
+            {group, http_errors}
+        ]},
+
+        {happy_cases, [], [
             t_happy_path_simple_put,
             t_happy_path_multi,
             t_abort_multi,
             t_abort_simple_put,
             t_signed_url_download,
-            t_signed_nonascii_url_download,
-
-            {group, noconn_errors},
-            {group, timeout_errors},
-            {group, http_errors}
+            t_signed_nonascii_url_download
         ]},
 
         {tcp_cases, [
@@ -97,12 +107,14 @@ end_per_suite(_Config) ->
 
 init_per_group(Group, Config) when Group =:= tcp orelse Group =:= tls ->
     [{conn_type, Group} | Config];
+init_per_group(vhost, Config) ->
+    [{conn_type, {tcp, direct}}, {access_method, vhost} | Config];
 init_per_group(noconn_errors, Config) ->
     [{failure, down} | Config];
 init_per_group(timeout_errors, Config) ->
     [{failure, timeout} | Config];
 init_per_group(http_errors, Config) ->
-    [{failure, ehttpc_500} | Config];
+    [{failure, httpc_500} | Config];
 init_per_group(_ConnType, Config) ->
     Config.
 
@@ -113,15 +125,21 @@ init_per_testcase(_TestCase, Config) ->
     ok = snabbkaffe:start_trace(),
     ConnType = ?config(conn_type, Config),
     TestAwsConfig = emqx_s3_test_helpers:aws_config(ConnType),
-
-    Bucket = emqx_s3_test_helpers:unique_bucket(),
-    ok = erlcloud_s3:create_bucket(Bucket, TestAwsConfig),
-
     ProfileBaseConfig = emqx_s3_test_helpers:base_config(ConnType),
-    ProfileConfig = ProfileBaseConfig#{bucket => Bucket},
+    AccessMethod = proplists:get_value(access_method, Config, path),
+    case AccessMethod of
+        path ->
+            Bucket = emqx_s3_test_helpers:unique_bucket();
+        vhost ->
+            Bucket = ?VHOST_BUCKET
+    end,
+    ProfileConfig = ProfileBaseConfig#{
+        bucket => Bucket,
+        access_method => AccessMethod
+    },
+    ok = emqx_s3_test_helpers:recreate_bucket(Bucket, TestAwsConfig),
     ok = emqx_s3:start_profile(profile_id(), ProfileConfig),
-
-    [{bucket, Bucket}, {test_aws_config, TestAwsConfig}, {profile_config, ProfileConfig} | Config].
+    [{bucket, Bucket}, {profile_config, ProfileConfig}, {test_aws_config, TestAwsConfig} | Config].
 
 end_per_testcase(_TestCase, _Config) ->
     ok = snabbkaffe:stop(),
@@ -205,7 +223,7 @@ t_signed_url_download(_Config) ->
         emqx_s3_client:uri(Client, Key)
     end),
 
-    HttpOpts = [{ssl, [{verify, verify_none}]}],
+    HttpOpts = [{ssl, [{verify, verify_none}]}, {timeout, 5000}],
     {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, HttpOpts, []),
 
     ?assertEqual(
@@ -223,7 +241,7 @@ t_signed_nonascii_url_download(_Config) ->
         emqx_s3_client:uri(Client, Key)
     end),
 
-    HttpOpts = [{ssl, [{verify, verify_none}]}],
+    HttpOpts = [{ssl, [{verify, verify_none}]}, {timeout, 5000}],
     {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, HttpOpts, []),
 
     ?assertEqual(