fix(ft-s3): use AWS4 signed urls for S3 export URIs

This will ensure that the S3 export URIs are valid in all AWS regions.

apps/emqx_s3/rebar.config

@@ -1,6 +1,6 @@
 {deps, [
     {emqx, {path, "../../apps/emqx"}},
-    {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}}
+    {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}}
 ]}.
 
 {project_plugins, [erlfmt]}.

apps/emqx_s3/src/emqx_s3_client.erl

@@ -189,7 +189,7 @@ list(#{bucket := Bucket, aws_config := AwsConfig}, Options) ->
 
 -spec uri(client(), key()) -> iodata().
 uri(#{bucket := Bucket, aws_config := AwsConfig, url_expire_time := ExpireTime}, Key) ->
-    erlcloud_s3:make_get_url(ExpireTime, Bucket, erlcloud_key(Key), AwsConfig).
+    erlcloud_s3:make_presigned_v4_url(ExpireTime, Bucket, get, erlcloud_key(Key), [], AwsConfig).
 
 -spec format(client()) -> term().
 format(#{aws_config := AwsConfig} = Client) ->

apps/emqx_s3/test/emqx_s3_uploader_SUITE.erl

@@ -54,6 +54,8 @@ groups() ->
             t_happy_path_multi,
             t_abort_multi,
             t_abort_simple_put,
+            t_signed_url_download,
+            t_signed_nonascii_url_download,
 
             {group, noconn_errors},
             {group, timeout_errors},
@@ -193,6 +195,40 @@ t_happy_path_multi(Config) ->
         Key
     ).
 
+t_signed_url_download(_Config) ->
+    Prefix = emqx_s3_test_helpers:unique_key(),
+    Key = Prefix ++ "/ascii.txt",
+
+    {ok, Data} = upload(Key, 1024, 5),
+
+    SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) ->
+        emqx_s3_client:uri(Client, Key)
+    end),
+
+    {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []),
+
+    ?assertEqual(
+        iolist_to_binary(Data),
+        iolist_to_binary(Body)
+    ).
+
+t_signed_nonascii_url_download(_Config) ->
+    Prefix = emqx_s3_test_helpers:unique_key(),
+    Key = Prefix ++ "/unicode-🫠.txt",
+
+    {ok, Data} = upload(Key, 1024 * 1024, 8),
+
+    SignedUrl = emqx_s3:with_client(profile_id(), fun(Client) ->
+        emqx_s3_client:uri(Client, Key)
+    end),
+
+    {ok, {_, _, Body}} = httpc:request(get, {SignedUrl, []}, [], []),
+
+    ?assertEqual(
+        iolist_to_binary(Data),
+        iolist_to_binary(Body)
+    ).
+
 t_abort_multi(Config) ->
     Key = emqx_s3_test_helpers:unique_key(),
     {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}),
@@ -532,3 +568,24 @@ data(Byte, ChunkSize, ChunkCount) ->
 list_objects(Config) ->
     Props = erlcloud_s3:list_objects(?config(bucket, Config), [], ?config(test_aws_config, Config)),
     proplists:get_value(contents, Props).
+
+upload(Key, ChunkSize, ChunkCount) ->
+    {ok, Pid} = emqx_s3:start_uploader(profile_id(), #{key => Key}),
+
+    _ = erlang:monitor(process, Pid),
+
+    Data = data($a, ChunkSize, ChunkCount),
+
+    ok = lists:foreach(
+        fun(Chunk) -> ?assertEqual(ok, emqx_s3_uploader:write(Pid, Chunk)) end,
+        Data
+    ),
+
+    ok = emqx_s3_uploader:complete(Pid),
+
+    ok = ?assertProcessExited(
+        normal,
+        Pid
+    ),
+
+    {ok, Data}.

lib-ee/emqx_ee_connector/rebar.config

@@ -4,7 +4,7 @@
     {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}},
     {tdengine, {git, "https://github.com/emqx/tdengine-client-erl", {tag, "0.1.6"}}},
     {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}},
-    {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.7-emqx-1"}}},
+    {erlcloud, {git, "https://github.com/emqx/erlcloud", {tag, "3.6.8-emqx-1"}}},
     {rocketmq, {git, "https://github.com/emqx/rocketmq-client-erl.git", {tag, "v0.5.1"}}},
     {emqx, {path, "../../apps/emqx"}}
 ]}.

mix.exs

@@ -94,10 +94,10 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by grpc and eetcd
       {:gpb, "4.19.5", override: true, runtime: false},
       {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true},
-      {:erlcloud, github: "emqx/erlcloud", tag: "3.6.7-emqx-1", override: true},
+      {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true},
       # erlcloud's rebar.config requires rebar3 and does not support Mix,
       # so it tries to fetch deps from git. We need to override this.
-      {:lhttpc, "1.6.2", override: true},
+      {:lhttpc, github: "https://github.com/erlcloud/lhttpc", tag: "1.6.2", override: true},
       {:eini, "1.2.9", override: true},
       {:base16, "1.0.0", override: true}
       # end of erlcloud's deps