Merge pull request #10622 from JimMoen/refactor-dynamo-bridge

refactor(dynamo): move dynamo bridge into its own app

apps/emqx_bridge_dynamo/docker-ct

@@ -0,0 +1,2 @@
+toxiproxy
+dynamo

apps/emqx_bridge_dynamo/rebar.config

@@ -0,0 +1,11 @@
+%% -*- mode: erlang; -*-
+{erl_opts, [debug_info]}.
+{deps, [ {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag, "3.5.16-emqx-1"}}}
+       , {emqx_connector, {path, "../../apps/emqx_connector"}}
+       , {emqx_resource, {path, "../../apps/emqx_resource"}}
+       , {emqx_bridge, {path, "../../apps/emqx_bridge"}}
+       ]}.
+
+{shell, [
+    {apps, [emqx_bridge_dynamo]}
+]}.

apps/emqx_bridge_dynamo/src/emqx_bridge_dynamo.app.src

@@ -1,8 +1,8 @@
 {application, emqx_bridge_dynamo, [
     {description, "EMQX Enterprise Dynamo Bridge"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
-    {applications, [kernel, stdlib]},
+    {applications, [kernel, stdlib, erlcloud]},
     {env, []},
     {modules, []},
     {links, []}

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_dynamo.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2022 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_bridge_dynamo).
+-module(emqx_bridge_dynamo).
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
@@ -89,7 +89,7 @@ fields("config") ->
                 }
             )}
     ] ++
-        (emqx_ee_connector_dynamo:fields(config) --
+        (emqx_bridge_dynamo_connector:fields(config) --
             emqx_connector_schema_lib:prepare_statement_fields());
 fields("creation_opts") ->
     emqx_resource_schema:fields("creation_opts");

lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo.erl

@@ -2,7 +2,7 @@
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_ee_connector_dynamo).
+-module(emqx_bridge_dynamo_connector).
 
 -behaviour(emqx_resource).
 
@@ -131,7 +131,7 @@ on_batch_query(_InstanceId, Query, _State) ->
 
 on_get_status(_InstanceId, #{pool_name := Pool}) ->
     Health = emqx_resource_pool:health_check_workers(
-        Pool, {emqx_ee_connector_dynamo_client, is_connected, []}
+        Pool, {emqx_bridge_dynamo_connector_client, is_connected, []}
     ),
     status_result(Health).
 
@@ -154,7 +154,7 @@ do_query(
     ),
     Result = ecpool:pick_and_do(
         PoolName,
-        {emqx_ee_connector_dynamo_client, query, [Table, Query, Templates]},
+        {emqx_bridge_dynamo_connector_client, query, [Table, Query, Templates]},
         no_handover
     ),
 
@@ -181,7 +181,7 @@ do_query(
 
 connect(Opts) ->
     Options = proplists:get_value(config, Opts),
-    {ok, _Pid} = Result = emqx_ee_connector_dynamo_client:start_link(Options),
+    {ok, _Pid} = Result = emqx_bridge_dynamo_connector_client:start_link(Options),
     Result.
 
 parse_template(Config) ->

lib-ee/emqx_ee_connector/src/emqx_ee_connector_dynamo_client.erl

@@ -1,7 +1,8 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_connector_dynamo_client).
+
+-module(emqx_bridge_dynamo_connector_client).
 
 -behaviour(gen_server).
 

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_dynamo_SUITE.erl

@@ -2,7 +2,7 @@
 %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_ee_bridge_dynamo_SUITE).
+-module(emqx_bridge_dynamo_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
@@ -24,6 +24,14 @@
 
 -define(GET_CONFIG(KEY__, CFG__), proplists:get_value(KEY__, CFG__)).
 
+%% How to run it locally (all commands are run in $PROJ_ROOT dir):
+%% run ct in docker container
+%% run script:
+%% ```bash
+%% ./scripts/ct/run.sh --ci --app apps/emqx_bridge_dynamo -- \
+%%                     --name 'test@127.0.0.1' -c -v --readable true \
+%%                     --suite apps/emqx_bridge_dynamo/test/emqx_bridge_dynamo_SUITE.erl
+
 %%------------------------------------------------------------------------------
 %% CT boilerplate
 %%------------------------------------------------------------------------------
@@ -224,7 +232,7 @@ query_resource(Config, Request) ->
     ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
     emqx_resource:query(ResourceID, Request, #{timeout => 1_000}).
 
-%% create a table, use the lib-ee/emqx_ee_bridge/priv/dynamo/mqtt_msg.json as template
+%% create a table, use the apps/emqx_bridge_dynamo/priv/dynamo/mqtt_msg.json as template
 create_table(Config) ->
     directly_setup_dynamo(),
     delete_table(Config),
@@ -251,7 +259,7 @@ directly_setup_dynamo() ->
 
 directly_query(Query) ->
     directly_setup_dynamo(),
-    emqx_ee_connector_dynamo_client:execute(Query, ?TABLE_BIN).
+    emqx_bridge_dynamo_connector_client:execute(Query, ?TABLE_BIN).
 
 directly_get_payload(Key) ->
     case directly_query({get_item, {<<"id">>, Key}}) of

lib-ee/emqx_ee_bridge/docker-ct

@@ -6,4 +6,3 @@ mysql
 redis
 redis_cluster
 clickhouse
-dynamo

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src

@@ -12,6 +12,7 @@
         emqx_bridge_cassandra,
         emqx_bridge_opents,
         emqx_bridge_pulsar,
+        emqx_bridge_dynamo,
         emqx_bridge_sqlserver,
         emqx_bridge_rocketmq,
         emqx_bridge_rabbitmq,

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl

@@ -33,7 +33,7 @@ api_schemas(Method) ->
         ref(emqx_bridge_matrix, Method),
         ref(emqx_bridge_tdengine, Method),
         ref(emqx_ee_bridge_clickhouse, Method),
-        ref(emqx_ee_bridge_dynamo, Method),
+        ref(emqx_bridge_dynamo, Method),
         ref(emqx_bridge_rocketmq, Method),
         ref(emqx_bridge_sqlserver, Method),
         ref(emqx_bridge_opents, Method),
@@ -58,7 +58,7 @@ schema_modules() ->
         emqx_bridge_matrix,
         emqx_bridge_tdengine,
         emqx_ee_bridge_clickhouse,
-        emqx_ee_bridge_dynamo,
+        emqx_bridge_dynamo,
         emqx_bridge_rocketmq,
         emqx_bridge_sqlserver,
         emqx_bridge_opents,
@@ -102,7 +102,7 @@ resource_type(timescale) -> emqx_connector_pgsql;
 resource_type(matrix) -> emqx_connector_pgsql;
 resource_type(tdengine) -> emqx_bridge_tdengine_connector;
 resource_type(clickhouse) -> emqx_ee_connector_clickhouse;
-resource_type(dynamo) -> emqx_ee_connector_dynamo;
+resource_type(dynamo) -> emqx_bridge_dynamo_connector;
 resource_type(rocketmq) -> emqx_bridge_rocketmq_connector;
 resource_type(sqlserver) -> emqx_bridge_sqlserver_connector;
 resource_type(opents) -> emqx_bridge_opents_connector;
@@ -147,7 +147,7 @@ fields(bridges) ->
             )},
         {dynamo,
             mk(
-                hoconsc:map(name, ref(emqx_ee_bridge_dynamo, "config")),
+                hoconsc:map(name, ref(emqx_bridge_dynamo, "config")),
                 #{
                     desc => <<"Dynamo Bridge Config">>,
                     required => false

lib-ee/emqx_ee_connector/rebar.config

@@ -1,9 +1,9 @@
+%% -*- mode: erlang -*-
 {erl_opts, [debug_info]}.
 {deps, [
   {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
   {influxdb, {git, "https://github.com/emqx/influxdb-client-erl", {tag, "1.1.9"}}},
   {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}},
-  {erlcloud, {git, "https://github.com/emqx/erlcloud.git", {tag,"3.5.16-emqx-1"}}},
   {emqx, {path, "../../apps/emqx"}},
   {emqx_utils, {path, "../../apps/emqx_utils"}}
 ]}.

lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src

@@ -8,8 +8,7 @@
         ecpool,
         hstreamdb_erl,
         influxdb,
-        clickhouse,
-        erlcloud
+        clickhouse
     ]},
     {env, []},
     {modules, []},

mix.exs

@@ -190,6 +190,13 @@ defmodule EMQXUmbrella.MixProject do
       {:snappyer, "1.2.8", override: true},
       {:crc32cer, "0.1.8", override: true},
       {:supervisor3, "1.1.12", override: true},
+      {:erlcloud, github: "emqx/erlcloud", tag: "3.5.16-emqx-1", override: true},
+      # erlcloud's rebar.config requires rebar3 and does not support Mix,
+      # so it tries to fetch deps from git. We need to override this.
+      {:lhttpc, tag: "1.6.2", override: true},
+      {:eini, "1.2.9", override: true},
+      {:base16, "1.0.0", override: true},
+      # end of erlcloud's deps
       {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true},
       # The following two are dependencies of rabbit_common. They are needed here to
       # make mix not complain about conflicting versions

rel/i18n/emqx_ee_bridge_dynamo.hocon

@@ -1,4 +1,4 @@
-emqx_ee_bridge_dynamo {
+emqx_bridge_dynamo {
 
 config_enable.desc:
 """Enable or disable this bridge"""

rel/i18n/emqx_ee_connector_dynamo.hocon

@@ -1,4 +1,4 @@
-emqx_ee_connector_dynamo {
+emqx_bridge_dynamo_connector {
 
 aws_access_key_id.desc:
 """Access Key ID for connecting to DynamoDB."""

rel/i18n/zh/emqx_ee_bridge_dynamo.hocon

@@ -1,4 +1,4 @@
-emqx_ee_bridge_dynamo {
+emqx_bridge_dynamo {
 
 config_enable.desc:
 """启用/禁用桥接"""

rel/i18n/zh/emqx_ee_connector_dynamo.hocon

@@ -1,4 +1,4 @@
-emqx_ee_connector_dynamo {
+emqx_bridge_dynamo_connector {
 
 aws_access_key_id.desc:
 """DynamoDB 的访问 ID。"""