Merge pull request #10654 from kjellwinblad/kjell/refactor/clickhouse_bridge/EMQX-9538

refactor: the clickhouse bridge to use the new bridge app structure

apps/emqx_bridge_clickhouse/README.md

@@ -23,7 +23,7 @@ User can create a rule and easily ingest IoT data into ClickHouse by leveraging
 - Several APIs are provided for bridge management, which includes create bridge,
   update bridge, get bridge, stop or restart bridge and list bridges etc.
 
-  Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges)
+- Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges)
   for more detailed information.
 
 

apps/emqx_bridge_clickhouse/docker-ct

@@ -0,0 +1 @@
+clickhouse

apps/emqx_bridge_clickhouse/rebar.config

@@ -0,0 +1,11 @@
+%% -*- mode: erlang; -*-
+{erl_opts, [debug_info]}.
+{deps, [ {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}}
+       , {emqx_connector, {path, "../../apps/emqx_connector"}}
+       , {emqx_resource, {path, "../../apps/emqx_resource"}}
+       , {emqx_bridge, {path, "../../apps/emqx_bridge"}}
+       ]}.
+
+{shell, [
+    {apps, [emqx_bridge_clickhouse]}
+]}.

apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse.app.src

@@ -1,8 +1,8 @@
 {application, emqx_bridge_clickhouse, [
     {description, "EMQX Enterprise ClickHouse Bridge"},
-    {vsn, "0.1.0"},
+    {vsn, "0.2.0"},
     {registered, []},
-    {applications, [kernel, stdlib]},
+    {applications, [kernel, stdlib, clickhouse]},
     {env, []},
     {modules, []},
     {links, []}

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_clickhouse.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_bridge_clickhouse).
+-module(emqx_bridge_clickhouse).
 
 -include_lib("emqx_bridge/include/emqx_bridge.hrl").
 -include_lib("typerefl/include/types.hrl").
@@ -101,7 +101,7 @@ fields("config") ->
                 }
             )}
     ] ++
-        emqx_ee_connector_clickhouse:fields(config);
+        emqx_bridge_clickhouse_connector:fields(config);
 fields("creation_opts") ->
     emqx_resource_schema:fields("creation_opts");
 fields("post") ->

lib-ee/emqx_ee_connector/src/emqx_ee_connector_clickhouse.erl

@@ -2,7 +2,7 @@
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_ee_connector_clickhouse).
+-module(emqx_bridge_clickhouse_connector).
 
 -include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("emqx_resource/include/emqx_resource.hrl").

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_clickhouse_SUITE.erl

@@ -2,17 +2,16 @@
 %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_ee_bridge_clickhouse_SUITE).
+-module(emqx_bridge_clickhouse_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
 
 -define(CLICKHOUSE_HOST, "clickhouse").
--define(CLICKHOUSE_RESOURCE_MOD, emqx_ee_connector_clickhouse).
 -include_lib("emqx_connector/include/emqx_connector.hrl").
 
 %% See comment in
-%% lib-ee/emqx_ee_connector/test/ee_connector_clickhouse_SUITE.erl for how to
+%% lib-ee/emqx_ee_connector/test/ee_bridge_clickhouse_connector_SUITE.erl for how to
 %% run this without bringing up the whole CI infrastucture
 
 %%------------------------------------------------------------------------------

lib-ee/emqx_ee_connector/test/emqx_ee_connector_clickhouse_SUITE.erl

@@ -2,7 +2,7 @@
 %% Copyright (c) 2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_ee_connector_clickhouse_SUITE).
+-module(emqx_bridge_clickhouse_connector_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
@@ -13,7 +13,7 @@
 -include_lib("stdlib/include/assert.hrl").
 
 -define(CLICKHOUSE_HOST, "clickhouse").
--define(CLICKHOUSE_RESOURCE_MOD, emqx_ee_connector_clickhouse).
+-define(CLICKHOUSE_RESOURCE_MOD, emqx_bridge_clickhouse_connector).
 
 %% This test SUITE requires a running clickhouse instance. If you don't want to
 %% bring up the whole CI infrastuctucture with the `scripts/ct/run.sh` script
@@ -21,7 +21,15 @@
 %% from root of the EMQX directory.). You also need to set ?CLICKHOUSE_HOST and
 %% ?CLICKHOUSE_PORT to appropriate values.
 %%
-%% docker run -d -p 18123:8123 -p19000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 -v "`pwd`/.ci/docker-compose-file/clickhouse/users.xml:/etc/clickhouse-server/users.xml" -v "`pwd`/.ci/docker-compose-file/clickhouse/config.xml:/etc/clickhouse-server/config.xml" clickhouse/clickhouse-server
+%% docker run \
+%%    -d \
+%%    -p 18123:8123 \
+%%    -p 19000:9000 \
+%%    --name some-clickhouse-server \
+%%    --ulimit nofile=262144:262144 \
+%%    -v "`pwd`/.ci/docker-compose-file/clickhouse/users.xml:/etc/clickhouse-server/users.xml" \
+%%    -v "`pwd`/.ci/docker-compose-file/clickhouse/config.xml:/etc/clickhouse-server/config.xml" \
+%%    clickhouse/clickhouse-server
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -119,7 +127,6 @@ perform_lifecycle_check(ResourceID, InitialConfig) ->
     ?assertEqual({ok, connected}, emqx_resource:health_check(ResourceID)),
     % % Perform query as further check that the resource is working as expected
     (fun() ->
-        erlang:display({pool_name, ResourceID}),
         QueryNoParamsResWrapper = emqx_resource:query(ResourceID, test_query_no_params()),
         ?assertMatch({ok, _}, QueryNoParamsResWrapper),
         {_, QueryNoParamsRes} = QueryNoParamsResWrapper,

changes/ee/refactor-10654.en.md

@@ -0,0 +1 @@
+The clickhouse bridge has been refactored so it is located in its own OTP application.

lib-ee/emqx_ee_bridge/docker-ct

@@ -4,4 +4,3 @@ mongo_rs_sharded
 mysql
 redis
 redis_cluster
-clickhouse

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.app.src

@@ -17,7 +17,8 @@
         emqx_bridge_rocketmq,
         emqx_bridge_rabbitmq,
         emqx_bridge_tdengine,
-        emqx_bridge_influxdb
+        emqx_bridge_influxdb,
+        emqx_bridge_clickhouse
     ]},
     {env, []},
     {modules, []},

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl

@@ -32,9 +32,9 @@ api_schemas(Method) ->
         ref(emqx_bridge_timescale, Method),
         ref(emqx_bridge_matrix, Method),
         ref(emqx_bridge_tdengine, Method),
-        ref(emqx_ee_bridge_clickhouse, Method),
         ref(emqx_bridge_dynamo, Method),
         ref(emqx_bridge_rocketmq, Method),
+        ref(emqx_bridge_clickhouse, Method),
         ref(emqx_bridge_sqlserver, Method),
         ref(emqx_bridge_opents, Method),
         ref(emqx_bridge_pulsar, Method ++ "_producer"),
@@ -57,7 +57,7 @@ schema_modules() ->
         emqx_bridge_timescale,
         emqx_bridge_matrix,
         emqx_bridge_tdengine,
-        emqx_ee_bridge_clickhouse,
+        emqx_bridge_clickhouse,
         emqx_bridge_dynamo,
         emqx_bridge_rocketmq,
         emqx_bridge_sqlserver,
@@ -101,7 +101,7 @@ resource_type(pgsql) -> emqx_connector_pgsql;
 resource_type(timescale) -> emqx_connector_pgsql;
 resource_type(matrix) -> emqx_connector_pgsql;
 resource_type(tdengine) -> emqx_bridge_tdengine_connector;
-resource_type(clickhouse) -> emqx_ee_connector_clickhouse;
+resource_type(clickhouse) -> emqx_bridge_clickhouse_connector;
 resource_type(dynamo) -> emqx_bridge_dynamo_connector;
 resource_type(rocketmq) -> emqx_bridge_rocketmq_connector;
 resource_type(sqlserver) -> emqx_bridge_sqlserver_connector;
@@ -297,7 +297,7 @@ clickhouse_structs() ->
     [
         {clickhouse,
             mk(
-                hoconsc:map(name, ref(emqx_ee_bridge_clickhouse, "config")),
+                hoconsc:map(name, ref(emqx_bridge_clickhouse, "config")),
                 #{
                     desc => <<"Clickhouse Bridge Config">>,
                     required => false

lib-ee/emqx_ee_connector/docker-ct

@@ -1,3 +1,2 @@
 toxiproxy
 influxdb
-clickhouse

lib-ee/emqx_ee_connector/rebar.config

@@ -2,7 +2,6 @@
 {erl_opts, [debug_info]}.
 {deps, [
   {hstreamdb_erl, {git, "https://github.com/hstreamdb/hstreamdb_erl.git", {tag, "0.2.5"}}},
-  {clickhouse, {git, "https://github.com/emqx/clickhouse-client-erl", {tag, "0.3"}}},
   {emqx, {path, "../../apps/emqx"}},
   {emqx_utils, {path, "../../apps/emqx_utils"}}
 ]}.

lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src

@@ -6,8 +6,7 @@
         kernel,
         stdlib,
         ecpool,
-        hstreamdb_erl,
-        clickhouse
+        hstreamdb_erl
     ]},
     {env, []},
     {modules, []},

mix.exs

@@ -158,7 +158,6 @@ defmodule EMQXUmbrella.MixProject do
       :emqx_bridge_gcp_pubsub,
       :emqx_bridge_cassandra,
       :emqx_bridge_opents,
-      :emqx_bridge_clickhouse,
       :emqx_bridge_dynamo,
       :emqx_bridge_hstreamdb,
       :emqx_bridge_influxdb,
@@ -175,7 +174,8 @@ defmodule EMQXUmbrella.MixProject do
       :emqx_bridge_pulsar,
       :emqx_oracle,
       :emqx_bridge_oracle,
-      :emqx_bridge_rabbitmq
+      :emqx_bridge_rabbitmq,
+      :emqx_bridge_clickhouse
     ])
   end
 

rel/i18n/emqx_ee_bridge_clickhouse.hocon

@@ -1,4 +1,4 @@
-emqx_ee_bridge_clickhouse {
+emqx_bridge_clickhouse {
 
 batch_value_separator.desc:
 """The default value ',' works for the VALUES format. You can also use other separator if other format is specified. See [INSERT INTO Statement](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)."""

rel/i18n/emqx_ee_connector_clickhouse.hocon

@@ -1,4 +1,4 @@
-emqx_ee_connector_clickhouse {
+emqx_bridge_clickhouse_connector {
 
 base_url.desc:
 """The HTTP URL to the Clickhouse server that you want to connect to (for example http://myhostname:8123)"""

rel/i18n/zh/emqx_ee_bridge_clickhouse.hocon

@@ -1,4 +1,4 @@
-emqx_ee_bridge_clickhouse {
+emqx_bridge_clickhouse {
 
 batch_value_separator.desc:
 """默认为逗号 ','，适用于 VALUE 格式。您也可以使用其他分隔符， 请参考 [INSERT INTO 语句](https://clickhouse.com/docs/en/sql-reference/statements/insert-into)。"""

rel/i18n/zh/emqx_ee_connector_clickhouse.hocon

@@ -1,4 +1,4 @@
-emqx_ee_connector_clickhouse {
+emqx_bridge_clickhouse_connector {
 
 base_url.desc:
 """你想连接到的Clickhouse服务器的HTTP URL（例如http://myhostname:8123）。"""