Merge pull request #11151 from kjellwinblad/kjell/refactor/mysql_bridge/EMQX-9533

refactor: MySQL bridge and connector to separate applications

apps/emqx_authn/rebar.config

@@ -5,7 +5,8 @@
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_connector, {path, "../emqx_connector"}},
     {emqx_mongodb, {path, "../emqx_mongodb"}},
-    {emqx_redis, {path, "../emqx_redis"}}
+    {emqx_redis, {path, "../emqx_redis"}},
+    {emqx_mysql, {path, "../emqx_mysql"}}
 ]}.
 
 {edoc_opts, [{preprocess, true}]}.

apps/emqx_authn/src/emqx_authn.app.src

@@ -14,7 +14,8 @@
         mysql,
         jose,
         emqx_mongodb,
-        emqx_redis
+        emqx_redis,
+        emqx_mysql
     ]},
     {mod, {emqx_authn_app, []}},
     {env, []},

apps/emqx_authn/src/simple_authn/emqx_authn_mysql.erl

@@ -62,7 +62,7 @@ fields(mysql) ->
         {query, fun query/1},
         {query_timeout, fun query_timeout/1}
     ] ++ emqx_authn_schema:common_fields() ++
-        proplists:delete(prepare_statement, emqx_connector_mysql:fields(config)).
+        proplists:delete(prepare_statement, emqx_mysql:fields(config)).
 
 desc(mysql) ->
     ?DESC(mysql);
@@ -92,12 +92,12 @@ create(_AuthenticatorID, Config) ->
 create(Config0) ->
     ResourceId = emqx_authn_utils:make_resource_id(?MODULE),
     {Config, State} = parse_config(Config0),
-    {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_connector_mysql, Config),
+    {ok, _Data} = emqx_authn_utils:create_resource(ResourceId, emqx_mysql, Config),
     {ok, State#{resource_id => ResourceId}}.
 
 update(Config0, #{resource_id := ResourceId} = _State) ->
     {Config, NState} = parse_config(Config0),
-    case emqx_authn_utils:update_resource(emqx_connector_mysql, Config, ResourceId) of
+    case emqx_authn_utils:update_resource(emqx_mysql, Config, ResourceId) of
         {error, Reason} ->
             error({load_config_error, Reason});
         {ok, _} ->

apps/emqx_authn/test/emqx_authn_mysql_SUITE.erl

@@ -62,7 +62,7 @@ init_per_suite(Config) ->
             {ok, _} = emqx_resource:create_local(
                 ?MYSQL_RESOURCE,
                 ?RESOURCE_GROUP,
-                emqx_connector_mysql,
+                emqx_mysql,
                 mysql_config(),
                 #{}
             ),

apps/emqx_authz/rebar.config

@@ -6,7 +6,8 @@
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_connector, {path, "../emqx_connector"}},
     {emqx_mongodb, {path, "../emqx_mongodb"}},
-    {emqx_redis, {path, "../emqx_redis"}}
+    {emqx_redis, {path, "../emqx_redis"}},
+    {emqx_mysql, {path, "../emqx_mysql"}}
 ]}.
 
 {shell, [

apps/emqx_authz/src/emqx_authz.app.src

@@ -11,7 +11,8 @@
         emqx_resource,
         emqx_connector,
         emqx_mongodb,
-        emqx_redis
+        emqx_redis,
+        emqx_mysql
     ]},
     {env, []},
     {modules, []},

apps/emqx_authz/src/emqx_authz_api_schema.erl

@@ -70,7 +70,7 @@ fields(mongo_sharded) ->
 fields(mysql) ->
     authz_common_fields(mysql) ++
         [{query, query()}] ++
-        proplists:delete(prepare_statement, emqx_connector_mysql:fields(config));
+        proplists:delete(prepare_statement, emqx_mysql:fields(config));
 fields(postgresql) ->
     authz_common_fields(postgresql) ++
         [{query, query()}] ++

apps/emqx_authz/src/emqx_authz_mysql.erl

@@ -54,13 +54,13 @@ create(#{query := SQL} = Source0) ->
     {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS),
     ResourceId = emqx_authz_utils:make_resource_id(?MODULE),
     Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}},
-    {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_mysql, Source),
+    {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mysql, Source),
     Source#{annotations => #{id => ResourceId, tmpl_oken => TmplToken}}.
 
 update(#{query := SQL} = Source0) ->
     {PrepareSQL, TmplToken} = emqx_authz_utils:parse_sql(SQL, '?', ?PLACEHOLDERS),
     Source = Source0#{prepare_statement => #{?PREPARE_KEY => PrepareSQL}},
-    case emqx_authz_utils:update_resource(emqx_connector_mysql, Source) of
+    case emqx_authz_utils:update_resource(emqx_mysql, Source) of
         {error, Reason} ->
             error({load_config_error, Reason});
         {ok, Id} ->

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -389,7 +389,7 @@ cmd() ->
 
 connector_fields(DB) ->
     connector_fields(DB, config).
-connector_fields(redis = DB, Fields) ->
+connector_fields(DB, Fields) when DB =:= redis; DB =:= mysql ->
     connector_fields(DB, Fields, emqx);
 connector_fields(DB, Fields) ->
     connector_fields(DB, Fields, emqx_connector).

apps/emqx_authz/test/emqx_authz_mysql_SUITE.erl

@@ -44,7 +44,7 @@ init_per_suite(Config) ->
             {ok, _} = emqx_resource:create_local(
                 ?MYSQL_RESOURCE,
                 ?RESOURCE_GROUP,
-                emqx_connector_mysql,
+                emqx_mysql,
                 mysql_config(),
                 #{}
             ),

apps/emqx_bridge_mongodb/test/emqx_bridge_mongodb_SUITE.erl

@@ -141,7 +141,9 @@ start_apps() ->
     %% we want to make sure they are loaded before
     %% ekka start in emqx_common_test_helpers:start_apps/1
     emqx_common_test_helpers:render_and_load_app_config(emqx_conf),
-    ok = emqx_common_test_helpers:start_apps([emqx_conf, emqx_rule_engine, emqx_bridge]).
+    ok = emqx_common_test_helpers:start_apps([
+        emqx_conf, emqx_rule_engine, emqx_bridge, emqx_mongodb
+    ]).
 
 ensure_loaded() ->
     _ = application:load(emqx_ee_bridge),

apps/emqx_bridge_mysql/docker-ct

@@ -0,0 +1,2 @@
+toxiproxy
+mysql

apps/emqx_bridge_mysql/rebar.config

@@ -0,0 +1,11 @@
+%% -*- mode: erlang; -*-
+{erl_opts, [debug_info]}.
+{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}}
+       , {emqx_resource, {path, "../../apps/emqx_resource"}}
+       , {emqx_bridge, {path, "../../apps/emqx_bridge"}}
+       , {emqx_mysql, {path, "../../apps/emqx_mysql"}}
+       ]}.
+
+{shell, [
+    {apps, [emqx_bridge_mysql]}
+]}.

apps/emqx_bridge_mysql/src/emqx_bridge_mysql.app.src

@@ -1,8 +1,8 @@
 {application, emqx_bridge_mysql, [
     {description, "EMQX Enterprise MySQL Bridge"},
-    {vsn, "0.1.0"},
+    {vsn, "0.1.1"},
     {registered, []},
-    {applications, [kernel, stdlib]},
+    {applications, [kernel, stdlib, emqx_connector, emqx_resource, emqx_bridge, emqx_mysql]},
     {env, []},
     {modules, []},
     {links, []}

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mysql.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_bridge_mysql).
+-module(emqx_bridge_mysql).
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
@@ -80,7 +80,7 @@ fields("config") ->
                 #{desc => ?DESC("local_topic"), default => undefined}
             )}
     ] ++ emqx_resource_schema:fields("resource_opts") ++
-        (emqx_connector_mysql:fields(config) --
+        (emqx_mysql:fields(config) --
             emqx_connector_schema_lib:prepare_statement_fields());
 fields("post") ->
     [type_field(), name_field() | fields("config")];

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mysql_SUITE.erl

@@ -2,7 +2,7 @@
 % Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_ee_bridge_mysql_SUITE).
+-module(emqx_bridge_mysql_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).

apps/emqx_connector/README.md

@@ -3,25 +3,4 @@
 This application is a collection of `connectors`.
 
 A `connector` is a callback module of `emqx_resource` that maintains the data related to
-external resources. Put all resource related callback modules in a single application is good as
-we can put some util functions/modules here for reusing purpose.
-
-For example, a MySQL connector is an emqx resource that maintains all the MySQL connection
-related parameters (configs) and the TCP connections to the MySQL server.
-
-An MySQL connector can be used as following:
-
-```
-(emqx@127.0.0.1)5> emqx_resource:list_instances_verbose().
-[#{config =>
-       #{cacertfile => [],certfile => [],
-         database => "mqtt",keyfile => [],password => "public",
-         pool_size => 1,
-         server => {{127,0,0,1},3306},
-         ssl => false,user => "root",verify => false},
-   id => <<"mysql-abc">>,mod => emqx_connector_mysql,
-   state => #{poolname => 'mysql-abc'},
-   status => connected}]
-(emqx@127.0.0.1)6> emqx_resource:query(<<"mysql-abc">>, {sql, <<"SELECT count(1)">>}).
-{ok,[<<"count(1)">>],[[1]]}
-```
+external resources.

apps/emqx_connector/docker-ct

@@ -1,2 +1 @@
-mysql
 pgsql

apps/emqx_connector/rebar.config

@@ -10,7 +10,6 @@
     {emqx_utils, {path, "../emqx_utils"}},
     {emqx_resource, {path, "../emqx_resource"}},
     {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
-    {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
     {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}}
 ]}.
 

apps/emqx_connector/src/emqx_connector.app.src

@@ -13,8 +13,6 @@
         eredis,
         epgsql,
         eldap2,
-        mysql,
-        mongodb,
         ehttpc,
         jose,
         emqx,

apps/emqx_mongodb/test/emqx_mongodb_SUITE.erl

@@ -38,6 +38,7 @@ init_per_suite(Config) ->
             ok = emqx_common_test_helpers:start_apps([emqx_conf]),
             ok = emqx_connector_test_helpers:start_apps([emqx_resource]),
             {ok, _} = application:ensure_all_started(emqx_connector),
+            {ok, _} = application:ensure_all_started(emqx_mongodb),
             Config;
         false ->
             {skip, no_mongo}

apps/emqx_mysql/README.md

@@ -0,0 +1,15 @@
+# MySQL Connector
+
+This application houses the MySQL Database connector.
+It provides the APIs to connect to MySQL Databases.
+
+It is used by the MySQL bridge to insert messages and by the emqx_authz and
+emqx_authn applications to check user permissions.
+
+## Contributing
+
+Please see our [contributing.md](../../CONTRIBUTING.md).
+
+## License
+
+See [APL](../../APL.txt).

apps/emqx_mysql/docker-ct

@@ -0,0 +1 @@
+mysql

apps/emqx_mysql/rebar.config

@@ -0,0 +1,9 @@
+%% -*- mode: erlang; -*-
+
+{erl_opts, [debug_info]}.
+{deps, [
+        %% NOTE: mind ecpool version when updating eredis_cluster version
+        {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
+        {emqx_connector, {path, "../../apps/emqx_connector"}},
+        {emqx_resource, {path, "../../apps/emqx_resource"}}
+]}.

apps/emqx_mysql/src/emqx_mysql.app.src

@@ -0,0 +1,14 @@
+{application, emqx_mysql, [
+    {description, "EMQX MySQL Database Connector"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib,
+        mysql
+    ]},
+    {env, []},
+    {modules, []},
+
+    {links, []}
+]}.

apps/emqx_connector/src/emqx_connector_mysql.erl

@@ -13,9 +13,9 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--module(emqx_connector_mysql).
+-module(emqx_mysql).
 
--include("emqx_connector.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").

apps/emqx_connector/test/emqx_connector_mysql_SUITE.erl

@@ -13,18 +13,18 @@
 % %% limitations under the License.
 % %%--------------------------------------------------------------------
 
--module(emqx_connector_mysql_SUITE).
+-module(emqx_mysql_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
 
--include("emqx_connector.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("eunit/include/eunit.hrl").
 -include_lib("emqx/include/emqx.hrl").
 -include_lib("stdlib/include/assert.hrl").
 
 -define(MYSQL_HOST, "mysql").
--define(MYSQL_RESOURCE_MOD, emqx_connector_mysql).
+-define(MYSQL_RESOURCE_MOD, emqx_mysql).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -60,7 +60,7 @@ end_per_testcase(_, _Config) ->
 
 t_lifecycle(_Config) ->
     perform_lifecycle_check(
-        <<"emqx_connector_mysql_SUITE">>,
+        <<"emqx_mysql_SUITE">>,
         mysql_config()
     ).
 

changes/ce/fix-11151.en.md

@@ -0,0 +1 @@
+The MySQL connector has been refactored to its own Erlang application to improve the code structure.

changes/ee/fix-11151.en.md

@@ -0,0 +1 @@
+The MySQL bridge has been refactored to its own Erlang application to improve the code structure and to make it easier to maintain.

lib-ee/emqx_ee_bridge/docker-ct

@@ -1,2 +1 @@
 toxiproxy
-mysql

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl

@@ -25,7 +25,7 @@ api_schemas(Method) ->
         %% to hocon; keeping this as just `kafka' for backwards compatibility.
         api_ref(emqx_bridge_kafka, <<"kafka">>, Method ++ "_producer"),
         api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method),
-        api_ref(emqx_ee_bridge_mysql, <<"mysql">>, Method),
+        api_ref(emqx_bridge_mysql, <<"mysql">>, Method),
         api_ref(emqx_bridge_pgsql, <<"pgsql">>, Method),
         api_ref(emqx_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"),
         api_ref(emqx_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"),
@@ -58,7 +58,7 @@ schema_modules() ->
         emqx_bridge_gcp_pubsub,
         emqx_bridge_influxdb,
         emqx_bridge_mongodb,
-        emqx_ee_bridge_mysql,
+        emqx_bridge_mysql,
         emqx_bridge_redis,
         emqx_bridge_pgsql,
         emqx_bridge_timescale,
@@ -99,7 +99,7 @@ resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer;
 resource_type(mongodb_rs) -> emqx_bridge_mongodb_connector;
 resource_type(mongodb_sharded) -> emqx_bridge_mongodb_connector;
 resource_type(mongodb_single) -> emqx_bridge_mongodb_connector;
-resource_type(mysql) -> emqx_connector_mysql;
+resource_type(mysql) -> emqx_mysql;
 resource_type(influxdb_api_v1) -> emqx_bridge_influxdb_connector;
 resource_type(influxdb_api_v2) -> emqx_bridge_influxdb_connector;
 resource_type(redis_single) -> emqx_bridge_redis_connector;
@@ -131,7 +131,7 @@ fields(bridges) ->
             )},
         {mysql,
             mk(
-                hoconsc:map(name, ref(emqx_ee_bridge_mysql, "config")),
+                hoconsc:map(name, ref(emqx_bridge_mysql, "config")),
                 #{
                     desc => <<"MySQL Bridge Config">>,
                     required => false

mix.exs

@@ -378,6 +378,7 @@ defmodule EMQXUmbrella.MixProject do
         emqx_slow_subs: :permanent,
         emqx_mongodb: :permanent,
         emqx_redis: :permanent,
+        emqx_mysql: :permanent,
         emqx_plugins: :permanent,
         emqx_mix: :none
       ] ++

rebar.config.erl

@@ -441,6 +441,7 @@ relx_apps(ReleaseType, Edition) ->
             emqx_slow_subs,
             emqx_mongodb,
             emqx_redis,
+            emqx_mysql,
             emqx_plugins
         ] ++
         [quicer || is_quicer_supported()] ++

rel/i18n/emqx_ee_bridge_mysql.hocon

@@ -1,4 +1,4 @@
-emqx_ee_bridge_mysql {
+emqx_bridge_mysql {
 
 config_enable.desc:
 """Enable or disable this bridge"""

rel/i18n/emqx_connector_mysql.hocon

@@ -1,4 +1,4 @@
-emqx_connector_mysql {
+emqx_mysql {
 
 server.desc:
 """The IPv4 or IPv6 address or the hostname to connect to.<br/>