Merge pull request #10667 from kjellwinblad/kjell/refactor/mongodb_bridge/EMQX-9535

Refactor the MongoDB birdge and the MongoDB connector into separate apps

apps/emqx_authn/rebar.config

@@ -3,7 +3,8 @@
 {deps, [
     {emqx, {path, "../emqx"}},
     {emqx_utils, {path, "../emqx_utils"}},
-    {emqx_connector, {path, "../emqx_connector"}}
+    {emqx_connector, {path, "../emqx_connector"}},
+    {emqx_mongodb, {path, "../emqx_mongodb"}}
 ]}.
 
 {edoc_opts, [{preprocess, true}]}.

apps/emqx_authn/src/emqx_authn.app.src

@@ -4,7 +4,9 @@
     {vsn, "0.1.22"},
     {modules, []},
     {registered, [emqx_authn_sup, emqx_authn_registry]},
-    {applications, [kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose]},
+    {applications, [
+        kernel, stdlib, emqx_resource, emqx_connector, ehttpc, epgsql, mysql, jose, emqx_mongodb
+    ]},
     {mod, {emqx_authn_app, []}},
     {env, []},
     {licenses, ["Apache-2.0"]},

apps/emqx_authn/src/simple_authn/emqx_authn_mongodb.erl

@@ -60,11 +60,11 @@ roots() ->
     ].
 
 fields(mongo_single) ->
-    common_fields() ++ emqx_connector_mongo:fields(single);
+    common_fields() ++ emqx_mongodb:fields(single);
 fields(mongo_rs) ->
-    common_fields() ++ emqx_connector_mongo:fields(rs);
+    common_fields() ++ emqx_mongodb:fields(rs);
 fields(mongo_sharded) ->
-    common_fields() ++ emqx_connector_mongo:fields(sharded).
+    common_fields() ++ emqx_mongodb:fields(sharded).
 
 desc(mongo_single) ->
     ?DESC(single);
@@ -140,14 +140,14 @@ create(Config0) ->
     {Config, State} = parse_config(Config0),
     {ok, _Data} = emqx_authn_utils:create_resource(
         ResourceId,
-        emqx_connector_mongo,
+        emqx_mongodb,
         Config
     ),
     {ok, State#{resource_id => ResourceId}}.
 
 update(Config0, #{resource_id := ResourceId} = _State) ->
     {Config, NState} = parse_config(Config0),
-    case emqx_authn_utils:update_resource(emqx_connector_mongo, Config, ResourceId) of
+    case emqx_authn_utils:update_resource(emqx_mongodb, Config, ResourceId) of
         {error, Reason} ->
             error({load_config_error, Reason});
         {ok, _} ->

apps/emqx_authn/test/emqx_authn_mongo_tls_SUITE.erl

@@ -64,7 +64,7 @@ end_per_suite(_Config) ->
 %% Tests
 %%------------------------------------------------------------------------------
 
-%% emqx_connector_mongo connects asynchronously,
+%% emqx_mongodb connects asynchronously,
 %% so we check failure/success indirectly (through snabbkaffe).
 
 %% openssl s_client -tls1_2 -cipher ECDHE-RSA-AES256-GCM-SHA384 \

apps/emqx_authz/rebar.config

@@ -4,7 +4,8 @@
 {deps, [
     {emqx, {path, "../emqx"}},
     {emqx_utils, {path, "../emqx_utils"}},
-    {emqx_connector, {path, "../emqx_connector"}}
+    {emqx_connector, {path, "../emqx_connector"}},
+    {emqx_mongodb, {path, "../emqx_mongodb"}}
 ]}.
 
 {shell, [

apps/emqx_authz/src/emqx_authz.app.src

@@ -1,7 +1,7 @@
 %% -*- mode: erlang -*-
 {application, emqx_authz, [
     {description, "An OTP application"},
-    {vsn, "0.1.22"},
+    {vsn, "0.1.23"},
     {registered, []},
     {mod, {emqx_authz_app, []}},
     {applications, [
@@ -9,7 +9,8 @@
         stdlib,
         crypto,
         emqx_resource,
-        emqx_connector
+        emqx_connector,
+        emqx_mongodb
     ]},
     {env, []},
     {modules, []},

apps/emqx_authz/src/emqx_authz_api_schema.erl

@@ -60,13 +60,13 @@ fields(built_in_database) ->
     authz_common_fields(built_in_database);
 fields(mongo_single) ->
     authz_mongo_common_fields() ++
-        emqx_connector_mongo:fields(single);
+        emqx_mongodb:fields(single);
 fields(mongo_rs) ->
     authz_mongo_common_fields() ++
-        emqx_connector_mongo:fields(rs);
+        emqx_mongodb:fields(rs);
 fields(mongo_sharded) ->
     authz_mongo_common_fields() ++
-        emqx_connector_mongo:fields(sharded);
+        emqx_mongodb:fields(sharded);
 fields(mysql) ->
     authz_common_fields(mysql) ++
         [{query, query()}] ++

apps/emqx_authz/src/emqx_authz_mongodb.erl

@@ -50,13 +50,13 @@ description() ->
 
 create(#{filter := Filter} = Source) ->
     ResourceId = emqx_authz_utils:make_resource_id(?MODULE),
-    {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_connector_mongo, Source),
+    {ok, _Data} = emqx_authz_utils:create_resource(ResourceId, emqx_mongodb, Source),
     FilterTemp = emqx_authz_utils:parse_deep(Filter, ?PLACEHOLDERS),
     Source#{annotations => #{id => ResourceId}, filter_template => FilterTemp}.
 
 update(#{filter := Filter} = Source) ->
     FilterTemp = emqx_authz_utils:parse_deep(Filter, ?PLACEHOLDERS),
-    case emqx_authz_utils:update_resource(emqx_connector_mongo, Source) of
+    case emqx_authz_utils:update_resource(emqx_mongodb, Source) of
         {error, Reason} ->
             error({load_config_error, Reason});
         {ok, Id} ->

apps/emqx_authz/src/emqx_authz_schema.erl

@@ -109,15 +109,15 @@ fields(builtin_db) ->
 fields(mongo_single) ->
     authz_common_fields(mongodb) ++
         mongo_common_fields() ++
-        emqx_connector_mongo:fields(single);
+        emqx_mongodb:fields(single);
 fields(mongo_rs) ->
     authz_common_fields(mongodb) ++
         mongo_common_fields() ++
-        emqx_connector_mongo:fields(rs);
+        emqx_mongodb:fields(rs);
 fields(mongo_sharded) ->
     authz_common_fields(mongodb) ++
         mongo_common_fields() ++
-        emqx_connector_mongo:fields(sharded);
+        emqx_mongodb:fields(sharded);
 fields(mysql) ->
     authz_common_fields(mysql) ++
         connector_fields(mysql) ++

apps/emqx_bridge_mongodb/README.md

@@ -25,7 +25,7 @@ User can create a rule and easily ingest IoT data into MongoDB by leveraging
 - Several APIs are provided for bridge management, which includes create bridge,
   update bridge, get bridge, stop or restart bridge and list bridges etc.
 
-  Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges)
+- Refer to [API Docs - Bridges](https://docs.emqx.com/en/enterprise/v5.0/admin/api-docs.html#tag/Bridges)
   for more detailed information.
 
 

apps/emqx_bridge_mongodb/docker-ct

@@ -0,0 +1,2 @@
+mongo
+mongo_rs_sharded

apps/emqx_bridge_mongodb/rebar.config

@@ -0,0 +1,11 @@
+%% -*- mode: erlang; -*-
+{erl_opts, [debug_info]}.
+{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}}
+       , {emqx_resource, {path, "../../apps/emqx_resource"}}
+       , {emqx_bridge, {path, "../../apps/emqx_bridge"}}
+       , {emqx_mongodb, {path, "../../apps/emqx_mongodb"}}
+       ]}.
+
+{shell, [
+    {apps, [emqx_bridge_mongodb]}
+]}.

apps/emqx_bridge_mongodb/src/emqx_bridge_mongodb.app.src

@@ -1,8 +1,16 @@
 {application, emqx_bridge_mongodb, [
     {description, "EMQX Enterprise MongoDB Bridge"},
-    {vsn, "0.1.0"},
+    {vsn, "0.2.0"},
     {registered, []},
-    {applications, [kernel, stdlib]},
+    {applications, [
+        kernel,
+        stdlib,
+        emqx_connector,
+        emqx_resource,
+        emqx_bridge,
+        emqx_ee_bridge,
+        emqx_mongodb
+    ]},
     {env, []},
     {modules, []},
     {links, []}

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge_mongodb.erl

@@ -1,7 +1,7 @@
 %%--------------------------------------------------------------------
 %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
--module(emqx_ee_bridge_mongodb).
+-module(emqx_bridge_mongodb).
 
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
@@ -56,11 +56,11 @@ fields("creation_opts") ->
         }}
     ]);
 fields(mongodb_rs) ->
-    emqx_connector_mongo:fields(rs) ++ fields("config");
+    emqx_mongodb:fields(rs) ++ fields("config");
 fields(mongodb_sharded) ->
-    emqx_connector_mongo:fields(sharded) ++ fields("config");
+    emqx_mongodb:fields(sharded) ++ fields("config");
 fields(mongodb_single) ->
-    emqx_connector_mongo:fields(single) ++ fields("config");
+    emqx_mongodb:fields(single) ++ fields("config");
 fields("post_rs") ->
     fields(mongodb_rs) ++ type_and_name_fields(mongodb_rs);
 fields("post_sharded") ->

lib-ee/emqx_ee_connector/src/emqx_ee_connector_mongodb.erl

@@ -2,7 +2,7 @@
 %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_ee_connector_mongodb).
+-module(emqx_bridge_mongodb_connector).
 
 -behaviour(emqx_resource).
 
@@ -25,10 +25,10 @@
 %% `emqx_resource' API
 %%========================================================================================
 
-callback_mode() -> emqx_connector_mongo:callback_mode().
+callback_mode() -> emqx_mongodb:callback_mode().
 
 on_start(InstanceId, Config) ->
-    case emqx_connector_mongo:on_start(InstanceId, Config) of
+    case emqx_mongodb:on_start(InstanceId, Config) of
         {ok, ConnectorState} ->
             PayloadTemplate0 = maps:get(payload_template, Config, undefined),
             PayloadTemplate = preprocess_template(PayloadTemplate0),
@@ -45,7 +45,7 @@ on_start(InstanceId, Config) ->
     end.
 
 on_stop(InstanceId, _State = #{connector_state := ConnectorState}) ->
-    emqx_connector_mongo:on_stop(InstanceId, ConnectorState).
+    emqx_mongodb:on_stop(InstanceId, ConnectorState).
 
 on_query(InstanceId, {send_message, Message0}, State) ->
     #{
@@ -57,14 +57,14 @@ on_query(InstanceId, {send_message, Message0}, State) ->
         collection => emqx_placeholder:proc_tmpl(CollectionTemplate, Message0)
     },
     Message = render_message(PayloadTemplate, Message0),
-    Res = emqx_connector_mongo:on_query(InstanceId, {send_message, Message}, NewConnectorState),
+    Res = emqx_mongodb:on_query(InstanceId, {send_message, Message}, NewConnectorState),
     ?tp(mongo_ee_connector_on_query_return, #{result => Res}),
     Res;
 on_query(InstanceId, Request, _State = #{connector_state := ConnectorState}) ->
-    emqx_connector_mongo:on_query(InstanceId, Request, ConnectorState).
+    emqx_mongodb:on_query(InstanceId, Request, ConnectorState).
 
 on_get_status(InstanceId, _State = #{connector_state := ConnectorState}) ->
-    emqx_connector_mongo:on_get_status(InstanceId, ConnectorState).
+    emqx_mongodb:on_get_status(InstanceId, ConnectorState).
 
 %%========================================================================================
 %% Helper fns

lib-ee/emqx_ee_bridge/test/emqx_ee_bridge_mongodb_SUITE.erl

@@ -2,7 +2,7 @@
 %% Copyright (c) 2022-2023 EMQ Technologies Co., Ltd. All Rights Reserved.
 %%--------------------------------------------------------------------
 
--module(emqx_ee_bridge_mongodb_SUITE).
+-module(emqx_bridge_mongodb_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).

apps/emqx_connector/docker-ct

@@ -1,4 +1,3 @@
-mongo
 redis
 redis_cluster
 mysql

apps/emqx_connector/rebar.config

@@ -12,7 +12,6 @@
     {eldap2, {git, "https://github.com/emqx/eldap2", {tag, "v0.2.2"}}},
     {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.2"}}},
     {epgsql, {git, "https://github.com/emqx/epgsql", {tag, "4.7.0.1"}}},
-    {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.19"}}},
     %% NOTE: mind ecpool version when updating eredis_cluster version
     {eredis_cluster, {git, "https://github.com/emqx/eredis_cluster", {tag, "0.8.1"}}}
 ]}.

apps/emqx_mongodb/README.md

@@ -0,0 +1,15 @@
+# MongoDB Connector
+
+This application houses the MongoDB connector. The MongoDB connector is used by
+emqx_authz, emqx_authn and emqx_bridge_mongodb applications to connect to
+MongoDB.
+
+## Contributing
+
+Please see our [contributing.md](../../CONTRIBUTING.md).
+
+## License
+
+Apache License, Version 2.0
+
+See [APL.txt](../../APL.txt).

apps/emqx_mongodb/docker-ct

@@ -0,0 +1 @@
+mongo

apps/emqx_mongodb/rebar.config

@@ -0,0 +1,7 @@
+%% -*- mode: erlang; -*-
+
+{erl_opts, [debug_info]}.
+{deps, [ {emqx_connector, {path, "../../apps/emqx_connector"}}
+       , {emqx_resource, {path, "../../apps/emqx_resource"}}
+       , {mongodb, {git, "https://github.com/emqx/mongodb-erlang", {tag, "v3.0.19"}}}
+       ]}.

apps/emqx_mongodb/src/emqx_mongodb.app.src

@@ -0,0 +1,16 @@
+{application, emqx_mongodb, [
+    {description, "EMQX MongoDB Connector"},
+    {vsn, "0.1.0"},
+    {registered, []},
+    {applications, [
+        kernel,
+        stdlib,
+        mongodb,
+        emqx_connector,
+        emqx_resource
+    ]},
+    {env, []},
+    {modules, []},
+
+    {links, []}
+]}.

apps/emqx_connector/src/emqx_connector_mongo.erl

@@ -13,9 +13,9 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 %%--------------------------------------------------------------------
--module(emqx_connector_mongo).
+-module(emqx_mongodb).
 
--include("emqx_connector.hrl").
+-include_lib("emqx_connector/include/emqx_connector.hrl").
 -include_lib("typerefl/include/types.hrl").
 -include_lib("hocon/include/hoconsc.hrl").
 -include_lib("emqx/include/logger.hrl").

apps/emqx_connector/test/emqx_connector_mongo_SUITE.erl

@@ -13,7 +13,7 @@
 % %% limitations under the License.
 % %%--------------------------------------------------------------------
 
--module(emqx_connector_mongo_SUITE).
+-module(emqx_mongodb_SUITE).
 
 -compile(nowarn_export_all).
 -compile(export_all).
@@ -24,7 +24,7 @@
 -include_lib("stdlib/include/assert.hrl").
 
 -define(MONGO_HOST, "mongo").
--define(MONGO_RESOURCE_MOD, emqx_connector_mongo).
+-define(MONGO_RESOURCE_MOD, emqx_mongodb).
 
 all() ->
     emqx_common_test_helpers:all(?MODULE).
@@ -60,7 +60,7 @@ end_per_testcase(_, _Config) ->
 
 t_lifecycle(_Config) ->
     perform_lifecycle_check(
-        <<"emqx_connector_mongo_SUITE">>,
+        <<"emqx_mongodb_SUITE">>,
         mongo_config()
     ).
 

apps/emqx_connector/test/emqx_connector_mongo_tests.erl

@@ -14,7 +14,7 @@
 %% limitations under the License.
 %%--------------------------------------------------------------------
 
--module(emqx_connector_mongo_tests).
+-module(emqx_mongodb_tests).
 
 -include_lib("eunit/include/eunit.hrl").
 
@@ -84,12 +84,12 @@ bad_query_string_test() ->
     ).
 
 resolve(Config) ->
-    emqx_connector_mongo:maybe_resolve_srv_and_txt_records(Config).
+    emqx_mongodb:maybe_resolve_srv_and_txt_records(Config).
 
 checked_config(Hocon) ->
     {ok, Config} = hocon:binary(Hocon),
     hocon_tconf:check_plain(
-        emqx_connector_mongo,
+        emqx_mongodb,
         #{<<"config">> => Config},
         #{atom_key => true}
     ).

apps/emqx_oracle/README.md

@@ -11,4 +11,4 @@ Please see our [contributing.md](../../CONTRIBUTING.md).
 
 ## License
 
-See [BSL](./BSL.txt).
+See [BSL](../../APL.txt).

changes/ce/feat-10667.en.md

@@ -0,0 +1 @@
+The MongoDB connector and bridge have been refactored to a separate app to improve code structure.

lib-ee/emqx_ee_bridge/docker-ct

@@ -1,6 +1,4 @@
 toxiproxy
-mongo
-mongo_rs_sharded
 mysql
 redis
 redis_cluster

lib-ee/emqx_ee_bridge/src/emqx_ee_bridge.erl

@@ -27,9 +27,9 @@ api_schemas(Method) ->
         api_ref(emqx_bridge_cassandra, <<"cassandra">>, Method),
         api_ref(emqx_ee_bridge_mysql, <<"mysql">>, Method),
         api_ref(emqx_bridge_pgsql, <<"pgsql">>, Method),
-        api_ref(emqx_ee_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"),
-        api_ref(emqx_ee_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"),
-        api_ref(emqx_ee_bridge_mongodb, <<"mongodb_single">>, Method ++ "_single"),
+        api_ref(emqx_bridge_mongodb, <<"mongodb_rs">>, Method ++ "_rs"),
+        api_ref(emqx_bridge_mongodb, <<"mongodb_sharded">>, Method ++ "_sharded"),
+        api_ref(emqx_bridge_mongodb, <<"mongodb_single">>, Method ++ "_single"),
         api_ref(emqx_ee_bridge_hstreamdb, <<"hstreamdb">>, Method),
         api_ref(emqx_bridge_influxdb, <<"influxdb_api_v1">>, Method ++ "_api_v1"),
         api_ref(emqx_bridge_influxdb, <<"influxdb_api_v2">>, Method ++ "_api_v2"),
@@ -57,7 +57,7 @@ schema_modules() ->
         emqx_ee_bridge_hstreamdb,
         emqx_bridge_gcp_pubsub,
         emqx_bridge_influxdb,
-        emqx_ee_bridge_mongodb,
+        emqx_bridge_mongodb,
         emqx_ee_bridge_mysql,
         emqx_ee_bridge_redis,
         emqx_bridge_pgsql,
@@ -96,9 +96,9 @@ resource_type(cassandra) -> emqx_bridge_cassandra_connector;
 resource_type(hstreamdb) -> emqx_ee_connector_hstreamdb;
 resource_type(gcp_pubsub) -> emqx_bridge_gcp_pubsub_impl_producer;
 resource_type(gcp_pubsub_consumer) -> emqx_bridge_gcp_pubsub_impl_consumer;
-resource_type(mongodb_rs) -> emqx_ee_connector_mongodb;
-resource_type(mongodb_sharded) -> emqx_ee_connector_mongodb;
-resource_type(mongodb_single) -> emqx_ee_connector_mongodb;
+resource_type(mongodb_rs) -> emqx_bridge_mongodb_connector;
+resource_type(mongodb_sharded) -> emqx_bridge_mongodb_connector;
+resource_type(mongodb_single) -> emqx_bridge_mongodb_connector;
 resource_type(mysql) -> emqx_connector_mysql;
 resource_type(influxdb_api_v1) -> emqx_bridge_influxdb_connector;
 resource_type(influxdb_api_v2) -> emqx_bridge_influxdb_connector;
@@ -203,7 +203,7 @@ mongodb_structs() ->
     [
         {Type,
             mk(
-                hoconsc:map(name, ref(emqx_ee_bridge_mongodb, Type)),
+                hoconsc:map(name, ref(emqx_bridge_mongodb, Type)),
                 #{
                     desc => <<"MongoDB Bridge Config">>,
                     required => false

lib-ee/emqx_ee_connector/src/emqx_ee_connector.app.src

@@ -1,6 +1,6 @@
 {application, emqx_ee_connector, [
     {description, "EMQX Enterprise connectors"},
-    {vsn, "0.1.14"},
+    {vsn, "0.1.15"},
     {registered, []},
     {applications, [
         kernel,

mix.exs

@@ -376,6 +376,7 @@ defmodule EMQXUmbrella.MixProject do
         emqx_prometheus: :permanent,
         emqx_psk: :permanent,
         emqx_slow_subs: :permanent,
+        emqx_mongodb: :permanent,
         emqx_plugins: :permanent,
         emqx_mix: :none
       ] ++

rebar.config.erl

@@ -439,6 +439,7 @@ relx_apps(ReleaseType, Edition) ->
             emqx_prometheus,
             emqx_psk,
             emqx_slow_subs,
+            emqx_mongodb,
             emqx_plugins
         ] ++
         [quicer || is_quicer_supported()] ++

rel/i18n/emqx_ee_bridge_mongodb.hocon

@@ -1,4 +1,4 @@
-emqx_ee_bridge_mongodb {
+emqx_bridge_mongodb {
 
 collection.desc:
 """The collection where data will be stored into"""

rel/i18n/emqx_connector_mongo.hocon

@@ -1,4 +1,4 @@
-emqx_connector_mongo {
+emqx_mongodb {
 
 auth_source.desc:
 """Database name associated with the user's credentials."""