feat(emqx_connector): add first emqx_connector for mysql

apps/emqx_connector/.gitignore

@@ -0,0 +1,19 @@
+.rebar3
+_*
+.eunit
+*.o
+*.beam
+*.plt
+*.swp
+*.swo
+.erlang.cookie
+ebin
+log
+erl_crash.dump
+.rebar
+logs
+_build
+.idea
+*.iml
+rebar3.crashdump
+*~

apps/emqx_connector/LICENSE

@@ -0,0 +1,191 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright 2021, Shawn <506895667@qq.com>.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

apps/emqx_connector/README.md

@@ -0,0 +1,9 @@
+emqx_connector
+=====
+
+An OTP application
+
+Build
+-----
+
+    $ rebar3 compile

apps/emqx_connector/rebar.config

@@ -0,0 +1,13 @@
+{erl_opts, [
+  nowarn_unused_import,
+  debug_info
+]}.
+
+{deps, [
+  {mysql, {git, "https://github.com/emqx/mysql-otp", {tag, "1.7.1"}}}
+]}.
+
+{shell, [
+  % {config, "config/sys.config"},
+    {apps, [emqx_connector]}
+]}.

apps/emqx_connector/src/emqx_connector.app.src

@@ -0,0 +1,16 @@
+{application, emqx_connector,
+ [{description, "An OTP application"},
+  {vsn, "0.1.0"},
+  {registered, []},
+  {mod, {emqx_connector_app, []}},
+  {applications,
+   [kernel,
+    stdlib,
+    emqx_resource
+   ]},
+  {env,[]},
+  {modules, []},
+
+  {licenses, ["Apache 2.0"]},
+  {links, []}
+ ]}.

apps/emqx_connector/src/emqx_connector_app.erl

@@ -0,0 +1,20 @@
+%%%-------------------------------------------------------------------
+%% @doc emqx_connector public API
+%% @end
+%%%-------------------------------------------------------------------
+
+-module(emqx_connector_app).
+
+-behaviour(application).
+
+-emqx_plugin(?MODULE).
+
+-export([start/2, stop/1]).
+
+start(_StartType, _StartArgs) ->
+    emqx_connector_sup:start_link().
+
+stop(_State) ->
+    ok.
+
+%% internal functions

apps/emqx_connector/src/emqx_connector_mysql.erl

@@ -0,0 +1,68 @@
+-module(emqx_connector_mysql).
+
+-include_lib("typerefl/include/types.hrl").
+-include_lib("emqx_resource/include/emqx_resource_behaviour.hrl").
+
+-emqx_resource_api_path("connectors/mysql").
+
+-export([fields/1]).
+
+%% callbacks of behaviour emqx_resource
+-export([ on_start/2
+        , on_stop/2
+        , on_query/4
+        , on_health_check/2
+        ]).
+
+-export([do_health_check/1]).
+
+fields("config") ->
+    emqx_connector_schema_lib:relational_db_fields() ++
+    emqx_connector_schema_lib:ssl_fields().
+
+%% ===================================================================
+
+on_start(InstId, #{server := {Host, Port},
+                   database := DB,
+                   user := User,
+                   password := Password,
+                   auto_reconnect := AutoReconn,
+                   pool_size := PoolSize} = Config) ->
+    logger:info("starting mysql connector: ~p, config: ~p", [InstId, Config]),
+    SslOpts = case maps:get(ssl, Config) of
+        true ->
+            [{ssl, [{server_name_indication, disable} |
+                    emqx_plugin_libs_ssl:save_files_return_opts(Config, "connectors", InstId)]}];
+        false ->
+            []
+    end,
+    Options = [{host, Host},
+               {port, Port},
+               {user, User},
+               {password, Password},
+               {database, DB},
+               {auto_reconnect, reconn_interval(AutoReconn)},
+               {pool_size, PoolSize}],
+    PoolName = emqx_plugin_libs_pool:pool_name(InstId),
+    _ = emqx_plugin_libs_pool:start_pool(PoolName, ?MODULE, Options ++ SslOpts),
+    {ok, #{poolname => PoolName}}.
+
+on_stop(InstId, #{poolname := PoolName}) ->
+    logger:info("stopping mysql connector: ~p", [InstId]),
+    emqx_plugin_libs_pool:stop_pool(PoolName).
+
+on_query(InstId, Request, AfterQuery, State) ->
+    io:format("== the demo log tracer ~p received request: ~p~nstate: ~p~n",
+        [InstId, Request, State]),
+    emqx_resource:query_success(AfterQuery),
+    "this is a demo log messages...".
+
+on_health_check(_InstId, #{poolname := PoolName} = State) ->
+    emqx_plugin_libs_pool:health_check(PoolName, fun ?MODULE:do_health_check/1, State).
+
+do_health_check(Conn) ->
+    ok == element(1, mysql:query(Conn, <<"SELECT count(1) AS T">>)).
+
+%% ===================================================================
+reconn_interval(true) -> 15;
+reconn_interval(false) -> false.

apps/emqx_connector/src/emqx_connector_schema_lib.erl

@@ -0,0 +1,102 @@
+-module(emqx_connector_schema_lib).
+-include_lib("typerefl/include/types.hrl").
+
+-export([ relational_db_fields/0
+        , ssl_fields/0
+        ]).
+
+-export([ to_ip_port/1
+        ]).
+
+-typerefl_from_string({ip_port/0, emqx_connector_schema_lib, to_ip_port}).
+
+-reflect_type([ip_port/0]).
+
+-type ip_port() :: tuple().
+
+-define(VALID, emqx_resource_validator).
+-define(REQUIRED(MSG), ?VALID:required(MSG)).
+-define(MAX(MAXV), ?VALID:max(number, MAXV)).
+-define(MIN(MINV), ?VALID:min(number, MINV)).
+
+relational_db_fields() ->
+    [ {server, fun server/1}
+    , {database, fun database/1}
+    , {pool_size, fun pool_size/1}
+    , {user, fun user/1}
+    , {password, fun password/1}
+    , {auto_reconnect, fun auto_reconnect/1}
+    ].
+
+ssl_fields() ->
+    [ {ssl, fun ssl/1}
+    , {cacertfile, fun cacertfile/1}
+    , {keyfile, fun keyfile/1}
+    , {certfile, fun certfile/1}
+    , {verify, fun verify/1}
+    ].
+
+server(mapping) -> "config.server";
+server(type) -> ip_port();
+server(validator) -> [?REQUIRED("the field 'server' is required")];
+server(_) -> undefined.
+
+database(mapping) -> "config.database";
+database(type) -> string();
+database(validator) -> [?REQUIRED("the field 'server' is required")];
+database(_) -> undefined.
+
+pool_size(mapping) -> "config.pool_size";
+pool_size(type) -> integer();
+pool_size(default) -> 8;
+pool_size(validator) -> [?MIN(1), ?MAX(64)];
+pool_size(_) -> undefined.
+
+user(mapping) -> "config.user";
+user(type) -> string();
+user(default) -> "root";
+user(_) -> undefined.
+
+password(mapping) -> "config.password";
+password(type) -> string();
+password(default) -> "";
+password(_) -> undefined.
+
+auto_reconnect(mapping) -> "config.auto_reconnect";
+auto_reconnect(type) -> boolean();
+auto_reconnect(default) -> true;
+auto_reconnect(_) -> undefined.
+ssl(mapping) -> "config.ssl";
+ssl(type) -> boolean();
+ssl(default) -> false;
+ssl(_) -> undefined.
+
+cacertfile(mapping) -> "config.cacertfile";
+cacertfile(type) -> string();
+cacertfile(default) -> "";
+cacertfile(_) -> undefined.
+
+keyfile(mapping) -> "config.keyfile";
+keyfile(type) -> string();
+keyfile(default) -> "";
+keyfile(_) -> undefined.
+
+certfile(mapping) -> "config.certfile";
+certfile(type) -> string();
+certfile(default) -> "";
+certfile(_) -> undefined.
+
+verify(mapping) -> "config.verify";
+verify(type) -> boolean();
+verify(default) -> false;
+verify(_) -> undefined.
+
+to_ip_port(Str) ->
+     case string:tokens(Str, ":") of
+         [Ip, Port] ->
+             case inet:parse_address(Ip) of
+                 {ok, R} -> {ok, {R, list_to_integer(Port)}};
+                 _ -> {error, Str}
+             end;
+         _ -> {error, Str}
+     end.

apps/emqx_connector/src/emqx_connector_sup.erl

@@ -0,0 +1,35 @@
+%%%-------------------------------------------------------------------
+%% @doc emqx_connector top level supervisor.
+%% @end
+%%%-------------------------------------------------------------------
+
+-module(emqx_connector_sup).
+
+-behaviour(supervisor).
+
+-export([start_link/0]).
+
+-export([init/1]).
+
+-define(SERVER, ?MODULE).
+
+start_link() ->
+    supervisor:start_link({local, ?SERVER}, ?MODULE, []).
+
+%% sup_flags() = #{strategy => strategy(),         % optional
+%%                 intensity => non_neg_integer(), % optional
+%%                 period => pos_integer()}        % optional
+%% child_spec() = #{id => child_id(),       % mandatory
+%%                  start => mfargs(),      % mandatory
+%%                  restart => restart(),   % optional
+%%                  shutdown => shutdown(), % optional
+%%                  type => worker(),       % optional
+%%                  modules => modules()}   % optional
+init([]) ->
+    SupFlags = #{strategy => one_for_all,
+                 intensity => 0,
+                 period => 1},
+    ChildSpecs = [],
+    {ok, {SupFlags, ChildSpecs}}.
+
+%% internal functions

apps/emqx_plugin_libs/src/emqx_plugin_libs_pool.erl

@@ -0,0 +1,57 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2021 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%%     http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%--------------------------------------------------------------------
+
+-module(emqx_plugin_libs_pool).
+
+-export([ start_pool/3
+        , stop_pool/1
+        , pool_name/1
+        , health_check/3
+        ]).
+
+pool_name(ID) when is_binary(ID) ->
+    list_to_atom(binary_to_list(ID)).
+
+start_pool(Name, Mod, Options) ->
+    case ecpool:start_sup_pool(Name, Mod, Options) of
+        {ok, _} -> logger:log(info, "Initiated ~0p Successfully", [Name]);
+        {error, {already_started, _Pid}} ->
+            stop_pool(Name),
+            start_pool(Name, Mod, Options);
+        {error, Reason} ->
+            logger:log(error, "Initiate ~0p failed ~0p", [Name, Reason]),
+            error({start_pool_failed, Name})
+    end.
+
+stop_pool(Name) ->
+    case ecpool:stop_sup_pool(Name) of
+        ok -> logger:log(info, "Destroyed ~0p Successfully", [Name]);
+        {error, Reason} ->
+            logger:log(error, "Destroy ~0p failed, ~0p", [Name, Reason]),
+            error({stop_pool_failed, Name})
+    end.
+
+health_check(PoolName, CheckFunc, State) when is_function(CheckFunc) ->
+    Status = [begin
+        case ecpool_worker:client(Worker) of
+            {ok, Conn} -> CheckFunc(Conn);
+            _ -> false
+        end
+    end || {_WorkerName, Worker} <- ecpool:workers(PoolName)],
+    case length(Status) > 0 andalso lists:all(fun(St) -> St =:= true end, Status) of
+        true -> {ok, State};
+        false -> {error, test_query_failed}
+    end.

data/loaded_plugins.tmpl

@@ -6,4 +6,5 @@
 {emqx_telemetry, {{enable_plugin_emqx_telemetry}}}.
 {emqx_rule_engine, {{enable_plugin_emqx_rule_engine}}}.
 {emqx_resource, {{enable_plugin_emqx_resource}}}.
+{emqx_connector, {{enable_plugin_emqx_connector}}}.
 {emqx_bridge_mqtt, {{enable_plugin_emqx_bridge_mqtt}}}.

rebar.config.erl

@@ -181,6 +181,7 @@ overlay_vars_rel(RelType) ->
     [ {enable_plugin_emqx_rule_engine, RelType =:= cloud}
     , {enable_plugin_emqx_bridge_mqtt, RelType =:= edge}
     , {enable_plugin_emqx_resource, true}
+    , {enable_plugin_emqx_connector, true}
     , {enable_plugin_emqx_modules, false} %% modules is not a plugin in ce
     , {enable_plugin_emqx_recon, true}
     , {enable_plugin_emqx_retainer, true}
@@ -275,6 +276,7 @@ relx_plugin_apps(ReleaseType) ->
     , emqx_web_hook
     , emqx_recon
     , emqx_resource
+    , emqx_connector
     , emqx_rule_engine
     , emqx_sasl
     ]