Merge remote-tracking branch 'origin/release-56' into sync-r56-m-20240304

.github/workflows/build_and_push_docker_images.yaml

@@ -175,7 +175,10 @@ jobs:
           EMQX_SOURCE_TYPE: tgz
         run: |
           ./build ${PROFILE} docker
+          echo "Built tags:"
+          echo "==========="
           cat .emqx_docker_image_tags
+          echo "==========="
           echo "_EMQX_DOCKER_IMAGE_TAG=$(head -n 1 .emqx_docker_image_tags)" >> $GITHUB_ENV
 
       - name: smoke test
@@ -204,5 +207,6 @@ jobs:
         if: inputs.publish || github.repository_owner != 'emqx'
         run: |
           for tag in $(cat .emqx_docker_image_tags); do
+            echo "Pushing tag $tag"
             docker push $tag
           done

.github/workflows/build_packages.yaml

@@ -196,9 +196,9 @@ jobs:
         set -eu
         cd packages/${{ matrix.profile }}
         # fix the .sha256 file format
-        for var in $(ls | grep emqx | grep -v sha256); do
-          dos2unix $var.sha256
-          echo "$(cat $var.sha256) $var" | sha256sum -c || exit 1
+        for f in *.sha256; do
+          dos2unix $f
+          echo "$(cat $f) ${f%.*}" | sha256sum -c || exit 1
         done
         cd -
     - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2

Makefile

@@ -21,7 +21,7 @@ endif
 # Dashboard version
 # from https://github.com/emqx/emqx-dashboard5
 export EMQX_DASHBOARD_VERSION ?= v1.7.0
-export EMQX_EE_DASHBOARD_VERSION ?= e1.6.0-beta.1
+export EMQX_EE_DASHBOARD_VERSION ?= e1.6.0-beta.2
 
 PROFILE ?= emqx
 REL_PROFILES := emqx emqx-enterprise

apps/emqx/include/emqx_release.hrl

@@ -32,7 +32,7 @@
 %% `apps/emqx/src/bpapi/README.md'
 
 %% Opensource edition
--define(EMQX_RELEASE_CE, "5.6.0-alpha.1").
+-define(EMQX_RELEASE_CE, "5.6.0-alpha.2").
 
 %% Enterprise edition
--define(EMQX_RELEASE_EE, "5.6.0-alpha.1").
+-define(EMQX_RELEASE_EE, "5.6.0-alpha.2").

apps/emqx/rebar.config

@@ -30,7 +30,7 @@
     {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.11.1"}}},
     {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.19.0"}}},
     {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "3.3.1"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.41.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.0"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
     {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {tag, "2.0.4"}}},
     {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}},

apps/emqx_bridge/src/emqx_action_info.erl

@@ -105,6 +105,7 @@ hard_coded_action_info_modules_ee() ->
         emqx_bridge_mysql_action_info,
         emqx_bridge_pgsql_action_info,
         emqx_bridge_syskeeper_action_info,
+        emqx_bridge_sqlserver_action_info,
         emqx_bridge_timescale_action_info,
         emqx_bridge_redis_action_info,
         emqx_bridge_iotdb_action_info,

apps/emqx_bridge_clickhouse/src/emqx_bridge_clickhouse_action_info.erl

@@ -1,3 +1,7 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
 -module(emqx_bridge_clickhouse_action_info).
 
 -behaviour(emqx_action_info).

apps/emqx_bridge_http/src/emqx_bridge_http_connector.erl

@@ -869,9 +869,9 @@ redact(Data) ->
 %% and we also can't know the body format and where the sensitive data will be
 %% so the easy way to keep data security is redacted the whole body
 redact_request({Path, Headers}) ->
-    {Path, redact(Headers)};
+    {Path, emqx_utils_redact:redact_headers(Headers)};
 redact_request({Path, Headers, _Body}) ->
-    {Path, redact(Headers), <<"******">>}.
+    {Path, emqx_utils_redact:redact_headers(Headers), <<"******">>}.
 
 clientid(Msg) -> maps:get(clientid, Msg, undefined).
 

apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.app.src

@@ -3,6 +3,7 @@
     {vsn, "0.1.6"},
     {registered, []},
     {applications, [kernel, stdlib, emqx_resource, odbc]},
+    {env, [{emqx_action_info_modules, [emqx_bridge_sqlserver_action_info]}]},
     {env, []},
     {modules, []},
     {links, []}

apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver.erl

@@ -11,6 +11,8 @@
 -import(hoconsc, [mk/2, enum/1, ref/2]).
 
 -export([
+    bridge_v2_examples/1,
+    connector_examples/1,
     conn_bridge_examples/1
 ]).
 
@@ -21,6 +23,9 @@
     desc/1
 ]).
 
+-define(CONNECTOR_TYPE, sqlserver).
+-define(ACTION_TYPE, ?CONNECTOR_TYPE).
+
 -define(DEFAULT_SQL, <<
     "insert into t_mqtt_msg(msgid, topic, qos, payload) "
     "values ( ${id}, ${topic}, ${qos}, ${payload} )"
@@ -28,6 +33,9 @@
 
 -define(DEFAULT_DRIVER, <<"ms-sql">>).
 
+%% -------------------------------------------------------------------------------------------------
+%% api.
+
 conn_bridge_examples(Method) ->
     [
         #{
@@ -65,12 +73,81 @@ values(post) ->
 values(put) ->
     values(post).
 
+%% ====================
+%% Bridge V2: Connector + Action
+
+connector_examples(Method) ->
+    [
+        #{
+            <<"sqlserver">> =>
+                #{
+                    summary => <<"Microsoft SQL Server Connector">>,
+                    value => emqx_connector_schema:connector_values(
+                        Method, ?CONNECTOR_TYPE, connector_values()
+                    )
+                }
+        }
+    ].
+
+connector_values() ->
+    #{
+        server => <<"127.0.0.1:1433">>,
+        database => <<"test">>,
+        pool_size => 8,
+        username => <<"sa">>,
+        password => <<"******">>,
+        driver => ?DEFAULT_DRIVER,
+        resource_opts => #{health_check_interval => <<"20s">>}
+    }.
+
+bridge_v2_examples(Method) ->
+    [
+        #{
+            <<"sqlserver">> =>
+                #{
+                    summary => <<"Microsoft SQL Server Action">>,
+                    value => emqx_bridge_v2_schema:action_values(
+                        Method, ?ACTION_TYPE, ?CONNECTOR_TYPE, action_values()
+                    )
+                }
+        }
+    ].
+
+action_values() ->
+    #{
+        <<"parameters">> =>
+            #{<<"sql">> => ?DEFAULT_SQL}
+    }.
+
 %% -------------------------------------------------------------------------------------------------
 %% Hocon Schema Definitions
 namespace() -> "bridge_sqlserver".
 
 roots() -> [].
 
+fields(Field) when
+    Field == "get_bridge_v2";
+    Field == "post_bridge_v2";
+    Field == "put_bridge_v2"
+->
+    emqx_bridge_v2_schema:api_fields(Field, ?ACTION_TYPE, fields(sqlserver_action));
+fields(Field) when
+    Field == "get_connector";
+    Field == "put_connector";
+    Field == "post_connector"
+->
+    emqx_connector_schema:api_fields(
+        Field,
+        ?CONNECTOR_TYPE,
+        fields("config_connector") -- emqx_connector_schema:common_fields()
+    );
+fields("config_connector") ->
+    driver_fields() ++
+        emqx_connector_schema:common_fields() ++
+        emqx_bridge_sqlserver_connector:fields(config) ++
+        emqx_connector_schema:resource_opts_ref(?MODULE, connector_resource_opts);
+fields(connector_resource_opts) ->
+    emqx_connector_schema:resource_opts_fields();
 fields("config") ->
     [
         {enable, mk(boolean(), #{desc => ?DESC("config_enable"), default => true})},
@@ -79,7 +156,6 @@ fields("config") ->
                 binary(),
                 #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>}
             )},
-        {driver, mk(binary(), #{desc => ?DESC("driver"), default => ?DEFAULT_DRIVER})},
         {local_topic,
             mk(
                 binary(),
@@ -94,9 +170,30 @@ fields("config") ->
                     desc => ?DESC(emqx_resource_schema, <<"resource_opts">>)
                 }
             )}
-    ] ++
+    ] ++ driver_fields() ++
         (emqx_bridge_sqlserver_connector:fields(config) --
             emqx_connector_schema_lib:prepare_statement_fields());
+fields(action) ->
+    {?ACTION_TYPE,
+        mk(
+            hoconsc:map(name, ref(?MODULE, sqlserver_action)),
+            #{desc => ?DESC("sqlserver_action"), required => false}
+        )};
+fields(sqlserver_action) ->
+    emqx_bridge_v2_schema:make_producer_action_schema(
+        mk(
+            ref(?MODULE, action_parameters),
+            #{required => true, desc => ?DESC(action_parameters)}
+        )
+    );
+fields(action_parameters) ->
+    [
+        {sql,
+            mk(
+                binary(),
+                #{desc => ?DESC("sql_template"), default => ?DEFAULT_SQL, format => <<"sql">>}
+            )}
+    ];
 fields("creation_opts") ->
     emqx_resource_schema:fields("creation_opts");
 fields("post") ->
@@ -109,12 +206,23 @@ fields("get") ->
 fields("post", Type) ->
     [type_field(Type), name_field() | fields("config")].
 
+driver_fields() ->
+    [{driver, mk(binary(), #{desc => ?DESC("driver"), default => ?DEFAULT_DRIVER})}].
+
 desc("config") ->
     ?DESC("desc_config");
 desc(Method) when Method =:= "get"; Method =:= "put"; Method =:= "post" ->
     ["Configuration for Microsoft SQL Server using `", string:to_upper(Method), "` method."];
 desc("creation_opts" = Name) ->
     emqx_resource_schema:desc(Name);
+desc("config_connector") ->
+    ?DESC("config_connector");
+desc(sqlserver_action) ->
+    ?DESC("sqlserver_action");
+desc(action_parameters) ->
+    ?DESC("action_parameters");
+desc(connector_resource_opts) ->
+    ?DESC(emqx_resource_schema, "resource_opts");
 desc(_) ->
     undefined.
 

apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_action_info.erl

@@ -0,0 +1,72 @@
+%%--------------------------------------------------------------------
+%% Copyright (c) 2024 EMQ Technologies Co., Ltd. All Rights Reserved.
+%%--------------------------------------------------------------------
+
+-module(emqx_bridge_sqlserver_action_info).
+
+-behaviour(emqx_action_info).
+
+-elvis([{elvis_style, invalid_dynamic_call, disable}]).
+
+-export([
+    bridge_v1_type_name/0,
+    action_type_name/0,
+    connector_type_name/0,
+    schema_module/0,
+    bridge_v1_config_to_action_config/2,
+    bridge_v1_config_to_connector_config/1,
+    connector_action_config_to_bridge_v1_config/2
+]).
+
+-import(emqx_utils_conv, [bin/1]).
+
+-define(ACTION_TYPE, sqlserver).
+-define(SCHEMA_MODULE, emqx_bridge_sqlserver).
+
+bridge_v1_type_name() -> ?ACTION_TYPE.
+
+action_type_name() -> ?ACTION_TYPE.
+
+connector_type_name() -> ?ACTION_TYPE.
+
+schema_module() -> ?SCHEMA_MODULE.
+
+bridge_v1_config_to_action_config(BridgeV1Config, ConnectorName) ->
+    ActionTopLevelKeys = schema_keys(sqlserver_action),
+    ActionParametersKeys = schema_keys(action_parameters),
+    ActionKeys = ActionTopLevelKeys ++ ActionParametersKeys,
+    ActionConfig = make_config_map(ActionKeys, ActionParametersKeys, BridgeV1Config),
+    emqx_utils_maps:update_if_present(
+        <<"resource_opts">>,
+        fun emqx_bridge_v2_schema:project_to_actions_resource_opts/1,
+        ActionConfig#{<<"connector">> => ConnectorName}
+    ).
+
+bridge_v1_config_to_connector_config(BridgeV1Config) ->
+    ActionTopLevelKeys = schema_keys(sqlserver_action),
+    ActionParametersKeys = schema_keys(action_parameters),
+    ActionKeys = ActionTopLevelKeys ++ ActionParametersKeys,
+    ConnectorTopLevelKeys = schema_keys("config_connector"),
+    ConnectorKeys = maps:keys(BridgeV1Config) -- (ActionKeys -- ConnectorTopLevelKeys),
+    ConnConfig0 = maps:with(ConnectorKeys, BridgeV1Config),
+    emqx_utils_maps:update_if_present(
+        <<"resource_opts">>,
+        fun emqx_connector_schema:project_to_connector_resource_opts/1,
+        ConnConfig0
+    ).
+
+connector_action_config_to_bridge_v1_config(ConnectorConfig, ActionConfig) ->
+    V1Config = emqx_action_info:connector_action_config_to_bridge_v1_config(
+        ConnectorConfig, ActionConfig
+    ),
+    maps:remove(<<"local_topic">>, V1Config).
+
+make_config_map(PickKeys, IndentKeys, Config) ->
+    Conf0 = maps:with(PickKeys, Config),
+    emqx_utils_maps:indent(<<"parameters">>, IndentKeys, Conf0).
+
+schema_keys(Name) ->
+    schema_keys(?SCHEMA_MODULE, Name).
+
+schema_keys(Mod, Name) ->
+    [bin(Key) || Key <- proplists:get_keys(Mod:fields(Name))].

apps/emqx_bridge_sqlserver/src/emqx_bridge_sqlserver_connector.erl

@@ -35,7 +35,11 @@
     on_stop/2,
     on_query/3,
     on_batch_query/3,
-    on_get_status/2
+    on_get_status/2,
+    on_add_channel/4,
+    on_remove_channel/3,
+    on_get_channels/1,
+    on_get_channel_status/3
 ]).
 
 %% callbacks for ecpool
@@ -124,9 +128,9 @@
 %% -type size() :: integer().
 
 -type state() :: #{
+    installed_channels := map(),
     pool_name := binary(),
-    resource_opts := map(),
-    sql_templates := map()
+    resource_opts := map()
 }.
 
 %%====================================================================
@@ -172,7 +176,7 @@ server() ->
 callback_mode() -> always_sync.
 
 on_start(
-    ResourceId = PoolName,
+    InstanceId = PoolName,
     #{
         server := Server,
         username := Username,
@@ -184,7 +188,7 @@ on_start(
 ) ->
     ?SLOG(info, #{
         msg => "starting_sqlserver_connector",
-        connector => ResourceId,
+        connector => InstanceId,
         config => emqx_utils:redact(Config)
     }),
 
@@ -199,7 +203,8 @@ on_start(
             ok
     end,
 
-    Options = [
+    %% odbc connection string required
+    ConnectOptions = [
         {server, to_bin(Server)},
         {username, Username},
         {password, maps:get(password, Config, emqx_secret:wrap(""))},
@@ -209,12 +214,12 @@ on_start(
     ],
 
     State = #{
-        %% also ResourceId
+        %% also InstanceId
         pool_name => PoolName,
-        sql_templates => parse_sql_template(Config),
+        installed_channels => #{},
         resource_opts => ResourceOpts
     },
-    case emqx_resource_pool:start(PoolName, ?MODULE, Options) of
+    case emqx_resource_pool:start(PoolName, ?MODULE, ConnectOptions) of
         ok ->
             {ok, State};
         {error, Reason} ->
@@ -225,23 +230,72 @@ on_start(
             {error, Reason}
     end.
 
-on_stop(ResourceId, _State) ->
+on_add_channel(
+    _InstId,
+    #{
+        installed_channels := InstalledChannels
+    } = OldState,
+    ChannelId,
+    ChannelConfig
+) ->
+    {ok, ChannelState} = create_channel_state(ChannelConfig),
+    NewInstalledChannels = maps:put(ChannelId, ChannelState, InstalledChannels),
+    %% Update state
+    NewState = OldState#{installed_channels => NewInstalledChannels},
+    {ok, NewState}.
+
+create_channel_state(
+    #{parameters := Conf} = _ChannelConfig
+) ->
+    State = #{sql_templates => parse_sql_template(Conf)},
+    {ok, State}.
+
+on_remove_channel(
+    _InstId,
+    #{
+        installed_channels := InstalledChannels
+    } = OldState,
+    ChannelId
+) ->
+    NewInstalledChannels = maps:remove(ChannelId, InstalledChannels),
+    %% Update state
+    NewState = OldState#{installed_channels => NewInstalledChannels},
+    {ok, NewState}.
+
+on_get_channel_status(
+    InstanceId,
+    ChannelId,
+    #{installed_channels := Channels} = State
+) ->
+    case maps:find(ChannelId, Channels) of
+        {ok, _} -> on_get_status(InstanceId, State);
+        error -> ?status_disconnected
+    end.
+
+on_get_channels(ResId) ->
+    emqx_bridge_v2:get_channels_for_connector(ResId).
+
+on_stop(InstanceId, _State) ->
+    ?tp(
+        sqlserver_connector_on_stop,
+        #{instance_id => InstanceId}
+    ),
     ?SLOG(info, #{
         msg => "stopping_sqlserver_connector",
-        connector => ResourceId
+        connector => InstanceId
     }),
-    emqx_resource_pool:stop(ResourceId).
+    emqx_resource_pool:stop(InstanceId).
 
 -spec on_query(
     resource_id(),
-    {?ACTION_SEND_MESSAGE, map()},
+    Query :: {channel_id(), map()},
     state()
 ) ->
     ok
     | {ok, list()}
     | {error, {recoverable_error, term()}}
     | {error, term()}.
-on_query(ResourceId, {?ACTION_SEND_MESSAGE, _Msg} = Query, State) ->
+on_query(ResourceId, {_ChannelId, _Msg} = Query, State) ->
     ?TRACE(
         "SINGLE_QUERY_SYNC",
         "bridge_sqlserver_received",
@@ -251,7 +305,7 @@ on_query(ResourceId, {?ACTION_SEND_MESSAGE, _Msg} = Query, State) ->
 
 -spec on_batch_query(
     resource_id(),
-    [{?ACTION_SEND_MESSAGE, map()}],
+    [{channel_id(), map()}],
     state()
 ) ->
     ok
@@ -273,8 +327,8 @@ on_get_status(_InstanceId, #{pool_name := PoolName} = _State) ->
     ),
     status_result(Health).
 
-status_result(_Status = true) -> connected;
-status_result(_Status = false) -> connecting.
+status_result(_Status = true) -> ?status_connected;
+status_result(_Status = false) -> ?status_connecting.
 %% TODO:
 %% case for disconnected
 
@@ -296,7 +350,7 @@ do_get_status(Conn) ->
     end.
 
 %%====================================================================
-%% Internal Helper fns
+%% Internal Functions
 %%====================================================================
 
 %% TODO && FIXME:
@@ -329,7 +383,7 @@ conn_str([{_, _} | Opts], Acc) ->
 %% Query with singe & batch sql statement
 -spec do_query(
     resource_id(),
-    Query :: {?ACTION_SEND_MESSAGE, map()} | [{?ACTION_SEND_MESSAGE, map()}],
+    Query :: {channel_id(), map()} | [{channel_id(), map()}],
     ApplyMode :: handover,
     state()
 ) ->
@@ -341,7 +395,10 @@ do_query(
     ResourceId,
     Query,
     ApplyMode,
-    #{pool_name := PoolName, sql_templates := Templates} = State
+    #{
+        pool_name := PoolName,
+        installed_channels := Channels
+    } = State
 ) ->
     ?TRACE(
         "SINGLE_QUERY_SYNC",
@@ -349,15 +406,19 @@ do_query(
         #{query => Query, connector => ResourceId, state => State}
     ),
 
+    ChannelId = get_channel_id(Query),
+    QueryTuple = get_query_tuple(Query),
+    #{sql_templates := Templates} = _ChannelState = maps:get(ChannelId, Channels),
+
     %% only insert sql statement for single query and batch query
-    case apply_template(Query, Templates) of
+    case apply_template(QueryTuple, Templates) of
         {?ACTION_SEND_MESSAGE, SQL} ->
             Result = ecpool:pick_and_do(
                 PoolName,
                 {?MODULE, worker_do_insert, [SQL, State]},
                 ApplyMode
             );
-        Query ->
+        QueryTuple ->
             Result = {error, {unrecoverable_error, invalid_query}};
         _ ->
             Result = {error, {unrecoverable_error, failed_to_apply_sql_template}}
@@ -426,8 +487,22 @@ execute(Conn, SQL) ->
 execute(Conn, SQL, Timeout) ->
     odbc:sql_query(Conn, str(SQL), Timeout).
 
-to_bin(List) when is_list(List) ->
-    unicode:characters_to_binary(List, utf8).
+get_channel_id([{ChannelId, _Req} | _]) ->
+    ChannelId;
+get_channel_id({ChannelId, _Req}) ->
+    ChannelId.
+
+get_query_tuple({_ChannelId, {QueryType, Data}} = _Query) ->
+    {QueryType, Data};
+get_query_tuple({_ChannelId, Data} = _Query) ->
+    {send_message, Data};
+get_query_tuple([{_ChannelId, {_QueryType, _Data}} | _]) ->
+    error(
+        {unrecoverable_error,
+            {invalid_request, <<"The only query type that supports batching is insert.">>}}
+    );
+get_query_tuple([InsertQuery | _]) ->
+    get_query_tuple(InsertQuery).
 
 %% for bridge data to sql server
 parse_sql_template(Config) ->
@@ -506,3 +581,6 @@ proc_batch_sql(BatchReqs, BatchInserts, Tokens) ->
         ])
     ),
     <<BatchInserts/binary, " values ", Values/binary>>.
+
+to_bin(List) when is_list(List) ->
+    unicode:characters_to_binary(List, utf8).

apps/emqx_bridge_sqlserver/test/emqx_bridge_sqlserver_SUITE.erl

@@ -249,6 +249,7 @@ t_create_disconnected(Config) ->
         ?assertMatch({ok, _}, create_bridge(Config)),
         health_check_resource_down(Config)
     end),
+    timer:sleep(10_000),
     health_check_resource_ok(Config),
     ok.
 
@@ -317,7 +318,8 @@ t_simple_query(Config) ->
         {ok, _},
         create_bridge(Config)
     ),
-    {Requests, Vals} = gen_batch_req(BatchSize),
+
+    {Requests, Vals} = gen_batch_req(Config, BatchSize),
     ?check_trace(
         begin
             ?wait_async_action(
@@ -519,14 +521,15 @@ create_bridge_http(Params) ->
 send_message(Config, Payload) ->
     Name = ?config(sqlserver_name, Config),
     BridgeType = ?config(sqlserver_bridge_type, Config),
-    BridgeID = emqx_bridge_resource:bridge_id(BridgeType, Name),
-    emqx_bridge:send_message(BridgeID, Payload).
+    ActionId = emqx_bridge_v2:id(BridgeType, Name),
+    emqx_bridge_v2:query(BridgeType, Name, {ActionId, Payload}, #{}).
 
 query_resource(Config, Request) ->
     Name = ?config(sqlserver_name, Config),
     BridgeType = ?config(sqlserver_bridge_type, Config),
-    ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name),
-    emqx_resource:query(ResourceID, Request, #{timeout => 1_000}).
+    ID = emqx_bridge_v2:id(BridgeType, Name),
+    ResID = emqx_connector_resource:resource_id(BridgeType, Name),
+    emqx_resource:query(ID, Request, #{timeout => 1_000, connector_resource_id => ResID}).
 
 query_resource_async(Config, Request) ->
     Name = ?config(sqlserver_name, Config),
@@ -545,7 +548,17 @@ resource_id(Config) ->
     _ResourceID = emqx_bridge_resource:resource_id(BridgeType, Name).
 
 health_check_resource_ok(Config) ->
-    ?assertEqual({ok, connected}, emqx_resource_manager:health_check(resource_id(Config))).
+    BridgeType = ?config(sqlserver_bridge_type, Config),
+    Name = ?config(sqlserver_name, Config),
+    % Wait for reconnection.
+    ?retry(
+        _Sleep = 1_000,
+        _Attempts = 10,
+        begin
+            ?assertEqual({ok, connected}, emqx_resource_manager:health_check(resource_id(Config))),
+            ?assertMatch(#{status := connected}, emqx_bridge_v2:health_check(BridgeType, Name))
+        end
+    ).
 
 health_check_resource_down(Config) ->
     case emqx_resource_manager:health_check(resource_id(Config)) of
@@ -666,13 +679,16 @@ sent_data(Payload) ->
         qos => 0
     }.
 
-gen_batch_req(Count) when
+gen_batch_req(Config, Count) when
     is_integer(Count) andalso Count > 0
 ->
+    BridgeType = ?config(sqlserver_bridge_type, Config),
+    Name = ?config(sqlserver_name, Config),
+    ActionId = emqx_bridge_v2:id(BridgeType, Name),
     Vals = [{str(erlang:unique_integer())} || _Seq <- lists:seq(1, Count)],
-    Requests = [{send_message, sent_data(Payload)} || {Payload} <- Vals],
+    Requests = [{ActionId, sent_data(Payload)} || {Payload} <- Vals],
     {Requests, Vals};
-gen_batch_req(Count) ->
+gen_batch_req(_Config, Count) ->
     ct:pal("Gen batch requests failed with unexpected Count: ~p", [Count]).
 
 str(List) when is_list(List) ->

apps/emqx_connector/src/schema/emqx_connector_ee_schema.erl

@@ -60,6 +60,8 @@ resource_type(syskeeper_forwarder) ->
     emqx_bridge_syskeeper_connector;
 resource_type(syskeeper_proxy) ->
     emqx_bridge_syskeeper_proxy_server;
+resource_type(sqlserver) ->
+    emqx_bridge_sqlserver_connector;
 resource_type(timescale) ->
     emqx_postgresql;
 resource_type(redis) ->
@@ -283,6 +285,14 @@ connector_structs() ->
                     required => false
                 }
             )},
+        {sqlserver,
+            mk(
+                hoconsc:map(name, ref(emqx_bridge_sqlserver, "config_connector")),
+                #{
+                    desc => <<"Microsoft SQL Server Connector Config">>,
+                    required => false
+                }
+            )},
         {timescale,
             mk(
                 hoconsc:map(name, ref(emqx_bridge_timescale, "config_connector")),
@@ -377,6 +387,7 @@ schema_modules() ->
         emqx_bridge_mysql,
         emqx_bridge_syskeeper_connector,
         emqx_bridge_syskeeper_proxy,
+        emqx_bridge_sqlserver,
         emqx_bridge_timescale,
         emqx_postgresql_connector_schema,
         emqx_bridge_redis_schema,
@@ -427,6 +438,7 @@ api_schemas(Method) ->
         api_ref(emqx_bridge_mysql, <<"mysql">>, Method ++ "_connector"),
         api_ref(emqx_bridge_syskeeper_connector, <<"syskeeper_forwarder">>, Method),
         api_ref(emqx_bridge_syskeeper_proxy, <<"syskeeper_proxy">>, Method),
+        api_ref(emqx_bridge_sqlserver, <<"sqlserver">>, Method ++ "_connector"),
         api_ref(emqx_bridge_timescale, <<"timescale">>, Method ++ "_connector"),
         api_ref(emqx_postgresql_connector_schema, <<"pgsql">>, Method ++ "_connector"),
         api_ref(emqx_bridge_redis_schema, <<"redis">>, Method ++ "_connector"),

apps/emqx_connector/src/schema/emqx_connector_schema.erl

@@ -166,6 +166,8 @@ connector_type_to_bridge_types(syskeeper_forwarder) ->
     [syskeeper_forwarder];
 connector_type_to_bridge_types(syskeeper_proxy) ->
     [];
+connector_type_to_bridge_types(sqlserver) ->
+    [sqlserver];
 connector_type_to_bridge_types(timescale) ->
     [timescale];
 connector_type_to_bridge_types(iotdb) ->

apps/emqx_management/src/emqx_mgmt_auth.erl

@@ -311,7 +311,7 @@ maybe_cleanup_api_key(#?APP{name = Name, api_key = ApiKey}) ->
 
             %% Note for EMQX-11844:
             %% emqx.conf has the highest priority
-            %% if there is a key conflict, delete the old one and keep the key which from the bootstrap filex
+            %% if there is a key conflict, delete the old one and keep the key which from the bootstrap file
             ?SLOG(info, #{
                 msg => "duplicated_apikey_detected",
                 info => <<"Delete duplicated apikeys and write a new one from bootstrap file">>,

apps/emqx_resource/src/emqx_resource.erl

@@ -37,8 +37,8 @@
 %% provisional solution: rpc:multicall to all the nodes for creating/updating/removing
 %% todo: replicate operations
 
-%% store the config and start the instance
 -export([
+    %% store the config and start the instance
     create_local/4,
     create_local/5,
     create_dry_run_local/2,

apps/emqx_utils/src/emqx_utils_redact.erl

@@ -16,7 +16,7 @@
 
 -module(emqx_utils_redact).
 
--export([redact/1, redact/2, is_redacted/2, is_redacted/3]).
+-export([redact/1, redact/2, redact_headers/1, is_redacted/2, is_redacted/3]).
 -export([deobfuscate/2]).
 
 -define(REDACT_VAL, "******").
@@ -62,6 +62,9 @@ redact(Term, Checker) ->
         is_sensitive_key(V) orelse Checker(V)
     end).
 
+redact_headers(Term) ->
+    do_redact_headers(Term).
+
 do_redact(L, Checker) when is_list(L) ->
     lists:map(fun(E) -> do_redact(E, Checker) end, L);
 do_redact(M, Checker) when is_map(M) ->
@@ -128,7 +131,7 @@ do_redact_headers(Value) ->
     Value.
 
 check_is_sensitive_header(Key) ->
-    Key1 = emqx_utils_conv:str(Key),
+    Key1 = string:trim(emqx_utils_conv:str(Key)),
     is_sensitive_header(string:lowercase(Key1)).
 
 is_sensitive_header("authorization") ->

build

@@ -385,6 +385,16 @@ docker_cleanup() {
     [ -f ./.dockerignore.bak ] && mv ./.dockerignore.bak ./.dockerignore >/dev/null || true
 }
 
+function is_ecr_and_enterprise() {
+  local registry="$1"
+  local profile="$2"
+  if [[ "$registry" == public.ecr.aws* ]] && [[ "$profile" == *enterprise* ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
 ## Build the default docker image based on debian 11.
 make_docker() {
     local EMQX_BUILDER_VERSION="${EMQX_BUILDER_VERSION:-5.3-2}"
@@ -460,8 +470,10 @@ make_docker() {
     )
     :> ./.emqx_docker_image_tags
     for r in "${DOCKER_REGISTRIES[@]}"; do
-        DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_IMAGE_TAG}")
-        echo "$r/${EMQX_IMAGE_TAG}" >> ./.emqx_docker_image_tags
+        if ! is_ecr_and_enterprise "$r" "$PROFILE"; then
+            DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_IMAGE_TAG}")
+            echo "$r/${EMQX_IMAGE_TAG}" >> ./.emqx_docker_image_tags
+        fi
     done
     if [ "${DOCKER_BUILD_NOCACHE:-false}" = true ]; then
         DOCKER_BUILDX_ARGS+=(--no-cache)
@@ -471,12 +483,14 @@ make_docker() {
     fi
     if [ "${DOCKER_LATEST:-false}" = true ]; then
         for r in "${DOCKER_REGISTRIES[@]}"; do
-          DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_BASE_DOCKER_TAG}:latest${SUFFIX}")
-          echo "$r/${EMQX_BASE_DOCKER_TAG}:latest${SUFFIX}" >> ./.emqx_docker_image_tags
-          DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}")
-          echo "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}" >> ./.emqx_docker_image_tags
-          DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}")
-          echo "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}" >> ./.emqx_docker_image_tags
+            if ! is_ecr_and_enterprise "$r" "$PROFILE"; then
+                DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_BASE_DOCKER_TAG}:latest${SUFFIX}")
+                echo "$r/${EMQX_BASE_DOCKER_TAG}:latest${SUFFIX}" >> ./.emqx_docker_image_tags
+                DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}")
+                echo "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}${SUFFIX}" >> ./.emqx_docker_image_tags
+                DOCKER_BUILDX_ARGS+=(--tag "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}")
+                echo "$r/${EMQX_BASE_DOCKER_TAG}:${VSN_MAJOR}.${VSN_MINOR}.${VSN_PATCH}${SUFFIX}" >> ./.emqx_docker_image_tags
+            fi
         done
     fi
     if [ "${DOCKER_PLATFORMS:-default}" != 'default' ]; then

changes/ce/breaking-12634.en.md

@@ -0,0 +1,10 @@
+Triple-quote string values in HOCON config files no longer support escape sequence.
+
+The detailed information can be found in [this pull request](https://github.com/emqx/hocon/pull/290).
+Here is a summary for the impact on EMQX users:
+
+- EMQX 5.6 is the first version to generate triple-quote strings in `cluster.hocon`,
+  meaning for generated configs, there is no compatibility issue.
+- For user hand-crafted configs (such as `emqx.conf`) a thorough review is needed
+  to inspect if escape sequences are used (such as `\n`, `\r`, `\t` and `\\`), if yes,
+  such strings should be changed to regular quotes (one pair of `"`) instead of triple-quotes.

changes/ce/feat-12517.en.md

@@ -12,4 +12,4 @@ rule_xlu4 {
   ~"""
 }
 ```
-See [HOCON 0.41.0](https://github.com/emqx/hocon/releases/tag/0.41.0) release note for more dtails.
+See [HOCON 0.42.0](https://github.com/emqx/hocon/releases/tag/0.42.0) release note for more dtails.

changes/e5.5.1.en.md

@@ -0,0 +1,29 @@
+# 5.5.1
+
+## Enhancements
+
+- [#12497](https://github.com/emqx/emqx/pull/12497) Improved MongoDB connector performance, resulting in more efficient database interactions. This enhancement is supported by improvements in the MongoDB Erlang driver as well ([mongodb-erlang PR](https://github.com/emqx/mongodb-erlang/pull/41)).
+
+## Bug Fixes
+
+- [#12471](https://github.com/emqx/emqx/pull/12471) Fixed an issue that data integration configurations failed to load correctly during upgrades from EMQX version 5.0.2 to newer releases.
+
+- [#12542](https://github.com/emqx/emqx/pull/12542) Redacted authorization headers to exclude basic authorization credentials from debug logs in the HTTP Server connector, mitigating potential security risks.
+
+- [#12598](https://github.com/emqx/emqx/pull/12598) Fixed an issue that users were unable to subscribe to or unsubscribe from shared topic filters via HTTP API.
+
+  The affected APIs include:
+
+  - `/clients/:clientid/subscribe`
+  - `/clients/:clientid/subscribe/bulk`
+
+  - `/clients/:clientid/unsubscribe`
+  - `/clients/:clientid/unsubscribe/bulk`
+
+- [#12601](https://github.com/emqx/emqx/pull/12601) Fixed an issue where logs of the LDAP driver were not being captured. Now, all logs are recorded at the `info` level.
+
+- [#12606](https://github.com/emqx/emqx/pull/12606) The Prometheus API experienced crashes when the specified SSL certificate file did not exist in the given path. Now, when an SSL certificate file is missing, the `emqx_cert_expiry_at` metric will report a value of 0, indicating the non-existence of the certificate.
+
+- [#12608](https://github.com/emqx/emqx/pull/12608) Fixed a `function_clause` error in the IoTDB action caused by the absence of a `payload` field in query data.
+
+- [#12610](https://github.com/emqx/emqx/pull/12610) Fixed an issue where connections to the LDAP connector could unexpectedly disconnect after a certain period of time.

changes/ee/feat-12619.en.md

@@ -0,0 +1 @@
+The Microsoft SQL Server bridge has been split into connector and action components. Old Microsoft SQL Server bridges will be upgraded automatically.

changes/v5.5.1.en.md

@@ -0,0 +1,21 @@
+# 5.5.1
+
+## Bug Fixes
+
+- [#12471](https://github.com/emqx/emqx/pull/12471) Fixed an issue that data integration configurations failed to load correctly during upgrades from EMQX version 5.0.2 to newer releases.
+
+- [#12542](https://github.com/emqx/emqx/pull/12542) Redacted authorization headers to exclude basic authorization credentials from debug logs in the HTTP Server connector, mitigating potential security risks.
+
+- [#12598](https://github.com/emqx/emqx/pull/12598) Fixed an issue that users were unable to subscribe to or unsubscribe from shared topic filters via HTTP API.
+
+  The affected APIs include:
+
+  - `/clients/:clientid/subscribe`
+  - `/clients/:clientid/subscribe/bulk`
+
+  - `/clients/:clientid/unsubscribe`
+  - `/clients/:clientid/unsubscribe/bulk`
+
+- [#12601](https://github.com/emqx/emqx/pull/12601) Fixed an issue where logs of the LDAP driver were not being captured. Now, all logs are recorded at the `info` level.
+
+- [#12606](https://github.com/emqx/emqx/pull/12606) The Prometheus API experienced crashes when the specified SSL certificate file did not exist in the given path. Now, when an SSL certificate file is missing, the `emqx_cert_expiry_at` metric will report a value of 0, indicating the non-existence of the certificate.

deploy/charts/emqx-enterprise/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.6.0-alpha.1
+version: 5.6.0-alpha.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.6.0-alpha.1
+appVersion: 5.6.0-alpha.2

deploy/charts/emqx/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 5.6.0-alpha.1
+version: 5.6.0-alpha.2
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 5.6.0-alpha.1
+appVersion: 5.6.0-alpha.2

mix.exs

@@ -72,7 +72,7 @@ defmodule EMQXUmbrella.MixProject do
       # in conflict by emqtt and hocon
       {:getopt, "1.0.2", override: true},
       {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true},
-      {:hocon, github: "emqx/hocon", tag: "0.41.0", override: true},
+      {:hocon, github: "emqx/hocon", tag: "0.42.0", override: true},
       {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true},
       {:esasl, github: "emqx/esasl", tag: "0.2.0"},
       {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},

rebar.config

@@ -97,7 +97,7 @@
     {system_monitor, {git, "https://github.com/ieQu1/system_monitor", {tag, "3.0.3"}}},
     {getopt, "1.0.2"},
     {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "1.0.8"}}},
-    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.41.0"}}},
+    {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.42.0"}}},
     {emqx_http_lib, {git, "https://github.com/emqx/emqx_http_lib.git", {tag, "0.5.3"}}},
     {esasl, {git, "https://github.com/emqx/esasl", {tag, "0.2.0"}}},
     {jose, {git, "https://github.com/potatosalad/erlang-jose", {tag, "1.11.2"}}},

rel/i18n/emqx_bridge_sqlserver.hocon

@@ -46,4 +46,22 @@ sql_template.desc:
 sql_template.label:
 """SQL Template"""
 
+action_parameters.desc:
+"""Action specific configuration."""
+
+action_parameters.label:
+"""Action"""
+
+sqlserver_action.desc:
+"""Configuration for Microsoft SOL Server action."""
+
+sqlserver_action.label:
+"""Microsoft SOL Server Action Configuration"""
+
+config_connector.desc:
+"""Configuration for a Microsoft SOL Server connector."""
+
+config_connector.label:
+"""Microsoft SOL Server Connector Configuration"""
+
 }