Merge branch 'main-v4.3' into fix-ctl-print-43

.github/workflows/build_packages.yaml

@@ -338,6 +338,11 @@ jobs:
     strategy:
       matrix:
         profile: ${{fromJSON(needs.prepare.outputs.profiles)}}
+        registry:
+          - 'docker.io'
+        include:
+          - profile: emqx
+            registry: 'public.ecr.aws'
 
     steps:
     - uses: actions/download-artifact@v2
@@ -351,10 +356,24 @@ jobs:
       with:
         image: tonistiigi/binfmt:latest
         platforms: all
+    - uses: aws-actions/configure-aws-credentials@v1
+      if: matrix.repository == 'public.ecr.aws'
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
+    - name: Docker login to aws ecr
+      if: matrix.repository == 'public.ecr.aws'
+      run: aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
+    - uses: docker/login-action@v1
+      if: matrix.repository == 'docker.io'
+      with:
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_TOKEN }}
     - uses: docker/metadata-action@v3
       id: meta
       with:
-        images: ${{ github.repository_owner }}/${{ matrix.profile }}
+        images: ${{ matrix.registry }}/${{ github.repository_owner }}/${{ matrix.profile }}
         flavor: |
           latest=${{ !github.event.release.prerelease }}
         tags: |
@@ -363,11 +382,6 @@ jobs:
           type=match,pattern=[v|e](.*),group=1
         labels:
           org.opencontainers.image.otp.version=${{ matrix.otp }}
-    - uses: docker/login-action@v1
-      if: github.event_name == 'release'
-      with:
-        username: ${{ secrets.DOCKER_HUB_USER }}
-        password: ${{ secrets.DOCKER_HUB_TOKEN }}
     - uses: docker/build-push-action@v2
       if: matrix.profile != 'emqx-ee'
       with:
@@ -398,20 +412,6 @@ jobs:
           EMQX_NAME=${{ matrix.profile }}
         file: source/deploy/docker/Dockerfile.enterprise
         context: source
-    - uses: aws-actions/configure-aws-credentials@v1
-      if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx'
-      with:
-        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
-        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-        aws-region: ${{ secrets.AWS_DEFAULT_REGION }}
-    - name: Push image to aws ecr
-      if: github.event_name == 'release' && !github.event.release.prerelease && matrix.profile == 'emqx'
-      run: |
-        version=${GITHUB_REF##*/}
-        docker pull emqx/emqx:${version#v}
-        docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v}
-        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
-        docker push public.ecr.aws/emqx/emqx:${version#v}
 
   delete-artifact:
     runs-on: ubuntu-20.04
@@ -491,13 +491,6 @@ jobs:
              -X POST \
              -d "{\"repo\":\"emqx/emqx\", \"tag\": \"${{ env.version }}\" }" \
              ${{ secrets.EMQX_IO_RELEASE_API }}
-    - name: push docker image to aws ecr
-      if: github.event_name == 'release' && matrix.profile == 'emqx'
-      run: |
-        set -e -x -u
-        aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws
-        docker tag emqx/emqx:${version#v} public.ecr.aws/emqx/emqx:${version#v}
-        docker push public.ecr.aws/emqx/emqx:${version#v}
     - name: update repo.emqx.io
       if: github.event_name == 'release' && endsWith(github.repository, 'enterprise') && matrix.profile == 'emqx-ee'
       run: |

CHANGES-4.3.md

@@ -17,6 +17,10 @@ File format:
 * Fix updating `emqx_auth_mnesia.conf` password and restarting the new password does not take effect [#6717]
 * Fix import data crash when emqx_auth_mnesia's record is not empty [#6717]
 * Fix `os_mon.sysmem_high_watermark` may not alert after reboot.
+* Enhancement: Log client status before killing it for holding the lock for too long.
+  [emqx-6959](https://github.com/emqx/emqx/pull/6959)
+  [ekka-144](https://github.com/emqx/ekka/pull/144)
+  [ekka-146](https://github.com/emqx/ekka/pull/146)
 
 ## v4.3.11
 
@@ -46,7 +50,7 @@ Important notes:
 * Clustering malfunction fixes [#6221, #6381]
   Mostly changes made in [ekka](https://github.com/emqx/ekka/pull/134)<br>
   From 0.8.1.4 to 0.8.1.6, fixes included intra-cluster RPC call timeouts,<br>
-  also fixed `ekka_locker` process crashe after killing a hanged lock owner.
+  also fixed `ekka_locker` process crashed after killing a hanged lock owner.
 
 * Improved log message when TCP proxy is in use but proxy_protocol configuration is not turned on [#6416]<br>
   "please check proxy_protocol config for specific listeners and zones" to hint a misconfiguration
@@ -101,6 +105,8 @@ Important notes:
   properties such as protocol name, protocol version, username (if any) peer-host<br>
   etc. are filled as MQTT message headers.
 
+* Format the message id to hex strings in the log message [#6961]
+
 ## v4.3.0~10
 
 Older version changes are not tracked here.

Makefile

@@ -7,7 +7,7 @@ export EMQX_DEFAULT_BUILDER = emqx/build-env:erl23.2.7.2-emqx-3-alpine
 export EMQX_DEFAULT_RUNNER = alpine:3.12
 export PKG_VSN ?= $(shell $(CURDIR)/pkg-vsn.sh)
 export EMQX_DESC ?= EMQ X
-export EMQX_CE_DASHBOARD_VERSION ?= v4.3.4
+export EMQX_CE_DASHBOARD_VERSION ?= v4.3.5
 export DOCKERFILE := deploy/docker/Dockerfile
 ifeq ($(OS),Windows_NT)
 	export REBAR_COLOR=none

apps/emqx_sn/test/emqx_sn_protocol_SUITE.erl

@@ -170,7 +170,6 @@ t_subscribe_case02(_) ->
     ReturnCode = 0,
     {ok, Socket} = gen_udp:open(0, [binary]),
 
-    ClientId = ?CLIENTID,
     send_connect_msg(Socket, ?CLIENTID),
     ?assertEqual(<<3, ?SN_CONNACK, 0>>, receive_response(Socket)),
 

apps/emqx_stomp/src/emqx_stomp_frame.erl

@@ -123,6 +123,8 @@ parse(<<>>, Parser) ->
 
 parse(Bytes, #{phase := body, length := Len, state := State}) ->
     parse(body, Bytes, State, Len);
+parse(<<?LF, Bytes/binary>>, #{phase := hdname, state := State}) ->
+    parse(body, Bytes, State, content_len(State));
 parse(Bytes, #{phase := Phase, state := State}) when Phase =/= none ->
     parse(Phase, Bytes, State);
 

apps/emqx_stomp/test/emqx_stomp_SUITE.erl

@@ -359,6 +359,35 @@ t_1000_msg_send(_) ->
         lists:foreach(fun(_) -> RecvFun() end, lists:seq(1, 1000))
     end).
 
+t_sticky_packets_truncate_after_headers(_) ->
+    with_connection(fun(Sock) ->
+        gen_tcp:send(Sock, serialize(<<"CONNECT">>,
+                                     [{<<"accept-version">>, ?STOMP_VER},
+                                      {<<"host">>, <<"127.0.0.1:61613">>},
+                                      {<<"login">>, <<"guest">>},
+                                      {<<"passcode">>, <<"guest">>},
+                                      {<<"heart-beat">>, <<"0,0">>}])),
+        {ok, Data} = gen_tcp:recv(Sock, 0),
+        {ok, #stomp_frame{command = <<"CONNECTED">>,
+                          headers = _,
+                          body    = _}, _} = parse(Data),
+
+        Topic = <<"/queue/foo">>,
+
+        emqx:subscribe(Topic),
+        gen_tcp:send(Sock, ["SEND\n",
+                            "content-length:3\n",
+                            "destination:/queue/foo\n"]),
+        timer:sleep(300),
+        gen_tcp:send(Sock, ["\nfoo",0]),
+        receive
+            {deliver, Topic, _Msg}->
+                ok
+        after 100 ->
+                  ?assert(false, "waiting message timeout")
+        end
+    end).
+
 with_connection(DoFun) ->
     {ok, Sock} = gen_tcp:connect({127, 0, 0, 1},
                                  61613,

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.3.11"}).
+-define(EMQX_RELEASE, {opensource, "4.3.12"}).
 
 -else.
 

rebar.config

@@ -43,7 +43,7 @@
     , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
     , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.4"}}}
-    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.7"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.1.8"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
     , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.3.6"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.7"}}}

src/emqx.appup.src

@@ -12,6 +12,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -30,6 +31,7 @@
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -48,6 +50,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -71,6 +74,7 @@
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -94,6 +98,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -119,6 +124,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -144,6 +150,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -170,6 +177,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -197,6 +205,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -225,6 +234,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -253,6 +263,7 @@
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -285,6 +296,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -320,6 +332,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}],
   [{"4.3.12",
@@ -333,6 +346,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.11",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -350,6 +364,7 @@
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
      {load_module,emqx_http_lib,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.10",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -367,6 +382,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_connection,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.9",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -389,6 +405,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.8",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -411,6 +428,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.7",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -435,6 +453,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.6",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -459,6 +478,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.5",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -484,6 +504,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.4",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -510,6 +531,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.3",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -537,6 +559,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.2",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -564,6 +587,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.1",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -595,6 +619,7 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {"4.3.0",
     [{load_module,emqx_vm_mon,brutal_purge,soft_purge,[]},
@@ -628,5 +653,6 @@
      {load_module,emqx_os_mon,brutal_purge,soft_purge,[]},
      {load_module,emqx,brutal_purge,soft_purge,[]},
      {load_module,emqx_app,brutal_purge,soft_purge,[]},
+     {load_module,emqx_message,brutal_purge,soft_purge,[]},
      {load_module,emqx_limiter,brutal_purge,soft_purge,[]}]},
    {<<".*">>,[]}]}.

src/emqx_message.erl

@@ -342,7 +342,13 @@ format(#message{id = Id,
                 flags = Flags,
                 headers = Headers}) ->
     io_lib:format("Message(Id=~s, QoS=~w, Topic=~s, From=~p, Flags=~s, Headers=~s)",
-                  [Id, QoS, Topic, From, format(flags, Flags), format(headers, Headers)]).
+                  [printable_msg_id(Id), QoS, Topic, From, format(flags, Flags),
+                   format(headers, Headers)]).
+
+printable_msg_id(undefined) ->
+    <<>>;
+printable_msg_id(Id) ->
+    emqx_guid:to_hexstr(Id).
 
 format(flags, Flags) ->
     io_lib:format("~p", [[Flag || {Flag, true} <- maps:to_list(Flags)]]);