Merge pull request #4947 from emqx/dev/v5.0

Merge dev/v5.0 to master

.ci/docker-compose-file/conf.env

@@ -11,3 +11,4 @@ EMQX_AUTH__PGSQL__DATABASE=mqtt
 EMQX_AUTH__REDIS__SERVER=redis_server:6379
 EMQX_AUTH__REDIS__PASSWORD=public
 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+HOCON_ENV_OVERRIDE_PREFIX=EMQX_

.ci/docker-compose-file/docker-compose-emqx-cluster.yaml

@@ -9,7 +9,7 @@ services:
       - emqx2
     volumes:
       - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
-      - ../../etc/certs:/usr/local/etc/haproxy/certs
+      - ../../apps/emqx/etc/certs:/usr/local/etc/haproxy/certs
     ports:
       - "18083:18083"
 #     - "1883:1883"

.github/workflows/run_cts_tests.yaml

@@ -48,7 +48,7 @@ jobs:
           export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap"
+          docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_ldap"
           docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
       - uses: actions/upload-artifact@v1
         if: failure()
@@ -86,7 +86,7 @@ jobs:
         if: matrix.connect_type == 'tls'
         run: |
           cat <<-EOF >> "$GITHUB_ENV"
-          EMQX_AUTH__MONGO__SSL=on
+          EMQX_AUTH__MONGO__SSL__ENABLE=on
           EMQX_AUTH__MONGO__SSL__CACERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem
           EMQX_AUTH__MONGO__SSL__CERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem
           EMQX_AUTH__MONGO__SSL__KEYFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem
@@ -98,7 +98,7 @@ jobs:
           MONGO_TAG: ${{ matrix.mongo_tag }}
         if: matrix.connect_type == 'tcp'
         run: |
-          echo EMQX_AUTH__MONGO__SSL=off >> "$GITHUB_ENV"
+          echo EMQX_AUTH__MONGO__SSL__ENABLE=off >> "$GITHUB_ENV"
       - name: setup
         if: matrix.network_type == 'ipv4'
         run: |
@@ -117,7 +117,7 @@ jobs:
           export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo"
+          docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_mongo"
           docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo"
       - uses: actions/upload-artifact@v1
         if: failure()
@@ -160,10 +160,10 @@ jobs:
         if: matrix.connect_type == 'tls'
         run: |
           cat <<-EOF >> "$GITHUB_ENV"
+            EMQX_AUTH__MYSQL__SSL__ENABLE=on
             EMQX_AUTH__MYSQL__USERNAME=ssluser
             EMQX_AUTH__MYSQL__PASSWORD=public
             EMQX_AUTH__MYSQL__DATABASE=mqtt
-            EMQX_AUTH__MYSQL__SSL=on
             EMQX_AUTH__MYSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem
             EMQX_AUTH__MYSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem
             EMQX_AUTH__MYSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem
@@ -179,7 +179,7 @@ jobs:
             EMQX_AUTH__MYSQL__USERNAME=root
             EMQX_AUTH__MYSQL__PASSWORD=public
             EMQX_AUTH__MYSQL__DATABASE=mqtt
-            EMQX_AUTH__MYSQL__SSL=off
+            EMQX_AUTH__MYSQL__SSL__ENABLE=off
           EOF
       - name: setup
         if: matrix.network_type == 'ipv4'
@@ -199,7 +199,7 @@ jobs:
           export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql"
+          docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_mysql"
           docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql"
       - uses: actions/upload-artifact@v1
         if: failure()
@@ -239,7 +239,7 @@ jobs:
         if: matrix.connect_type == 'tls'
         run: |
           cat <<-EOF >> "$GITHUB_ENV"
-          EMQX_AUTH__PGSQL__SSL=on
+          EMQX_AUTH__PGSQL__SSL__ENABLE=on
           EMQX_AUTH__PGSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/ca.pem
           EMQX_AUTH__PGSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-cert.pem
           EMQX_AUTH__PGSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/client-key.pem
@@ -251,7 +251,7 @@ jobs:
           PGSQL_TAG: ${{ matrix.pgsql_tag }}
         if: matrix.connect_type == 'tcp'
         run: |
-          echo EMQX_AUTH__PGSQL__SSL=off >> "$GITHUB_ENV"
+          echo EMQX_AUTH__PGSQL__SSL__ENABLE=off >> "$GITHUB_ENV"
       - name: setup
         if: matrix.network_type == 'ipv4'
         run: |
@@ -273,7 +273,7 @@ jobs:
                  CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
           printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql"
+          docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_pgsql"
           docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql"
       - uses: actions/upload-artifact@v1
         if: failure()
@@ -318,7 +318,7 @@ jobs:
         if: matrix.connect_type == 'tls'
         run: |
           cat <<-EOF >> "$GITHUB_ENV"
-          EMQX_AUTH__REDIS__SSL=on
+          EMQX_AUTH__REDIS__SSL__ENABLE=on
           EMQX_AUTH__REDIS__SSL__CACERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt
           EMQX_AUTH__REDIS__SSL__CERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt
           EMQX_AUTH__REDIS__SSL__KEYFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key
@@ -330,7 +330,7 @@ jobs:
           REDIS_TAG: ${{ matrix.redis_tag }}
         if: matrix.connect_type == 'tcp'
         run: |
-          echo EMQX_AUTH__REDIS__SSL=off >> "$GITHUB_ENV"
+          echo EMQX_AUTH__REDIS__SSL__ENABLE=off >> "$GITHUB_ENV"
       - name: get server address
         run: |
           ipv4_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' redis)
@@ -394,7 +394,7 @@ jobs:
           export EMQX_AUTH__REIDS__PASSWORD=public
           printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
-          docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis"
+          docker exec -i erlang sh -c "./rebar3 eunit --application=emqx_auth_redis"
           docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis"
       - uses: actions/upload-artifact@v1
         if: failure()

.github/workflows/run_emqx_app_tests.yaml

@@ -0,0 +1,26 @@
+name: Check emqx app standalone
+
+on:
+  push:
+    tags:
+      - v*
+      - e*
+  pull_request:
+
+jobs:
+    check_all:
+        runs-on: ubuntu-20.04
+        container: emqx/build-env:erl23.2.7.2-emqx-2-ubuntu20.04
+
+        steps:
+        - uses: actions/checkout@v2
+        - name: run
+          run: |
+            make ensure-rebar3
+            cp rebar3 apps/emqx/
+            cd apps/emqx
+            ./rebar3 xref
+            ./rebar3 dialyzer
+            ./rebar3 eunit -v
+            ./rebar3 ct -v
+            ./rebar3 proper -d test/props

include/emqx_release.hrl

@@ -29,7 +29,7 @@
 
 -ifndef(EMQX_ENTERPRISE).
 
--define(EMQX_RELEASE, {opensource, "4.3.3"}).
+-define(EMQX_RELEASE, {opensource, "5.0-pre"}).
 
 -else.
 

apps/emqx/rebar.config

@@ -0,0 +1,47 @@
+{erl_opts, [warn_unused_vars,warn_shadow_vars,warn_unused_import,
+            warn_obsolete_guard,compressed]}.
+
+{xref_checks,[undefined_function_calls,undefined_functions,locals_not_used,
+              deprecated_function_calls,warnings_as_errors,deprecated_functions]}.
+
+%% Deps here may duplicate with emqx.git root level rebar.config
+%% but there not be any descrpancy.
+%% This rebar.config is necessary because the app may be used as a
+%% `git_subdir` dependency in other projects.
+{deps,
+    [ {gproc, {git, "https://github.com/uwiger/gproc", {tag, "0.8.0"}}}
+    , {jiffy, {git, "https://github.com/emqx/jiffy", {tag, "1.0.5"}}}
+    , {cowboy, {git, "https://github.com/emqx/cowboy", {tag, "2.8.2"}}}
+    , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}}
+    , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.9.0"}}}
+    , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.1"}}}
+    , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v4.0.1"}}} %% todo delete when plugins use hocon
+    , {hocon, {git, "https://github.com/emqx/hocon.git", {tag, "0.5.0"}}}
+    , {pbkdf2, {git, "https://github.com/emqx/erlang-pbkdf2.git", {branch, "2.0.4"}}}
+    , {recon, {git, "https://github.com/ferd/recon", {tag, "2.5.1"}}}
+    , {snabbkaffe, {git, "https://github.com/kafka4beam/snabbkaffe.git", {tag, "0.13.0"}}}
+    ]}.
+
+{plugins, [rebar3_proper]}.
+{extra_src_dirs, [{"etc", [recursive]}]}.
+{profiles, [
+   {test,
+       [{deps,
+           [ meck
+           , {bbmustache,"1.10.0"}
+           , {emqx_ct_helpers, {git,"https://github.com/zmstone/emqx-ct-helpers", {branch,"hocon"}}}
+           , {emqtt, {git, "https://github.com/emqx/emqtt", {tag, "1.2.3.1"}}}
+           ]},
+         {extra_src_dirs, [{"test",[recursive]}]}
+       ]}
+]}.
+
+{dialyzer, [
+    {warnings, [unmatched_returns, error_handling, race_conditions]},
+    {plt_location, "."},
+    {plt_prefix, "emqx_dialyzer"},
+    {plt_apps, all_apps},
+    {plt_extra_apps, [hocon]},
+    {statistics, true}
+   ]
+}.

apps/emqx/rebar.config.script

@@ -0,0 +1,11 @@
+Bcrypt = {bcrypt, {git, "https://github.com/emqx/erlang-bcrypt.git", {branch, "0.6.0"}}},
+AddBcrypt = fun(C) ->
+    {deps, Deps0} = lists:keyfind(deps, 1, C),
+    Deps = [Bcrypt | Deps0],
+    lists:keystore(deps, 1, C, {deps, Deps})
+end,
+
+case os:type() of
+    {win32, _} -> CONFIG;
+    _ -> AddBcrypt(CONFIG)
+end.

src/emqx.app.src

@@ -1,7 +1,7 @@
 {application, emqx,
  [{id, "emqx"},
   {description, "EMQ X"},
-  {vsn, "4.3.3"}, % strict semver, bump manually!
+  {vsn, "5.0.0"}, % strict semver, bump manually!
   {modules, []},
   {registered, []},
   {applications, [kernel,stdlib,gproc,gen_rpc,esockd,cowboy,sasl,os_mon]},

src/emqx_logger_jsonfmt.erl

@@ -32,6 +32,9 @@
 -export([format/2]).
 
 -ifdef(TEST).
+-include_lib("proper/include/proper.hrl").
+-include_lib("eunit/include/eunit.hrl").
+
 -export([report_cb_1/1, report_cb_2/2, report_cb_crash/2]).
 -endif.
 
@@ -220,8 +223,6 @@ json_key(Term) ->
     end.
 
 -ifdef(TEST).
--include_lib("proper/include/proper.hrl").
--include_lib("eunit/include/eunit.hrl").
 
 no_crash_test_() ->
     Opts = [{numtests, 1000}, {to_file, user}],