Merge pull request #4195 from z8674558/env-override

feat(bin): let cuttlefish read env var

.ci/build_packages/tests.sh

@@ -23,8 +23,8 @@ emqx_test(){
             "zip")
                 packagename=$(basename "${PACKAGE_PATH}/${EMQX_NAME}"-*.zip)
                 unzip -q "${PACKAGE_PATH}/${packagename}"
-                sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" "${PACKAGE_PATH}"/emqx/etc/emqx.conf
-                sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" "${PACKAGE_PATH}"/emqx/etc/emqx.conf
+                export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
+                       EMQX_MQTT__MAX_TOPIC_ALIAS=10
                 sed -i '/emqx_telemetry/d' "${PACKAGE_PATH}"/emqx/data/loaded_plugins
 
                 echo "running ${packagename} start"
@@ -96,8 +96,8 @@ emqx_test(){
 }
 
 running_test(){
-    sed -i "/zone.external.server_keepalive/c zone.external.server_keepalive = 60" /etc/emqx/emqx.conf
-    sed -i "/mqtt.max_topic_alias/c mqtt.max_topic_alias = 10" /etc/emqx/emqx.conf
+    export EMQX_ZONE__EXTERNAL__SERVER__KEEPALIVE=60 \
+           EMQX_MQTT__MAX_TOPIC_ALIAS=10
     sed -i '/emqx_telemetry/d' /var/lib/emqx/loaded_plugins
 
     emqx start || tail /var/log/emqx/erlang.log.1

.github/workflows/run_cts_tests.yaml

@@ -36,18 +36,18 @@ jobs:
       - name: setup
         if: matrix.network_type == 'ipv4'
         run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap)
-          sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = $server_address|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf
+          echo EMQX_AUTH__LDAP__SERVERS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ldap) >> "$GITHUB_ENV"
       - name: setup
         if: matrix.network_type == 'ipv6'
         run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap)
-          sed -i "s|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = $server_address|g" apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf
+          echo EMQX_AUTH__LDAP__SERVERS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' ldap) >> "$GITHUB_ENV"
       - name: run test cases
         run: |
+          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
           docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_ldap"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_ldap"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_ldap"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -78,30 +78,34 @@ jobs:
         if: matrix.connect_type == 'tls'
         run: |
           docker-compose -f .ci/compatibility_tests/docker-compose-mongo-tls.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.mongo.ssl[[:space:]]*=.*|auth.mongo.ssl = on|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
-          sed -i 's|^[#[:space:]]*auth.mongo.cacertfile[[:space:]]*=.*|auth.mongo.cacertfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
-          sed -i 's|^[#[:space:]]*auth.mongo.certfile[[:space:]]*=.*|auth.mongo.certfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
-          sed -i 's|^[#[:space:]]*auth.mongo.keyfile[[:space:]]*=.*|auth.mongo.keyfile = /emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__MONGO__SSL=on
+          EMQX_AUTH__MONGO__CACERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/ca.pem
+          EMQX_AUTH__MONGO__CERTFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-cert.pem
+          EMQX_AUTH__MONGO__KEYFILE=/emqx/apps/emqx_auth_mongo/test/emqx_auth_mongo_SUITE_data/client-key.pem
+          EOF
       - name: setup
         env:
           MONGO_TAG: ${{ matrix.mongo_tag }}
         if: matrix.connect_type == 'tcp'
-        run: docker-compose -f .ci/compatibility_tests/docker-compose-mongo.yaml up -d
+        run: |
+          docker-compose -f .ci/compatibility_tests/docker-compose-mongo.yaml up -d
+          echo EMQX_AUTH__MONGO__SSL=off >> "$GITHUB_ENV"
       - name: setup
         if: matrix.network_type == 'ipv4'
         run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo)
-          sed -i "s|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = $server_address:27017|g" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
+          echo "EMQX_AUTH__MONGO__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mongo):27017" >> "$GITHUB_ENV"
       - name: setup
         if: matrix.network_type == 'ipv6'
         run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo)
-          sed -i "s|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = $server_address:27017|g" apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
+          echo "EMQX_AUTH__MONGO__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mongo):27017" >> "$GITHUB_ENV"
       - name: run test cases
         run: |
+          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
           docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mongo"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_mongo"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mongo"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -132,35 +136,37 @@ jobs:
         if: matrix.connect_type == 'tls'
         run: |
           docker-compose -f .ci/compatibility_tests/docker-compose-mysql-tls.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.mysql.ssl[[:space:]]*=.*|auth.mysql.ssl = on|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.ssl.cacertfile[[:space:]]*=.*|auth.mysql.ssl.cacertfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.ssl.certfile[[:space:]]*=.*|auth.mysql.ssl.certfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.ssl.keyfile[[:space:]]*=.*|auth.mysql.ssl.keyfile = /emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+            EMQX_AUTH__MYSQL__SSL=on
+            EMQX_AUTH__MYSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/ca.pem
+            EMQX_AUTH__MYSQL__SSL__CERTFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-cert.pem
+            EMQX_AUTH__MYSQL__SSL__KEYFILE=/emqx/apps/emqx_auth_mysql/test/emqx_auth_mysql_SUITE_data/client-key.pem
+          EOF
       - name: setup
         env:
           MYSQL_TAG: ${{ matrix.mysql_tag }}
         if: matrix.connect_type == 'tcp'
-        run: docker-compose -f .ci/compatibility_tests/docker-compose-mysql.yaml up -d
+        run: |
+          docker-compose -f .ci/compatibility_tests/docker-compose-mysql.yaml up -d
+          echo EMQX_AUTH__MYSQL__SSL=off >> "$GITHUB_ENV"
       - name: setup
         if: matrix.network_type == 'ipv4'
         run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql)
-          sed -i "/auth.mysql.server/c auth.mysql.server = $server_address:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
+          echo "EMQX_AUTH__MYSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' mysql):3306" >> "$GITHUB_ENV"
       - name: setup
         if: matrix.network_type == 'ipv6'
         run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql)
-          sed -i "/auth.mysql.server/c auth.mysql.server = $server_address:3306" apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-      - name: setup
-        run: |
-          sed -i 's|^[#[:space:]]*auth.mysql.username[[:space:]]*=.*|auth.mysql.username = root|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.password[[:space:]]*=.*|auth.mysql.password = public|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-          sed -i 's|^[#[:space:]]*auth.mysql.database[[:space:]]*=.*|auth.mysql.database = mqtt|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
+          echo "EMQX_AUTH__MYSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' mysql):3306" >> "$GITHUB_ENV"
       - name: run test cases
         run: |
+          export EMQX_AUTH__MYSQL__USERNAME=root \
+                 EMQX_AUTH__MYSQL__PASSWORD=public \
+                 EMQX_AUTH__MYSQL__DATABASE=mqtt \
+                 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
           docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_mysql"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_mysql"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_mysql"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -194,35 +200,35 @@ jobs:
         run: |
           docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml build --no-cache
           docker-compose -f .ci/compatibility_tests/docker-compose-pgsql-tls.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.ssl[ \t]*=.*|auth.pgsql.ssl = on|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.cacertfile[ \t]*=.*|auth.pgsql.cacertfile = /emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__PGSQL__SSL=on
+          EMQX_AUTH__PGSQL__SSL__CACERTFILE=/emqx/apps/emqx_auth_pgsql/test/emqx_auth_pgsql_SUITE_data/root.crt
+          EOF
       - name: setup
         env:
           PGSQL_TAG: ${{ matrix.pgsql_tag }}
         if: matrix.connect_type == 'tcp'
         run: |
           docker-compose -f .ci/compatibility_tests/docker-compose-pgsql.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.pgsql.username[ \t]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.password[ \t]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-          sed -i 's|^[#[:space:]]*auth.pgsql.database[ \t]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
+          echo EMQX_AUTH__PGSQL__SSL=off >> "$GITHUB_ENV"
       - name: setup
         if: matrix.network_type == 'ipv4'
         run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql)
-          sed -i "s|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = $server_address:5432|g" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
+          echo "EMQX_AUTH__PGSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' pgsql):5432" >> "$GITHUB_ENV"
       - name: setup
         if: matrix.network_type == 'ipv6'
         run: |
-          server_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql)
-          sed -i "s|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = $server_address:5432|g" apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
+          echo "EMQX_AUTH__PGSQL__SERVER=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' pgsql):5432" >> "$GITHUB_ENV"
       - name: run test cases
         run: |
+          export EMQX_AUTH__PGSQL__USERNAME=root \
+                 EMQX_AUTH__PGSQL__PASSWORD=public \
+                 EMQX_AUTH__PGSQL__DATABASE=mqtt \
+                 CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
           docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_pgsql"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_pgsql"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_pgsql"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:
@@ -257,51 +263,69 @@ jobs:
         run: |
           set -exu
           docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}-tls.yaml up -d
-          sed -i 's|^[#[:space:]]*auth.redis.ssl[[:space:]]*=.*|auth.redis.ssl = on|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i 's|^[#[:space:]]*auth.redis.ssl.cacertfile[[:space:]]*=.*|auth.redis.ssl.cacertfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i 's|^[#[:space:]]*auth.redis.ssl.certfile[[:space:]]*=.*|auth.redis.ssl.certfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i 's|^[#[:space:]]*auth.redis.ssl.keyfile[[:space:]]*=.*|auth.redis.ssl.keyfile = /emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__SSL=on
+          EMQX_AUTH__REDIS__SSL__CACERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/ca.crt
+          EMQX_AUTH__REDIS__SSL__CERTFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.crt
+          EMQX_AUTH__REDIS__SSL__KEYFILE=/emqx/apps/emqx_auth_redis/test/emqx_auth_redis_SUITE_data/certs/redis.key
+          EOF
       - name: setup
         env:
           REDIS_TAG: ${{ matrix.redis_tag }}
         if: matrix.connect_type == 'tcp'
-        run: docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}.yaml up -d
+        run: |
+          docker-compose -f .ci/compatibility_tests/docker-compose-redis-${{ matrix.node_type }}.yaml up -d
+          echo EMQX_AUTH__REDIS__SSL=off >> "$GITHUB_ENV"
       - name: get server address
         if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5')
         run: |
           set -exu
           ipv4_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' redis)
           ipv6_address=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.GlobalIPv6Address}}{{end}}' redis)
-          echo "redis_ipv4_address=$ipv4_address" >> $GITHUB_ENV
-          echo "redis_ipv6_address=$ipv6_address" >> $GITHUB_ENV
+          cat <<-EOF >> "$GITHUB_ENV"
+          redis_ipv4_address=$ipv4_address
+          redis_ipv6_address=$ipv6_address
+          EOF
       - name: setup
         if: matrix.node_type == 'single' && matrix.connect_type == 'tcp'
         run: |
           set -exu
-          sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6379|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__TYPE=single
+          EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6379
+          EOF
       - name: setup
         if: matrix.node_type == 'single' && matrix.connect_type == 'tls' && matrix.redis_tag != '5'
         run: |
           set -exu
-          sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:6380|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__TYPE=single
+          EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:6380
+          EOF
       - name: setup
         if: matrix.node_type == 'cluster' && matrix.connect_type == 'tcp'
         run: |
           set -exu
-          sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:7000, ${redis_${{ matrix.network_type }}_address}:7001, ${redis_${{ matrix.network_type }}_address}:7002|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__TYPE=cluster
+          EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:7000
+          EOF
       - name: setup
         if: matrix.node_type == 'cluster' && matrix.connect_type == 'tls' && matrix.redis_tag != '5'
         run: |
           set -exu
-          sed -i 's|^[#[:space:]]*auth.redis.type[[:space:]]*=.*|auth.redis.type = cluster|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-          sed -i "s|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = ${redis_${{ matrix.network_type }}_address}:8000, ${redis_${{ matrix.network_type }}_address}:8001, ${redis_${{ matrix.network_type }}_address}:8002|g" apps/emqx_auth_redis/etc/emqx_auth_redis.conf
+          cat <<-EOF >> "$GITHUB_ENV"
+          EMQX_AUTH__REDIS__TYPE=cluster
+          EMQX_AUTH__REDIS__SERVER=${redis_${{ matrix.network_type }}_address}:8000
+          EOF
       - name: run test cases
         if: matrix.connect_type == 'tcp' || (matrix.connect_type == 'tls' && matrix.redis_tag != '5')
         run: |
+          export CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+          printenv > .env
           docker exec -i erlang sh -c "make ensure-rebar3"
           docker exec -i erlang sh -c "./rebar3 eunit --dir apps/emqx_auth_redis"
-          docker exec -i erlang sh -c "./rebar3 ct    --dir apps/emqx_auth_redis"
+          docker exec --env-file .env -i erlang sh -c "./rebar3 ct --dir apps/emqx_auth_redis"
       - uses: actions/upload-artifact@v1
         if: failure()
         with:

.github/workflows/run_test_cases.yaml

@@ -29,25 +29,24 @@ jobs:
           run: |
             docker-compose -f .ci/apps_tests/docker-compose.yaml build --no-cache
             docker-compose -f .ci/apps_tests/docker-compose.yaml up -d
-        - name: set config files
-          run: |
-            sed -i 's|^[#[:space:]]*auth.ldap.servers[[:space:]]*=.*|auth.ldap.servers = ldap_server|g' apps/emqx_auth_ldap/etc/emqx_auth_ldap.conf
-            sed -i 's|^[#[:space:]]*auth.mongo.server[[:space:]]*=.*|auth.mongo.server = mongo_server:27017|g' apps/emqx_auth_mongo/etc/emqx_auth_mongo.conf
-            sed -i 's|^[#[:space:]]*auth.redis.server[[:space:]]*=.*|auth.redis.server = redis_server:6379|g' apps/emqx_auth_redis/etc/emqx_auth_redis.conf
-
-            sed -i 's|^[#[:space:]]*auth.mysql.server[[:space:]]*=.*|auth.mysql.server = mysql_server:3306|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-            sed -i 's|^[#[:space:]]*auth.mysql.username[[:space:]]*=.*|auth.mysql.username = root|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-            sed -i 's|^[#[:space:]]*auth.mysql.password[[:space:]]*=.*|auth.mysql.password = public|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
-            sed -i 's|^[#[:space:]]*auth.mysql.database[[:space:]]*=.*|auth.mysql.database = mqtt|g' apps/emqx_auth_mysql/etc/emqx_auth_mysql.conf
 
-            sed -i 's|^[#[:space:]]*auth.pgsql.server[[:space:]]*=.*|auth.pgsql.server = pgsql_server:5432|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-            sed -i 's|^[#[:space:]]*auth.pgsql.username[[:space:]]*=.*|auth.pgsql.username = root|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-            sed -i 's|^[#[:space:]]*auth.pgsql.password[[:space:]]*=.*|auth.pgsql.password = public|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
-            sed -i 's|^[#[:space:]]*auth.pgsql.database[[:space:]]*=.*|auth.pgsql.database = mqtt|g' apps/emqx_auth_pgsql/etc/emqx_auth_pgsql.conf
         - name: run tests
           run: |
+            export EMQX_AUTH__LDAP__SERVERS=ldap_server \
+                   EMQX_AUTH__MONGO__SERVER=mongo_server:27017 \
+                   EMQX_AUTH__REDIS__SERVER=redis_server:6379 \
+                   EMQX_AUTH__MYSQL__SERVER=mysql_server:3306 \
+                   EMQX_AUTH__MYSQL__USERNAME=root \
+                   EMQX_AUTH__MYSQL__PASSWORD=public \
+                   EMQX_AUTH__MYSQL__DATABASE=mqtt \
+                   EMQX_AUTH__PGSQL__SERVER=pgsql_server:5432 \
+                   EMQX_AUTH__PGSQL__USERNAME=root \
+                   EMQX_AUTH__PGSQL__PASSWORD=public \
+                   EMQX_AUTH__PGSQL__DATABASE=mqtt \
+                   CUTTLEFISH_ENV_OVERRIDE_PREFIX=EMQX_
+            printenv > .env
             docker exec -i erlang bash -c "make xref"
-            docker exec -i erlang bash -c "make ct"
+            docker exec --env-file .env -i erlang bash -c "make ct"
             docker exec -i erlang bash -c "make cover"
             docker exec -i erlang bash -c "make coveralls"
         - uses: actions/upload-artifact@v1

bin/emqx

@@ -20,6 +20,9 @@ mkdir -p "$RUNNER_LOG_DIR"
 # Make sure data directory exists
 mkdir -p "$RUNNER_DATA_DIR"
 
+# cuttlefish try to read environment variables starting with "EMQX_", if not specified
+export CUTTLEFISH_ENV_OVERRIDE_PREFIX="${CUTTLEFISH_ENV_OVERRIDE_PREFIX:-EMQX_}"
+
 relx_usage() {
     command="$1"
 

deploy/docker/README.md

@@ -41,11 +41,9 @@ The emqx broker runs as linux user `emqx` in the docker container.
 
 Use the environment variable to configure the EMQ X docker container.
 
-The environment variables which with ``EMQX_`` prefix are mapped to configuration fils.
+By default, the environment variables with ``EMQX_`` prefix are mapped to key-value pairs in configuration files.
 
-+ Prefix ``EMQX_`` is removed
-+ All upper case letters is replaced with lower case letters
-+ ``__`` is replaced with ``.``
+You can change the prefix by overriding "CUTTLEFISH_ENV_OVERRIDE_PREFIX".
 
 Example:
 
@@ -54,6 +52,17 @@ EMQX_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors
 EMQX_MQTT__MAX_PACKET_SIZE              <--> mqtt.max_packet_size
 ```
 
++ Prefix ``EMQX_`` is removed
++ All upper case letters is replaced with lower case letters
++ ``__`` is replaced with ``.``
+
+If `CUTTLEFISH_ENV_OVERRIDE_PREFIX=DEV_` is set:
+
+```bash
+DEV_LISTENER__SSL__EXTERNAL__ACCEPTORS <--> listener.ssl.external.acceptors
+DEV_MQTT__MAX_PACKET_SIZE              <--> mqtt.max_packet_size
+```
+
 Non mapped environment variables:
 
 ```bash
@@ -189,16 +198,6 @@ docker run -d --name emqx -p 18083:18083 -p 1883:1883 -p 4369:4369 \
     emqx/emqx:latest
 ```
 
-#### Mask Sensitive Configuration
-
-Use ``MASK_CONFIG_FILTER`` to hide senstive configuration values from leaking to logging system.
-
-For example, set ``MASK_CONFIG_FILTER="password|token"`` to hide all configuration names containing those keywords.
-
-By default emqx masks the configuration using following filter `"password|passwd|key|token|secret"`. Setting ``MASK_CONFIG_FILTER`` will be merged with the default filter.
-
-The configuration should match whole word (after splitting it by '.') with `MASK_CONFIG_FILTER`. You can use commas, spaces or other required separators to separate different words.
-
 ### Cluster
 
 EMQ X supports a variety of clustering methods, see our [documentation](https://docs.emqx.io/broker/latest/en/advanced/cluster.html#emqx-service-discovery) for details.
@@ -234,7 +233,7 @@ Let's create a static node list cluster from docker-compose.
         emqx-bridge:
           aliases:
           - node2.emqx.io
-  
+
   networks:
     emqx-bridge:
       driver: bridge

deploy/docker/docker-entrypoint.sh

@@ -90,79 +90,6 @@ if [[ -z "$EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS" ]]; then
     export EMQX_LISTENER__WSS__EXTERNAL__MAX_CONNECTIONS=102400
 fi
 
-# Fix issue #42 - export env EMQX_DASHBOARD__DEFAULT_USER__PASSWORD to configure
-# 'dashboard.default_user.password' in etc/plugins/emqx_dashboard.conf
-if [[ -n "$EMQX_ADMIN_PASSWORD" ]]; then
-    export EMQX_DASHBOARD__DEFAULT_USER__PASSWORD=$EMQX_ADMIN_PASSWORD
-fi
-
-# echo value of $VAR hiding secrets if any
-# SYNOPSIS
-#     echo_value KEY VALUE
-echo_value() {
-    # get MASK_CONFIG
-    MASK_CONFIG_FILTER="$MASK_CONFIG_FILTER|password|passwd|key|token|secret"
-    FORMAT_MASK_CONFIG_FILTER=$(echo "$MASK_CONFIG_FILTER" | sed -r -e 's/^[^A-Za-z0-9_]+//' -e 's/[^A-Za-z0-9_]+$//' -e 's/[^A-Za-z0-9_]+/|/g')
-    local key=$1
-    local value=$2
-    # check if contains sensitive value
-    if echo "$key" | grep -iqwE "$FORMAT_MASK_CONFIG_FILTER"; then
-        echo "$key=***secret***"
-    else
-        echo "$key=$value"
-    fi
-}
-
-# fill config on specific file if the key exists
-# SYNOPSIS
-#     try_fill_config FILE KEY VALUE
-try_fill_config() {
-    local file=$1
-    local key=$2
-    local value=$3
-    local escaped_key
-    # shellcheck disable=SC2001
-    escaped_key=$(echo "$key" | sed 's/[^a-zA-Z0-9_]/\\&/g')
-    local escaped_value
-    escaped_value=$(echo "$value" | sed 's/[\/&]/\\&/g')
-    if grep -qE "^[#[:space:]]*$escaped_key\s*=" "$file"; then
-        echo_value "$key" "$value"
-        if [[ -z "$value" ]]; then
-            sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/# \1 = \2/" "$file" > tmpfile && cat tmpfile > "$file"
-        else
-            sed -r "s/^[#[:space:]]*($escaped_key)\s*=\s*(.*)/\1 = $escaped_value/" "$file" > tmpfile && cat tmpfile > "$file"
-        fi
-    # Check if config has a numbering system, but no existing configuration line in file
-    elif echo "$key" | grep -qE '\.\d+|\d+\.'; then
-        if [[ -n "$value" ]]; then
-            local template
-            template="$(echo "$escaped_key" | sed -r -e 's/\\\.[0-9]+/\\.[0-9]+/g' -e 's/[0-9]+\\\./[0-9]+\\./g')"
-            if grep -qE "^[#[:space:]]*$template\s*=" "$file"; then
-                echo_value "$key" "$value"
-                sed '$a'\\ "$file" > tmpfile && cat tmpfile > "$file"
-                echo "$key = $value" >> "$file"
-            fi
-        fi
-    fi
-}
-
-# Catch all EMQX_ prefix environment variable and match it in configure file
-CONFIG_FILE="$_EMQX_HOME/etc/emqx.conf"
-CONFIG_PLUGINS="$_EMQX_HOME/etc/plugins"
-for VAR in $(compgen -e); do
-    # Config normal keys such like node.name = emqx@127.0.0.1
-    if echo "$VAR" | grep -q '^EMQX_'; then
-        VAR_NAME=$(echo "$VAR" | sed -e 's/^EMQX_//' -e 's/__/./g' | tr '[:upper:]' '[:lower:]' | tr -d '[:cntrl:]')
-        VAR_VALUE=$(echo "${!VAR}" | tr -d '[:cntrl:]')
-        # Config in emqx.conf
-        try_fill_config "$CONFIG_FILE" "$VAR_NAME" "$VAR_VALUE"
-        # Config in plugins/*
-        for CONFIG_PLUGINS_FILE in "$CONFIG_PLUGINS"/*; do
-            try_fill_config "$CONFIG_PLUGINS_FILE" "$VAR_NAME" "$VAR_VALUE"
-        done
-    fi
-done
-
 # fill tuples on specific file
 # SYNOPSIS
 #     fill_tuples FILE [ELEMENTS ...]

lib-opensource/emqx_dashboard/priv/emqx_dashboard.schema

@@ -6,7 +6,8 @@
 ]}.
 
 {mapping, "dashboard.default_user.password", "emqx_dashboard.default_user_passwd", [
-  {datatype, string}
+  {datatype, string},
+  {override_env, "ADMIN_PASSWORD"}
 ]}.
 
 {mapping, "dashboard.listener.http", "emqx_dashboard.listeners", [

priv/emqx.schema

@@ -202,7 +202,8 @@ end}.
 
 %% @doc Node name
 {mapping, "node.name", "vm_args.-name", [
-  {default, "emqx@127.0.0.1"}
+  {default, "emqx@127.0.0.1"},
+  {override_env, "NODE_NAME"}
 ]}.
 
 %% @doc Specify SSL Options in the file if using SSL for erlang distribution

rebar.config

@@ -46,7 +46,7 @@
     , {esockd, {git, "https://github.com/emqx/esockd", {tag, "5.8.0"}}}
     , {ekka, {git, "https://github.com/emqx/ekka", {tag, "0.8.0"}}}
     , {gen_rpc, {git, "https://github.com/emqx/gen_rpc", {tag, "2.5.0"}}}
-    , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.0.0"}}}
+    , {cuttlefish, {git, "https://github.com/emqx/cuttlefish", {tag, "v3.1.0"}}}
     , {minirest, {git, "https://github.com/emqx/minirest", {tag, "0.3.3"}}}
     , {ecpool, {git, "https://github.com/emqx/ecpool", {tag, "0.5.0"}}}
     , {replayq, {git, "https://github.com/emqx/replayq", {tag, "0.3.1"}}}