Merge pull request #8287 from terry-xiaoyu/gen_api_schema_json_for_connector_and_bridge

feat: generate API schema files for connectors and bridges

.ci/fvt_tests/relup.lux

@@ -85,6 +85,27 @@
 
     !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/
 
+    ## 1. upgrade to the new version
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## 2. downgrade to the old version
+    !./bin/emqx install $old_vsn
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent | grep -qs "$old_vsn"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## 3. again, upgrade to the new version
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
@@ -110,6 +131,27 @@
 
     !cp -f ../$PROFILE-$VSN-*-ubuntu20.04-amd64.tar.gz releases/
 
+    ## 1. upgrade to the new version
+    !./bin/emqx install $VSN
+    ?Made release permanent: "$VSN"
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent
+    ?(.*)$VSN
+    ?SH-PROMPT
+
+    ## 2. downgrade to the old version
+    !./bin/emqx install $old_vsn
+    ?Made release permanent:.*
+    ?SH-PROMPT
+
+    !./bin/emqx versions |grep permanent | grep -qs "$old_vsn"
+    ?SH-PROMPT:
+    !echo ==$$?==
+    ?^==0==
+    ?SH-PROMPT:
+
+    ## 3. again, upgrade to the new version
     !./bin/emqx install $VSN
     ?Made release permanent: "$VSN"
     ?SH-PROMPT
@@ -129,18 +171,20 @@
     ?Plugin\(emqx_management.*active=true\)
     ?SH-PROMPT
 
+## We don't guarantee not to lose a single message!
+## So even if we received 290~300 messages, we consider it as success
 [shell bench]
     ???publish complete
     ??SH-PROMPT:
     !sleep 5
     ?SH-PROMPT
 
-    !curl --user admin:public --silent --show-error http://localhost:8081/api/v4/rules | jq --raw-output ".data[0].metrics[] | select(.node==\"emqx@127.0.0.1\").matched"
+    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.matched"
     ?300
     ?SH-PROMPT
 
-    !curl http://127.0.0.1:8080/counter
-    ???{"data":300,"code":0}
+    !curl --user admin:public --silent --show-error http://localhost:18083/api/v5/rules | jq --raw-output ".[0].node_metrics[] | select(.node==\"emqx@127.0.0.1\") | .metrics.\"actions.success\""
+    ?\{"data":(29[0-9])|(300),"code":0\}
     ?SH-PROMPT
 
 [shell emqx2]

.github/workflows/run_relup_tests.yaml

@@ -71,6 +71,7 @@ jobs:
 
     - name: Get old vsn
       run: echo "OLD_VSNS=$(emqx/scripts/relup-base-vsns.sh ${{ matrix.profile }} | xargs echo -n)" >> $GITHUB_ENV
+      run: echo "VSN=$(emqx/pkg-vsn.sh ${{ matrix.profile }})" >> $GITHUB_ENV
 
     - name: build emqx
       env:

apps/emqx_conf/src/emqx_conf.appup.src

@@ -1,4 +1,3 @@
-%% -*- mode: erlang -*-
 %% Unless you know what you are doing, DO NOT edit manually!!
 {VSN,
   [{"0.1.0",

apps/emqx_conf/src/emqx_conf.erl

@@ -145,7 +145,7 @@ dump_schema(Dir, SchemaModule, I18nFile) ->
     lists:foreach(
         fun(Lang) ->
             gen_config_md(Dir, I18nFile, SchemaModule, Lang),
-            gen_hot_conf_schema_json(Dir, I18nFile, Lang),
+            gen_api_schema_json(Dir, I18nFile, Lang),
             gen_example_conf(filename:dirname(I18nFile), I18nFile, SchemaModule, Lang)
         end,
         [en, zh]
@@ -161,14 +161,32 @@ gen_schema_json(Dir, I18nFile, SchemaModule) ->
     IoData = jsx:encode(JsonMap, [space, {indent, 4}]),
     ok = file:write_file(SchemaJsonFile, IoData).
 
-gen_hot_conf_schema_json(Dir, I18nFile, Lang) ->
+gen_api_schema_json(Dir, I18nFile, Lang) ->
     emqx_dashboard:init_i18n(I18nFile, Lang),
-    JsonFile = "hot-config-schema-" ++ atom_to_list(Lang) ++ ".json",
-    HotConfigSchemaFile = filename:join([Dir, JsonFile]),
-    io:format(user, "===< Generating: ~s~n", [HotConfigSchemaFile]),
-    ok = gen_hot_conf_schema(HotConfigSchemaFile),
+    gen_api_schema_json_hotconf(Dir, Lang),
+    gen_api_schema_json_connector(Dir, Lang),
+    gen_api_schema_json_bridge(Dir, Lang),
     emqx_dashboard:clear_i18n().
 
+gen_api_schema_json_hotconf(Dir, Lang) ->
+    SchemaInfo = #{title => <<"EMQX Hot Conf API Schema">>, version => <<"0.1.0">>},
+    File = schema_filename(Dir, "hot-config-schema-", Lang),
+    ok = do_gen_api_schema_json(File, emqx_mgmt_api_configs, SchemaInfo).
+
+gen_api_schema_json_connector(Dir, Lang) ->
+    SchemaInfo = #{title => <<"EMQX Connector API Schema">>, version => <<"0.1.0">>},
+    File = schema_filename(Dir, "connector-api-", Lang),
+    ok = do_gen_api_schema_json(File, emqx_connector_api, SchemaInfo).
+
+gen_api_schema_json_bridge(Dir, Lang) ->
+    SchemaInfo = #{title => <<"EMQX Data Bridge API Schema">>, version => <<"0.1.0">>},
+    File = schema_filename(Dir, "bridge-api-", Lang),
+    ok = do_gen_api_schema_json(File, emqx_bridge_api, SchemaInfo).
+
+schema_filename(Dir, Prefix, Lang) ->
+    Filename = Prefix ++ atom_to_list(Lang) ++ ".json",
+    filename:join([Dir, Filename]).
+
 gen_config_md(Dir, I18nFile, SchemaModule, Lang0) ->
     Lang = atom_to_list(Lang0),
     SchemaMdFile = filename:join([Dir, "config-" ++ Lang ++ ".md"]),
@@ -216,9 +234,10 @@ gen_example(File, SchemaModule, I18nFile, Lang) ->
     file:write_file(File, Example).
 
 %% Only gen hot_conf schema, not all configuration fields.
-gen_hot_conf_schema(File) ->
+do_gen_api_schema_json(File, SchemaMod, SchemaInfo) ->
+    io:format(user, "===< Generating: ~s~n", [File]),
     {ApiSpec0, Components0} = emqx_dashboard_swagger:spec(
-        emqx_mgmt_api_configs,
+        SchemaMod,
         #{schema_converter => fun hocon_schema_to_spec/2}
     ),
     ApiSpec = lists:foldl(
@@ -250,7 +269,7 @@ gen_hot_conf_schema(File) ->
     Components = lists:foldl(fun(M, Acc) -> maps:merge(M, Acc) end, #{}, Components0),
     IoData = jsx:encode(
         #{
-            info => #{title => <<"EMQX Hot Conf Schema">>, version => <<"0.1.0">>},
+            info => SchemaInfo,
             paths => ApiSpec,
             components => #{schemas => Components}
         },