Merge pull request #12210 from keynslug/feat/EMQX-11579/schema-type-doc

feat(schema): provide type-level documentation snippets

apps/emqx_conf/src/emqx_conf.erl

@@ -28,7 +28,7 @@
 -export([remove/2, remove/3]).
 -export([tombstone/2]).
 -export([reset/2, reset/3]).
--export([dump_schema/2, reformat_schema_dump/1]).
+-export([dump_schema/2, reformat_schema_dump/2]).
 -export([schema_module/0]).
 
 %% TODO: move to emqx_dashboard when we stop building api schema at build time
@@ -186,7 +186,7 @@ gen_schema_json(Dir, SchemaModule, Lang) ->
     ok = gen_preformat_md_json_files(Dir, StructsJsonArray, Lang).
 
 gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
-    NestedStruct = reformat_schema_dump(StructsJsonArray),
+    NestedStruct = reformat_schema_dump(StructsJsonArray, Lang),
     %% write to files
     NestedJsonFile = filename:join([Dir, "schema-v2-" ++ Lang ++ ".json"]),
     io:format(user, "===< Generating: ~s~n", [NestedJsonFile]),
@@ -196,15 +196,17 @@ gen_preformat_md_json_files(Dir, StructsJsonArray, Lang) ->
     ok.
 
 %% @doc This function is exported for scripts/schema-dump-reformat.escript
-reformat_schema_dump(StructsJsonArray0) ->
+reformat_schema_dump(StructsJsonArray0, Lang) ->
     %% prepare
+    DescResolver = make_desc_resolver(Lang),
     StructsJsonArray = deduplicate_by_full_name(StructsJsonArray0),
     #{fields := RootFields} = hd(StructsJsonArray),
     RootNames0 = lists:map(fun(#{name := RootName}) -> RootName end, RootFields),
     RootNames = lists:map(fun to_bin/1, RootNames0),
     %% reformat
     [Root | FlatStructs0] = lists:map(
-        fun(Struct) -> gen_flat_doc(RootNames, Struct) end, StructsJsonArray
+        fun(Struct) -> gen_flat_doc(RootNames, Struct, DescResolver) end,
+        StructsJsonArray
     ),
     FlatStructs = [Root#{text => <<"root">>, hash => <<"root">>} | FlatStructs0],
     gen_nested_doc(FlatStructs).
@@ -302,7 +304,7 @@ expand_ref(#{hash := FullName}, FindFn, Path) ->
 
 %% generate flat docs for each struct.
 %% using references to link to other structs.
-gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
+gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S, DescResolver) ->
     ShortName = short_name(FullName),
     case is_missing_namespace(ShortName, to_bin(FullName), RootNames) of
         true ->
@@ -314,18 +316,19 @@ gen_flat_doc(RootNames, #{full_name := FullName, fields := Fields} = S) ->
         text => short_name(FullName),
         hash => format_hash(FullName),
         doc => maps:get(desc, S, <<"">>),
-        fields => format_fields(Fields)
+        fields => format_fields(Fields, DescResolver)
     }.
 
-format_fields([]) ->
-    [];
-format_fields([Field | Fields]) ->
-    [format_field(Field) | format_fields(Fields)].
+format_fields(Fields, DescResolver) ->
+    [format_field(F, DescResolver) || F <- Fields].
 
-format_field(#{name := Name, aliases := Aliases, type := Type} = F) ->
+format_field(#{name := Name, aliases := Aliases, type := Type} = F, DescResolver) ->
+    TypeDoc = format_type_desc(Type, DescResolver),
     L = [
         {text, Name},
         {type, format_type(Type)},
+        %% TODO: Make it into a separate field.
+        %% {typedoc, format_type_desc(Type, DescResolver)},
         {refs, format_refs(Type)},
         {aliases,
             case Aliases of
@@ -333,7 +336,7 @@ format_field(#{name := Name, aliases := Aliases, type := Type} = F) ->
                 _ -> Aliases
             end},
         {default, maps:get(hocon, maps:get(default, F, #{}), undefined)},
-        {doc, maps:get(desc, F, undefined)}
+        {doc, join_format([maps:get(desc, F, undefined), TypeDoc])}
     ],
     maps:from_list([{K, V} || {K, V} <- L, V =/= undefined]).
 
@@ -393,10 +396,26 @@ format_union_members([Member | Members], Acc) ->
     NewAcc = [format_type(Member) | Acc],
     format_union_members(Members, NewAcc).
 
+format_type_desc(#{kind := primitive, name := Name}, DescResolver) ->
+    format_primitive_type_desc(Name, DescResolver);
+format_type_desc(#{}, _DescResolver) ->
+    undefined.
+
 format_primitive_type(TypeStr) ->
-    Spec = emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr),
+    Spec = get_primitive_typespec(TypeStr),
     to_bin(maps:get(type, Spec)).
 
+format_primitive_type_desc(TypeStr, DescResolver) ->
+    case get_primitive_typespec(TypeStr) of
+        #{desc := Desc} ->
+            DescResolver(Desc);
+        #{} ->
+            undefined
+    end.
+
+get_primitive_typespec(TypeStr) ->
+    emqx_conf_schema_types:readable_docgen(?MODULE, TypeStr).
+
 %% All types should have a namespace to avlid name clashing.
 is_missing_namespace(ShortName, FullName, RootNames) ->
     case lists:member(ShortName, RootNames) of
@@ -560,6 +579,14 @@ hocon_schema_to_spec(Atom, _LocalModule) when is_atom(Atom) ->
 typename_to_spec(TypeStr, Module) ->
     emqx_conf_schema_types:readable_dashboard(Module, TypeStr).
 
+join_format(Snippets) ->
+    case [S || S <- Snippets, S =/= undefined] of
+        [] ->
+            undefined;
+        NonEmpty ->
+            to_bin(lists:join("<br/>", NonEmpty))
+    end.
+
 to_bin(List) when is_list(List) -> iolist_to_binary(List);
 to_bin(Boolean) when is_boolean(Boolean) -> Boolean;
 to_bin(Atom) when is_atom(Atom) -> atom_to_binary(Atom, utf8);

apps/emqx_conf/src/emqx_conf_schema_types.erl

@@ -16,6 +16,8 @@
 
 -module(emqx_conf_schema_types).
 
+-include_lib("hocon/include/hocon_types.hrl").
+
 -export([readable/2]).
 -export([readable_swagger/2, readable_dashboard/2, readable_docgen/2]).
 
@@ -165,37 +167,37 @@ readable("duration()") ->
     #{
         swagger => #{type => string, example => <<"12m">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"12m">>}
+        docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)}
     };
 readable("duration_s()") ->
     #{
         swagger => #{type => string, example => <<"1h">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"1h">>}
+        docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)}
     };
 readable("duration_ms()") ->
     #{
         swagger => #{type => string, example => <<"32s">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"32s">>}
+        docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)}
     };
 readable("timeout_duration()") ->
     #{
         swagger => #{type => string, example => <<"12m">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"12m">>}
+        docgen => #{type => "Duration", example => <<"12m">>, desc => ?DESC(duration)}
     };
 readable("timeout_duration_s()") ->
     #{
         swagger => #{type => string, example => <<"1h">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"1h">>}
+        docgen => #{type => "Duration(s)", example => <<"1h">>, desc => ?DESC(duration)}
     };
 readable("timeout_duration_ms()") ->
     #{
         swagger => #{type => string, example => <<"32s">>},
         dashboard => #{type => duration},
-        docgen => #{type => "String", example => <<"32s">>}
+        docgen => #{type => "Duration", example => <<"32s">>, desc => ?DESC(duration)}
     };
 readable("percent()") ->
     #{
@@ -219,13 +221,13 @@ readable("bytesize()") ->
     #{
         swagger => #{type => string, example => <<"32MB">>},
         dashboard => #{type => 'byteSize'},
-        docgen => #{type => "String", example => <<"32MB">>}
+        docgen => #{type => "Bytesize", example => <<"32MB">>, desc => ?DESC(bytesize)}
     };
 readable("wordsize()") ->
     #{
         swagger => #{type => string, example => <<"1024KB">>},
         dashboard => #{type => 'wordSize'},
-        docgen => #{type => "String", example => <<"1024KB">>}
+        docgen => #{type => "Bytesize", example => <<"1024KB">>, desc => ?DESC(bytesize)}
     };
 readable("map(" ++ Map) ->
     [$) | _MapArgs] = lists:reverse(Map),
@@ -287,7 +289,11 @@ readable("secret()") ->
     #{
         swagger => #{type => string, example => <<"R4ND0M/S∃CЯ∃T"/utf8>>},
         dashboard => #{type => string},
-        docgen => #{type => "String", example => <<"R4ND0M/S∃CЯ∃T"/utf8>>}
+        docgen => #{
+            type => "Secret",
+            example => <<"R4ND0M/S∃CЯ∃T"/utf8>>,
+            desc => ?DESC(secret)
+        }
     };
 readable(TypeStr0) ->
     case string:split(TypeStr0, ":") of

rel/i18n/emqx_conf_schema_types.hocon

@@ -0,0 +1,12 @@
+emqx_conf_schema_types {
+
+    duration.desc:
+    """A string that represents a time duration, for example: <code>10s</code>, <code>2.5m</code>, <code>1h30m</code>, <code>1W2D</code>, or <code>2345ms</code>, which is the smallest unit. When precision is specified, finer portions of the duration may be ignored: writing <code>1200ms</code> for <code>Duration(s)</code> is equivalent to writing <code>1s</code>. The unit part is case-insensitive."""
+
+    bytesize.desc:
+    """A string that represents a number of bytes, for example: <code>10B</code>, <code>640kb</code>, <code>4MB</code>, <code>1GB</code>. Units are interpreted as powers of 1024, and the unit part is case-insensitive."""
+
+    secret.desc:
+    """A string holding some sensitive information, such as a password. When secret starts with <code>file://</code>, the rest of the string is interpreted as a path to a file containing the secret itself: whole content of the file except any trailing whitespace characters is considered a secret value. Note: when clustered, all EMQX nodes should have the same file present before using <code>file://</code> secrets."""
+
+}

scripts/schema-dump-reformat.escript

@@ -1,132 +0,0 @@
-#!/usr/bin/env escript
-
-%% This script translates the hocon_schema_json's schema dump to a new format.
-%% It is used to convert older version EMQX's schema dumps to the new format
-%% after all files are upgraded to the new format, this script can be removed.
-
--mode(compile).
-
-main([Input]) ->
-    ok = add_libs(),
-    _ = atoms(),
-    {ok, Data} = file:read_file(Input),
-    Json = jsx:decode(Data),
-    NewJson = reformat(Json),
-    io:format("~s~n", [jsx:encode(NewJson)]);
-main(_) ->
-    io:format("Usage: schema-dump-reformat.escript <input.json>~n"),
-    halt(1).
-
-reformat(Json) ->
-    emqx_conf:reformat_schema_dump(fix(Json)).
-
-%% fix old type specs to make them compatible with new type specs
-fix(#{
-    <<"kind">> := <<"union">>,
-    <<"members">> := [#{<<"name">> := <<"string()">>}, #{<<"name">> := <<"function()">>}]
-}) ->
-    %% s3_exporter.secret_access_key
-    #{
-        kind => primitive,
-        name => <<"string()">>
-    };
-fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_conf_schema:log_level()">>}) ->
-    #{
-        kind => enum,
-        symbols => [emergency, alert, critical, error, warning, notice, info, debug, none, all]
-    };
-fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_connector_http:pool_type()">>}) ->
-    #{kind => enum, symbols => [random, hash]};
-fix(#{<<"kind">> := <<"primitive">>, <<"name">> := <<"emqx_bridge_http_connector:pool_type()">>}) ->
-    #{kind => enum, symbols => [random, hash]};
-fix(Map) when is_map(Map) ->
-    maps:from_list(fix(maps:to_list(Map)));
-fix(List) when is_list(List) ->
-    lists:map(fun fix/1, List);
-fix({<<"kind">>, Kind}) ->
-    {kind, binary_to_atom(Kind, utf8)};
-fix({<<"name">>, Type}) ->
-    {name, fix_type(Type)};
-fix({K, V}) ->
-    {binary_to_atom(K, utf8), fix(V)};
-fix(V) when is_number(V) ->
-    V;
-fix(V) when is_atom(V) ->
-    V;
-fix(V) when is_binary(V) ->
-    V.
-
-%% ensure below ebin dirs are added to code path:
-%% _build/default/lib/*/ebin
-%% _build/emqx/lib/*/ebin
-%% _build/emqx-enterprise/lib/*/ebin
-add_libs() ->
-    Profile = os:getenv("PROFILE"),
-    case Profile of
-        "emqx" ->
-            ok;
-        "emqx-enterprise" ->
-            ok;
-        _ ->
-            io:format("PROFILE is not set~n"),
-            halt(1)
-    end,
-    Dirs =
-        filelib:wildcard("_build/default/lib/*/ebin") ++
-            filelib:wildcard("_build/" ++ Profile ++ "/lib/*/ebin"),
-    lists:foreach(fun add_lib/1, Dirs).
-
-add_lib(Dir) ->
-    code:add_patha(Dir),
-    Beams = filelib:wildcard(Dir ++ "/*.beam"),
-    _ = spawn(fun() -> lists:foreach(fun load_beam/1, Beams) end),
-    ok.
-
-load_beam(BeamFile) ->
-    ModuleName = filename:basename(BeamFile, ".beam"),
-    Module = list_to_atom(ModuleName),
-    %% load the beams to make sure the atoms are existing
-    code:ensure_loaded(Module),
-    ok.
-
-fix_type(<<"[{string(), string()}]">>) ->
-    <<"map()">>;
-fix_type(<<"[{binary(), binary()}]">>) ->
-    <<"map()">>;
-fix_type(<<"emqx_limiter_schema:rate()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_limiter_schema:burst_rate()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_limiter_schema:capacity()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_limiter_schema:initial()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_limiter_schema:failure_strategy()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_conf_schema:file()">>) ->
-    <<"string()">>;
-fix_type(<<"#{term() => binary()}">>) ->
-    <<"map()">>;
-fix_type(<<"[term()]">>) ->
-    %% jwt claims
-    <<"map()">>;
-fix_type(<<"emqx_ee_bridge_influxdb:write_syntax()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_bridge_influxdb:write_syntax()">>) ->
-    <<"string()">>;
-fix_type(<<"emqx_schema:mqtt_max_packet_size()">>) ->
-    <<"non_neg_integer()">>;
-fix_type(<<"emqx_s3_schema:secret_access_key()">>) ->
-    <<"string()">>;
-fix_type(Type) ->
-    Type.
-
-%% ensure atoms are loaded
-%% these atoms are from older version of emqx
-atoms() ->
-    [
-        emqx_ee_connector_clickhouse,
-        emqx_ee_bridge_gcp_pubsub,
-        emqx_ee_bridge_influxdb,
-        emqx_connector_http
-    ].