Merge pull request #13455 from thalesmg/20240711-m-mix-umbrella-part-III-no-ci

sync new mix build work to master

.github/workflows/.zipignore2

@@ -0,0 +1 @@
+*/.github/*

Makefile

@@ -28,6 +28,8 @@ CT_COVER_EXPORT_PREFIX ?= $(PROFILE)
 
 export REBAR_GIT_CLONE_OPTIONS += --depth=1
 
+ELIXIR_COMMON_DEPS := ensure-hex ensure-mix-rebar3 ensure-mix-rebar
+
 .PHONY: default
 default: $(REBAR) $(PROFILE)
 
@@ -58,8 +60,12 @@ ensure-mix-rebar3: $(REBAR)
 ensure-mix-rebar: $(REBAR)
 	@mix local.rebar --if-missing --force
 
+
+.PHONY: elixir-common-deps
+elixir-common-deps: $(ELIXIR_COMMON_DEPS)
+
 .PHONY: mix-deps-get
-mix-deps-get: $(ELIXIR_COMMON_DEPS)
+mix-deps-get: elixir-common-deps
 	@mix deps.get
 
 .PHONY: eunit

apps/emqx/mix.exs

@@ -8,7 +8,7 @@ defmodule EMQX.MixProject do
       app: :emqx,
       version: "0.1.0",
       build_path: "../../_build",
-      erlc_paths: UMP.erlc_paths(),
+      erlc_paths: erlc_paths(),
       erlc_options: [
         {:i, "src"}
         | UMP.erlc_options()
@@ -36,8 +36,9 @@ defmodule EMQX.MixProject do
   def deps() do
     ## FIXME!!! go though emqx.app.src and add missing stuff...
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx_utils, in_umbrella: true},
-      # {:emqx_ds_backends, in_umbrella: true},
+      {:emqx_ds_backends, in_umbrella: true},
 
       UMP.common_dep(:gproc),
       UMP.common_dep(:gen_rpc),
@@ -53,6 +54,15 @@ defmodule EMQX.MixProject do
     ] ++ UMP.quicer_dep()
   end
 
+  defp erlc_paths() do
+    paths = UMP.erlc_paths()
+    if UMP.test_env?() do
+      ["integration_test" | paths]
+    else
+      paths
+    end
+  end
+
   defp extra_dirs() do
     dirs = ["src", "etc"]
     if UMP.test_env?() do

apps/emqx_auth/mix.exs

@@ -28,6 +28,7 @@ defmodule EMQXAuth.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       {:emqx_utils, in_umbrella: true}
     ]

apps/emqx_bridge/mix.exs

@@ -28,6 +28,7 @@ defmodule EMQXBridge.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       {:emqx_resource, in_umbrella: true},
       {:emqx_connector, in_umbrella: true},

apps/emqx_bridge_hstreamdb/mix.exs

@@ -24,7 +24,8 @@ defmodule EMQXBridgeHstreamdb.MixProject do
   def deps() do
     [
       {:hstreamdb_erl,
-       github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"},
+       github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1",
+       system_env: UMP.emqx_app_system_env()},
       {:emqx, in_umbrella: true},
       {:emqx_utils, in_umbrella: true},
       {:emqx_connector, in_umbrella: true, runtime: false},

apps/emqx_dashboard_sso/mix.exs

@@ -26,7 +26,8 @@ defmodule EMQXDashboardSso.MixProject do
       {:emqx_ctl, in_umbrella: true},
       {:emqx_ldap, in_umbrella: true},
       {:emqx_dashboard, in_umbrella: true},
-      {:esaml, github: "emqx/esaml", tag: "v1.1.3"}
+      {:esaml, github: "emqx/esaml", tag: "v1.1.3"},
+      {:oidcc, github: "emqx/oidcc", tag: "v3.2.0-1"},
     ]
   end
 end

apps/emqx_durable_storage/mix.exs

@@ -7,9 +7,14 @@ defmodule EMQXDurableStorage.MixProject do
       app: :emqx_durable_storage,
       version: "0.1.0",
       build_path: "../../_build",
-      # config_path: "../../config/config.exs",
+      compilers: [:yecc, :leex, :elixir, :asn1, :erlang, :app],
       erlc_options: UMP.erlc_options(),
-      erlc_paths: UMP.erlc_paths(),
+      erlc_paths: ["gen_src" | UMP.erlc_paths()],
+      # used by our `compile.asn1` compiler
+      asn1_srcs: [
+        %{src: "./asn.1/DurableMessage.asn",
+          compile_opts: [:per, :noobj, outdir: ~c"gen_src"]}
+      ],
       deps_path: "../../deps",
       lockfile: "../../mix.lock",
       elixir: "~> 1.14",
@@ -28,6 +33,7 @@ defmodule EMQXDurableStorage.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx_utils, in_umbrella: true},
       UMP.common_dep(:rocksdb),
       UMP.common_dep(:gproc),

apps/emqx_exhook/mix.exs

@@ -36,6 +36,7 @@ defmodule EMQXExhook.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       {:emqx_utils, in_umbrella: true},
       UMP.common_dep(:grpc)

apps/emqx_gateway/mix.exs

@@ -26,6 +26,7 @@ defmodule EMQXGateway.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       {:emqx_utils, in_umbrella: true},
       {:emqx_ctl, in_umbrella: true},

apps/emqx_gateway/test/emqx_gateway_api_SUITE.erl

@@ -48,6 +48,8 @@ init_per_suite(Conf) ->
     Apps = emqx_cth_suite:start(
         [
             emqx_conf,
+            emqx_auth,
+            emqx_auth_mnesia,
             emqx_management,
             {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"},
             {emqx_gateway, ?CONF_DEFAULT}

apps/emqx_gateway/test/emqx_gateway_authn_SUITE.erl

@@ -57,6 +57,7 @@ init_per_group(AuthName, Conf) ->
     Apps = emqx_cth_suite:start(
         [
             emqx_conf,
+            emqx_auth_http,
             emqx_management,
             {emqx_dashboard, "dashboard.listeners.http { enable = true, bind = 18083 }"},
             {emqx_gateway, emqx_gateway_auth_ct:list_gateway_conf()}

apps/emqx_gateway/test/emqx_gateway_authz_SUITE.erl

@@ -57,6 +57,7 @@ init_per_group(AuthName, Conf) ->
     Apps = emqx_cth_suite:start(
         [
             {emqx_conf, "authorization { no_match = deny, cache { enable = false } }"},
+            emqx_auth_http,
             {emqx_gateway, emqx_gateway_auth_ct:list_gateway_conf()}
             | emqx_gateway_test_utils:all_gateway_apps()
         ],

apps/emqx_gateway_exproto/mix.exs

@@ -34,6 +34,7 @@ defmodule EMQXGatewayExproto.MixProject do
   def deps() do
     test_deps = if UMP.test_env?(), do: [{:emqx_exhook, in_umbrella: true, runtime: false}], else: []
     test_deps ++ [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       {:emqx_utils, in_umbrella: true},
       {:emqx_gateway, in_umbrella: true},

apps/emqx_gcp_device/mix.exs

@@ -27,6 +27,7 @@ defmodule EMQXGCPDevice.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       {:emqx_auth, in_umbrella: true},
       UMP.common_dep(:jose),

apps/emqx_license/mix.exs

@@ -26,6 +26,7 @@ defmodule EMQXLicense.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       {:emqx_utils, in_umbrella: true},
       {:emqx_ctl, in_umbrella: true},

apps/emqx_machine/mix.exs

@@ -30,6 +30,8 @@ defmodule EMQXMachine.MixProject do
       {:emqx_dashboard, in_umbrella: true, runtime: false},
       {:emqx_management, in_umbrella: true, runtime: false},
       UMP.common_dep(:covertool),
+      UMP.common_dep(:system_monitor),
+      UMP.common_dep(:redbug),
     ]
   end
 end

apps/emqx_mix_utils/lib/mix/tasks/compile.asn1.ex

@@ -0,0 +1,82 @@
+defmodule Mix.Tasks.Compile.Asn1 do
+  use Mix.Task.Compiler
+
+  @recursive true
+  @manifest_vsn 1
+  @manifest "compile.asn1"
+  # TODO: use manifest to track generated files?
+
+  @impl true
+  def manifests(), do: [manifest()]
+  defp manifest(), do: Path.join(Mix.Project.manifest_path(), @manifest)
+
+  @impl true
+  def run(_args) do
+    add_to_path_and_cache(:asn1)
+
+    Mix.Project.get!()
+    config = Mix.Project.config()
+    app_root = File.cwd!()
+
+    asn1_srcs = config[:asn1_srcs] || []
+    manifest_data = read_manifest(manifest())
+    manifest_modified_time = Mix.Utils.last_modified(manifest())
+    Enum.each(asn1_srcs, &compile(&1, app_root, manifest_modified_time))
+    write_manifest(manifest(), manifest_data)
+
+    {:noop, []}
+  end
+
+  defp compile(src, app_root, manifest_modified_time) do
+    %{
+      src: src_path,
+      compile_opts: compile_opts
+    } = src
+    src_path =
+      app_root
+      |> Path.join(src_path)
+      |> Path.expand()
+    if stale?(src_path, manifest_modified_time) do
+      Mix.shell().info("compiling asn1 file: #{src_path}")
+      :ok = :asn1ct.compile(to_charlist(src_path), compile_opts)
+    else
+      Mix.shell().info("file is up to date, not compiling: #{src_path}")
+    end
+  end
+
+  defp stale?(file, manifest_modified_time) do
+    with true <- File.exists?(file),
+         false <- Mix.Utils.stale?([file], [manifest_modified_time]) do
+      false
+    else
+      _ -> true
+    end
+  end
+
+  defp read_manifest(file) do
+    try do
+      file |> File.read!() |> :erlang.binary_to_term()
+    rescue
+      _ -> %{}
+    else
+      {@manifest_vsn, data} when is_map(data) -> data
+      _ -> %{}
+    end
+  end
+
+  defp write_manifest(file, data) do
+    Mix.shell().info("writing manifest #{file}")
+    File.mkdir_p!(Path.dirname(file))
+    File.write!(file, :erlang.term_to_binary({@manifest_vsn, data}))
+  end
+
+  def add_to_path_and_cache(lib_name) do
+    :code.lib_dir()
+    |> Path.join("#{lib_name}-*")
+    |> Path.wildcard()
+    |> hd()
+    |> Path.join("ebin")
+    |> to_charlist()
+    |> :code.add_path(:cache)
+  end
+end

apps/emqx_exhook/lib/mix/tasks/compile.grpc.ex

@@ -1,6 +1,8 @@
 defmodule Mix.Tasks.Compile.Grpc do
   use Mix.Task.Compiler
 
+  alias EMQXUmbrella.MixProject, as: UMP
+
   @recursive true
   @manifest_vsn 1
   @manifest "compile.grpc"
@@ -46,6 +48,9 @@ defmodule Mix.Tasks.Compile.Grpc do
     write_manifest(manifest(), manifest_data)
 
     {:noop, []}
+  after
+    Application.unload(:gpb)
+    Application.unload(:syntax_tools)
   end
 
   defp compile_pb(proto_src, context) do
@@ -95,7 +100,7 @@ defmodule Mix.Tasks.Compile.Grpc do
           :return_errors,
           i: to_charlist(gpb_include_dir),
           outdir: to_charlist(ebin_path)
-        ]
+        ] ++ UMP.erlc_options()
       )
       # todo: error handling & logging
       case compile_res do

lib/mix/tasks/emqx.ct.ex

@@ -48,10 +48,11 @@ defmodule Mix.Tasks.Emqx.Ct do
       abort_if_missing_suites: true,
       auto_compile: false,
       suite: opts |> Map.fetch!(:suites) |> Enum.map(&to_charlist/1),
+      group: opts |> Map.fetch!(:group_paths) |> Enum.map(fn gp -> Enum.map(gp, &String.to_atom/1) end),
       testcase: opts |> Map.fetch!(:cases) |> Enum.map(&to_charlist/1),
       readable: 'true',
       name: node_name,
-      ct_hooks: [:cth_readable_shell],
+      ct_hooks: [:cth_readable_shell, :cth_readable_failonly],
       logdir: to_charlist(logdir)
     )
 
@@ -198,11 +199,13 @@ defmodule Mix.Tasks.Emqx.Ct do
       args,
       strict: [
         suites: :string,
-        groups: :string,
+        group_paths: :string,
         cases: :string])
-    |> IO.inspect(label: :opts)
     suites = get_name_list(opts, :suites)
-    groups = get_name_list(opts, :groups)
+    group_paths =
+      opts
+      |> get_name_list(:group_paths)
+      |> Enum.map(& String.split(&1, ".", trim: true))
     cases = get_name_list(opts, :cases)
 
     if suites == [] do
@@ -211,7 +214,7 @@ defmodule Mix.Tasks.Emqx.Ct do
 
     %{
       suites: suites,
-      groups: groups,
+      group_paths: group_paths,
       cases: cases
     }
   end

apps/emqx_mix_utils/lib/mix/tasks/emqx.dialyzer.ex

@@ -0,0 +1,158 @@
+defmodule Mix.Tasks.Emqx.Dialyzer do
+  use Mix.Task
+
+  alias Mix.Tasks.Emqx.Ct, as: ECt
+
+  @requirements ["compile", "loadpaths"]
+
+  @excluded_mods (
+    [
+      :emqx_exproto_v_1_connection_unary_handler_bhvr,
+      :emqx_exproto_v_1_connection_handler_client,
+      :emqx_exproto_v_1_connection_handler_bhvr,
+      :emqx_exproto_v_1_connection_adapter_client,
+      :emqx_exproto_v_1_connection_adapter_bhvr,
+      :emqx_exproto_v_1_connection_unary_handler_client,
+      :emqx_exhook_v_2_hook_provider_client,
+      :emqx_exhook_v_2_hook_provider_bhvr,
+      Mix.Tasks.Compile.Grpc,
+      Mix.Tasks.Compile.CopySrcs,
+    ]
+    |> MapSet.new(&to_string/1)
+  )
+
+  @impl true
+  def run(_args) do
+    ECt.add_to_path_and_cache(:dialyzer)
+
+    %{
+      umbrella_apps: umbrella_apps,
+      dep_apps: dep_apps
+    } = resolve_apps()
+    umbrella_files = Enum.flat_map(umbrella_apps, & resolve_files/1)
+    dep_files = Enum.flat_map(dep_apps, & resolve_files/1)
+    files =
+      (umbrella_files ++ dep_files)
+      |> Enum.reject(fn path ->
+        name = Path.basename(path, ".beam")
+        MapSet.member?(@excluded_mods, name)
+      end)
+      |> Enum.map(&to_charlist/1)
+    warning_files =
+      umbrella_files
+      |> Enum.reject(fn path ->
+        name = Path.basename(path, ".beam")
+        MapSet.member?(@excluded_mods, name)
+      end)
+      |> Enum.map(&to_charlist/1)
+    warning_apps = Enum.sort(umbrella_apps)
+
+    try do
+      :dialyzer.run(
+        analysis_type: :incremental,
+        warnings: [
+          :unmatched_returns,
+          :error_handling
+        ],
+        # plt_location: ~c".",
+        # plt_prefix: ~c"emqx_dialyzer",
+        warning_files: warning_files,
+        warning_files_rec: warning_files,
+        # apps: umbrella_apps ++ dep_apps,
+        # warning_apps: warning_apps,
+        get_warnings: false,
+        files: files,
+        files_rec: files
+      )
+    catch
+      {:dialyzer_error, msg} ->
+        {:dialyzer_error, to_string(msg)}
+      err ->
+        {:throw, err}
+    end
+    |> IO.inspect(limit: :infinity)
+  end
+
+  defp resolve_apps() do
+    base_apps = MapSet.new([:erts, :crypto])
+    # excluded_apps = MapSet.new([:elixir])
+    excluded_apps = MapSet.new()
+    acc = %{
+      umbrella_apps: [],
+      dep_apps: base_apps
+    }
+
+    Mix.Dep.Umbrella.loaded()
+    |> Enum.reduce(acc, fn dep, acc ->
+      # IO.inspect(dep)
+      props = dep.opts[:app_properties]
+      optional_apps = Keyword.get(props, :optional_applications, [])
+      apps = Keyword.get(props, :applications, [])
+      included_apps = Keyword.get(props, :included_applications, [])
+      dep_apps = MapSet.new(optional_apps ++ apps ++ included_apps)
+      acc
+      |> Map.update!(:umbrella_apps, & [dep.app | &1])
+      |> Map.update!(:dep_apps, & MapSet.union(&1, dep_apps))
+    end)
+    |> then(fn acc ->
+      dep_apps =
+        acc.dep_apps
+        |> MapSet.difference(MapSet.new(acc.umbrella_apps))
+        |> MapSet.difference(excluded_apps)
+        |> Enum.reduce(MapSet.new(), &find_nested_apps/2)
+        |> MapSet.difference(excluded_apps)
+        |> Enum.filter(&app_present?/1)
+      %{acc | dep_apps: dep_apps}
+    end)
+  end
+
+  defp app_present?(app) do
+    match?({:ok, _}, ebin_dir(app))
+  end
+
+  defp find_nested_apps(app, seen) do
+    if MapSet.member?(seen, app) do
+      seen
+    else
+      seen = MapSet.put(seen, app)
+      apps = case :application.get_key(app, :applications) do
+        {:ok, apps} -> apps
+        :undefined -> []
+      end
+      included_apps = case :application.get_key(app, :included_applications) do
+        {:ok, apps} -> apps
+        :undefined -> []
+      end
+      optional_apps = case :application.get_key(app, :optional_applications) do
+        {:ok, apps} -> apps
+        :undefined -> []
+      end
+      Enum.reduce(apps ++ included_apps, seen, &find_nested_apps/2)
+    end
+  end
+
+  defp resolve_files(app) do
+    with {:ok, dir} <- ebin_dir(app) do
+      Mix.Utils.extract_files([dir], [:beam])
+    else
+      _ -> []
+    end
+  end
+
+  defp ebin_dir(app) do
+    with dir when is_list(dir) <- :code.lib_dir(app, :ebin),
+         dir = to_string(dir),
+         true <- File.dir?(dir) || {:error, :not_a_dir} do
+      {:ok, to_string(dir)}
+    else
+      error ->
+        Mix.shell().info(IO.ANSI.format([
+              [:yellow,
+               "Unknown application: #{app}; error: #{inspect(error)}",
+               "; if this is is an optional application, ignore."
+              ],
+            ]))
+        :error
+    end
+  end
+end

apps/emqx_mix_utils/lib/mix/tasks/emqx.eunit.ex

@@ -0,0 +1,122 @@
+defmodule Mix.Tasks.Emqx.Eunit do
+  use Mix.Task
+
+  alias Mix.Tasks.Emqx.Ct, as: ECt
+
+  # todo: invoke the equivalent of `make merge-config` as a requirement...
+  @requirements ["compile", "loadpaths"]
+
+  @impl true
+  def run(args) do
+    Mix.debug(true)
+    IO.inspect(args)
+
+
+    Enum.each([:common_test, :eunit, :mnesia], &ECt.add_to_path_and_cache/1)
+
+    ECt.ensure_whole_emqx_project_is_loaded!()
+    ECt.unload_emqx_applications!()
+
+    {_, 0} = System.cmd("epmd", ["-daemon"])
+    node_name = :"test@127.0.0.1"
+    :net_kernel.start([node_name, :longnames])
+
+    # unmangle PROFILE env because some places (`:emqx_conf.resolve_schema_module`) expect
+    # the version without the `-test` suffix.
+    System.fetch_env!("PROFILE")
+    |> String.replace_suffix("-test", "")
+    |> then(& System.put_env("PROFILE", &1))
+
+    args
+    |> parse_args!()
+    |> discover_tests()
+    |> :eunit.test(
+      verbose: true,
+      print_depth: 100
+    )
+    |> case do
+         :ok -> :ok
+         :error -> Mix.raise("errors found in tests")
+       end
+  end
+
+  defp add_to_path_and_cache(lib_name) do
+    :code.lib_dir()
+    |> Path.join("#{lib_name}-*")
+    |> Path.wildcard()
+    |> hd()
+    |> Path.join("ebin")
+    |> to_charlist()
+    |> :code.add_path(:cache)
+  end
+
+  defp parse_args!(args) do
+    {opts, _rest} = OptionParser.parse!(
+      args,
+      strict: [
+        cases: :string,
+        modules: :string,
+      ]
+    )
+    cases =
+      opts
+      |> get_name_list(:cases)
+      |> Enum.flat_map(&resolve_test_fns!/1)
+    modules =
+      opts
+      |> get_name_list(:modules)
+      |> Enum.map(&String.to_atom/1)
+
+    %{
+      cases: cases,
+      modules: modules,
+    }
+  end
+
+  defp get_name_list(opts, key) do
+    opts
+    |> Keyword.get(key, "")
+    |> String.split(",", trim: true)
+  end
+
+  defp resolve_test_fns!(mod_fn_str) do
+    {mod, fun} = case String.split(mod_fn_str, ":") do
+                   [mod, fun] ->
+                     {String.to_atom(mod), String.to_atom(fun)}
+                   _ ->
+                     Mix.raise("Bad test case spec; must of `MOD:FUN` form.  Got: #{mod_fn_str}`")
+                 end
+    if not has_test_case?(mod, fun) do
+      Mix.raise("Module #{mod} does not export test case #{fun}")
+    end
+
+    if to_string(fun) =~ ~r/_test_$/ do
+      apply(mod, fun, [])
+    else
+      [Function.capture(mod, fun, 0)]
+    end
+  end
+
+  defp has_test_case?(mod, fun) do
+    try do
+      mod.module_info(:functions)
+      |> Enum.find(& &1 == {fun, 0})
+      |> then(& !! &1)
+    rescue
+      UndefinedFunctionError -> false
+    end
+  end
+
+  defp discover_tests(%{cases: [], modules: []} = _opts) do
+    Mix.Dep.Umbrella.cached()
+    |> Enum.map(& {:application, &1.app})
+  end
+  defp discover_tests(%{cases: cases, modules: modules}) do
+    Enum.concat(
+      [
+        cases,
+        Enum.map(modules, & {:module, &1})
+      ]
+    )
+  end
+end

lib/mix/tasks/emqx.proper.ex

@@ -1,8 +1,6 @@
 defmodule Mix.Tasks.Emqx.Proper do
   use Mix.Task
 
-  # Code.require_file("emqx.ct.ex", __DIR__)
-
   alias Mix.Tasks.Emqx.Ct, as: ECt
 
   # todo: invoke the equivalent of `make merge-config` as a requirement...

apps/emqx_mix_utils/mix.exs

@@ -0,0 +1,26 @@
+defmodule EMQXMixUtils.MixProject do
+  use Mix.Project
+  alias EMQXUmbrella.MixProject, as: UMP
+
+  def project do
+    [
+      app: :emqx_mix_utils,
+      version: "0.1.0",
+      build_path: "../../_build",
+      deps_path: "../../deps",
+      lockfile: "../../mix.lock",
+      elixir: "~> 1.14",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications
+  def application do
+    [extra_applications: UMP.extra_applications()]
+  end
+
+  def deps() do
+    []
+  end
+end

apps/emqx_modules/mix.exs

@@ -26,7 +26,8 @@ defmodule EMQXModules.MixProject do
       {:emqx, in_umbrella: true},
       {:emqx_ctl, in_umbrella: true},
       {:emqx_utils, in_umbrella: true},
-      {:emqx_conf, in_umbrella: true}
+      {:emqx_conf, in_umbrella: true},
+      UMP.common_dep(:observer_cli)
     ]
   end
 end

apps/emqx_rule_engine/mix.exs

@@ -22,7 +22,7 @@ defmodule EMQXRuleEngine.MixProject do
   end
 
   def deps() do
-    [
+    UMP.jq_dep() ++ [
       {:emqx, in_umbrella: true},
       {:emqx_ctl, in_umbrella: true},
       {:emqx_utils, in_umbrella: true},
@@ -31,6 +31,7 @@ defmodule EMQXRuleEngine.MixProject do
       {:emqx_bridge, in_umbrella: true},
       UMP.common_dep(:rulesql),
       UMP.common_dep(:emqtt),
+      UMP.common_dep(:uuid),
     ]
   end
 end

apps/emqx_s3/mix.exs

@@ -26,6 +26,7 @@ defmodule EMQXS3.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       UMP.common_dep(:gproc),
       UMP.common_dep(:ehttpc),

apps/emqx_schema_registry/mix.exs

@@ -28,7 +28,7 @@ defmodule EMQXSchemaRegistry.MixProject do
       {:emqx_rule_engine, in_umbrella: true},
       {:erlavro, github: "emqx/erlavro", tag: "2.10.0"},
       {:jesse, github: "emqx/jesse", tag: "1.8.0"},
-      UMP.common_dep(:gpb),
+      UMP.common_dep(:gpb, runtime: true),
     ]
   end
 end

apps/emqx_telemetry/mix.exs

@@ -26,6 +26,7 @@ defmodule EMQXTelemetry.MixProject do
 
   def deps() do
     [
+      {:emqx_mix_utils, in_umbrella: true, runtime: false},
       {:emqx, in_umbrella: true},
       {:emqx_utils, in_umbrella: true},
       {:emqx_conf, in_umbrella: true}

lib/mix/tasks/emqx.eunit.ex

@@ -1,58 +0,0 @@
-defmodule Mix.Tasks.Emqx.Eunit do
-  use Mix.Task
-
-  # Code.require_file("emqx.ct.ex", __DIR__)
-
-  alias Mix.Tasks.Emqx.Ct, as: ECt
-
-  # todo: invoke the equivalent of `make merge-config` as a requirement...
-  @requirements ["compile", "loadpaths"]
-
-  @impl true
-  def run(args) do
-    Mix.debug(true)
-    IO.inspect(args)
-
-
-    Enum.each([:common_test, :eunit, :mnesia], &ECt.add_to_path_and_cache/1)
-
-    ECt.ensure_whole_emqx_project_is_loaded!()
-    ECt.unload_emqx_applications!()
-
-    {_, 0} = System.cmd("epmd", ["-daemon"])
-    node_name = :"test@127.0.0.1"
-    :net_kernel.start([node_name, :longnames])
-
-    # unmangle PROFILE env because some places (`:emqx_conf.resolve_schema_module`) expect
-    # the version without the `-test` suffix.
-    System.fetch_env!("PROFILE")
-    |> String.replace_suffix("-test", "")
-    |> then(& System.put_env("PROFILE", &1))
-
-    discover_tests()
-    |> :eunit.test(
-      verbose: true,
-      print_depth: 100
-    )
-    |> case do
-         :ok -> :ok
-         :error -> Mix.raise("errors found in tests")
-       end
-  end
-
-  defp add_to_path_and_cache(lib_name) do
-    :code.lib_dir()
-    |> Path.join("#{lib_name}-*")
-    |> Path.wildcard()
-    |> hd()
-    |> Path.join("ebin")
-    |> to_charlist()
-    |> :code.add_path(:cache)
-  end
-
-  ## TODO: allow filtering modules and test names
-  defp discover_tests() do
-    Mix.Dep.Umbrella.cached()
-    |> Enum.map(& {:application, &1.app})
-  end
-end

mix.exs

@@ -40,11 +40,7 @@ defmodule EMQXUmbrella.MixProject do
 
     if new_mix_build?() do
       [
-        # TODO: these lines will be uncommented when we switch to using mix as the manager
-        # for all umbrella apps.
         apps_path: "apps",
-        apps:
-          applications(profile_info.release_type, profile_info.edition_type) |> Keyword.keys(),
         erlc_options: erlc_options(profile_info, version),
         version: version,
         deps: deps(profile_info, version),
@@ -160,7 +156,7 @@ defmodule EMQXUmbrella.MixProject do
       {:ssl_verify_fun, "1.1.7", override: true},
       common_dep(:rfc3339),
       common_dep(:bcrypt),
-      {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true},
+      common_dep(:uuid),
       {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true},
       common_dep(:ra),
       {:mimerl, "1.2.0", override: true}
@@ -169,12 +165,22 @@ defmodule EMQXUmbrella.MixProject do
 
   def extra_release_apps() do
     [
-      {:redbug, github: "emqx/redbug", tag: "2.0.10"},
-      {:observer_cli, "1.7.1"},
-      {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"}
+      common_dep(:redbug),
+      common_dep(:observer_cli),
+      common_dep(:system_monitor)
     ]
   end
 
+  def common_dep(dep_name, overrides) do
+    case common_dep(dep_name) do
+      {^dep_name, opts} ->
+        {dep_name, Keyword.merge(opts, overrides)}
+
+      {^dep_name, tag, opts} when is_binary(tag) ->
+        {dep_name, tag, Keyword.merge(opts, overrides)}
+    end
+  end
+
   def common_dep(:ekka), do: {:ekka, github: "emqx/ekka", tag: "0.19.5", override: true}
   def common_dep(:esockd), do: {:esockd, github: "emqx/esockd", tag: "5.11.2", override: true}
   def common_dep(:gproc), do: {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true}
@@ -186,7 +192,12 @@ defmodule EMQXUmbrella.MixProject do
   def common_dep(:ranch), do: {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true}
   def common_dep(:ehttpc), do: {:ehttpc, github: "emqx/ehttpc", tag: "0.4.14", override: true}
   def common_dep(:jiffy), do: {:jiffy, github: "emqx/jiffy", tag: "1.0.6", override: true}
-  def common_dep(:grpc), do: {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true}
+
+  def common_dep(:grpc),
+    do:
+      {:grpc,
+       github: "emqx/grpc-erl", tag: "0.6.12", override: true, system_env: emqx_app_system_env()}
+
   def common_dep(:cowboy), do: {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true}
   def common_dep(:jsone), do: {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true}
   def common_dep(:ecpool), do: {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true}
@@ -207,6 +218,13 @@ defmodule EMQXUmbrella.MixProject do
   def common_dep(:esasl), do: {:esasl, github: "emqx/esasl", tag: "0.2.1"}
   def common_dep(:gen_rpc), do: {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true}
 
+  def common_dep(:system_monitor),
+    do: {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"}
+
+  def common_dep(:uuid), do: {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true}
+  def common_dep(:redbug), do: {:redbug, github: "emqx/redbug", tag: "2.0.10"}
+  def common_dep(:observer_cli), do: {:observer_cli, "1.7.1"}
+
   def common_dep(:jose),
     do: {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2", override: true}
 
@@ -252,7 +270,7 @@ defmodule EMQXUmbrella.MixProject do
       github: "kafka4beam/snabbkaffe",
       tag: "1.0.10",
       override: true,
-      system_env: emqx_app_system_env(profile_info(), pkg_vsn())
+      system_env: emqx_app_system_env()
     }
 
   ###############################################################################################
@@ -293,6 +311,7 @@ defmodule EMQXUmbrella.MixProject do
           false
       end
     end)
+    |> Enum.reject(fn {app, _} -> app == :emqx_mix_utils end)
     |> Enum.reject(fn {app, _} -> app in excluded_apps end)
   end
 
@@ -416,16 +435,24 @@ defmodule EMQXUmbrella.MixProject do
     )
   end
 
-  ###############################################################################################
-  # END DEPRECATED FOR MIX BLOCK
-  ###############################################################################################
-
   def emqx_app_system_env(profile_info, version) do
     erlc_options(profile_info, version)
     |> dump_as_erl()
     |> then(&[{"ERL_COMPILER_OPTIONS", &1}])
   end
 
+  def emqx_app_system_env() do
+    k = {__MODULE__, :emqx_app_system_env}
+
+    get_memoized(k, fn ->
+      emqx_app_system_env(profile_info(), pkg_vsn())
+    end)
+  end
+
+  ###############################################################################################
+  # END DEPRECATED FOR MIX BLOCK
+  ###############################################################################################
+
   defp erlc_options(%{edition_type: edition_type}, version) do
     [
       :debug_info,
@@ -542,6 +569,7 @@ defmodule EMQXUmbrella.MixProject do
         } = check_profile!()
 
         base_steps = [
+          &merge_config/1,
           &make_docs/1,
           :assemble,
           &create_RELEASES/1,
@@ -780,6 +808,12 @@ defmodule EMQXUmbrella.MixProject do
   #  Custom Steps
   #############################################################################
 
+  # Gathers i18n files and merge them before producing docs and schemas.
+  defp merge_config(release) do
+    {_, 0} = System.cmd("bash", ["-c", "./scripts/merge-config.escript"])
+    release
+  end
+
   defp make_docs(release) do
     profile = System.get_env("MIX_ENV")
     os_cmd("build", [profile, "docs"])
@@ -1104,7 +1138,7 @@ defmodule EMQXUmbrella.MixProject do
   defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema
   defp emqx_schema_mod(:community), do: :emqx_conf_schema
 
-  defp jq_dep() do
+  def jq_dep() do
     if enable_jq?(),
       do: [{:jq, github: "emqx/jq", tag: "v0.3.12", override: true}],
       else: []
@@ -1267,7 +1301,8 @@ defmodule EMQXUmbrella.MixProject do
     [
       ct: &do_ct/1,
       eunit: &do_eunit/1,
-      proper: &do_proper/1
+      proper: &do_proper/1,
+      dialyzer: &do_dialyzer/1
     ]
   end
 
@@ -1278,24 +1313,25 @@ defmodule EMQXUmbrella.MixProject do
     ensure_test_mix_env!()
     set_test_env!(true)
 
-    Code.require_file("lib/mix/tasks/emqx.ct.ex")
     Mix.Task.run("emqx.ct", args)
   end
 
   defp do_eunit(args) do
     ensure_test_mix_env!()
     set_test_env!(true)
-    Code.require_file("lib/mix/tasks/emqx.eunit.ex")
     Mix.Task.run("emqx.eunit", args)
   end
 
   defp do_proper(args) do
     ensure_test_mix_env!()
     set_test_env!(true)
-    Code.require_file("lib/mix/tasks/emqx.proper.ex")
     Mix.Task.run("emqx.proper", args)
   end
 
+  defp do_dialyzer(args) do
+    Mix.Task.run("emqx.dialyzer", args)
+  end
+
   defp ensure_test_mix_env!() do
     Mix.env()
     |> to_string()