emqx4.3/mix.exs

defmodule EMQXUmbrella.MixProject do
  use Mix.Project

  @moduledoc """

  The purpose of this file is to configure the release of EMQX under
  Mix.  Since EMQX uses its own configuration conventions and startup
  procedures, one cannot simply use `iex -S mix`.  Instead, it's
  recommendd to build and use the release.

  ## Release Environment Variables

  The release build is controlled by a few environment variables.

    * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
      tarball along with the release.
    * `EMQX_RELEASE_TYPE` - Must be one of `cloud | edge`.  Controls a
      few dependencies and the `vm.args` to be used.  Defaults to
      `cloud`.
    * `EMQX_PACKAGE_TYPE` - Must be one of `bin | pkg`.  Controls
      whether the build is intended for direct usage or for packaging.
      Defaults to `bin`.
    * `EMQX_EDITION_TYPE` - Must be one of `community | enterprise`.
      Defaults to `community`.
  """

  def project() do
    [
      app: :emqx_mix,
      version: pkg_vsn(),
      deps: deps(),
      releases: releases()
    ]
  end

  defp deps() do
    # we need several overrides here because dependencies specify
    # other exact versions, and not ranges.
    [
      {:lc, github: "qzhuyan/lc", tag: "0.1.2"},
      {:redbug, "2.0.7"},
      {:typerefl, github: "k32/typerefl", tag: "0.8.6", override: true},
      {:ehttpc, github: "emqx/ehttpc", tag: "0.1.12"},
      {:gproc, github: "uwiger/gproc", tag: "0.8.0", override: true},
      {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
      {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
      {:esockd, github: "emqx/esockd", tag: "5.9.0", override: true},
      {:mria, github: "emqx/mria", tag: "0.1.5", override: true},
      {:ekka, github: "emqx/ekka", tag: "0.11.3", override: true},
      {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.0", override: true},
      {:minirest, github: "emqx/minirest", tag: "1.2.10", override: true},
      {:ecpool, github: "emqx/ecpool", tag: "0.5.2"},
      {:replayq, "0.3.3", override: true},
      {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
      {:emqtt, github: "emqx/emqtt", tag: "1.4.3", override: true},
      {:rulesql, github: "emqx/rulesql", tag: "0.1.4"},
      {:observer_cli, "1.7.1"},
      {:system_monitor, github: "k32/system_monitor", tag: "2.2.1"},
      # in conflict by emqtt and hocon
      {:getopt, "1.0.2", override: true},
      {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "0.16.0", override: true},
      {:hocon, github: "emqx/hocon", tag: "0.23.0", override: true},
      {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.4.1", override: true},
      {:esasl, github: "emqx/esasl", tag: "0.2.0"},
      {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
      # in conflict by ehttpc and emqtt
      {:gun, github: "emqx/gun", tag: "1.3.6", override: true},
      # in conflict by emqx_connectior and system_monitor
      {:epgsql, github: "emqx/epgsql", tag: "4.7-emqx.2", override: true},
      # in conflict by mongodb and eredis_cluster
      {:poolboy, github: "emqx/poolboy", tag: "1.5.2", override: true},
      # in conflict by emqx and observer_cli
      {:recon, github: "ferd/recon", tag: "2.5.1", override: true},
      {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
      # dependencies of dependencies; we choose specific refs to match
      # what rebar3 chooses.
      # in conflict by gun and emqtt
      {:cowlib,
       github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
      # in conflict by cowboy_swagger and cowboy
      {:ranch,
       github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
      # in conflict by grpc and eetcd
      {:gpb, "4.11.2", override: true}
    ] ++ umbrella_apps() ++ bcrypt_dep() ++ quicer_dep()
  end

  defp umbrella_apps() do
    "apps/*"
    |> Path.wildcard()
    |> Enum.map(fn path ->
      app =
        path
        |> String.trim_leading("apps/")
        |> String.to_atom()

      {app, path: path, manager: :rebar3, override: true}
    end)
  end

  defp releases() do
    [
      emqx: fn ->
        %{
          release_type: release_type,
          package_type: package_type,
          edition_type: edition_type
        } = read_inputs()

        base_steps = [
          :assemble,
          &create_RELEASES/1,
          &copy_files(&1, release_type, package_type, edition_type),
          &copy_escript(&1, "nodetool"),
          &copy_escript(&1, "install_upgrade.escript")
        ]

        steps =
          if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
            base_steps ++ [&prepare_tar_overlays/1, :tar]
          else
            base_steps
          end

        [
          applications: applications(release_type),
          skip_mode_validation_for: [
            :emqx_gateway,
            :emqx_dashboard,
            :emqx_resource,
            :emqx_connector,
            :emqx_exhook,
            :emqx_bridge,
            :emqx_modules,
            :emqx_management,
            :emqx_statsd,
            :emqx_retainer,
            :emqx_prometheus,
            :emqx_plugins
          ],
          steps: steps,
          strip_beams: false
        ]
      end
    ]
  end

  def applications(release_type) do
    [
      crypto: :permanent,
      public_key: :permanent,
      asn1: :permanent,
      syntax_tools: :permanent,
      ssl: :permanent,
      os_mon: :permanent,
      inets: :permanent,
      compiler: :permanent,
      runtime_tools: :permanent,
      redbug: :permanent,
      hocon: :load,
      emqx: :load,
      emqx_conf: :load,
      emqx_machine: :permanent,
      mria: :load,
      mnesia: :load,
      ekka: :load,
      emqx_plugin_libs: :load,
      esasl: :load,
      observer_cli: :permanent,
      system_monitor: :permanent,
      emqx_http_lib: :permanent,
      emqx_resource: :permanent,
      emqx_connector: :permanent,
      emqx_authn: :permanent,
      emqx_authz: :permanent,
      emqx_auto_subscribe: :permanent,
      emqx_gateway: :permanent,
      emqx_exhook: :permanent,
      emqx_bridge: :permanent,
      emqx_rule_engine: :permanent,
      emqx_modules: :permanent,
      emqx_management: :permanent,
      emqx_dashboard: :permanent,
      emqx_retainer: :permanent,
      emqx_statsd: :permanent,
      emqx_prometheus: :permanent,
      emqx_psk: :permanent,
      emqx_slow_subs: :permanent,
      emqx_plugins: :permanent,
      emqx_mix: :none
    ] ++
      if(enable_quicer?(), do: [quicer: :permanent], else: []) ++
      if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++
      if(release_type == :cloud,
        do: [xmerl: :permanent, observer: :load],
        else: []
      )
  end

  defp read_inputs() do
    release_type =
      read_enum_env_var(
        "EMQX_RELEASE_TYPE",
        [:cloud, :edge],
        :cloud
      )

    package_type =
      read_enum_env_var(
        "EMQX_PACKAGE_TYPE",
        [:bin, :pkg],
        :bin
      )

    edition_type =
      read_enum_env_var(
        "EMQX_EDITION_TYPE",
        [:community, :enterprise],
        :community
      )

    %{
      release_type: release_type,
      package_type: package_type,
      edition_type: edition_type
    }
  end

  #############################################################################
  #  Custom Steps
  #############################################################################

  defp copy_files(release, release_type, package_type, edition_type) do
    overwrite? = Keyword.get(release.options, :overwrite, false)

    bin = Path.join(release.path, "bin")
    etc = Path.join(release.path, "etc")
    log = Path.join(release.path, "log")

    Mix.Generator.create_directory(bin)
    Mix.Generator.create_directory(etc)
    Mix.Generator.create_directory(log)
    Mix.Generator.create_directory(Path.join(etc, "certs"))

    Enum.each(
      ["mnesia", "configs", "patches", "scripts"],
      fn dir ->
        path = Path.join([release.path, "data", dir])
        Mix.Generator.create_directory(path)
      end
    )

    Mix.Generator.copy_file(
      "apps/emqx_authz/etc/acl.conf",
      Path.join(etc, "acl.conf"),
      force: overwrite?
    )

    # required by emqx_authz
    File.cp_r!(
      "apps/emqx/etc/certs",
      Path.join(etc, "certs")
    )

    # this is required by the produced escript / nodetool
    Mix.Generator.copy_file(
      Path.join(release.version_path, "start_clean.boot"),
      Path.join(bin, "no_dot_erlang.boot"),
      force: overwrite?
    )

    assigns = template_vars(release, release_type, package_type, edition_type)

    # This is generated by `scripts/merge-config.escript` or `make
    # conf-segs`.  So, this should be run before the release.
    # TODO: run as a "compiler" step???
    conf_rendered =
      File.read!("apps/emqx_conf/etc/emqx.conf.all")
      |> from_rebar_to_eex_template()
      |> EEx.eval_string(assigns)

    File.write!(
      Path.join(etc, "emqx.conf"),
      conf_rendered
    )

    vars_rendered =
      File.read!("rel/emqx_vars")
      |> from_rebar_to_eex_template()
      |> EEx.eval_string(assigns)

    File.write!(
      Path.join([release.path, "releases", "emqx_vars"]),
      vars_rendered
    )

    vm_args_template_path =
      case release_type do
        :cloud ->
          "apps/emqx/etc/emqx_cloud/vm.args"

        :edge ->
          "apps/emqx/etc/emqx_edge/vm.args"
      end

    vm_args_rendered =
      File.read!(vm_args_template_path)
      |> from_rebar_to_eex_template()
      |> EEx.eval_string(assigns)

    File.write!(
      Path.join(etc, "vm.args"),
      vm_args_rendered
    )

    File.write!(
      Path.join(release.version_path, "vm.args"),
      vm_args_rendered
    )

    for name <- [
          "emqx",
          "emqx_ctl"
        ] do
      Mix.Generator.copy_file(
        "bin/#{name}",
        Path.join(bin, name),
        force: overwrite?
      )

      # Files with the version appended are expected by the release
      # upgrade script `install_upgrade.escript`
      Mix.Generator.copy_file(
        Path.join(bin, name),
        Path.join(bin, name <> "-#{release.version}"),
        force: overwrite?
      )
    end

    for base_name <- ["emqx", "emqx_ctl"],
        suffix <- ["", "-#{release.version}"] do
      name = base_name <> suffix
      File.chmod!(Path.join(bin, name), 0o755)
    end

    built_on_rendered =
      File.read!("rel/BUILT_ON")
      |> from_rebar_to_eex_template()
      |> EEx.eval_string(assigns)

    File.write!(
      Path.join([release.version_path, "BUILT_ON"]),
      built_on_rendered
    )

    release
  end

  # needed by nodetool and by release_handler
  defp create_RELEASES(release) do
    apps =
      Enum.map(release.applications, fn {app_name, app_props} ->
        app_vsn = Keyword.fetch!(app_props, :vsn)

        app_path =
          "./lib"
          |> Path.join("#{app_name}-#{app_vsn}")
          |> to_charlist()

        {app_name, app_vsn, app_path}
      end)

    release_entry = [
      {
        :release,
        to_charlist(release.name),
        to_charlist(release.version),
        release.erts_version,
        apps,
        :permanent
      }
    ]

    release.path
    |> Path.join("releases")
    |> Path.join("RELEASES")
    |> File.open!([:write, :utf8], fn handle ->
      IO.puts(handle, "%% coding: utf-8")
      :io.format(handle, '~tp.~n', [release_entry])
    end)

    release
  end

  defp copy_escript(release, escript_name) do
    [shebang, rest] =
      "bin/#{escript_name}"
      |> File.read!()
      |> String.split("\n", parts: 2)

    # the elixir version of escript + start.boot required the boot_var
    # RELEASE_LIB to be defined.
    boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib"

    # Files with the version appended are expected by the release
    # upgrade script `install_upgrade.escript`
    Enum.each(
      [escript_name, escript_name <> "-" <> release.version],
      fn name ->
        path = Path.join([release.path, "bin", name])
        File.write!(path, [shebang, "\n", boot_var, "\n", rest])
      end
    )

    release
  end

  # The `:tar` built-in step in Mix Release does not currently add the
  # `etc` directory into the resulting tarball.  The workaround is to
  # add those to the `:overlays` key before running `:tar`.
  # See: https://hexdocs.pm/mix/1.13.2/Mix.Release.html#__struct__/0
  defp prepare_tar_overlays(release) do
    Map.update!(release, :overlays, &["etc", "data" | &1])
  end

  #############################################################################
  #  Helper functions
  #############################################################################

  defp template_vars(release, release_type, :bin = _package_type, edition_type) do
    [
      platform_bin_dir: "bin",
      platform_data_dir: "data",
      platform_etc_dir: "etc",
      platform_lib_dir: "lib",
      platform_log_dir: "log",
      platform_plugins_dir: "plugins",
      runner_root_dir: "$(cd $(dirname $(readlink $0 || echo $0))/..; pwd -P)",
      runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
      runner_etc_dir: "$RUNNER_ROOT_DIR/etc",
      runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
      runner_log_dir: "$RUNNER_ROOT_DIR/log",
      runner_data_dir: "$RUNNER_ROOT_DIR/data",
      runner_user: "",
      release_version: release.version,
      erts_vsn: release.erts_version,
      # FIXME: this is empty in `make emqx` ???
      erl_opts: "",
      emqx_description: emqx_description(release_type, edition_type),
      built_on_arch: built_on(),
      is_elixir: "yes"
    ]
  end

  defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
    [
      platform_bin_dir: "",
      platform_data_dir: "/var/lib/emqx",
      platform_etc_dir: "/etc/emqx",
      platform_lib_dir: "",
      platform_log_dir: "/var/log/emqx",
      platform_plugins_dir: "/var/lib/emqx/plugins",
      runner_root_dir: "/usr/lib/emqx",
      runner_bin_dir: "/usr/bin",
      runner_etc_dir: "/etc/emqx",
      runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
      runner_log_dir: "/var/log/emqx",
      runner_data_dir: "/var/lib/emqx",
      runner_user: "emqx",
      release_version: release.version,
      erts_vsn: release.erts_version,
      # FIXME: this is empty in `make emqx` ???
      erl_opts: "",
      emqx_description: emqx_description(release_type, edition_type),
      built_on_arch: built_on(),
      is_elixir: "yes"
    ]
  end

  defp read_enum_env_var(env_var, allowed_values, default_value) do
    case System.fetch_env(env_var) do
      :error ->
        default_value

      {:ok, raw_value} ->
        value =
          raw_value
          |> String.downcase()
          |> String.to_atom()

        if value not in allowed_values do
          Mix.raise("""
          Invalid value #{raw_value} for variable #{env_var}.
          Allowed values are: #{inspect(allowed_values)}
          """)
        end

        value
    end
  end

  defp emqx_description(release_type, edition_type) do
    case {release_type, edition_type} do
      {:cloud, :enterprise} ->
        "EMQ X Enterprise Edition"

      {:cloud, :community} ->
        "EMQ X Community Edition"

      {:edge, :community} ->
        "EMQ X Edge Edition"
    end
  end

  defp bcrypt_dep() do
    if enable_bcrypt?(),
      do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0", override: true}],
      else: []
  end

  defp quicer_dep() do
    if enable_quicer?(),
      # in conflict with emqx and emqtt
      do: [{:quicer, github: "emqx/quic", tag: "0.0.9", override: true}],
      else: []
  end

  defp enable_bcrypt?() do
    not win32?()
  end

  defp enable_quicer?() do
    not Enum.any?([
      build_without_quic?(),
      win32?(),
      centos6?()
    ])
  end

  defp pkg_vsn() do
    %{edition_type: edition_type} = read_inputs()
    basedir = Path.dirname(__ENV__.file)
    script = Path.join(basedir, "pkg-vsn.sh")
    {str_vsn, 0} = System.cmd(script, [Atom.to_string(edition_type)])

    String.trim(str_vsn)
  end

  defp win32?(),
    do: match?({:win_32, _}, :os.type())

  defp centos6?() do
    case File.read("/etc/centos-release") do
      {:ok, "CentOS release 6" <> _} ->
        true

      _ ->
        false
    end
  end

  defp build_without_quic?() do
    opt = System.get_env("BUILD_WITHOUT_QUIC", "false")

    String.downcase(opt) != "false"
  end

  defp from_rebar_to_eex_template(str) do
    # we must not consider surrounding space in the template var name
    # because some help strings contain informative variables that
    # should not be interpolated, and those have no spaces.
    Regex.replace(
      ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/,
      str,
      "<%= \\g{1} %>"
    )
  end

  defp built_on() do
    system_architecture = to_string(:erlang.system_info(:system_architecture))
    elixir_version = System.version()
    words = wordsize()

    "#{elixir_version}-#{otp_release()}-#{system_architecture}-#{words}"
  end

  # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142
  defp wordsize() do
    size =
      try do
        :erlang.system_info({:wordsize, :external})
      rescue
        ErlangError ->
          :erlang.system_info(:wordsize)
      end

    to_string(8 * size)
  end

  # As from Erlang/OTP 17, the OTP release number corresponds to the
  # major OTP version number. No erlang:system_info() argument gives
  # the exact OTP version.
  # https://www.erlang.org/doc/man/erlang.html#system_info_otp_release
  # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577
  defp otp_release() do
    major_version = System.otp_release()
    root_dir = to_string(:code.root_dir())

    [root_dir, "releases", major_version, "OTP_VERSION"]
    |> Path.join()
    |> File.read()
    |> case do
      {:error, _} ->
        major_version

      {:ok, version} ->
        version
        |> String.trim()
        |> String.split("**")
        |> List.first()
    end
  end
end