mix.exs 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963
  1. defmodule EMQXUmbrella.MixProject do
  2. use Mix.Project
  3. @moduledoc """
  4. The purpose of this file is to configure the release of EMQX under
  5. Mix. Since EMQX uses its own configuration conventions and startup
  6. procedures, one cannot simply use `iex -S mix`. Instead, it's
  7. recommended to build and use the release.
  8. ## Profiles
  9. To control the profile and edition to build, we case split on the
  10. MIX_ENV value.
  11. The following profiles are valid:
  12. * `emqx`
  13. * `emqx-enterprise`
  14. * `emqx-pkg`
  15. * `emqx-enterprise-pkg`
  16. * `dev` -> same as `emqx`, for convenience
  17. ## Release Environment Variables
  18. The release build is controlled by a few environment variables.
  19. * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
  20. tarball along with the release.
  21. """
  22. def project() do
  23. profile_info = check_profile!()
  24. version = pkg_vsn()
  25. [
  26. app: :emqx_mix,
  27. version: version,
  28. deps: deps(profile_info, version),
  29. releases: releases()
  30. ]
  31. end
  32. defp deps(profile_info, version) do
  33. # we need several overrides here because dependencies specify
  34. # other exact versions, and not ranges.
  35. [
  36. {:lc, github: "emqx/lc", tag: "0.3.2", override: true},
  37. {:redbug, github: "emqx/redbug", tag: "2.0.10"},
  38. {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true},
  39. {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
  40. {:ehttpc, github: "emqx/ehttpc", tag: "0.4.13", override: true},
  41. {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true},
  42. {:jiffy, github: "emqx/jiffy", tag: "1.0.6", override: true},
  43. {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true},
  44. {:esockd, github: "emqx/esockd", tag: "5.11.2", override: true},
  45. {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-5", override: true},
  46. {:ekka, github: "emqx/ekka", tag: "0.19.3", override: true},
  47. {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true},
  48. {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true},
  49. {:minirest, github: "emqx/minirest", tag: "1.4.3", override: true},
  50. {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true},
  51. {:replayq, github: "emqx/replayq", tag: "0.3.8", override: true},
  52. {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
  53. # maybe forbid to fetch quicer
  54. {:emqtt,
  55. github: "emqx/emqtt", tag: "1.10.1", override: true, system_env: maybe_no_quic_env()},
  56. {:rulesql, github: "emqx/rulesql", tag: "0.2.1"},
  57. {:observer_cli, "1.7.1"},
  58. {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.5"},
  59. {:telemetry, "1.1.0"},
  60. # in conflict by emqtt and hocon
  61. {:getopt, "1.0.2", override: true},
  62. {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.10", override: true},
  63. {:hocon, github: "emqx/hocon", tag: "0.42.2", override: true},
  64. {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true},
  65. {:esasl, github: "emqx/esasl", tag: "0.2.1"},
  66. {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
  67. # in conflict by ehttpc and emqtt
  68. {:gun, github: "emqx/gun", tag: "1.3.11", override: true},
  69. # in conflict by emqx_connector and system_monitor
  70. {:epgsql, github: "emqx/epgsql", tag: "4.7.1.2", override: true},
  71. # in conflict by emqx and observer_cli
  72. {:recon, github: "ferd/recon", tag: "2.5.1", override: true},
  73. {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
  74. # in conflict by erlavro and rocketmq
  75. {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true},
  76. # dependencies of dependencies; we choose specific refs to match
  77. # what rebar3 chooses.
  78. # in conflict by gun and emqtt
  79. {:cowlib,
  80. github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
  81. # in conflict by cowboy_swagger and cowboy
  82. {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true},
  83. # in conflict by grpc and eetcd
  84. {:gpb, "4.19.9", override: true, runtime: false},
  85. {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true},
  86. # set by hackney (dependency)
  87. {:ssl_verify_fun, "1.1.7", override: true},
  88. {:rfc3339, github: "emqx/rfc3339", tag: "0.2.3", override: true},
  89. {:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.2", override: true},
  90. {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true},
  91. {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true},
  92. {:ra, "2.7.3", override: true},
  93. {:mimerl, "1.2.0", override: true}
  94. ] ++
  95. emqx_apps(profile_info, version) ++
  96. enterprise_deps(profile_info) ++ jq_dep() ++ quicer_dep()
  97. end
  98. defp emqx_apps(profile_info, version) do
  99. apps = umbrella_apps(profile_info) ++ enterprise_apps(profile_info)
  100. set_emqx_app_system_env(apps, profile_info, version)
  101. end
  102. defp umbrella_apps(profile_info = %{release_type: release_type}) do
  103. enterprise_apps = enterprise_umbrella_apps(release_type)
  104. excluded_apps = excluded_apps(release_type)
  105. "apps/*"
  106. |> Path.wildcard()
  107. |> Enum.map(fn path ->
  108. app =
  109. path
  110. |> Path.basename()
  111. |> String.to_atom()
  112. {app, path: path, manager: :rebar3, override: true}
  113. end)
  114. |> Enum.reject(fn dep_spec ->
  115. dep_spec
  116. |> elem(0)
  117. |> then(&MapSet.member?(enterprise_apps, &1))
  118. end)
  119. |> Enum.reject(fn {app, _} ->
  120. case profile_info do
  121. %{edition_type: :enterprise} ->
  122. app == :emqx_telemetry
  123. _ ->
  124. false
  125. end
  126. end)
  127. |> Enum.reject(fn {app, _} -> app in excluded_apps end)
  128. end
  129. defp enterprise_apps(_profile_info = %{release_type: release_type, edition_type: :enterprise}) do
  130. Enum.map(enterprise_umbrella_apps(release_type), fn app_name ->
  131. path = "apps/#{app_name}"
  132. {app_name, path: path, manager: :rebar3, override: true}
  133. end)
  134. end
  135. defp enterprise_apps(_profile_info) do
  136. []
  137. end
  138. # need to remove those when listing `/apps/`...
  139. defp enterprise_umbrella_apps(_release_type) do
  140. MapSet.new([
  141. :emqx_connector_aggregator,
  142. :emqx_bridge_kafka,
  143. :emqx_bridge_confluent,
  144. :emqx_bridge_gcp_pubsub,
  145. :emqx_bridge_cassandra,
  146. :emqx_bridge_opents,
  147. :emqx_bridge_dynamo,
  148. :emqx_bridge_greptimedb,
  149. :emqx_bridge_hstreamdb,
  150. :emqx_bridge_influxdb,
  151. :emqx_bridge_iotdb,
  152. :emqx_bridge_es,
  153. :emqx_bridge_matrix,
  154. :emqx_bridge_mongodb,
  155. :emqx_bridge_mysql,
  156. :emqx_bridge_pgsql,
  157. :emqx_bridge_redis,
  158. :emqx_bridge_rocketmq,
  159. :emqx_bridge_tdengine,
  160. :emqx_bridge_timescale,
  161. :emqx_bridge_sqlserver,
  162. :emqx_bridge_pulsar,
  163. :emqx_oracle,
  164. :emqx_bridge_oracle,
  165. :emqx_bridge_rabbitmq,
  166. :emqx_bridge_clickhouse,
  167. :emqx_ft,
  168. :emqx_license,
  169. :emqx_s3,
  170. :emqx_bridge_s3,
  171. :emqx_schema_registry,
  172. :emqx_schema_validation,
  173. :emqx_message_transformation,
  174. :emqx_enterprise,
  175. :emqx_bridge_kinesis,
  176. :emqx_bridge_azure_event_hub,
  177. :emqx_gcp_device,
  178. :emqx_dashboard_rbac,
  179. :emqx_dashboard_sso,
  180. :emqx_audit,
  181. :emqx_gateway_gbt32960,
  182. :emqx_gateway_ocpp,
  183. :emqx_gateway_jt808,
  184. :emqx_bridge_syskeeper,
  185. :emqx_auth_ext
  186. ])
  187. end
  188. defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
  189. [
  190. {:hstreamdb_erl,
  191. github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"},
  192. {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.13", override: true},
  193. {:wolff, github: "kafka4beam/wolff", tag: "1.10.5"},
  194. {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},
  195. {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"},
  196. {:brod, github: "kafka4beam/brod", tag: "3.18.0"},
  197. {:snappyer, "1.2.9", override: true},
  198. {:crc32cer, "0.1.8", override: true},
  199. {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true},
  200. {:greptimedb,
  201. github: "GreptimeTeam/greptimedb-ingester-erl", tag: "v0.1.8", override: true},
  202. # The following two are dependencies of rabbit_common. They are needed here to
  203. # make mix not complain about conflicting versions
  204. {:thoas, github: "emqx/thoas", tag: "v1.0.0", override: true},
  205. {:credentials_obfuscation,
  206. github: "emqx/credentials-obfuscation", tag: "v3.2.0", override: true},
  207. {:rabbit_common,
  208. github: "emqx/rabbitmq-server",
  209. tag: "v3.11.13.2",
  210. sparse: "deps/rabbit_common",
  211. override: true},
  212. {:amqp_client,
  213. github: "emqx/rabbitmq-server",
  214. tag: "v3.11.13.2",
  215. sparse: "deps/amqp_client",
  216. override: true}
  217. ]
  218. end
  219. defp enterprise_deps(_profile_info) do
  220. []
  221. end
  222. defp set_emqx_app_system_env(apps, profile_info, version) do
  223. system_env = emqx_app_system_env(profile_info, version) ++ maybe_no_quic_env()
  224. Enum.map(
  225. apps,
  226. fn {app, opts} ->
  227. {app,
  228. Keyword.update(
  229. opts,
  230. :system_env,
  231. system_env,
  232. &Keyword.merge(&1, system_env)
  233. )}
  234. end
  235. )
  236. end
  237. def emqx_app_system_env(profile_info, version) do
  238. erlc_options(profile_info, version)
  239. |> dump_as_erl()
  240. |> then(&[{"ERL_COMPILER_OPTIONS", &1}])
  241. end
  242. defp erlc_options(%{edition_type: edition_type}, version) do
  243. [
  244. :debug_info,
  245. {:compile_info, [{:emqx_vsn, String.to_charlist(version)}]},
  246. {:d, :EMQX_RELEASE_EDITION, erlang_edition(edition_type)},
  247. {:d, :EMQX_ELIXIR},
  248. {:d, :snk_kind, :msg}
  249. ]
  250. end
  251. def maybe_no_quic_env() do
  252. if not enable_quicer?() do
  253. [{"BUILD_WITHOUT_QUIC", "true"}]
  254. else
  255. []
  256. end
  257. end
  258. defp releases() do
  259. [
  260. emqx: fn ->
  261. %{
  262. release_type: release_type,
  263. package_type: package_type,
  264. edition_type: edition_type
  265. } = check_profile!()
  266. base_steps = [
  267. &make_docs(&1),
  268. :assemble,
  269. &create_RELEASES/1,
  270. &copy_files(&1, release_type, package_type, edition_type),
  271. &copy_escript(&1, "nodetool"),
  272. &copy_escript(&1, "install_upgrade.escript")
  273. ]
  274. steps =
  275. if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
  276. base_steps ++ [&prepare_tar_overlays/1, :tar]
  277. else
  278. base_steps
  279. end
  280. [
  281. applications: applications(release_type, edition_type),
  282. skip_mode_validation_for: [
  283. :emqx_mix,
  284. :emqx_gateway,
  285. :emqx_gateway_stomp,
  286. :emqx_gateway_mqttsn,
  287. :emqx_gateway_coap,
  288. :emqx_gateway_lwm2m,
  289. :emqx_gateway_exproto,
  290. :emqx_dashboard,
  291. :emqx_dashboard_sso,
  292. :emqx_audit,
  293. :emqx_resource,
  294. :emqx_connector,
  295. :emqx_exhook,
  296. :emqx_bridge,
  297. :emqx_bridge_mqtt,
  298. :emqx_modules,
  299. :emqx_management,
  300. :emqx_retainer,
  301. :emqx_prometheus,
  302. :emqx_rule_engine,
  303. :emqx_auto_subscribe,
  304. :emqx_slow_subs,
  305. :emqx_plugins,
  306. :emqx_ft,
  307. :emqx_s3,
  308. :emqx_opentelemetry,
  309. :emqx_durable_storage,
  310. :rabbit_common,
  311. :emqx_eviction_agent,
  312. :emqx_node_rebalance
  313. ],
  314. steps: steps,
  315. strip_beams: false
  316. ]
  317. end
  318. ]
  319. end
  320. def applications(release_type, edition_type) do
  321. {:ok,
  322. [
  323. %{
  324. db_apps: db_apps,
  325. system_apps: system_apps,
  326. common_business_apps: common_business_apps,
  327. ee_business_apps: ee_business_apps,
  328. ce_business_apps: ce_business_apps
  329. }
  330. ]} = :file.consult("apps/emqx_machine/priv/reboot_lists.eterm")
  331. edition_specific_apps =
  332. if edition_type == :enterprise do
  333. ee_business_apps
  334. else
  335. ce_business_apps
  336. end
  337. business_apps = common_business_apps ++ edition_specific_apps
  338. excluded_apps = excluded_apps(release_type)
  339. system_apps =
  340. Enum.map(system_apps, fn app ->
  341. if is_atom(app), do: {app, :permanent}, else: app
  342. end)
  343. db_apps = Enum.map(db_apps, &{&1, :load})
  344. business_apps = Enum.map(business_apps, &{&1, :load})
  345. [system_apps, db_apps, [emqx_machine: :permanent], business_apps]
  346. |> List.flatten()
  347. |> Keyword.reject(fn {app, _type} -> app in excluded_apps end)
  348. end
  349. defp excluded_apps(_release_type) do
  350. %{
  351. mnesia_rocksdb: enable_rocksdb?(),
  352. quicer: enable_quicer?(),
  353. jq: enable_jq?(),
  354. observer: is_app?(:observer)
  355. }
  356. |> Enum.reject(&elem(&1, 1))
  357. |> Enum.map(&elem(&1, 0))
  358. end
  359. defp is_app?(name) do
  360. case Application.load(name) do
  361. :ok ->
  362. true
  363. {:error, {:already_loaded, _}} ->
  364. true
  365. _ ->
  366. false
  367. end
  368. end
  369. def check_profile!() do
  370. valid_envs = [
  371. :emqx,
  372. :"emqx-pkg",
  373. :"emqx-enterprise",
  374. :"emqx-enterprise-pkg"
  375. ]
  376. if Mix.env() == :dev do
  377. env_profile = System.get_env("PROFILE")
  378. if env_profile do
  379. # copy from PROFILE env var
  380. System.get_env("PROFILE")
  381. |> String.to_atom()
  382. |> Mix.env()
  383. else
  384. IO.puts(
  385. IO.ANSI.format([
  386. :yellow,
  387. "Warning: env var PROFILE is unset; defaulting to emqx"
  388. ])
  389. )
  390. Mix.env(:emqx)
  391. end
  392. end
  393. if Mix.env() not in valid_envs do
  394. formatted_envs =
  395. valid_envs
  396. |> Enum.map(&" * #{&1}")
  397. |> Enum.join("\n")
  398. Mix.raise("""
  399. Invalid env #{Mix.env()}. Valid options are:
  400. #{formatted_envs}
  401. """)
  402. end
  403. {
  404. release_type,
  405. package_type,
  406. edition_type
  407. } =
  408. case Mix.env() do
  409. :dev ->
  410. {:standard, :bin, :community}
  411. :emqx ->
  412. {:standard, :bin, :community}
  413. :"emqx-enterprise" ->
  414. {:standard, :bin, :enterprise}
  415. :"emqx-pkg" ->
  416. {:standard, :pkg, :community}
  417. :"emqx-enterprise-pkg" ->
  418. {:standard, :pkg, :enterprise}
  419. end
  420. normalize_env!()
  421. %{
  422. release_type: release_type,
  423. package_type: package_type,
  424. edition_type: edition_type
  425. }
  426. end
  427. #############################################################################
  428. # Custom Steps
  429. #############################################################################
  430. defp make_docs(release) do
  431. profile = System.get_env("MIX_ENV")
  432. os_cmd("build", [profile, "docs"])
  433. release
  434. end
  435. defp copy_files(release, release_type, package_type, edition_type) do
  436. overwrite? = Keyword.get(release.options, :overwrite, false)
  437. bin = Path.join(release.path, "bin")
  438. etc = Path.join(release.path, "etc")
  439. log = Path.join(release.path, "log")
  440. plugins = Path.join(release.path, "plugins")
  441. Mix.Generator.create_directory(bin)
  442. Mix.Generator.create_directory(etc)
  443. Mix.Generator.create_directory(log)
  444. Mix.Generator.create_directory(plugins)
  445. Mix.Generator.create_directory(Path.join(etc, "certs"))
  446. Enum.each(
  447. ["mnesia", "configs", "patches", "scripts"],
  448. fn dir ->
  449. path = Path.join([release.path, "data", dir])
  450. Mix.Generator.create_directory(path)
  451. end
  452. )
  453. Mix.Generator.copy_file(
  454. "apps/emqx_auth/etc/acl.conf",
  455. Path.join(etc, "acl.conf"),
  456. force: overwrite?
  457. )
  458. # required by emqx_auth
  459. File.cp_r!(
  460. "apps/emqx/etc/certs",
  461. Path.join(etc, "certs")
  462. )
  463. profile = System.get_env("MIX_ENV")
  464. File.cp_r!(
  465. "rel/config/examples",
  466. Path.join(etc, "examples"),
  467. force: overwrite?
  468. )
  469. # copy /rel/config/ee-examples if profile is enterprise
  470. case profile do
  471. "emqx-enterprise" ->
  472. File.cp_r!(
  473. "rel/config/ee-examples",
  474. Path.join(etc, "examples"),
  475. force: overwrite?
  476. )
  477. _ ->
  478. :ok
  479. end
  480. # this is required by the produced escript / nodetool
  481. Mix.Generator.copy_file(
  482. Path.join(release.version_path, "start_clean.boot"),
  483. Path.join(bin, "no_dot_erlang.boot"),
  484. force: overwrite?
  485. )
  486. assigns = template_vars(release, release_type, package_type, edition_type)
  487. # This is generated by `scripts/merge-config.escript` or `make merge-config`
  488. # So, this should be run before the release.
  489. # TODO: run as a "compiler" step???
  490. render_template(
  491. "apps/emqx_conf/etc/emqx.conf.all",
  492. assigns,
  493. Path.join(etc, "emqx.conf")
  494. )
  495. render_template(
  496. "rel/emqx_vars",
  497. assigns,
  498. Path.join([release.path, "releases", "emqx_vars"])
  499. )
  500. vm_args_template_path =
  501. case release_type do
  502. _ ->
  503. "apps/emqx/etc/vm.args.cloud"
  504. end
  505. render_template(
  506. vm_args_template_path,
  507. assigns,
  508. [
  509. Path.join(etc, "vm.args"),
  510. Path.join(release.version_path, "vm.args")
  511. ]
  512. )
  513. for name <- [
  514. "emqx",
  515. "emqx_ctl"
  516. ] do
  517. Mix.Generator.copy_file(
  518. "bin/#{name}",
  519. Path.join(bin, name),
  520. force: overwrite?
  521. )
  522. # Files with the version appended are expected by the release
  523. # upgrade script `install_upgrade.escript`
  524. Mix.Generator.copy_file(
  525. Path.join(bin, name),
  526. Path.join(bin, name <> "-#{release.version}"),
  527. force: overwrite?
  528. )
  529. end
  530. for base_name <- ["emqx", "emqx_ctl"],
  531. suffix <- ["", "-#{release.version}"] do
  532. name = base_name <> suffix
  533. File.chmod!(Path.join(bin, name), 0o755)
  534. end
  535. Mix.Generator.copy_file(
  536. "bin/node_dump",
  537. Path.join(bin, "node_dump"),
  538. force: overwrite?
  539. )
  540. File.chmod!(Path.join(bin, "node_dump"), 0o755)
  541. Mix.Generator.copy_file(
  542. "bin/emqx_cluster_rescue",
  543. Path.join(bin, "emqx_cluster_rescue"),
  544. force: overwrite?
  545. )
  546. File.chmod!(Path.join(bin, "emqx_cluster_rescue"), 0o755)
  547. render_template(
  548. "rel/BUILD_INFO",
  549. assigns,
  550. Path.join(release.version_path, "BUILD_INFO")
  551. )
  552. release
  553. end
  554. defp render_template(template, assigns, target) when is_binary(target) do
  555. render_template(template, assigns, [target])
  556. end
  557. defp render_template(template, assigns, tartgets) when is_list(tartgets) do
  558. rendered =
  559. File.read!(template)
  560. |> from_rebar_to_eex_template()
  561. |> EEx.eval_string(assigns)
  562. for target <- tartgets do
  563. File.write!(target, rendered)
  564. end
  565. end
  566. # needed by nodetool and by release_handler
  567. defp create_RELEASES(release) do
  568. apps =
  569. Enum.map(release.applications, fn {app_name, app_props} ->
  570. app_vsn = Keyword.fetch!(app_props, :vsn)
  571. app_path =
  572. "./lib"
  573. |> Path.join("#{app_name}-#{app_vsn}")
  574. |> to_charlist()
  575. {app_name, app_vsn, app_path}
  576. end)
  577. release_entry = [
  578. {
  579. :release,
  580. to_charlist(release.name),
  581. to_charlist(release.version),
  582. release.erts_version,
  583. apps,
  584. :permanent
  585. }
  586. ]
  587. release.path
  588. |> Path.join("releases")
  589. |> Path.join("RELEASES")
  590. |> File.open!([:write, :utf8], fn handle ->
  591. IO.puts(handle, "%% coding: utf-8")
  592. :io.format(handle, ~c"~tp.~n", [release_entry])
  593. end)
  594. release
  595. end
  596. defp copy_escript(release, escript_name) do
  597. [shebang, rest] =
  598. "bin/#{escript_name}"
  599. |> File.read!()
  600. |> String.split("\n", parts: 2)
  601. # the elixir version of escript + start.boot required the boot_var
  602. # RELEASE_LIB to be defined.
  603. # enable-feature is not required when 1.6.x
  604. boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib -enable-feature maybe_expr"
  605. # Files with the version appended are expected by the release
  606. # upgrade script `install_upgrade.escript`
  607. Enum.each(
  608. [escript_name, escript_name <> "-" <> release.version],
  609. fn name ->
  610. path = Path.join([release.path, "bin", name])
  611. File.write!(path, [shebang, "\n", boot_var, "\n", rest])
  612. end
  613. )
  614. release
  615. end
  616. # The `:tar` built-in step in Mix Release does not currently add the
  617. # `etc` directory into the resulting tarball. The workaround is to
  618. # add those to the `:overlays` key before running `:tar`.
  619. # See: https://hexdocs.pm/mix/1.13.4/Mix.Release.html#__struct__/0
  620. defp prepare_tar_overlays(release) do
  621. Map.update!(
  622. release,
  623. :overlays,
  624. &[
  625. "etc",
  626. "data",
  627. "plugins",
  628. "bin/node_dump"
  629. | &1
  630. ]
  631. )
  632. end
  633. #############################################################################
  634. # Helper functions
  635. #############################################################################
  636. defp template_vars(release, release_type, :bin = _package_type, edition_type) do
  637. [
  638. emqx_default_erlang_cookie: default_cookie(),
  639. emqx_configuration_doc: emqx_configuration_doc(edition_type, :root),
  640. emqx_configuration_doc_log: emqx_configuration_doc(edition_type, :log),
  641. platform_data_dir: "data",
  642. platform_etc_dir: "etc",
  643. platform_plugins_dir: "plugins",
  644. runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
  645. emqx_etc_dir: "$RUNNER_ROOT_DIR/etc",
  646. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  647. runner_log_dir: "$RUNNER_ROOT_DIR/log",
  648. runner_user: "",
  649. release_version: release.version,
  650. erts_vsn: release.erts_version,
  651. # FIXME: this is empty in `make emqx` ???
  652. erl_opts: "",
  653. emqx_description: emqx_description(release_type, edition_type),
  654. emqx_schema_mod: emqx_schema_mod(edition_type),
  655. is_elixir: "yes",
  656. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  657. ] ++ build_info()
  658. end
  659. defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
  660. [
  661. emqx_default_erlang_cookie: default_cookie(),
  662. emqx_configuration_doc: emqx_configuration_doc(edition_type, :root),
  663. emqx_configuration_doc_log: emqx_configuration_doc(edition_type, :log),
  664. platform_data_dir: "/var/lib/emqx",
  665. platform_etc_dir: "/etc/emqx",
  666. platform_plugins_dir: "/var/lib/emqx/plugins",
  667. runner_bin_dir: "/usr/bin",
  668. emqx_etc_dir: "/etc/emqx",
  669. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  670. runner_log_dir: "/var/log/emqx",
  671. runner_user: "emqx",
  672. release_version: release.version,
  673. erts_vsn: release.erts_version,
  674. # FIXME: this is empty in `make emqx` ???
  675. erl_opts: "",
  676. emqx_description: emqx_description(release_type, edition_type),
  677. emqx_schema_mod: emqx_schema_mod(edition_type),
  678. is_elixir: "yes",
  679. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  680. ] ++ build_info()
  681. end
  682. defp default_cookie() do
  683. "emqx50elixir"
  684. end
  685. defp emqx_description(release_type, edition_type) do
  686. case {release_type, edition_type} do
  687. {_, :enterprise} ->
  688. "EMQX Enterprise"
  689. {_, :community} ->
  690. "EMQX"
  691. end
  692. end
  693. defp emqx_configuration_doc(:enterprise, :root),
  694. do: "https://docs.emqx.com/en/enterprise/latest/configuration/configuration.html"
  695. defp emqx_configuration_doc(:enterprise, :log),
  696. do: "https://docs.emqx.com/en/enterprise/latest/configuration/logs.html"
  697. defp emqx_configuration_doc(:community, :root),
  698. do: "https://www.emqx.io/docs/en/latest/configuration/configuration.html"
  699. defp emqx_configuration_doc(:community, :log),
  700. do: "https://www.emqx.io/docs/en/latest/configuration/logs.html"
  701. defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema
  702. defp emqx_schema_mod(:community), do: :emqx_conf_schema
  703. defp jq_dep() do
  704. if enable_jq?(),
  705. do: [{:jq, github: "emqx/jq", tag: "v0.3.12", override: true}],
  706. else: []
  707. end
  708. defp quicer_dep() do
  709. if enable_quicer?(),
  710. # in conflict with emqx and emqtt
  711. do: [{:quicer, github: "emqx/quic", tag: "0.0.313", override: true}],
  712. else: []
  713. end
  714. defp enable_jq?() do
  715. not Enum.any?([
  716. build_without_jq?()
  717. ])
  718. end
  719. defp enable_quicer?() do
  720. "1" == System.get_env("BUILD_WITH_QUIC") or
  721. not Enum.any?([
  722. macos?(),
  723. build_without_quic?()
  724. ])
  725. end
  726. defp enable_rocksdb?() do
  727. not Enum.any?([
  728. raspbian?(),
  729. build_without_rocksdb?()
  730. ])
  731. end
  732. defp pkg_vsn() do
  733. %{edition_type: edition_type} = check_profile!()
  734. basedir = Path.dirname(__ENV__.file)
  735. script = Path.join(basedir, "pkg-vsn.sh")
  736. os_cmd(script, [Atom.to_string(edition_type)])
  737. end
  738. defp os_cmd(script, args) do
  739. {str, 0} = System.cmd("bash", [script | args])
  740. String.trim(str)
  741. end
  742. defp macos?() do
  743. {:unix, :darwin} == :os.type()
  744. end
  745. defp raspbian?() do
  746. os_cmd("./scripts/get-distro.sh", []) =~ "raspbian"
  747. end
  748. defp build_without_jq?() do
  749. opt = System.get_env("BUILD_WITHOUT_JQ", "false")
  750. String.downcase(opt) != "false"
  751. end
  752. defp build_without_quic?() do
  753. opt = System.get_env("BUILD_WITHOUT_QUIC", "false")
  754. String.downcase(opt) != "false"
  755. end
  756. defp build_without_rocksdb?() do
  757. opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false")
  758. String.downcase(opt) != "false"
  759. end
  760. defp from_rebar_to_eex_template(str) do
  761. # we must not consider surrounding space in the template var name
  762. # because some help strings contain informative variables that
  763. # should not be interpolated, and those have no spaces.
  764. Regex.replace(
  765. ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/,
  766. str,
  767. "<%= \\g{1} %>"
  768. )
  769. end
  770. defp build_info() do
  771. [
  772. build_info_arch: to_string(:erlang.system_info(:system_architecture)),
  773. build_info_wordsize: wordsize(),
  774. build_info_os: os_cmd("./scripts/get-distro.sh", []),
  775. build_info_erlang: otp_release(),
  776. build_info_elixir: System.version(),
  777. build_info_relform: System.get_env("EMQX_REL_FORM", "tgz")
  778. ]
  779. end
  780. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142
  781. defp wordsize() do
  782. size =
  783. try do
  784. :erlang.system_info({:wordsize, :external})
  785. rescue
  786. ErlangError ->
  787. :erlang.system_info(:wordsize)
  788. end
  789. to_string(8 * size)
  790. end
  791. defp normalize_env!() do
  792. env =
  793. case Mix.env() do
  794. :dev ->
  795. :emqx
  796. env ->
  797. env
  798. end
  799. Mix.env(env)
  800. end
  801. # As from Erlang/OTP 17, the OTP release number corresponds to the
  802. # major OTP version number. No erlang:system_info() argument gives
  803. # the exact OTP version.
  804. # https://www.erlang.org/doc/man/erlang.html#system_info_otp_release
  805. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577
  806. defp otp_release() do
  807. major_version = System.otp_release()
  808. root_dir = to_string(:code.root_dir())
  809. [root_dir, "releases", major_version, "OTP_VERSION"]
  810. |> Path.join()
  811. |> File.read()
  812. |> case do
  813. {:error, _} ->
  814. major_version
  815. {:ok, version} ->
  816. version
  817. |> String.trim()
  818. |> String.split("**")
  819. |> List.first()
  820. end
  821. end
  822. defp dump_as_erl(term) do
  823. term
  824. |> then(&:io_lib.format("~0p", [&1]))
  825. |> :erlang.iolist_to_binary()
  826. end
  827. defp erlang_edition(:community), do: :ce
  828. defp erlang_edition(:enterprise), do: :ee
  829. end