mix.exs 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960
  1. defmodule EMQXUmbrella.MixProject do
  2. use Mix.Project
  3. @moduledoc """
  4. The purpose of this file is to configure the release of EMQX under
  5. Mix. Since EMQX uses its own configuration conventions and startup
  6. procedures, one cannot simply use `iex -S mix`. Instead, it's
  7. recommended to build and use the release.
  8. ## Profiles
  9. To control the profile and edition to build, we case split on the
  10. MIX_ENV value.
  11. The following profiles are valid:
  12. * `emqx`
  13. * `emqx-enterprise`
  14. * `emqx-pkg`
  15. * `emqx-enterprise-pkg`
  16. * `dev` -> same as `emqx`, for convenience
  17. ## Release Environment Variables
  18. The release build is controlled by a few environment variables.
  19. * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
  20. tarball along with the release.
  21. """
  22. def project() do
  23. profile_info = check_profile!()
  24. version = pkg_vsn()
  25. [
  26. app: :emqx_mix,
  27. version: version,
  28. deps: deps(profile_info, version),
  29. releases: releases()
  30. ]
  31. end
  32. defp deps(profile_info, version) do
  33. # we need several overrides here because dependencies specify
  34. # other exact versions, and not ranges.
  35. [
  36. {:lc, github: "emqx/lc", tag: "0.3.2", override: true},
  37. {:redbug, github: "emqx/redbug", tag: "2.0.10"},
  38. {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true},
  39. {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
  40. {:ehttpc, github: "emqx/ehttpc", tag: "0.4.13", override: true},
  41. {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true},
  42. {:jiffy, github: "emqx/jiffy", tag: "1.0.6", override: true},
  43. {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true},
  44. {:esockd, github: "emqx/esockd", tag: "5.11.2", override: true},
  45. {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-2", override: true},
  46. {:ekka, github: "emqx/ekka", tag: "0.19.3", override: true},
  47. {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true},
  48. {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true},
  49. {:minirest, github: "emqx/minirest", tag: "1.4.0", override: true},
  50. {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true},
  51. {:replayq, github: "emqx/replayq", tag: "0.3.8", override: true},
  52. {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
  53. # maybe forbid to fetch quicer
  54. {:emqtt,
  55. github: "emqx/emqtt", tag: "1.10.1", override: true, system_env: maybe_no_quic_env()},
  56. {:rulesql, github: "emqx/rulesql", tag: "0.2.1"},
  57. {:observer_cli, "1.7.1"},
  58. {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
  59. {:telemetry, "1.1.0"},
  60. # in conflict by emqtt and hocon
  61. {:getopt, "1.0.2", override: true},
  62. {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true},
  63. {:hocon, github: "emqx/hocon", tag: "0.42.2", override: true},
  64. {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true},
  65. {:esasl, github: "emqx/esasl", tag: "0.2.1"},
  66. {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
  67. # in conflict by ehttpc and emqtt
  68. {:gun, github: "emqx/gun", tag: "1.3.11", override: true},
  69. # in conflict by emqx_connector and system_monitor
  70. {:epgsql, github: "emqx/epgsql", tag: "4.7.1.1", override: true},
  71. # in conflict by emqx and observer_cli
  72. {:recon, github: "ferd/recon", tag: "2.5.1", override: true},
  73. {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
  74. # in conflict by erlavro and rocketmq
  75. {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true},
  76. # dependencies of dependencies; we choose specific refs to match
  77. # what rebar3 chooses.
  78. # in conflict by gun and emqtt
  79. {:cowlib,
  80. github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
  81. # in conflict by cowboy_swagger and cowboy
  82. {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true},
  83. # in conflict by grpc and eetcd
  84. {:gpb, "4.19.9", override: true, runtime: false},
  85. {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true},
  86. # set by hackney (dependency)
  87. {:ssl_verify_fun, "1.1.7", override: true},
  88. {:rfc3339, github: "emqx/rfc3339", tag: "0.2.3", override: true},
  89. {:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.2", override: true},
  90. {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true},
  91. {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true},
  92. {:ra, "2.7.3", override: true},
  93. {:mimerl, "1.2.0", override: true}
  94. ] ++
  95. emqx_apps(profile_info, version) ++
  96. enterprise_deps(profile_info) ++ jq_dep() ++ quicer_dep()
  97. end
  98. defp emqx_apps(profile_info, version) do
  99. apps = umbrella_apps(profile_info) ++ enterprise_apps(profile_info)
  100. set_emqx_app_system_env(apps, profile_info, version)
  101. end
  102. defp umbrella_apps(profile_info = %{release_type: release_type}) do
  103. enterprise_apps = enterprise_umbrella_apps(release_type)
  104. excluded_apps = excluded_apps(release_type)
  105. "apps/*"
  106. |> Path.wildcard()
  107. |> Enum.map(fn path ->
  108. app =
  109. path
  110. |> Path.basename()
  111. |> String.to_atom()
  112. {app, path: path, manager: :rebar3, override: true}
  113. end)
  114. |> Enum.reject(fn dep_spec ->
  115. dep_spec
  116. |> elem(0)
  117. |> then(&MapSet.member?(enterprise_apps, &1))
  118. end)
  119. |> Enum.reject(fn {app, _} ->
  120. case profile_info do
  121. %{edition_type: :enterprise} ->
  122. app == :emqx_telemetry
  123. _ ->
  124. false
  125. end
  126. end)
  127. |> Enum.reject(fn {app, _} -> app in excluded_apps end)
  128. end
  129. defp enterprise_apps(_profile_info = %{release_type: release_type, edition_type: :enterprise}) do
  130. Enum.map(enterprise_umbrella_apps(release_type), fn app_name ->
  131. path = "apps/#{app_name}"
  132. {app_name, path: path, manager: :rebar3, override: true}
  133. end)
  134. end
  135. defp enterprise_apps(_profile_info) do
  136. []
  137. end
  138. # need to remove those when listing `/apps/`...
  139. defp enterprise_umbrella_apps(_release_type) do
  140. MapSet.new([
  141. :emqx_bridge_kafka,
  142. :emqx_bridge_confluent,
  143. :emqx_bridge_gcp_pubsub,
  144. :emqx_bridge_cassandra,
  145. :emqx_bridge_opents,
  146. :emqx_bridge_dynamo,
  147. :emqx_bridge_greptimedb,
  148. :emqx_bridge_hstreamdb,
  149. :emqx_bridge_influxdb,
  150. :emqx_bridge_iotdb,
  151. :emqx_bridge_es,
  152. :emqx_bridge_matrix,
  153. :emqx_bridge_mongodb,
  154. :emqx_bridge_mysql,
  155. :emqx_bridge_pgsql,
  156. :emqx_bridge_redis,
  157. :emqx_bridge_rocketmq,
  158. :emqx_bridge_tdengine,
  159. :emqx_bridge_timescale,
  160. :emqx_bridge_sqlserver,
  161. :emqx_bridge_pulsar,
  162. :emqx_oracle,
  163. :emqx_bridge_oracle,
  164. :emqx_bridge_rabbitmq,
  165. :emqx_bridge_clickhouse,
  166. :emqx_ft,
  167. :emqx_license,
  168. :emqx_s3,
  169. :emqx_bridge_s3,
  170. :emqx_schema_registry,
  171. :emqx_message_validation,
  172. :emqx_enterprise,
  173. :emqx_bridge_kinesis,
  174. :emqx_bridge_azure_event_hub,
  175. :emqx_gcp_device,
  176. :emqx_dashboard_rbac,
  177. :emqx_dashboard_sso,
  178. :emqx_audit,
  179. :emqx_gateway_gbt32960,
  180. :emqx_gateway_ocpp,
  181. :emqx_gateway_jt808,
  182. :emqx_bridge_syskeeper
  183. ])
  184. end
  185. defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
  186. [
  187. {:hstreamdb_erl,
  188. github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"},
  189. {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.13", override: true},
  190. {:wolff, github: "kafka4beam/wolff", tag: "1.10.3"},
  191. {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},
  192. {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"},
  193. {:brod, github: "kafka4beam/brod", tag: "3.16.8"},
  194. {:snappyer, "1.2.9", override: true},
  195. {:crc32cer, "0.1.8", override: true},
  196. {:supervisor3, "1.1.12", override: true},
  197. {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true},
  198. {:greptimedb,
  199. github: "GreptimeTeam/greptimedb-ingester-erl", tag: "v0.1.8", override: true},
  200. # The following two are dependencies of rabbit_common. They are needed here to
  201. # make mix not complain about conflicting versions
  202. {:thoas, github: "emqx/thoas", tag: "v1.0.0", override: true},
  203. {:credentials_obfuscation,
  204. github: "emqx/credentials-obfuscation", tag: "v3.2.0", override: true},
  205. {:rabbit_common,
  206. github: "emqx/rabbitmq-server",
  207. tag: "v3.11.13.2",
  208. sparse: "deps/rabbit_common",
  209. override: true},
  210. {:amqp_client,
  211. github: "emqx/rabbitmq-server",
  212. tag: "v3.11.13.2",
  213. sparse: "deps/amqp_client",
  214. override: true}
  215. ]
  216. end
  217. defp enterprise_deps(_profile_info) do
  218. []
  219. end
  220. defp set_emqx_app_system_env(apps, profile_info, version) do
  221. system_env = emqx_app_system_env(profile_info, version) ++ maybe_no_quic_env()
  222. Enum.map(
  223. apps,
  224. fn {app, opts} ->
  225. {app,
  226. Keyword.update(
  227. opts,
  228. :system_env,
  229. system_env,
  230. &Keyword.merge(&1, system_env)
  231. )}
  232. end
  233. )
  234. end
  235. def emqx_app_system_env(profile_info, version) do
  236. erlc_options(profile_info, version)
  237. |> dump_as_erl()
  238. |> then(&[{"ERL_COMPILER_OPTIONS", &1}])
  239. end
  240. defp erlc_options(%{edition_type: edition_type}, version) do
  241. [
  242. :debug_info,
  243. {:compile_info, [{:emqx_vsn, String.to_charlist(version)}]},
  244. {:d, :EMQX_RELEASE_EDITION, erlang_edition(edition_type)},
  245. {:d, :snk_kind, :msg}
  246. ]
  247. end
  248. def maybe_no_quic_env() do
  249. if not enable_quicer?() do
  250. [{"BUILD_WITHOUT_QUIC", "true"}]
  251. else
  252. []
  253. end
  254. end
  255. defp releases() do
  256. [
  257. emqx: fn ->
  258. %{
  259. release_type: release_type,
  260. package_type: package_type,
  261. edition_type: edition_type
  262. } = check_profile!()
  263. base_steps = [
  264. &make_docs(&1),
  265. :assemble,
  266. &create_RELEASES/1,
  267. &copy_files(&1, release_type, package_type, edition_type),
  268. &copy_escript(&1, "nodetool"),
  269. &copy_escript(&1, "install_upgrade.escript")
  270. ]
  271. steps =
  272. if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
  273. base_steps ++ [&prepare_tar_overlays/1, :tar]
  274. else
  275. base_steps
  276. end
  277. [
  278. applications: applications(release_type, edition_type),
  279. skip_mode_validation_for: [
  280. :emqx_mix,
  281. :emqx_gateway,
  282. :emqx_gateway_stomp,
  283. :emqx_gateway_mqttsn,
  284. :emqx_gateway_coap,
  285. :emqx_gateway_lwm2m,
  286. :emqx_gateway_exproto,
  287. :emqx_dashboard,
  288. :emqx_dashboard_sso,
  289. :emqx_audit,
  290. :emqx_resource,
  291. :emqx_connector,
  292. :emqx_exhook,
  293. :emqx_bridge,
  294. :emqx_bridge_mqtt,
  295. :emqx_modules,
  296. :emqx_management,
  297. :emqx_retainer,
  298. :emqx_prometheus,
  299. :emqx_rule_engine,
  300. :emqx_auto_subscribe,
  301. :emqx_slow_subs,
  302. :emqx_plugins,
  303. :emqx_ft,
  304. :emqx_s3,
  305. :emqx_opentelemetry,
  306. :emqx_durable_storage,
  307. :rabbit_common,
  308. :emqx_eviction_agent,
  309. :emqx_node_rebalance
  310. ],
  311. steps: steps,
  312. strip_beams: false
  313. ]
  314. end
  315. ]
  316. end
  317. def applications(release_type, edition_type) do
  318. {:ok,
  319. [
  320. %{
  321. db_apps: db_apps,
  322. system_apps: system_apps,
  323. common_business_apps: common_business_apps,
  324. ee_business_apps: ee_business_apps,
  325. ce_business_apps: ce_business_apps
  326. }
  327. ]} = :file.consult("apps/emqx_machine/priv/reboot_lists.eterm")
  328. edition_specific_apps =
  329. if edition_type == :enterprise do
  330. ee_business_apps
  331. else
  332. ce_business_apps
  333. end
  334. business_apps = common_business_apps ++ edition_specific_apps
  335. excluded_apps = excluded_apps(release_type)
  336. system_apps =
  337. Enum.map(system_apps, fn app ->
  338. if is_atom(app), do: {app, :permanent}, else: app
  339. end)
  340. db_apps = Enum.map(db_apps, &{&1, :load})
  341. business_apps = Enum.map(business_apps, &{&1, :load})
  342. [system_apps, db_apps, [emqx_machine: :permanent], business_apps]
  343. |> List.flatten()
  344. |> Keyword.reject(fn {app, _type} -> app in excluded_apps end)
  345. end
  346. defp excluded_apps(_release_type) do
  347. %{
  348. mnesia_rocksdb: enable_rocksdb?(),
  349. quicer: enable_quicer?(),
  350. jq: enable_jq?(),
  351. observer: is_app?(:observer)
  352. }
  353. |> Enum.reject(&elem(&1, 1))
  354. |> Enum.map(&elem(&1, 0))
  355. end
  356. defp is_app?(name) do
  357. case Application.load(name) do
  358. :ok ->
  359. true
  360. {:error, {:already_loaded, _}} ->
  361. true
  362. _ ->
  363. false
  364. end
  365. end
  366. def check_profile!() do
  367. valid_envs = [
  368. :emqx,
  369. :"emqx-pkg",
  370. :"emqx-enterprise",
  371. :"emqx-enterprise-pkg"
  372. ]
  373. if Mix.env() == :dev do
  374. env_profile = System.get_env("PROFILE")
  375. if env_profile do
  376. # copy from PROFILE env var
  377. System.get_env("PROFILE")
  378. |> String.to_atom()
  379. |> Mix.env()
  380. else
  381. IO.puts(
  382. IO.ANSI.format([
  383. :yellow,
  384. "Warning: env var PROFILE is unset; defaulting to emqx"
  385. ])
  386. )
  387. Mix.env(:emqx)
  388. end
  389. end
  390. if Mix.env() not in valid_envs do
  391. formatted_envs =
  392. valid_envs
  393. |> Enum.map(&" * #{&1}")
  394. |> Enum.join("\n")
  395. Mix.raise("""
  396. Invalid env #{Mix.env()}. Valid options are:
  397. #{formatted_envs}
  398. """)
  399. end
  400. {
  401. release_type,
  402. package_type,
  403. edition_type
  404. } =
  405. case Mix.env() do
  406. :dev ->
  407. {:standard, :bin, :community}
  408. :emqx ->
  409. {:standard, :bin, :community}
  410. :"emqx-enterprise" ->
  411. {:standard, :bin, :enterprise}
  412. :"emqx-pkg" ->
  413. {:standard, :pkg, :community}
  414. :"emqx-enterprise-pkg" ->
  415. {:standard, :pkg, :enterprise}
  416. end
  417. normalize_env!()
  418. %{
  419. release_type: release_type,
  420. package_type: package_type,
  421. edition_type: edition_type
  422. }
  423. end
  424. #############################################################################
  425. # Custom Steps
  426. #############################################################################
  427. defp make_docs(release) do
  428. profile = System.get_env("MIX_ENV")
  429. os_cmd("build", [profile, "docs"])
  430. release
  431. end
  432. defp copy_files(release, release_type, package_type, edition_type) do
  433. overwrite? = Keyword.get(release.options, :overwrite, false)
  434. bin = Path.join(release.path, "bin")
  435. etc = Path.join(release.path, "etc")
  436. log = Path.join(release.path, "log")
  437. plugins = Path.join(release.path, "plugins")
  438. Mix.Generator.create_directory(bin)
  439. Mix.Generator.create_directory(etc)
  440. Mix.Generator.create_directory(log)
  441. Mix.Generator.create_directory(plugins)
  442. Mix.Generator.create_directory(Path.join(etc, "certs"))
  443. Enum.each(
  444. ["mnesia", "configs", "patches", "scripts"],
  445. fn dir ->
  446. path = Path.join([release.path, "data", dir])
  447. Mix.Generator.create_directory(path)
  448. end
  449. )
  450. Mix.Generator.copy_file(
  451. "apps/emqx_auth/etc/acl.conf",
  452. Path.join(etc, "acl.conf"),
  453. force: overwrite?
  454. )
  455. # required by emqx_auth
  456. File.cp_r!(
  457. "apps/emqx/etc/certs",
  458. Path.join(etc, "certs")
  459. )
  460. profile = System.get_env("MIX_ENV")
  461. File.cp_r!(
  462. "rel/config/examples",
  463. Path.join(etc, "examples"),
  464. force: overwrite?
  465. )
  466. # copy /rel/config/ee-examples if profile is enterprise
  467. case profile do
  468. "emqx-enterprise" ->
  469. File.cp_r!(
  470. "rel/config/ee-examples",
  471. Path.join(etc, "examples"),
  472. force: overwrite?
  473. )
  474. _ ->
  475. :ok
  476. end
  477. # this is required by the produced escript / nodetool
  478. Mix.Generator.copy_file(
  479. Path.join(release.version_path, "start_clean.boot"),
  480. Path.join(bin, "no_dot_erlang.boot"),
  481. force: overwrite?
  482. )
  483. assigns = template_vars(release, release_type, package_type, edition_type)
  484. # This is generated by `scripts/merge-config.escript` or `make merge-config`
  485. # So, this should be run before the release.
  486. # TODO: run as a "compiler" step???
  487. render_template(
  488. "apps/emqx_conf/etc/emqx.conf.all",
  489. assigns,
  490. Path.join(etc, "emqx.conf")
  491. )
  492. render_template(
  493. "rel/emqx_vars",
  494. assigns,
  495. Path.join([release.path, "releases", "emqx_vars"])
  496. )
  497. vm_args_template_path =
  498. case release_type do
  499. _ ->
  500. "apps/emqx/etc/vm.args.cloud"
  501. end
  502. render_template(
  503. vm_args_template_path,
  504. assigns,
  505. [
  506. Path.join(etc, "vm.args"),
  507. Path.join(release.version_path, "vm.args")
  508. ]
  509. )
  510. for name <- [
  511. "emqx",
  512. "emqx_ctl"
  513. ] do
  514. Mix.Generator.copy_file(
  515. "bin/#{name}",
  516. Path.join(bin, name),
  517. force: overwrite?
  518. )
  519. # Files with the version appended are expected by the release
  520. # upgrade script `install_upgrade.escript`
  521. Mix.Generator.copy_file(
  522. Path.join(bin, name),
  523. Path.join(bin, name <> "-#{release.version}"),
  524. force: overwrite?
  525. )
  526. end
  527. for base_name <- ["emqx", "emqx_ctl"],
  528. suffix <- ["", "-#{release.version}"] do
  529. name = base_name <> suffix
  530. File.chmod!(Path.join(bin, name), 0o755)
  531. end
  532. Mix.Generator.copy_file(
  533. "bin/node_dump",
  534. Path.join(bin, "node_dump"),
  535. force: overwrite?
  536. )
  537. File.chmod!(Path.join(bin, "node_dump"), 0o755)
  538. Mix.Generator.copy_file(
  539. "bin/emqx_cluster_rescue",
  540. Path.join(bin, "emqx_cluster_rescue"),
  541. force: overwrite?
  542. )
  543. File.chmod!(Path.join(bin, "emqx_cluster_rescue"), 0o755)
  544. render_template(
  545. "rel/BUILD_INFO",
  546. assigns,
  547. Path.join(release.version_path, "BUILD_INFO")
  548. )
  549. release
  550. end
  551. defp render_template(template, assigns, target) when is_binary(target) do
  552. render_template(template, assigns, [target])
  553. end
  554. defp render_template(template, assigns, tartgets) when is_list(tartgets) do
  555. rendered =
  556. File.read!(template)
  557. |> from_rebar_to_eex_template()
  558. |> EEx.eval_string(assigns)
  559. for target <- tartgets do
  560. File.write!(target, rendered)
  561. end
  562. end
  563. # needed by nodetool and by release_handler
  564. defp create_RELEASES(release) do
  565. apps =
  566. Enum.map(release.applications, fn {app_name, app_props} ->
  567. app_vsn = Keyword.fetch!(app_props, :vsn)
  568. app_path =
  569. "./lib"
  570. |> Path.join("#{app_name}-#{app_vsn}")
  571. |> to_charlist()
  572. {app_name, app_vsn, app_path}
  573. end)
  574. release_entry = [
  575. {
  576. :release,
  577. to_charlist(release.name),
  578. to_charlist(release.version),
  579. release.erts_version,
  580. apps,
  581. :permanent
  582. }
  583. ]
  584. release.path
  585. |> Path.join("releases")
  586. |> Path.join("RELEASES")
  587. |> File.open!([:write, :utf8], fn handle ->
  588. IO.puts(handle, "%% coding: utf-8")
  589. :io.format(handle, ~c"~tp.~n", [release_entry])
  590. end)
  591. release
  592. end
  593. defp copy_escript(release, escript_name) do
  594. [shebang, rest] =
  595. "bin/#{escript_name}"
  596. |> File.read!()
  597. |> String.split("\n", parts: 2)
  598. # the elixir version of escript + start.boot required the boot_var
  599. # RELEASE_LIB to be defined.
  600. # enable-feature is not required when 1.6.x
  601. boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib -enable-feature maybe_expr"
  602. # Files with the version appended are expected by the release
  603. # upgrade script `install_upgrade.escript`
  604. Enum.each(
  605. [escript_name, escript_name <> "-" <> release.version],
  606. fn name ->
  607. path = Path.join([release.path, "bin", name])
  608. File.write!(path, [shebang, "\n", boot_var, "\n", rest])
  609. end
  610. )
  611. release
  612. end
  613. # The `:tar` built-in step in Mix Release does not currently add the
  614. # `etc` directory into the resulting tarball. The workaround is to
  615. # add those to the `:overlays` key before running `:tar`.
  616. # See: https://hexdocs.pm/mix/1.13.4/Mix.Release.html#__struct__/0
  617. defp prepare_tar_overlays(release) do
  618. Map.update!(
  619. release,
  620. :overlays,
  621. &[
  622. "etc",
  623. "data",
  624. "plugins",
  625. "bin/node_dump"
  626. | &1
  627. ]
  628. )
  629. end
  630. #############################################################################
  631. # Helper functions
  632. #############################################################################
  633. defp template_vars(release, release_type, :bin = _package_type, edition_type) do
  634. [
  635. emqx_default_erlang_cookie: default_cookie(),
  636. emqx_configuration_doc: emqx_configuration_doc(edition_type, :root),
  637. emqx_configuration_doc_log: emqx_configuration_doc(edition_type, :log),
  638. platform_data_dir: "data",
  639. platform_etc_dir: "etc",
  640. platform_plugins_dir: "plugins",
  641. runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
  642. emqx_etc_dir: "$RUNNER_ROOT_DIR/etc",
  643. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  644. runner_log_dir: "$RUNNER_ROOT_DIR/log",
  645. runner_user: "",
  646. release_version: release.version,
  647. erts_vsn: release.erts_version,
  648. # FIXME: this is empty in `make emqx` ???
  649. erl_opts: "",
  650. emqx_description: emqx_description(release_type, edition_type),
  651. emqx_schema_mod: emqx_schema_mod(edition_type),
  652. is_elixir: "yes",
  653. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  654. ] ++ build_info()
  655. end
  656. defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
  657. [
  658. emqx_default_erlang_cookie: default_cookie(),
  659. emqx_configuration_doc: emqx_configuration_doc(edition_type, :root),
  660. emqx_configuration_doc_log: emqx_configuration_doc(edition_type, :log),
  661. platform_data_dir: "/var/lib/emqx",
  662. platform_etc_dir: "/etc/emqx",
  663. platform_plugins_dir: "/var/lib/emqx/plugins",
  664. runner_bin_dir: "/usr/bin",
  665. emqx_etc_dir: "/etc/emqx",
  666. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  667. runner_log_dir: "/var/log/emqx",
  668. runner_user: "emqx",
  669. release_version: release.version,
  670. erts_vsn: release.erts_version,
  671. # FIXME: this is empty in `make emqx` ???
  672. erl_opts: "",
  673. emqx_description: emqx_description(release_type, edition_type),
  674. emqx_schema_mod: emqx_schema_mod(edition_type),
  675. is_elixir: "yes",
  676. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  677. ] ++ build_info()
  678. end
  679. defp default_cookie() do
  680. "emqx50elixir"
  681. end
  682. defp emqx_description(release_type, edition_type) do
  683. case {release_type, edition_type} do
  684. {_, :enterprise} ->
  685. "EMQX Enterprise"
  686. {_, :community} ->
  687. "EMQX"
  688. end
  689. end
  690. defp emqx_configuration_doc(:enterprise, :root),
  691. do: "https://docs.emqx.com/en/enterprise/latest/configuration/configuration.html"
  692. defp emqx_configuration_doc(:enterprise, :log),
  693. do: "https://docs.emqx.com/en/enterprise/latest/configuration/logs.html"
  694. defp emqx_configuration_doc(:community, :root),
  695. do: "https://www.emqx.io/docs/en/latest/configuration/configuration.html"
  696. defp emqx_configuration_doc(:community, :log),
  697. do: "https://www.emqx.io/docs/en/latest/configuration/logs.html"
  698. defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema
  699. defp emqx_schema_mod(:community), do: :emqx_conf_schema
  700. defp jq_dep() do
  701. if enable_jq?(),
  702. do: [{:jq, github: "emqx/jq", tag: "v0.3.12", override: true}],
  703. else: []
  704. end
  705. defp quicer_dep() do
  706. if enable_quicer?(),
  707. # in conflict with emqx and emqtt
  708. do: [{:quicer, github: "emqx/quic", tag: "0.0.313", override: true}],
  709. else: []
  710. end
  711. defp enable_jq?() do
  712. not Enum.any?([
  713. build_without_jq?()
  714. ])
  715. end
  716. defp enable_quicer?() do
  717. "1" == System.get_env("BUILD_WITH_QUIC") or
  718. not Enum.any?([
  719. macos?(),
  720. build_without_quic?()
  721. ])
  722. end
  723. defp enable_rocksdb?() do
  724. not Enum.any?([
  725. raspbian?(),
  726. build_without_rocksdb?()
  727. ])
  728. end
  729. defp pkg_vsn() do
  730. %{edition_type: edition_type} = check_profile!()
  731. basedir = Path.dirname(__ENV__.file)
  732. script = Path.join(basedir, "pkg-vsn.sh")
  733. os_cmd(script, [Atom.to_string(edition_type)])
  734. end
  735. defp os_cmd(script, args) do
  736. {str, 0} = System.cmd("bash", [script | args])
  737. String.trim(str)
  738. end
  739. defp macos?() do
  740. {:unix, :darwin} == :os.type()
  741. end
  742. defp raspbian?() do
  743. os_cmd("./scripts/get-distro.sh", []) =~ "raspbian"
  744. end
  745. defp build_without_jq?() do
  746. opt = System.get_env("BUILD_WITHOUT_JQ", "false")
  747. String.downcase(opt) != "false"
  748. end
  749. defp build_without_quic?() do
  750. opt = System.get_env("BUILD_WITHOUT_QUIC", "false")
  751. String.downcase(opt) != "false"
  752. end
  753. defp build_without_rocksdb?() do
  754. opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false")
  755. String.downcase(opt) != "false"
  756. end
  757. defp from_rebar_to_eex_template(str) do
  758. # we must not consider surrounding space in the template var name
  759. # because some help strings contain informative variables that
  760. # should not be interpolated, and those have no spaces.
  761. Regex.replace(
  762. ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/,
  763. str,
  764. "<%= \\g{1} %>"
  765. )
  766. end
  767. defp build_info() do
  768. [
  769. build_info_arch: to_string(:erlang.system_info(:system_architecture)),
  770. build_info_wordsize: wordsize(),
  771. build_info_os: os_cmd("./scripts/get-distro.sh", []),
  772. build_info_erlang: otp_release(),
  773. build_info_elixir: System.version(),
  774. build_info_relform: System.get_env("EMQX_REL_FORM", "tgz")
  775. ]
  776. end
  777. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142
  778. defp wordsize() do
  779. size =
  780. try do
  781. :erlang.system_info({:wordsize, :external})
  782. rescue
  783. ErlangError ->
  784. :erlang.system_info(:wordsize)
  785. end
  786. to_string(8 * size)
  787. end
  788. defp normalize_env!() do
  789. env =
  790. case Mix.env() do
  791. :dev ->
  792. :emqx
  793. env ->
  794. env
  795. end
  796. Mix.env(env)
  797. end
  798. # As from Erlang/OTP 17, the OTP release number corresponds to the
  799. # major OTP version number. No erlang:system_info() argument gives
  800. # the exact OTP version.
  801. # https://www.erlang.org/doc/man/erlang.html#system_info_otp_release
  802. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577
  803. defp otp_release() do
  804. major_version = System.otp_release()
  805. root_dir = to_string(:code.root_dir())
  806. [root_dir, "releases", major_version, "OTP_VERSION"]
  807. |> Path.join()
  808. |> File.read()
  809. |> case do
  810. {:error, _} ->
  811. major_version
  812. {:ok, version} ->
  813. version
  814. |> String.trim()
  815. |> String.split("**")
  816. |> List.first()
  817. end
  818. end
  819. defp dump_as_erl(term) do
  820. term
  821. |> then(&:io_lib.format("~0p", [&1]))
  822. |> :erlang.iolist_to_binary()
  823. end
  824. defp erlang_edition(:community), do: :ce
  825. defp erlang_edition(:enterprise), do: :ee
  826. end