mix.exs 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957
  1. defmodule EMQXUmbrella.MixProject do
  2. use Mix.Project
  3. @moduledoc """
  4. The purpose of this file is to configure the release of EMQX under
  5. Mix. Since EMQX uses its own configuration conventions and startup
  6. procedures, one cannot simply use `iex -S mix`. Instead, it's
  7. recommended to build and use the release.
  8. ## Profiles
  9. To control the profile and edition to build, we case split on the
  10. MIX_ENV value.
  11. The following profiles are valid:
  12. * `emqx`
  13. * `emqx-enterprise`
  14. * `emqx-pkg`
  15. * `emqx-enterprise-pkg`
  16. * `dev` -> same as `emqx`, for convenience
  17. ## Release Environment Variables
  18. The release build is controlled by a few environment variables.
  19. * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
  20. tarball along with the release.
  21. """
  22. def project() do
  23. profile_info = check_profile!()
  24. version = pkg_vsn()
  25. [
  26. app: :emqx_mix,
  27. version: version,
  28. deps: deps(profile_info, version),
  29. releases: releases()
  30. ]
  31. end
  32. defp deps(profile_info, version) do
  33. # we need several overrides here because dependencies specify
  34. # other exact versions, and not ranges.
  35. [
  36. {:lc, github: "emqx/lc", tag: "0.3.2", override: true},
  37. {:redbug, github: "emqx/redbug", tag: "2.0.10"},
  38. {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true},
  39. {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
  40. {:ehttpc, github: "emqx/ehttpc", tag: "0.4.13", override: true},
  41. {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true},
  42. {:jiffy, github: "emqx/jiffy", tag: "1.0.6", override: true},
  43. {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true},
  44. {:esockd, github: "emqx/esockd", tag: "5.11.1", override: true},
  45. {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-2", override: true},
  46. {:ekka, github: "emqx/ekka", tag: "0.19.3", override: true},
  47. {:gen_rpc, github: "emqx/gen_rpc", tag: "3.3.1", override: true},
  48. {:grpc, github: "emqx/grpc-erl", tag: "0.6.12", override: true},
  49. {:minirest, github: "emqx/minirest", tag: "1.4.0", override: true},
  50. {:ecpool, github: "emqx/ecpool", tag: "0.5.7", override: true},
  51. {:replayq, github: "emqx/replayq", tag: "0.3.8", override: true},
  52. {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
  53. # maybe forbid to fetch quicer
  54. {:emqtt,
  55. github: "emqx/emqtt", tag: "1.10.1", override: true, system_env: maybe_no_quic_env()},
  56. {:rulesql, github: "emqx/rulesql", tag: "0.2.0"},
  57. {:observer_cli, "1.7.1"},
  58. {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
  59. {:telemetry, "1.1.0"},
  60. # in conflict by emqtt and hocon
  61. {:getopt, "1.0.2", override: true},
  62. {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true},
  63. {:hocon, github: "emqx/hocon", tag: "0.42.1", override: true},
  64. {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true},
  65. {:esasl, github: "emqx/esasl", tag: "0.2.1"},
  66. {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
  67. # in conflict by ehttpc and emqtt
  68. {:gun, github: "emqx/gun", tag: "1.3.11", override: true},
  69. # in conflict by emqx_connector and system_monitor
  70. {:epgsql, github: "emqx/epgsql", tag: "4.7.1.1", override: true},
  71. # in conflict by emqx and observer_cli
  72. {:recon, github: "ferd/recon", tag: "2.5.1", override: true},
  73. {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
  74. # in conflict by erlavro and rocketmq
  75. {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true},
  76. # dependencies of dependencies; we choose specific refs to match
  77. # what rebar3 chooses.
  78. # in conflict by gun and emqtt
  79. {:cowlib,
  80. github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
  81. # in conflict by cowboy_swagger and cowboy
  82. {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true},
  83. # in conflict by grpc and eetcd
  84. {:gpb, "4.19.9", override: true, runtime: false},
  85. {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true},
  86. # set by hackney (dependency)
  87. {:ssl_verify_fun, "1.1.7", override: true},
  88. {:rfc3339, github: "emqx/rfc3339", tag: "0.2.3", override: true},
  89. {:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.2", override: true},
  90. {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true},
  91. {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true},
  92. {:ra, "2.7.3", override: true}
  93. ] ++
  94. emqx_apps(profile_info, version) ++
  95. enterprise_deps(profile_info) ++ jq_dep() ++ quicer_dep()
  96. end
  97. defp emqx_apps(profile_info, version) do
  98. apps = umbrella_apps(profile_info) ++ enterprise_apps(profile_info)
  99. set_emqx_app_system_env(apps, profile_info, version)
  100. end
  101. defp umbrella_apps(profile_info) do
  102. enterprise_apps = enterprise_umbrella_apps()
  103. "apps/*"
  104. |> Path.wildcard()
  105. |> Enum.map(fn path ->
  106. app =
  107. path
  108. |> Path.basename()
  109. |> String.to_atom()
  110. {app, path: path, manager: :rebar3, override: true}
  111. end)
  112. |> Enum.reject(fn dep_spec ->
  113. dep_spec
  114. |> elem(0)
  115. |> then(&MapSet.member?(enterprise_apps, &1))
  116. end)
  117. |> Enum.reject(fn {app, _} ->
  118. case profile_info do
  119. %{edition_type: :enterprise} ->
  120. app == :emqx_telemetry
  121. _ ->
  122. false
  123. end
  124. end)
  125. end
  126. defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do
  127. Enum.map(enterprise_umbrella_apps(), fn app_name ->
  128. path = "apps/#{app_name}"
  129. {app_name, path: path, manager: :rebar3, override: true}
  130. end)
  131. end
  132. defp enterprise_apps(_profile_info) do
  133. []
  134. end
  135. # need to remove those when listing `/apps/`...
  136. defp enterprise_umbrella_apps() do
  137. MapSet.new([
  138. :emqx_bridge_kafka,
  139. :emqx_bridge_confluent,
  140. :emqx_bridge_gcp_pubsub,
  141. :emqx_bridge_cassandra,
  142. :emqx_bridge_opents,
  143. :emqx_bridge_dynamo,
  144. :emqx_bridge_greptimedb,
  145. :emqx_bridge_hstreamdb,
  146. :emqx_bridge_influxdb,
  147. :emqx_bridge_iotdb,
  148. :emqx_bridge_es,
  149. :emqx_bridge_matrix,
  150. :emqx_bridge_mongodb,
  151. :emqx_bridge_mysql,
  152. :emqx_bridge_pgsql,
  153. :emqx_bridge_redis,
  154. :emqx_bridge_rocketmq,
  155. :emqx_bridge_tdengine,
  156. :emqx_bridge_timescale,
  157. :emqx_bridge_sqlserver,
  158. :emqx_bridge_pulsar,
  159. :emqx_oracle,
  160. :emqx_bridge_oracle,
  161. :emqx_bridge_rabbitmq,
  162. :emqx_bridge_clickhouse,
  163. :emqx_ft,
  164. :emqx_license,
  165. :emqx_s3,
  166. :emqx_bridge_s3,
  167. :emqx_schema_registry,
  168. :emqx_message_validation,
  169. :emqx_enterprise,
  170. :emqx_bridge_kinesis,
  171. :emqx_bridge_azure_event_hub,
  172. :emqx_gcp_device,
  173. :emqx_dashboard_rbac,
  174. :emqx_dashboard_sso,
  175. :emqx_audit,
  176. :emqx_gateway_gbt32960,
  177. :emqx_gateway_ocpp,
  178. :emqx_gateway_jt808,
  179. :emqx_bridge_syskeeper
  180. ])
  181. end
  182. defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
  183. [
  184. {:hstreamdb_erl,
  185. github: "hstreamdb/hstreamdb_erl", tag: "0.5.18+v0.18.1+ezstd-v1.0.5-emqx1"},
  186. {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.13", override: true},
  187. {:wolff, github: "kafka4beam/wolff", tag: "1.10.2"},
  188. {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.5", override: true},
  189. {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.1"},
  190. {:brod, github: "kafka4beam/brod", tag: "3.16.8"},
  191. {:snappyer, "1.2.9", override: true},
  192. {:crc32cer, "0.1.8", override: true},
  193. {:supervisor3, "1.1.12", override: true},
  194. {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true},
  195. {:greptimedb,
  196. github: "GreptimeTeam/greptimedb-ingester-erl", tag: "v0.1.8", override: true},
  197. # The following two are dependencies of rabbit_common. They are needed here to
  198. # make mix not complain about conflicting versions
  199. {:thoas, github: "emqx/thoas", tag: "v1.0.0", override: true},
  200. {:credentials_obfuscation,
  201. github: "emqx/credentials-obfuscation", tag: "v3.2.0", override: true},
  202. {:rabbit_common,
  203. github: "emqx/rabbitmq-server",
  204. tag: "v3.11.13.2",
  205. sparse: "deps/rabbit_common",
  206. override: true},
  207. {:amqp_client,
  208. github: "emqx/rabbitmq-server",
  209. tag: "v3.11.13.2",
  210. sparse: "deps/amqp_client",
  211. override: true}
  212. ]
  213. end
  214. defp enterprise_deps(_profile_info) do
  215. []
  216. end
  217. defp set_emqx_app_system_env(apps, profile_info, version) do
  218. system_env = emqx_app_system_env(profile_info, version) ++ maybe_no_quic_env()
  219. Enum.map(
  220. apps,
  221. fn {app, opts} ->
  222. {app,
  223. Keyword.update(
  224. opts,
  225. :system_env,
  226. system_env,
  227. &Keyword.merge(&1, system_env)
  228. )}
  229. end
  230. )
  231. end
  232. def emqx_app_system_env(profile_info, version) do
  233. erlc_options(profile_info, version)
  234. |> dump_as_erl()
  235. |> then(&[{"ERL_COMPILER_OPTIONS", &1}])
  236. end
  237. defp erlc_options(%{edition_type: edition_type}, version) do
  238. [
  239. :debug_info,
  240. {:compile_info, [{:emqx_vsn, String.to_charlist(version)}]},
  241. {:d, :EMQX_RELEASE_EDITION, erlang_edition(edition_type)},
  242. {:d, :snk_kind, :msg}
  243. ]
  244. end
  245. def maybe_no_quic_env() do
  246. if not enable_quicer?() do
  247. [{"BUILD_WITHOUT_QUIC", "true"}]
  248. else
  249. []
  250. end
  251. end
  252. defp releases() do
  253. [
  254. emqx: fn ->
  255. %{
  256. release_type: release_type,
  257. package_type: package_type,
  258. edition_type: edition_type
  259. } = check_profile!()
  260. base_steps = [
  261. &make_docs(&1),
  262. :assemble,
  263. &create_RELEASES/1,
  264. &copy_files(&1, release_type, package_type, edition_type),
  265. &copy_escript(&1, "nodetool"),
  266. &copy_escript(&1, "install_upgrade.escript")
  267. ]
  268. steps =
  269. if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
  270. base_steps ++ [&prepare_tar_overlays/1, :tar]
  271. else
  272. base_steps
  273. end
  274. [
  275. applications: applications(edition_type),
  276. skip_mode_validation_for: [
  277. :emqx_mix,
  278. :emqx_gateway,
  279. :emqx_gateway_stomp,
  280. :emqx_gateway_mqttsn,
  281. :emqx_gateway_coap,
  282. :emqx_gateway_lwm2m,
  283. :emqx_gateway_exproto,
  284. :emqx_dashboard,
  285. :emqx_dashboard_sso,
  286. :emqx_audit,
  287. :emqx_resource,
  288. :emqx_connector,
  289. :emqx_exhook,
  290. :emqx_bridge,
  291. :emqx_bridge_mqtt,
  292. :emqx_modules,
  293. :emqx_management,
  294. :emqx_retainer,
  295. :emqx_prometheus,
  296. :emqx_rule_engine,
  297. :emqx_auto_subscribe,
  298. :emqx_slow_subs,
  299. :emqx_plugins,
  300. :emqx_ft,
  301. :emqx_s3,
  302. :emqx_opentelemetry,
  303. :emqx_durable_storage,
  304. :rabbit_common,
  305. :emqx_eviction_agent,
  306. :emqx_node_rebalance
  307. ],
  308. steps: steps,
  309. strip_beams: false
  310. ]
  311. end
  312. ]
  313. end
  314. def applications(edition_type) do
  315. {:ok,
  316. [
  317. %{
  318. db_apps: db_apps,
  319. system_apps: system_apps,
  320. common_business_apps: common_business_apps,
  321. ee_business_apps: ee_business_apps,
  322. ce_business_apps: ce_business_apps
  323. }
  324. ]} = :file.consult("apps/emqx_machine/priv/reboot_lists.eterm")
  325. edition_specific_apps =
  326. if edition_type == :enterprise do
  327. ee_business_apps
  328. else
  329. ce_business_apps
  330. end
  331. business_apps = common_business_apps ++ edition_specific_apps
  332. excluded_apps = excluded_apps()
  333. system_apps =
  334. Enum.map(system_apps, fn app ->
  335. if is_atom(app), do: {app, :permanent}, else: app
  336. end)
  337. db_apps = Enum.map(db_apps, &{&1, :load})
  338. business_apps = Enum.map(business_apps, &{&1, :load})
  339. [system_apps, db_apps, [emqx_machine: :permanent], business_apps]
  340. |> List.flatten()
  341. |> Keyword.reject(fn {app, _type} -> app in excluded_apps end)
  342. end
  343. defp excluded_apps() do
  344. %{
  345. mnesia_rocksdb: enable_rocksdb?(),
  346. quicer: enable_quicer?(),
  347. jq: enable_jq?(),
  348. observer: is_app?(:observer)
  349. }
  350. |> Enum.reject(&elem(&1, 1))
  351. |> Enum.map(&elem(&1, 0))
  352. end
  353. defp is_app?(name) do
  354. case Application.load(name) do
  355. :ok ->
  356. true
  357. {:error, {:already_loaded, _}} ->
  358. true
  359. _ ->
  360. false
  361. end
  362. end
  363. def check_profile!() do
  364. valid_envs = [
  365. :emqx,
  366. :"emqx-pkg",
  367. :"emqx-enterprise",
  368. :"emqx-enterprise-pkg"
  369. ]
  370. if Mix.env() == :dev do
  371. env_profile = System.get_env("PROFILE")
  372. if env_profile do
  373. # copy from PROFILE env var
  374. System.get_env("PROFILE")
  375. |> String.to_atom()
  376. |> Mix.env()
  377. else
  378. IO.puts(
  379. IO.ANSI.format([
  380. :yellow,
  381. "Warning: env var PROFILE is unset; defaulting to emqx"
  382. ])
  383. )
  384. Mix.env(:emqx)
  385. end
  386. end
  387. if Mix.env() not in valid_envs do
  388. formatted_envs =
  389. valid_envs
  390. |> Enum.map(&" * #{&1}")
  391. |> Enum.join("\n")
  392. Mix.raise("""
  393. Invalid env #{Mix.env()}. Valid options are:
  394. #{formatted_envs}
  395. """)
  396. end
  397. {
  398. release_type,
  399. package_type,
  400. edition_type
  401. } =
  402. case Mix.env() do
  403. :dev ->
  404. {:cloud, :bin, :community}
  405. :emqx ->
  406. {:cloud, :bin, :community}
  407. :"emqx-enterprise" ->
  408. {:cloud, :bin, :enterprise}
  409. :"emqx-pkg" ->
  410. {:cloud, :pkg, :community}
  411. :"emqx-enterprise-pkg" ->
  412. {:cloud, :pkg, :enterprise}
  413. end
  414. normalize_env!()
  415. %{
  416. release_type: release_type,
  417. package_type: package_type,
  418. edition_type: edition_type
  419. }
  420. end
  421. #############################################################################
  422. # Custom Steps
  423. #############################################################################
  424. defp make_docs(release) do
  425. profile = System.get_env("MIX_ENV")
  426. os_cmd("build", [profile, "docs"])
  427. release
  428. end
  429. defp copy_files(release, release_type, package_type, edition_type) do
  430. overwrite? = Keyword.get(release.options, :overwrite, false)
  431. bin = Path.join(release.path, "bin")
  432. etc = Path.join(release.path, "etc")
  433. log = Path.join(release.path, "log")
  434. plugins = Path.join(release.path, "plugins")
  435. Mix.Generator.create_directory(bin)
  436. Mix.Generator.create_directory(etc)
  437. Mix.Generator.create_directory(log)
  438. Mix.Generator.create_directory(plugins)
  439. Mix.Generator.create_directory(Path.join(etc, "certs"))
  440. Enum.each(
  441. ["mnesia", "configs", "patches", "scripts"],
  442. fn dir ->
  443. path = Path.join([release.path, "data", dir])
  444. Mix.Generator.create_directory(path)
  445. end
  446. )
  447. Mix.Generator.copy_file(
  448. "apps/emqx_auth/etc/acl.conf",
  449. Path.join(etc, "acl.conf"),
  450. force: overwrite?
  451. )
  452. # required by emqx_auth
  453. File.cp_r!(
  454. "apps/emqx/etc/certs",
  455. Path.join(etc, "certs")
  456. )
  457. profile = System.get_env("MIX_ENV")
  458. File.cp_r!(
  459. "rel/config/examples",
  460. Path.join(etc, "examples"),
  461. force: overwrite?
  462. )
  463. # copy /rel/config/ee-examples if profile is enterprise
  464. case profile do
  465. "emqx-enterprise" ->
  466. File.cp_r!(
  467. "rel/config/ee-examples",
  468. Path.join(etc, "examples"),
  469. force: overwrite?
  470. )
  471. _ ->
  472. :ok
  473. end
  474. # this is required by the produced escript / nodetool
  475. Mix.Generator.copy_file(
  476. Path.join(release.version_path, "start_clean.boot"),
  477. Path.join(bin, "no_dot_erlang.boot"),
  478. force: overwrite?
  479. )
  480. assigns = template_vars(release, release_type, package_type, edition_type)
  481. # This is generated by `scripts/merge-config.escript` or `make merge-config`
  482. # So, this should be run before the release.
  483. # TODO: run as a "compiler" step???
  484. render_template(
  485. "apps/emqx_conf/etc/emqx.conf.all",
  486. assigns,
  487. Path.join(etc, "emqx.conf")
  488. )
  489. render_template(
  490. "rel/emqx_vars",
  491. assigns,
  492. Path.join([release.path, "releases", "emqx_vars"])
  493. )
  494. vm_args_template_path =
  495. case release_type do
  496. :cloud ->
  497. "apps/emqx/etc/vm.args.cloud"
  498. end
  499. render_template(
  500. vm_args_template_path,
  501. assigns,
  502. [
  503. Path.join(etc, "vm.args"),
  504. Path.join(release.version_path, "vm.args")
  505. ]
  506. )
  507. for name <- [
  508. "emqx",
  509. "emqx_ctl"
  510. ] do
  511. Mix.Generator.copy_file(
  512. "bin/#{name}",
  513. Path.join(bin, name),
  514. force: overwrite?
  515. )
  516. # Files with the version appended are expected by the release
  517. # upgrade script `install_upgrade.escript`
  518. Mix.Generator.copy_file(
  519. Path.join(bin, name),
  520. Path.join(bin, name <> "-#{release.version}"),
  521. force: overwrite?
  522. )
  523. end
  524. for base_name <- ["emqx", "emqx_ctl"],
  525. suffix <- ["", "-#{release.version}"] do
  526. name = base_name <> suffix
  527. File.chmod!(Path.join(bin, name), 0o755)
  528. end
  529. Mix.Generator.copy_file(
  530. "bin/node_dump",
  531. Path.join(bin, "node_dump"),
  532. force: overwrite?
  533. )
  534. File.chmod!(Path.join(bin, "node_dump"), 0o755)
  535. Mix.Generator.copy_file(
  536. "bin/emqx_cluster_rescue",
  537. Path.join(bin, "emqx_cluster_rescue"),
  538. force: overwrite?
  539. )
  540. File.chmod!(Path.join(bin, "emqx_cluster_rescue"), 0o755)
  541. render_template(
  542. "rel/BUILD_INFO",
  543. assigns,
  544. Path.join(release.version_path, "BUILD_INFO")
  545. )
  546. release
  547. end
  548. defp render_template(template, assigns, target) when is_binary(target) do
  549. render_template(template, assigns, [target])
  550. end
  551. defp render_template(template, assigns, tartgets) when is_list(tartgets) do
  552. rendered =
  553. File.read!(template)
  554. |> from_rebar_to_eex_template()
  555. |> EEx.eval_string(assigns)
  556. for target <- tartgets do
  557. File.write!(target, rendered)
  558. end
  559. end
  560. # needed by nodetool and by release_handler
  561. defp create_RELEASES(release) do
  562. apps =
  563. Enum.map(release.applications, fn {app_name, app_props} ->
  564. app_vsn = Keyword.fetch!(app_props, :vsn)
  565. app_path =
  566. "./lib"
  567. |> Path.join("#{app_name}-#{app_vsn}")
  568. |> to_charlist()
  569. {app_name, app_vsn, app_path}
  570. end)
  571. release_entry = [
  572. {
  573. :release,
  574. to_charlist(release.name),
  575. to_charlist(release.version),
  576. release.erts_version,
  577. apps,
  578. :permanent
  579. }
  580. ]
  581. release.path
  582. |> Path.join("releases")
  583. |> Path.join("RELEASES")
  584. |> File.open!([:write, :utf8], fn handle ->
  585. IO.puts(handle, "%% coding: utf-8")
  586. :io.format(handle, ~c"~tp.~n", [release_entry])
  587. end)
  588. release
  589. end
  590. defp copy_escript(release, escript_name) do
  591. [shebang, rest] =
  592. "bin/#{escript_name}"
  593. |> File.read!()
  594. |> String.split("\n", parts: 2)
  595. # the elixir version of escript + start.boot required the boot_var
  596. # RELEASE_LIB to be defined.
  597. # enable-feature is not required when 1.6.x
  598. boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib -enable-feature maybe_expr"
  599. # Files with the version appended are expected by the release
  600. # upgrade script `install_upgrade.escript`
  601. Enum.each(
  602. [escript_name, escript_name <> "-" <> release.version],
  603. fn name ->
  604. path = Path.join([release.path, "bin", name])
  605. File.write!(path, [shebang, "\n", boot_var, "\n", rest])
  606. end
  607. )
  608. release
  609. end
  610. # The `:tar` built-in step in Mix Release does not currently add the
  611. # `etc` directory into the resulting tarball. The workaround is to
  612. # add those to the `:overlays` key before running `:tar`.
  613. # See: https://hexdocs.pm/mix/1.13.4/Mix.Release.html#__struct__/0
  614. defp prepare_tar_overlays(release) do
  615. Map.update!(
  616. release,
  617. :overlays,
  618. &[
  619. "etc",
  620. "data",
  621. "plugins",
  622. "bin/node_dump"
  623. | &1
  624. ]
  625. )
  626. end
  627. #############################################################################
  628. # Helper functions
  629. #############################################################################
  630. defp template_vars(release, release_type, :bin = _package_type, edition_type) do
  631. [
  632. emqx_default_erlang_cookie: default_cookie(),
  633. emqx_configuration_doc: emqx_configuration_doc(edition_type, :root),
  634. emqx_configuration_doc_log: emqx_configuration_doc(edition_type, :log),
  635. platform_data_dir: "data",
  636. platform_etc_dir: "etc",
  637. platform_plugins_dir: "plugins",
  638. runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
  639. emqx_etc_dir: "$RUNNER_ROOT_DIR/etc",
  640. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  641. runner_log_dir: "$RUNNER_ROOT_DIR/log",
  642. runner_user: "",
  643. release_version: release.version,
  644. erts_vsn: release.erts_version,
  645. # FIXME: this is empty in `make emqx` ???
  646. erl_opts: "",
  647. emqx_description: emqx_description(release_type, edition_type),
  648. emqx_schema_mod: emqx_schema_mod(edition_type),
  649. is_elixir: "yes",
  650. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  651. ] ++ build_info()
  652. end
  653. defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
  654. [
  655. emqx_default_erlang_cookie: default_cookie(),
  656. emqx_configuration_doc: emqx_configuration_doc(edition_type, :root),
  657. emqx_configuration_doc_log: emqx_configuration_doc(edition_type, :log),
  658. platform_data_dir: "/var/lib/emqx",
  659. platform_etc_dir: "/etc/emqx",
  660. platform_plugins_dir: "/var/lib/emqx/plugins",
  661. runner_bin_dir: "/usr/bin",
  662. emqx_etc_dir: "/etc/emqx",
  663. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  664. runner_log_dir: "/var/log/emqx",
  665. runner_user: "emqx",
  666. release_version: release.version,
  667. erts_vsn: release.erts_version,
  668. # FIXME: this is empty in `make emqx` ???
  669. erl_opts: "",
  670. emqx_description: emqx_description(release_type, edition_type),
  671. emqx_schema_mod: emqx_schema_mod(edition_type),
  672. is_elixir: "yes",
  673. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  674. ] ++ build_info()
  675. end
  676. defp default_cookie() do
  677. "emqx50elixir"
  678. end
  679. defp emqx_description(release_type, edition_type) do
  680. case {release_type, edition_type} do
  681. {:cloud, :enterprise} ->
  682. "EMQX Enterprise"
  683. {:cloud, :community} ->
  684. "EMQX"
  685. end
  686. end
  687. defp emqx_configuration_doc(:enterprise, :root),
  688. do: "https://docs.emqx.com/en/enterprise/latest/configuration/configuration.html"
  689. defp emqx_configuration_doc(:enterprise, :log),
  690. do: "https://docs.emqx.com/en/enterprise/latest/configuration/logs.html"
  691. defp emqx_configuration_doc(:community, :root),
  692. do: "https://www.emqx.io/docs/en/latest/configuration/configuration.html"
  693. defp emqx_configuration_doc(:community, :log),
  694. do: "https://www.emqx.io/docs/en/latest/configuration/logs.html"
  695. defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema
  696. defp emqx_schema_mod(:community), do: :emqx_conf_schema
  697. defp jq_dep() do
  698. if enable_jq?(),
  699. do: [{:jq, github: "emqx/jq", tag: "v0.3.12", override: true}],
  700. else: []
  701. end
  702. defp quicer_dep() do
  703. if enable_quicer?(),
  704. # in conflict with emqx and emqtt
  705. do: [{:quicer, github: "emqx/quic", tag: "0.0.313", override: true}],
  706. else: []
  707. end
  708. defp enable_jq?() do
  709. not Enum.any?([
  710. build_without_jq?()
  711. ])
  712. end
  713. defp enable_quicer?() do
  714. "1" == System.get_env("BUILD_WITH_QUIC") or
  715. not Enum.any?([
  716. macos?(),
  717. build_without_quic?()
  718. ])
  719. end
  720. defp enable_rocksdb?() do
  721. not Enum.any?([
  722. raspbian?(),
  723. build_without_rocksdb?()
  724. ])
  725. end
  726. defp pkg_vsn() do
  727. %{edition_type: edition_type} = check_profile!()
  728. basedir = Path.dirname(__ENV__.file)
  729. script = Path.join(basedir, "pkg-vsn.sh")
  730. os_cmd(script, [Atom.to_string(edition_type)])
  731. end
  732. defp os_cmd(script, args) do
  733. {str, 0} = System.cmd("bash", [script | args])
  734. String.trim(str)
  735. end
  736. defp macos?() do
  737. {:unix, :darwin} == :os.type()
  738. end
  739. defp raspbian?() do
  740. os_cmd("./scripts/get-distro.sh", []) =~ "raspbian"
  741. end
  742. defp build_without_jq?() do
  743. opt = System.get_env("BUILD_WITHOUT_JQ", "false")
  744. String.downcase(opt) != "false"
  745. end
  746. defp build_without_quic?() do
  747. opt = System.get_env("BUILD_WITHOUT_QUIC", "false")
  748. String.downcase(opt) != "false"
  749. end
  750. defp build_without_rocksdb?() do
  751. opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false")
  752. String.downcase(opt) != "false"
  753. end
  754. defp from_rebar_to_eex_template(str) do
  755. # we must not consider surrounding space in the template var name
  756. # because some help strings contain informative variables that
  757. # should not be interpolated, and those have no spaces.
  758. Regex.replace(
  759. ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/,
  760. str,
  761. "<%= \\g{1} %>"
  762. )
  763. end
  764. defp build_info() do
  765. [
  766. build_info_arch: to_string(:erlang.system_info(:system_architecture)),
  767. build_info_wordsize: wordsize(),
  768. build_info_os: os_cmd("./scripts/get-distro.sh", []),
  769. build_info_erlang: otp_release(),
  770. build_info_elixir: System.version(),
  771. build_info_relform: System.get_env("EMQX_REL_FORM", "tgz")
  772. ]
  773. end
  774. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142
  775. defp wordsize() do
  776. size =
  777. try do
  778. :erlang.system_info({:wordsize, :external})
  779. rescue
  780. ErlangError ->
  781. :erlang.system_info(:wordsize)
  782. end
  783. to_string(8 * size)
  784. end
  785. defp normalize_env!() do
  786. env =
  787. case Mix.env() do
  788. :dev ->
  789. :emqx
  790. env ->
  791. env
  792. end
  793. Mix.env(env)
  794. end
  795. # As from Erlang/OTP 17, the OTP release number corresponds to the
  796. # major OTP version number. No erlang:system_info() argument gives
  797. # the exact OTP version.
  798. # https://www.erlang.org/doc/man/erlang.html#system_info_otp_release
  799. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577
  800. defp otp_release() do
  801. major_version = System.otp_release()
  802. root_dir = to_string(:code.root_dir())
  803. [root_dir, "releases", major_version, "OTP_VERSION"]
  804. |> Path.join()
  805. |> File.read()
  806. |> case do
  807. {:error, _} ->
  808. major_version
  809. {:ok, version} ->
  810. version
  811. |> String.trim()
  812. |> String.split("**")
  813. |> List.first()
  814. end
  815. end
  816. defp dump_as_erl(term) do
  817. term
  818. |> then(&:io_lib.format("~0p", [&1]))
  819. |> :erlang.iolist_to_binary()
  820. end
  821. defp erlang_edition(:community), do: :ce
  822. defp erlang_edition(:enterprise), do: :ee
  823. end