mix.exs 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004
  1. defmodule EMQXUmbrella.MixProject do
  2. use Mix.Project
  3. @moduledoc """
  4. The purpose of this file is to configure the release of EMQX under
  5. Mix. Since EMQX uses its own configuration conventions and startup
  6. procedures, one cannot simply use `iex -S mix`. Instead, it's
  7. recommended to build and use the release.
  8. ## Profiles
  9. To control the profile and edition to build, we case split on the
  10. MIX_ENV value.
  11. The following profiles are valid:
  12. * `emqx`
  13. * `emqx-enterprise`
  14. * `emqx-pkg`
  15. * `emqx-enterprise-pkg`
  16. * `dev` -> same as `emqx`, for convenience
  17. ## Release Environment Variables
  18. The release build is controlled by a few environment variables.
  19. * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
  20. tarball along with the release.
  21. """
  22. def project() do
  23. profile_info = check_profile!()
  24. version = pkg_vsn()
  25. [
  26. app: :emqx_mix,
  27. version: version,
  28. deps: deps(profile_info, version),
  29. releases: releases()
  30. ]
  31. end
  32. defp deps(profile_info, version) do
  33. # we need several overrides here because dependencies specify
  34. # other exact versions, and not ranges.
  35. [
  36. {:lc, github: "emqx/lc", tag: "0.3.2", override: true},
  37. {:redbug, "2.0.8"},
  38. {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true},
  39. {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
  40. {:ehttpc, github: "emqx/ehttpc", tag: "0.4.11", override: true},
  41. {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true},
  42. {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
  43. {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true},
  44. {:esockd, github: "emqx/esockd", tag: "5.9.7", override: true},
  45. {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true},
  46. {:ekka, github: "emqx/ekka", tag: "0.15.14", override: true},
  47. {:gen_rpc, github: "emqx/gen_rpc", tag: "3.1.0", override: true},
  48. {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true},
  49. {:minirest, github: "emqx/minirest", tag: "1.3.13", override: true},
  50. {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true},
  51. {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true},
  52. {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
  53. # maybe forbid to fetch quicer
  54. {:emqtt,
  55. github: "emqx/emqtt", tag: "1.9.0", override: true, system_env: maybe_no_quic_env()},
  56. {:rulesql, github: "emqx/rulesql", tag: "0.1.7"},
  57. {:observer_cli, "1.7.1"},
  58. {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
  59. {:telemetry, "1.1.0"},
  60. # in conflict by emqtt and hocon
  61. {:getopt, "1.0.2", override: true},
  62. {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true},
  63. {:hocon, github: "emqx/hocon", tag: "0.39.16", override: true},
  64. {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true},
  65. {:esasl, github: "emqx/esasl", tag: "0.2.0"},
  66. {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
  67. # in conflict by ehttpc and emqtt
  68. {:gun, github: "emqx/gun", tag: "1.3.9", override: true},
  69. # in conflict by emqx_connector and system_monitor
  70. {:epgsql, github: "emqx/epgsql", tag: "4.7.0.1", override: true},
  71. # in conflict by emqx and observer_cli
  72. {:recon, github: "ferd/recon", tag: "2.5.1", override: true},
  73. {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
  74. # in conflict by erlavro and rocketmq
  75. {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true},
  76. # dependencies of dependencies; we choose specific refs to match
  77. # what rebar3 chooses.
  78. # in conflict by gun and emqtt
  79. {:cowlib,
  80. github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
  81. # in conflict by cowboy_swagger and cowboy
  82. {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true},
  83. # in conflict by grpc and eetcd
  84. {:gpb, "4.19.9", override: true, runtime: false},
  85. {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true},
  86. # set by hackney (dependency)
  87. {:ssl_verify_fun, "1.1.6", override: true},
  88. {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true},
  89. {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true},
  90. {:opentelemetry_api,
  91. github: "emqx/opentelemetry-erlang",
  92. sparse: "apps/opentelemetry_api",
  93. tag: "v1.3.0-emqx",
  94. override: true,
  95. runtime: false},
  96. {:opentelemetry,
  97. github: "emqx/opentelemetry-erlang",
  98. sparse: "apps/opentelemetry",
  99. tag: "v1.3.0-emqx",
  100. override: true,
  101. runtime: false},
  102. {:opentelemetry_api_experimental,
  103. github: "emqx/opentelemetry-erlang",
  104. sparse: "apps/opentelemetry_api_experimental",
  105. tag: "v1.3.0-emqx",
  106. override: true,
  107. runtime: false},
  108. {:opentelemetry_experimental,
  109. github: "emqx/opentelemetry-erlang",
  110. sparse: "apps/opentelemetry_experimental",
  111. tag: "v1.3.0-emqx",
  112. override: true,
  113. runtime: false},
  114. {:opentelemetry_exporter,
  115. github: "emqx/opentelemetry-erlang",
  116. sparse: "apps/opentelemetry_exporter",
  117. tag: "v1.3.0-emqx",
  118. override: true,
  119. runtime: false}
  120. ] ++
  121. emqx_apps(profile_info, version) ++
  122. enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
  123. end
  124. defp emqx_apps(profile_info, version) do
  125. apps = umbrella_apps(profile_info) ++ enterprise_apps(profile_info)
  126. set_emqx_app_system_env(apps, profile_info, version)
  127. end
  128. defp umbrella_apps(profile_info) do
  129. enterprise_apps = enterprise_umbrella_apps()
  130. "apps/*"
  131. |> Path.wildcard()
  132. |> Enum.map(fn path ->
  133. app =
  134. path
  135. |> Path.basename()
  136. |> String.to_atom()
  137. {app, path: path, manager: :rebar3, override: true}
  138. end)
  139. |> Enum.reject(fn dep_spec ->
  140. dep_spec
  141. |> elem(0)
  142. |> then(&MapSet.member?(enterprise_apps, &1))
  143. end)
  144. |> Enum.reject(fn {app, _} ->
  145. case profile_info do
  146. %{edition_type: :enterprise} ->
  147. app == :emqx_telemetry
  148. _ ->
  149. false
  150. end
  151. end)
  152. end
  153. defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do
  154. umbrella_apps =
  155. Enum.map(enterprise_umbrella_apps(), fn app_name ->
  156. path = "apps/#{app_name}"
  157. {app_name, path: path, manager: :rebar3, override: true}
  158. end)
  159. "lib-ee/*"
  160. |> Path.wildcard()
  161. |> Enum.filter(&File.dir?/1)
  162. |> Enum.map(fn path ->
  163. app =
  164. path
  165. |> Path.basename()
  166. |> String.to_atom()
  167. {app, path: path, manager: :rebar3, override: true}
  168. end)
  169. |> Enum.concat(umbrella_apps)
  170. end
  171. defp enterprise_apps(_profile_info) do
  172. []
  173. end
  174. # need to remove those when listing `/apps/`...
  175. defp enterprise_umbrella_apps() do
  176. MapSet.new([
  177. :emqx_bridge_kafka,
  178. :emqx_bridge_gcp_pubsub,
  179. :emqx_bridge_cassandra,
  180. :emqx_bridge_opents,
  181. :emqx_bridge_dynamo,
  182. :emqx_bridge_greptimedb,
  183. :emqx_bridge_hstreamdb,
  184. :emqx_bridge_influxdb,
  185. :emqx_bridge_iotdb,
  186. :emqx_bridge_matrix,
  187. :emqx_bridge_mongodb,
  188. :emqx_bridge_mysql,
  189. :emqx_bridge_pgsql,
  190. :emqx_bridge_redis,
  191. :emqx_bridge_rocketmq,
  192. :emqx_bridge_tdengine,
  193. :emqx_bridge_timescale,
  194. :emqx_bridge_sqlserver,
  195. :emqx_bridge_pulsar,
  196. :emqx_oracle,
  197. :emqx_bridge_oracle,
  198. :emqx_bridge_rabbitmq,
  199. :emqx_bridge_clickhouse,
  200. :emqx_ft,
  201. :emqx_s3,
  202. :emqx_schema_registry,
  203. :emqx_enterprise,
  204. :emqx_bridge_kinesis,
  205. :emqx_bridge_azure_event_hub,
  206. :emqx_ldap,
  207. :emqx_gcp_device,
  208. :emqx_dashboard_rbac,
  209. :emqx_dashboard_sso
  210. ])
  211. end
  212. defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
  213. [
  214. {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.4.5+v0.16.1"},
  215. {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.11", override: true},
  216. {:wolff, github: "kafka4beam/wolff", tag: "1.7.7"},
  217. {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true},
  218. {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0"},
  219. {:brod, github: "kafka4beam/brod", tag: "3.16.8"},
  220. {:snappyer, "1.2.9", override: true},
  221. {:crc32cer, "0.1.8", override: true},
  222. {:supervisor3, "1.1.12", override: true},
  223. {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true},
  224. {:greptimedb, github: "GreptimeTeam/greptimedb-client-erl", tag: "v0.1.2", override: true},
  225. # The following two are dependencies of rabbit_common. They are needed here to
  226. # make mix not complain about conflicting versions
  227. {:thoas, github: "emqx/thoas", tag: "v1.0.0", override: true},
  228. {:credentials_obfuscation,
  229. github: "emqx/credentials-obfuscation", tag: "v3.2.0", override: true},
  230. {:rabbit_common,
  231. github: "emqx/rabbitmq-server",
  232. tag: "v3.11.13-emqx",
  233. sparse: "deps/rabbit_common",
  234. override: true},
  235. {:amqp_client,
  236. github: "emqx/rabbitmq-server",
  237. tag: "v3.11.13-emqx",
  238. sparse: "deps/amqp_client",
  239. override: true}
  240. ]
  241. end
  242. defp enterprise_deps(_profile_info) do
  243. []
  244. end
  245. defp set_emqx_app_system_env(apps, profile_info, version) do
  246. system_env = emqx_app_system_env(profile_info, version) ++ maybe_no_quic_env()
  247. Enum.map(
  248. apps,
  249. fn {app, opts} ->
  250. {app,
  251. Keyword.update(
  252. opts,
  253. :system_env,
  254. system_env,
  255. &Keyword.merge(&1, system_env)
  256. )}
  257. end
  258. )
  259. end
  260. def emqx_app_system_env(profile_info, version) do
  261. erlc_options(profile_info, version)
  262. |> dump_as_erl()
  263. |> then(&[{"ERL_COMPILER_OPTIONS", &1}])
  264. end
  265. defp erlc_options(%{edition_type: edition_type}, version) do
  266. [
  267. :debug_info,
  268. {:compile_info, [{:emqx_vsn, String.to_charlist(version)}]},
  269. {:d, :EMQX_RELEASE_EDITION, erlang_edition(edition_type)},
  270. {:d, :snk_kind, :msg}
  271. ]
  272. end
  273. def maybe_no_quic_env() do
  274. if not enable_quicer?() do
  275. [{"BUILD_WITHOUT_QUIC", "true"}]
  276. else
  277. []
  278. end
  279. end
  280. defp releases() do
  281. [
  282. emqx: fn ->
  283. %{
  284. release_type: release_type,
  285. package_type: package_type,
  286. edition_type: edition_type
  287. } = check_profile!()
  288. base_steps = [
  289. &make_docs(&1),
  290. :assemble,
  291. &create_RELEASES/1,
  292. &copy_files(&1, release_type, package_type, edition_type),
  293. &copy_escript(&1, "nodetool"),
  294. &copy_escript(&1, "install_upgrade.escript")
  295. ]
  296. steps =
  297. if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
  298. base_steps ++ [&prepare_tar_overlays/1, :tar]
  299. else
  300. base_steps
  301. end
  302. [
  303. applications: applications(edition_type),
  304. skip_mode_validation_for: [
  305. :emqx_mix,
  306. :emqx_gateway,
  307. :emqx_gateway_stomp,
  308. :emqx_gateway_mqttsn,
  309. :emqx_gateway_coap,
  310. :emqx_gateway_lwm2m,
  311. :emqx_gateway_exproto,
  312. :emqx_dashboard,
  313. :emqx_resource,
  314. :emqx_connector,
  315. :emqx_exhook,
  316. :emqx_bridge,
  317. :emqx_bridge_mqtt,
  318. :emqx_modules,
  319. :emqx_management,
  320. :emqx_retainer,
  321. :emqx_prometheus,
  322. :emqx_auto_subscribe,
  323. :emqx_slow_subs,
  324. :emqx_plugins,
  325. :emqx_ft,
  326. :emqx_s3,
  327. :emqx_opentelemetry,
  328. :emqx_durable_storage,
  329. :rabbit_common
  330. ],
  331. steps: steps,
  332. strip_beams: false
  333. ]
  334. end
  335. ]
  336. end
  337. def applications(edition_type) do
  338. {:ok,
  339. [
  340. %{
  341. db_apps: db_apps,
  342. system_apps: system_apps,
  343. common_business_apps: common_business_apps,
  344. ee_business_apps: ee_business_apps,
  345. ce_business_apps: ce_business_apps
  346. }
  347. ]} = :file.consult("apps/emqx_machine/priv/reboot_lists.eterm")
  348. edition_specific_apps =
  349. if edition_type == :enterprise do
  350. ee_business_apps
  351. else
  352. ce_business_apps
  353. end
  354. business_apps = common_business_apps ++ edition_specific_apps
  355. excluded_apps = excluded_apps()
  356. system_apps =
  357. Enum.map(system_apps, fn app ->
  358. if is_atom(app), do: {app, :permanent}, else: app
  359. end)
  360. db_apps = Enum.map(db_apps, &{&1, :load})
  361. business_apps = Enum.map(business_apps, &{&1, :load})
  362. [system_apps, db_apps, [emqx_machine: :permanent], business_apps]
  363. |> List.flatten()
  364. |> Keyword.reject(fn {app, _type} -> app in excluded_apps end)
  365. end
  366. defp excluded_apps() do
  367. %{
  368. mnesia_rocksdb: enable_rocksdb?(),
  369. quicer: enable_quicer?(),
  370. bcrypt: enable_bcrypt?(),
  371. jq: enable_jq?(),
  372. observer: is_app?(:observer),
  373. os_mon: enable_os_mon?()
  374. }
  375. |> Enum.reject(&elem(&1, 1))
  376. |> Enum.map(&elem(&1, 0))
  377. end
  378. defp is_app?(name) do
  379. case Application.load(name) do
  380. :ok ->
  381. true
  382. {:error, {:already_loaded, _}} ->
  383. true
  384. _ ->
  385. false
  386. end
  387. end
  388. def check_profile!() do
  389. valid_envs = [
  390. :emqx,
  391. :"emqx-pkg",
  392. :"emqx-enterprise",
  393. :"emqx-enterprise-pkg"
  394. ]
  395. if Mix.env() == :dev do
  396. env_profile = System.get_env("PROFILE")
  397. if env_profile do
  398. # copy from PROFILE env var
  399. System.get_env("PROFILE")
  400. |> String.to_atom()
  401. |> Mix.env()
  402. else
  403. IO.puts(
  404. IO.ANSI.format([
  405. :yellow,
  406. "Warning: env var PROFILE is unset; defaulting to emqx"
  407. ])
  408. )
  409. Mix.env(:emqx)
  410. end
  411. end
  412. if Mix.env() not in valid_envs do
  413. formatted_envs =
  414. valid_envs
  415. |> Enum.map(&" * #{&1}")
  416. |> Enum.join("\n")
  417. Mix.raise("""
  418. Invalid env #{Mix.env()}. Valid options are:
  419. #{formatted_envs}
  420. """)
  421. end
  422. {
  423. release_type,
  424. package_type,
  425. edition_type
  426. } =
  427. case Mix.env() do
  428. :dev ->
  429. {:cloud, :bin, :community}
  430. :emqx ->
  431. {:cloud, :bin, :community}
  432. :"emqx-enterprise" ->
  433. {:cloud, :bin, :enterprise}
  434. :"emqx-pkg" ->
  435. {:cloud, :pkg, :community}
  436. :"emqx-enterprise-pkg" ->
  437. {:cloud, :pkg, :enterprise}
  438. end
  439. normalize_env!()
  440. %{
  441. release_type: release_type,
  442. package_type: package_type,
  443. edition_type: edition_type
  444. }
  445. end
  446. #############################################################################
  447. # Custom Steps
  448. #############################################################################
  449. defp make_docs(release) do
  450. profile = System.get_env("MIX_ENV")
  451. os_cmd("build", [profile, "docs"])
  452. release
  453. end
  454. defp copy_files(release, release_type, package_type, edition_type) do
  455. overwrite? = Keyword.get(release.options, :overwrite, false)
  456. bin = Path.join(release.path, "bin")
  457. etc = Path.join(release.path, "etc")
  458. log = Path.join(release.path, "log")
  459. plugins = Path.join(release.path, "plugins")
  460. Mix.Generator.create_directory(bin)
  461. Mix.Generator.create_directory(etc)
  462. Mix.Generator.create_directory(log)
  463. Mix.Generator.create_directory(plugins)
  464. Mix.Generator.create_directory(Path.join(etc, "certs"))
  465. Enum.each(
  466. ["mnesia", "configs", "patches", "scripts"],
  467. fn dir ->
  468. path = Path.join([release.path, "data", dir])
  469. Mix.Generator.create_directory(path)
  470. end
  471. )
  472. Mix.Generator.copy_file(
  473. "apps/emqx_authz/etc/acl.conf",
  474. Path.join(etc, "acl.conf"),
  475. force: overwrite?
  476. )
  477. # required by emqx_authz
  478. File.cp_r!(
  479. "apps/emqx/etc/certs",
  480. Path.join(etc, "certs")
  481. )
  482. profile = System.get_env("MIX_ENV")
  483. File.cp_r!(
  484. "rel/config/examples",
  485. Path.join(etc, "examples"),
  486. force: overwrite?
  487. )
  488. # copy /rel/config/ee-examples if profile is enterprise
  489. case profile do
  490. "emqx-enterprise" ->
  491. File.cp_r!(
  492. "rel/config/ee-examples",
  493. Path.join(etc, "examples"),
  494. force: overwrite?
  495. )
  496. _ ->
  497. :ok
  498. end
  499. # this is required by the produced escript / nodetool
  500. Mix.Generator.copy_file(
  501. Path.join(release.version_path, "start_clean.boot"),
  502. Path.join(bin, "no_dot_erlang.boot"),
  503. force: overwrite?
  504. )
  505. assigns = template_vars(release, release_type, package_type, edition_type)
  506. # This is generated by `scripts/merge-config.escript` or `make merge-config`
  507. # So, this should be run before the release.
  508. # TODO: run as a "compiler" step???
  509. render_template(
  510. "apps/emqx_conf/etc/emqx.conf.all",
  511. assigns,
  512. Path.join(etc, "emqx.conf")
  513. )
  514. render_template(
  515. "rel/emqx_vars",
  516. assigns,
  517. Path.join([release.path, "releases", "emqx_vars"])
  518. )
  519. vm_args_template_path =
  520. case release_type do
  521. :cloud ->
  522. "apps/emqx/etc/vm.args.cloud"
  523. end
  524. render_template(
  525. vm_args_template_path,
  526. assigns,
  527. [
  528. Path.join(etc, "vm.args"),
  529. Path.join(release.version_path, "vm.args")
  530. ]
  531. )
  532. for name <- [
  533. "emqx",
  534. "emqx_ctl"
  535. ] do
  536. Mix.Generator.copy_file(
  537. "bin/#{name}",
  538. Path.join(bin, name),
  539. force: overwrite?
  540. )
  541. # Files with the version appended are expected by the release
  542. # upgrade script `install_upgrade.escript`
  543. Mix.Generator.copy_file(
  544. Path.join(bin, name),
  545. Path.join(bin, name <> "-#{release.version}"),
  546. force: overwrite?
  547. )
  548. end
  549. for base_name <- ["emqx", "emqx_ctl"],
  550. suffix <- ["", "-#{release.version}"] do
  551. name = base_name <> suffix
  552. File.chmod!(Path.join(bin, name), 0o755)
  553. end
  554. Mix.Generator.copy_file(
  555. "bin/node_dump",
  556. Path.join(bin, "node_dump"),
  557. force: overwrite?
  558. )
  559. File.chmod!(Path.join(bin, "node_dump"), 0o755)
  560. Mix.Generator.copy_file(
  561. "bin/emqx_cluster_rescue",
  562. Path.join(bin, "emqx_cluster_rescue"),
  563. force: overwrite?
  564. )
  565. File.chmod!(Path.join(bin, "emqx_cluster_rescue"), 0o755)
  566. render_template(
  567. "rel/BUILD_INFO",
  568. assigns,
  569. Path.join(release.version_path, "BUILD_INFO")
  570. )
  571. release
  572. end
  573. defp render_template(template, assigns, target) when is_binary(target) do
  574. render_template(template, assigns, [target])
  575. end
  576. defp render_template(template, assigns, tartgets) when is_list(tartgets) do
  577. rendered =
  578. File.read!(template)
  579. |> from_rebar_to_eex_template()
  580. |> EEx.eval_string(assigns)
  581. for target <- tartgets do
  582. File.write!(target, rendered)
  583. end
  584. end
  585. # needed by nodetool and by release_handler
  586. defp create_RELEASES(release) do
  587. apps =
  588. Enum.map(release.applications, fn {app_name, app_props} ->
  589. app_vsn = Keyword.fetch!(app_props, :vsn)
  590. app_path =
  591. "./lib"
  592. |> Path.join("#{app_name}-#{app_vsn}")
  593. |> to_charlist()
  594. {app_name, app_vsn, app_path}
  595. end)
  596. release_entry = [
  597. {
  598. :release,
  599. to_charlist(release.name),
  600. to_charlist(release.version),
  601. release.erts_version,
  602. apps,
  603. :permanent
  604. }
  605. ]
  606. release.path
  607. |> Path.join("releases")
  608. |> Path.join("RELEASES")
  609. |> File.open!([:write, :utf8], fn handle ->
  610. IO.puts(handle, "%% coding: utf-8")
  611. :io.format(handle, ~c"~tp.~n", [release_entry])
  612. end)
  613. release
  614. end
  615. defp copy_escript(release, escript_name) do
  616. [shebang, rest] =
  617. "bin/#{escript_name}"
  618. |> File.read!()
  619. |> String.split("\n", parts: 2)
  620. # the elixir version of escript + start.boot required the boot_var
  621. # RELEASE_LIB to be defined.
  622. boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib"
  623. # Files with the version appended are expected by the release
  624. # upgrade script `install_upgrade.escript`
  625. Enum.each(
  626. [escript_name, escript_name <> "-" <> release.version],
  627. fn name ->
  628. path = Path.join([release.path, "bin", name])
  629. File.write!(path, [shebang, "\n", boot_var, "\n", rest])
  630. end
  631. )
  632. release
  633. end
  634. # The `:tar` built-in step in Mix Release does not currently add the
  635. # `etc` directory into the resulting tarball. The workaround is to
  636. # add those to the `:overlays` key before running `:tar`.
  637. # See: https://hexdocs.pm/mix/1.13.4/Mix.Release.html#__struct__/0
  638. defp prepare_tar_overlays(release) do
  639. Map.update!(
  640. release,
  641. :overlays,
  642. &[
  643. "etc",
  644. "data",
  645. "plugins",
  646. "bin/node_dump"
  647. | &1
  648. ]
  649. )
  650. end
  651. #############################################################################
  652. # Helper functions
  653. #############################################################################
  654. defp template_vars(release, release_type, :bin = _package_type, edition_type) do
  655. [
  656. emqx_default_erlang_cookie: default_cookie(),
  657. emqx_configuration_doc: emqx_configuration_doc(edition_type),
  658. platform_data_dir: "data",
  659. platform_etc_dir: "etc",
  660. platform_plugins_dir: "plugins",
  661. runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
  662. emqx_etc_dir: "$RUNNER_ROOT_DIR/etc",
  663. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  664. runner_log_dir: "$RUNNER_ROOT_DIR/log",
  665. runner_user: "",
  666. release_version: release.version,
  667. erts_vsn: release.erts_version,
  668. # FIXME: this is empty in `make emqx` ???
  669. erl_opts: "",
  670. emqx_description: emqx_description(release_type, edition_type),
  671. emqx_schema_mod: emqx_schema_mod(edition_type),
  672. is_elixir: "yes",
  673. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  674. ] ++ build_info()
  675. end
  676. defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
  677. [
  678. emqx_default_erlang_cookie: default_cookie(),
  679. emqx_configuration_doc: emqx_configuration_doc(edition_type),
  680. platform_data_dir: "/var/lib/emqx",
  681. platform_etc_dir: "/etc/emqx",
  682. platform_plugins_dir: "/var/lib/emqx/plugins",
  683. runner_bin_dir: "/usr/bin",
  684. emqx_etc_dir: "/etc/emqx",
  685. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  686. runner_log_dir: "/var/log/emqx",
  687. runner_user: "emqx",
  688. release_version: release.version,
  689. erts_vsn: release.erts_version,
  690. # FIXME: this is empty in `make emqx` ???
  691. erl_opts: "",
  692. emqx_description: emqx_description(release_type, edition_type),
  693. emqx_schema_mod: emqx_schema_mod(edition_type),
  694. is_elixir: "yes",
  695. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  696. ] ++ build_info()
  697. end
  698. defp default_cookie() do
  699. "emqx50elixir"
  700. end
  701. defp emqx_description(release_type, edition_type) do
  702. case {release_type, edition_type} do
  703. {:cloud, :enterprise} ->
  704. "EMQX Enterprise"
  705. {:cloud, :community} ->
  706. "EMQX"
  707. end
  708. end
  709. defp emqx_configuration_doc(:enterprise),
  710. do: "https://docs.emqx.com/en/enterprise/v5.0/configuration/configuration.html"
  711. defp emqx_configuration_doc(:community),
  712. do: "https://www.emqx.io/docs/en/v5.0/configuration/configuration.html"
  713. defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema
  714. defp emqx_schema_mod(:community), do: :emqx_conf_schema
  715. defp bcrypt_dep() do
  716. if enable_bcrypt?(),
  717. do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.1", override: true}],
  718. else: []
  719. end
  720. defp jq_dep() do
  721. if enable_jq?(),
  722. do: [{:jq, github: "emqx/jq", tag: "v0.3.10", override: true}],
  723. else: []
  724. end
  725. defp quicer_dep() do
  726. if enable_quicer?(),
  727. # in conflict with emqx and emqtt
  728. do: [{:quicer, github: "emqx/quic", tag: "0.0.201", override: true}],
  729. else: []
  730. end
  731. defp enable_bcrypt?() do
  732. not win32?()
  733. end
  734. defp enable_os_mon?() do
  735. not win32?()
  736. end
  737. defp enable_jq?() do
  738. not Enum.any?([
  739. build_without_jq?(),
  740. win32?()
  741. ]) or "1" == System.get_env("BUILD_WITH_JQ")
  742. end
  743. defp enable_quicer?() do
  744. not Enum.any?([
  745. build_without_quic?(),
  746. win32?(),
  747. centos6?(),
  748. macos?()
  749. ]) or "1" == System.get_env("BUILD_WITH_QUIC")
  750. end
  751. defp enable_rocksdb?() do
  752. not Enum.any?([
  753. build_without_rocksdb?(),
  754. raspbian?()
  755. ]) or "1" == System.get_env("BUILD_WITH_ROCKSDB")
  756. end
  757. defp pkg_vsn() do
  758. %{edition_type: edition_type} = check_profile!()
  759. basedir = Path.dirname(__ENV__.file)
  760. script = Path.join(basedir, "pkg-vsn.sh")
  761. os_cmd(script, [Atom.to_string(edition_type)])
  762. end
  763. defp os_cmd(script, args) do
  764. {str, 0} = System.cmd("bash", [script | args])
  765. String.trim(str)
  766. end
  767. defp win32?(),
  768. do: match?({:win_32, _}, :os.type())
  769. defp centos6?() do
  770. case File.read("/etc/centos-release") do
  771. {:ok, "CentOS release 6" <> _} ->
  772. true
  773. _ ->
  774. false
  775. end
  776. end
  777. defp macos?() do
  778. {:unix, :darwin} == :os.type()
  779. end
  780. defp raspbian?() do
  781. os_cmd("./scripts/get-distro.sh", []) =~ "raspbian"
  782. end
  783. defp build_without_jq?() do
  784. opt = System.get_env("BUILD_WITHOUT_JQ", "false")
  785. String.downcase(opt) != "false"
  786. end
  787. defp build_without_quic?() do
  788. opt = System.get_env("BUILD_WITHOUT_QUIC", "false")
  789. String.downcase(opt) != "false"
  790. end
  791. defp build_without_rocksdb?() do
  792. opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false")
  793. String.downcase(opt) != "false"
  794. end
  795. defp from_rebar_to_eex_template(str) do
  796. # we must not consider surrounding space in the template var name
  797. # because some help strings contain informative variables that
  798. # should not be interpolated, and those have no spaces.
  799. Regex.replace(
  800. ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/,
  801. str,
  802. "<%= \\g{1} %>"
  803. )
  804. end
  805. defp build_info() do
  806. [
  807. build_info_arch: to_string(:erlang.system_info(:system_architecture)),
  808. build_info_wordsize: wordsize(),
  809. build_info_os: os_cmd("./scripts/get-distro.sh", []),
  810. build_info_erlang: otp_release(),
  811. build_info_elixir: System.version(),
  812. build_info_relform: System.get_env("EMQX_REL_FORM", "tgz")
  813. ]
  814. end
  815. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142
  816. defp wordsize() do
  817. size =
  818. try do
  819. :erlang.system_info({:wordsize, :external})
  820. rescue
  821. ErlangError ->
  822. :erlang.system_info(:wordsize)
  823. end
  824. to_string(8 * size)
  825. end
  826. defp normalize_env!() do
  827. env =
  828. case Mix.env() do
  829. :dev ->
  830. :emqx
  831. env ->
  832. env
  833. end
  834. Mix.env(env)
  835. end
  836. # As from Erlang/OTP 17, the OTP release number corresponds to the
  837. # major OTP version number. No erlang:system_info() argument gives
  838. # the exact OTP version.
  839. # https://www.erlang.org/doc/man/erlang.html#system_info_otp_release
  840. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577
  841. defp otp_release() do
  842. major_version = System.otp_release()
  843. root_dir = to_string(:code.root_dir())
  844. [root_dir, "releases", major_version, "OTP_VERSION"]
  845. |> Path.join()
  846. |> File.read()
  847. |> case do
  848. {:error, _} ->
  849. major_version
  850. {:ok, version} ->
  851. version
  852. |> String.trim()
  853. |> String.split("**")
  854. |> List.first()
  855. end
  856. end
  857. defp dump_as_erl(term) do
  858. term
  859. |> then(&:io_lib.format("~0p", [&1]))
  860. |> :erlang.iolist_to_binary()
  861. end
  862. defp erlang_edition(:community), do: :ce
  863. defp erlang_edition(:enterprise), do: :ee
  864. end