mix.exs 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971
  1. defmodule EMQXUmbrella.MixProject do
  2. use Mix.Project
  3. @moduledoc """
  4. The purpose of this file is to configure the release of EMQX under
  5. Mix. Since EMQX uses its own configuration conventions and startup
  6. procedures, one cannot simply use `iex -S mix`. Instead, it's
  7. recommendd to build and use the release.
  8. ## Profiles
  9. To control the profile and edition to build, we case split on the
  10. MIX_ENV value.
  11. The following profiles are valid:
  12. * `emqx`
  13. * `emqx-enterprise`
  14. * `emqx-pkg`
  15. * `emqx-enterprise-pkg`
  16. * `dev` -> same as `emqx`, for convenience
  17. ## Release Environment Variables
  18. The release build is controlled by a few environment variables.
  19. * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
  20. tarball along with the release.
  21. """
  22. def project() do
  23. profile_info = check_profile!()
  24. version = pkg_vsn()
  25. [
  26. app: :emqx_mix,
  27. version: version,
  28. deps: deps(profile_info, version),
  29. releases: releases()
  30. ]
  31. end
  32. defp deps(profile_info, version) do
  33. # we need several overrides here because dependencies specify
  34. # other exact versions, and not ranges.
  35. [
  36. {:lc, github: "emqx/lc", tag: "0.3.2", override: true},
  37. {:redbug, "2.0.8"},
  38. {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true},
  39. {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
  40. {:ehttpc, github: "emqx/ehttpc", tag: "0.4.10", override: true},
  41. {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true},
  42. {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
  43. {:cowboy, github: "emqx/cowboy", tag: "2.9.0", override: true},
  44. {:esockd, github: "emqx/esockd", tag: "5.9.6", override: true},
  45. {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.7.2-emqx-9", override: true},
  46. {:ekka, github: "emqx/ekka", tag: "0.15.1", override: true},
  47. {:gen_rpc, github: "emqx/gen_rpc", tag: "2.8.1", override: true},
  48. {:grpc, github: "emqx/grpc-erl", tag: "0.6.7", override: true},
  49. {:minirest, github: "emqx/minirest", tag: "1.3.9", override: true},
  50. {:ecpool, github: "emqx/ecpool", tag: "0.5.3", override: true},
  51. {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true},
  52. {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
  53. # maybe forbid to fetch quicer
  54. {:emqtt,
  55. github: "emqx/emqtt", tag: "1.8.5", override: true, system_env: maybe_no_quic_env()},
  56. {:rulesql, github: "emqx/rulesql", tag: "0.1.6"},
  57. {:observer_cli, "1.7.1"},
  58. {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
  59. {:telemetry, "1.1.0"},
  60. # in conflict by emqtt and hocon
  61. {:getopt, "1.0.2", override: true},
  62. {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true},
  63. {:hocon, github: "emqx/hocon", tag: "0.39.4", override: true},
  64. {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.2", override: true},
  65. {:esasl, github: "emqx/esasl", tag: "0.2.0"},
  66. {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
  67. # in conflict by ehttpc and emqtt
  68. {:gun, github: "emqx/gun", tag: "1.3.9", override: true},
  69. # in conflict by emqx_connector and system_monitor
  70. {:epgsql, github: "emqx/epgsql", tag: "4.7.0.1", override: true},
  71. # in conflict by emqx and observer_cli
  72. {:recon, github: "ferd/recon", tag: "2.5.1", override: true},
  73. {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
  74. # in conflict by erlavro and rocketmq
  75. {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true},
  76. # dependencies of dependencies; we choose specific refs to match
  77. # what rebar3 chooses.
  78. # in conflict by gun and emqtt
  79. {:cowlib,
  80. github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
  81. # in conflict by cowboy_swagger and cowboy
  82. {:ranch,
  83. github: "ninenines/ranch", ref: "a692f44567034dacf5efcaa24a24183788594eb7", override: true},
  84. # in conflict by grpc and eetcd
  85. {:gpb, "4.19.7", override: true, runtime: false},
  86. {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true}
  87. ] ++
  88. emqx_apps(profile_info, version) ++
  89. enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
  90. end
  91. defp emqx_apps(profile_info, version) do
  92. apps = umbrella_apps() ++ enterprise_apps(profile_info)
  93. set_emqx_app_system_env(apps, profile_info, version)
  94. end
  95. defp umbrella_apps() do
  96. enterprise_apps = enterprise_umbrella_apps()
  97. "apps/*"
  98. |> Path.wildcard()
  99. |> Enum.map(fn path ->
  100. app =
  101. path
  102. |> Path.basename()
  103. |> String.to_atom()
  104. {app, path: path, manager: :rebar3, override: true}
  105. end)
  106. |> Enum.reject(fn dep_spec ->
  107. dep_spec
  108. |> elem(0)
  109. |> then(&MapSet.member?(enterprise_apps, &1))
  110. end)
  111. end
  112. defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do
  113. umbrella_apps =
  114. Enum.map(enterprise_umbrella_apps(), fn app_name ->
  115. path = "apps/#{app_name}"
  116. {app_name, path: path, manager: :rebar3, override: true}
  117. end)
  118. "lib-ee/*"
  119. |> Path.wildcard()
  120. |> Enum.filter(&File.dir?/1)
  121. |> Enum.map(fn path ->
  122. app =
  123. path
  124. |> Path.basename()
  125. |> String.to_atom()
  126. {app, path: path, manager: :rebar3, override: true}
  127. end)
  128. |> Enum.concat(umbrella_apps)
  129. end
  130. defp enterprise_apps(_profile_info) do
  131. []
  132. end
  133. # need to remove those when listing `/apps/`...
  134. defp enterprise_umbrella_apps() do
  135. MapSet.new([
  136. :emqx_bridge_kafka,
  137. :emqx_bridge_gcp_pubsub,
  138. :emqx_bridge_cassandra,
  139. :emqx_bridge_opents,
  140. :emqx_bridge_clickhouse,
  141. :emqx_bridge_dynamo,
  142. :emqx_bridge_hstreamdb,
  143. :emqx_bridge_influxdb,
  144. :emqx_bridge_iotdb,
  145. :emqx_bridge_matrix,
  146. :emqx_bridge_mongodb,
  147. :emqx_bridge_mysql,
  148. :emqx_bridge_pgsql,
  149. :emqx_bridge_redis,
  150. :emqx_bridge_rocketmq,
  151. :emqx_bridge_tdengine,
  152. :emqx_bridge_timescale,
  153. :emqx_bridge_sqlserver,
  154. :emqx_bridge_pulsar,
  155. :emqx_oracle,
  156. :emqx_bridge_oracle,
  157. :emqx_bridge_rabbitmq,
  158. :emqx_ft,
  159. :emqx_s3
  160. ])
  161. end
  162. defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
  163. [
  164. {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.2.5"},
  165. {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.9", override: true},
  166. {:wolff, github: "kafka4beam/wolff", tag: "1.7.5"},
  167. {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.2", override: true},
  168. {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0"},
  169. {:brod, github: "kafka4beam/brod", tag: "3.16.8"},
  170. {:snappyer, "1.2.8", override: true},
  171. {:crc32cer, "0.1.8", override: true},
  172. {:supervisor3, "1.1.12", override: true},
  173. {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true},
  174. # The following two are dependencies of rabbit_common. They are needed here to
  175. # make mix not complain about conflicting versions
  176. {:thoas, github: "emqx/thoas", tag: "v1.0.0", override: true},
  177. {:credentials_obfuscation,
  178. github: "emqx/credentials-obfuscation", tag: "v3.2.0", override: true},
  179. {:rabbit_common,
  180. github: "emqx/rabbitmq-server",
  181. tag: "v3.11.13-emqx",
  182. sparse: "deps/rabbit_common",
  183. override: true},
  184. {:amqp_client,
  185. github: "emqx/rabbitmq-server",
  186. tag: "v3.11.13-emqx",
  187. sparse: "deps/amqp_client",
  188. override: true},
  189. {:erlcloud, github: "emqx/erlcloud", tag: "3.6.8-emqx-1", override: true},
  190. # erlcloud's rebar.config requires rebar3 and does not support Mix,
  191. # so it tries to fetch deps from git. We need to override this.
  192. {:lhttpc, github: "erlcloud/lhttpc", tag: "1.6.2", override: true},
  193. {:eini, "1.2.9", override: true},
  194. {:base16, "1.0.0", override: true}
  195. # end of erlcloud's deps
  196. ]
  197. end
  198. defp enterprise_deps(_profile_info) do
  199. []
  200. end
  201. defp set_emqx_app_system_env(apps, profile_info, version) do
  202. system_env = emqx_app_system_env(profile_info, version) ++ maybe_no_quic_env()
  203. Enum.map(
  204. apps,
  205. fn {app, opts} ->
  206. {app,
  207. Keyword.update(
  208. opts,
  209. :system_env,
  210. system_env,
  211. &Keyword.merge(&1, system_env)
  212. )}
  213. end
  214. )
  215. end
  216. def emqx_app_system_env(profile_info, version) do
  217. erlc_options(profile_info, version)
  218. |> dump_as_erl()
  219. |> then(&[{"ERL_COMPILER_OPTIONS", &1}])
  220. end
  221. defp erlc_options(%{edition_type: edition_type}, version) do
  222. [
  223. :debug_info,
  224. {:compile_info, [{:emqx_vsn, String.to_charlist(version)}]},
  225. {:d, :EMQX_RELEASE_EDITION, erlang_edition(edition_type)},
  226. {:d, :snk_kind, :msg}
  227. ]
  228. end
  229. def maybe_no_quic_env() do
  230. if not enable_quicer?() do
  231. [{"BUILD_WITHOUT_QUIC", "true"}]
  232. else
  233. []
  234. end
  235. end
  236. defp releases() do
  237. [
  238. emqx: fn ->
  239. %{
  240. release_type: release_type,
  241. package_type: package_type,
  242. edition_type: edition_type
  243. } = check_profile!()
  244. base_steps = [
  245. &make_docs(&1),
  246. :assemble,
  247. &create_RELEASES/1,
  248. &copy_files(&1, release_type, package_type, edition_type),
  249. &copy_escript(&1, "nodetool"),
  250. &copy_escript(&1, "install_upgrade.escript")
  251. ]
  252. steps =
  253. if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
  254. base_steps ++ [&prepare_tar_overlays/1, :tar]
  255. else
  256. base_steps
  257. end
  258. [
  259. applications: applications(edition_type),
  260. skip_mode_validation_for: [
  261. :emqx_gateway,
  262. :emqx_gateway_stomp,
  263. :emqx_gateway_mqttsn,
  264. :emqx_gateway_coap,
  265. :emqx_gateway_lwm2m,
  266. :emqx_gateway_exproto,
  267. :emqx_dashboard,
  268. :emqx_resource,
  269. :emqx_connector,
  270. :emqx_exhook,
  271. :emqx_bridge,
  272. :emqx_modules,
  273. :emqx_management,
  274. :emqx_statsd,
  275. :emqx_retainer,
  276. :emqx_prometheus,
  277. :emqx_auto_subscribe,
  278. :emqx_slow_subs,
  279. :emqx_plugins
  280. ],
  281. steps: steps,
  282. strip_beams: false
  283. ]
  284. end
  285. ]
  286. end
  287. def applications(edition_type) do
  288. [
  289. crypto: :permanent,
  290. public_key: :permanent,
  291. asn1: :permanent,
  292. syntax_tools: :permanent,
  293. ssl: :permanent,
  294. os_mon: :permanent,
  295. inets: :permanent,
  296. compiler: :permanent,
  297. runtime_tools: :permanent,
  298. redbug: :permanent,
  299. xmerl: :permanent,
  300. hocon: :load,
  301. telemetry: :permanent,
  302. emqx: :load,
  303. emqx_conf: :load,
  304. emqx_machine: :permanent
  305. ] ++
  306. if(enable_rocksdb?(),
  307. do: [mnesia_rocksdb: :load],
  308. else: []
  309. ) ++
  310. [
  311. mnesia: :load,
  312. ekka: :load,
  313. emqx_plugin_libs: :load,
  314. esasl: :load,
  315. observer_cli: :permanent,
  316. tools: :permanent,
  317. covertool: :load,
  318. system_monitor: :load,
  319. emqx_utils: :load,
  320. emqx_http_lib: :permanent,
  321. emqx_resource: :permanent,
  322. emqx_connector: :permanent,
  323. emqx_authn: :permanent,
  324. emqx_authz: :permanent,
  325. emqx_auto_subscribe: :permanent,
  326. emqx_gateway: :permanent,
  327. emqx_gateway_stomp: :permanent,
  328. emqx_gateway_mqttsn: :permanent,
  329. emqx_gateway_coap: :permanent,
  330. emqx_gateway_lwm2m: :permanent,
  331. emqx_gateway_exproto: :permanent,
  332. emqx_exhook: :permanent,
  333. emqx_bridge: :permanent,
  334. emqx_rule_engine: :permanent,
  335. emqx_modules: :permanent,
  336. emqx_management: :permanent,
  337. emqx_dashboard: :permanent,
  338. emqx_retainer: :permanent,
  339. emqx_statsd: :permanent,
  340. emqx_prometheus: :permanent,
  341. emqx_psk: :permanent,
  342. emqx_slow_subs: :permanent,
  343. emqx_plugins: :permanent,
  344. emqx_mix: :none
  345. ] ++
  346. if(enable_quicer?(), do: [quicer: :permanent], else: []) ++
  347. if(enable_bcrypt?(), do: [bcrypt: :permanent], else: []) ++
  348. if(enable_jq?(), do: [jq: :load], else: []) ++
  349. if(is_app(:observer),
  350. do: [observer: :load],
  351. else: []
  352. ) ++
  353. if(edition_type == :enterprise,
  354. do: [
  355. emqx_license: :permanent,
  356. emqx_ee_conf: :load,
  357. emqx_ee_connector: :permanent,
  358. emqx_ee_bridge: :permanent,
  359. emqx_bridge_kafka: :permanent,
  360. emqx_bridge_pulsar: :permanent,
  361. emqx_bridge_gcp_pubsub: :permanent,
  362. emqx_bridge_cassandra: :permanent,
  363. emqx_bridge_opents: :permanent,
  364. emqx_bridge_clickhouse: :permanent,
  365. emqx_bridge_dynamo: :permanent,
  366. emqx_bridge_hstreamdb: :permanent,
  367. emqx_bridge_influxdb: :permanent,
  368. emqx_bridge_iotdb: :permanent,
  369. emqx_bridge_matrix: :permanent,
  370. emqx_bridge_mongodb: :permanent,
  371. emqx_bridge_mysql: :permanent,
  372. emqx_bridge_pgsql: :permanent,
  373. emqx_bridge_redis: :permanent,
  374. emqx_bridge_rocketmq: :permanent,
  375. emqx_bridge_tdengine: :permanent,
  376. emqx_bridge_timescale: :permanent,
  377. emqx_bridge_sqlserver: :permanent,
  378. emqx_oracle: :permanent,
  379. emqx_bridge_oracle: :permanent,
  380. emqx_bridge_rabbitmq: :permanent,
  381. emqx_ee_schema_registry: :permanent,
  382. emqx_eviction_agent: :permanent,
  383. emqx_node_rebalance: :permanent,
  384. emqx_ft: :permanent
  385. ],
  386. else: []
  387. )
  388. end
  389. defp is_app(name) do
  390. case Application.load(name) do
  391. :ok ->
  392. true
  393. {:error, {:already_loaded, _}} ->
  394. true
  395. _ ->
  396. false
  397. end
  398. end
  399. def check_profile!() do
  400. valid_envs = [
  401. :dev,
  402. :emqx,
  403. :"emqx-pkg",
  404. :"emqx-enterprise",
  405. :"emqx-enterprise-pkg"
  406. ]
  407. if Mix.env() not in valid_envs do
  408. formatted_envs =
  409. valid_envs
  410. |> Enum.map(&" * #{&1}")
  411. |> Enum.join("\n")
  412. Mix.raise("""
  413. Invalid env #{Mix.env()}. Valid options are:
  414. #{formatted_envs}
  415. """)
  416. end
  417. {
  418. release_type,
  419. package_type,
  420. edition_type
  421. } =
  422. case Mix.env() do
  423. :dev ->
  424. {:cloud, :bin, :community}
  425. :emqx ->
  426. {:cloud, :bin, :community}
  427. :"emqx-enterprise" ->
  428. {:cloud, :bin, :enterprise}
  429. :"emqx-pkg" ->
  430. {:cloud, :pkg, :community}
  431. :"emqx-enterprise-pkg" ->
  432. {:cloud, :pkg, :enterprise}
  433. end
  434. normalize_env!()
  435. %{
  436. release_type: release_type,
  437. package_type: package_type,
  438. edition_type: edition_type
  439. }
  440. end
  441. #############################################################################
  442. # Custom Steps
  443. #############################################################################
  444. defp make_docs(release) do
  445. profile = System.get_env("MIX_ENV")
  446. os_cmd("build", [profile, "docs"])
  447. release
  448. end
  449. defp copy_files(release, release_type, package_type, edition_type) do
  450. overwrite? = Keyword.get(release.options, :overwrite, false)
  451. bin = Path.join(release.path, "bin")
  452. etc = Path.join(release.path, "etc")
  453. log = Path.join(release.path, "log")
  454. plugins = Path.join(release.path, "plugins")
  455. Mix.Generator.create_directory(bin)
  456. Mix.Generator.create_directory(etc)
  457. Mix.Generator.create_directory(log)
  458. Mix.Generator.create_directory(plugins)
  459. Mix.Generator.create_directory(Path.join(etc, "certs"))
  460. Enum.each(
  461. ["mnesia", "configs", "patches", "scripts"],
  462. fn dir ->
  463. path = Path.join([release.path, "data", dir])
  464. Mix.Generator.create_directory(path)
  465. end
  466. )
  467. Mix.Generator.copy_file(
  468. "apps/emqx_authz/etc/acl.conf",
  469. Path.join(etc, "acl.conf"),
  470. force: overwrite?
  471. )
  472. # required by emqx_authz
  473. File.cp_r!(
  474. "apps/emqx/etc/certs",
  475. Path.join(etc, "certs")
  476. )
  477. profile = System.get_env("MIX_ENV")
  478. Mix.Generator.copy_file(
  479. "_build/docgen/#{profile}/emqx.conf.example",
  480. Path.join(etc, "emqx.conf.example"),
  481. force: overwrite?
  482. )
  483. # this is required by the produced escript / nodetool
  484. Mix.Generator.copy_file(
  485. Path.join(release.version_path, "start_clean.boot"),
  486. Path.join(bin, "no_dot_erlang.boot"),
  487. force: overwrite?
  488. )
  489. assigns = template_vars(release, release_type, package_type, edition_type)
  490. # This is generated by `scripts/merge-config.escript` or `make merge-config`
  491. # So, this should be run before the release.
  492. # TODO: run as a "compiler" step???
  493. render_template(
  494. "apps/emqx_conf/etc/emqx.conf.all",
  495. assigns,
  496. Path.join(etc, "emqx.conf")
  497. )
  498. if edition_type == :enterprise do
  499. render_template(
  500. "apps/emqx_conf/etc/emqx-enterprise.conf.all",
  501. assigns,
  502. Path.join(etc, "emqx-enterprise.conf")
  503. )
  504. end
  505. render_template(
  506. "rel/emqx_vars",
  507. assigns,
  508. Path.join([release.path, "releases", "emqx_vars"])
  509. )
  510. vm_args_template_path =
  511. case release_type do
  512. :cloud ->
  513. "apps/emqx/etc/vm.args.cloud"
  514. end
  515. render_template(
  516. vm_args_template_path,
  517. assigns,
  518. [
  519. Path.join(etc, "vm.args"),
  520. Path.join(release.version_path, "vm.args")
  521. ]
  522. )
  523. for name <- [
  524. "emqx",
  525. "emqx_ctl"
  526. ] do
  527. Mix.Generator.copy_file(
  528. "bin/#{name}",
  529. Path.join(bin, name),
  530. force: overwrite?
  531. )
  532. # Files with the version appended are expected by the release
  533. # upgrade script `install_upgrade.escript`
  534. Mix.Generator.copy_file(
  535. Path.join(bin, name),
  536. Path.join(bin, name <> "-#{release.version}"),
  537. force: overwrite?
  538. )
  539. end
  540. for base_name <- ["emqx", "emqx_ctl"],
  541. suffix <- ["", "-#{release.version}"] do
  542. name = base_name <> suffix
  543. File.chmod!(Path.join(bin, name), 0o755)
  544. end
  545. Mix.Generator.copy_file(
  546. "bin/node_dump",
  547. Path.join(bin, "node_dump"),
  548. force: overwrite?
  549. )
  550. File.chmod!(Path.join(bin, "node_dump"), 0o755)
  551. Mix.Generator.copy_file(
  552. "bin/emqx_cluster_rescue",
  553. Path.join(bin, "emqx_cluster_rescue"),
  554. force: overwrite?
  555. )
  556. File.chmod!(Path.join(bin, "emqx_cluster_rescue"), 0o755)
  557. render_template(
  558. "rel/BUILD_INFO",
  559. assigns,
  560. Path.join(release.version_path, "BUILD_INFO")
  561. )
  562. release
  563. end
  564. defp render_template(template, assigns, target) when is_binary(target) do
  565. render_template(template, assigns, [target])
  566. end
  567. defp render_template(template, assigns, tartgets) when is_list(tartgets) do
  568. rendered =
  569. File.read!(template)
  570. |> from_rebar_to_eex_template()
  571. |> EEx.eval_string(assigns)
  572. for target <- tartgets do
  573. File.write!(target, rendered)
  574. end
  575. end
  576. # needed by nodetool and by release_handler
  577. defp create_RELEASES(release) do
  578. apps =
  579. Enum.map(release.applications, fn {app_name, app_props} ->
  580. app_vsn = Keyword.fetch!(app_props, :vsn)
  581. app_path =
  582. "./lib"
  583. |> Path.join("#{app_name}-#{app_vsn}")
  584. |> to_charlist()
  585. {app_name, app_vsn, app_path}
  586. end)
  587. release_entry = [
  588. {
  589. :release,
  590. to_charlist(release.name),
  591. to_charlist(release.version),
  592. release.erts_version,
  593. apps,
  594. :permanent
  595. }
  596. ]
  597. release.path
  598. |> Path.join("releases")
  599. |> Path.join("RELEASES")
  600. |> File.open!([:write, :utf8], fn handle ->
  601. IO.puts(handle, "%% coding: utf-8")
  602. :io.format(handle, ~c"~tp.~n", [release_entry])
  603. end)
  604. release
  605. end
  606. defp copy_escript(release, escript_name) do
  607. [shebang, rest] =
  608. "bin/#{escript_name}"
  609. |> File.read!()
  610. |> String.split("\n", parts: 2)
  611. # the elixir version of escript + start.boot required the boot_var
  612. # RELEASE_LIB to be defined.
  613. boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib"
  614. # Files with the version appended are expected by the release
  615. # upgrade script `install_upgrade.escript`
  616. Enum.each(
  617. [escript_name, escript_name <> "-" <> release.version],
  618. fn name ->
  619. path = Path.join([release.path, "bin", name])
  620. File.write!(path, [shebang, "\n", boot_var, "\n", rest])
  621. end
  622. )
  623. release
  624. end
  625. # The `:tar` built-in step in Mix Release does not currently add the
  626. # `etc` directory into the resulting tarball. The workaround is to
  627. # add those to the `:overlays` key before running `:tar`.
  628. # See: https://hexdocs.pm/mix/1.13.4/Mix.Release.html#__struct__/0
  629. defp prepare_tar_overlays(release) do
  630. Map.update!(
  631. release,
  632. :overlays,
  633. &[
  634. "etc",
  635. "data",
  636. "plugins",
  637. "bin/node_dump"
  638. | &1
  639. ]
  640. )
  641. end
  642. #############################################################################
  643. # Helper functions
  644. #############################################################################
  645. defp template_vars(release, release_type, :bin = _package_type, edition_type) do
  646. [
  647. emqx_default_erlang_cookie: default_cookie(),
  648. platform_data_dir: "data",
  649. platform_etc_dir: "etc",
  650. platform_plugins_dir: "plugins",
  651. runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
  652. emqx_etc_dir: "$RUNNER_ROOT_DIR/etc",
  653. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  654. runner_log_dir: "$RUNNER_ROOT_DIR/log",
  655. runner_user: "",
  656. release_version: release.version,
  657. erts_vsn: release.erts_version,
  658. # FIXME: this is empty in `make emqx` ???
  659. erl_opts: "",
  660. emqx_description: emqx_description(release_type, edition_type),
  661. emqx_schema_mod: emqx_schema_mod(edition_type),
  662. is_elixir: "yes",
  663. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  664. ] ++ build_info()
  665. end
  666. defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
  667. [
  668. emqx_default_erlang_cookie: default_cookie(),
  669. platform_data_dir: "/var/lib/emqx",
  670. platform_etc_dir: "/etc/emqx",
  671. platform_plugins_dir: "/var/lib/emqx/plugins",
  672. runner_bin_dir: "/usr/bin",
  673. emqx_etc_dir: "/etc/emqx",
  674. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  675. runner_log_dir: "/var/log/emqx",
  676. runner_user: "emqx",
  677. release_version: release.version,
  678. erts_vsn: release.erts_version,
  679. # FIXME: this is empty in `make emqx` ???
  680. erl_opts: "",
  681. emqx_description: emqx_description(release_type, edition_type),
  682. emqx_schema_mod: emqx_schema_mod(edition_type),
  683. is_elixir: "yes",
  684. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  685. ] ++ build_info()
  686. end
  687. defp default_cookie() do
  688. "emqx50elixir"
  689. end
  690. defp emqx_description(release_type, edition_type) do
  691. case {release_type, edition_type} do
  692. {:cloud, :enterprise} ->
  693. "EMQX Enterprise"
  694. {:cloud, :community} ->
  695. "EMQX"
  696. end
  697. end
  698. defp emqx_schema_mod(:enterprise), do: :emqx_ee_conf_schema
  699. defp emqx_schema_mod(:community), do: :emqx_conf_schema
  700. defp bcrypt_dep() do
  701. if enable_bcrypt?(),
  702. do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.0", override: true}],
  703. else: []
  704. end
  705. defp jq_dep() do
  706. if enable_jq?(),
  707. do: [{:jq, github: "emqx/jq", tag: "v0.3.9", override: true}],
  708. else: []
  709. end
  710. defp quicer_dep() do
  711. if enable_quicer?(),
  712. # in conflict with emqx and emqtt
  713. do: [{:quicer, github: "emqx/quic", tag: "0.0.114", override: true}],
  714. else: []
  715. end
  716. defp enable_bcrypt?() do
  717. not win32?()
  718. end
  719. defp enable_jq?() do
  720. not Enum.any?([
  721. build_without_jq?(),
  722. win32?()
  723. ]) or "1" == System.get_env("BUILD_WITH_JQ")
  724. end
  725. defp enable_quicer?() do
  726. not Enum.any?([
  727. build_without_quic?(),
  728. win32?(),
  729. centos6?(),
  730. macos?()
  731. ]) or "1" == System.get_env("BUILD_WITH_QUIC")
  732. end
  733. defp enable_rocksdb?() do
  734. not Enum.any?([
  735. build_without_rocksdb?(),
  736. raspbian?()
  737. ]) or "1" == System.get_env("BUILD_WITH_ROCKSDB")
  738. end
  739. defp pkg_vsn() do
  740. %{edition_type: edition_type} = check_profile!()
  741. basedir = Path.dirname(__ENV__.file)
  742. script = Path.join(basedir, "pkg-vsn.sh")
  743. os_cmd(script, [Atom.to_string(edition_type)])
  744. end
  745. defp os_cmd(script, args) do
  746. {str, 0} = System.cmd("bash", [script | args])
  747. String.trim(str)
  748. end
  749. defp win32?(),
  750. do: match?({:win_32, _}, :os.type())
  751. defp centos6?() do
  752. case File.read("/etc/centos-release") do
  753. {:ok, "CentOS release 6" <> _} ->
  754. true
  755. _ ->
  756. false
  757. end
  758. end
  759. defp macos?() do
  760. {:unix, :darwin} == :os.type()
  761. end
  762. defp raspbian?() do
  763. os_cmd("./scripts/get-distro.sh", []) =~ "raspbian"
  764. end
  765. defp build_without_jq?() do
  766. opt = System.get_env("BUILD_WITHOUT_JQ", "false")
  767. String.downcase(opt) != "false"
  768. end
  769. defp build_without_quic?() do
  770. opt = System.get_env("BUILD_WITHOUT_QUIC", "false")
  771. String.downcase(opt) != "false"
  772. end
  773. defp build_without_rocksdb?() do
  774. opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false")
  775. String.downcase(opt) != "false"
  776. end
  777. defp from_rebar_to_eex_template(str) do
  778. # we must not consider surrounding space in the template var name
  779. # because some help strings contain informative variables that
  780. # should not be interpolated, and those have no spaces.
  781. Regex.replace(
  782. ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/,
  783. str,
  784. "<%= \\g{1} %>"
  785. )
  786. end
  787. defp build_info() do
  788. [
  789. build_info_arch: to_string(:erlang.system_info(:system_architecture)),
  790. build_info_wordsize: wordsize(),
  791. build_info_os: os_cmd("./scripts/get-distro.sh", []),
  792. build_info_erlang: otp_release(),
  793. build_info_elixir: System.version(),
  794. build_info_relform: System.get_env("EMQX_REL_FORM", "tgz")
  795. ]
  796. end
  797. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142
  798. defp wordsize() do
  799. size =
  800. try do
  801. :erlang.system_info({:wordsize, :external})
  802. rescue
  803. ErlangError ->
  804. :erlang.system_info(:wordsize)
  805. end
  806. to_string(8 * size)
  807. end
  808. defp normalize_env!() do
  809. env =
  810. case Mix.env() do
  811. :dev ->
  812. :emqx
  813. env ->
  814. env
  815. end
  816. Mix.env(env)
  817. end
  818. # As from Erlang/OTP 17, the OTP release number corresponds to the
  819. # major OTP version number. No erlang:system_info() argument gives
  820. # the exact OTP version.
  821. # https://www.erlang.org/doc/man/erlang.html#system_info_otp_release
  822. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577
  823. defp otp_release() do
  824. major_version = System.otp_release()
  825. root_dir = to_string(:code.root_dir())
  826. [root_dir, "releases", major_version, "OTP_VERSION"]
  827. |> Path.join()
  828. |> File.read()
  829. |> case do
  830. {:error, _} ->
  831. major_version
  832. {:ok, version} ->
  833. version
  834. |> String.trim()
  835. |> String.split("**")
  836. |> List.first()
  837. end
  838. end
  839. defp dump_as_erl(term) do
  840. term
  841. |> then(&:io_lib.format("~0p", [&1]))
  842. |> :erlang.iolist_to_binary()
  843. end
  844. defp erlang_edition(:community), do: :ce
  845. defp erlang_edition(:enterprise), do: :ee
  846. end