mix.exs 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005
  1. defmodule EMQXUmbrella.MixProject do
  2. use Mix.Project
  3. @moduledoc """
  4. The purpose of this file is to configure the release of EMQX under
  5. Mix. Since EMQX uses its own configuration conventions and startup
  6. procedures, one cannot simply use `iex -S mix`. Instead, it's
  7. recommended to build and use the release.
  8. ## Profiles
  9. To control the profile and edition to build, we case split on the
  10. MIX_ENV value.
  11. The following profiles are valid:
  12. * `emqx`
  13. * `emqx-enterprise`
  14. * `emqx-pkg`
  15. * `emqx-enterprise-pkg`
  16. * `dev` -> same as `emqx`, for convenience
  17. ## Release Environment Variables
  18. The release build is controlled by a few environment variables.
  19. * `ELIXIR_MAKE_TAR` - If set to `yes`, will produce a `.tar.gz`
  20. tarball along with the release.
  21. """
  22. def project() do
  23. profile_info = check_profile!()
  24. version = pkg_vsn()
  25. [
  26. app: :emqx_mix,
  27. version: version,
  28. deps: deps(profile_info, version),
  29. releases: releases()
  30. ]
  31. end
  32. defp deps(profile_info, version) do
  33. # we need several overrides here because dependencies specify
  34. # other exact versions, and not ranges.
  35. [
  36. {:lc, github: "emqx/lc", tag: "0.3.2", override: true},
  37. {:redbug, "2.0.8"},
  38. {:covertool, github: "zmstone/covertool", tag: "2.0.4.1", override: true},
  39. {:typerefl, github: "ieQu1/typerefl", tag: "0.9.1", override: true},
  40. {:ehttpc, github: "emqx/ehttpc", tag: "0.4.11", override: true},
  41. {:gproc, github: "emqx/gproc", tag: "0.9.0.1", override: true},
  42. {:jiffy, github: "emqx/jiffy", tag: "1.0.5", override: true},
  43. {:cowboy, github: "emqx/cowboy", tag: "2.9.2", override: true},
  44. {:esockd, github: "emqx/esockd", tag: "5.9.7", override: true},
  45. {:rocksdb, github: "emqx/erlang-rocksdb", tag: "1.8.0-emqx-1", override: true},
  46. {:ekka, github: "emqx/ekka", tag: "0.15.16", override: true},
  47. {:gen_rpc, github: "emqx/gen_rpc", tag: "3.2.1", override: true},
  48. {:grpc, github: "emqx/grpc-erl", tag: "0.6.8", override: true},
  49. {:minirest, github: "emqx/minirest", tag: "1.3.13", override: true},
  50. {:ecpool, github: "emqx/ecpool", tag: "0.5.4", override: true},
  51. {:replayq, github: "emqx/replayq", tag: "0.3.7", override: true},
  52. {:pbkdf2, github: "emqx/erlang-pbkdf2", tag: "2.0.4", override: true},
  53. # maybe forbid to fetch quicer
  54. {:emqtt,
  55. github: "emqx/emqtt", tag: "1.9.1", override: true, system_env: maybe_no_quic_env()},
  56. {:rulesql, github: "emqx/rulesql", tag: "0.1.7"},
  57. {:observer_cli, "1.7.1"},
  58. {:system_monitor, github: "ieQu1/system_monitor", tag: "3.0.3"},
  59. {:telemetry, "1.1.0"},
  60. # in conflict by emqtt and hocon
  61. {:getopt, "1.0.2", override: true},
  62. {:snabbkaffe, github: "kafka4beam/snabbkaffe", tag: "1.0.8", override: true},
  63. {:hocon, github: "emqx/hocon", tag: "0.39.19", override: true},
  64. {:emqx_http_lib, github: "emqx/emqx_http_lib", tag: "0.5.3", override: true},
  65. {:esasl, github: "emqx/esasl", tag: "0.2.0"},
  66. {:jose, github: "potatosalad/erlang-jose", tag: "1.11.2"},
  67. # in conflict by ehttpc and emqtt
  68. {:gun, github: "emqx/gun", tag: "1.3.9", override: true},
  69. # in conflict by emqx_connector and system_monitor
  70. {:epgsql, github: "emqx/epgsql", tag: "4.7.0.1", override: true},
  71. # in conflict by emqx and observer_cli
  72. {:recon, github: "ferd/recon", tag: "2.5.1", override: true},
  73. {:jsx, github: "talentdeficit/jsx", tag: "v3.1.0", override: true},
  74. # in conflict by erlavro and rocketmq
  75. {:jsone, github: "emqx/jsone", tag: "1.7.1", override: true},
  76. # dependencies of dependencies; we choose specific refs to match
  77. # what rebar3 chooses.
  78. # in conflict by gun and emqtt
  79. {:cowlib,
  80. github: "ninenines/cowlib", ref: "c6553f8308a2ca5dcd69d845f0a7d098c40c3363", override: true},
  81. # in conflict by cowboy_swagger and cowboy
  82. {:ranch, github: "emqx/ranch", tag: "1.8.1-emqx", override: true},
  83. # in conflict by grpc and eetcd
  84. {:gpb, "4.19.9", override: true, runtime: false},
  85. {:hackney, github: "emqx/hackney", tag: "1.18.1-1", override: true},
  86. # set by hackney (dependency)
  87. {:ssl_verify_fun, "1.1.6", override: true},
  88. {:uuid, github: "okeuday/uuid", tag: "v2.0.6", override: true},
  89. {:quickrand, github: "okeuday/quickrand", tag: "v2.0.6", override: true},
  90. {:opentelemetry_api,
  91. github: "emqx/opentelemetry-erlang",
  92. sparse: "apps/opentelemetry_api",
  93. tag: "v1.3.2-emqx",
  94. override: true,
  95. runtime: false},
  96. {:opentelemetry,
  97. github: "emqx/opentelemetry-erlang",
  98. sparse: "apps/opentelemetry",
  99. tag: "v1.3.2-emqx",
  100. override: true,
  101. runtime: false},
  102. {:opentelemetry_api_experimental,
  103. github: "emqx/opentelemetry-erlang",
  104. sparse: "apps/opentelemetry_api_experimental",
  105. tag: "v1.3.2-emqx",
  106. override: true,
  107. runtime: false},
  108. {:opentelemetry_experimental,
  109. github: "emqx/opentelemetry-erlang",
  110. sparse: "apps/opentelemetry_experimental",
  111. tag: "v1.3.2-emqx",
  112. override: true,
  113. runtime: false},
  114. {:opentelemetry_exporter,
  115. github: "emqx/opentelemetry-erlang",
  116. sparse: "apps/opentelemetry_exporter",
  117. tag: "v1.3.2-emqx",
  118. override: true,
  119. runtime: false}
  120. ] ++
  121. emqx_apps(profile_info, version) ++
  122. enterprise_deps(profile_info) ++ bcrypt_dep() ++ jq_dep() ++ quicer_dep()
  123. end
  124. defp emqx_apps(profile_info, version) do
  125. apps = umbrella_apps(profile_info) ++ enterprise_apps(profile_info)
  126. set_emqx_app_system_env(apps, profile_info, version)
  127. end
  128. defp umbrella_apps(profile_info) do
  129. enterprise_apps = enterprise_umbrella_apps()
  130. "apps/*"
  131. |> Path.wildcard()
  132. |> Enum.map(fn path ->
  133. app =
  134. path
  135. |> Path.basename()
  136. |> String.to_atom()
  137. {app, path: path, manager: :rebar3, override: true}
  138. end)
  139. |> Enum.reject(fn dep_spec ->
  140. dep_spec
  141. |> elem(0)
  142. |> then(&MapSet.member?(enterprise_apps, &1))
  143. end)
  144. |> Enum.reject(fn {app, _} ->
  145. case profile_info do
  146. %{edition_type: :enterprise} ->
  147. app == :emqx_telemetry
  148. _ ->
  149. false
  150. end
  151. end)
  152. end
  153. defp enterprise_apps(_profile_info = %{edition_type: :enterprise}) do
  154. umbrella_apps =
  155. Enum.map(enterprise_umbrella_apps(), fn app_name ->
  156. path = "apps/#{app_name}"
  157. {app_name, path: path, manager: :rebar3, override: true}
  158. end)
  159. "lib-ee/*"
  160. |> Path.wildcard()
  161. |> Enum.filter(&File.dir?/1)
  162. |> Enum.map(fn path ->
  163. app =
  164. path
  165. |> Path.basename()
  166. |> String.to_atom()
  167. {app, path: path, manager: :rebar3, override: true}
  168. end)
  169. |> Enum.concat(umbrella_apps)
  170. end
  171. defp enterprise_apps(_profile_info) do
  172. []
  173. end
  174. # need to remove those when listing `/apps/`...
  175. defp enterprise_umbrella_apps() do
  176. MapSet.new([
  177. :emqx_bridge_kafka,
  178. :emqx_bridge_gcp_pubsub,
  179. :emqx_bridge_cassandra,
  180. :emqx_bridge_opents,
  181. :emqx_bridge_dynamo,
  182. :emqx_bridge_greptimedb,
  183. :emqx_bridge_hstreamdb,
  184. :emqx_bridge_influxdb,
  185. :emqx_bridge_iotdb,
  186. :emqx_bridge_matrix,
  187. :emqx_bridge_mongodb,
  188. :emqx_bridge_mysql,
  189. :emqx_bridge_pgsql,
  190. :emqx_bridge_redis,
  191. :emqx_bridge_rocketmq,
  192. :emqx_bridge_tdengine,
  193. :emqx_bridge_timescale,
  194. :emqx_bridge_sqlserver,
  195. :emqx_bridge_pulsar,
  196. :emqx_oracle,
  197. :emqx_bridge_oracle,
  198. :emqx_bridge_rabbitmq,
  199. :emqx_bridge_clickhouse,
  200. :emqx_ft,
  201. :emqx_s3,
  202. :emqx_schema_registry,
  203. :emqx_enterprise,
  204. :emqx_bridge_kinesis,
  205. :emqx_bridge_azure_event_hub,
  206. :emqx_ldap,
  207. :emqx_auth_ldap,
  208. :emqx_gcp_device,
  209. :emqx_dashboard_rbac,
  210. :emqx_dashboard_sso
  211. ])
  212. end
  213. defp enterprise_deps(_profile_info = %{edition_type: :enterprise}) do
  214. [
  215. {:hstreamdb_erl, github: "hstreamdb/hstreamdb_erl", tag: "0.4.5+v0.16.1"},
  216. {:influxdb, github: "emqx/influxdb-client-erl", tag: "1.1.11", override: true},
  217. {:wolff, github: "kafka4beam/wolff", tag: "1.8.0"},
  218. {:kafka_protocol, github: "kafka4beam/kafka_protocol", tag: "4.1.3", override: true},
  219. {:brod_gssapi, github: "kafka4beam/brod_gssapi", tag: "v0.1.0"},
  220. {:brod, github: "kafka4beam/brod", tag: "3.16.8"},
  221. {:snappyer, "1.2.9", override: true},
  222. {:crc32cer, "0.1.8", override: true},
  223. {:supervisor3, "1.1.12", override: true},
  224. {:opentsdb, github: "emqx/opentsdb-client-erl", tag: "v0.5.1", override: true},
  225. {:greptimedb, github: "GreptimeTeam/greptimedb-client-erl", tag: "v0.1.2", override: true},
  226. # The following two are dependencies of rabbit_common. They are needed here to
  227. # make mix not complain about conflicting versions
  228. {:thoas, github: "emqx/thoas", tag: "v1.0.0", override: true},
  229. {:credentials_obfuscation,
  230. github: "emqx/credentials-obfuscation", tag: "v3.2.0", override: true},
  231. {:rabbit_common,
  232. github: "emqx/rabbitmq-server",
  233. tag: "v3.11.13-emqx",
  234. sparse: "deps/rabbit_common",
  235. override: true},
  236. {:amqp_client,
  237. github: "emqx/rabbitmq-server",
  238. tag: "v3.11.13-emqx",
  239. sparse: "deps/amqp_client",
  240. override: true}
  241. ]
  242. end
  243. defp enterprise_deps(_profile_info) do
  244. []
  245. end
  246. defp set_emqx_app_system_env(apps, profile_info, version) do
  247. system_env = emqx_app_system_env(profile_info, version) ++ maybe_no_quic_env()
  248. Enum.map(
  249. apps,
  250. fn {app, opts} ->
  251. {app,
  252. Keyword.update(
  253. opts,
  254. :system_env,
  255. system_env,
  256. &Keyword.merge(&1, system_env)
  257. )}
  258. end
  259. )
  260. end
  261. def emqx_app_system_env(profile_info, version) do
  262. erlc_options(profile_info, version)
  263. |> dump_as_erl()
  264. |> then(&[{"ERL_COMPILER_OPTIONS", &1}])
  265. end
  266. defp erlc_options(%{edition_type: edition_type}, version) do
  267. [
  268. :debug_info,
  269. {:compile_info, [{:emqx_vsn, String.to_charlist(version)}]},
  270. {:d, :EMQX_RELEASE_EDITION, erlang_edition(edition_type)},
  271. {:d, :snk_kind, :msg}
  272. ]
  273. end
  274. def maybe_no_quic_env() do
  275. if not enable_quicer?() do
  276. [{"BUILD_WITHOUT_QUIC", "true"}]
  277. else
  278. []
  279. end
  280. end
  281. defp releases() do
  282. [
  283. emqx: fn ->
  284. %{
  285. release_type: release_type,
  286. package_type: package_type,
  287. edition_type: edition_type
  288. } = check_profile!()
  289. base_steps = [
  290. &make_docs(&1),
  291. :assemble,
  292. &create_RELEASES/1,
  293. &copy_files(&1, release_type, package_type, edition_type),
  294. &copy_escript(&1, "nodetool"),
  295. &copy_escript(&1, "install_upgrade.escript")
  296. ]
  297. steps =
  298. if System.get_env("ELIXIR_MAKE_TAR") == "yes" do
  299. base_steps ++ [&prepare_tar_overlays/1, :tar]
  300. else
  301. base_steps
  302. end
  303. [
  304. applications: applications(edition_type),
  305. skip_mode_validation_for: [
  306. :emqx_mix,
  307. :emqx_gateway,
  308. :emqx_gateway_stomp,
  309. :emqx_gateway_mqttsn,
  310. :emqx_gateway_coap,
  311. :emqx_gateway_lwm2m,
  312. :emqx_gateway_exproto,
  313. :emqx_dashboard,
  314. :emqx_resource,
  315. :emqx_connector,
  316. :emqx_exhook,
  317. :emqx_bridge,
  318. :emqx_bridge_mqtt,
  319. :emqx_modules,
  320. :emqx_management,
  321. :emqx_retainer,
  322. :emqx_prometheus,
  323. :emqx_auto_subscribe,
  324. :emqx_slow_subs,
  325. :emqx_plugins,
  326. :emqx_ft,
  327. :emqx_s3,
  328. :emqx_opentelemetry,
  329. :emqx_durable_storage,
  330. :rabbit_common
  331. ],
  332. steps: steps,
  333. strip_beams: false
  334. ]
  335. end
  336. ]
  337. end
  338. def applications(edition_type) do
  339. {:ok,
  340. [
  341. %{
  342. db_apps: db_apps,
  343. system_apps: system_apps,
  344. common_business_apps: common_business_apps,
  345. ee_business_apps: ee_business_apps,
  346. ce_business_apps: ce_business_apps
  347. }
  348. ]} = :file.consult("apps/emqx_machine/priv/reboot_lists.eterm")
  349. edition_specific_apps =
  350. if edition_type == :enterprise do
  351. ee_business_apps
  352. else
  353. ce_business_apps
  354. end
  355. business_apps = common_business_apps ++ edition_specific_apps
  356. excluded_apps = excluded_apps()
  357. system_apps =
  358. Enum.map(system_apps, fn app ->
  359. if is_atom(app), do: {app, :permanent}, else: app
  360. end)
  361. db_apps = Enum.map(db_apps, &{&1, :load})
  362. business_apps = Enum.map(business_apps, &{&1, :load})
  363. [system_apps, db_apps, [emqx_machine: :permanent], business_apps]
  364. |> List.flatten()
  365. |> Keyword.reject(fn {app, _type} -> app in excluded_apps end)
  366. end
  367. defp excluded_apps() do
  368. %{
  369. mnesia_rocksdb: enable_rocksdb?(),
  370. quicer: enable_quicer?(),
  371. bcrypt: enable_bcrypt?(),
  372. jq: enable_jq?(),
  373. observer: is_app?(:observer),
  374. os_mon: enable_os_mon?()
  375. }
  376. |> Enum.reject(&elem(&1, 1))
  377. |> Enum.map(&elem(&1, 0))
  378. end
  379. defp is_app?(name) do
  380. case Application.load(name) do
  381. :ok ->
  382. true
  383. {:error, {:already_loaded, _}} ->
  384. true
  385. _ ->
  386. false
  387. end
  388. end
  389. def check_profile!() do
  390. valid_envs = [
  391. :emqx,
  392. :"emqx-pkg",
  393. :"emqx-enterprise",
  394. :"emqx-enterprise-pkg"
  395. ]
  396. if Mix.env() == :dev do
  397. env_profile = System.get_env("PROFILE")
  398. if env_profile do
  399. # copy from PROFILE env var
  400. System.get_env("PROFILE")
  401. |> String.to_atom()
  402. |> Mix.env()
  403. else
  404. IO.puts(
  405. IO.ANSI.format([
  406. :yellow,
  407. "Warning: env var PROFILE is unset; defaulting to emqx"
  408. ])
  409. )
  410. Mix.env(:emqx)
  411. end
  412. end
  413. if Mix.env() not in valid_envs do
  414. formatted_envs =
  415. valid_envs
  416. |> Enum.map(&" * #{&1}")
  417. |> Enum.join("\n")
  418. Mix.raise("""
  419. Invalid env #{Mix.env()}. Valid options are:
  420. #{formatted_envs}
  421. """)
  422. end
  423. {
  424. release_type,
  425. package_type,
  426. edition_type
  427. } =
  428. case Mix.env() do
  429. :dev ->
  430. {:cloud, :bin, :community}
  431. :emqx ->
  432. {:cloud, :bin, :community}
  433. :"emqx-enterprise" ->
  434. {:cloud, :bin, :enterprise}
  435. :"emqx-pkg" ->
  436. {:cloud, :pkg, :community}
  437. :"emqx-enterprise-pkg" ->
  438. {:cloud, :pkg, :enterprise}
  439. end
  440. normalize_env!()
  441. %{
  442. release_type: release_type,
  443. package_type: package_type,
  444. edition_type: edition_type
  445. }
  446. end
  447. #############################################################################
  448. # Custom Steps
  449. #############################################################################
  450. defp make_docs(release) do
  451. profile = System.get_env("MIX_ENV")
  452. os_cmd("build", [profile, "docs"])
  453. release
  454. end
  455. defp copy_files(release, release_type, package_type, edition_type) do
  456. overwrite? = Keyword.get(release.options, :overwrite, false)
  457. bin = Path.join(release.path, "bin")
  458. etc = Path.join(release.path, "etc")
  459. log = Path.join(release.path, "log")
  460. plugins = Path.join(release.path, "plugins")
  461. Mix.Generator.create_directory(bin)
  462. Mix.Generator.create_directory(etc)
  463. Mix.Generator.create_directory(log)
  464. Mix.Generator.create_directory(plugins)
  465. Mix.Generator.create_directory(Path.join(etc, "certs"))
  466. Enum.each(
  467. ["mnesia", "configs", "patches", "scripts"],
  468. fn dir ->
  469. path = Path.join([release.path, "data", dir])
  470. Mix.Generator.create_directory(path)
  471. end
  472. )
  473. Mix.Generator.copy_file(
  474. "apps/emqx_auth/etc/acl.conf",
  475. Path.join(etc, "acl.conf"),
  476. force: overwrite?
  477. )
  478. # required by emqx_auth
  479. File.cp_r!(
  480. "apps/emqx/etc/certs",
  481. Path.join(etc, "certs")
  482. )
  483. profile = System.get_env("MIX_ENV")
  484. File.cp_r!(
  485. "rel/config/examples",
  486. Path.join(etc, "examples"),
  487. force: overwrite?
  488. )
  489. # copy /rel/config/ee-examples if profile is enterprise
  490. case profile do
  491. "emqx-enterprise" ->
  492. File.cp_r!(
  493. "rel/config/ee-examples",
  494. Path.join(etc, "examples"),
  495. force: overwrite?
  496. )
  497. _ ->
  498. :ok
  499. end
  500. # this is required by the produced escript / nodetool
  501. Mix.Generator.copy_file(
  502. Path.join(release.version_path, "start_clean.boot"),
  503. Path.join(bin, "no_dot_erlang.boot"),
  504. force: overwrite?
  505. )
  506. assigns = template_vars(release, release_type, package_type, edition_type)
  507. # This is generated by `scripts/merge-config.escript` or `make merge-config`
  508. # So, this should be run before the release.
  509. # TODO: run as a "compiler" step???
  510. render_template(
  511. "apps/emqx_conf/etc/emqx.conf.all",
  512. assigns,
  513. Path.join(etc, "emqx.conf")
  514. )
  515. render_template(
  516. "rel/emqx_vars",
  517. assigns,
  518. Path.join([release.path, "releases", "emqx_vars"])
  519. )
  520. vm_args_template_path =
  521. case release_type do
  522. :cloud ->
  523. "apps/emqx/etc/vm.args.cloud"
  524. end
  525. render_template(
  526. vm_args_template_path,
  527. assigns,
  528. [
  529. Path.join(etc, "vm.args"),
  530. Path.join(release.version_path, "vm.args")
  531. ]
  532. )
  533. for name <- [
  534. "emqx",
  535. "emqx_ctl"
  536. ] do
  537. Mix.Generator.copy_file(
  538. "bin/#{name}",
  539. Path.join(bin, name),
  540. force: overwrite?
  541. )
  542. # Files with the version appended are expected by the release
  543. # upgrade script `install_upgrade.escript`
  544. Mix.Generator.copy_file(
  545. Path.join(bin, name),
  546. Path.join(bin, name <> "-#{release.version}"),
  547. force: overwrite?
  548. )
  549. end
  550. for base_name <- ["emqx", "emqx_ctl"],
  551. suffix <- ["", "-#{release.version}"] do
  552. name = base_name <> suffix
  553. File.chmod!(Path.join(bin, name), 0o755)
  554. end
  555. Mix.Generator.copy_file(
  556. "bin/node_dump",
  557. Path.join(bin, "node_dump"),
  558. force: overwrite?
  559. )
  560. File.chmod!(Path.join(bin, "node_dump"), 0o755)
  561. Mix.Generator.copy_file(
  562. "bin/emqx_cluster_rescue",
  563. Path.join(bin, "emqx_cluster_rescue"),
  564. force: overwrite?
  565. )
  566. File.chmod!(Path.join(bin, "emqx_cluster_rescue"), 0o755)
  567. render_template(
  568. "rel/BUILD_INFO",
  569. assigns,
  570. Path.join(release.version_path, "BUILD_INFO")
  571. )
  572. release
  573. end
  574. defp render_template(template, assigns, target) when is_binary(target) do
  575. render_template(template, assigns, [target])
  576. end
  577. defp render_template(template, assigns, tartgets) when is_list(tartgets) do
  578. rendered =
  579. File.read!(template)
  580. |> from_rebar_to_eex_template()
  581. |> EEx.eval_string(assigns)
  582. for target <- tartgets do
  583. File.write!(target, rendered)
  584. end
  585. end
  586. # needed by nodetool and by release_handler
  587. defp create_RELEASES(release) do
  588. apps =
  589. Enum.map(release.applications, fn {app_name, app_props} ->
  590. app_vsn = Keyword.fetch!(app_props, :vsn)
  591. app_path =
  592. "./lib"
  593. |> Path.join("#{app_name}-#{app_vsn}")
  594. |> to_charlist()
  595. {app_name, app_vsn, app_path}
  596. end)
  597. release_entry = [
  598. {
  599. :release,
  600. to_charlist(release.name),
  601. to_charlist(release.version),
  602. release.erts_version,
  603. apps,
  604. :permanent
  605. }
  606. ]
  607. release.path
  608. |> Path.join("releases")
  609. |> Path.join("RELEASES")
  610. |> File.open!([:write, :utf8], fn handle ->
  611. IO.puts(handle, "%% coding: utf-8")
  612. :io.format(handle, ~c"~tp.~n", [release_entry])
  613. end)
  614. release
  615. end
  616. defp copy_escript(release, escript_name) do
  617. [shebang, rest] =
  618. "bin/#{escript_name}"
  619. |> File.read!()
  620. |> String.split("\n", parts: 2)
  621. # the elixir version of escript + start.boot required the boot_var
  622. # RELEASE_LIB to be defined.
  623. boot_var = "%%!-boot_var RELEASE_LIB $RUNNER_ROOT_DIR/lib"
  624. # Files with the version appended are expected by the release
  625. # upgrade script `install_upgrade.escript`
  626. Enum.each(
  627. [escript_name, escript_name <> "-" <> release.version],
  628. fn name ->
  629. path = Path.join([release.path, "bin", name])
  630. File.write!(path, [shebang, "\n", boot_var, "\n", rest])
  631. end
  632. )
  633. release
  634. end
  635. # The `:tar` built-in step in Mix Release does not currently add the
  636. # `etc` directory into the resulting tarball. The workaround is to
  637. # add those to the `:overlays` key before running `:tar`.
  638. # See: https://hexdocs.pm/mix/1.13.4/Mix.Release.html#__struct__/0
  639. defp prepare_tar_overlays(release) do
  640. Map.update!(
  641. release,
  642. :overlays,
  643. &[
  644. "etc",
  645. "data",
  646. "plugins",
  647. "bin/node_dump"
  648. | &1
  649. ]
  650. )
  651. end
  652. #############################################################################
  653. # Helper functions
  654. #############################################################################
  655. defp template_vars(release, release_type, :bin = _package_type, edition_type) do
  656. [
  657. emqx_default_erlang_cookie: default_cookie(),
  658. emqx_configuration_doc: emqx_configuration_doc(edition_type),
  659. platform_data_dir: "data",
  660. platform_etc_dir: "etc",
  661. platform_plugins_dir: "plugins",
  662. runner_bin_dir: "$RUNNER_ROOT_DIR/bin",
  663. emqx_etc_dir: "$RUNNER_ROOT_DIR/etc",
  664. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  665. runner_log_dir: "$RUNNER_ROOT_DIR/log",
  666. runner_user: "",
  667. release_version: release.version,
  668. erts_vsn: release.erts_version,
  669. # FIXME: this is empty in `make emqx` ???
  670. erl_opts: "",
  671. emqx_description: emqx_description(release_type, edition_type),
  672. emqx_schema_mod: emqx_schema_mod(edition_type),
  673. is_elixir: "yes",
  674. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  675. ] ++ build_info()
  676. end
  677. defp template_vars(release, release_type, :pkg = _package_type, edition_type) do
  678. [
  679. emqx_default_erlang_cookie: default_cookie(),
  680. emqx_configuration_doc: emqx_configuration_doc(edition_type),
  681. platform_data_dir: "/var/lib/emqx",
  682. platform_etc_dir: "/etc/emqx",
  683. platform_plugins_dir: "/var/lib/emqx/plugins",
  684. runner_bin_dir: "/usr/bin",
  685. emqx_etc_dir: "/etc/emqx",
  686. runner_lib_dir: "$RUNNER_ROOT_DIR/lib",
  687. runner_log_dir: "/var/log/emqx",
  688. runner_user: "emqx",
  689. release_version: release.version,
  690. erts_vsn: release.erts_version,
  691. # FIXME: this is empty in `make emqx` ???
  692. erl_opts: "",
  693. emqx_description: emqx_description(release_type, edition_type),
  694. emqx_schema_mod: emqx_schema_mod(edition_type),
  695. is_elixir: "yes",
  696. is_enterprise: if(edition_type == :enterprise, do: "yes", else: "no")
  697. ] ++ build_info()
  698. end
  699. defp default_cookie() do
  700. "emqx50elixir"
  701. end
  702. defp emqx_description(release_type, edition_type) do
  703. case {release_type, edition_type} do
  704. {:cloud, :enterprise} ->
  705. "EMQX Enterprise"
  706. {:cloud, :community} ->
  707. "EMQX"
  708. end
  709. end
  710. defp emqx_configuration_doc(:enterprise),
  711. do: "https://docs.emqx.com/en/enterprise/v5.0/configuration/configuration.html"
  712. defp emqx_configuration_doc(:community),
  713. do: "https://www.emqx.io/docs/en/v5.0/configuration/configuration.html"
  714. defp emqx_schema_mod(:enterprise), do: :emqx_enterprise_schema
  715. defp emqx_schema_mod(:community), do: :emqx_conf_schema
  716. defp bcrypt_dep() do
  717. if enable_bcrypt?(),
  718. do: [{:bcrypt, github: "emqx/erlang-bcrypt", tag: "0.6.1", override: true}],
  719. else: []
  720. end
  721. defp jq_dep() do
  722. if enable_jq?(),
  723. do: [{:jq, github: "emqx/jq", tag: "v0.3.11", override: true}],
  724. else: []
  725. end
  726. defp quicer_dep() do
  727. if enable_quicer?(),
  728. # in conflict with emqx and emqtt
  729. do: [{:quicer, github: "emqx/quic", tag: "0.0.202", override: true}],
  730. else: []
  731. end
  732. defp enable_bcrypt?() do
  733. not win32?()
  734. end
  735. defp enable_os_mon?() do
  736. not win32?()
  737. end
  738. defp enable_jq?() do
  739. not Enum.any?([
  740. build_without_jq?(),
  741. win32?()
  742. ]) or "1" == System.get_env("BUILD_WITH_JQ")
  743. end
  744. defp enable_quicer?() do
  745. not Enum.any?([
  746. build_without_quic?(),
  747. win32?(),
  748. centos6?(),
  749. macos?()
  750. ]) or "1" == System.get_env("BUILD_WITH_QUIC")
  751. end
  752. defp enable_rocksdb?() do
  753. not Enum.any?([
  754. build_without_rocksdb?(),
  755. raspbian?()
  756. ]) or "1" == System.get_env("BUILD_WITH_ROCKSDB")
  757. end
  758. defp pkg_vsn() do
  759. %{edition_type: edition_type} = check_profile!()
  760. basedir = Path.dirname(__ENV__.file)
  761. script = Path.join(basedir, "pkg-vsn.sh")
  762. os_cmd(script, [Atom.to_string(edition_type)])
  763. end
  764. defp os_cmd(script, args) do
  765. {str, 0} = System.cmd("bash", [script | args])
  766. String.trim(str)
  767. end
  768. defp win32?(),
  769. do: match?({:win_32, _}, :os.type())
  770. defp centos6?() do
  771. case File.read("/etc/centos-release") do
  772. {:ok, "CentOS release 6" <> _} ->
  773. true
  774. _ ->
  775. false
  776. end
  777. end
  778. defp macos?() do
  779. {:unix, :darwin} == :os.type()
  780. end
  781. defp raspbian?() do
  782. os_cmd("./scripts/get-distro.sh", []) =~ "raspbian"
  783. end
  784. defp build_without_jq?() do
  785. opt = System.get_env("BUILD_WITHOUT_JQ", "false")
  786. String.downcase(opt) != "false"
  787. end
  788. defp build_without_quic?() do
  789. opt = System.get_env("BUILD_WITHOUT_QUIC", "false")
  790. String.downcase(opt) != "false"
  791. end
  792. defp build_without_rocksdb?() do
  793. opt = System.get_env("BUILD_WITHOUT_ROCKSDB", "false")
  794. String.downcase(opt) != "false"
  795. end
  796. defp from_rebar_to_eex_template(str) do
  797. # we must not consider surrounding space in the template var name
  798. # because some help strings contain informative variables that
  799. # should not be interpolated, and those have no spaces.
  800. Regex.replace(
  801. ~r/\{\{ ([a-zA-Z0-9_]+) \}\}/,
  802. str,
  803. "<%= \\g{1} %>"
  804. )
  805. end
  806. defp build_info() do
  807. [
  808. build_info_arch: to_string(:erlang.system_info(:system_architecture)),
  809. build_info_wordsize: wordsize(),
  810. build_info_os: os_cmd("./scripts/get-distro.sh", []),
  811. build_info_erlang: otp_release(),
  812. build_info_elixir: System.version(),
  813. build_info_relform: System.get_env("EMQX_REL_FORM", "tgz")
  814. ]
  815. end
  816. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L142
  817. defp wordsize() do
  818. size =
  819. try do
  820. :erlang.system_info({:wordsize, :external})
  821. rescue
  822. ErlangError ->
  823. :erlang.system_info(:wordsize)
  824. end
  825. to_string(8 * size)
  826. end
  827. defp normalize_env!() do
  828. env =
  829. case Mix.env() do
  830. :dev ->
  831. :emqx
  832. env ->
  833. env
  834. end
  835. Mix.env(env)
  836. end
  837. # As from Erlang/OTP 17, the OTP release number corresponds to the
  838. # major OTP version number. No erlang:system_info() argument gives
  839. # the exact OTP version.
  840. # https://www.erlang.org/doc/man/erlang.html#system_info_otp_release
  841. # https://github.com/erlang/rebar3/blob/e3108ac187b88fff01eca6001a856283a3e0ec87/src/rebar_utils.erl#L572-L577
  842. defp otp_release() do
  843. major_version = System.otp_release()
  844. root_dir = to_string(:code.root_dir())
  845. [root_dir, "releases", major_version, "OTP_VERSION"]
  846. |> Path.join()
  847. |> File.read()
  848. |> case do
  849. {:error, _} ->
  850. major_version
  851. {:ok, version} ->
  852. version
  853. |> String.trim()
  854. |> String.split("**")
  855. |> List.first()
  856. end
  857. end
  858. defp dump_as_erl(term) do
  859. term
  860. |> then(&:io_lib.format("~0p", [&1]))
  861. |> :erlang.iolist_to_binary()
  862. end
  863. defp erlang_edition(:community), do: :ce
  864. defp erlang_edition(:enterprise), do: :ee
  865. end