feat(update_appup): Compare beam files

scripts/update_appup.escript

@@ -1,41 +1,41 @@
 #!/usr/bin/env -S escript -c
+%% -*- erlang-indent-level:4 -*-
 %% A script that adds changed modules to the corresponding appup files
 
-main(_Args) ->
-    ChangedFiles = string:lexemes(os:cmd("git diff --name-only origin/master..HEAD"), "\n"),
-    AppModules0 = lists:filtermap(fun filter_erlang_modules/1, ChangedFiles),
-    %% emqx_app must always be included as we bump version number in emqx_release.hrl for each release
-    AppModules1 = [{emqx, emqx_app} | AppModules0],
-    AppModules = group_modules(AppModules1),
-    io:format("Changed modules: ~p~n", [AppModules]),
-    _ = maps:map(fun process_app/2, AppModules),
-    ok.
-
-process_app(App, Modules) ->
-    AppupFiles = filelib:wildcard(lists:concat(["{src,apps,lib-*}/**/", App, ".appup.src"])),
-    case AppupFiles of
-        [AppupFile] ->
-          update_appup(AppupFile, Modules);
-        []          ->
-          io:format("~nWARNING: Please create an stub appup src file for ~p~n", [App])
+main(Args) ->
+    #{check := Check, current_release := CurrentRelease, prepare := Prepare} =
+        parse_args(Args, #{check => false, prepare => true}),
+    case find_pred_tag(CurrentRelease) of
+        {ok, Baseline} ->
+            {CurrDir, PredDir} = prepare(Baseline, Prepare),
+            Changed = diff_releases(CurrDir, PredDir),
+            _ = maps:map(fun(App, Changes) -> process_app(Baseline, Check, App, Changes) end, Changed),
+            ok;
+        undefined ->
+            io:format(standard_error, "No appup update is needed for this release, done~n", []),
+            ok
     end.
 
-filter_erlang_modules(Filename) ->
-    case lists:reverse(filename:split(Filename)) of
-        [Module, "src"] ->
-            erl_basename("emqx", Module);
-        [Module, "src", App|_] ->
-            erl_basename(App, Module);
-        [Module, _, "src", App|_] ->
-            erl_basename(App, Module);
-        _ ->
-            false
-    end.
+parse_args([CurrentRelease = [A|_]], State) when A =/= $- ->
+    State#{current_release => CurrentRelease};
+parse_args(["--check"|Rest], State) ->
+    parse_args(Rest, State#{check => true});
+parse_args(["--skip-build"|Rest], State) ->
+    parse_args(Rest, State#{prepare => false});
+parse_args([], _) ->
+    fail("Usage:~n  update_appup.escript [--check] [--skip-build] <current_release_tag>
+
+  --check       Don't update the appup files, just check that they are complete
+  --skip-build  Don't rebuild the releases. May produce wrong appup files.
+").
 
-erl_basename(App, Name) ->
-    case filename:basename(Name, ".erl") of
-        Name   -> false;
-        Module -> {true, {list_to_atom(App), list_to_atom(Module)}}
+process_app(PredVersion, _Check, App, Changes) ->
+    AppupFiles = filelib:wildcard(lists:concat(["{src,apps,lib-*}/**/", App, ".appup.src"])),
+    case AppupFiles of
+        [AppupFile] ->
+            update_appup(PredVersion, AppupFile, Changes);
+        [] ->
+            io:format("~nWARNING: Please create an stub appup src file for ~p~n", [App])
     end.
 
 group_modules(L) ->
@@ -43,33 +43,173 @@ group_modules(L) ->
                         maps:update_with(App, fun(Tl) -> [Mod|Tl] end, [Mod], Acc)
                 end, #{}, L).
 
-update_appup(File, Modules) ->
+update_appup(_, File, {[], [], []}) ->
+    %% No changes in the app. Just check syntax of the existing appup:
+    _ = read_appup(File);
+update_appup(PredVersion, File, Changes) ->
     io:format("~nUpdating appup: ~p~n", [File]),
     {_, Upgrade0, Downgrade0} = read_appup(File),
-    Upgrade = update_actions(Modules, Upgrade0),
-    Downgrade = update_actions(Modules, Downgrade0),
+    Upgrade = update_actions(PredVersion, Changes, Upgrade0),
+    Downgrade = update_actions(PredVersion, Changes, Downgrade0),
     IOList = io_lib:format("%% -*- mode: erlang -*-
 {VSN,~n  ~p,~n  ~p}.~n", [Upgrade, Downgrade]),
-    ok = file:write_file(File, IOList).
+    ok = file:write_file(File, IOList),
+    %% Check appup syntax:
+    _ = read_appup(File).
 
-update_actions(Modules, Versions) ->
-    lists:map(fun(L) -> do_update_actions(Modules, L) end, Versions).
+update_actions(PredVersion, Changes, Versions) ->
+    lists:map(fun(L) -> do_update_actions(Changes, L) end, ensure_pred_version(PredVersion, Versions)).
 
 do_update_actions(_, Ret = {<<".*">>, _}) ->
     Ret;
-do_update_actions(Modules, {Vsn, Actions}) ->
-    {Vsn, add_modules(Modules, Actions)}.
+do_update_actions(Changes, {Vsn, Actions}) ->
+    {Vsn, process_changes(Changes, Actions)}.
+
+process_changes({New0, Changed0, Deleted0}, OldActions) ->
+    AlreadyHandled = lists:map(fun(It) -> element(2, It) end, OldActions),
+    New = New0 -- AlreadyHandled,
+    Changed = Changed0 -- AlreadyHandled,
+    Deleted = Deleted0 -- AlreadyHandled,
+    OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Changed ++ New]
+               ++ [{delete_module, M} || M <- Deleted].
 
-add_modules(NewModules, OldActions) ->
-    OldModules = lists:map(fun(It) -> element(2, It) end, OldActions),
-    Modules = NewModules -- OldModules,
-    OldActions ++ [{load_module, M, brutal_purge, soft_purge, []} || M <- Modules].
+ensure_pred_version(PredVersion, Versions) ->
+    case lists:keyfind(PredVersion, 1, Versions) of
+        false ->
+            [{PredVersion, []}|Versions];
+        _ ->
+            Versions
+    end.
 
 read_appup(File) ->
-    {ok, Bin0} = file:read_file(File),
-    %% Hack:
-    Bin1 = re:replace(Bin0, "VSN", "\"VSN\""),
-    TmpFile = filename:join("/tmp", filename:basename(File)),
-    ok = file:write_file(TmpFile, Bin1),
-    {ok, [Terms]} = file:consult(TmpFile),
-    Terms.
+    case file:script(File, [{'VSN', "VSN"}]) of
+        {ok, Terms} ->
+            Terms;
+        Error ->
+            fail("Failed to parse appup file ~s: ~p", [File, Error])
+    end.
+
+diff_releases(CurrDir, OldDir) ->
+    Curr = hashsums(find_beams(CurrDir)),
+    Old = hashsums(find_beams(OldDir)),
+    Fun = fun(App, Modules, Acc) ->
+                  OldModules = maps:get(App, Old, #{}),
+                  Acc#{App => diff_app_modules(Modules, OldModules)}
+          end,
+    maps:fold(Fun, #{}, Curr).
+
+diff_app_modules(Modules, OldModules) ->
+    {New, Changed} =
+        maps:fold( fun(Mod, MD5, {New, Changed}) ->
+                           case OldModules of
+                               #{Mod := OldMD5} when MD5 =:= OldMD5 ->
+                                   {New, Changed};
+                               #{Mod := _} ->
+                                   {[Mod|New], Changed};
+                               _ -> {New, [Mod|Changed]}
+                           end
+                   end
+                 , {[], []}
+                 , Modules
+                 ),
+    Deleted = maps:keys(maps:without(maps:keys(Modules), OldModules)),
+    {New, Changed, Deleted}.
+
+find_beams(Dir) ->
+    [filename:join(Dir, I) || I <- filelib:wildcard("**/ebin/*.beam", Dir)].
+
+prepare(Baseline, Prepare) ->
+    io:format("~n===================================~n"
+              "Baseline: ~s"
+              "~n===================================~n", [Baseline]),
+    io:format("Building the current version...~n"),
+    Prepare andalso success(cmd("make", #{args => ["emqx-rel"]}), "Failed to build HEAD"),
+    io:format("Downloading the preceding release...~n"),
+    {ok, PredRootDir} = build_pred_release(Baseline, Prepare),
+    BeamDir = "_build/emqx/rel/emqx/lib/",
+    {BeamDir, filename:join(PredRootDir, BeamDir)}.
+
+build_pred_release(Baseline, Prepare) ->
+    Repo = find_upstream_repo(),
+    BaseDir = "/tmp/emqx-baseline/",
+    Dir = filename:basename(Repo, ".git") ++ [$-|Baseline],
+    %% TODO: shallow clone
+    Script = "mkdir -p ${BASEDIR} && cd ${BASEDIR} && { git clone --branch ${TAG} ${REPO} ${DIR} || true; } && cd ${DIR} && make emqx-rel",
+    Env = [{"REPO", Repo}, {"TAG", Baseline}, {"BASEDIR", BaseDir}, {"DIR", Dir}],
+    Prepare andalso
+        success( cmd("bash", #{ args => ["-c", Script]
+                              , env  => Env
+                              })
+               , "Failed to build the baseline release"
+               ),
+    {ok, filename:join(BaseDir, Dir)}.
+
+%% @doc Find whether we are in emqx or emqx-ee
+find_upstream_repo() ->
+    Str = os:cmd("git remote get-url origin"),
+    case re:run(Str, "/([^/]+).git$", [{capture, all_but_first, list}]) of
+        {match, ["emqx"]}    -> "git@github.com:emqx/emqx.git";
+        {match, ["emqx-ee"]} -> "git@github.com:emqx/emqx-ee.git";
+        Ret                  -> fail("Cannot detect the correct upstream repo: ~p", [Ret])
+    end.
+
+find_pred_tag(CurrentRelease) ->
+    case re:run(CurrentRelease, "^([0-9]+)\.([0-9]+)\.([0-9]+)$", [{capture, all_but_first, list}]) of
+        {match, [Maj, Min, Patch]} ->
+            case list_to_integer(Patch) of
+                0 -> undefined;
+                P -> {ok, lists:flatten(io_lib:format("~s.~s.~p", [Maj, Min, P - 1]))}
+            end;
+        Err ->
+            fail("The current release tag doesn't follow semver pattern: ~p", [Err])
+    end.
+
+-spec hashsums(file:filename()) -> #{App => #{module() => binary()}}
+              when App :: atom().
+hashsums(Files) ->
+    hashsums(Files, #{}).
+
+hashsums([], Acc) ->
+    Acc;
+hashsums([File|Rest], Acc0) ->
+    [_, "ebin", Dir|_] = lists:reverse(filename:split(File)),
+    {match, [AppStr]} = re:run(Dir, "^(.*)-[^-]+$", [{capture, all_but_first, list}]),
+    App = list_to_atom(AppStr),
+    {ok, {Module, MD5}} = beam_lib:md5(File),
+    Acc = maps:update_with( App
+                          , fun(Old) -> Old #{Module => MD5} end
+                          , #{Module => MD5}
+                          , Acc0
+                          ),
+    hashsums(Rest, Acc).
+
+%% Spawn an executable and return the exit status
+cmd(Exec, Params) ->
+    case os:find_executable(Exec) of
+        false ->
+            fail("Executable not found in $PATH: ~s", [Exec]);
+        Path ->
+            Params1 = maps:to_list(maps:with([env, args, cd], Params)),
+            Port = erlang:open_port( {spawn_executable, Path}
+                                   , [ exit_status
+                                     , nouse_stdio
+                                     | Params1
+                                     ]
+                                   ),
+            receive
+                {Port, {exit_status, Status}} ->
+                    Status
+            end
+    end.
+
+success(0, _) ->
+    true;
+success(_, Msg) ->
+    fail(Msg).
+
+fail(Str) ->
+    fail(Str, []).
+
+fail(Str, Args) ->
+    io:format(standard_error, Str ++ "~n", Args),
+    halt(1).