Merge pull request #6307 from emqx/update-appup-ext-deps-diff

chore(appup): make update_appup.escript output only differences for external dependencies

Currently, the update_appup.escript outputs as an error the full appup
file for external dependencies, even if all the changes are already
contained in the depency.  Here, we make it only output the missing
actions to be inserted, to aid in seeing what are the differences.

<details>
<summary> Output before: </summary>

```
ERROR: Appup file for the external dependency 'ekka' is not complete.
       Missing changes: [{"0.8.1.4",
                          [{load_module,ekka_cluster_dns,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_node_monitor,brutal_purge,
                               soft_purge,[]}]},
                         {"0.8.1.3",
                          [{load_module,ekka_node_monitor,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_autocluster,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_autoheal,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_locker,brutal_purge,soft_purge,
                               []},
                           {load_module,ekka_cluster_dns,brutal_purge,
                               soft_purge,[]}]},
                         {"0.8.1.2",
                          [{load_module,ekka_ring,brutal_purge,soft_purge,[]},
                           {load_module,ekka_cluster_dns,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_node_monitor,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_autocluster,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_autoheal,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_locker,brutal_purge,soft_purge,
                               []},
                           {load_module,ekka_httpc,brutal_purge,soft_purge,
                               []}]},
                         {"0.8.1.1",
                          [{load_module,ekka_cluster_dns,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_ring,brutal_purge,soft_purge,[]},
                           {load_module,ekka_node_monitor,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_httpc,brutal_purge,soft_purge,[]},
                           {load_module,ekka_autocluster,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_autoheal,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_locker,brutal_purge,soft_purge,
                               []}]},
                         {"0.8.1",
                          [{load_module,ekka_cluster_dns,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_ring,brutal_purge,soft_purge,[]},
                           {load_module,ekka_node_monitor,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_autocluster,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_autoheal,brutal_purge,
                               soft_purge,[]},
                           {load_module,ekka_locker,brutal_purge,soft_purge,
                               []},
                           {load_module,ekka_httpc,brutal_purge,soft_purge,[]},
                           {load_module,ekka_mnesia,brutal_purge,soft_purge,
                               []}]}]
ERROR: Appup file for the external dependency 'ehttpc' is not complete.
       Missing changes: [{"0.1.10",
                          [{load_module,ehttpc_pool,brutal_purge,soft_purge,
                               []},
                           {load_module,ehttpc,brutal_purge,soft_purge,[]}]},
                         {<<"0\\.1\\.[0-7]">>,
                          [{load_module,ehttpc_pool,brutal_purge,soft_purge,
                               []},
                           {load_module,ehttpc,brutal_purge,soft_purge,[]},
                           {update,ehttpc,{advanced,[]}}]},
                         {<<"0\\.1\\.([8-9]|(1[0-1]))">>,
                          [{load_module,ehttpc,brutal_purge,soft_purge,[]},
                           {load_module,ehttpc_pool,brutal_purge,soft_purge,
                               []}]},
                         {<<".*">>,[]}]
```

</details>


<details>
<summary> Output after: </summary>

```
ERROR: Appup file for the external dependency 'ekka' is not complete.
       Missing changes: #{down =>
                              [{"0.8.1.4",
                                [{load_module,ekka_cluster_dns,brutal_purge,soft_purge,[]}]}],
                          up => []}
NOTE: Some changes above might be already covered by regexes.ERROR: Appup file for the external dependency 'ehttpc' is not complete.
       Missing changes: #{down =>
                              [{"0.1.10",
                                [{load_module,ehttpc_pool,brutal_purge,soft_purge,[]},
                                 {load_module,ehttpc,brutal_purge,soft_purge,[]}]},
                               {<<"0\\.1\\.0">>,
                                [{load_module,ehttpc_pool,brutal_purge,soft_purge,[]},
                                 {load_module,ehttpc,brutal_purge,soft_purge,[]}]},
                               {<<"0\\.1\\.[1-7]">>,
                                [{load_module,ehttpc_pool,brutal_purge,soft_purge,[]},
                                 {load_module,ehttpc,brutal_purge,soft_purge,[]}]}],
                          up =>
                              [{"0.1.10",
                                [{load_module,ehttpc_pool,brutal_purge,soft_purge,[]},
                                 {load_module,ehttpc,brutal_purge,soft_purge,[]}]},
                               {<<"0\\.1\\.[0-7]">>,
                                [{load_module,ehttpc_pool,brutal_purge,soft_purge,[]},
                                 {load_module,ehttpc,brutal_purge,soft_purge,[]}]}]}
NOTE: Some changes above might be already covered by regexes.
ERROR: Incomplete appups found. Please inspect the output for more details.
```

</details>

scripts/update_appup.escript

@@ -99,8 +99,26 @@ main(Options, Baseline) ->
                 [] ->
                     ok;
                 _ ->
-                    set_invalid(),
-                    log("ERROR: The appup files are incomplete. Missing changes:~n   ~p", [AppupChanges])
+                    Diffs =
+                        lists:filtermap(
+                          fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) ->
+                                  case parse_appup_diffs(Upgrade, OldUpgrade,
+                                                         Downgrade, OldDowngrade) of
+                                      ok ->
+                                          false;
+                                      {diffs, Diffs} ->
+                                          {true, {App, Diffs}}
+                                  end
+                          end,
+                          AppupChanges),
+                    case Diffs =:= [] of
+                        true ->
+                            ok;
+                        false ->
+                            set_invalid(),
+                            log("ERROR: The appup files are incomplete. Missing changes:~n   ~p",
+                                [Diffs])
+                    end
             end;
         false ->
             update_appups(AppupChanges)
@@ -189,9 +207,52 @@ find_appup_actions(App, CurrAppIdx, PrevAppIdx = #app{version = PrevVersion}) ->
             %% The appup file has been already updated:
             [];
        true ->
-            [{App, {Upgrade, Downgrade}}]
+            [{App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}]
     end.
 
+%% For external dependencies, show only the changes that are missing
+%% in their current appup.
+diff_appup_instructions(ComputedChanges, PresentChanges) ->
+    lists:foldr(
+      fun({Vsn, ComputedActions}, Acc) ->
+              case find_matching_version(Vsn, PresentChanges) of
+                  undefined ->
+                      [{Vsn, ComputedActions} | Acc];
+                  PresentActions ->
+                      DiffActions = ComputedActions -- PresentActions,
+                      case DiffActions of
+                          [] ->
+                              %% no diff
+                              Acc;
+                          _ ->
+                              [{Vsn, DiffActions} | Acc]
+                      end
+              end
+      end,
+      [],
+      ComputedChanges).
+
+%% For external dependencies, checks if any missing diffs are present
+%% and groups them by `up' and `down' types.
+parse_appup_diffs(Upgrade, OldUpgrade, Downgrade, OldDowngrade) ->
+    DiffUp = diff_appup_instructions(Upgrade, OldUpgrade),
+    DiffDown = diff_appup_instructions(Downgrade, OldDowngrade),
+    case {DiffUp, DiffDown} of
+        {[], []} ->
+            %% no diff for external dependency; ignore
+            ok;
+        _ ->
+            set_invalid(),
+            Diffs = #{ up => DiffUp
+                     , down => DiffDown
+                     },
+            {diffs, Diffs}
+    end.
+
+%% TODO: handle regexes
+find_matching_version(Vsn, PresentChanges) ->
+    proplists:get_value(Vsn, PresentChanges).
+
 find_old_appup_actions(App, PrevVersion) ->
     {Upgrade0, Downgrade0} =
         case locate(ebin_current, App, ".appup") of
@@ -270,12 +331,12 @@ check_appup_files() ->
 
 update_appups(Changes) ->
     lists:foreach(
-      fun({App, {Upgrade, Downgrade}}) ->
-              do_update_appup(App, Upgrade, Downgrade)
+      fun({App, {Upgrade, Downgrade, OldUpgrade, OldDowngrade}}) ->
+              do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade)
       end,
       Changes).
 
-do_update_appup(App, Upgrade, Downgrade) ->
+do_update_appup(App, Upgrade, Downgrade, OldUpgrade, OldDowngrade) ->
     case locate(src, App, ".appup.src") of
         {ok, AppupFile} ->
             render_appfile(AppupFile, Upgrade, Downgrade);
@@ -284,8 +345,16 @@ do_update_appup(App, Upgrade, Downgrade) ->
                 {ok, AppupFile} ->
                     render_appfile(AppupFile, Upgrade, Downgrade);
                 false ->
-                    set_invalid(),
-                    log("ERROR: Appup file for the external dependency '~p' is not complete.~n       Missing changes: ~p~n", [App, Upgrade])
+                    case parse_appup_diffs(Upgrade, OldUpgrade,
+                                           Downgrade, OldDowngrade) of
+                        ok ->
+                            %% no diff for external dependency; ignore
+                            ok;
+                        {diffs, Diffs} ->
+                            set_invalid(),
+                            log("ERROR: Appup file for the external dependency '~p' is not complete.~n       Missing changes: ~100p~n", [App, Diffs]),
+                            log("NOTE: Some changes above might be already covered by regexes.~n")
+                    end
             end
     end.