Merge branch 'pu/json-hooks-21.04' into 'main'

JSON Hooks 0.2

See merge request apt-team/apt!166

apt-private/private-install.cc

@@ -100,7 +100,7 @@ static void RemoveDownloadNeedingItemsFromFetcher(pkgAcquire &Fetcher, bool &Tra
       I = Fetcher.ItemsBegin();
    }
 }
-bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
+bool InstallPackages(CacheFile &Cache, bool ShwKept, bool Ask, bool Safety, std::string const &Hook, CommandLine const &CmdL)
 {
    if (not RunScripts("APT::Install::Pre-Invoke"))
       return false;
@@ -168,7 +168,12 @@ bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask, bool Safety)
    if (_config->FindB("APT::Get::Download-Only",false) == false)
         Essential = !ShowEssential(c1out,Cache);
 
+   if (not Hook.empty())
+      RunJsonHook(Hook, "org.debian.apt.hooks.install.package-list", CmdL.FileList, Cache);
+
    Stats(c1out,Cache);
+   if (not Hook.empty())
+      RunJsonHook(Hook, "org.debian.apt.hooks.install.statistics", CmdL.FileList, Cache);
 
    // Sanity check
    if (Cache->BrokenCount() != 0)
@@ -944,9 +949,9 @@ bool DoInstall(CommandLine &CmdL)
    // See if we need to prompt
    // FIXME: check if really the packages in the set are going to be installed
    if (Cache->InstCount() == verset[MOD_INSTALL].size() && Cache->DelCount() == 0)
-      result = InstallPackages(Cache, false, false);
+      result = InstallPackages(Cache, false, false, true, "AptCli::Hooks::Install", CmdL);
    else
-      result = InstallPackages(Cache, false);
+      result = InstallPackages(Cache, false, true, true, "AptCli::Hooks::Install", CmdL);
 
    if (result)
       result = RunJsonHook("AptCli::Hooks::Install", "org.debian.apt.hooks.install.post", CmdL.FileList, Cache);

apt-private/private-install.h

@@ -36,8 +36,10 @@ bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, std::vector<PseudoPkg
 bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, std::vector<PseudoPkg> &VolatileCmdL, CacheFile &Cache, int UpgradeMode);
 bool DoCacheManipulationFromCommandLine(CommandLine &CmdL, CacheFile &Cache, int UpgradeMode);
 
-APT_PUBLIC bool InstallPackages(CacheFile &Cache,bool ShwKept,bool Ask = true,
-                        bool Safety = true);
+APT_PUBLIC bool InstallPackages(CacheFile &Cache, bool ShwKept, bool Ask = true,
+				bool Safety = true,
+				std::string const &Hook = "",
+				CommandLine const &CmdL = {});
 
 bool CheckNothingBroken(CacheFile &Cache);
 bool DoAutomaticRemove(CacheFile &Cache);

apt-private/private-json-hooks.cc

@@ -17,6 +17,7 @@
 #include <ostream>
 #include <sstream>
 #include <stack>
+#include <unordered_map>
 
 #include <signal.h>
 #include <sys/socket.h>
@@ -198,6 +199,33 @@ class APT_HIDDEN JsonWriter
    }
 };
 
+/**
+ * @brief Write a VerFileIterator to a JsonWriter
+ */
+static void verFiletoJson(JsonWriter &writer, CacheFile &, pkgCache::VerFileIterator const &vf)
+{
+   auto pf = vf.File(); // Packages file
+   auto rf = pf.ReleaseFile(); // release file
+
+   writer.beginObject();
+   if (not rf.end()) {
+      if (rf->Archive != 0)
+	 writer.name("archive").value(rf.Archive());
+      if (rf->Codename != 0)
+	 writer.name("codename").value(rf.Codename());
+      if (rf->Version != 0)
+	 writer.name("version").value(rf.Version());
+      if (rf->Origin != 0)
+	 writer.name("origin").value(rf.Origin());
+      if (rf->Label != 0)
+	 writer.name("label").value(rf.Label());
+      if (rf->Site != 0)
+	 writer.name("site").value(rf.Site());
+   }
+
+   writer.endObject();
+}
+
 /**
  * @brief Write a VerIterator to a JsonWriter
  */
@@ -208,6 +236,14 @@ static void verIterToJson(JsonWriter &writer, CacheFile &Cache, pkgCache::VerIte
    writer.name("version").value(Ver.VerStr());
    writer.name("architecture").value(Ver.Arch());
    writer.name("pin").value(Cache->GetPolicy().GetPriority(Ver));
+
+   writer.name("origins");
+   writer.beginArray();
+   for (auto vf = Ver.FileList(); !vf.end(); vf++)
+      if ((vf.File()->Flags & pkgCache::Flag::NotSource) == 0)
+         verFiletoJson(writer, Cache, vf);
+   writer.endArray();
+
    writer.endObject();
 }
 
@@ -230,7 +266,7 @@ static void DpkgChrootDirectory()
 /**
  * @brief Send a notification to the hook's stream
  */
-static void NotifyHook(std::ostream &os, std::string const &method, const char **FileList, CacheFile &Cache, std::set<std::string> const &UnknownPackages)
+static void NotifyHook(std::ostream &os, std::string const &method, const char **FileList, CacheFile &Cache, std::set<std::string> const &UnknownPackages, int hookVersion)
 {
    SortedPackageUniverse Universe(Cache);
    JsonWriter jsonWriter{os};
@@ -242,11 +278,14 @@ static void NotifyHook(std::ostream &os, std::string const &method, const char *
 
    /* Build params */
    jsonWriter.name("params").beginObject();
-   jsonWriter.name("command").value(FileList[0]);
-   jsonWriter.name("search-terms").beginArray();
-   for (int i = 1; FileList[i] != NULL; i++)
-      jsonWriter.value(FileList[i]);
-   jsonWriter.endArray();
+   if (FileList != nullptr)
+   {
+      jsonWriter.name("command").value(FileList[0]);
+      jsonWriter.name("search-terms").beginArray();
+      for (int i = 1; FileList[i] != NULL; i++)
+	 jsonWriter.value(FileList[i]);
+      jsonWriter.endArray();
+   }
    jsonWriter.name("unknown-packages").beginArray();
    for (auto const &PkgName : UnknownPackages)
       jsonWriter.value(PkgName);
@@ -273,7 +312,14 @@ static void NotifyHook(std::ostream &os, std::string const &method, const char *
       switch (Cache[Pkg].Mode)
       {
       case pkgDepCache::ModeInstall:
-	 jsonWriter.name("mode").value("install");
+	 if (Pkg->CurrentVer != 0 && Cache[Pkg].Upgrade() && hookVersion >= 0x020)
+	    jsonWriter.name("mode").value("upgrade");
+	 else if (Pkg->CurrentVer != 0 && Cache[Pkg].Downgrade() && hookVersion >= 0x020)
+	    jsonWriter.name("mode").value("downgrade");
+	 else if (Pkg->CurrentVer != 0 && Cache[Pkg].ReInstall() && hookVersion >= 0x020)
+	    jsonWriter.name("mode").value("reinstall");
+	 else
+	    jsonWriter.name("mode").value("install");
 	 break;
       case pkgDepCache::ModeDelete:
 	 jsonWriter.name("mode").value(Cache[Pkg].Purge() ? "purge" : "deinstall");
@@ -306,7 +352,7 @@ static void NotifyHook(std::ostream &os, std::string const &method, const char *
 static std::string BuildHelloMessage()
 {
    std::stringstream Hello;
-   JsonWriter(Hello).beginObject().name("jsonrpc").value("2.0").name("method").value("org.debian.apt.hooks.hello").name("id").value(0).name("params").beginObject().name("versions").beginArray().value("0.1").endArray().endObject().endObject();
+   JsonWriter(Hello).beginObject().name("jsonrpc").value("2.0").name("method").value("org.debian.apt.hooks.hello").name("id").value(0).name("params").beginObject().name("versions").beginArray().value("0.1").value("0.2").endArray().endObject().endObject();
 
    return Hello.str();
 }
@@ -323,11 +369,10 @@ static std::string BuildByeMessage()
 /// @brief Run the Json hook processes in the given option.
 bool RunJsonHook(std::string const &option, std::string const &method, const char **FileList, CacheFile &Cache, std::set<std::string> const &UnknownPackages)
 {
-   std::stringstream ss;
-   NotifyHook(ss, method, FileList, Cache, UnknownPackages);
-   std::string TheData = ss.str();
+   std::unordered_map<int, std::string> notifications;
    std::string HelloData = BuildHelloMessage();
    std::string ByeData = BuildByeMessage();
+   int hookVersion;
 
    bool result = true;
 
@@ -427,6 +472,21 @@ bool RunJsonHook(std::string const &option, std::string const &method, const cha
 	 goto out;
       }
 
+      if (strstr(line, "\"0.1\""))
+      {
+	 _error->Warning("Hook %s uses deprecated 0.1 protocol", Opts->Value.c_str());
+	 hookVersion = 0x010;
+      }
+      else if (strstr(line, "\"0.2\""))
+      {
+	 hookVersion = 0x020;
+      }
+      else
+      {
+	 _error->Error("Unknown hook version in handshake from hook %s: %s", Opts->Value.c_str(), line);
+	 goto out;
+      }
+
       size = getline(&line, &linesize, F);
       if (size < 0)
       {
@@ -438,9 +498,18 @@ bool RunJsonHook(std::string const &option, std::string const &method, const cha
 	 _error->Error("Expected empty line after handshake from %s, received %s", Opts->Value.c_str(), line);
 	 goto out;
       }
-
-      fwrite(TheData.data(), TheData.size(), 1, F);
-      fwrite("\n\n", 2, 1, F);
+      {
+	 std::string &data = notifications[hookVersion];
+	 if (data.empty())
+	 {
+	    std::stringstream ss;
+	    NotifyHook(ss, method, FileList, Cache, UnknownPackages, hookVersion);
+	    ;
+	    data = ss.str();
+	 }
+	 fwrite(data.data(), data.size(), 1, F);
+	 fwrite("\n\n", 2, 1, F);
+      }
 
       fwrite(ByeData.data(), ByeData.size(), 1, F);
       fwrite("\n\n", 2, 1, F);
@@ -452,6 +521,7 @@ bool RunJsonHook(std::string const &option, std::string const &method, const cha
 	 result = _error->Error("Failure running hook %s", Opts->Value.c_str());
 	 break;
       }
+
    }
    signal(SIGINT, old_sigint);
    signal(SIGPIPE, old_sigpipe);

apt-private/private-upgrade.cc

@@ -1,12 +1,14 @@
 // Includes								/*{{{*/
 #include <config.h>
 
+#include <apt-pkg/cmndline.h>
 #include <apt-pkg/configuration.h>
 #include <apt-pkg/error.h>
 #include <apt-pkg/upgrade.h>
 
 #include <apt-private/private-cachefile.h>
 #include <apt-private/private-install.h>
+#include <apt-private/private-json-hooks.h>
 #include <apt-private/private-output.h>
 #include <apt-private/private-upgrade.h>
 
@@ -24,10 +26,18 @@ static bool UpgradeHelper(CommandLine &CmdL, int UpgradeFlags)
    if (Cache.OpenForInstall() == false || Cache.CheckDeps() == false)
       return false;
 
-   if(!DoCacheManipulationFromCommandLine(CmdL, VolatileCmdL,  Cache, UpgradeFlags))
+   std::map<unsigned short, APT::VersionSet> verset;
+   std::set<std::string> UnknownPackages;
+   if (!DoCacheManipulationFromCommandLine(CmdL, VolatileCmdL, Cache, verset, UpgradeFlags, UnknownPackages))
+   {
+      RunJsonHook("AptCli::Hooks::Upgrade", "org.debian.apt.hooks.install.fail", CmdL.FileList, Cache, UnknownPackages);
       return false;
-
-   return InstallPackages(Cache,true);
+   }
+   RunJsonHook("AptCli::Hooks::Upgrade", "org.debian.apt.hooks.install.pre-prompt", CmdL.FileList, Cache);
+   if (InstallPackages(Cache, true, true, true, "AptCli::Hooks::Upgrade", CmdL))
+      return RunJsonHook("AptCli::Hooks::Upgrade", "org.debian.apt.hooks.install.post", CmdL.FileList, Cache);
+   else
+      return RunJsonHook("AptCli::Hooks::Upgrade", "org.debian.apt.hooks.install.fail", CmdL.FileList, Cache);
 }
 
 // DoDistUpgrade - Automatic smart upgrader				/*{{{*/

doc/json-hooks-protocol.md

@@ -1,3 +1,5 @@
+Version: 0.2
+
 ## JSON Hooks
 
 APT 1.6 introduces support for hooks that talk JSON-RPC 2.0. Hooks act
@@ -57,7 +59,9 @@ method `org.debian.apt.hooks.bye`.
 
 The following methods are supported:
 
-1. `org.debian.apt.hooks.install.pre-prompt` - Run before the y/n prompt
+1. `org.debian.apt.hooks.install.pre-prompt` - Run before the package list and y/n prompt
+1. `org.debian.apt.hooks.install.package-list` - (optional in 0.1) Run after the package list. You could display additional lists of packages here
+1. `org.debian.apt.hooks.install.statistics` - (optional in 0.1) Run after the count of packages to upgrade/install. You could display additional information here, such as `5 security upgrades`
 1. `org.debian.apt.hooks.install.post` - Run after success
 1. `org.debian.apt.hooks.install.fail` - Run after failed install
 1. `org.debian.apt.hooks.search.pre` - Run before search
@@ -88,24 +92,39 @@ install. Each package has the following attributes:
 - *id*: An unsigned integer describing the package
 - *name*: The name of the package
 - *architecture*: The architecture of the package. For `"all"` packages, this will be the native architecture;
-                  use per-version architecture fields to see `"all"`.
-
-- *mode*: One of `install`, `deinstall`, `purge`, or `keep`. `keep`
-          is not exposed in 0.1. To determine an upgrade, check
-          that a current version is installed.
+			  use per-version architecture fields to see `"all"`.
+
+- *mode*: One of `install`, `upgrade`, `downgrade`, `reinstall`, `deinstall`, `purge`, `keep`.
+  Version 0.1 does not implement `upgrade`, `downgrade`, and `reinstall` - all of them are represented
+  as `install`, and you have to compare the `current` version to the `install` version to figure out if
+  is one of those.
+- One of the following optional fields may be set to true to indicate a change relative to an installed version:
+- *downgrade*: true if downgrading
+- *upgrade*: true if upgrading
+- *reinstall*: true if reinstall flag is set
 - *automatic*: Whether the package is/will be automatically installed
 - *versions*: An array with up to 3 fields:
 
-  - *candidate*: The candidate version
-  - *install*: The version to be installed
-  - *current*: The version currently installed
+- *candidate*: The candidate version
+- *install*: The version to be installed
+- *current*: The version currently installed
+
+Each version is represented as an object with the following fields:
+
+- *id*: An unsigned integer
+- *version*: The version as a string
+- *architecture*: Architecture of the version
+- *pin*: The pin priority (optional)
+- *origins*: Sources from which the package is retrieved (since 0.2)
 
-  Each version is represented as an object with the following fields:
+  Each origin is represented as an object with the following fields:
 
-  - *id*: An unsigned integer
-  - *version*: The version as a string
-  - *architecture*: Architecture of the version
-  - *pin*: The pin priority (optional)
+  - *archive*: string (optional)
+  - *codename*: string (optional)
+  - *version*: string (optional)
+  - *origin*: string (optional)
+  - *label*: string (optional)
+  - *site*: string, empty for local repositories or when using mirror+file:/ method (optional)
 
 #### Example
 
@@ -157,3 +176,14 @@ protocol version only (for example, 1.7 may only support 0.2).
 
 Additional fields may be added to objects without bumping the protocol
 version.
+
+# Changes:
+
+## Version 0.2
+
+The 0.2 protocol makes one incompatible change, and adds several new compatible changes, all of which are optional in 0.1,
+but mandatory in 0.2.
+
+* (incompatible change) The `mode` flag of arguments gained `upgrade`, `downgrade`, `reinstall` modes. All of these are `install`
+* (compatible change) The hooks `org.debian.apt.hooks.install.package-list` and `org.debian.apt.hooks.install.statistics` have been added
+* (compatible change) Version objects gained a new `origins` array