report packages from security-pocket (#57)

Makefile.travis

@@ -11,6 +11,7 @@ pipinstallpythonapt: pipinstallpythonapt_deps
 pipinstallpythonapt_deps:
 	pip install pyopenssl
 	pip install service_identity
+	sudo apt-get update
 	sudo apt-get -y build-dep python-apt python3-apt
 	sudo apt-get -y install libapt-pkg-dev
 

landscape/client/package/reporter.py

@@ -609,6 +609,9 @@ def _compute_packages_changes(self):
         - are now set, and were not;
         - were previously set but are not anymore;
 
+        Also, packages coming from the security pocket will be
+        reported as such.
+
         In all cases, the server is notified of the new situation
         with a "packages" message.
 
@@ -622,14 +625,17 @@ def _compute_packages_changes(self):
         old_upgrades = set(self._store.get_available_upgrades())
         old_locked = set(self._store.get_locked())
         old_autoremovable = set(self._store.get_autoremovable())
+        old_security = set(self._store.get_security())
 
         current_installed = set()
         current_available = set()
         current_upgrades = set()
         current_locked = set()
         current_autoremovable = set()
+        current_security = set()
         lsb = parse_lsb_release(LSB_RELEASE_FILENAME)
         backports_archive = "{}-backports".format(lsb["code-name"])
+        security_archive = "{}-security".format(lsb["code-name"])
 
         for package in self._facade.get_packages():
             # Don't include package versions from the official backports
@@ -664,6 +670,13 @@ def _compute_packages_changes(self):
                 if self._facade.is_package_upgrade(package):
                     current_upgrades.add(id)
 
+                # Is this package present in the security pocket?
+                security_origins = any(
+                    origin for origin in package.origins
+                    if origin.archive == security_archive)
+                if security_origins:
+                    current_security.add(id)
+
         for package in self._facade.get_locked_packages():
             hash = self._facade.get_package_hash(package)
             id = self._store.get_hash_id(hash)
@@ -675,12 +688,14 @@ def _compute_packages_changes(self):
         new_upgrades = current_upgrades - old_upgrades
         new_locked = current_locked - old_locked
         new_autoremovable = current_autoremovable - old_autoremovable
+        new_security = current_security - old_security
 
         not_installed = old_installed - current_installed
         not_available = old_available - current_available
         not_upgrades = old_upgrades - current_upgrades
         not_locked = old_locked - current_locked
         not_autoremovable = old_autoremovable - current_autoremovable
+        not_security = old_security - current_security
 
         message = {}
         if new_installed:
@@ -703,6 +718,13 @@ def _compute_packages_changes(self):
             message["not-autoremovable"] = list(
                 sequence_to_ranges(sorted(not_autoremovable)))
 
+        if new_security:
+            message["security"] = list(
+                sequence_to_ranges(sorted(new_security)))
+        if not_security:
+            message["not-security"] = list(
+                sequence_to_ranges(sorted(not_security)))
+
         if not_installed:
             message["not-installed"] = \
                 list(sequence_to_ranges(sorted(not_installed)))
@@ -722,17 +744,25 @@ def _compute_packages_changes(self):
         message["type"] = "packages"
         result = self.send_message(message)
 
-        logging.info("Queuing message with changes in known packages: "
-                     "%d installed, %d available, %d available upgrades, "
-                     "%d locked, %d autoremovable, %d not installed, "
-                     "%d not available, %d not available upgrades, "
-                     "%d not locked, %d not autoremovable. "
-                     % (len(new_installed), len(new_available),
-                        len(new_upgrades), len(new_locked),
-                        len(new_autoremovable),
-                        len(not_installed), len(not_available),
-                        len(not_upgrades), len(not_locked),
-                        len(not_autoremovable)))
+        logging.info(
+            "Queuing message with changes in known packages: "
+            "%(installed)d installed, %(available)d available, "
+            "%(upgrades)d available upgrades, %(locked)d locked, "
+            "%(auto)d autoremovable, %(security)d security, "
+            "%(not_installed)d not installed, "
+            "%(not_available)d not available, "
+            "%(not_upgrades)d not available upgrades, "
+            "%(not_locked)d not locked, "
+            "%(not_auto)d not autoremovable, "
+            "%(not_security)d not security.",
+            extra=dict(
+                installed=len(new_installed), available=len(new_available),
+                upgrades=len(new_upgrades), locked=len(new_locked),
+                auto=len(new_autoremovable), not_installed=len(not_installed),
+                not_available=len(not_available),
+                not_upgrades=len(not_upgrades), not_locked=len(not_locked),
+                not_auto=len(not_autoremovable), security=len(new_security),
+                not_security=len(not_security)))
 
         def update_currently_known(result):
             if new_installed:
@@ -755,6 +785,10 @@ def update_currently_known(result):
                 self._store.remove_locked(not_locked)
             if not_autoremovable:
                 self._store.remove_autoremovable(not_autoremovable)
+            if new_security:
+                self._store.add_security(new_security)
+            if not_security:
+                self._store.remove_security(not_security)
             # Something has changed wrt the former run, let's update the
             # timestamp and return True.
             stamp_file = self._config.detect_package_changes_stamp

landscape/client/package/tests/test_reporter.py

@@ -999,6 +999,47 @@ def test_detect_packages_changes_with_known_autoremovable(self):
         self.assertFalse(result)
         self.assertEqual([1], self.store.get_autoremovable())
 
+    @inlineCallbacks
+    def test_detect_packages_from_security_pocket(self):
+        """Packages versions coming from security are reported as such."""
+        message_store = self.broker_service.message_store
+        message_store.set_accepted_types(["packages"])
+        lsb = parse_lsb_release(LSB_RELEASE_FILENAME)
+        release_path = os.path.join(self.repository_dir, "Release")
+        with open(release_path, "w") as release:
+            release.write("Suite: {}-security".format(lsb["code-name"]))
+
+        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
+
+        yield self.reporter.detect_packages_changes()
+
+        self.assertMessages(message_store.get_pending_messages(), [{
+            "type": "packages",
+            "available": [(1, 3)],
+            "security": [(1, 3)],
+        }])
+        self.assertEqual(sorted(self.store.get_available()), [1, 2, 3])
+        self.assertEqual(sorted(self.store.get_security()), [1, 2, 3])
+
+    @inlineCallbacks
+    def test_detect_packages_not_from_security_pocket(self):
+        """Packages versions removed from security are reported as such."""
+        message_store = self.broker_service.message_store
+        message_store.set_accepted_types(["packages"])
+
+        self.store.set_hash_ids({HASH1: 1, HASH2: 2, HASH3: 3})
+        self.store.add_available([1, 2, 3])
+        self.store.add_security([1, 2])
+
+        yield self.reporter.detect_packages_changes()
+
+        self.assertMessages(message_store.get_pending_messages(), [{
+            "type": "packages",
+            "not-security": [1, 2],
+        }])
+        self.assertEqual(sorted(self.store.get_available()), [1, 2, 3])
+        self.assertEqual(self.store.get_security(), [])
+
     def test_detect_packages_changes_with_backports(self):
         """
         Package versions coming from backports aren't considered to be

landscape/lib/apt/package/store.py

@@ -232,6 +232,25 @@ def get_autoremovable(self, cursor):
         cursor.execute("SELECT id FROM autoremovable")
         return [row[0] for row in cursor.fetchall()]
 
+    @with_cursor
+    def add_security(self, cursor, ids):
+        for id in ids:
+            cursor.execute("REPLACE INTO security VALUES (?)", (id,))
+
+    @with_cursor
+    def remove_security(self, cursor, ids):
+        id_list = ",".join(str(int(id)) for id in ids)
+        cursor.execute("DELETE FROM security WHERE id IN (%s)" % id_list)
+
+    @with_cursor
+    def clear_security(self, cursor):
+        cursor.execute("DELETE FROM security")
+
+    @with_cursor
+    def get_security(self, cursor):
+        cursor.execute("SELECT id FROM security")
+        return [row[0] for row in cursor.fetchall()]
+
     @with_cursor
     def add_installed(self, cursor, ids):
         for id in ids:
@@ -450,6 +469,8 @@ def ensure_package_schema(db):
     #       try block.
     cursor = db.cursor()
     try:
+        cursor.execute("CREATE TABLE security"
+                       " (id INTEGER PRIMARY KEY)")
         cursor.execute("CREATE TABLE autoremovable"
                        " (id INTEGER PRIMARY KEY)")
         cursor.execute("CREATE TABLE locked"

landscape/message_schemas/server_bound.py

@@ -367,13 +367,16 @@
      "locked": package_ids_or_ranges,
      "autoremovable": package_ids_or_ranges,
      "not-autoremovable": package_ids_or_ranges,
+     "security": package_ids_or_ranges,
      "not-installed": package_ids_or_ranges,
      "not-available": package_ids_or_ranges,
      "not-available-upgrades": package_ids_or_ranges,
-     "not-locked": package_ids_or_ranges},
+     "not-locked": package_ids_or_ranges,
+     "not-security": package_ids_or_ranges},
     optional=["installed", "available", "available-upgrades", "locked",
               "not-available", "not-installed", "not-available-upgrades",
-              "not-locked", "autoremovable", "not-autoremovable"])
+              "not-locked", "autoremovable", "not-autoremovable", "security",
+              "not-security"])
 
 package_locks = List(Tuple(Unicode(), Unicode(), Unicode()))
 PACKAGE_LOCKS = Message(