Add entitlements to fingerprinting engine. (#794)

quipucords/api/common/serializer.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2017 Red Hat, Inc.
+# Copyright (c) 2017-2018 Red Hat, Inc.
 #
 # This software is licensed to you under the GNU General Public License,
 # version 3 (GPLv3). There is NO WARRANTY for this software, express or
@@ -79,6 +79,6 @@ def to_internal_value(self, data):
 
     def to_representation(self, value):
         """Transform JSON str to python object."""
-        if bool(value):
-            return json.loads(value)
-        return None
+        if value == '':
+            return value
+        return json.loads(value)

quipucords/api/fingerprint/model.py

@@ -115,7 +115,7 @@ def __str__(self):
             'vm_host_cpu_threads:{}, '\
             'vm_host_socket_count:{}, '\
             'vm_datacenter:{}, '\
-            'vm_cluster:{} '\
+            'vm_cluster:{}, '\
             'metadata:{} '.format(self.id,
                                   self.report_id.id,
                                   self.name,
@@ -161,7 +161,9 @@ class Product(models.Model):
         ('unknown', 'Unknown')
     )
 
-    fingerprint = models.ForeignKey(SystemFingerprint, models.CASCADE)
+    fingerprint = models.ForeignKey(SystemFingerprint,
+                                    models.CASCADE,
+                                    related_name='products')
     name = models.CharField(max_length=256, unique=False, null=False)
     version = models.CharField(max_length=256, unique=False, null=True)
     presence = models.CharField(
@@ -187,7 +189,9 @@ def __str__(self):
 class Entitlement(models.Model):
     """Represents a Entitlement."""
 
-    fingerprint = models.ForeignKey(SystemFingerprint, models.CASCADE)
+    fingerprint = models.ForeignKey(SystemFingerprint,
+                                    models.CASCADE,
+                                    related_name='entitlements')
     name = models.CharField(max_length=256, unique=False, null=True)
     entitlement_id = models.CharField(max_length=256, unique=False, null=True)
 

quipucords/api/report/renderer.py

@@ -141,7 +141,7 @@ def render(self,
 
         headers = csv_helper.generate_headers(
             systems_list, exclude=set([
-                'id', 'report_id', 'metadata']))
+                'id', 'report_id', 'metadata', 'products', 'entitlements']))
         csv_writer.writerow(headers)
         for system in systems_list:
             row = []

quipucords/api/report/tests_report.py

@@ -271,7 +271,7 @@ def test_get_fact_collection_group_report(self):
         self.assertEqual(response.status_code, status.HTTP_200_OK)
         report = response.json()
         self.assertIsInstance(report, dict)
-        self.assertEqual(len(report['report'][0].keys()), 33)
+        self.assertEqual(len(report['report'][0].keys()), 35)
 
     def test_get_fact_collection_filter_report(self):
         """Get a specific group count report with filter."""

quipucords/api/source/tests_source.py

@@ -130,6 +130,7 @@ def test_format_source(self):
         start = datetime.now()
         source = Source(
             name='source1',
+            hosts=json.dumps(['1.2.3.4']),
             source_type='network',
             port=22)
         source.save()
@@ -156,6 +157,7 @@ def test_format_source(self):
                     'name': 'source1',
                     'source_type': 'network',
                     'port': 22,
+                    'hosts': ['1.2.3.4'],
                     'connection': {'id': 1, 'start_time': start,
                                    'end_time': end, 'status': 'completed',
                                    'systems_count': 10,

quipucords/fingerprinter/__init__.py

@@ -42,6 +42,8 @@
 
 META_DATA_KEY = 'metadata'
 
+ENTITLEMENTS_KEY = 'entitlements'
+
 
 def process_fact_collection(sender, instance, **kwargs):
     """Process the fact collection.
@@ -418,14 +420,19 @@ def _merge_fingerprint(priority_fingerprint, to_merge_fingerprint):
     if META_DATA_KEY in keys_to_add_list:
         keys_to_add_list.remove(META_DATA_KEY)
 
-    # Add vcenter facts
     for fact_key in keys_to_add_list:
         to_merge_fact = to_merge_fingerprint.get(fact_key)
         if to_merge_fact:
             priority_fingerprint[META_DATA_KEY][fact_key] = \
                 to_merge_fingerprint[META_DATA_KEY][fact_key]
             priority_fingerprint[fact_key] = to_merge_fact
 
+    if to_merge_fingerprint.get(ENTITLEMENTS_KEY):
+        if ENTITLEMENTS_KEY not in priority_fingerprint:
+            priority_fingerprint[ENTITLEMENTS_KEY] = []
+        for entitlement in to_merge_fingerprint.get(ENTITLEMENTS_KEY):
+            priority_fingerprint[ENTITLEMENTS_KEY].append(entitlement)
+
     return priority_fingerprint
 
 
@@ -468,6 +475,52 @@ def add_fact_to_fingerprint(source,
         }
 
 
+# pylint: disable=C0103
+def add_fact_entitlements_to_fingerprint(source,
+                                         raw_fact_key,
+                                         raw_fact,
+                                         fingerprint):
+    """Create the fingerprint entitlements with fact and metadata.
+
+    :param source: Source used to gather raw facts.
+    :param raw_fact_key: Raw fact key used to obtain value
+    :param raw_fact: Raw fact used used to obtain value
+    :param fingerprint: dict containing all fingerprint facts
+    this fact.
+    """
+    # pylint: disable=too-many-arguments
+    source_object = Source.objects.filter(id=source.get('source_id')).first()
+    if source_object:
+        source_name = source_object.name
+    else:
+        source_name = None
+    actual_fact_value = None
+    if raw_fact.get(raw_fact_key) is not None:
+        actual_fact_value = raw_fact.get(raw_fact_key)
+
+    if actual_fact_value is not None and isinstance(actual_fact_value, list):
+        entitlements = []
+        for entitlement in actual_fact_value:
+            add = False
+            f_ent = {}
+            if entitlement.get('name'):
+                f_ent['name'] = entitlement.get('name')
+                add = True
+            if entitlement.get('entitlement_id'):
+                f_ent['entitlement_id'] = entitlement.get('entitlement_id')
+                add = True
+            if add:
+                f_ent[META_DATA_KEY] = {
+                    'source_id': source['source_id'],
+                    'source_name': source_name,
+                    'source_type': source['source_type'],
+                    'raw_fact_key': raw_fact_key
+                }
+                entitlements.append(f_ent)
+
+        fingerprint[ENTITLEMENTS_KEY] = entitlements
+
+
 def _process_network_fact(source, fact):
     """Process a fact and convert to a fingerprint.
 
@@ -580,6 +633,9 @@ def _process_network_fact(source, fact):
                             fact, 'virtualized_num_running_guests',
                             fingerprint)
 
+    add_fact_entitlements_to_fingerprint(source, 'subman_consumed',
+                                         fact, fingerprint)
+
     return fingerprint
 
 
@@ -692,6 +748,9 @@ def _process_satellite_fact(source, fact):
     add_fact_to_fingerprint(source, 'num_sockets', fact,
                             'cpu_socket_count', fingerprint)
 
+    add_fact_entitlements_to_fingerprint(source, 'entitlements',
+                                         fact, fingerprint)
+
     return fingerprint
 
 

quipucords/fingerprinter/tests_engine.py

@@ -25,12 +25,17 @@
 from api.models import Source
 
 
+SUBMAN_CONSUMED = [{'name': 'Red Hat JBoss Fuse (Pilot)',
+                    'entitlement_id': 'ESA0009'}]
+SAT_ENTITLEMENTS = [{'name': 'Satellite Tools 6.3'}]
+
+
 class EngineTest(TestCase):
     """Tests Engine class."""
 
     # pylint: disable=no-self-use,too-many-arguments
     # pylint: disable=too-many-locals,too-many-branches,invalid-name
-    # pylint: disable=protected-access
+    # pylint: disable=protected-access, W0102
 
     ################################################################
     # Helper functions
@@ -48,6 +53,7 @@ def _create_network_fc_json(
             ifconfig_mac_addresses=None,
             dmi_system_uuid='1234',
             subman_virt_uuid='4567',
+            subman_consumed=SUBMAN_CONSUMED,
             connection_uuid='a037f26f-2988-57bd-85d8-de7617a3aab0',
             connection_host='1.2.3.4',
             connection_port=22,
@@ -93,6 +99,8 @@ def _create_network_fc_json(
             fact['dmi_system_uuid'] = dmi_system_uuid
         if subman_virt_uuid:
             fact['subman_virt_uuid'] = subman_virt_uuid
+        if subman_consumed:
+            fact['subman_consumed'] = subman_consumed
         if connection_uuid:
             fact['connection_uuid'] = connection_uuid
         if connection_host:
@@ -210,7 +218,8 @@ def _create_satellite_fc_json(
             virt_type='lxc',
             is_virtualized=True,
             virtual_host='9.3.4.6',
-            num_sockets=8):
+            num_sockets=8,
+            entitlements=SAT_ENTITLEMENTS):
         """Create an in memory FactCollection for tests."""
         fact = {}
         if source_id:
@@ -248,6 +257,8 @@ def _create_satellite_fc_json(
             fact['virtual_host'] = virtual_host
         if num_sockets:
             fact['num_sockets'] = num_sockets
+        if entitlements:
+            fact['entitlements'] = entitlements
 
         fact_collection = {'id': report_id, 'facts': [fact]}
         return fact_collection

quipucords/scanner/network/processing/__init__.py

@@ -15,3 +15,4 @@
 from . import karaf
 from . import cpu
 from . import date
+from . import subman

quipucords/scanner/network/processing/process.py

@@ -10,7 +10,6 @@
 """Infrastructure for the initial data postprocessing."""
 
 import abc
-import json
 from logging import (ERROR, DEBUG)
 import traceback
 
@@ -140,9 +139,7 @@ def process(scan_task, previous_host_facts, fact_key, fact_value, host):
         scan_task.log_message(log_message, log_level=ERROR)
         return NO_DATA
 
-    # https://docs.python.org/3/library/json.html suggests that
-    # these separators will give the most compact representation.
-    return json.dumps(processor_out, separators=(',', ':'))
+    return processor_out
 
 
 class ProcessorMeta(abc.ABCMeta):

quipucords/scanner/network/processing/subman.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2018 Red Hat, Inc.
+#
+# This software is licensed to you under the GNU General Public License,
+# version 3 (GPLv3). There is NO WARRANTY for this software, express or
+# implied, including the implied warranties of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv3
+# along with this software; if not, see
+# https://www.gnu.org/licenses/gpl-3.0.txt.
+
+"""Initial processing of the shell output from the subman role."""
+
+import logging
+from scanner.network.processing import process
+
+logger = logging.getLogger(__name__)  # pylint: disable=invalid-name
+
+# pylint: disable=too-few-public-methods
+
+# #### Processors ####
+
+
+class ProcessSubmanConsumed(process.Processor):
+    """Process the subman_consumed fact."""
+
+    KEY = 'subman_consumed'
+
+    @staticmethod
+    def process(output):
+        """Pass the output back through."""
+        entitlements_data = []
+        entitlements = output.get('stdout_lines', [])
+        for entitlement in entitlements:
+            name, entitlement_id = entitlement.split(' - ')
+            entitlement_dict = {'name': name, 'entitlement_id': entitlement_id}
+            entitlements_data.append(entitlement_dict)
+        return entitlements_data

quipucords/scanner/network/processing/tests_process.py

@@ -180,7 +180,7 @@ def test_simple_processor(self):
         self.assertEqual(
             process.process(self.scan_task, {},
                             TEST_KEY, ansible_result(process.NO_DATA),
-                            HOST), '1')
+                            HOST), 1)
 
     def test_missing_dependency(self):
         """Test a key whose processor is missing a dependency."""

quipucords/scanner/network/processing/tests_subman.py

@@ -0,0 +1,36 @@
+# Copyright (c) 2018 Red Hat, Inc.
+#
+# This software is licensed to you under the GNU General Public License,
+# version 2 (GPLv2). There is NO WARRANTY for this software, express or
+# implied, including the implied warranties of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv2
+# along with this software; if not, see
+# http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt.
+
+
+"""Unit tests for initial processing of subman facts."""
+
+
+import unittest
+from scanner.network.processing import subman
+from scanner.network.processing.test_util import ansible_result
+
+
+class TestProcessSubmanConsumed(unittest.TestCase):
+    """Test ProcessSubmanConsumed."""
+
+    def test_success_case(self):
+        """Found subman_consumed."""
+        self.assertEqual(
+            subman.ProcessSubmanConsumed.process(
+                ansible_result('subnameA - 1\nsubnameB - 2\nsubnameC - 3')),
+            [{'name': 'subnameA', 'entitlement_id': '1'},
+             {'name': 'subnameB', 'entitlement_id': '2'},
+             {'name': 'subnameC', 'entitlement_id': '3'}])
+
+    def test_not_found(self):
+        """Did not find subman_consumed."""
+        self.assertEqual(
+            subman.ProcessSubmanConsumed.process(
+                ansible_result('')),
+            [])

roles/subman/tasks/main.yml

@@ -105,13 +105,7 @@
 # subman_consumed fact
 - name: gather subman.consumed fact
   raw: subscription-manager list --consumed | grep -e '^SKU' -e '^Subscription Name' | sed -n -e 's/SKU\s*.\s*//p' -e 's/Subscription Name\s*.\s*//p' | awk '{ ORS = (NR%2 ? " - " {{":"}} RS) } 1'
-  register: internal_subman_consumed_cmd
+  register: subman_consumed
   ignore_errors: yes
   become: yes
   when: 'internal_have_subscription_manager'
-
-- name: extract result value for subman.consumed
-  set_fact:
-    subman_consumed: "{{  internal_subman_consumed_cmd['stdout'] | trim | default(None) }}"
-  ignore_errors: yes
-  when: '"stdout" in internal_subman_consumed_cmd'