From 2a00e3c8a11b954ac445af192e06f2cf4f769748 Mon Sep 17 00:00:00 2001 From: Kevan Holdaway <32331547+kholdaway@users.noreply.github.com> Date: Tue, 17 Oct 2017 15:13:45 -0400 Subject: [PATCH] Process and persist system fingerprints when facts are published (#107) Process and persist system fingerprints when facts are published --- Makefile | 2 +- quipucords/api/admin.py | 2 +- quipucords/api/fact_views.py | 55 ++++++++++++++++++- .../{report_model.py => fingerprint_model.py} | 14 ++--- quipucords/api/fingerprint_serializer.py | 23 ++++++++ quipucords/api/report_views.py | 12 ++-- quipucords/api/tests_report.py | 4 +- quipucords/fingerprinter/__init__.py | 47 ++++++++++++++++ quipucords/fingerprinter/engine.py | 50 ----------------- quipucords/fingerprinter/tests_engine.py | 45 +++++++++++---- 10 files changed, 175 insertions(+), 79 deletions(-) rename quipucords/api/{report_model.py => fingerprint_model.py} (73%) create mode 100644 quipucords/api/fingerprint_serializer.py delete mode 100644 quipucords/fingerprinter/engine.py diff --git a/Makefile b/Makefile index f0c0c9e6c..caf4561ab 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ build: clean $(PYTHON) setup.py build -f clean: - -rm -rf dist/ build/ quipucords.egg-info/ + -rm -rf dist/ build/ quipucords.egg-info/;rm -rf quipucords/api/migrations/*;rm quipucords/db.sqlite3 install: build $(PYTHON) setup.py install -f diff --git a/quipucords/api/admin.py b/quipucords/api/admin.py index b83f8c23b..6b76d12a1 100644 --- a/quipucords/api/admin.py +++ b/quipucords/api/admin.py @@ -12,9 +12,9 @@ from django.contrib import admin from api.fact_model import FactCollection +from api.fingerprint_model import SystemFingerprint from api.hostcredential_model import HostCredential from api.networkprofile_model import NetworkProfile -from api.report_model import SystemFingerprint from api.scanjob_model import ScanJob from api.scanresults_model import ScanJobResults diff --git a/quipucords/api/fact_views.py b/quipucords/api/fact_views.py index 80e6142e8..ce450166b 100644 --- a/quipucords/api/fact_views.py +++ b/quipucords/api/fact_views.py @@ -11,16 +11,69 @@ """Viewset for system facts models""" +import logging from rest_framework import viewsets, mixins from api.fact_model import FactCollection from api.fact_serializer import FactCollectionSerializer +from api.fingerprint_serializer import FingerprintSerializer +from fingerprinter import Engine # pylint: disable=too-many-ancestors class FactViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet): """ - List all facts, or create a new snippet. + ModelViewSet to publish system facts. """ + + # Get an instance of a logger + logger = logging.getLogger(__name__) # pylint: disable=invalid-name + queryset = FactCollection.objects.all() serializer_class = FactCollectionSerializer + + def __init__(self, *args, **kwargs): + super(FactViewSet, self).__init__(*args, **kwargs) + self.engine = Engine() + + def create(self, request, *args, **kwargs): + """ + Method to publish system facts, process facts, + and persist resulting fingerprints + + :param request: http request + :param args: original ModelViewSet args + :param kwargs: arg count + :returns: http response + """ + + response = super().create(request) + self.persist_fingerprints(response.data) + return response + + def persist_fingerprints(self, data): + """ + Method to process facts and persist + resulting fingerprints + + :param data: fact collection dict response + :returns: fingerprints produced from fact + collection + """ + + fact_collection_id = data['id'] + facts = data['facts'] + fingerprints_list = self.engine.process_facts( + fact_collection_id, facts) + + fingerprints = [] + for fingerprint_dict in fingerprints_list: + serializer = FingerprintSerializer(data=fingerprint_dict) + if serializer.is_valid(): + fingerprint = serializer.save() + fingerprints.append(fingerprint) + else: + self.logger.error('%s could not persist fingerprints', + self.__class__) + self.logger.error(serializer.errors) + return fingerprints diff --git a/quipucords/api/report_model.py b/quipucords/api/fingerprint_model.py similarity index 73% rename from quipucords/api/report_model.py rename to quipucords/api/fingerprint_model.py index d0622fa85..c3cda0d77 100644 --- a/quipucords/api/report_model.py +++ b/quipucords/api/fingerprint_model.py @@ -9,26 +9,26 @@ # https://www.gnu.org/licenses/gpl-3.0.txt. # -"""Models to retrieve system reports.""" +"""Models system fingerprints.""" from django.db import models from api.fact_model import FactCollection class SystemFingerprint(models.Model): - """Represents os installation count""" - fact_collection = models.ForeignKey(FactCollection, - models.CASCADE) + """Represents system fingerprint""" + fact_collection_id = models.ForeignKey(FactCollection, + models.CASCADE) os_name = models.CharField(max_length=128, unique=False) os_release = models.CharField(max_length=64, unique=False) os_version = models.CharField(max_length=64, unique=False) def __str__(self): - return 'id:{}, fact_collection:{}, ' \ + return '{' + 'id:{}, fact_collection:{}, ' \ 'os_name:{}, os_release:{}, '\ 'os_version:{}' \ .format(self.id, - self.fact_collection, + self.fact_collection_id, self.os_name, self.os_release, - self.os_version) + self.os_version) + '}' diff --git a/quipucords/api/fingerprint_serializer.py b/quipucords/api/fingerprint_serializer.py new file mode 100644 index 000000000..15706d20d --- /dev/null +++ b/quipucords/api/fingerprint_serializer.py @@ -0,0 +1,23 @@ +# +# Copyright (c) 2017 Red Hat, Inc. +# +# This software is licensed to you under the GNU General Public License, +# version 3 (GPLv3). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv3 +# along with this software; if not, see +# https://www.gnu.org/licenses/gpl-3.0.txt. +# + +"""Serializer for system fingerprint models""" + +from rest_framework.serializers import ModelSerializer +from api.fingerprint_model import SystemFingerprint + + +class FingerprintSerializer(ModelSerializer): + """Serializer for the Fingerprint model.""" + class Meta: + """Meta class for FingerprintSerializer.""" + model = SystemFingerprint + fields = '__all__' diff --git a/quipucords/api/report_views.py b/quipucords/api/report_views.py index 6be5d1be8..4e816466b 100644 --- a/quipucords/api/report_views.py +++ b/quipucords/api/report_views.py @@ -9,10 +9,10 @@ # https://www.gnu.org/licenses/gpl-3.0.txt. # -"""Viewset for system report models""" +"""Viewset for system reports""" import logging from django.db.models import Count -from api.report_model import SystemFingerprint +from api.fingerprint_model import SystemFingerprint from rest_framework.views import APIView from rest_framework.response import Response from rest_framework import status @@ -32,11 +32,12 @@ def get(self, request): collection_report_list = [] # Find all distinct fact_collection_ids fact_collection_value_set = SystemFingerprint.objects.all().values( - 'fact_collection').distinct() + 'fact_collection_id').distinct() # For each id, build a report and add to results array for fact_collection_value in fact_collection_value_set: - fact_collection_id = fact_collection_value['fact_collection'] + fact_collection_id = fact_collection_value[ + 'fact_collection_id'] report = self.build_report(fact_collection_id) if report is not None: collection_report_list.append(report) @@ -57,8 +58,9 @@ def build_report(fact_collection_id): """Lookup system report by fact_collection_id.""" # We want aggregate counts on the fact collection groups # Find all fingerprints with this fact_collection_id + fc_fingerprints = SystemFingerprint.objects.filter( - fact_collection__id=fact_collection_id) + fact_collection_id__id=fact_collection_id) # Group by os_release and count counts_by_os = fc_fingerprints.values( diff --git a/quipucords/api/tests_report.py b/quipucords/api/tests_report.py index ecd0530fa..881e23196 100644 --- a/quipucords/api/tests_report.py +++ b/quipucords/api/tests_report.py @@ -13,7 +13,7 @@ import uuid from django.test import TestCase from api.fact_model import FactCollection, Fact -from api.report_model import SystemFingerprint +from api.fingerprint_model import SystemFingerprint from rest_framework import status @@ -50,7 +50,7 @@ def create_fingerprints(self, fact_collection): fingerprints = [] for fact in fact_collection.facts.all(): fingerprint = SystemFingerprint \ - .objects.create(fact_collection=fact_collection, + .objects.create(fact_collection_id=fact_collection, os_name=fact.etc_release_name, os_release=fact.etc_release_release, os_version=fact.etc_release_version) diff --git a/quipucords/fingerprinter/__init__.py b/quipucords/fingerprinter/__init__.py index e69de29bb..70b779fda 100644 --- a/quipucords/fingerprinter/__init__.py +++ b/quipucords/fingerprinter/__init__.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2017 Red Hat, Inc. +# +# This software is licensed to you under the GNU General Public License, +# version 3 (GPLv3). There is NO WARRANTY for this software, express or +# implied, including the implied warranties of MERCHANTABILITY or FITNESS +# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv3 +# along with this software; if not, see +# https://www.gnu.org/licenses/gpl-3.0.txt. +# + +"""Fingerprint engine ingests raw facts and produces system finger prints""" + + +class Engine(): + """Engine that produces fingerprints from + facts""" + # pylint: disable= no-self-use + + def process_facts(self, fact_collection_id, facts): + """Process facts and convert to fingerprints + + :param fact_collection_id: id of fact collection + associated with facts + :param facts: facts to process + :returns: fingerprints produced from facts + """ + + fingerprints = [] + for fact in facts: + fingerprints.append(self.process_fact(fact_collection_id, fact)) + return fingerprints + + def process_fact(self, fact_collection_id, fact): + """Process a fact and convert to a fingerprint + + :param fact_collection_id: id of fact collection + associated with facts + :param facts: fact to process + :returns: fingerprint produced from fact + """ + fingerprint = {'fact_collection_id': fact_collection_id, + 'os_name': fact['etc_release_name'], + 'os_release': fact['etc_release_release'], + 'os_version': fact['etc_release_version']} + + return fingerprint diff --git a/quipucords/fingerprinter/engine.py b/quipucords/fingerprinter/engine.py deleted file mode 100644 index c0bcfb5a9..000000000 --- a/quipucords/fingerprinter/engine.py +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright (c) 2017 Red Hat, Inc. -# -# This software is licensed to you under the GNU General Public License, -# version 3 (GPLv3). There is NO WARRANTY for this software, express or -# implied, including the implied warranties of MERCHANTABILITY or FITNESS -# FOR A PARTICULAR PURPOSE. You should have received a copy of GPLv3 -# along with this software; if not, see -# https://www.gnu.org/licenses/gpl-3.0.txt. -# - -"""Fingerprint engine ingests raw facts and produces system finger prints""" - - -class BasicEngine(): - """Engine that produces fingerprints from - facts""" - # pylint: disable= no-self-use - - def __init__(self): - """Create instance of fingerprint engine.""" - pass - - def process_facts(self, fact_collection_id, facts): - """Process facts and convert to fingerprints - - :param fact_collection_id: id of fact collection - associated with facts - :param facts: facts to process - :returns: fingerprints produced from facts - """ - fingerprints = [] - for fact in facts: - fingerprints.append(self.process_fact(fact_collection_id, fact)) - return fingerprints - - def process_fact(self, fact_collection_id, fact): - """Process a fact and convert to a fingerprint - - :param fact_collection_id: id of fact collection - associated with facts - :param facts: fact to process - :returns: fingerprint produced from fact - """ - fingerprint = {'fact_collection_id': fact_collection_id, - 'os_name': fact['etc_release_name'], - 'os_release': fact['etc_release_release'], - 'os_version': fact['etc_release_version']} - - return fingerprint diff --git a/quipucords/fingerprinter/tests_engine.py b/quipucords/fingerprinter/tests_engine.py index b7c3c4680..9c8f26766 100644 --- a/quipucords/fingerprinter/tests_engine.py +++ b/quipucords/fingerprinter/tests_engine.py @@ -13,14 +13,15 @@ import uuid from django.test import TestCase -from fingerprinter.engine import BasicEngine +from api.fact_model import Fact, FactCollection +from fingerprinter import Engine -class BasicEngineTest(TestCase): - """Tests BasicEngine class""" - # pylint: disable= no-self-use +class EngineTest(TestCase): + """Tests Engine class""" + # pylint: disable= no-self-use, too-many-arguments - def create_json_fc(self, etc_release_name='RHEL', + def create_json_fc(self, fc_id=1, etc_release_name='RHEL', etc_release_release='RHEL 7.4 (Maipo)', etc_release_version='7.4 (Maipo)', connection_uuid=str(uuid.uuid4())): @@ -30,12 +31,32 @@ def create_json_fc(self, etc_release_name='RHEL', 'etc_release_release': etc_release_release, 'etc_release_version': etc_release_version, 'connection_uuid': connection_uuid} - fact_collection = {'id': 1, 'facts': [fact]} + fact_collection = {'id': fc_id, 'facts': [fact]} return fact_collection - def test_process_facts(self): - """ Test model creation not via API.""" - engine = BasicEngine() + def create_fc(self, etc_release_name='RHEL', + etc_release_release='RHEL 7.4 (Maipo)', + etc_release_version='7.4 (Maipo)', + connection_uuid=str(uuid.uuid4())): + """Creates a FactCollection model for use within test cases + + :param etc_release_name: name of the release + :param etc_release_release: the release string + :param etc_release_version: the version of the release + :returns: A FactCollection model + """ + fact = Fact.objects.create(etc_release_name=etc_release_name, + etc_release_release=etc_release_release, + etc_release_version=etc_release_version, + connection_uuid=connection_uuid) + fact_collection = FactCollection.objects.create() + fact_collection.facts.add(fact) + fact_collection.save() + return fact_collection + + def test_basic_engine_process_facts(self): + """ Test basic engine process_facts.""" + engine = Engine() fact_collection = self.create_json_fc() fact = fact_collection['facts'][0] fingerprints = engine.process_facts(fact_collection['id'], @@ -49,9 +70,9 @@ def test_process_facts(self): self.assertEqual(fact['etc_release_version'], fingerprint['os_version']) - def test_process_fact(self): - """ Test model creation not via API.""" - engine = BasicEngine() + def test_basic_engine_process_fact(self): + """ Test basic engine process_fact.""" + engine = Engine() fact_collection = self.create_json_fc() fact = fact_collection['facts'][0] fingerprint = engine.process_fact(fact_collection['id'], fact)