Syncing 'develop' branch.

.gitignore

@@ -27,3 +27,4 @@
 /src/ralph/static
 /src/ralph/logs
 /src/ralph/uploads
+/src/ralph/urls_local.py

src/ralph/scan/manual.py

@@ -214,12 +214,21 @@ def _get_ip_addresses_from_results(results):
     return result
 
 
-def _get_cleaned_results(data):
-    UNNECESSARY_KEYS = set(['status', 'date', 'messages'])
+def _get_cleaned_results(
+    data, remove_device_data=False,
+    unnecessary_keys=['status', 'date', 'messages'],
+):
     data = copy.deepcopy(data)
     for plugin_name, plugin_results in data.iteritems():
-        for key in UNNECESSARY_KEYS:
+        for key in unnecessary_keys:
             plugin_results.pop(key, None)
+        if all((
+            remove_device_data,
+            'device' in plugin_results,
+            'device' not in unnecessary_keys,
+        )):
+            for key in unnecessary_keys:
+                plugin_results['device'].pop(key, None)
     return data
 
 
@@ -293,20 +302,11 @@ def _scan_postprocessing(results, job, ip_address=None):
             job.save()
             results.update(updated_results)
     # calculate new checksum
-    cleaned_results = _get_cleaned_results(results)
-    checksum = _get_results_checksum(cleaned_results)
-    job.meta['results_checksum'] = checksum
-    job.save()
-    # calculate new status
-    if all((
-        checksum != scan_summary.previous_checksum,
-        checksum != scan_summary.false_positive_checksum,
-    )):
-        job.meta['changed'] = True
-    else:
-        job.meta['changed'] = False
-        scan_summary.false_positive_checksum = None
-    job.save()
+    cleaned_results = _get_cleaned_results(
+        data=results, remove_device_data=True,
+        unnecessary_keys=['status', 'date', 'messages', 'installed_software'],
+    )
+    scan_summary.current_checksum = _get_results_checksum(cleaned_results)
     scan_summary.save()
     ip_address.save()
     # cancel old job (if exists)

src/ralph/scan/migrations/0002_auto__add_field_scansummary_current_checksum__add_field_scansummary_ch.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding field 'ScanSummary.current_checksum'
+        db.add_column('scan_scansummary', 'current_checksum',
+                      self.gf('django.db.models.fields.CharField')(max_length=32, null=True, blank=True),
+                      keep_default=False)
+
+        # Adding field 'ScanSummary.changed'
+        db.add_column('scan_scansummary', 'changed',
+                      self.gf('django.db.models.fields.BooleanField')(default=False, db_index=True),
+                      keep_default=False)
+
+
+        # Changing field 'ScanSummary.previous_checksum'
+        db.alter_column('scan_scansummary', 'previous_checksum', self.gf('django.db.models.fields.CharField')(max_length=32, null=True))
+
+    def backwards(self, orm):
+        # Deleting field 'ScanSummary.current_checksum'
+        db.delete_column('scan_scansummary', 'current_checksum')
+
+        # Deleting field 'ScanSummary.changed'
+        db.delete_column('scan_scansummary', 'changed')
+
+
+        # Changing field 'ScanSummary.previous_checksum'
+        db.alter_column('scan_scansummary', 'previous_checksum', self.gf('django.db.models.fields.CharField')(default='-', max_length=32))
+
+    models = {
+        'scan.scansummary': {
+            'Meta': {'object_name': 'ScanSummary'},
+            'changed': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'db_index': 'True'}),
+            'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}),
+            'current_checksum': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}),
+            'false_positive_checksum': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'}),
+            'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'job_id': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '36'}),
+            'modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'auto_now_add': 'True', 'blank': 'True'}),
+            'previous_checksum': ('django.db.models.fields.CharField', [], {'max_length': '32', 'null': 'True', 'blank': 'True'})
+        }
+    }
+
+    complete_apps = ['scan']

src/ralph/scan/models.py

@@ -19,13 +19,15 @@ class ScanSummary(db.Model, WithConcurrentGetOrCreate):
     """
 
     job_id = db.CharField(unique=True, max_length=36)
-    previous_checksum = db.CharField(max_length=32)
+    current_checksum = db.CharField(max_length=32, blank=True, null=True)
+    previous_checksum = db.CharField(max_length=32, blank=True, null=True)
     false_positive_checksum = db.CharField(
         blank=True,
         null=True,
         max_length=32,
         verbose_name="Ignored checksum",
     )
+    changed = db.BooleanField(default=False, db_index=True)
     created = db.DateTimeField(auto_now=False, auto_now_add=True)
     modified = db.DateTimeField(auto_now=True, auto_now_add=True)
 
@@ -40,3 +42,11 @@ def device(self):
         ipaddress = self.ipaddress
         if ipaddress:
             return self.ipaddress.device
+
+    def save(self, *args, **kwargs):
+        self.changed = all((
+            self.current_checksum,
+            self.current_checksum != self.previous_checksum,
+            self.current_checksum != self.false_positive_checksum,
+        ))
+        return super(ScanSummary, self).save(*args, **kwargs)

src/ralph/scan/util.py

@@ -32,14 +32,15 @@ def update_scan_summary(job):
     except ScanSummary.DoesNotExist:
         return
     else:
-        scan_summary.previous_checksum = job.meta.get('results_checksum')
+        scan_summary.previous_checksum = scan_summary.current_checksum
+        scan_summary.current_checksum = None
         scan_summary.false_positive_checksum = None
         scan_summary.save()
-        job.meta['changed'] = False
-        job.save()
 
 
-PendingChanges = namedtuple('PendingChanges', ['new_devices', 'changed_devices'])
+PendingChanges = namedtuple(
+    'PendingChanges', ['new_devices', 'changed_devices'],
+)
 
 
 def get_pending_changes():
@@ -49,6 +50,6 @@ def get_pending_changes():
     all_changes = ScanSummary.objects.filter(modified__gt=delta)
     new, changed = (
         all_changes.filter(ipaddress__device=None).count(),
-        all_changes.exclude(ipaddress__device=None).count(),
+        all_changes.filter(changed=True).count(),
     )
     return PendingChanges(new, changed)

src/ralph/ui/templates/ui/scan-status.html

@@ -28,7 +28,7 @@
                 <a href="../../{{ address }}/" class="btn btn-small">
                     {% icon 'fugue-arrow-circle-double' %}&nbsp;Rescan
                 </a>
-                {% if job.meta.changed %}
+                {% if scan_summary and scan_summary.changed %}
                     <button type="submit" value="no-changes" class="btn btn-small" name="no-changes">
                         {% icon 'fugue-flashlight--exclamation' %}&nbsp;No changes
                     </button>

src/ralph/ui/views/common.py

@@ -575,21 +575,10 @@ def get_running_deployment_info(self):
 
     def get_changed_addresses(self):
         delta = timezone.now() - datetime.timedelta(days=1)
-        result = []
-        for ip_address in self.object.ipaddress.filter(
+        return self.object.ipaddress.filter(
             scan_summary__modified__gt=delta,
-        ):
-            try:
-                job = rq.job.Job.fetch(
-                    ip_address.scan_summary.job_id,
-                    django_rq.get_connection(),
-                )
-            except rq.exceptions.NoSuchJobError:
-                continue
-            else:
-                if job.meta.get('changed', False):
-                    result.append(ip_address)
-        return result
+            scan_summary__changed=True,
+        )
 
     def get_context_data(self, **kwargs):
         ret = super(Info, self).get_context_data(**kwargs)
@@ -1624,7 +1613,7 @@ def handle_search_data(self, *args, **kwargs):
         if kwargs['change_type'] == 'new':
             return all_changes.filter(ipaddress__device=None)
         else:
-            return all_changes.exclude(ipaddress__device=None)
+            return all_changes.filter(changed=True)
 
 
 class ScanStatus(BaseMixin, TemplateView):
@@ -1757,6 +1746,7 @@ def get_context_data(self, **kwargs):
                     ) for p in plugins
                 ],
                 'task_size': 100 / len(plugins),
+                'scan_summary': self.get_scan_summary(self.job),
                 'job': self.job,
             })
             if self.job.is_finished:
@@ -1790,18 +1780,18 @@ def get(self, *args, **kwargs):
                     )
         return super(ScanStatus, self).get(*args, **kwargs)
 
-    def mark_scan_as_nochanges(self, job):
+    def get_scan_summary(self, job):
         try:
-            scan_summary = ScanSummary.objects.get(job_id=job.id)
+            return ScanSummary.objects.get(job_id=job.id)
         except ScanSummary.DoesNotExist:
             return
-        else:
-            scan_summary.false_positive_checksum = job.meta.get(
-                'results_checksum',
-            )
+
+    def mark_scan_as_nochanges(self, job):
+        scan_summary = self.get_scan_summary(job)
+        if scan_summary:
+            current_checksum = scan_summary.current_checksum
+            scan_summary.false_positive_checksum = current_checksum
             scan_summary.save()
-            job.meta['changed'] = False
-            job.save()
 
     def post(self, *args, **kwargs):
         self.device_id = self.request.POST.get('save')

src/ralph/ui/views/search.py

@@ -8,9 +8,6 @@
 import ipaddr
 import re
 
-import django_rq
-import rq
-
 from django.conf import settings
 from django.contrib import messages
 from django.core.urlresolvers import reverse
@@ -20,8 +17,8 @@
 from powerdns.models import Record
 
 from ralph.account.models import Perm
-from ralph.discovery.models import ReadOnlyDevice, Device
-from ralph.scan.models import ScanSummary
+from ralph.discovery.models import ReadOnlyDevice, Device, IPAddress
+from ralph.scan.api import SCAN_RESULT_TTL
 from ralph.ui.forms.search import SearchForm, SearchFormWithAssets
 from ralph.ui.views.common import (
     Addresses,
@@ -109,25 +106,14 @@ def __init__(self, *args, **kwargs):
         self.query = None
 
     def _get_changed_devices_ids(self):
-        delta = timezone.now() - datetime.timedelta(days=1)
-        ids = set()
-        for scan_summary in ScanSummary.objects.filter(modified__gt=delta):
-            try:
-                job = rq.job.Job.fetch(
-                    scan_summary.job_id,
-                    django_rq.get_connection(),
-                )
-            except rq.exceptions.NoSuchJobError:
-                continue
-            else:
-                if job.meta.get('changed', False):
-                    for device_id in scan_summary.ipaddress_set.values_list(
-                        'device__id',
-                        flat=True,
-                    ):
-                        if device_id:
-                            ids.add(device_id)
-        return sorted(list(ids))
+        delta = timezone.now() - datetime.timedelta(seconds=SCAN_RESULT_TTL)
+        return set(
+            IPAddress.objects.filter(
+                scan_summary__modified__gt=delta,
+                scan_summary__changed=True,
+                device__isnull=False,
+            ).values_list('device__id', flat=True)
+        )
 
     def user_allowed(self):
         return True

src/ralph/urls.py

@@ -165,3 +165,9 @@
 )
 
 urlpatterns += pluggableapp.patterns()
+
+try:
+    from ralph.urls_local import urlpatterns as local_urlpatterns
+    urlpatterns += local_urlpatterns
+except ImportError:
+    pass