bug 1181140 - Add issue level to import status

New statuses for FeaturePage:
* STATUS_PARSED_CRITICAL: Worst issue is a critical error
* STATUS_PARSED_ERROR: Worst issue is an error
* STATUS_PARSED_WARNINGL Worst issues is a warning

STATUS_PARSED becomes "parsed with no issues". Status is set after
scrape, displayed in list view. Because it depends on mdn/issues.py and
not database data, it has to be updated by reparsing the page.

mdn/migrations/0007_expand_status_choices.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# flake8: noqa
+from __future__ import unicode_literals
+
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('mdn', '0006_update_issue_choices'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='featurepage',
+            name='status',
+            field=models.IntegerField(default=0, help_text='Status of MDN Parsing process', choices=[(0, 'Starting Import'), (1, 'Fetching Metadata'), (2, 'Fetching MDN pages'), (3, 'Parsing MDN pages'), (4, 'Parsing Complete'), (5, 'Scraping Failed'), (6, 'No Compat Data'), (7, 'Has Warnings'), (8, 'Has Errors'), (9, 'Has Critical Errors')]),
+            preserve_default=True,
+        ),
+    ]

mdn/models.py

@@ -51,6 +51,9 @@ class FeaturePage(models.Model):
     STATUS_PARSED = 4
     STATUS_ERROR = 5
     STATUS_NO_DATA = 6
+    STATUS_PARSED_WARNING = 7
+    STATUS_PARSED_ERROR = 8
+    STATUS_PARSED_CRITICAL = 9
     STATUS_CHOICES = (
         (STATUS_STARTING, "Starting Import"),
         (STATUS_META, "Fetching Metadata"),
@@ -59,6 +62,9 @@ class FeaturePage(models.Model):
         (STATUS_PARSED, "Parsing Complete"),
         (STATUS_ERROR, "Scraping Failed"),
         (STATUS_NO_DATA, "No Compat Data"),
+        (STATUS_PARSED_WARNING, "Has Warnings"),
+        (STATUS_PARSED_ERROR, "Has Errors"),
+        (STATUS_PARSED_CRITICAL, "Has Critical Errors"),
     )
     status = models.IntegerField(
         help_text="Status of MDN Parsing process",
@@ -275,6 +281,14 @@ def add_issue(self, issue, locale=None):
     def get_absolute_url(self):
         return ('mdn.views.feature_page_detail', [str(self.id)])
 
+    @property
+    def is_parsed(self):
+        return self.status in (
+            self.STATUS_PARSED,
+            self.STATUS_PARSED_WARNING,
+            self.STATUS_PARSED_ERROR,
+            self.STATUS_PARSED_CRITICAL)
+
 
 @python_2_unicode_compatible
 class Issue(models.Model):

mdn/scrape.py

@@ -42,7 +42,14 @@ def scrape_feature_page(feature_page):
     has_data = (scraped_data['specs'] or scraped_data['compat'] or
                 scraped_data['issues'])
     if has_data:
-        feature_page.status = feature_page.STATUS_PARSED
+        if feature_page.critical:
+            feature_page.status = feature_page.STATUS_PARSED_CRITICAL
+        elif feature_page.errors:
+            feature_page.status = feature_page.STATUS_PARSED_ERROR
+        elif feature_page.warnings:
+            feature_page.status = feature_page.STATUS_PARSED_WARNING
+        else:
+            feature_page.status = feature_page.STATUS_PARSED
     else:
         feature_page.status = feature_page.STATUS_NO_DATA
     merged_data['meta']['scrape']['phase'] = feature_page.get_status_display()

mdn/templates/mdn/feature_page_list.jinja2

@@ -68,7 +68,7 @@
       <td><a href="{{ url('feature_page_detail', pk=page.id) }}">{{ page.get_status_display() }}</a></td>
       <td>
         <a href="{{ url('feature_page_detail', pk=page.id) }}">
-        {%- if page.status == page.STATUS_PARSED -%}
+        {%- if page.is_parsed -%}
           {%- if page.has_issues %}
             {{ page.warnings }} / {{ page.errors }} / {{ page.critical }}
           {% else %}

mdn/tests/test_scrape.py

@@ -739,7 +739,7 @@ def test_empty_page(self):
             [], fp.data['meta']['scrape']['raw']['issues'])
         self.assertFalse(fp.has_issues)
 
-    def test_parse_issue(self):
+    def test_parse_warning(self):
         bad_page = '''\
 <p>The page has a bad specification section.</p>
 <h2 id="Specifications">Specifications</h2>
@@ -748,9 +748,81 @@ def test_parse_issue(self):
         self.set_content(bad_page)
         scrape_feature_page(self.page)
         fp = FeaturePage.objects.get(id=self.page.id)
-        self.assertEqual(fp.STATUS_PARSED, fp.status)
+        self.assertEqual(fp.STATUS_PARSED_WARNING, fp.status)
         expected_issues = [['skipped_content', 93, 108, {}]]
         self.assertEqual(
             expected_issues,
             fp.data['meta']['scrape']['raw']['issues'])
         self.assertTrue(fp.has_issues)
+
+    def test_parse_error(self):
+        self.get_instance('Specification', 'css3_backgrounds')
+        bad_page = '''\
+<p>The page has an error in the specification section.</p>
+<h2 id="Specifications">Specifications</h2>
+<table class="standard-table">
+ <thead>
+  <tr>
+   <th scope="col">Specification</th>
+   <th scope="col">Status</th>
+   <th scope="col">Comment</th>
+  </tr>
+ </thead>
+ <tbody>
+   <tr>
+     <td>{{SpecName('CSS3 Backgrounds', '#the-background-size',\
+ 'background-size')}}</td>
+     <td>{{SpecName('CSS3 Backgrounds')}}</td>
+     <td></td>
+   </tr>"
+ </tbody>
+</table>
+'''
+        self.set_content(bad_page)
+        scrape_feature_page(self.page)
+        fp = FeaturePage.objects.get(id=self.page.id)
+        self.assertEqual(fp.STATUS_PARSED_ERROR, fp.status)
+        self.assertTrue(fp.has_issues)
+
+    def test_parse_critical(self):
+        bad_page = '''\
+<p>The page has a div element wrapping the content.</p>
+<div>
+  <h2 id="Specifications">Specifications</h2>
+  <p>No specs</p>
+</div>
+'''
+        self.set_content(bad_page)
+        scrape_feature_page(self.page)
+        fp = FeaturePage.objects.get(id=self.page.id)
+        self.assertEqual(fp.STATUS_PARSED_CRITICAL, fp.status)
+        self.assertTrue(fp.has_issues)
+
+    def test_parse_ok(self):
+        self.get_instance('Specification', 'css3_backgrounds')
+        good_page = '''\
+<p>This page is OK.</p>
+<h2 id="Specifications">Specifications</h2>
+<table class="standard-table">
+ <thead>
+  <tr>
+   <th scope="col">Specification</th>
+   <th scope="col">Status</th>
+   <th scope="col">Comment</th>
+  </tr>
+ </thead>
+ <tbody>
+   <tr>
+     <td>{{SpecName('CSS3 Backgrounds', '#the-background-size',\
+ 'background-size')}}</td>
+     <td>{{Spec2('CSS3 Backgrounds')}}</td>
+     <td></td>
+   </tr>"
+ </tbody>
+</table>
+'''
+        self.set_content(good_page)
+        scrape_feature_page(self.page)
+        fp = FeaturePage.objects.get(id=self.page.id)
+        self.assertEqual(fp.STATUS_PARSED, fp.status)
+        self.assertFalse(fp.has_issues)