Skip to content
This repository has been archived by the owner on Nov 10, 2017. It is now read-only.

Commit

Permalink
bug 1216786 - Use references in MDN importer
Browse files Browse the repository at this point in the history
  • Loading branch information
@jwhitlock
jwhitlock committed Feb 25, 2016
1 parent 0c31fa4 commit 3c06922
Show file tree
Hide file tree
Showing 4 changed files with 79 additions and 32 deletions.
3 changes: 2 additions & 1 deletion mdn/models.py
View file
Expand Up @@ -201,7 +201,7 @@ def reset_data(self, keep_issues=False):
('obsolete', self.feature.obsolete),
('name', self.feature.name),
('links', OrderedDict((
('sections', []),
('references', []),
('supports', []),
('parent', str(self.feature.parent_id)),
('children', []),
Expand Down Expand Up @@ -237,6 +237,7 @@ def reset_data(self, keep_issues=False):
('specifications', []),
('sections', []),
('features', []),
('references', []),
))),
('meta', OrderedDict((
('compat_table', OrderedDict((
Expand Down
48 changes: 35 additions & 13 deletions mdn/scrape.py
View file
Expand Up @@ -6,7 +6,7 @@
from django.utils.six import text_type
from parsimonious import IncompleteParseError
from webplatformcompat.models import (
Browser, Feature, Section, Specification, Support, Version)
Browser, Feature, Reference, Section, Specification, Support, Version)

from .compatibility import CompatSectionExtractor
from .data import Data
Expand Down Expand Up @@ -325,7 +325,7 @@ def __init__(self, feature_page, scraped_data):
self.resources = OrderedDict()
for resource_type in (
'specifications', 'maturities', 'sections', 'browsers',
'versions', 'features', 'supports'):
'versions', 'features', 'supports', 'references'):
self.resources[resource_type] = OrderedDict()
self.tabs = []
self.compat_table_supports = OrderedDict(
Expand All @@ -350,8 +350,9 @@ def generate_data(self):
fp_data['features']['links']['children'] = list(
f_id for f_id in self.compat_table_supports.keys()
if f_id != main_id)
fp_data['features']['links']['sections'] = list(
self.resources['sections'].keys())
fp_data['features']['links']['references'] = list(
reference['id'] for reference in fp_data['linked']['references']
if reference['links']['feature'] == main_id)
fp_data['features']['links']['supports'] = sorted(
support['id'] for support in fp_data['linked']['supports']
if support['links']['feature'] == main_id)
Expand Down Expand Up @@ -382,11 +383,15 @@ def load_specification_row(self, spec_row):
section_content = self.load_section(spec_row['section.id'])
section_content['name']['en'] = spec_row['section.name']
section_content['subpath']['en'] = spec_row['section.subpath']
section_content['note']['en'] = spec_row['section.note']
else:
section_content = self.new_section(spec_row, spec_content['id'])
self.add_resource('sections', section_content)

# Load Reference
reference_content = self.load_or_new_reference(section_content['id'])
reference_content['note']['en'] = spec_row['section.note']
self.add_resource('references', reference_content)

def load_compat_table(self, table):
"""Load a compat table."""
tab = OrderedDict((
Expand Down Expand Up @@ -515,7 +520,6 @@ def load_section(self, section_id):
('number', section.number or None),
('name', section.name),
('subpath', section.subpath),
('note', section.note),
('links', OrderedDict((
('specification', text_type(section.specification_id)),)))))
return section_content
Expand All @@ -529,12 +533,10 @@ def new_section(self, spec_row, spec_id):
('number', None),
('name', OrderedDict()),
('subpath', OrderedDict()),
('note', OrderedDict()),
('links', OrderedDict((
('specification', text_type(spec_id)),)))))
section_content['name']['en'] = spec_row['section.name']
section_content['subpath']['en'] = spec_row['section.subpath']
section_content['note']['en'] = spec_row['section.note']
return section_content

def load_browser(self, browser_id):
Expand All @@ -561,9 +563,9 @@ def new_browser(self, browser_entry):
def load_feature(self, feature_id):
"""Serialize an existing feature."""
feature = Feature.objects.get(id=feature_id)
section_ids = [
text_type(s_id) for s_id in
feature.sections.values_list('pk', flat=True)]
reference_ids = [
text_type(r_id) for r_id in
feature.references.values_list('pk', flat=True)]
support_ids = [
text_type(s_id) for s_id in
sorted(feature.supports.values_list('pk', flat=True))]
Expand All @@ -583,7 +585,7 @@ def load_feature(self, feature_id):
('obsolete', feature.obsolete),
('name', feature.name),
('links', OrderedDict((
('sections', section_ids),
('references', reference_ids),
('supports', support_ids),
('parent', parent_id),
('children', children_ids))))))
Expand All @@ -607,7 +609,7 @@ def new_feature(self, feature_entry):
('obsolete', feature_entry.get('obsolete', False)),
('name', fname),
('links', OrderedDict((
('sections', []),
('references', []),
('supports', []),
('parent', text_type(self.feature.id)),
('children', []))))))
Expand Down Expand Up @@ -689,3 +691,23 @@ def new_support(self, support_entry):
if support_entry.get('footnote'):
support_content['note'] = {'en': support_entry['footnote']}
return support_content

def load_or_new_reference(self, section_id):
try:
reference = Reference.objects.get(
feature_id=self.feature.id, section_id=section_id)
except (Reference.DoesNotExist, ValueError):
reference_content = OrderedDict((
('id', '_%s_%s' % (self.feature.id, section_id)),
('note', OrderedDict()),
('links', OrderedDict((
('feature', text_type(self.feature.id)),
('section', text_type(section_id)))))))
else:
reference_content = OrderedDict((
('id', text_type(reference.id)),
('note', reference.note),
('links', OrderedDict((
('feature', text_type(reference.feature_id)),
('section', text_type(reference.section_id)))))))
return reference_content
26 changes: 21 additions & 5 deletions mdn/tests/base.py
View file
Expand Up @@ -2,7 +2,7 @@
"""Define base TestCase for MDN tests."""
from webplatformcompat.tests.base import TestCase as BaseTestCase
from webplatformcompat.models import (
Browser, Feature, Maturity, Section, Specification, Version)
Browser, Feature, Maturity, Reference, Section, Specification, Version)


class TestCase(BaseTestCase):
Expand Down Expand Up @@ -50,6 +50,7 @@ class TestCase(BaseTestCase):
('Section', 'background-size'): {
'_req': {'specification': ('Specification', 'css3_backgrounds')},
'subpath': '{"en": "#the-background-size"}'},
('Reference', ('web-css-background-size', 'background-size')): {},
('Feature', 'web'): {'name': '{"en": "Web"}'},
('Feature', 'web-css'): {
'_req': {'parent': ('Feature', 'web')}, 'name': '{"en": "CSS"}'},
Expand All @@ -68,27 +69,42 @@ class TestCase(BaseTestCase):
('Version', ('firefox_desktop', '1.0')): {},
}

name_to_class = {
'Browser': Browser,
'Feature': Feature,
'Maturity': Maturity,
'Reference': Reference,
'Section': Section,
'Specification': Specification,
'Version': Version,
}

def get_instance(self, model_cls_name, slug):
"""Get a test fixture instance, creating on first access."""
instance_key = (model_cls_name, slug)
if not hasattr(self, '_instances'):
self._instances = {}
if instance_key not in self._instances:
attrs = self._instance_specs[instance_key].copy()

# Load implied and explicit required instances
req = attrs.pop('_req', {})
if model_cls_name == 'Version':
browser_slug, version = slug
attrs['browser'] = self.get_instance('Browser', browser_slug)
attrs['version'] = version
elif model_cls_name == 'Section':
attrs['name'] = '{"en": "%s"}' % slug
elif model_cls_name == 'Reference':
feature_slug, section_slug = slug
attrs['feature'] = self.get_instance('Feature', feature_slug)
attrs['section'] = self.get_instance('Section', section_slug)
else:
attrs['slug'] = slug
for req_name, (req_type, req_slug) in req.items():
attrs[req_name] = self.get_instance(req_type, req_slug)
model_cls = {
'Browser': Browser, 'Feature': Feature, 'Maturity': Maturity,
'Section': Section, 'Specification': Specification,
'Version': Version}[model_cls_name]

# Create the instance
model_cls = self.name_to_class[model_cls_name]
self._instances[instance_key] = self.create(model_cls, **attrs)
return self._instances[instance_key]
34 changes: 21 additions & 13 deletions mdn/tests/test_scrape.py
View file
Expand Up @@ -290,7 +290,7 @@ def empty_view(self, scraped_data):
'links': {
'children': [],
'parent': str(self.feature.parent.id),
'sections': [],
'references': [],
'supports': []},
'mdn_uri': {
'en': ('https://developer.mozilla.org/en-US/docs/'
Expand All @@ -307,6 +307,7 @@ def empty_view(self, scraped_data):
'browsers': [],
'features': [],
'maturities': [],
'references': [],
'sections': [],
'specifications': [],
'supports': [],
Expand Down Expand Up @@ -374,8 +375,7 @@ def test_load_section(self):
view = ScrapedViewFeature(self.page, self.empty_scrape())
section_content = view.load_section(section.id)
expected = {
'id': str(section.id),
'name': section.name, 'note': {},
'id': str(section.id), 'name': section.name,
'number': None, 'subpath': section.subpath,
'links': {'specification': str(section.specification.id)}}
self.assertDataEqual(expected, section_content)
Expand All @@ -391,8 +391,7 @@ def test_new_section(self):
view = ScrapedViewFeature(self.page, self.empty_scrape())
section_content = view.new_section(spec_row, '_CSS3_UI')
expected = {
'id': '__CSS3_UI_#section',
'name': {'en': 'section'}, 'note': {'en': 'section note'},
'id': '__CSS3_UI_#section', 'name': {'en': 'section'},
'number': None, 'subpath': {'en': '#section'},
'links': {'specification': '_CSS3_UI'}}
self.assertDataEqual(expected, section_content)
Expand Down Expand Up @@ -476,7 +475,7 @@ def test_load_feature(self):
'stable': True, 'standardized': True, 'experimental': False,
'links': {
'children': [], 'parent': str(self.feature.id),
'sections': [], 'supports': []}}
'references': [], 'supports': []}}
self.assertDataEqual(expected, feature_content)

def test_load_feature_canonical_name(self):
Expand All @@ -491,7 +490,7 @@ def test_load_feature_canonical_name(self):
'stable': True, 'standardized': True, 'experimental': False,
'links': {
'children': [], 'parent': str(self.feature.id),
'sections': [], 'supports': []}}
'references': [], 'supports': []}}
self.assertDataEqual(expected, feature_content)

def test_new_feature(self):
Expand All @@ -509,7 +508,7 @@ def test_new_feature(self):
'standardized': True, 'experimental': False,
'links': {
'children': [], 'parent': str(self.feature.id),
'sections': [], 'supports': []}}
'references': [], 'supports': []}}
self.assertDataEqual(expected, feature_content)

def test_new_feature_canonical(self):
Expand All @@ -525,7 +524,7 @@ def test_new_feature_canonical(self):
'standardized': True, 'experimental': False,
'links': {
'children': [], 'parent': str(self.feature.id),
'sections': [], 'supports': []}}
'references': [], 'supports': []}}
self.assertDataEqual(expected, feature_content)

def test_load_support(self):
Expand Down Expand Up @@ -585,15 +584,22 @@ def test_load_specification_row_new_resources(self):
out = view.generate_data()
spec_content, mat_content = view.new_specification(scraped_spec)
section_content = view.new_section(scraped_spec, spec_content['id'])
section_content['name']['en'] = 'section'
section_content['subpath']['en'] = '#section'
reference_content = view.load_or_new_reference(section_content['id'])
reference_content['note']['en'] = 'section note'
expected = self.empty_view(scraped_data)
expected['features']['links']['sections'] = [section_content['id']]
expected['features']['links']['references'] = [reference_content['id']]
expected['linked']['maturities'] = [mat_content]
expected['linked']['specifications'] = [spec_content]
expected['linked']['sections'] = [section_content]
expected['linked']['references'] = [reference_content]
self.assertDataEqual(expected, out)

def test_load_specification_row_existing_resources(self):
section = self.get_instance('Section', 'background-size')
reference = self.get_instance(
'Reference', ('web-css-background-size', 'background-size'))
section = reference.section
spec = section.specification
scraped_spec = {
'section.note': 'new note',
Expand All @@ -609,11 +615,13 @@ def test_load_specification_row_existing_resources(self):
expected = self.empty_view(scraped_data)
spec_content, mat_content = view.load_specification(spec.id)
section_content = view.load_section(section.id)
section_content['note'] = {'en': 'new note'}
expected['features']['links']['sections'] = [str(section.id)]
reference_content = view.load_or_new_reference(section.id)
reference_content['note'] = {'en': 'new note'}
expected['features']['links']['references'] = [reference_content['id']]
expected['linked']['maturities'] = [mat_content]
expected['linked']['specifications'] = [spec_content]
expected['linked']['sections'] = [section_content]
expected['linked']['references'] = [reference_content]
self.assertDataEqual(expected, out)

def test_load_compat_table_new_resources(self):
Expand Down

0 comments on commit 3c06922

Please sign in to comment.