Skip to content

Commit

Permalink
Merge 6f2da05 into 7925f53
Browse files Browse the repository at this point in the history
  • Loading branch information
yolile committed Jun 26, 2020
2 parents 7925f53 + 6f2da05 commit b41d699
Show file tree
Hide file tree
Showing 2 changed files with 12 additions and 10 deletions.
1 change: 1 addition & 0 deletions kingfisher_scrapy/item_schema/item.json
@@ -1,4 +1,5 @@
{
"$id": "item.json",
"$schema": "http://json-schema.org/draft-04/schema#",
"definitions": {
"KingfisherItem": {
Expand Down
21 changes: 11 additions & 10 deletions kingfisher_scrapy/pipelines.py
@@ -1,27 +1,28 @@
# https://docs.scrapy.org/en/latest/topics/item-pipeline.html
# https://docs.scrapy.org/en/latest/topics/signals.html#item-signals
import json
import pkgutil

import os
import pathlib

import jsonref as jsonref
from jsonschema import FormatChecker
from jsonschema.validators import Draft4Validator
from jsonschema.validators import Draft4Validator, RefResolver

from kingfisher_scrapy.items import File, FileItem


def _json_loads(basename):
return json.loads(pkgutil.get_data('kingfisher_scrapy', f'item_schema/{basename}.json'))


class Validate:
def __init__(self):
self.validators = {}
self.files = set()
self.file_items = set()
schema_path = pathlib.Path(os.path.dirname(os.path.abspath(__file__)), 'item_schema')

resolver = RefResolver.from_schema(_json_loads('item'))
checker = FormatChecker()
for item in ('File', 'FileError', 'FileItem'):
filename = os.path.join(schema_path, f'{item}.json')
with open(filename) as f:
schema = jsonref.load(f, base_uri=schema_path.as_uri() + '/')
self.validators[item] = Draft4Validator(schema, format_checker=FormatChecker())
self.validators[item] = Draft4Validator(_json_loads(item), resolver=resolver, format_checker=checker)

def process_item(self, item, spider):
if hasattr(item, 'validate'):
Expand Down

0 comments on commit b41d699

Please sign in to comment.