Merge pull request #85 from shiquanwang/#84

[MRG+2] Fix: #84 try to parse JSON-LD with control characters
scrapinghub · Aug 8, 2018 · cd56da2 · cd56da2
2 parents 79dff34 + d839faa
commit cd56da2
Show file tree

Hide file tree

Showing 9 changed files with 77 additions and 15 deletions.
diff --git a/extruct/jsonld.py b/extruct/jsonld.py
@@ -30,10 +30,11 @@ def extract_items(self, document, base_url=None):
     def _extract_items(self, node):
         script = node.xpath('string()')
         try:
-            data = json.loads(script)
+            # TODO: `strict=False` can be configurable if needed
+            data = json.loads(script, strict=False)
         except ValueError:
             # sometimes JSON-decoding errors are due to leading HTML or JavaScript comments
-            data = json.loads(HTML_OR_JS_COMMENTLINE.sub('', script))
+            data = json.loads(HTML_OR_JS_COMMENTLINE.sub('', script), strict=False)
         if isinstance(data, list):
             return data
         elif isinstance(data, dict):

diff --git a/tests/samples/custom.invalid/JSONLD_with_control_characters.html b/tests/samples/custom.invalid/JSONLD_with_control_characters.html
@@ -0,0 +1,17 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <script type="application/ld+json">
+    {
+        "data": "line 1
+        line 2
+        line 3
+"
+    }
+    </script>
+</head>
+
+<body></body>
+
+</html>
diff --git a/tests/samples/custom.invalid/JSONLD_with_control_characters.jsonld b/tests/samples/custom.invalid/JSONLD_with_control_characters.jsonld
@@ -0,0 +1,5 @@
+[
+    {
+        "data": "line 1\n        line 2\n        line 3\n"
+    }
+]
diff --git a/tests/samples/custom.invalid/JSONLD_with_control_characters_comment.html b/tests/samples/custom.invalid/JSONLD_with_control_characters_comment.html
@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <script type="application/ld+json">
+    {
+        "data": "<!-- John joined the festival. --><b>some
+        text</b>"
+    }
+    </script>
+</head>
+
+<body></body>
+
+</html>
diff --git a/tests/samples/custom.invalid/JSONLD_with_control_characters_comment.jsonld b/tests/samples/custom.invalid/JSONLD_with_control_characters_comment.jsonld
@@ -0,0 +1,5 @@
+[
+    {
+        "data": "<!-- John joined the festival. --><b>some\n        text</b>"
+    }
+]
diff --git a/tests/samples/misc/microformat_flat_test.json b/tests/samples/misc/microformat_flat_test.json
@@ -1,8 +1,8 @@
 [
     {
         "@type": [
-            "h-hidden-tablet",
-            "h-hidden-phone"
+            "h-hidden-phone",
+            "h-hidden-tablet"
         ],
         "name": [
             ""
@@ -17,8 +17,8 @@
         "children": [
             {
                 "@type": [
-                    "h-hidden-tablet",
-                    "h-hidden-phone"
+                    "h-hidden-phone",
+                    "h-hidden-tablet"
                 ],
                 "name": [
                     ""
@@ -72,4 +72,4 @@
             "2013-06-13 12:00:00"
         ]
     }
-]
+]
diff --git a/tests/samples/misc/microformat_test.json b/tests/samples/misc/microformat_test.json
@@ -6,8 +6,8 @@
             ]
         },
         "type": [
-            "h-hidden-tablet",
-            "h-hidden-phone"
+            "h-hidden-phone",
+            "h-hidden-tablet"
         ]
     },
     {
@@ -20,8 +20,8 @@
                     ]
                 },
                 "type": [
-                    "h-hidden-tablet",
-                    "h-hidden-phone"
+                    "h-hidden-phone",
+                    "h-hidden-tablet"
                 ]
             },
             {
@@ -80,4 +80,4 @@
             "h-entry"
         ]
     }
-]
+]
diff --git a/tests/test_jsonld.py b/tests/test_jsonld.py
@@ -5,6 +5,7 @@
 from extruct.jsonld import JsonLdExtractor
 from tests import get_testdata
 
+
 class TestJsonLD(unittest.TestCase):
 
     def test_schemaorg_CreativeWork(self):
@@ -42,3 +43,21 @@ def test_jsonld_with_comments(self):
             jsonlde = JsonLdExtractor()
             data = jsonlde.extract(body)
             self.assertEqual(data, expected)
+
+    def test_jsonld_with_control_characters(self):
+        page = 'JSONLD_with_control_characters'
+        body = get_testdata('custom.invalid', '{}.html'.format(page))
+        expected = json.loads(get_testdata('custom.invalid', '{}.jsonld'.format(page)).decode('UTF-8'))
+
+        jsonlde = JsonLdExtractor()
+        data = jsonlde.extract(body)
+        self.assertEqual(data, expected)
+
+    def test_jsonld_with_control_characters_comment(self):
+        page = 'JSONLD_with_control_characters_comment'
+        body = get_testdata('custom.invalid', '{}.html'.format(page))
+        expected = json.loads(get_testdata('custom.invalid', '{}.jsonld'.format(page)).decode('UTF-8'))
+
+        jsonlde = JsonLdExtractor()
+        data = jsonlde.extract(body)
+        self.assertEqual(data, expected)
diff --git a/tests/test_uniform.py b/tests/test_uniform.py
@@ -29,12 +29,12 @@ def test_uopengraph(self):
 
     def test_umicroformat(self):
         expected = [ { '@context': 'http://microformats.org/wiki/',
-                     '@type': ['h-hidden-tablet', 'h-hidden-phone'],
+                     '@type': ['h-hidden-phone', 'h-hidden-tablet'],
                      'name': ['']},
                    { '@context': 'http://microformats.org/wiki/',
                      '@type': ['h-hidden-phone'],
-                     'children': [ { '@type': [ 'h-hidden-tablet',
-                                                'h-hidden-phone'],
+                     'children': [ { '@type': [ 'h-hidden-phone',
+                                                'h-hidden-tablet'],
                                      'name': ['']},
                                    { '@type': ['h-hidden-phone'],
                                      'name': [ 'aJ Styles FastLane 2018 15 x '