Basic Lists (#36)

* Adding example * wip * wip * wip * wip * wip * wip
spekulatius · Nov 24, 2020 · 0aac528 · 0aac528
1 parent 47d6f8a
commit 0aac528
Show file tree

Hide file tree

Showing 6 changed files with 213 additions and 0 deletions.
diff --git a/src/phpscraper.php b/src/phpscraper.php
@@ -509,6 +509,50 @@ public function headings()
         ];
     }
 
+    /**
+     * Get all lists on the page
+     *
+     * @return array
+     */
+    public function lists()
+    {
+        $lists = [];
+
+        foreach ($this->current_page->filter('ol, ul') as $list) {
+            $lists[] = [
+                'type' => $list->tagName,
+                'children' => $list->childNodes,
+                'children_plain' => array_filter(array_map('trim', explode("\n", $list->textContent))),
+            ];
+        }
+
+        return $lists;
+    }
+
+    /**
+     * Get all ordered lists on the page
+     *
+     * @return array
+     */
+    public function orderedLists()
+    {
+        return array_values(array_filter($this->lists(), function ($list) {
+            return $list['type'] === 'ol';
+        }));
+    }
+
+    /**
+     * Get all unordered lists on the page
+     *
+     * @return array
+     */
+    public function unorderedLists()
+    {
+        return array_values(array_filter($this->lists(), function ($list) {
+            return $list['type'] === 'ul';
+        }));
+    }
+
     /**
      * Get all paragraphs of the page
      *

diff --git a/tests/ListsTest.php b/tests/ListsTest.php
@@ -0,0 +1,51 @@
+<?php
+
+namespace Tests;
+
+class ListsTest extends BaseTest
+{
+    /**
+     * @test
+     */
+    public function checkCountTest()
+    {
+        $web = new \spekulatius\phpscraper();
+
+        /**
+         * Navigate to the test page. This page contains:
+         *
+         * <h2>Example 1: Unordered List</h2>
+         * <ul>
+         *     <li>Unordered item 1</li>
+         *     <li>Unordered item 2</li>
+         *     <li>Unordered item with <b>HTML</b></li>
+         * </ul>
+         *
+         * <h2>Example 2: Ordered List</h2>
+         * <ol>
+         *     <li>Order list item 1</li>
+         *     <li>Order list item 2</li>
+         *     <li>Order list item with <i>HTML</i></li>
+         * </ol>
+         */
+        $web->go($this->url . '/content/lists.html');
+
+        // Check all lists are recognized
+        $this->assertSame(count($web->lists), 2);
+        $this->assertSame(count($web->unorderedLists), 1);
+        $this->assertSame(count($web->orderedLists), 1);
+
+        // Check the contents
+        $this->assertSame([
+            'Ordered list item 1',
+            'Ordered list item 2',
+            'Ordered list item with HTML',
+        ], $web->orderedLists[0]['children_plain']);
+
+        $this->assertSame([
+            'Unordered list item 1',
+            'Unordered list item 2',
+            'Unordered list item with HTML',
+        ], $web->unorderedLists[0]['children_plain']);
+    }
+}
diff --git a/tests/resources/content/lists.html b/tests/resources/content/lists.html
@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>List Tests</title>
+    </head>
+    <body>
+        <h1>We are testing here!</h1>
+        <p>This page contains example lists.</p>
+
+        <h2>Example 1: Unordered List</h2>
+        <ul>
+            <li>Unordered list item 1</li>
+            <li>Unordered list item 2</li>
+            <li>Unordered list item with <b>HTML</b></li>
+        </ul>
+
+        <h2>Example 2: Ordered List</h2>
+        <ol>
+            <li>Ordered list item 1</li>
+            <li>Ordered list item 2</li>
+            <li>Ordered list item with <i>HTML</i></li>
+        </ol>
+    </body>
+</html>
diff --git a/websites/.vuepress/config.js b/websites/.vuepress/config.js
@@ -53,6 +53,7 @@ module.exports = {
         children: [
           'examples/headings',
           'examples/paragraphs',
+          'examples/lists',
           'examples/outline',
           'examples/extract-keywords',
           'examples/scrape-images',

diff --git a/websites/examples/lists.md b/websites/examples/lists.md
@@ -0,0 +1,69 @@
+---
+image: https://api.imageee.com/bold?text=PHP:%20Scraping%20Lists&bg_image=https://images.unsplash.com/photo-1542762933-ab3502717ce7
+---
+
+# Scraping Lists
+
+Scraping lists follows a similar approach as other scraping:
+
+```php
+$web = new \spekulatius\phpscraper();
+
+/**
+ * Navigate to the test page. This page contains:
+ *
+ * <h2>Example 1: Unordered List</h2>
+ * <ul>
+ *     <li>Unordered list item 1</li>
+ *     <li>Unordered list item 2</li>
+ *     <li>Unordered list item with <b>HTML</b></li>
+ * </ul>
+ *
+ * <h2>Example 2: Ordered List</h2>
+ * <ol>
+ *     <li>Ordered list item 1</li>
+ *     <li>Ordered list item 2</li>
+ *     <li>Ordered list item with <i>HTML</i></li>
+ * </ol>
+ */
+$web->go('https://test-pages.phpscraper.de/content/lists.html');
+
+var_dump($web->unorderedLists);
+/**
+ * Only unordered lists (<ul>)
+ *
+ * [
+ *     "type" => "ul",
+ *     "children" => ... // List of childNodes
+ *     "children_plain" =>
+ *     [
+ *         "Unordered list item 1"
+ *         "Unordered list item 2"
+ *         "Unordered list item with HTML"
+ *     ]
+ * ]
+ */
+
+var_dump($web->orderedLists);
+/**
+ * Only ordered lists (<ol>)
+ *
+ * [
+ *     "type" => "ul",
+ *     "children" => ... // List of childNodes
+ *     "children_plain" =>
+ *     [
+ *         "Ordered list item 1"
+ *         "Ordered list item 2"
+ *         "Ordered list item with HTML"
+ *     ]
+ * ]
+ */
+
+// Both lists combined (as above)
+var_dump($web->lists);
+```
+
+::: warning Nested Lists
+At the moment, this doesn't handle nested lists well. To allow further processing nested lists are included in the result as `children`.
+:::
diff --git a/websites/test-pages/content/lists.html b/websites/test-pages/content/lists.html
@@ -0,0 +1,24 @@
+<!DOCTYPE html>
+<html>
+    <head>
+        <title>List Tests</title>
+    </head>
+    <body>
+        <h1>We are testing here!</h1>
+        <p>This page contains example lists.</p>
+
+        <h2>Example 1: Unordered List</h2>
+        <ul>
+            <li>Unordered list item 1</li>
+            <li>Unordered list item 2</li>
+            <li>Unordered list item with <b>HTML</b></li>
+        </ul>
+
+        <h2>Example 2: Ordered List</h2>
+        <ol>
+            <li>Ordered list item 1</li>
+            <li>Ordered list item 2</li>
+            <li>Ordered list item with <i>HTML</i></li>
+        </ol>
+    </body>
+</html>