...

.gitignore

@@ -1,8 +1,8 @@
 # WiP
 /packages/perso
-/packages/seo-pocket-crawler
 
 # ...
+/packages/crawler/data
 build
 tmp.csv
 var

composer.json

@@ -9,7 +9,7 @@
     "php": ">=8.1",
     "ext-ctype": "*",
     "ext-iconv": "*",
-    "danielstjules/stringy": "^3.1",
+    "voku/stringy": "^6.5",
     "jeremykendall/php-domain-parser": "^6.1",
     "league/csv": "^9.8",
     "league/uri": "^6.5",
@@ -44,7 +44,7 @@
       "PiedWeb\\Google\\": "packages/google/src",
       "PiedWeb\\GoogleSpreadsheetSeoScraper\\": "packages/google-spreadsheet-seo-scraper/src",
       "PiedWeb\\Extractor\\": "packages/extractor/src",
-      "PiedWeb\\SeoPocketCrawler\\": "packages/seo-pocket-crawler/src",
+      "PiedWeb\\Crawler\\": "packages/crawler/src",
       "PiedWeb\\TextAnalyzer\\": "packages/text-analyzer/src"
     }
   },
@@ -53,7 +53,7 @@
       "PiedWeb\\Curl\\Test\\": "packages/curl/tests",
       "PiedWeb\\Google\\Test\\": "packages/google/tests",
       "PiedWeb\\Extractor\\Test\\": "packages/extractor/tests",
-      "PiedWeb\\SeoPocketCrawler\\Test\\": "packages/seo-pocket-crawler/tests",
+      "PiedWeb\\Crawler\\Test\\": "packages/crawler/tests",
       "PiedWeb\\TextAnalyzer\\Test\\": "packages/text-analyzer/tests"
     }
   },

packages/crawler/LICENSE

@@ -0,0 +1,22 @@
+MIT License
+===========
+
+Copyright (c) Robin Delattre https://piedweb.com <contact@robin-d.fr>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

packages/crawler/README.md

@@ -0,0 +1,127 @@
+<p align="center"><a href="https://dev.piedweb.com">
+<img src="https://raw.githubusercontent.com/PiedWeb/piedweb-devoluix-theme/master/src/img/logo_title.png" width="200" height="200" alt="Open Source Package" />
+</a></p>
+
+# CLI Seo Pocket Crawler
+
+[![Latest Version](https://img.shields.io/github/tag/PiedWeb/Crawler.svg?style=flat&label=release)](https://github.com/PiedWeb/Crawler/tags)
+[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](LICENSE)
+[![GitHub Tests Action Status](https://img.shields.io/github/workflow/status/PiedWeb/Crawler/Tests?label=tests)](https://github.com/PiedWeb/PiedWeb/actions)
+[![Quality Score](https://img.shields.io/scrutinizer/g/PiedWeb/PiedWeb.svg?style=flat)](https://scrutinizer-ci.com/g/PiedWeb/PiedWeb)
+[![Code Coverage](https://codecov.io/gh/PiedWeb/PiedWeb/branch/main/graph/badge.svg)](https://codecov.io/gh/PiedWeb/PiedWeb/branch/main)
+[![Type Coverage](https://shepherd.dev/github/PiedWeb/PiedWeb/coverage.svg)](https://shepherd.dev/github/PiedWeb/PiedWeb)
+[![Total Downloads](https://img.shields.io/packagist/dt/piedweb/crawler.svg?style=flat)](https://packagist.org/packages/piedweb/crawler)
+
+Web Crawler to check few SEO basics.
+
+Use the collected data in your favorite spreadsheet software or retrieve them via your favorite language.
+
+French documentation available :
+https://piedweb.com/seo/crawler
+
+## Install
+
+Via [Packagist](https://img.shields.io/packagist/dt/piedweb/crawler.svg?style=flat)
+
+```bash
+$ composer create-project piedweb/crawler
+```
+
+## Usage
+
+### Crawler CLI
+
+```bash
+$ bin/console crawler:go $start
+```
+
+#### Arguments:
+
+```
+  start                            Define where the crawl start. Eg: https://piedweb.com
+                                   You can specify an id from a previous crawl. Other options will not be listen.
+                                   You can use `last` to continue the last crawl (just stopped)
+```
+
+#### Options:
+
+```
+  -l, --limit=LIMIT                Define where a depth limit [default: 5]
+  -i, --ignore=IGNORE              Virtual Robots.txt to respect (could be a string or an URL).
+  -u, --user-agent=USER-AGENT      Define the user-agent used during the crawl. [default: "SEO Pocket Crawler - PiedWeb.com/seo/crawler"]
+  -w, --wait=WAIT                  In Microseconds, the time to wait between 2 requests. Default 0,1s. [default: 100000]
+  -c, --cache-method=CACHE-METHOD  In Microseconds, the time to wait between two request. Default : 100000 (0,1s). [default: 2]
+  -r, --restart=RESTART            Permit to restart a previous crawl. Values 1 = fresh restart, 2 = restart from cache
+  -h, --help                       Display this help message
+  -q, --quiet                      Do not output any message
+  -V, --version                    Display this application version
+      --ansi                       Force ANSI output
+      --no-ansi                    Disable ANSI output
+  -n, --no-interaction             Do not ask any interactive question
+  -v|vv|vvv, --verbose             Increase the verbosity of messages: 1 for normal output, 2 for more verbose output and 3 for debug
+
+
+
+```
+
+### Extract All External Links in 1s from a previous crawl
+
+```bash
+$ bin/console crawler:external $id [--host]
+```
+
+```
+    --id
+        id from a previous crawl
+        You can use  `last` too show external links from the last crawl.
+
+    --host -ho
+        flag permitting to get only host
+```
+
+### Calcul Page Rank
+
+Will update the previous `data.csv` generated. Then you can explore your website with the PoC `pagerank.html`
+(in a server `npx http-server -c-1 --port 3000`).
+
+```bash
+$ bin/console crawler:pagerank $id
+```
+
+```
+    --id
+        id from a previous crawl
+        You can use `last` too calcul page rank from the last crawl.
+```
+
+## Testing
+
+```bash
+$ composer test
+```
+
+## Todo
+
+- [ ] Better Links Harvesting and Recording (record context (list, nav, sentence...))
+- [ ] Transform the PoC (Page Rank Visualizer)
+- [ ] Complex Page Rank Calculator (with 301, canonical, nofollow, etc.)
+
+## Contributing
+
+Please see [contributing](https://dev.piedweb.com/contributing)
+
+## Credits
+
+- [PiedWeb](https://piedweb.com) ak [Robind4](https://twitter.com/Robind4)
+- [All Contributors](https://github.com/PiedWeb/:package_skake/graphs/contributors)
+
+## License
+
+The MIT License (MIT). Please see [License File](LICENSE) for more information.
+
+[![Latest Version](https://img.shields.io/github/tag/PiedWeb/PiedWeb.svg?style=flat&label=release)](https://github.com/PiedWeb/PiedWeb/tags)
+[![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](https://github.com/PiedWeb/PiedWeb/blob/master/LICENSE)
+[![Build Status](https://img.shields.io/travis/PiedWeb/PiedWeb/master.svg?style=flat)](https://travis-ci.org/PiedWeb/PiedWeb)
+[![Quality Score](https://img.shields.io/scrutinizer/g/PiedWeb/PiedWeb.svg?style=flat)](https://scrutinizer-ci.com/g/PiedWeb/PiedWeb)
+[![Code Coverage](https://img.shields.io/scrutinizer/coverage/g/PiedWeb/PiedWeb.svg?style=flat)](https://scrutinizer-ci.com/g/PiedWeb/PiedWeb/code-structure)
+[![Total Downloads](https://img.shields.io/packagist/dt/piedweb/crawler.svg?style=flat)](https://packagist.org/packages/piedweb/crawler)

packages/crawler/bin/console

@@ -0,0 +1,25 @@
+#!/usr/bin/env php
+<?php
+
+use Symfony\Component\Console\Application;
+use Symfony\Component\Console\Input\ArgvInput;
+use Symfony\Component\ErrorHandler\Debug;
+
+
+if (false === in_array(\PHP_SAPI, ['cli', 'phpdbg', 'embed'], true)) {
+    echo 'Warning: The console should be invoked via the CLI version of PHP, not the '.\PHP_SAPI.' SAPI'.\PHP_EOL;
+}
+
+set_time_limit(0);
+
+require dirname(__DIR__).'/../../vendor/autoload.php';
+
+$input = new ArgvInput();
+
+$application = new Application();
+
+$application->add(new \PiedWeb\Crawler\Command\CrawlerCommand());
+$application->add(new \PiedWeb\Crawler\Command\ShowExternalLinksCommand());
+$application->add(new \PiedWeb\Crawler\Command\PageRankCommand());
+
+$application->run($input);

packages/crawler/composer.json

@@ -0,0 +1,35 @@
+{
+  "name": "piedweb/crawler",
+  "type": "library",
+  "description": "Web Crawler to check few SEO basics.",
+  "keywords": [
+    "Pied Web",
+    "Crawler"
+  ],
+  "homepage": "https://dev.piedweb.com",
+  "license": "MIT",
+  "authors": [
+    {
+      "name": "Robin D. (ak Pied Web)",
+      "email": "contact@robin-d.fr",
+      "homepage": "https://piedweb.com"
+    }
+  ],
+  "require": {
+    "php": ">=8.1",
+    "piedweb/url-harvester": "*",
+    "league/csv": "^9.8",
+    "piedweb/curl": "*",
+    "symfony/console": "^6.1"
+  },
+  "autoload": {
+    "psr-4": {
+      "PiedWeb\\Crawler\\": "src"
+    }
+  },
+  "autoload-dev": {
+    "psr-4": {
+      "PiedWeb\\Crawler\\": "tests"
+    }
+  }
+}