update dependencies & fix cs

.github/workflows/split-monorepo.yaml

@@ -18,18 +18,12 @@ jobs:
             fail-fast: false
             matrix:
                 package:
-                    -
-                        package: 'crawler'
-                    -
-                        package: 'curl'
-                    -
-                        package: 'extractor'
-                    -
-                        package: 'google'
-                    -
-                        package: 'google-spreedsheet-seo-scraper'
-                    -
-                        package: 'text-analyzer'
+                    - crawler
+                    - curl
+                    - extractor
+                    - google
+                    - google-spreedsheet-seo-scraper
+                    - text-analyzer
 
         steps:
             -
@@ -39,29 +33,29 @@ jobs:
             -
                 if: "!startsWith(github.ref, 'refs/tags/')"
                 # Uses an action in the root directory
-                name: Monorepo Split of ${{ matrix.package.package }}
+                name: Monorepo Split of ${{ matrix.package }}
                 uses: danharrin/monorepo-split-github-action@v2.3.0
                 env:
                     GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
                 with:
-                    package_directory: 'packages/${{ matrix.package.package }}'
+                    package_directory: 'packages/${{ matrix.package }}'
                     repository_organization: 'piedweb'
-                    repository_name: '${{ matrix.package.package }}'
+                    repository_name: '${{ matrix.package }}'
                     user_name: "GitHub Action"
                     user_email: "action@github.com"
 
             # with tag
             -
                 if: "startsWith(github.ref, 'refs/tags/')"
                 # Uses an action in the root directory
-                name: Monorepo Tagged Split of ${{ matrix.package.package }}
+                name: Monorepo Tagged Split of ${{ matrix.package }}
                 uses: danharrin/monorepo-split-github-action@v2.3.0
                 env:
                     GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}
                 with:
                     tag: ${GITHUB_REF#refs/tags/}
-                    package_directory: 'packages/${{ matrix.package.package }}'
+                    package_directory: 'packages/${{ matrix.package }}'
                     repository_organization: 'piedweb'
-                    repository_name: '${{ matrix.package.package }}'
+                    repository_name: '${{ matrix.package }}'
                     user_name: "GitHub Action"
                     user_email: "action@github.com"

composer.json

@@ -18,7 +18,7 @@
     "symfony/cache": "^6.1",
     "symfony/console": "^6.1",
     "symfony/css-selector": "^6.1",
-    "symfony/dom-crawler": "^6.2",
+    "symfony/dom-crawler": "^6.3",
     "symfony/serializer": "^6.2",
     "thecodingmachine/safe": "^2.1",
     "vierbergenlars/php-semver": "^3.0.3",
@@ -49,7 +49,8 @@
       "PiedWeb\\GoogleSpreadsheetSeoScraper\\": "packages/google-spreadsheet-seo-scraper/src",
       "PiedWeb\\Extractor\\": "packages/extractor/src",
       "PiedWeb\\Crawler\\": "packages/crawler/src",
-      "PiedWeb\\TextAnalyzer\\": "packages/text-analyzer/src"
+      "PiedWeb\\TextAnalyzer\\": "packages/text-analyzer/src",
+      "PiedWeb\\Perso\\": "packages/perso"
     }
   },
   "autoload-dev": {
@@ -63,11 +64,11 @@
   },
   "scripts": {
     "stan": "vendor/bin/phpstan analyze packages/*/src",
+    "format": "vendor/bin/php-cs-fixer fix && vendor/bin/php-cs-fixer fix rector.php",
     "psalm": "vendor/bin/psalm --no-suggestions --no-cache",
     "rector": "vendor/bin/rector process",
     "test": "vendor/bin/phpunit --stop-on-failure",
-    "test-google": "vendor/bin/phpunit --stop-on-failure --testsuite google",
-    "format": "vendor/bin/php-cs-fixer fix && vendor/bin/php-cs-fixer fix rector.php"
+    "test-google": "vendor/bin/phpunit --stop-on-failure --testsuite google"
   },
   "config": {
     "sort-packages": true,
@@ -80,7 +81,7 @@
   "repositories": [
     {
       "type": "vcs",
-      "url": "https://github.com/zoonru/puphpeteer.git"
+      "url": "https://github.com/RobinDev/puphpeteer.git"
     },
     {
       "type": "vcs",

package.json

@@ -1,5 +1,8 @@
 {
   "dependencies": {
-    "@nesk/puphpeteer": "https://github.com/zoonru/puphpeteer"
+    "@nesk/puphpeteer": "https://github.com/zoonru/puphpeteer",
+    "puppeteer": "^20",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
   }
 }

packages/crawler/src/Crawler.php

@@ -27,7 +27,7 @@ final class Crawler
 
     private bool $nothingUpdated = true;
 
-    public readonly CrawlerConfig  $config;
+    public readonly CrawlerConfig $config;
 
     public function __construct(
         CrawlerConfig|string $config,
@@ -39,7 +39,7 @@ public function __construct(
     public static function continue(
         string $id,
         bool $debug = true,
-        ?string $dataDirectory = null
+        string $dataDirectory = null
     ): self {
         $config = CrawlerConfig::loadFrom($id, $dataDirectory);
         $current = new self($config, $debug);
@@ -56,7 +56,7 @@ public static function restart(
         string $id,
         bool $fromCache = false,
         bool $debug = true,
-        ?string $dataDirectory = null
+        string $dataDirectory = null
     ): self {
         $config = CrawlerConfig::loadFrom($id, $dataDirectory);
         $current = new self($config, $debug);

packages/crawler/src/CrawlerConfig.php

@@ -42,12 +42,12 @@ final class CrawlerConfig
      */
     public function __construct(
         public readonly int $depthLimit = 0,
-        ?string $userAgent = null,
-        ?int $cacheMethod = null,
-        ?int $sleepBetweenReqInMs = null, // ms
-        ?string $virtualRobotsTxtRules = null,
-        ?array $toHarvest = null,
-        ?string $dataDirectory = null,
+        string $userAgent = null,
+        int $cacheMethod = null,
+        int $sleepBetweenReqInMs = null, // ms
+        string $virtualRobotsTxtRules = null,
+        array $toHarvest = null,
+        string $dataDirectory = null,
         public readonly int $autosave = 500 // number of Urls we can crawled before saving (0 = autosaving disabled)
     ) {
         $this->userAgent = $userAgent ?? 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)';
@@ -78,7 +78,7 @@ public function getStartUrl(): UrlManipuler
         return $this->startUrl;
     }
 
-    public static function dataDirectory(?string $dataDirectory = null): string
+    public static function dataDirectory(string $dataDirectory = null): string
     {
         $dataDirectory = (string) $dataDirectory;
 
@@ -110,7 +110,7 @@ public static function getLastCrawl(string $dataDirectory): string
         return $lastCrawl;
     }
 
-    public static function loadFrom(string $crawlId, ?string $dataDirectory = null): self
+    public static function loadFrom(string $crawlId, string $dataDirectory = null): self
     {
         $dataDirectory = self::dataDirectory($dataDirectory);
 

packages/crawler/src/ExtractExternalLinks.php

@@ -15,7 +15,7 @@ final class ExtractExternalLinks
 
     public function __construct(
         string $id,
-        ?string $dataDirectory = null
+        string $dataDirectory = null
     ) {
         $this->config = CrawlerConfig::loadFrom($id, $dataDirectory);
         $this->dir = $this->config->getDataFolder().'/links';

packages/crawler/src/LinksVisualizer.php

@@ -16,7 +16,7 @@ class LinksVisualizer
      */
     protected array $results = ['nodes' => [], 'links' => []];
 
-    public function __construct(string $id, ?string $dataDirectory = null)
+    public function __construct(string $id, string $dataDirectory = null)
     {
         $this->config = CrawlerConfig::loadFrom($id, $dataDirectory);
 

packages/crawler/src/SimplePageRankCalculator.php

@@ -32,7 +32,7 @@ final class SimplePageRankCalculator
 
     private float $dampingFactor = 0.85;
 
-    public function __construct(string $id, ?string $dataDirectory = null)
+    public function __construct(string $id, string $dataDirectory = null)
     {
         $this->config = CrawlerConfig::loadFrom($id, $dataDirectory);
 

packages/curl/src/Client.php

@@ -14,7 +14,7 @@ class Client
 
     protected Response $response;
 
-    public function __construct(?string $target = null)
+    public function __construct(string $target = null)
     {
         if (null !== $target) {
             $this->setTarget($target);
@@ -133,7 +133,7 @@ public function resetError(): void
         $this->errorMessage = '';
     }
 
-    public function request(?string $target = null): bool
+    public function request(string $target = null): bool
     {
         if (null !== $target) {
             $this->setTarget($target);

packages/curl/src/ExtendedClient.php

@@ -13,7 +13,7 @@ class ExtendedClient extends Client
      */
     final public const DEFAULT_USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.104 Safari/537.36';
 
-    private bool  $fakeBrowserHeader = false;
+    private bool $fakeBrowserHeader = false;
 
     public ?string $referer = null;
 
@@ -262,7 +262,7 @@ public function checkHeader(\CurlHandle $handle, string $line): int
     /**
      * Execute the request.
      */
-    public function request(?string $target = null, bool $updateRefererAndCookies = true): bool
+    public function request(string $target = null, bool $updateRefererAndCookies = true): bool
     {
         if ($this->fakeBrowserHeader) {
             $this->setBrowserHeader();

packages/curl/src/Response.php

@@ -146,7 +146,7 @@ public function getCookies(): ?string
      *
      * @return int|string|array<string, string|int>|null
      */
-    public function getInfo(?string $key = null): int|string|array|null
+    public function getInfo(string $key = null): int|string|array|null
     {
         return null !== $key && '' !== $key ? ($this->info[$key] ?? null) : $this->info;
     }

packages/curl/src/ResponseFromCache.php

@@ -12,7 +12,7 @@ class ResponseFromCache extends Response
      */
     public function __construct(
         string $filePathOrContent,
-        ?string $url = null,
+        string $url = null,
         array $info = [],
         mixed $headersSeparator = \PHP_EOL.\PHP_EOL
     ) {

packages/extractor/src/Link.php

@@ -64,7 +64,7 @@ public static function initialize(
         string $url,
         Url $parentUrl,
         bool $parentMayFollow = true,
-        ?\DOMElement $element = null
+        \DOMElement $element = null
     ): self {
         $self = new self();
         $self->element = $element;

packages/google/package.json

@@ -1,5 +1,8 @@
 {
   "dependencies": {
-    "@nesk/puphpeteer": "https://github.com/zoonru/puphpeteer.git"
+    "@nesk/puphpeteer": "https://github.com/zoonru/puphpeteer.git",
+    "puppeteer": "^20",
+    "puppeteer-extra": "^3.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
   }
 }

packages/google/src/Extractor/SERPExtractor.php

@@ -307,7 +307,7 @@ public function getRelatedSearches(): array
     /**
      * @param string[] $xpaths
      */
-    public function exists(array $xpaths, ?\DOMNode &$node = null): bool
+    public function exists(array $xpaths, \DOMNode &$node = null): bool
     {
         try {
             $node = $this->getNode($xpaths);

packages/google/src/GoogleRequester.php

@@ -37,7 +37,7 @@ public function getPuppeteerClient(): Puphpeteer
         return $this->puppeteerClient;
     }
 
-    public function requestGoogleWithCurl(GoogleSERPManager $Google, ?callable $manageProxy = null): string
+    public function requestGoogleWithCurl(GoogleSERPManager $Google, callable $manageProxy = null): string
     {
         $this->getCurlClient()->setLanguage($Google->language.';q=0.9');
 
@@ -54,11 +54,13 @@ public function requestGoogleWithCurl(GoogleSERPManager $Google, ?callable $mana
         return $this->getCurlClient()->getResponse()->getBody();
     }
 
-    public function requestGoogleWithPuppeteer(GoogleSERPManager $manager, ?callable $manageProxy = null): string
+    public function requestGoogleWithPuppeteer(GoogleSERPManager $manager, callable $manageProxy = null): string
     {
-        $this->getPuppeteerClient()
-            ->instantiate(Puphpeteer::EMULATE_OPTIONS_MOBILE, $manager->language)
-            ->setCookie('CONSENT', 'YES+', '.google.fr');
+        $pClient = $this->getPuppeteerClient();
+
+        $pClient->instantiate(Puphpeteer::EMULATE_OPTIONS_MOBILE, $manager->language);
+
+        $pClient->setCookie('CONSENT', 'YES+', '.google.fr');
 
         if (null !== $manageProxy) {
             \call_user_func($manageProxy, $this->getPuppeteerClient());

packages/google/src/Helper/Puphpeteer.php

@@ -51,6 +51,7 @@ public function getLogger(): PuppeteerLogger
             'isLandscape' => false,
         ],
         'userAgent' => 'Mozilla/5.0 (Linux; Android 10; SM-A305N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.210 Mobile Safari/537.36',
+        'headless' => 'new',
     ];
 
     /**
@@ -78,7 +79,7 @@ public function instantiate(
         array $emulateOptions = [],
         string $language = '',
         array $userOptions = [
-            'headless' => true,
+            'headless' => 'new',
             'slowMo' => 250,
             'read_timeout' => 9000,
             'idle_timeout' => 9000, ]
@@ -97,14 +98,23 @@ public function instantiate(
             return $this;
         }
 
+        $userOptions['js_extra'] = "
+            const puppeteer = require('puppeteer-extra');
+            const StealthPlugin = require('puppeteer-extra-plugin-stealth');
+            puppeteer.use(StealthPlugin());
+            instruction.setDefaultResource(puppeteer);
+        ";
+
         Logger::log('launching new Puppeteer instance `'.self::$currentKey.'`');
+
         self::$puppeteer[self::$currentKey] = new Puppeteer($userOptions);
         self::$browser[self::$currentKey] = self::$puppeteer[self::$currentKey]->launch(
             array_merge(
                 [] !== $emulateOptions ? $emulateOptions : self::EMULATE_OPTIONS_MOBILE,
                 // ['executablePath' => '/snap/bin/chromium',]
             )
         );
+
         self::$browserPage[self::$currentKey] = $this->getBrowserPage(true);
         self::$browserPage[self::$currentKey]->emulate([] !== $emulateOptions ? $emulateOptions : self::EMULATE_OPTIONS_MOBILE);
 
@@ -131,8 +141,7 @@ public function getBrowserPage(bool $new = false): Page
         if (self::$browserPage[self::$currentKey]::class === BasicResource::class) {
             dump($new);
             dump(self::$browser[self::$currentKey]::class);
-            dump(self::$browserPage[self::$currentKey]::class);
-            // self::$browserPage[self::$currentKey] = self::$browserPage[self::$currentKey]->newPage();
+            // self::$browserPage[self::$currentKey] = self::$browser[self::$currentKey]->newPage();
             dd(self::$browserPage[self::$currentKey]::class);
         }
 

packages/google/tests/GoogleSerpTest.php

@@ -34,7 +34,7 @@ public function testPuphpeteerMobile(): void
         $manager = $this->getSerpManager();
 
         $googleRequester = new GoogleRequester();
-        $rawHtml = $manager->getCache() ?? $manager->setCache($googleRequester->requestGoogleWithPuppeteer($manager));
+        $rawHtml = $googleRequester->requestGoogleWithPuppeteer($manager); // $manager->getCache() ?? $manager->setCache($googleRequester->requestGoogleWithPuppeteer($manager));
         file_put_contents('debug.html', $rawHtml);
         $googleRequester->getPuppeteerClient()->getBrowserPage()->screenshot(['path' => 'debug.png']);
 

packages/text-analyzer/src/Analysis.php

@@ -34,7 +34,7 @@ public function getWordNumber(): int
     /**
      * @return array<string, int>
      */
-    public function getExpressions(?int $minFound = null): array
+    public function getExpressions(int $minFound = null): array
     {
         return $minFound ? array_filter(
             $this->getExpressions(),