~ hide my ip

vantoozz · Jul 1, 2017 · b19f37a · b19f37a
1 parent 6bca090
commit b19f37a
Show file tree

Hide file tree

Showing 10 changed files with 348 additions and 388 deletions.
diff --git a/src/HttpClient/HttpClientInterface.php b/src/HttpClient/HttpClientInterface.php
@@ -10,8 +10,7 @@ interface HttpClientInterface
 {
     /**
      * @param string $uri
-     * @param array $headers
      * @return string
      */
-    public function get(string $uri, array $headers): string;
+    public function get(string $uri): string;
 }
diff --git a/src/HttpClient/HttplugHttpClient.php b/src/HttpClient/HttplugHttpClient.php
@@ -38,13 +38,12 @@ public function __construct(Client $httpClient, MessageFactory $messageFactory)
 
     /**
      * @param string $uri
-     * @param array $headers
      * @return string
      * @throws HttpClientException
      */
-    public function get(string $uri, array $headers): string
+    public function get(string $uri): string
     {
-        $request = $this->messageFactory->createRequest(Http::GET, $uri, $headers);
+        $request = $this->messageFactory->createRequest(Http::GET, $uri);
         try {
             return $this->httpClient->sendRequest($request)->getBody()->getContents();
         } catch (ClientException  $e) {

diff --git a/src/Scrapers/AbstractFreeProxyListScraper.php b/src/Scrapers/AbstractFreeProxyListScraper.php
@@ -38,7 +38,7 @@ public function __construct(HttpClientInterface $httpClient)
     public function get(): \Generator
     {
         try {
-            $html = $this->httpClient->get($this->baseUrl(), []);
+            $html = $this->httpClient->get($this->baseUrl());
         } catch (HttpClientException $e) {
             throw new ScraperException($e->getMessage(), $e->getCode(), $e);
         }

diff --git a/src/Scrapers/HideMyIpScraper.php b/src/Scrapers/HideMyIpScraper.php
@@ -2,6 +2,8 @@
 
 namespace Vantoozz\ProxyScraper\Scrapers;
 
+use Faker\Generator as Faker;
+use Faker\Provider\UserAgent;
 use Vantoozz\ProxyScraper\Exceptions\HttpClientException;
 use Vantoozz\ProxyScraper\Exceptions\InvalidArgumentException;
 use Vantoozz\ProxyScraper\Exceptions\ScraperException;
@@ -24,7 +26,7 @@ final class HideMyIpScraper implements ScraperInterface
     /**
      *
      */
-    private const URL = 'https://www.hide-my-ip.com/proxylist.shtml';
+    private const URL = 'https://www.hide-my-ip.com/%s/proxylist.shtml';
 
     /**
      * FreeProxyListScraper constructor.
@@ -42,14 +44,7 @@ public function __construct(HttpClientInterface $httpClient)
     public function get(): \Generator
     {
         try {
-            $html = $this->httpClient->get(
-                static::URL,
-                [
-                    'User-Agent' =>
-                        'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) ' .
-                        'Chrome/41.0.2228.0 Safari/537.36',
-                ]
-            );
+            $html = $this->httpClient->get($this->makeUrl());
         } catch (HttpClientException $e) {
             throw new ScraperException($e->getMessage(), $e->getCode(), $e);
         }
@@ -66,6 +61,21 @@ public function get(): \Generator
         }
     }
 
+    /**
+     * @return string
+     */
+    private function makeUrl(): string
+    {
+        $languages = [
+            'es', 'fr', 'it', 'pt', 'nl', 'de', 'se',
+            'dk', 'pl', 'tr', 'ar', 'ru', 'ro',
+            'cn', 'kr', 'jp', 'vn', 'th', 'sr',
+
+        ];
+
+        return sprintf(static::URL, $languages[array_rand($languages)]);
+    }
+
     /**
      * @param string $html
      * @return array
@@ -78,7 +88,7 @@ private function extractData(string $html): array
         if ($expectedPartsCount !== count($parts)) {
             throw new ScraperException('Unknown markup');
         }
-        $json = trim(explode(';<!-- proxylist -->', $parts[1])[0]);
+        $json = trim(explode(";\n\n", $parts[1])[0]);
         $data = json_decode($json, true);
         if (!$data) {
             throw new ScraperException('Cannot parse json: ' . json_last_error_msg());

diff --git a/src/Scrapers/ProxyDbScraper.php b/src/Scrapers/ProxyDbScraper.php
@@ -58,7 +58,7 @@ public function get(): \Generator
     private function getPage(int $offset, int $pageSize): \Generator
     {
         try {
-            $html = $this->httpClient->get(sprintf(static::PAGE_URL, $pageSize, $offset), []);
+            $html = $this->httpClient->get(sprintf(static::PAGE_URL, $pageSize, $offset));
         } catch (HttpClientException $e) {
             throw new ScraperException($e->getMessage(), $e->getCode(), $e);
         }

diff --git a/src/Scrapers/RemoteTextScraper.php b/src/Scrapers/RemoteTextScraper.php
@@ -34,7 +34,7 @@ public function __construct(HttpClientInterface $httpClient)
     public function get(): \Generator
     {
         try {
-            $text = $this->httpClient->get($this->remoteTextUrl(), []);
+            $text = $this->httpClient->get($this->remoteTextUrl());
         } catch (HttpClientException $e) {
             throw new ScraperException($e->getMessage(), $e->getCode(), $e);
         }

diff --git a/tests/fixtures/hideMyIp.html b/tests/fixtures/hideMyIp.html
diff --git a/tests/systemTests.php b/tests/systemTests.php
@@ -41,8 +41,13 @@
          ] as $class) {
     $miner->addScraper($container->get($class));
 }
+
+$cacheFilename = __DIR__ . '/.cached_proxies';
+if (in_array('--refresh', $argv, true)) {
+    unlink($cacheFilename);
+}
 if (in_array('--cached', $argv, true)) {
-    $miner = new Cached($miner, __DIR__ . '/.cached_proxies');
+    $miner = new Cached($miner, $cacheFilename);
 }
 
 $pipeline = new ReportsPipeline;

diff --git a/tests/unit/HttpClient/HttplugHttpClientTest.php b/tests/unit/HttpClient/HttplugHttpClientTest.php
@@ -37,7 +37,7 @@ public function it_throws_an_exception_if_an_error_happens(): void
             ->willThrowException(new \Exception('error message'));
 
         $httpClient = new HttplugHttpClient($client, $messageFactory);
-        $httpClient->get('some url', []);
+        $httpClient->get('some url');
     }
 
     /**
@@ -67,7 +67,7 @@ public function it_throws_an_exception_if_processing_the_request_is_impossible()
             });
 
         $httpClient = new HttplugHttpClient($client, $messageFactory);
-        $httpClient->get('some url', []);
+        $httpClient->get('some url');
     }
 
     /**
@@ -110,6 +110,6 @@ public function it_returns_a_string(): void
 
         $httpClient = new HttplugHttpClient($client, $messageFactory);
 
-        $this->assertEquals('some string', $httpClient->get('some url', []));
+        $this->assertEquals('some string', $httpClient->get('some url'));
     }
 }
diff --git a/tests/unit/Scrapers/HideMyIpScraperTest.php b/tests/unit/Scrapers/HideMyIpScraperTest.php
@@ -84,7 +84,7 @@ public function it_returns_a_proxy(): void
         $proxy = $scraper->get()->current();
 
         $this->assertInstanceOf(Proxy::class, $proxy);
-        $this->assertSame('104.41.154.213:8118', (string)$proxy);
+        $this->assertSame('218.161.1.189:3128', (string)$proxy);
     }
 
     /**