Skip to content
Permalink
Browse files

Merge pull request #586 from zachborboa/master

Add ability to make multicurl requests with multiple proxies
  • Loading branch information...
zachborboa committed May 12, 2019
2 parents 494d8a7 + 0857023 commit 98cad71d345d655ee03196689af8094407d121c8
Showing with 144 additions and 0 deletions.
  1. +1 −0 README.md
  2. +24 −0 examples/multi_curl_proxies.php
  3. +13 −0 src/Curl/ArrayUtil.php
  4. +25 −0 src/Curl/MultiCurl.php
  5. +81 −0 tests/PHPCurlClass/PHPMultiCurlClassTest.php
@@ -320,6 +320,7 @@ MultiCurl::setJsonDecoder($mixed)
MultiCurl::setOpt($option, $value)
MultiCurl::setOpts($options)
MultiCurl::setPort($port)
MultiCurl::setProxies($proxies)
MultiCurl::setProxy($proxy, $port = null, $username = null, $password = null)
MultiCurl::setProxyAuth($auth)
MultiCurl::setProxyTunnel($tunnel = true)
@@ -0,0 +1,24 @@
<?php
require __DIR__ . '/../vendor/autoload.php';
use \Curl\MultiCurl;
$multi_curl = new MultiCurl();
$multi_curl->setProxies(array(
'someproxy.com:9999',
'someproxy.com:80',
'someproxy.com:443',
'someproxy.com:1080',
'someproxy.com:3128',
'someproxy.com:8080',
));
$multi_curl->addGet('https://httpbin.org/ip');
$multi_curl->addGet('https://httpbin.org/ip');
$multi_curl->addGet('https://httpbin.org/ip');
$multi_curl->complete(function ($instance) {
echo
'curl id ' . $instance->id . ' used proxy ' .
$instance->getOpt(CURLOPT_PROXY) . ' and ' .
'ip is ' . $instance->response->origin . "\n";
});
$multi_curl->start();
@@ -77,4 +77,17 @@ public static function array_flatten_multidim($array, $prefix = false)
}
return $return;
}
/**
* Array Random
*
* @access public
* @param $array
*
* @return mixed
*/
public static function array_random($array)
{
return $array[mt_rand(0, count($array) - 1)];
}
}
@@ -2,6 +2,8 @@
namespace Curl;
use Curl\ArrayUtil;
class MultiCurl
{
public $baseUrl = null;
@@ -23,6 +25,7 @@ class MultiCurl
private $cookies = array();
private $headers = array();
private $options = array();
private $proxies = null;
private $jsonDecoder = null;
private $xmlDecoder = null;
@@ -571,6 +574,21 @@ public function setProxy($proxy, $port = null, $username = null, $password = nul
}
}
/**
* Set Proxies
*
* Set proxies to tunnel requests through. When set, a random proxy will be
* used for the request.
*
* @access public
* @param $proxies array - A list of HTTP proxies to tunnel requests
* through. May include port number.
*/
public function setProxies($proxies)
{
$this->proxies = $proxies;
}
/**
* Set Proxy Auth
*
@@ -931,6 +949,13 @@ private function initHandle($curl)
$curl->setRetry($this->retry);
$curl->setCookies($this->cookies);
// Use a random proxy for the curl instance when proxies have been set
// and the curl instance doesn't already have a proxy set.
if (is_array($this->proxies) && $curl->getOpt(CURLOPT_PROXY) === null) {
$random_proxy = ArrayUtil::array_random($this->proxies);
$curl->setProxy($random_proxy);
}
$curlm_error_code = curl_multi_add_handle($this->multiCurl, $curl->curl);
if (!($curlm_error_code === CURLM_OK)) {
throw new \ErrorException('cURL multi add handle error: ' . curl_multi_strerror($curlm_error_code));
@@ -2919,4 +2919,85 @@ public function testSetProxyTunnel()
$multi_curl->setProxyTunnel($tunnel);
$this->assertEquals($tunnel, $multi_curl->getOpt(CURLOPT_HTTPPROXYTUNNEL));
}
public function testSetProxiesRandomProxy()
{
$proxies = array(
'example.com:80',
'example.com:443',
'example.com:1080',
'example.com:3128',
'example.com:8080',
);
$multi_curl = new MultiCurl();
$multi_curl->setProxies($proxies);
$multi_curl->addGet(Test::TEST_URL);
$multi_curl->addGet(Test::TEST_URL);
$multi_curl->addGet(Test::TEST_URL);
// Make MultiCurl::curls accessible and MultiCurl::initHandle()
// callable.
$reflector = new \ReflectionClass('\Curl\MultiCurl');
$property = $reflector->getProperty('curls');
$property->setAccessible(true);
$multi_curl_curls = $property->getValue($multi_curl);
$multi_curl_initHandle = $reflector->getMethod('initHandle');
$multi_curl_initHandle->setAccessible(true);
// Ensure we have the requests queued.
$this->assertCount(3, $multi_curl_curls);
// Invoke MultiCurl::initHandle() so that proxies are set.
foreach ($multi_curl_curls as $curl) {
$multi_curl_initHandle->invoke($multi_curl, $curl);
}
// Ensure each request is set to one of the proxies.
foreach ($multi_curl_curls as $curl) {
$this->assertContains($curl->getOpt(CURLOPT_PROXY), $proxies);
}
}
public function testSetProxiesAlreadySet()
{
$proxies = array(
'example.com:80',
'example.com:443',
'example.com:1080',
'example.com:3128',
'example.com:8080',
);
$multi_curl = new MultiCurl();
$multi_curl->setProxies($proxies);
$get_1 = $multi_curl->addGet(Test::TEST_URL);
$get_2 = $multi_curl->addGet(Test::TEST_URL);
$get_2->setProxy('example.com:9999');
$get_3 = $multi_curl->addGet(Test::TEST_URL);
// Make MultiCurl::curls accessible and MultiCurl::initHandle()
// callable.
$reflector = new \ReflectionClass('\Curl\MultiCurl');
$property = $reflector->getProperty('curls');
$property->setAccessible(true);
$multi_curl_curls = $property->getValue($multi_curl);
$multi_curl_initHandle = $reflector->getMethod('initHandle');
$multi_curl_initHandle->setAccessible(true);
// Ensure we have the requests queued.
$this->assertCount(3, $multi_curl_curls);
// Invoke MultiCurl::initHandle() so that proxies are set.
foreach ($multi_curl_curls as $curl) {
$multi_curl_initHandle->invoke($multi_curl, $curl);
}
// Ensure requests are set to one of the random proxies.
$this->assertContains($get_1->getOpt(CURLOPT_PROXY), $proxies);
$this->assertContains($get_3->getOpt(CURLOPT_PROXY), $proxies);
// Ensure request with specific proxy is not set to one of the random proxies.
$this->assertNotContains($get_2->getOpt(CURLOPT_PROXY), $proxies);
}
}

0 comments on commit 98cad71

Please sign in to comment.
You can’t perform that action at this time.