From 0857023319952130235926248101ced5a548d9e6 Mon Sep 17 00:00:00 2001 From: Zach Borboa Date: Sun, 12 May 2019 01:01:57 -0700 Subject: [PATCH] Fix #572,#582: Add MultiCurl::setProxies() to make multicurl requests with multiple proxies --- README.md | 1 + examples/multi_curl_proxies.php | 24 ++++++ src/Curl/ArrayUtil.php | 13 ++++ src/Curl/MultiCurl.php | 25 ++++++ tests/PHPCurlClass/PHPMultiCurlClassTest.php | 81 ++++++++++++++++++++ 5 files changed, 144 insertions(+) create mode 100644 examples/multi_curl_proxies.php diff --git a/README.md b/README.md index 83f70632e6..b1e6c43b5e 100644 --- a/README.md +++ b/README.md @@ -320,6 +320,7 @@ MultiCurl::setJsonDecoder($mixed) MultiCurl::setOpt($option, $value) MultiCurl::setOpts($options) MultiCurl::setPort($port) +MultiCurl::setProxies($proxies) MultiCurl::setProxy($proxy, $port = null, $username = null, $password = null) MultiCurl::setProxyAuth($auth) MultiCurl::setProxyTunnel($tunnel = true) diff --git a/examples/multi_curl_proxies.php b/examples/multi_curl_proxies.php new file mode 100644 index 0000000000..cb5bf824e7 --- /dev/null +++ b/examples/multi_curl_proxies.php @@ -0,0 +1,24 @@ +setProxies(array( + 'someproxy.com:9999', + 'someproxy.com:80', + 'someproxy.com:443', + 'someproxy.com:1080', + 'someproxy.com:3128', + 'someproxy.com:8080', +)); +$multi_curl->addGet('https://httpbin.org/ip'); +$multi_curl->addGet('https://httpbin.org/ip'); +$multi_curl->addGet('https://httpbin.org/ip'); +$multi_curl->complete(function ($instance) { + echo + 'curl id ' . $instance->id . ' used proxy ' . + $instance->getOpt(CURLOPT_PROXY) . ' and ' . + 'ip is ' . $instance->response->origin . "\n"; +}); +$multi_curl->start(); diff --git a/src/Curl/ArrayUtil.php b/src/Curl/ArrayUtil.php index 0cd5088af3..c93af5167c 100644 --- a/src/Curl/ArrayUtil.php +++ b/src/Curl/ArrayUtil.php @@ -77,4 +77,17 @@ public static function array_flatten_multidim($array, $prefix = false) } return $return; } + + /** + * Array Random + * + * @access public + * @param $array + * + * @return mixed + */ + public static function array_random($array) + { + return $array[mt_rand(0, count($array) - 1)]; + } } diff --git a/src/Curl/MultiCurl.php b/src/Curl/MultiCurl.php index cfa1b23a4f..bacc3b613c 100644 --- a/src/Curl/MultiCurl.php +++ b/src/Curl/MultiCurl.php @@ -2,6 +2,8 @@ namespace Curl; +use Curl\ArrayUtil; + class MultiCurl { public $baseUrl = null; @@ -23,6 +25,7 @@ class MultiCurl private $cookies = array(); private $headers = array(); private $options = array(); + private $proxies = null; private $jsonDecoder = null; private $xmlDecoder = null; @@ -571,6 +574,21 @@ public function setProxy($proxy, $port = null, $username = null, $password = nul } } + /** + * Set Proxies + * + * Set proxies to tunnel requests through. When set, a random proxy will be + * used for the request. + * + * @access public + * @param $proxies array - A list of HTTP proxies to tunnel requests + * through. May include port number. + */ + public function setProxies($proxies) + { + $this->proxies = $proxies; + } + /** * Set Proxy Auth * @@ -931,6 +949,13 @@ private function initHandle($curl) $curl->setRetry($this->retry); $curl->setCookies($this->cookies); + // Use a random proxy for the curl instance when proxies have been set + // and the curl instance doesn't already have a proxy set. + if (is_array($this->proxies) && $curl->getOpt(CURLOPT_PROXY) === null) { + $random_proxy = ArrayUtil::array_random($this->proxies); + $curl->setProxy($random_proxy); + } + $curlm_error_code = curl_multi_add_handle($this->multiCurl, $curl->curl); if (!($curlm_error_code === CURLM_OK)) { throw new \ErrorException('cURL multi add handle error: ' . curl_multi_strerror($curlm_error_code)); diff --git a/tests/PHPCurlClass/PHPMultiCurlClassTest.php b/tests/PHPCurlClass/PHPMultiCurlClassTest.php index 83b27a3394..4e22b4e425 100644 --- a/tests/PHPCurlClass/PHPMultiCurlClassTest.php +++ b/tests/PHPCurlClass/PHPMultiCurlClassTest.php @@ -2919,4 +2919,85 @@ public function testSetProxyTunnel() $multi_curl->setProxyTunnel($tunnel); $this->assertEquals($tunnel, $multi_curl->getOpt(CURLOPT_HTTPPROXYTUNNEL)); } + + public function testSetProxiesRandomProxy() + { + $proxies = array( + 'example.com:80', + 'example.com:443', + 'example.com:1080', + 'example.com:3128', + 'example.com:8080', + ); + + $multi_curl = new MultiCurl(); + $multi_curl->setProxies($proxies); + $multi_curl->addGet(Test::TEST_URL); + $multi_curl->addGet(Test::TEST_URL); + $multi_curl->addGet(Test::TEST_URL); + + // Make MultiCurl::curls accessible and MultiCurl::initHandle() + // callable. + $reflector = new \ReflectionClass('\Curl\MultiCurl'); + $property = $reflector->getProperty('curls'); + $property->setAccessible(true); + $multi_curl_curls = $property->getValue($multi_curl); + $multi_curl_initHandle = $reflector->getMethod('initHandle'); + $multi_curl_initHandle->setAccessible(true); + + // Ensure we have the requests queued. + $this->assertCount(3, $multi_curl_curls); + + // Invoke MultiCurl::initHandle() so that proxies are set. + foreach ($multi_curl_curls as $curl) { + $multi_curl_initHandle->invoke($multi_curl, $curl); + } + + // Ensure each request is set to one of the proxies. + foreach ($multi_curl_curls as $curl) { + $this->assertContains($curl->getOpt(CURLOPT_PROXY), $proxies); + } + } + + public function testSetProxiesAlreadySet() + { + $proxies = array( + 'example.com:80', + 'example.com:443', + 'example.com:1080', + 'example.com:3128', + 'example.com:8080', + ); + + $multi_curl = new MultiCurl(); + $multi_curl->setProxies($proxies); + $get_1 = $multi_curl->addGet(Test::TEST_URL); + $get_2 = $multi_curl->addGet(Test::TEST_URL); + $get_2->setProxy('example.com:9999'); + $get_3 = $multi_curl->addGet(Test::TEST_URL); + + // Make MultiCurl::curls accessible and MultiCurl::initHandle() + // callable. + $reflector = new \ReflectionClass('\Curl\MultiCurl'); + $property = $reflector->getProperty('curls'); + $property->setAccessible(true); + $multi_curl_curls = $property->getValue($multi_curl); + $multi_curl_initHandle = $reflector->getMethod('initHandle'); + $multi_curl_initHandle->setAccessible(true); + + // Ensure we have the requests queued. + $this->assertCount(3, $multi_curl_curls); + + // Invoke MultiCurl::initHandle() so that proxies are set. + foreach ($multi_curl_curls as $curl) { + $multi_curl_initHandle->invoke($multi_curl, $curl); + } + + // Ensure requests are set to one of the random proxies. + $this->assertContains($get_1->getOpt(CURLOPT_PROXY), $proxies); + $this->assertContains($get_3->getOpt(CURLOPT_PROXY), $proxies); + + // Ensure request with specific proxy is not set to one of the random proxies. + $this->assertNotContains($get_2->getOpt(CURLOPT_PROXY), $proxies); + } }