-
Notifications
You must be signed in to change notification settings - Fork 44
/
_batch.php
252 lines (231 loc) · 9.22 KB
/
_batch.php
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
<?php
/**
* Various batch-related helper functions.
*/
define('BATCH_SCRIPT', true);
function require_batch_key() {
global $argv;
if (isset($argv)) {
if ($argv[1] !== get_site_config("automated_key")) {
throw new Exception("Invalid key");
}
} else {
if (require_get("key") != get_site_config("automated_key")) {
throw new Exception("Invalid key");
}
}
}
function batch_header($page_name, $page_id) {
if (require_get("key", false)) {
// we're running from a web browser
require(__DIR__ . "/../layout/templates.php");
$options = array();
if (require_get("refresh", false)) {
$options["refresh"] = require_get("refresh");
}
page_header($page_name, $page_id, $options);
}
}
function batch_footer() {
if (require_get("key", false)) {
// we're running from a web browser
// include page gen times etc
page_footer();
} else {
// we are running from the CLI
// we still need to calculate performance metrics
performance_metrics_page_end();
}
}
class JobException extends Exception { }
function crypto_log($log) {
echo "\n<li>$log</li>";
// flush();
}
class ExternalAPIException extends Exception { } // expected exceptions
class EmptyResponseException extends ExternalAPIException { } // expected exception; allows us to handle e.g BitMinter
class CloudFlareException extends ExternalAPIException { } // expected exception; TODO implement some code to handle CloudFlare blocking
class IncapsulaException extends ExternalAPIException { } // expected exception; TODO implement some code to handle Incapsula blocking
class BlockchainException extends ExternalAPIException { } // expected exceptions
/**
* Extends {@link #curl_init()} to also set {@code CURLOPT_TIMEOUT}
* and {@code CURLOPT_CONNECTTIMEOUT} appropriately.
*/
function crypto_curl_init() {
$ch = curl_init();
curl_setopt($ch, CURLOPT_TIMEOUT, get_site_config('get_contents_timeout') /* in sec */);
curl_setopt($ch, CURLOPT_CONNECTTIMEOUT, get_site_config('get_contents_timeout') /* in sec */);
return $ch;
}
function crypto_wrap_url($url) {
// remove API keys etc
$url_clean = $url;
$url_clean = preg_replace('#key=([^&]{3})[^&]+#im', 'key=\\1...', $url_clean);
$url_clean = preg_replace('#hash=([^&]{3})[^&]+#im', 'hash=\\1...', $url_clean);
crypto_log("Requesting <a href=\"" . htmlspecialchars($url_clean) . "\">" . htmlspecialchars($url_clean) . "</a>...");
return $url;
}
/**
* Wraps {@link #file_get_contents()} with timeout information etc.
* May throw a {@link ExternalAPIException} if something unexpected occured.
*/
function crypto_get_contents($url, $options = array()) {
\Openclerk\Events::trigger('curl_start', $url);
// normally file_get_contents is OK, but if URLs are down etc, the timeout has no value and we can just stall here forever
// this also means we don't have to enable OpenSSL on windows for file_get_contents('https://...'), which is just a bit of a mess
$ch = crypto_curl_init();
curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
curl_setopt($ch, CURLOPT_USERAGENT, 'Mozilla/4.0 (compatible; Openclerk PHP client; '.php_uname('s').'; PHP/'.phpversion().')');
curl_setopt($ch, CURLOPT_URL, $url);
// curl_setopt($ch, CURLOPT_POSTFIELDS, $post_data);
// curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
curl_setopt($ch, CURLOPT_ENCODING, "gzip,deflate"); // enable gzip decompression if necessary
curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, FALSE);
foreach ($options as $key => $value) {
curl_setopt($ch, $key, $value);
}
// run the query
$res = curl_exec($ch);
\Openclerk\Events::trigger('curl_end', $url);
if ($res === false) throw new ExternalAPIException('Could not get reply: '.curl_error($ch));
crypto_check_response($res);
return $res;
}
/**
* @throws a {@link CloudFlareException} or {@link IncapsulaException} if the given
* remote response suggests something about CloudFlare or Incapsula.
* @throws an {@link ExternalAPIException} if the response suggests something else that was unexpected
*/
function crypto_check_response($string, $message = false) {
if (strpos($string, 'DDoS protection by CloudFlare') !== false) {
throw new CloudFlareException('Throttled by CloudFlare' . ($message ? " $message" : ""));
}
if (strpos($string, 'CloudFlare') !== false) {
if (strpos($string, 'The origin web server timed out responding to this request.') !== false) {
throw new CloudFlareException('CloudFlare reported: The origin web server timed out responding to this request.');
}
if (strpos($string, 'Web server is down') !== false) {
throw new CloudFlareException('CloudFlare reported: Web server is down.');
}
}
if (strpos($string, 'Incapsula incident') !== false) {
throw new IncapsulaException('Blocked by Incapsula' . ($message ? " $message" : ""));
}
if (strpos($string, '_Incapsula_Resource') !== false) {
throw new IncapsulaException('Throttled by Incapsula' . ($message ? " $message" : ""));
}
if (strpos(strtolower($string), '301 moved permanently') !== false) {
throw new ExternalAPIException("API location has been moved permanently" . ($message ? " $message" : ""));
}
if (strpos($string, "Access denied for user '") !== false) {
throw new ExternalAPIException("Remote database host returned 'Access denied'" . ($message ? " $message" : ""));
}
if (strpos(strtolower($string), "502 bad gateway") !== false) {
throw new ExternalAPIException("Bad gateway" . ($message ? " $message" : ""));
}
if (strpos(strtolower($string), "503 service unavailable") !== false) {
throw new ExternalAPIException("Service unavailable" . ($message ? " $message" : ""));
}
if (strpos(strtolower($string), "connection timed out") !== false) {
throw new ExternalAPIException("Connection timed out" . ($message ? " $message" : ""));
}
}
/**
* Try to decode a JSON string, or try and work out why it failed to decode but throw an exception
* if it was not a valid JSON string.
*
* @param empty_is_ok if true, then don't bail if the returned JSON is an empty array
*/
function crypto_json_decode($string, $message = false, $empty_array_is_ok = false) {
$json = json_decode($string, true);
if (!$json) {
if ($empty_array_is_ok && is_array($json)) {
// the result is an empty array
return $json;
}
crypto_log(htmlspecialchars($string));
crypto_check_response($string);
if (substr($string, 0, 1) == "<") {
throw new ExternalAPIException("Unexpectedly received HTML instead of JSON" . ($message ? " $message" : ""));
}
if (strpos(strtolower($string), "invalid key") !== false) {
throw new ExternalAPIException("Invalid key" . ($message ? " $message" : ""));
}
if (strpos(strtolower($string), "bad api key") !== false) {
throw new ExternalAPIException("Bad API key" . ($message ? " $message" : ""));
}
if (strpos(strtolower($string), "access denied") !== false) {
throw new ExternalAPIException("Access denied" . ($message ? " $message" : ""));
}
if (strpos(strtolower($string), "parameter error") !== false) {
// for 796 Exchange
throw new ExternalAPIException("Parameter error" . ($message ? " $message" : ""));
}
if (!$string) {
throw new EmptyResponseException('Response was empty' . ($message ? " $message" : ""));
}
throw new ExternalAPIException('Invalid data received' . ($message ? " $message" : ""));
}
return $json;
}
class JSendException extends ExternalAPIException { }
/**
* Checks the JSON to make sure it adheres to the JSend format http://labs.omniti.com/labs/jsend.
* Throws an JSendException if the JSON returned a 'fail', otherwise returns the wrapped data.
*
* @return $json['data'] if there were no problems
* @throws JSendException if there was a failure in the response
*/
function crypto_jsend($json) {
if (isset($json['status'])) {
if ($json['status'] == 'fail') {
if (isset($json['message']) && $json['message']) {
throw new JSendException("External API failed: " . $json['message']);
}
if (isset($json['data'])) {
throw new JSendException("External API failed: " . implode(", ", $json['data']));
}
throw new JSendException("External API failed with no message");
}
}
if (isset($json['data'])) {
return $json['data'];
}
throw new JSendException("Empty JSend response");
}
class WrappedJobException extends Exception {
public $job_id;
public $cause;
public function __construct($cause, $job_id) {
parent::__construct($cause->getMessage());
$this->cause = $cause;
$this->job_id = $job_id;
}
public function getCause() {
return $this->cause;
}
public function getJobId() {
return $this->job_id;
}
}
// provides our own logger implementation
use Monolog\Logger;
class CryptoLogHandler extends \Monolog\Handler\AbstractHandler {
function handle(array $record) {
$message = $record['message'];
if (is_valid_url($message)) {
return crypto_wrap_url($message);
}
if ($record['level'] >= Logger::WARNING) {
if ($record['level'] >= Logger::ERROR) {
$message = "[ERROR] " . $message;
} else {
$message = "[Warning] " . $message;
}
}
crypto_log($message);
}
}
global $logger;
$logger = new Logger("batch");
$logger->pushHandler(new CryptoLogHandler());