Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Newer
Older
100644 450 lines (388 sloc) 16.084 kb
df1fb37 @gholms Update GPL file headers
gholms authored
1 /*************************************************************************
2 * Copyright 2009-2012 Eucalyptus Systems, Inc.
3 *
4 * This program is free software: you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License as published by
6 * the Free Software Foundation; version 3 of the License.
7 *
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 * GNU General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program. If not, see http://www.gnu.org/licenses/.
15 *
16 * Please contact Eucalyptus Systems, Inc., 6755 Hollister Ave., Goleta
17 * CA 93117, USA or visit http://www.eucalyptus.com/licenses/ if you need
18 * additional information or have any questions.
47b5fb5 @gholms Tweak GPL headers
gholms authored
19 *
20 * This file may incorporate work covered under the following copyright
21 * and permission notice:
22 *
23 * Software License Agreement (BSD License)
24 *
25 * Copyright (c) 2008, Regents of the University of California
26 * All rights reserved.
27 *
28 * Redistribution and use of this software in source and binary forms,
29 * with or without modification, are permitted provided that the
30 * following conditions are met:
31 *
32 * Redistributions of source code must retain the above copyright
33 * notice, this list of conditions and the following disclaimer.
34 *
35 * Redistributions in binary form must reproduce the above copyright
36 * notice, this list of conditions and the following disclaimer
37 * in the documentation and/or other materials provided with the
38 * distribution.
39 *
40 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
41 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
42 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
43 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
44 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
45 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
46 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
47 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
48 * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
50 * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
51 * POSSIBILITY OF SUCH DAMAGE. USERS OF THIS SOFTWARE ACKNOWLEDGE
52 * THE POSSIBLE PRESENCE OF OTHER OPEN SOURCE LICENSED MATERIAL,
53 * COPYRIGHTED MATERIAL OR PATENTED MATERIAL IN THIS SOFTWARE,
54 * AND IF ANY SUCH MATERIAL IS DISCOVERED THE PARTY DISCOVERING
55 * IT MAY INFORM DR. RICH WOLSKI AT THE UNIVERSITY OF CALIFORNIA,
56 * SANTA BARBARA WHO WILL THEN ASCERTAIN THE MOST APPROPRIATE REMEDY,
57 * WHICH IN THE REGENTS' DISCRETION MAY INCLUDE, WITHOUT LIMITATION,
58 * REPLACEMENT OF THE CODE SO IDENTIFIED, LICENSING OF THE CODE SO
59 * IDENTIFIED, OR WITHDRAWAL OF THE CODE CAPABILITY TO THE EXTENT
60 * NEEDED TO COMPLY WITH ANY SUCH LICENSES OR RIGHTS.
df1fb37 @gholms Update GPL file headers
gholms authored
61 ************************************************************************/
4a3d9eb sync back-end
root authored
62
63 #include <stdlib.h>
64 #include <time.h>
65 #include <unistd.h> // close, stat
66 #include <assert.h>
67 #include <string.h>
68 #include <strings.h>
69 #include <fcntl.h> // open
70 #include <ctype.h> // tolower, isdigit
71 #include <sys/types.h> // stat
72 #include <sys/stat.h> // stat
73 #include <curl/curl.h>
74 #include <curl/easy.h>
75
76 #ifndef _UNIT_TEST // http_ functions aren't part of the unit test
77 #include "config.h"
78 #include "eucalyptus.h"
79 #include "misc.h"
2dacaa6 some logfile/compile warning cleanup
root authored
80 #include "http.h"
4a3d9eb sync back-end
root authored
81
82 #define TOTAL_RETRIES 3 /* download is retried in case of connection problems */
83 #define FIRST_TIMEOUT 4 /* in seconds, goes in powers of two afterwards */
84 #define STRSIZE 245 /* for short strings: files, hosts, URLs */
85
86 static size_t read_data (char *bufptr, size_t size, size_t nitems, void *userp);
87
6d9bbd8 network restoration logic in place
root authored
88 struct read_request {
4a3d9eb sync back-end
root authored
89 FILE * fp; /* input file pointer to be used by curl READERs */
90 long long total_read; /* bytes written during the operation */
91 long long total_calls; /* write calls made during the operation */
92 time_t timestamp; // timestamp for periodically printing progress messages
93 long long file_size; // file size in bytes, to print in progress messages
94 };
95
6d9bbd8 network restoration logic in place
root authored
96 struct write_request {
97 FILE * fp; /* output file pointer to be used by curl WRITERs */
98 long long total_wrote; /* bytes written during the operation */
99 long long total_calls; /* write calls made during the operation */
100 #if defined (CAN_GZIP)
101 z_stream strm; /* stream struct used by zlib */
102 int ret; /* return value of last inflate() call */
103 #endif
104 };
105
4a3d9eb sync back-end
root authored
106 static int curl_initialized = 0;
107
108 int http_put (const char * file_path, const char * url, const char * login, const char * password)
109 {
110 int code = ERROR;
111
112 if (curl_initialized!=1) {
113 curl_global_init(CURL_GLOBAL_SSL);
114 curl_initialized = 1;
115 }
116
117 struct stat64 mystat;
118 if (stat64 (file_path, &mystat)) {
119 logprintfl (EUCAERROR, "http_put(): failed to stat %s\n", file_path);
120 return code;
121 }
122 if (!S_ISREG(mystat.st_mode)) {
123 logprintfl (EUCAERROR, "http_put(): %s is not a regular file\n", file_path);
124 return code;
125 }
126
127 FILE * fp = fopen64 (file_path, "r");
128 if (fp==NULL) {
129 logprintfl (EUCAERROR, "http_put(): failed to open %s for reading\n", file_path);
130 return code;
131 }
132
133 CURL * curl;
134 CURLcode result;
135 curl = curl_easy_init ();
136 if (curl==NULL) {
137 logprintfl (EUCAERROR, "http_put(): could not initialize libcurl\n");
138 fclose (fp);
139 return code;
140 }
141
142 logprintfl (EUCAINFO, "http_put(): uploading %s\n", file_path);
143 logprintfl (EUCAINFO, " to %s\n", url);
144
145 char error_msg [CURL_ERROR_SIZE];
146 curl_easy_setopt (curl, CURLOPT_ERRORBUFFER, error_msg);
147 curl_easy_setopt (curl, CURLOPT_URL, url);
148 curl_easy_setopt (curl, CURLOPT_UPLOAD, 1L);
149 curl_easy_setopt (curl, CURLOPT_INFILESIZE_LARGE, (curl_off_t)mystat.st_size);
150 curl_easy_setopt (curl, CURLOPT_SSL_VERIFYPEER, 0L); // TODO: make this optional?
151 curl_easy_setopt (curl, CURLOPT_SSL_VERIFYHOST, 0L);
152 curl_easy_setopt (curl, CURLOPT_LOW_SPEED_LIMIT, 360L); // must have at least a 360 baud modem
153 curl_easy_setopt (curl, CURLOPT_LOW_SPEED_TIME, 10L); // abort if below speed limit for this many seconds
154
155 if (login!=NULL && password!=NULL) {
156 char userpwd [STRSIZE];
157 snprintf (userpwd, STRSIZE, "%s:%s", login, password);
158 curl_easy_setopt (curl, CURLOPT_USERPWD, userpwd);
159 }
160
6d9bbd8 network restoration logic in place
root authored
161 struct read_request params;
4a3d9eb sync back-end
root authored
162 params.fp = fp;
163 params.timestamp = time(NULL);
164 params.file_size = (long long)mystat.st_size;
165 curl_easy_setopt (curl, CURLOPT_READDATA, &params);
166 curl_easy_setopt (curl, CURLOPT_READFUNCTION, read_data);
167
168 int retries = TOTAL_RETRIES;
169 int timeout = FIRST_TIMEOUT;
170 do {
171 params.total_read = 0L;
172 params.total_calls = 0L;
173 result = curl_easy_perform (curl); /* do it */
174 logprintfl (EUCADEBUG, "http_put(): uploaded %ld bytes in %ld sends\n", params.total_read, params.total_calls);
175
176 if (result) { // curl error (connection or transfer failed)
177 logprintfl (EUCAERROR, "http_put(): %s (%d)\n", error_msg, result);
178
179 } else {
180 long httpcode;
181 curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &httpcode);
182 // TODO: pull out response message, too?
183
184 switch (httpcode) {
185 case 200L: // all good
186 logprintfl (EUCAINFO, "http_put(): file updated sucessfully\n");
187 code = OK;
188 break;
189 case 201L: // all good, created
190 logprintfl (EUCAINFO, "http_put(): file created sucessfully\n");
191 code = OK;
192 break;
193 case 408L: // timeout, retry
194 logprintfl (EUCAWARN, "http_put(): server responded with HTTP code %ld (timeout)\n", httpcode);
195 break;
196 case 500L: // internal server error (could be a fluke, so we'll retry)
197 logprintfl (EUCAWARN, "http_put(): server responded with HTTP code %ld (transient?)\n", httpcode);
198 break;
199 default: // some kind of error, will not retry
200 logprintfl (EUCAERROR, "http_put(): server responded with HTTP code %ld\n", httpcode);
201 retries = 0;
202 }
203 }
204
205 if (code!=OK && retries > 0) {
206 logprintfl (EUCAERROR, " upload retry %d of %d will commence in %d seconds\n", TOTAL_RETRIES-retries+1, TOTAL_RETRIES, timeout);
207 sleep (timeout);
208 fseek (fp, 0L, SEEK_SET);
209 timeout <<= 1;
210 }
211
212 retries--;
213 } while (code!=OK && retries>0);
214 fclose (fp);
215
216 curl_easy_cleanup (curl);
217 return code;
218 }
219
220 /* libcurl read handler */
221 static size_t read_data (char *buffer, size_t size, size_t nitems, void *params)
222 {
223 assert (params != NULL);
224
6d9bbd8 network restoration logic in place
root authored
225 FILE * fp = ((struct read_request *)params)->fp;
4a3d9eb sync back-end
root authored
226 int items_read = 0;
227 do {
228 items_read += fread (buffer, size, nitems-items_read, fp);
229 } while (items_read!=nitems && !feof(fp));
230
6d9bbd8 network restoration logic in place
root authored
231 ((struct read_request *)params)->total_read += items_read * size;
232 ((struct read_request *)params)->total_calls++;
4a3d9eb sync back-end
root authored
233
6d9bbd8 network restoration logic in place
root authored
234 if (((struct read_request *)params)->total_calls%50==0) {
235 time_t prev = ((struct read_request *)params)->timestamp;
4a3d9eb sync back-end
root authored
236 time_t now = time(NULL);
237 if ((now-prev)>10) {
6d9bbd8 network restoration logic in place
root authored
238 ((struct read_request *)params)->timestamp = now;
239 long long bytes_read = ((struct read_request *)params)->total_read;
240 long long bytes_file = ((struct read_request *)params)->file_size;
4a3d9eb sync back-end
root authored
241 int percent = (int)((bytes_read*100)/bytes_file);
242 logprintfl (EUCADEBUG, "http_put(): upload progress %ld/%ld bytes (%d%%)\n", bytes_read, bytes_file, percent);
243 }
244 }
245
246 return items_read;
247 }
248
6d9bbd8 network restoration logic in place
root authored
249 /* libcurl write handler */
250 static size_t write_data (void *buffer, size_t size, size_t nmemb, void *params)
251 {
252 assert (params !=NULL);
253 FILE * fp = ((struct write_request *)params)->fp;
254 int wrote = fwrite (buffer, size, nmemb, fp);
255 ((struct write_request *)params)->total_wrote += wrote;
256 ((struct write_request *)params)->total_calls++;
257
258 return wrote;
259 }
260
4a3d9eb sync back-end
root authored
261 #endif
262
263 // converts hex character to integer
264 static char hch_to_int (char ch) {
265 return isdigit (ch) ? (ch - '0') : (10 + tolower (ch) - 'a');
266 }
267
268 // converts integer to hex character
269 static char int_to_hch (char i) {
270 static char hex[] = "0123456789ABCDEF";
271 return hex [i & 15];
272 }
273
274 // converts a string to url-encoded string (which must be freed)
275 char * url_encode (const char * unencoded) {
276 char * encoded = malloc (strlen (unencoded) * 3 + 1);
277 if (encoded==NULL) return NULL;
278
279 const char * pu = unencoded;
280 char * pe = encoded;
281 while (*pu) {
282 if (isalnum (*pu)
283 || *pu == '-'
284 || *pu == '_'
285 || *pu == '.'
286 || *pu == '~')
287 *pe++ = *pu;
288 else if (*pu == ' ')
289 *pe++ = '+';
290 else {
291 *pe++ = '%';
292 *pe++ = int_to_hch (*pu >> 4);
293 *pe++ = int_to_hch (*pu & 15);
294 }
295 pu++;
296 }
297 *pe = '\0';
298
299 return encoded;
300 }
301
302 // converts a url-encoded string to regular (which must be freed)
303 char * url_decode (const char * encoded) {
304 char * unencoded = malloc (strlen (encoded) + 1);
305 if (unencoded==NULL) return NULL;
306
307 const char * pe = encoded;
308 char * pu = unencoded;
309 while (*pe) {
310 if (*pe == '%') {
311 if (pe[1] && pe[2]) {
312 *pu++ = hch_to_int (pe[1]) << 4 | hch_to_int (pe[2]);
313 pe += 2;
314 }
315 } else if (*pe == '+') {
316 *pu++ = ' ';
317 } else {
318 *pu++ = *pe;
319 }
320 pe++;
321 }
322 *pu = '\0';
323
324 return unencoded;
325 }
326
6d9bbd8 network restoration logic in place
root authored
327 int http_get (const char * url, const char * outfile)
328 {
8a6938c add ability to specify timeout to http_get (connection and overall), fix...
root authored
329 return(http_get_timeout(url, outfile, TOTAL_RETRIES, FIRST_TIMEOUT, 0, 0));
6d9bbd8 network restoration logic in place
root authored
330 }
331
8a6938c add ability to specify timeout to http_get (connection and overall), fix...
root authored
332 int http_get_timeout (const char * url, const char * outfile, int total_retries, int first_timeout, int connect_timeout, int total_timeout)
6d9bbd8 network restoration logic in place
root authored
333 {
334 int code = ERROR;
335
336 logprintfl (EUCAINFO, "http_get(): downloading %s\n", outfile);
337 logprintfl (EUCAINFO, " from %s\n", url);
338
339 /* isolate the PATH in the URL as it will be needed for signing */
340 if (strncasecmp (url, "http://", 7)!=0) {
341 logprintfl (EUCAERROR, "http_get(): URL must start with http://...\n");
342 return code;
343 }
344
345 FILE * fp = fopen64 (outfile, "w");
346 if (fp==NULL) {
347 logprintfl (EUCAERROR, "http_get(): failed to open %s for writing\n", outfile);
348 return code;
349 }
350
351 CURL * curl;
352 CURLcode result;
353 curl = curl_easy_init ();
354 if (curl==NULL) {
355 logprintfl (EUCAERROR, "http_get(): could not initialize libcurl\n");
356 fclose(fp);
357 return code;
358 }
359
360 char error_msg [CURL_ERROR_SIZE];
361 curl_easy_setopt (curl, CURLOPT_ERRORBUFFER, error_msg);
362 curl_easy_setopt (curl, CURLOPT_URL, url);
363 // curl_easy_setopt (curl, CURLOPT_HEADERFUNCTION, write_header);
364
365 curl_easy_setopt (curl, CURLOPT_HTTPGET, 1L);
366
367 /* set up the default write function, but possibly override it below, if compression is desired and possible */
368 struct write_request params;
369 params.fp = fp;
370 curl_easy_setopt (curl, CURLOPT_WRITEDATA, &params);
371 curl_easy_setopt (curl, CURLOPT_WRITEFUNCTION, write_data);
372
8a6938c add ability to specify timeout to http_get (connection and overall), fix...
root authored
373 if (connect_timeout > 0) {
374 curl_easy_setopt (curl, CURLOPT_CONNECTTIMEOUT, connect_timeout);
375 }
376 if (total_timeout > 0) {
377 curl_easy_setopt (curl, CURLOPT_TIMEOUT, total_timeout);
378 }
6d9bbd8 network restoration logic in place
root authored
379 // curl_easy_setopt (curl, CURLOPT_HTTPHEADER, headers); /* register headers */
380
381 logprintfl (EUCADEBUG, "http_get(): writing %s output to %s\n", "GET", outfile);
382
383 int retries = total_retries;
384 int timeout = first_timeout;
385 do {
386 params.total_wrote = 0L;
387 params.total_calls = 0L;
388
389 result = curl_easy_perform (curl); /* do it */
390 logprintfl (EUCADEBUG, "http_get(): wrote %ld bytes in %ld writes\n", params.total_wrote, params.total_calls);
391
392
393 if (result) { // curl error (connection or transfer failed)
394 logprintfl (EUCAERROR, "http_get(): %s (%d)\n", error_msg, result);
395
396 } else {
397 long httpcode;
398 curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &httpcode);
399 /* TODO: pull out response message, too */
400
401 switch (httpcode) {
402 case 200L: /* all good */
403 logprintfl (EUCAINFO, "http_get(): saved image in %s\n", outfile);
404 code = OK;
405 break;
406 case 408L: /* timeout, retry */
407 logprintfl (EUCAWARN, "http_get(): server responded with HTTP code %ld (timeout)\n", httpcode);
408 //logcat (EUCADEBUG, outfile); /* dump the error from outfile into the log */
409 break;
410 case 404L:
411 logprintfl (EUCAWARN, "http_get(): server responded with HTTP code %ld (file not found)\n", httpcode);
412 break;
413 default: /* some kind of error */
414 logprintfl (EUCAERROR, "http_get(): server responded with HTTP code %ld\n", httpcode);
415 //logcat (EUCADEBUG, outfile); /* dump the error from outfile into the log */
416 retries=0;
417 }
418 }
419
420 if (code!=OK && retries>0) {
421 logprintfl (EUCAERROR, " download retry %d of %d will commence in %d seconds\n", retries, total_retries, timeout);
422 sleep (timeout);
423 fseek (fp, 0L, SEEK_SET);
424 timeout <<= 1;
425 }
426
427 retries--;
428 } while (code!=OK && retries>0);
429 fclose (fp);
430
431 if ( code != OK ) {
432 logprintfl (EUCAINFO, "http_get(): due to error, removing %s\n", outfile);
433 remove (outfile);
434 }
435
436 // curl_slist_free_all (headers);
437 curl_easy_cleanup (curl);
438 return code;
439 }
440
4a3d9eb sync back-end
root authored
441 #ifdef _UNIT_TEST
442 int main (int argc, char ** argv)
443 {
444 #define _T(_S) { char * e = url_encode (_S); char * u = url_decode (e); printf ("orig: %s\nenco: %s\ndeco: %s\n\n", _S, e, u); free (e); free (u); }
445 _T("hello world");
446 _T("~`!1@2#3$4%5^6&7*8(9)0_-+={[}]|\\:;\"'<,>.?/");
447 _T("[datastore1 (1)] windows 2003 enterprise/windows 2003 enterprise.vmx");
448 }
449 #endif
Something went wrong with that request. Please try again.