xref: /illumos-kvm-cmd/block/curl.c (revision 68396ea9)
1 /*
2  * QEMU Block driver for CURL images
3  *
4  * Copyright (c) 2009 Alexander Graf <agraf@suse.de>
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a copy
7  * of this software and associated documentation files (the "Software"), to deal
8  * in the Software without restriction, including without limitation the rights
9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10  * copies of the Software, and to permit persons to whom the Software is
11  * furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice shall be included in
14  * all copies or substantial portions of the Software.
15  *
16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22  * THE SOFTWARE.
23  */
24 #include "qemu-common.h"
25 #include "block_int.h"
26 #include <curl/curl.h>
27 
28 // #define DEBUG
29 // #define DEBUG_VERBOSE
30 
31 #ifdef DEBUG_CURL
32 #define DPRINTF(fmt, ...) do { printf(fmt, ## __VA_ARGS__); } while (0)
33 #else
34 #define DPRINTF(fmt, ...) do { } while (0)
35 #endif
36 
37 #define CURL_NUM_STATES 8
38 #define CURL_NUM_ACB    8
39 #define SECTOR_SIZE     512
40 #define READ_AHEAD_SIZE (256 * 1024)
41 
42 #define FIND_RET_NONE   0
43 #define FIND_RET_OK     1
44 #define FIND_RET_WAIT   2
45 
46 struct BDRVCURLState;
47 
48 typedef struct CURLAIOCB {
49     BlockDriverAIOCB common;
50     QEMUIOVector *qiov;
51     size_t start;
52     size_t end;
53 } CURLAIOCB;
54 
55 typedef struct CURLState
56 {
57     struct BDRVCURLState *s;
58     CURLAIOCB *acb[CURL_NUM_ACB];
59     CURL *curl;
60     char *orig_buf;
61     size_t buf_start;
62     size_t buf_off;
63     size_t buf_len;
64     char range[128];
65     char errmsg[CURL_ERROR_SIZE];
66     char in_use;
67 } CURLState;
68 
69 typedef struct BDRVCURLState {
70     CURLM *multi;
71     size_t len;
72     CURLState states[CURL_NUM_STATES];
73     char *url;
74     size_t readahead_size;
75 } BDRVCURLState;
76 
77 static void curl_clean_state(CURLState *s);
78 static void curl_multi_do(void *arg);
79 
curl_sock_cb(CURL * curl,curl_socket_t fd,int action,void * s,void * sp)80 static int curl_sock_cb(CURL *curl, curl_socket_t fd, int action,
81                         void *s, void *sp)
82 {
83     DPRINTF("CURL (AIO): Sock action %d on fd %d\n", action, fd);
84     switch (action) {
85         case CURL_POLL_IN:
86             qemu_aio_set_fd_handler(fd, curl_multi_do, NULL, NULL, NULL, s);
87             break;
88         case CURL_POLL_OUT:
89             qemu_aio_set_fd_handler(fd, NULL, curl_multi_do, NULL, NULL, s);
90             break;
91         case CURL_POLL_INOUT:
92             qemu_aio_set_fd_handler(fd, curl_multi_do,
93                                     curl_multi_do, NULL, NULL, s);
94             break;
95         case CURL_POLL_REMOVE:
96             qemu_aio_set_fd_handler(fd, NULL, NULL, NULL, NULL, NULL);
97             break;
98     }
99 
100     return 0;
101 }
102 
curl_size_cb(void * ptr,size_t size,size_t nmemb,void * opaque)103 static size_t curl_size_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
104 {
105     CURLState *s = ((CURLState*)opaque);
106     size_t realsize = size * nmemb;
107     size_t fsize;
108 
109     if(sscanf(ptr, "Content-Length: %zd", &fsize) == 1) {
110         s->s->len = fsize;
111     }
112 
113     return realsize;
114 }
115 
curl_read_cb(void * ptr,size_t size,size_t nmemb,void * opaque)116 static size_t curl_read_cb(void *ptr, size_t size, size_t nmemb, void *opaque)
117 {
118     CURLState *s = ((CURLState*)opaque);
119     size_t realsize = size * nmemb;
120     int i;
121 
122     DPRINTF("CURL: Just reading %zd bytes\n", realsize);
123 
124     if (!s || !s->orig_buf)
125         goto read_end;
126 
127     memcpy(s->orig_buf + s->buf_off, ptr, realsize);
128     s->buf_off += realsize;
129 
130     for(i=0; i<CURL_NUM_ACB; i++) {
131         CURLAIOCB *acb = s->acb[i];
132 
133         if (!acb)
134             continue;
135 
136         if ((s->buf_off >= acb->end)) {
137             qemu_iovec_from_buffer(acb->qiov, s->orig_buf + acb->start,
138                                    acb->end - acb->start);
139             acb->common.cb(acb->common.opaque, 0);
140             qemu_aio_release(acb);
141             s->acb[i] = NULL;
142         }
143     }
144 
145 read_end:
146     return realsize;
147 }
148 
curl_find_buf(BDRVCURLState * s,size_t start,size_t len,CURLAIOCB * acb)149 static int curl_find_buf(BDRVCURLState *s, size_t start, size_t len,
150                          CURLAIOCB *acb)
151 {
152     int i;
153     size_t end = start + len;
154 
155     for (i=0; i<CURL_NUM_STATES; i++) {
156         CURLState *state = &s->states[i];
157         size_t buf_end = (state->buf_start + state->buf_off);
158         size_t buf_fend = (state->buf_start + state->buf_len);
159 
160         if (!state->orig_buf)
161             continue;
162         if (!state->buf_off)
163             continue;
164 
165         // Does the existing buffer cover our section?
166         if ((start >= state->buf_start) &&
167             (start <= buf_end) &&
168             (end >= state->buf_start) &&
169             (end <= buf_end))
170         {
171             char *buf = state->orig_buf + (start - state->buf_start);
172 
173             qemu_iovec_from_buffer(acb->qiov, buf, len);
174             acb->common.cb(acb->common.opaque, 0);
175 
176             return FIND_RET_OK;
177         }
178 
179         // Wait for unfinished chunks
180         if ((start >= state->buf_start) &&
181             (start <= buf_fend) &&
182             (end >= state->buf_start) &&
183             (end <= buf_fend))
184         {
185             int j;
186 
187             acb->start = start - state->buf_start;
188             acb->end = acb->start + len;
189 
190             for (j=0; j<CURL_NUM_ACB; j++) {
191                 if (!state->acb[j]) {
192                     state->acb[j] = acb;
193                     return FIND_RET_WAIT;
194                 }
195             }
196         }
197     }
198 
199     return FIND_RET_NONE;
200 }
201 
curl_multi_do(void * arg)202 static void curl_multi_do(void *arg)
203 {
204     BDRVCURLState *s = (BDRVCURLState *)arg;
205     int running;
206     int r;
207     int msgs_in_queue;
208 
209     if (!s->multi)
210         return;
211 
212     do {
213         r = curl_multi_socket_all(s->multi, &running);
214     } while(r == CURLM_CALL_MULTI_PERFORM);
215 
216     /* Try to find done transfers, so we can free the easy
217      * handle again. */
218     do {
219         CURLMsg *msg;
220         msg = curl_multi_info_read(s->multi, &msgs_in_queue);
221 
222         if (!msg)
223             break;
224         if (msg->msg == CURLMSG_NONE)
225             break;
226 
227         switch (msg->msg) {
228             case CURLMSG_DONE:
229             {
230                 CURLState *state = NULL;
231                 curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, (char**)&state);
232                 curl_clean_state(state);
233                 break;
234             }
235             default:
236                 msgs_in_queue = 0;
237                 break;
238         }
239     } while(msgs_in_queue);
240 }
241 
curl_init_state(BDRVCURLState * s)242 static CURLState *curl_init_state(BDRVCURLState *s)
243 {
244     CURLState *state = NULL;
245     int i, j;
246 
247     do {
248         for (i=0; i<CURL_NUM_STATES; i++) {
249             for (j=0; j<CURL_NUM_ACB; j++)
250                 if (s->states[i].acb[j])
251                     continue;
252             if (s->states[i].in_use)
253                 continue;
254 
255             state = &s->states[i];
256             state->in_use = 1;
257             break;
258         }
259         if (!state) {
260             usleep(100);
261             curl_multi_do(s);
262         }
263     } while(!state);
264 
265     if (state->curl)
266         goto has_curl;
267 
268     state->curl = curl_easy_init();
269     if (!state->curl)
270         return NULL;
271     curl_easy_setopt(state->curl, CURLOPT_URL, s->url);
272     curl_easy_setopt(state->curl, CURLOPT_TIMEOUT, 5);
273     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
274     curl_easy_setopt(state->curl, CURLOPT_WRITEDATA, (void *)state);
275     curl_easy_setopt(state->curl, CURLOPT_PRIVATE, (void *)state);
276     curl_easy_setopt(state->curl, CURLOPT_AUTOREFERER, 1);
277     curl_easy_setopt(state->curl, CURLOPT_FOLLOWLOCATION, 1);
278     curl_easy_setopt(state->curl, CURLOPT_NOSIGNAL, 1);
279     curl_easy_setopt(state->curl, CURLOPT_ERRORBUFFER, state->errmsg);
280 
281 #ifdef DEBUG_VERBOSE
282     curl_easy_setopt(state->curl, CURLOPT_VERBOSE, 1);
283 #endif
284 
285 has_curl:
286 
287     state->s = s;
288 
289     return state;
290 }
291 
curl_clean_state(CURLState * s)292 static void curl_clean_state(CURLState *s)
293 {
294     if (s->s->multi)
295         curl_multi_remove_handle(s->s->multi, s->curl);
296     s->in_use = 0;
297 }
298 
curl_open(BlockDriverState * bs,const char * filename,int flags)299 static int curl_open(BlockDriverState *bs, const char *filename, int flags)
300 {
301     BDRVCURLState *s = bs->opaque;
302     CURLState *state = NULL;
303     double d;
304 
305     #define RA_OPTSTR ":readahead="
306     char *file;
307     char *ra;
308     const char *ra_val;
309     int parse_state = 0;
310 
311     static int inited = 0;
312 
313     file = qemu_strdup(filename);
314     s->readahead_size = READ_AHEAD_SIZE;
315 
316     /* Parse a trailing ":readahead=#:" param, if present. */
317     ra = file + strlen(file) - 1;
318     while (ra >= file) {
319         if (parse_state == 0) {
320             if (*ra == ':')
321                 parse_state++;
322             else
323                 break;
324         } else if (parse_state == 1) {
325             if (*ra > '9' || *ra < '0') {
326                 char *opt_start = ra - strlen(RA_OPTSTR) + 1;
327                 if (opt_start > file &&
328                     strncmp(opt_start, RA_OPTSTR, strlen(RA_OPTSTR)) == 0) {
329                     ra_val = ra + 1;
330                     ra -= strlen(RA_OPTSTR) - 1;
331                     *ra = '\0';
332                     s->readahead_size = atoi(ra_val);
333                     break;
334                 } else {
335                     break;
336                 }
337             }
338         }
339         ra--;
340     }
341 
342     if ((s->readahead_size & 0x1ff) != 0) {
343         fprintf(stderr, "HTTP_READAHEAD_SIZE %zd is not a multiple of 512\n",
344                 s->readahead_size);
345         goto out_noclean;
346     }
347 
348     if (!inited) {
349         curl_global_init(CURL_GLOBAL_ALL);
350         inited = 1;
351     }
352 
353     DPRINTF("CURL: Opening %s\n", file);
354     s->url = file;
355     state = curl_init_state(s);
356     if (!state)
357         goto out_noclean;
358 
359     // Get file size
360 
361     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 1);
362     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_size_cb);
363     if (curl_easy_perform(state->curl))
364         goto out;
365     curl_easy_getinfo(state->curl, CURLINFO_CONTENT_LENGTH_DOWNLOAD, &d);
366     curl_easy_setopt(state->curl, CURLOPT_WRITEFUNCTION, (void *)curl_read_cb);
367     curl_easy_setopt(state->curl, CURLOPT_NOBODY, 0);
368     if (d)
369         s->len = (size_t)d;
370     else if(!s->len)
371         goto out;
372     DPRINTF("CURL: Size = %zd\n", s->len);
373 
374     curl_clean_state(state);
375     curl_easy_cleanup(state->curl);
376     state->curl = NULL;
377 
378     // Now we know the file exists and its size, so let's
379     // initialize the multi interface!
380 
381     s->multi = curl_multi_init();
382     curl_multi_setopt( s->multi, CURLMOPT_SOCKETDATA, s);
383     curl_multi_setopt( s->multi, CURLMOPT_SOCKETFUNCTION, curl_sock_cb );
384     curl_multi_do(s);
385 
386     return 0;
387 
388 out:
389     fprintf(stderr, "CURL: Error opening file: %s\n", state->errmsg);
390     curl_easy_cleanup(state->curl);
391     state->curl = NULL;
392 out_noclean:
393     qemu_free(file);
394     return -EINVAL;
395 }
396 
curl_aio_cancel(BlockDriverAIOCB * blockacb)397 static void curl_aio_cancel(BlockDriverAIOCB *blockacb)
398 {
399     // Do we have to implement canceling? Seems to work without...
400 }
401 
402 static AIOPool curl_aio_pool = {
403     .aiocb_size         = sizeof(CURLAIOCB),
404     .cancel             = curl_aio_cancel,
405 };
406 
curl_aio_readv(BlockDriverState * bs,int64_t sector_num,QEMUIOVector * qiov,int nb_sectors,BlockDriverCompletionFunc * cb,void * opaque)407 static BlockDriverAIOCB *curl_aio_readv(BlockDriverState *bs,
408         int64_t sector_num, QEMUIOVector *qiov, int nb_sectors,
409         BlockDriverCompletionFunc *cb, void *opaque)
410 {
411     BDRVCURLState *s = bs->opaque;
412     CURLAIOCB *acb;
413     size_t start = sector_num * SECTOR_SIZE;
414     size_t end;
415     CURLState *state;
416 
417     acb = qemu_aio_get(&curl_aio_pool, bs, cb, opaque);
418     if (!acb)
419         return NULL;
420 
421     acb->qiov = qiov;
422 
423     // In case we have the requested data already (e.g. read-ahead),
424     // we can just call the callback and be done.
425 
426     switch (curl_find_buf(s, start, nb_sectors * SECTOR_SIZE, acb)) {
427         case FIND_RET_OK:
428             qemu_aio_release(acb);
429             // fall through
430         case FIND_RET_WAIT:
431             return &acb->common;
432         default:
433             break;
434     }
435 
436     // No cache found, so let's start a new request
437 
438     state = curl_init_state(s);
439     if (!state)
440         return NULL;
441 
442     acb->start = 0;
443     acb->end = (nb_sectors * SECTOR_SIZE);
444 
445     state->buf_off = 0;
446     if (state->orig_buf)
447         qemu_free(state->orig_buf);
448     state->buf_start = start;
449     state->buf_len = acb->end + s->readahead_size;
450     end = MIN(start + state->buf_len, s->len) - 1;
451     state->orig_buf = qemu_malloc(state->buf_len);
452     state->acb[0] = acb;
453 
454     snprintf(state->range, 127, "%zd-%zd", start, end);
455     DPRINTF("CURL (AIO): Reading %d at %zd (%s)\n",
456             (nb_sectors * SECTOR_SIZE), start, state->range);
457     curl_easy_setopt(state->curl, CURLOPT_RANGE, state->range);
458 
459     curl_multi_add_handle(s->multi, state->curl);
460     curl_multi_do(s);
461 
462     return &acb->common;
463 }
464 
curl_close(BlockDriverState * bs)465 static void curl_close(BlockDriverState *bs)
466 {
467     BDRVCURLState *s = bs->opaque;
468     int i;
469 
470     DPRINTF("CURL: Close\n");
471     for (i=0; i<CURL_NUM_STATES; i++) {
472         if (s->states[i].in_use)
473             curl_clean_state(&s->states[i]);
474         if (s->states[i].curl) {
475             curl_easy_cleanup(s->states[i].curl);
476             s->states[i].curl = NULL;
477         }
478         if (s->states[i].orig_buf) {
479             qemu_free(s->states[i].orig_buf);
480             s->states[i].orig_buf = NULL;
481         }
482     }
483     if (s->multi)
484         curl_multi_cleanup(s->multi);
485     if (s->url)
486         free(s->url);
487 }
488 
curl_getlength(BlockDriverState * bs)489 static int64_t curl_getlength(BlockDriverState *bs)
490 {
491     BDRVCURLState *s = bs->opaque;
492     return s->len;
493 }
494 
495 static BlockDriver bdrv_http = {
496     .format_name     = "http",
497     .protocol_name   = "http",
498 
499     .instance_size   = sizeof(BDRVCURLState),
500     .bdrv_file_open  = curl_open,
501     .bdrv_close      = curl_close,
502     .bdrv_getlength  = curl_getlength,
503 
504     .bdrv_aio_readv  = curl_aio_readv,
505 };
506 
507 static BlockDriver bdrv_https = {
508     .format_name     = "https",
509     .protocol_name   = "https",
510 
511     .instance_size   = sizeof(BDRVCURLState),
512     .bdrv_file_open  = curl_open,
513     .bdrv_close      = curl_close,
514     .bdrv_getlength  = curl_getlength,
515 
516     .bdrv_aio_readv  = curl_aio_readv,
517 };
518 
519 static BlockDriver bdrv_ftp = {
520     .format_name     = "ftp",
521     .protocol_name   = "ftp",
522 
523     .instance_size   = sizeof(BDRVCURLState),
524     .bdrv_file_open  = curl_open,
525     .bdrv_close      = curl_close,
526     .bdrv_getlength  = curl_getlength,
527 
528     .bdrv_aio_readv  = curl_aio_readv,
529 };
530 
531 static BlockDriver bdrv_ftps = {
532     .format_name     = "ftps",
533     .protocol_name   = "ftps",
534 
535     .instance_size   = sizeof(BDRVCURLState),
536     .bdrv_file_open  = curl_open,
537     .bdrv_close      = curl_close,
538     .bdrv_getlength  = curl_getlength,
539 
540     .bdrv_aio_readv  = curl_aio_readv,
541 };
542 
543 static BlockDriver bdrv_tftp = {
544     .format_name     = "tftp",
545     .protocol_name   = "tftp",
546 
547     .instance_size   = sizeof(BDRVCURLState),
548     .bdrv_file_open  = curl_open,
549     .bdrv_close      = curl_close,
550     .bdrv_getlength  = curl_getlength,
551 
552     .bdrv_aio_readv  = curl_aio_readv,
553 };
554 
curl_block_init(void)555 static void curl_block_init(void)
556 {
557     bdrv_register(&bdrv_http);
558     bdrv_register(&bdrv_https);
559     bdrv_register(&bdrv_ftp);
560     bdrv_register(&bdrv_ftps);
561     bdrv_register(&bdrv_tftp);
562 }
563 
564 block_init(curl_block_init);
565