doom3-gpl
Doom 3 GPL source release
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
fopen.c
Go to the documentation of this file.
1 /*****************************************************************************
2  *
3  * This example source code introduces a c library buffered I/O interface to
4  * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
5  * rewind(). Supported functions have identical prototypes to their normal c
6  * lib namesakes and are preceaded by url_ .
7  *
8  * Using this code you can replace your program's fopen() with url_fopen()
9  * and fread() with url_fread() and it become possible to read remote streams
10  * instead of (only) local files. Local files (ie those that can be directly
11  * fopened) will drop back to using the underlying clib implementations
12  *
13  * See the main() function at the bottom that shows an app that retrives from a
14  * specified url using fgets() and fread() and saves as two output files.
15  *
16  * Coyright (c)2003 Simtec Electronics
17  *
18  * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
19  * reference to original curl example code
20  *
21  * Redistribution and use in source and binary forms, with or without
22  * modification, are permitted provided that the following conditions
23  * are met:
24  * 1. Redistributions of source code must retain the above copyright
25  * notice, this list of conditions and the following disclaimer.
26  * 2. Redistributions in binary form must reproduce the above copyright
27  * notice, this list of conditions and the following disclaimer in the
28  * documentation and/or other materials provided with the distribution.
29  * 3. The name of the author may not be used to endorse or promote products
30  * derived from this software without specific prior written permission.
31  *
32  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
33  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
34  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
35  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
36  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
41  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42  *
43  * This example requires libcurl 7.9.7 or later.
44  */
45 
46 #include <stdio.h>
47 #include <string.h>
48 #include <sys/time.h>
49 #include <stdlib.h>
50 #include <errno.h>
51 
52 #include <curl/curl.h>
53 
55 
56 struct fcurl_data
57 {
58  enum fcurl_type_e type; /* type of handle */
59  union {
61  FILE *file;
62  } handle; /* handle */
63 
64  char *buffer; /* buffer to store cached data*/
65  int buffer_len; /* currently allocated buffers length */
66  int buffer_pos; /* end of data in buffer*/
67  int still_running; /* Is background url fetch still in progress */
68 };
69 
70 typedef struct fcurl_data URL_FILE;
71 
72 /* exported functions */
73 URL_FILE *url_fopen(char *url,const char *operation);
75 int url_feof(URL_FILE *file);
76 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
77 char * url_fgets(char *ptr, int size, URL_FILE *file);
78 void url_rewind(URL_FILE *file);
79 
80 /* we use a global one for convenience */
82 
83 /* curl calls this routine to get more data */
84 static size_t
85 write_callback(char *buffer,
86  size_t size,
87  size_t nitems,
88  void *userp)
89 {
90  char *newbuff;
91  int rembuff;
92 
93  URL_FILE *url = (URL_FILE *)userp;
94  size *= nitems;
95 
96  rembuff=url->buffer_len - url->buffer_pos;//remaining space in buffer
97 
98  if(size > rembuff)
99  {
100  //not enuf space in buffer
101  newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
102  if(newbuff==NULL)
103  {
104  fprintf(stderr,"callback buffer grow failed\n");
105  size=rembuff;
106  }
107  else
108  {
109  /* realloc suceeded increase buffer size*/
110  url->buffer_len+=size - rembuff;
111  url->buffer=newbuff;
112 
113  /*printf("Callback buffer grown to %d bytes\n",url->buffer_len);*/
114  }
115  }
116 
117  memcpy(&url->buffer[url->buffer_pos], buffer, size);
118  url->buffer_pos += size;
119 
120  /*fprintf(stderr, "callback %d size bytes\n", size);*/
121 
122  return size;
123 }
124 
125 /* use to attempt to fill the read buffer up to requested number of bytes */
126 static int
127 curl_fill_buffer(URL_FILE *file,int want,int waittime)
128 {
129  fd_set fdread;
130  fd_set fdwrite;
131  fd_set fdexcep;
132  int maxfd;
133  struct timeval timeout;
134  int rc;
135 
136  /* only attempt to fill buffer if transactions still running and buffer
137  * doesnt exceed required size already
138  */
139  if((!file->still_running) || (file->buffer_pos > want))
140  return 0;
141 
142  /* attempt to fill buffer */
143  do
144  {
145  FD_ZERO(&fdread);
146  FD_ZERO(&fdwrite);
147  FD_ZERO(&fdexcep);
148 
149  /* set a suitable timeout to fail on */
150  timeout.tv_sec = 60; /* 1 minute */
151  timeout.tv_usec = 0;
152 
153  /* get file descriptors from the transfers */
154  curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
155 
156  rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
157 
158  switch(rc) {
159  case -1:
160  /* select error */
161  break;
162 
163  case 0:
164  break;
165 
166  default:
167  /* timeout or readable/writable sockets */
168  /* note we *could* be more efficient and not wait for
169  * CURLM_CALL_MULTI_PERFORM to clear here and check it on re-entry
170  * but that gets messy */
173 
174  break;
175  }
176  } while(file->still_running && (file->buffer_pos < want));
177  return 1;
178 }
179 
180 /* use to remove want bytes from the front of a files buffer */
181 static int
182 curl_use_buffer(URL_FILE *file,int want)
183 {
184  /* sort out buffer */
185  if((file->buffer_pos - want) <=0)
186  {
187  /* ditch buffer - write will recreate */
188  if(file->buffer)
189  free(file->buffer);
190 
191  file->buffer=NULL;
192  file->buffer_pos=0;
193  file->buffer_len=0;
194  }
195  else
196  {
197  /* move rest down make it available for later */
198  memmove(file->buffer,
199  &file->buffer[want],
200  (file->buffer_pos - want));
201 
202  file->buffer_pos -= want;
203  }
204  return 0;
205 }
206 
207 
208 
209 URL_FILE *
210 url_fopen(char *url,const char *operation)
211 {
212  /* this code could check for URLs or types in the 'url' and
213  basicly use the real fopen() for standard files */
214 
215  URL_FILE *file;
216  (void)operation;
217 
218  file = (URL_FILE *)malloc(sizeof(URL_FILE));
219  if(!file)
220  return NULL;
221 
222  memset(file, 0, sizeof(URL_FILE));
223 
224  if((file->handle.file=fopen(url,operation)))
225  {
226  file->type = CFTYPE_FILE; /* marked as URL */
227  }
228  else
229  {
230  file->type = CFTYPE_CURL; /* marked as URL */
231  file->handle.curl = curl_easy_init();
232 
233  curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
235  curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, FALSE);
236  curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
237 
238  if(!multi_handle)
240 
242 
243  /* lets start the fetch */
246 
247  if((file->buffer_pos == 0) && (!file->still_running))
248  {
249  /* if still_running is 0 now, we should return NULL */
250 
251  /* make sure the easy handle is not in the multi handle anymore */
253 
254  /* cleanup */
256 
257  free(file);
258 
259  file = NULL;
260  }
261  }
262  return file;
263 }
264 
265 int
267 {
268  int ret=0;/* default is good return */
269 
270  switch(file->type)
271  {
272  case CFTYPE_FILE:
273  ret=fclose(file->handle.file); /* passthrough */
274  break;
275 
276  case CFTYPE_CURL:
277  /* make sure the easy handle is not in the multi handle anymore */
279 
280  /* cleanup */
282  break;
283 
284  default: /* unknown or supported type - oh dear */
285  ret=EOF;
286  errno=EBADF;
287  break;
288 
289  }
290 
291  if(file->buffer)
292  free(file->buffer);/* free any allocated buffer space */
293 
294  free(file);
295 
296  return ret;
297 }
298 
299 int
301 {
302  int ret=0;
303 
304  switch(file->type)
305  {
306  case CFTYPE_FILE:
307  ret=feof(file->handle.file);
308  break;
309 
310  case CFTYPE_CURL:
311  if((file->buffer_pos == 0) && (!file->still_running))
312  ret = 1;
313  break;
314  default: /* unknown or supported type - oh dear */
315  ret=-1;
316  errno=EBADF;
317  break;
318  }
319  return ret;
320 }
321 
322 size_t
323 url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
324 {
325  size_t want;
326 
327  switch(file->type)
328  {
329  case CFTYPE_FILE:
330  want=fread(ptr,size,nmemb,file->handle.file);
331  break;
332 
333  case CFTYPE_CURL:
334  want = nmemb * size;
335 
336  curl_fill_buffer(file,want,1);
337 
338  /* check if theres data in the buffer - if not curl_fill_buffer()
339  * either errored or EOF */
340  if(!file->buffer_pos)
341  return 0;
342 
343  /* ensure only available data is considered */
344  if(file->buffer_pos < want)
345  want = file->buffer_pos;
346 
347  /* xfer data to caller */
348  memcpy(ptr, file->buffer, want);
349 
350  curl_use_buffer(file,want);
351 
352  want = want / size; /* number of items - nb correct op - checked
353  * with glibc code*/
354 
355  /*printf("(fread) return %d bytes %d left\n", want,file->buffer_pos);*/
356  break;
357 
358  default: /* unknown or supported type - oh dear */
359  want=0;
360  errno=EBADF;
361  break;
362 
363  }
364  return want;
365 }
366 
367 char *
368 url_fgets(char *ptr, int size, URL_FILE *file)
369 {
370  int want = size - 1;/* always need to leave room for zero termination */
371  int loop;
372 
373  switch(file->type)
374  {
375  case CFTYPE_FILE:
376  ptr = fgets(ptr,size,file->handle.file);
377  break;
378 
379  case CFTYPE_CURL:
380  curl_fill_buffer(file,want,1);
381 
382  /* check if theres data in the buffer - if not fill either errored or
383  * EOF */
384  if(!file->buffer_pos)
385  return NULL;
386 
387  /* ensure only available data is considered */
388  if(file->buffer_pos < want)
389  want = file->buffer_pos;
390 
391  /*buffer contains data */
392  /* look for newline or eof */
393  for(loop=0;loop < want;loop++)
394  {
395  if(file->buffer[loop] == '\n')
396  {
397  want=loop+1;/* include newline */
398  break;
399  }
400  }
401 
402  /* xfer data to caller */
403  memcpy(ptr, file->buffer, want);
404  ptr[want]=0;/* allways null terminate */
405 
406  curl_use_buffer(file,want);
407 
408  /*printf("(fgets) return %d bytes %d left\n", want,file->buffer_pos);*/
409  break;
410 
411  default: /* unknown or supported type - oh dear */
412  ptr=NULL;
413  errno=EBADF;
414  break;
415  }
416 
417  return ptr;/*success */
418 }
419 
420 void
422 {
423  switch(file->type)
424  {
425  case CFTYPE_FILE:
426  rewind(file->handle.file); /* passthrough */
427  break;
428 
429  case CFTYPE_CURL:
430  /* halt transaction */
432 
433  /* restart */
435 
436  /* ditch buffer - write will recreate - resets stream pos*/
437  if(file->buffer)
438  free(file->buffer);
439 
440  file->buffer=NULL;
441  file->buffer_pos=0;
442  file->buffer_len=0;
443 
444  break;
445 
446  default: /* unknown or supported type - oh dear */
447  break;
448 
449  }
450 
451 }
452 
453 
454 /* Small main program to retrive from a url using fgets and fread saving the
455  * output to two test files (note the fgets method will corrupt binary files if
456  * they contain 0 chars */
457 int
458 main(int argc, char *argv[])
459 {
460  URL_FILE *handle;
461  FILE *outf;
462 
463  int nread;
464  char buffer[256];
465  char *url;
466 
467  if(argc < 2)
468  {
469  url="http://192.168.7.3/testfile";/* default to testurl */
470  }
471  else
472  {
473  url=argv[1];/* use passed url */
474  }
475 
476  /* copy from url line by line with fgets */
477  outf=fopen("fgets.test","w+");
478  if(!outf)
479  {
480  perror("couldnt open fgets output file\n");
481  return 1;
482  }
483 
484  handle = url_fopen(url, "r");
485  if(!handle)
486  {
487  printf("couldn't url_fopen()\n");
488  fclose(outf);
489  return 2;
490  }
491 
492  while(!url_feof(handle))
493  {
494  url_fgets(buffer,sizeof(buffer),handle);
495  fwrite(buffer,1,strlen(buffer),outf);
496  }
497 
498  url_fclose(handle);
499 
500  fclose(outf);
501 
502 
503  /* Copy from url with fread */
504  outf=fopen("fread.test","w+");
505  if(!outf)
506  {
507  perror("couldnt open fread output file\n");
508  return 1;
509  }
510 
511  handle = url_fopen("testfile", "r");
512  if(!handle) {
513  printf("couldn't url_fopen()\n");
514  fclose(outf);
515  return 2;
516  }
517 
518  do {
519  nread = url_fread(buffer, 1,sizeof(buffer), handle);
520  fwrite(buffer,1,nread,outf);
521  } while(nread);
522 
523  url_fclose(handle);
524 
525  fclose(outf);
526 
527 
528  /* Test rewind */
529  outf=fopen("rewind.test","w+");
530  if(!outf)
531  {
532  perror("couldnt open fread output file\n");
533  return 1;
534  }
535 
536  handle = url_fopen("testfile", "r");
537  if(!handle) {
538  printf("couldn't url_fopen()\n");
539  fclose(outf);
540  return 2;
541  }
542 
543  nread = url_fread(buffer, 1,sizeof(buffer), handle);
544  fwrite(buffer,1,nread,outf);
545  url_rewind(handle);
546 
547  buffer[0]='\n';
548  fwrite(buffer,1,1,outf);
549 
550  nread = url_fread(buffer, 1,sizeof(buffer), handle);
551  fwrite(buffer,1,nread,outf);
552 
553 
554  url_fclose(handle);
555 
556  fclose(outf);
557 
558 
559  return 0;/* all done */
560 }
void url_rewind(URL_FILE *file)
Definition: fopen.c:421
CURLMcode curl_multi_fdset(CURLM *multi_handle, fd_set *read_fd_set, fd_set *write_fd_set, fd_set *exc_fd_set, int *max_fd)
Definition: multi.c:231
int url_fclose(URL_FILE *file)
Definition: fopen.c:266
int buffer_pos
Definition: fopen.c:66
char * buffer
Definition: fopen.c:64
FILE * file
Definition: fopen.c:61
fcurl_type_e
Definition: fopen.c:54
CURLcode curl_easy_setopt(CURL *curl, CURLoption option,...)
Definition: easy.c:217
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
Definition: fopen.c:323
int buffer_len
Definition: fopen.c:65
CURLM * multi_handle
Definition: fopen.c:81
CURLMcode curl_multi_remove_handle(CURLM *multi_handle, CURL *curl_handle)
Definition: multi.c:182
union fcurl_data::@0 handle
#define NULL
Definition: Lib.h:88
#define select(args...)
Definition: amigaos.h:39
GLuint buffer
Definition: glext.h:3108
CURLM * curl_multi_init(void)
Definition: multi.c:114
int still_running
Definition: fopen.c:67
size_t fread(void *, size_t, size_t, FILE *)
enum fcurl_type_e type
Definition: fopen.c:58
CURLMcode curl_multi_perform(CURLM *multi_handle, int *running_handles)
Definition: multi.c:306
char * url_fgets(char *ptr, int size, URL_FILE *file)
Definition: fopen.c:368
void CURLM
Definition: multi.h:76
GLsizeiptr size
Definition: glext.h:3112
int url_feof(URL_FILE *file)
Definition: fopen.c:300
typedef void(APIENTRYP PFNGLBLENDCOLORPROC)(GLclampf red
#define FALSE
Definition: mprintf.c:70
void CURL
Definition: types.h:25
CURLMcode curl_multi_add_handle(CURLM *multi_handle, CURL *curl_handle)
Definition: multi.c:134
URL_FILE * url_fopen(char *url, const char *operation)
Definition: fopen.c:210
void curl_easy_cleanup(CURL *curl)
Definition: easy.c:288
#define CURLOPT_WRITEDATA
Definition: curl.h:805
CURL * curl
Definition: fopen.c:60
size_t fwrite(const void *, size_t, size_t, FILE *)
int main(int argc, char *argv[])
Definition: fopen.c:458
CURL * curl_easy_init(void)
Definition: easy.c:195