Commit 14ca732a authored by Daniel Stenberg's avatar Daniel Stenberg
Browse files

Multiple URL support added

parent 53c27c77

src/main.c

+391 −308
Original line number Diff line number Diff line
@@ -39,7 +39,6 @@ 
#include "writeout.h"



#define CURLseparator	"--_curl_--"

#define MIMEseparator	"_curl_"



/* This define make use of the "Curlseparator" as opposed to the

   MIMEseparator. We might add support for the latter one in the
@@ -222,6 +221,20 @@ static void helpf(char *fmt, ...) 
  fprintf(stderr, "curl: try 'curl --help' for more information\n");

}



/*

 * A chain of these nodes contain URL to get and where to put the URL's

 * contents.

 */

struct getout {

  struct getout *next;

  char *url;

  char *outfile;

  int flags;

};

#define GETOUT_OUTFILE (1<<0) /* set when outfile is deemed done */

#define GETOUT_URL     (1<<1) /* set when URL is deemed done */

#define GETOUT_USEREMOTE (1<<2) /* use remote file name locally */



static void help(void)

{

  printf(CURL_ID "%s\n"
@@ -303,9 +316,7 @@ struct Configurable { 
  char *referer;

  long timeout;

  long maxredirs;

  char *outfile;

  char *headerfile;

  char remotefile;

  char *ftpport;

  char *iface;

  unsigned short porttouse;
@@ -320,7 +331,13 @@ struct Configurable { 
  bool configread;

  bool proxytunnel;

  long conf;

  char *url;



  struct getout *url_list; /* point to the first node */

  struct getout *url_last; /* point to the last/current node */



  struct getout *url_get;  /* point to the node to fill in URL */

  struct getout *url_out;  /* point to the node to fill in outfile */



  char *cert;

  char *cacert;

  char *cert_passwd;
@@ -426,6 +443,29 @@ static char *file2memory(FILE *file, long *size) 
    return NULL; /* no string */

}



struct getout *new_getout(struct Configurable *config)

{

  struct getout *node =malloc(sizeof(struct getout));

  struct getout *last= config->url_last;

  if(node) {

    /* clear the struct */

    memset(node, 0, sizeof(struct getout));

        

    /* append this new node last in the list */

    if(last)

      last->next = node;

    else

      config->url_list = node; /* first node */

            

    /* move the last pointer */

    config->url_last = node;

  }

  return node;

}









typedef enum {

  PARAM_OK,

  PARAM_OPTION_AMBIGUOUS,
@@ -610,7 +650,30 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ 
      break;

    case '5':

      /* the URL! */

      GetStr(&config->url, nextarg);

      {

        struct getout *url;

        if(config->url_get || (config->url_get=config->url_list)) {

          /* there's a node here, if it already is filled-in continue to find

             an "empty" node */

          while(config->url_get && (config->url_get->flags&GETOUT_URL))

            config->url_get = config->url_get->next;

        }



        /* now there might or might not be an available node to fill in! */



        if(config->url_get)

          /* existing node */

          url = config->url_get;

        else

          /* there was no free node, create one! */

          url=new_getout(config);



        if(url) {

          /* fill in the URL */

          GetStr(&url->url, nextarg);

          url->flags |= GETOUT_URL;

        }

      }

      break;

    case '#': /* added 19990617 larsa */

      config->progressmode ^= CURL_PROGRESS_BAR;
@@ -794,12 +857,37 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ 
      config->nobuffer ^= 1;

      break;

    case 'o':

      /* output file */

      GetStr(&config->outfile, nextarg); /* write to this file */

      break;

    case 'O':

      /* output file */

      config->remotefile ^= TRUE;

      {

        struct getout *url;

        if(config->url_out || (config->url_out=config->url_list)) {

          /* there's a node here, if it already is filled-in continue to find

             an "empty" node */

          while(config->url_out && (config->url_out->flags&GETOUT_OUTFILE))

            config->url_out = config->url_out->next;

        }



        /* now there might or might not be an available node to fill in! */



        if(config->url_out)

          /* existing node */

          url = config->url_out;

        else

          /* there was no free node, create one! */

          url=new_getout(config);



        if(url) {

          /* fill in the outfile */

          if('o' == letter)

            GetStr(&url->outfile, nextarg);

          else {

            url->outfile=NULL; /* leave it */

            url->flags |= GETOUT_USEREMOTE;

          }

          url->flags |= GETOUT_OUTFILE;

        }

      }

      break;

    case 'P':

      /* This makes the FTP sessions use PORT instead of PASV */
@@ -1253,8 +1341,6 @@ void progressbarinit(struct ProgressData *bar) 


void free_config_fields(struct Configurable *config)

{

  if(config->url)

    free(config->url);

  if(config->userpwd)

    free(config->userpwd);

  if(config->postfields)
@@ -1271,8 +1357,6 @@ void free_config_fields(struct Configurable *config) 
    free(config->krb4level);

  if(config->headerfile)

    free(config->headerfile);

  if(config->outfile)

    free(config->outfile);

  if(config->ftpport)

    free(config->ftpport);

  if(config->infile)
@@ -1300,6 +1384,8 @@ operate(struct Configurable *config, int argc, char *argv[]) 
  char errorbuffer[CURL_ERROR_SIZE];

  char useragent[128]; /* buah, we don't want a larger default user agent */

  struct ProgressData progressbar;

  struct getout *urlnode;

  struct getout *nextnode;



  struct OutStruct outs;

  struct OutStruct heads;
@@ -1323,9 +1409,6 @@ operate(struct Configurable *config, int argc, char *argv[]) 
  int res;

  int i;



  outs.stream = stdout;

  outs.config = config;



#ifdef MALLOCDEBUG

  /* this sends all memory debug messages to a logfile named memdump */

  curl_memdebug("memdump");
@@ -1356,7 +1439,7 @@ operate(struct Configurable *config, int argc, char *argv[]) 
      return res;

  }



  if ((argc < 2)  && !config->url) {

  if ((argc < 2)  && !config->url_list) {

    helpf(NULL);

    return CURLE_FAILED_INIT;

  }
@@ -1405,20 +1488,15 @@ operate(struct Configurable *config, int argc, char *argv[]) 
      }

    }

    else {

      if(url) {

	helpf("only one URL is supported!\n");

	return CURLE_FAILED_INIT;

      }

      url = argv[i];

      bool used;

      /* just add the URL please */

      res = getparameter("--url", argv[i], &used, config);

      if(res)

        return res;

    }

  }



  /* if no URL was specified and there was one in the config file, get that

     one */

  if(!url && config->url)

    url = config->url;

  

  if(!url) {

  if(!config->url_list) {

    helpf("no URL specified!\n");

    return CURLE_FAILED_INIT;

  }
@@ -1430,9 +1508,18 @@ operate(struct Configurable *config, int argc, char *argv[]) 
  }

  else

    allocuseragent = TRUE;

#if 0

  fprintf(stderr, "URL: %s PROXY: %s\n", url, config->proxy?config->proxy:"none");

#endif



  urlnode = config->url_list;



  /* loop through the list of given URLs */

  while(urlnode) {



    /* get the full URL */

    url=urlnode->url;



    /* default output stream is stdout */

    outs.stream = stdout;

    outs.config = config;

    

    /* expand '{...}' and '[...]' expressions and return total number of URLs

       in pattern set */
@@ -1440,34 +1527,27 @@ operate(struct Configurable *config, int argc, char *argv[]) 
    if(res != CURLE_OK)

      return res;



  /* save outfile pattern befor expansion */

  outfiles = config->outfile?strdup(config->outfile):NULL;

    /* save outfile pattern before expansion */

    outfiles = urlnode->outfile?strdup(urlnode->outfile):NULL;



  if (!outfiles && !config->remotefile && urlnum > 1) {

#ifdef CURL_SEPARATORS

    if (outfiles && strequal(outfiles, "-") && urlnum > 1) {

      /* multiple files extracted to stdout, insert separators! */

      separator = 1;

#endif

#ifdef MIME_SEPARATORS

    /* multiple files extracted to stdout, insert MIME separators! */

    separator = 1;

    printf("MIME-Version: 1.0\n");

    printf("Content-Type: multipart/mixed; boundary=%s\n\n", MIMEseparator);

#endif

    }

    for (i = 0; (url = next_url(urls)); ++i) {

    if (config->outfile) {

      free(config->outfile);

      config->outfile = outfiles?strdup(outfiles):NULL;

    }

      char *outfile;

      outfile = outfiles?strdup(outfiles):NULL;



 

      if((urlnode->flags&GETOUT_USEREMOTE) ||

         (outfile && !strequal("-", outfile)) ) {



    if (config->outfile || config->remotefile) {

        /* 

         * We have specified a file name to store the result in, or we have

         * decided we want to use the remote file name.

         */

      

      if(!config->outfile && config->remotefile) {

        if(!outfile) {

          /* Find and get the remote file name */

          char * pc =strstr(url, "://");

          if(pc)
@@ -1475,25 +1555,26 @@ operate(struct Configurable *config, int argc, char *argv[]) 
          else

            pc=url;

          pc = strrchr(pc, '/');

        config->outfile = (char *) NULL == pc ? NULL : strdup(pc+1) ;

        if(!config->outfile || !strlen(config->outfile)) {

          outfile = (char *) NULL == pc ? NULL : strdup(pc+1) ;

          if(!outfile) {

            helpf("Remote file name has no length!\n");

            return CURLE_WRITE_ERROR;

          }

        }

        else {

          /* fill '#1' ... '#9' terms from URL pattern */

        char *outfile = config->outfile;

        config->outfile = match_url(config->outfile, urls);

        free(outfile);

          char *storefile = outfile;

          outfile = match_url(storefile, urls);

          free(storefile);

        }

      

        if((0 == config->resume_from) && config->use_resume) {

        /* we're told to continue where we are now, then we get the size of the

           file as it is now and open it for append instead */

          /* we're told to continue where we are now, then we get the size of

             the file as it is now and open it for append instead */



          struct stat fileinfo;



        if(0 == stat(config->outfile, &fileinfo)) {

          if(0 == stat(outfile, &fileinfo)) {

            /* set offset to current file size: */

            config->resume_from = fileinfo.st_size;

          }
@@ -1502,14 +1583,14 @@ operate(struct Configurable *config, int argc, char *argv[]) 
      

        if(config->resume_from) {

          /* open file for output: */

        outs.stream=(FILE *) fopen(config->outfile, config->resume_from?"ab":"wb");

          outs.stream=(FILE *) fopen(outfile, config->resume_from?"ab":"wb");

          if (!outs.stream) {

          helpf("Can't open '%s'!\n", config->outfile);

            helpf("Can't open '%s'!\n", outfile);

            return CURLE_WRITE_ERROR;

          }

        }

        else {

        outs.filename = config->outfile;

          outs.filename = outfile;

          outs.stream = NULL; /* open when needed */

        }

      }
@@ -1527,8 +1608,9 @@ operate(struct Configurable *config, int argc, char *argv[]) 
          ptr=url;

        ptr = strrchr(ptr, '/');

        if(!ptr || !strlen(++ptr)) {

        /* The URL has no file name part, add the local file name. In order to

           be able to do so, we have to create a new URL in another buffer.*/

          /* The URL has no file name part, add the local file name. In order

             to be able to do so, we have to create a new URL in another

             buffer.*/



          urlbuffer=(char *)malloc(strlen(url) + strlen(config->infile) + 3);

          if(!urlbuffer) {
@@ -1579,23 +1661,16 @@ operate(struct Configurable *config, int argc, char *argv[]) 


      if (urlnum > 1) {

        fprintf(stderr, "\n[%d/%d]: %s --> %s\n",

              i+1, urlnum, url, config->outfile ? config->outfile : "<stdout>");

      if (separator) {

#ifdef CURL_SEPARATORS

                i+1, urlnum, url, outfile ? outfile : "<stdout>");

        if (separator)

          printf("%s%s\n", CURLseparator, url);

#endif

#ifdef MIME_SEPARATORS

        printf("--%s\n", MIMEseparator);

        printf("Content-ID: %s\n\n", url); 

#endif

      }

      }



      if(!config->errors)

        config->errors = stderr;



#ifdef WIN32

    if(!config->outfile && !(config->conf & CONF_GETTEXT)) {

      if(!outfile && !(config->conf & CONF_GETTEXT)) {

        /* We get the output to stdout and we have not got the ASCII/text flag,

           then set stdout to be binary */

        setmode( 1, O_BINARY );
@@ -1605,7 +1680,6 @@ operate(struct Configurable *config, int argc, char *argv[]) 


      main_init();



    /* The new, v7-style easy-interface! */

      curl = curl_easy_init();

      if(curl) {

        curl_easy_setopt(curl, CURLOPT_FILE, (FILE *)&outs); /* where to store */
@@ -1676,7 +1750,6 @@ operate(struct Configurable *config, int argc, char *argv[]) 
        else 

          curl_easy_setopt(curl, CURLOPT_MAXREDIRS, DEFAULT_MAXREDIRS); 

 



        curl_easy_setopt(curl, CURLOPT_CRLF, config->crlf);

        curl_easy_setopt(curl, CURLOPT_QUOTE, config->quote);

        curl_easy_setopt(curl, CURLOPT_POSTQUOTE, config->postquote);
@@ -1730,7 +1803,7 @@ operate(struct Configurable *config, int argc, char *argv[]) 


      if(urlbuffer)

        free(urlbuffer);

    if (config->outfile && outs.stream)

      if (outfile && !strequal(outfile, "-") && outs.stream)

        fclose(outs.stream);

      if (config->infile)

        fclose(infd);
@@ -1740,21 +1813,31 @@ operate(struct Configurable *config, int argc, char *argv[]) 
      if(url)

        free(url);



      if(outfile)

        free(outfile);

    }

    if(outfiles)

      free(outfiles);



#ifdef MIME_SEPARATORS

  if (separator)

    printf("--%s--\n", MIMEseparator);

#endif

    /* cleanup memory used for URL globbing patterns */

    glob_cleanup(urls);



    /* empty this urlnode struct */

    if(urlnode->url)

      free(urlnode->url);

    if(urlnode->outfile)

      free(urlnode->outfile);

    

    /* move on to the next URL */

    nextnode=urlnode->next;

    free(urlnode); /* free the node */

    urlnode = nextnode;



  } /* while-loop through all URLs */



  if(allocuseragent)

    free(config->useragent);



  /* cleanup memory used for URL globbing patterns */

  glob_cleanup(urls);



  return res;

}

src/urlglob.c

+3 −3
Original line number Diff line number Diff line
@@ -213,6 +213,7 @@ int glob_url(URLGlob** glob, char* url, int *urlnum) 
  glob_expand->size = 0;

  glob_expand->urllen = strlen(url);

  glob_expand->glob_buffer = glob_buffer;

  glob_expand->beenhere=0;

  *urlnum = glob_word(glob_expand, url, 1);

  *glob = glob_expand;

  return CURLE_OK;
@@ -240,15 +241,14 @@ void glob_cleanup(URLGlob* glob) 


char *next_url(URLGlob *glob)

{

  static int beenhere = 0;

  char *buf = glob->glob_buffer;

  URLPattern *pat;

  char *lit;

  signed int i;

  int carry;



  if (!beenhere)

    beenhere = 1;

  if (!glob->beenhere)

    glob->beenhere = 1;

  else {

    carry = 1;

src/urlglob.h

+1 −0
Original line number Diff line number Diff line
@@ -50,6 +50,7 @@ typedef struct { 
  int size;

  int urllen;

  char *glob_buffer;

  char beenhere;

} URLGlob;



int glob_url(URLGlob**, char*, int *);