transfer.c

/***************************************************************************
 *                                  _   _ ____  _     
 *  Project                     ___| | | |  _ \| |    
 *                             / __| | | | |_) | |    
 *                            | (__| |_| |  _ <| |___ 
 *                             \___|\___/|_| \_\_____|
 *
 * Copyright (C) 1998 - 2003, Daniel Stenberg, <daniel@haxx.se>, et al.
 *
 * This software is licensed as described in the file COPYING, which
 * you should have received as part of this distribution. The terms
 * are also available at http://curl.haxx.se/docs/copyright.html.
 * 
 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
 * copies of the Software, and permit persons to whom the Software is
 * furnished to do so, under the terms of the COPYING file.
 *
 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
 * KIND, either express or implied.
 *
 * $Id$
 ***************************************************************************/

#include "setup.h"

/* -- WIN32 approved -- */
#include <stdio.h>
#include <string.h>
#include <stdarg.h>
#include <stdlib.h>
#include <ctype.h>
#include <sys/types.h>
#include <sys/stat.h>

#include <errno.h>

#include "strequal.h"

#if defined(WIN32) && !defined(__GNUC__) || defined(__MINGW32__)
#include <winsock.h>
#include <time.h>
#include <io.h>
#else
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#include <netinet/in.h>
#include <sys/time.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <netdb.h>
#ifdef HAVE_ARPA_INET_H
#include <arpa/inet.h>
#endif
#ifdef HAVE_NET_IF_H
#include <net/if.h>
#endif
#include <sys/ioctl.h>
#include <signal.h>

#ifdef HAVE_SYS_PARAM_H
#include <sys/param.h>
#endif

#ifdef HAVE_SYS_SELECT_H
#include <sys/select.h>
#endif

#ifndef HAVE_SELECT
#error "We can't compile without select() support!"
#endif
#ifndef HAVE_SOCKET
#error "We can't compile without socket() support!"
#endif

#endif

#include "urldata.h"
#include <curl/curl.h>
#include <curl/types.h>
#include "netrc.h"

#include "content_encoding.h"   /* content encoding support. 08/27/02 jhrg */

#include "hostip.h"
#include "transfer.h"
#include "sendf.h"
#include "speedcheck.h"
#include "getpass.h"
#include "progress.h"
#include "getdate.h"
#include "http.h"
#include "url.h"
#include "getinfo.h"
#include "ssluse.h"
#include "http_digest.h"
#include "http_ntlm.h"
#include "http_negotiate.h"
#include "share.h"

#define _MPRINTF_REPLACE /* use our functions only */
#include <curl/mprintf.h>

/* The last #include file should be: */
#ifdef CURLDEBUG
#include "memdebug.h"
#endif

#ifndef min
#define min(a, b)   ((a) < (b) ? (a) : (b))
#endif

#define CURL_TIMEOUT_EXPECT_100 1000 /* counting ms here */

enum {
  KEEP_NONE,
  KEEP_READ,
  KEEP_WRITE
};

/* We keep this static and global since this is read-only and NEVER
   changed. It should just remain a blanked-out timeout value. */
static struct timeval notimeout={0,0};

/*
 * This function will call the read callback to fill our buffer with data
 * to upload.
 */
static int fillbuffer(struct connectdata *conn,
                      int bytes)
{
  int buffersize = bytes;
  int nread;

  if(conn->bits.upload_chunky) {
    /* if chunked Transfer-Encoding */
    buffersize -= (8 + 2 + 2);   /* 32bit hex + CRLF + CRLF */
    conn->upload_fromhere += 10; /* 32bit hex + CRLF */
  }
  
  nread = conn->fread(conn->upload_fromhere, 1,
                      buffersize, conn->fread_in);
          
  if(!conn->bits.forbidchunk && conn->bits.upload_chunky) {
    /* if chunked Transfer-Encoding */
    char hexbuffer[11];
    int hexlen = snprintf(hexbuffer, sizeof(hexbuffer),
                          "%x\r\n", nread);
    /* move buffer pointer */
    conn->upload_fromhere -= hexlen;
    nread += hexlen;

    /* copy the prefix to the buffer */
    memcpy(conn->upload_fromhere, hexbuffer, hexlen);
    if(nread>hexlen) {
      /* append CRLF to the data */
      memcpy(conn->upload_fromhere +
             nread, "\r\n", 2);
      nread+=2;
    }
    else {
      /* mark this as done once this chunk is transfered */
      conn->keep.upload_done = TRUE;
    }
  }
  return nread;
}

/*
 * checkhttpprefix()
 *
 * Returns TRUE if member of the list matches prefix of string
 */
static bool
checkhttpprefix(struct SessionHandle *data,
                const char *s)
{
  struct curl_slist *head = data->set.http200aliases;

  while (head) {
    if (checkprefix(head->data, s))
      return TRUE;
    head = head->next;
  }

  if(checkprefix("HTTP/", s))
    return TRUE;

  return FALSE;
}

CURLcode Curl_readwrite(struct connectdata *conn,
                        bool *done)
{
  struct Curl_transfer_keeper *k = &conn->keep;
  struct SessionHandle *data = conn->data;
  int result;
  ssize_t nread; /* number of bytes read */
  int didwhat=0;
  
  /* These two are used only if no other select() or _fdset() have been
     invoked before this. This typicly happens if you use the multi interface
     and call curl_multi_perform() without calling curl_multi_fdset()
     first. */
  fd_set extrareadfd;
  fd_set extrawritefd;

  fd_set *readfdp = k->readfdp;
  fd_set *writefdp = k->writefdp;
  
  if((k->keepon & KEEP_READ) && !readfdp) {
    /* reading is requested, but no socket descriptor pointer was set */
    FD_ZERO(&extrareadfd);
    FD_SET(conn->sockfd, &extrareadfd);
    readfdp = &extrareadfd;

    /* no write, no exceptions, no timeout */
    select(conn->sockfd+1, readfdp, NULL, NULL, &notimeout);
  }
  if((k->keepon & KEEP_WRITE) && !writefdp) {
    /* writing is requested, but no socket descriptor pointer was set */
    FD_ZERO(&extrawritefd);
    FD_SET(conn->writesockfd, &extrawritefd);
    writefdp = &extrawritefd;

    /* no read, no exceptions, no timeout */
    select(conn->writesockfd+1, NULL, writefdp, NULL, &notimeout);
  }

  do {
    /* If we still have reading to do, we check if we have a readable
       socket. Sometimes the reafdp is NULL, if no fd_set was done using
       the multi interface and then we can do nothing but to attempt a
       read to be sure. */
    if((k->keepon & KEEP_READ) &&
       (!readfdp || FD_ISSET(conn->sockfd, readfdp))) {

      bool readdone = TRUE;

      /* This is where we loop until we have read everything there is to
         read or we get a EWOULDBLOCK */
      do {
        int buffersize = data->set.buffer_size?
          data->set.buffer_size:BUFSIZE -1;

        /* receive data from the network! */
        result = Curl_read(conn, conn->sockfd, k->buf, buffersize, &nread);

        if(0>result)
          break; /* get out of loop */
        if(result>0)
          return result;

        if ((k->bytecount == 0) && (k->writebytecount == 0)) {
          Curl_pgrsTime(data, TIMER_STARTTRANSFER);
          if(k->wait100_after_headers)
            /* set time stamp to compare with when waiting for the 100 */
            k->start100 = Curl_tvnow();
        }

        didwhat |= KEEP_READ;

        /* NULL terminate, allowing string ops to be used */
        if (0 < nread)
          k->buf[nread] = 0;

        /* if we receive 0 or less here, the server closed the connection and
           we bail out from this! */
        else if (0 >= nread) {
          k->keepon &= ~KEEP_READ;
          FD_ZERO(&k->rkeepfd);
          readdone = TRUE;
          break;
        }

        /* Default buffer to use when we write the buffer, it may be changed
           in the flow below before the actual storing is done. */
        k->str = k->buf;

        /* Since this is a two-state thing, we check if we are parsing
           headers at the moment or not. */          
        if (k->header) {
          /* we are in parse-the-header-mode */
          bool stop_reading = FALSE;

          /* header line within buffer loop */
          do {
            int hbufp_index;
            int rest_length;
            int full_length;
            int writetype;
              
            /* str_start is start of line within buf */
            k->str_start = k->str;
              
            k->end_ptr = strchr (k->str_start, '\n');
              
            if (!k->end_ptr) {
              /* Not a complete header line within buffer, append the data to
                 the end of the headerbuff. */

              if (k->hbuflen + nread >= data->state.headersize) {
                /* We enlarge the header buffer as it is too small */
                char *newbuff;
                long newsize=MAX((k->hbuflen+nread)*3/2,
                                 data->state.headersize*2);
                hbufp_index = k->hbufp - data->state.headerbuff;
                newbuff = (char *)realloc(data->state.headerbuff, newsize);
                if(!newbuff) {
                  failf (data, "Failed to alloc memory for big header!");
                  return CURLE_OUT_OF_MEMORY;
                }
                data->state.headersize=newsize;
                data->state.headerbuff = newbuff;
                k->hbufp = data->state.headerbuff + hbufp_index;
              }
              memcpy(k->hbufp, k->str, nread);
              k->hbufp += nread;
              k->hbuflen += nread;
              if (!k->headerline && (k->hbuflen>5)) {
                /* make a first check that this looks like a HTTP header */
                if(!checkhttpprefix(data, data->state.headerbuff)) {
                  /* this is not the beginning of a HTTP first header line */
                  k->header = FALSE;
                  k->badheader = HEADER_ALLBAD;
                  break;
                }
              }

              break; /* read more and try again */
            }

            /* decrease the size of the remaining (supposed) header line */
            rest_length = (k->end_ptr - k->str)+1;
            nread -= rest_length; 

            k->str = k->end_ptr + 1; /* move past new line */

            full_length = k->str - k->str_start;

            /*
             * We're about to copy a chunk of data to the end of the
             * already received header. We make sure that the full string
             * fit in the allocated header buffer, or else we enlarge 
             * it.
             */
            if (k->hbuflen + full_length >=
                data->state.headersize) {
              char *newbuff;
              long newsize=MAX((k->hbuflen+full_length)*3/2,
                               data->state.headersize*2);
              hbufp_index = k->hbufp - data->state.headerbuff;
              newbuff = (char *)realloc(data->state.headerbuff, newsize);
              if(!newbuff) {
                failf (data, "Failed to alloc memory for big header!");
                return CURLE_OUT_OF_MEMORY;
              }
              data->state.headersize= newsize;
              data->state.headerbuff = newbuff;
              k->hbufp = data->state.headerbuff + hbufp_index;
            }

            /* copy to end of line */
            strncpy (k->hbufp, k->str_start, full_length);
            k->hbufp += full_length;
            k->hbuflen += full_length;
            *k->hbufp = 0;
            k->end_ptr = k->hbufp;
              
            k->p = data->state.headerbuff;
              
            /****
             * We now have a FULL header line that p points to
             *****/

            if(!k->headerline) {
              /* the first read header */
              if((k->hbuflen>5) &&
                 !checkhttpprefix(data, data->state.headerbuff)) {
                /* this is not the beginning of a HTTP first header line */
                k->header = FALSE;
                if(nread)
                  /* since there's more, this is a partial bad header */
                  k->badheader = HEADER_PARTHEADER;
                else {
                  /* this was all we read so its all a bad header */
                  k->badheader = HEADER_ALLBAD;
                  nread = rest_length;
                }
                break;
              }
            }

            if (('\n' == *k->p) || ('\r' == *k->p)) {
              int headerlen;
              /* Zero-length header line means end of headers! */

              if ('\r' == *k->p)
                k->p++; /* pass the \r byte */
              if ('\n' == *k->p)
                k->p++; /* pass the \n byte */

              if(100 == k->httpcode) {
                /*
                 * We have made a HTTP PUT or POST and this is 1.1-lingo
                 * that tells us that the server is OK with this and ready
                 * to receive the data.
                 * However, we'll get more headers now so we must get
                 * back into the header-parsing state!
                 */
                k->header = TRUE;
                k->headerline = 0; /* restart the header line counter */
                /* if we did wait for this do enable write now! */
                if (k->write_after_100_header) {

                  k->write_after_100_header = FALSE;
                  FD_SET (conn->writesockfd, &k->writefd); /* write */
                  k->keepon |= KEEP_WRITE;
                  k->wkeepfd = k->writefd;
                }
              }
              else
                k->header = FALSE; /* no more header to parse! */

              if (417 == k->httpcode) {
                /*
                 * we got: "417 Expectation Failed" this means:
                 * we have made a HTTP call and our Expect Header
                 * seems to cause a problem => abort the write operations
                 * (or prevent them from starting).
                 */
                k->write_after_100_header = FALSE;
                k->keepon &= ~KEEP_WRITE;
                FD_ZERO(&k->wkeepfd);
              }

              /* now, only output this if the header AND body are requested:
               */
              writetype = CLIENTWRITE_HEADER;
              if (data->set.http_include_header)
                writetype |= CLIENTWRITE_BODY;

              headerlen = k->p - data->state.headerbuff;

              result = Curl_client_write(data, writetype,
                                         data->state.headerbuff,
                                         headerlen);
              if(result)
                return result;

              data->info.header_size += headerlen;
              conn->headerbytecount += headerlen;

              if(!k->header) {
                /*
                 * really end-of-headers.
                 *
                 * If we requested a "no body", this is a good time to get
                 * out and return home.
                 */
                if(data->set.no_body)
                  stop_reading = TRUE;
                else {
                  /* If we know the expected size of this document, we set the
                     maximum download size to the size of the expected
                     document or else, we won't know when to stop reading!

                     Note that we set the download maximum even if we read a
                     "Connection: close" header, to make sure that
                     "Content-Length: 0" still prevents us from attempting to
                     read the (missing) response-body.
                  */
                  if(-1 != conn->size)
                    conn->maxdownload = conn->size;
                }
                /* If max download size is *zero* (nothing) we already
                   have nothing and can safely return ok now! */
                if(0 == conn->maxdownload)
                  stop_reading = TRUE;
                    
                if(stop_reading) {
                  /* we make sure that this socket isn't read more now */
                  k->keepon &= ~KEEP_READ;
                  FD_ZERO(&k->rkeepfd);
                }

                break;          /* exit header line loop */
              }

              /* We continue reading headers, so reset the line-based
                 header parsing variables hbufp && hbuflen */
              k->hbufp = data->state.headerbuff;
              k->hbuflen = 0;
              continue;
            }

            /*
             * Checks for special headers coming up.
             */
              
            if (!k->headerline++) {
              /* This is the first header, it MUST be the error code line
                 or else we consiser this to be the body right away! */
              int httpversion_major;
              int nc=sscanf (k->p, " HTTP/%d.%d %3d",
                             &httpversion_major,
                             &k->httpversion,
                             &k->httpcode);
              if (nc==3) {
                k->httpversion += 10 * httpversion_major;
              }
              else {
                /* this is the real world, not a Nirvana
                   NCSA 1.5.x returns this crap when asked for HTTP/1.1
                */
                nc=sscanf (k->p, " HTTP %3d", &k->httpcode);
                k->httpversion = 10;
               
               /* If user has set option HTTP200ALIASES,
                  compare header line against list of aliases
               */
                if (!nc) {
                  if (checkhttpprefix(data, k->p)) {
                    nc = 1;
                    k->httpcode = 200;
                    k->httpversion =
                      (data->set.httpversion==CURL_HTTP_VERSION_1_0)? 10 : 11;
                  }
                }
              }

              if (nc) {
                data->info.httpcode = k->httpcode;
                data->info.httpversion = k->httpversion;

                /* 404 -> URL not found! */
                if (data->set.http_fail_on_error &&
                    (k->httpcode >= 400)) {
                  /* If we have been told to fail hard on HTTP-errors,
                     here is the check for that: */
                  /* serious error, go home! */
                  failf (data, "The requested URL returned error: %d",
                         k->httpcode);
                  return CURLE_HTTP_RETURNED_ERROR;
                }

                if(k->httpversion == 10)
                  /* Default action for HTTP/1.0 must be to close, unless
                     we get one of those fancy headers that tell us the
                     server keeps it open for us! */
                  conn->bits.close = TRUE;

                switch(k->httpcode) {
                case 204:
                  /* (quote from RFC2616, section 10.2.5): The server has
                   * fulfilled the request but does not need to return an
                   * entity-body ... The 204 response MUST NOT include a
                   * message-body, and thus is always terminated by the first
                   * empty line after the header fields. */
                  /* FALLTHROUGH */
                case 304:
                  /* (quote from RFC2616, section 10.3.5): The 304 response MUST
                   * NOT contain a message-body, and thus is always terminated
                   * by the first empty line after the header fields.  */
                  conn->size=0;
                  conn->maxdownload=0;
                  break;
                default:
                  /* nothing */
                  break;
                }
              }
              else {
                k->header = FALSE;   /* this is not a header line */
                break;
              }
            }

            /* check for Content-Length: header lines to get size */
            if (checkprefix("Content-Length:", k->p) &&
                sscanf (k->p+15, " %ld", &k->contentlength)) {
              conn->size = k->contentlength;
              Curl_pgrsSetDownloadSize(data, k->contentlength);
            }
            /* check for Content-Type: header lines to get the mime-type */
            else if (checkprefix("Content-Type:", k->p)) {
              char *start;
              char *end;
              int len;
              
              /* Find the first non-space letter */
              for(start=k->p+13;
                  *start && isspace((int)*start);
                  start++);

              end = strchr(start, '\r');
              if(!end)
                end = strchr(start, '\n');

              if(end) {
                /* skip all trailing space letters */
                for(; isspace((int)*end) && (end > start); end--);

                /* get length of the type */
                len = end-start+1;
              
                /* allocate memory of a cloned copy */
                Curl_safefree(data->info.contenttype);
              
                data->info.contenttype = malloc(len + 1);
                if (NULL == data->info.contenttype)
                  return CURLE_OUT_OF_MEMORY;

                /* copy the content-type string */
                memcpy(data->info.contenttype, start, len);
                data->info.contenttype[len] = 0; /* zero terminate */
              }
            }
            else if((k->httpversion == 10) &&
                    conn->bits.httpproxy &&
                    Curl_compareheader(k->p,
                                       "Proxy-Connection:", "keep-alive")) {
              /*
               * When a HTTP/1.0 reply comes when using a proxy, the
               * 'Proxy-Connection: keep-alive' line tells us the
               * connection will be kept alive for our pleasure.
               * Default action for 1.0 is to close.
               */
              conn->bits.close = FALSE; /* don't close when done */
              infof(data, "HTTP/1.0 proxy connection set to keep alive!\n");
            }
            else if((k->httpversion == 10) &&
                    Curl_compareheader(k->p, "Connection:", "keep-alive")) {
              /*
               * A HTTP/1.0 reply with the 'Connection: keep-alive' line
               * tells us the connection will be kept alive for our
               * pleasure.  Default action for 1.0 is to close.
               *
               * [RFC2068, section 19.7.1] */
              conn->bits.close = FALSE; /* don't close when done */
              infof(data, "HTTP/1.0 connection set to keep alive!\n");
            }
            else if (Curl_compareheader(k->p, "Connection:", "close")) {
              /*
               * [RFC 2616, section 8.1.2.1]
               * "Connection: close" is HTTP/1.1 language and means that
               * the connection will close when this request has been
               * served.
               */
              conn->bits.close = TRUE; /* close when done */
            }
            else if (Curl_compareheader(k->p,
                                        "Transfer-Encoding:", "chunked")) {
              /*
               * [RFC 2616, section 3.6.1] A 'chunked' transfer encoding
               * means that the server will send a series of "chunks". Each
               * chunk starts with line with info (including size of the
               * coming block) (terminated with CRLF), then a block of data
               * with the previously mentioned size. There can be any amount
               * of chunks, and a chunk-data set to zero signals the
               * end-of-chunks. */
              conn->bits.chunk = TRUE; /* chunks coming our way */

              /* init our chunky engine */
              Curl_httpchunk_init(conn);
            }
            else if (checkprefix("Content-Encoding:", k->p) &&
                     data->set.encoding) {
              /*
               * Process Content-Encoding. Look for the values: identity,
               * gzip, deflate, compress, x-gzip and x-compress. x-gzip and
               * x-compress are the same as gzip and compress. (Sec 3.5 RFC
               * 2616). zlib cannot handle compress.  However, errors are
               * handled further down when the response body is processed
               */
              char *start;

              /* Find the first non-space letter */
              for(start=k->p+17;
                  *start && isspace((int)*start);
                  start++);

              /* Record the content-encoding for later use. 08/27/02 jhrg */
              if (checkprefix("identity", start))
                k->content_encoding = IDENTITY;
              else if (checkprefix("deflate", start))
                k->content_encoding = DEFLATE;
              else if (checkprefix("gzip", start) 
                       || checkprefix("x-gzip", start))
                k->content_encoding = GZIP;
              else if (checkprefix("compress", start) 
                       || checkprefix("x-compress", start))
                k->content_encoding = COMPRESS;
            }
            else if (checkprefix("Content-Range:", k->p)) {
              if (sscanf (k->p+14, " bytes %d-", &k->offset) ||
                  sscanf (k->p+14, " bytes: %d-", &k->offset)) {
                /* This second format was added August 1st 2000 by Igor
                   Khristophorov since Sun's webserver JavaWebServer/1.1.1
                   obviously sends the header this way! :-( */
                if (conn->resume_from == k->offset) {
                  /* we asked for a resume and we got it */
                  k->content_range = TRUE;
                }
              }
            }
            else if(data->cookies &&
                    checkprefix("Set-Cookie:", k->p)) {
              Curl_share_lock(data, CURL_LOCK_DATA_COOKIE,
                              CURL_LOCK_ACCESS_SINGLE);
              Curl_cookie_add(data,
                              data->cookies, TRUE, k->p+11,
                              /* If there is a custom-set Host: name, use it
                                 here, or else use real peer host name. */
                              conn->allocptr.cookiehost?
                              conn->allocptr.cookiehost:conn->name,
                              conn->ppath);
              Curl_share_unlock(data, CURL_LOCK_DATA_COOKIE);
            }
            else if(checkprefix("Last-Modified:", k->p) &&
                    (data->set.timecondition || data->set.get_filetime) ) {
              time_t secs=time(NULL);
              k->timeofdoc = curl_getdate(k->p+strlen("Last-Modified:"),
                                          &secs);
              if(data->set.get_filetime)
                data->info.filetime = k->timeofdoc;
            }
            else if((checkprefix("WWW-Authenticate:", k->p) &&
                    (401 == k->httpcode)) ||
                    (checkprefix("Proxy-authenticate:", k->p) &&
                    (407 == k->httpcode))) {
              /*
               * This page requires authentication
               */
              char *start = (k->httpcode == 407) ? 
                            k->p+strlen("Proxy-authenticate:"): 
                            k->p+strlen("WWW-Authenticate:");
              /*
               * Switch from proxy to web authentication and back if needed
               */
              if (k->httpcode == 407 && data->state.authstage != 407)
                Curl_http_auth_stage(data, 407);
              
              else if (k->httpcode == 401 && data->state.authstage != 401)
                Curl_http_auth_stage(data, 401);

              /* pass all white spaces */
              while(*start && isspace((int)*start))
                start++;

#ifdef GSSAPI
              if (checkprefix("GSS-Negotiate", start)) {
                if(data->state.authwant == CURLAUTH_GSSNEGOTIATE) {
                  /* if exactly this is wanted, go */
                  int neg = Curl_input_negotiate(conn, start);
                  if (neg == 0)
                    conn->newurl = strdup(data->change.url);
                }
                else
                  if(data->state.authwant & CURLAUTH_GSSNEGOTIATE)
                    data->state.authavail |= CURLAUTH_GSSNEGOTIATE;
              }
              else
#endif
#ifdef USE_SSLEAY
            /* NTLM support requires the SSL crypto libs */
              if(checkprefix("NTLM", start)) {
                if(data->state.authwant == CURLAUTH_NTLM) {
                  /* NTLM authentication is activated */
                  CURLntlm ntlm =
                    Curl_input_ntlm(conn, (bool)(k->httpcode == 407), start);
                  
                  if(CURLNTLM_BAD != ntlm)
                    conn->newurl = strdup(data->change.url); /* clone string */
                  else
                    infof(data, "Authentication problem. Ignoring this.\n");
                }
                else
                  if(data->state.authwant & CURLAUTH_NTLM)
                    data->state.authavail |= CURLAUTH_NTLM;
              }
              else
#endif
              if(checkprefix("Digest", start)) {
                if(data->state.authwant == CURLAUTH_DIGEST) {
                  /* Digest authentication is activated */
                  CURLdigest dig = CURLDIGEST_BAD;

                  if(data->state.digest.nonce)
                    infof(data, "Authentication problem. Ignoring this.\n");
                  else
                    dig = Curl_input_digest(conn, start);

                  if(CURLDIGEST_FINE == dig)
                    /* We act on it. Store our new url, which happens to be
                       the same one we already use! */
                    conn->newurl = strdup(data->change.url); /* clone string */
                }
                else
                  if(data->state.authwant & CURLAUTH_DIGEST) {
                    /* We don't know if Digest is what we're gonna use, but we
                       call this function anyway to store the digest data that
                       is provided on this line, to skip the extra round-trip
                       we need to do otherwise. We must sure to free this
                       data! */
                    Curl_input_digest(conn, start);
                    data->state.authavail |= CURLAUTH_DIGEST;
                  }
              }
              else if(checkprefix("Basic", start)) {
                if((data->state.authwant == CURLAUTH_BASIC) &&
                   (k->httpcode == 401)) {
                  /* We asked for Basic authentication but got a 401 back
                     anyway, which basicly means our name+password isn't
                     valid. */
                  data->state.authavail = CURLAUTH_NONE;
                  infof(data, "Authentication problem. Ignoring this.\n");
                }
                else if(data->state.authwant & CURLAUTH_BASIC) {
                  data->state.authavail |= CURLAUTH_BASIC;
                }
              }
            }
            else if ((k->httpcode >= 300 && k->httpcode < 400) &&
                     checkprefix("Location:", k->p)) {
              if(data->set.http_follow_location) {
                /* this is the URL that the server advices us to get instead */
                char *ptr;
                char *start=k->p;
                char backup;

                start += 9; /* pass "Location:" */

                /* Skip spaces and tabs. We do this to support multiple
                   white spaces after the "Location:" keyword. */
                while(*start && isspace((int)*start ))
                  start++;
                
                /* Scan through the string from the end to find the last
                   non-space. k->end_ptr points to the actual terminating zero
                   letter, move pointer one letter back and start from
                   there. This logic strips off trailing whitespace, but keeps
                   any embedded whitespace. */
                ptr = k->end_ptr-1;
                while((ptr>=start) && isspace((int)*ptr))
                  ptr--;
                ptr++;

                backup = *ptr; /* store the ending letter */
                if(ptr != start) {
                  *ptr = '\0';   /* zero terminate */
                  conn->newurl = strdup(start); /* clone string */
                  *ptr = backup; /* restore ending letter */
                }
              }
#if 0 /* for consideration */
              else {
                /* This is a Location: but we have not been instructed to
                   follow it */
                infof(data, "We ignore this location header as instructed\n");
              }
#endif
            }

            /*
             * End of header-checks. Write them to the client.
             */

            writetype = CLIENTWRITE_HEADER;
            if (data->set.http_include_header)
              writetype |= CLIENTWRITE_BODY;

            if(data->set.verbose)
              Curl_debug(data, CURLINFO_HEADER_IN,
                         k->p, k->hbuflen);

            result = Curl_client_write(data, writetype, k->p, k->hbuflen);
            if(result)
              return result;

            data->info.header_size += k->hbuflen;
            conn->headerbytecount += k->hbuflen;
              
            /* reset hbufp pointer && hbuflen */
            k->hbufp = data->state.headerbuff;
            k->hbuflen = 0;
          }
          while (!stop_reading && *k->str); /* header line within buffer */

          if(stop_reading)
            /* We've stopped dealing with input, get out of the do-while loop */
            break;

          /* We might have reached the end of the header part here, but
             there might be a non-header part left in the end of the read
             buffer. */

        }                       /* end if header mode */

        /* This is not an 'else if' since it may be a rest from the header
           parsing, where the beginning of the buffer is headers and the end
           is non-headers. */
        if (k->str && !k->header && (nread > 0)) {
            
          if(0 == k->bodywrites) {
            /* These checks are only made the first time we are about to
               write a piece of the body */
            if(conn->protocol&PROT_HTTP) {
              /* HTTP-only checks */
              
              if(data->state.authavail) {
                if(data->state.authavail & CURLAUTH_GSSNEGOTIATE)
                  data->state.authwant = CURLAUTH_GSSNEGOTIATE;
                else if(data->state.authavail & CURLAUTH_DIGEST)
                  data->state.authwant = CURLAUTH_DIGEST;
                else if(data->state.authavail & CURLAUTH_NTLM)
                  data->state.authwant = CURLAUTH_NTLM;
                else if(data->state.authavail & CURLAUTH_BASIC)
                  data->state.authwant = CURLAUTH_BASIC;
                else
                  data->state.authwant = CURLAUTH_NONE; /* none */

                if(data->state.authwant)
                  conn->newurl = strdup(data->change.url); /* clone string */

                data->state.authavail = CURLAUTH_NONE; /* clear it here */
              }

              if (conn->newurl) {
                if(conn->bits.close) {
                  /* Abort after the headers if "follow Location" is set
                     and we're set to close anyway. */
                  k->keepon &= ~KEEP_READ;
                  FD_ZERO(&k->rkeepfd);
                  *done = TRUE;
                  return CURLE_OK;
                }
                /* We have a new url to load, but since we want to be able
                   to re-use this connection properly, we read the full
                   response in "ignore more" */
                k->ignorebody = TRUE;
                infof(data, "Ignoring the response-body\n");
              }
              if (conn->resume_from &&
                       !k->content_range &&
                       (data->set.httpreq==HTTPREQ_GET)) {
                /* we wanted to resume a download, although the server
                   doesn't seem to support this and we did this with a GET
                   (if it wasn't a GET we did a POST or PUT resume) */
                failf (data, "HTTP server doesn't seem to support "
                       "byte ranges. Cannot resume.");
                return CURLE_HTTP_RANGE_ERROR;
              }
              else if(data->set.timecondition && !conn->range) {
                /* A time condition has been set AND no ranges have been
                   requested. This seems to be what chapter 13.3.4 of
                   RFC 2616 defines to be the correct action for a
                   HTTP/1.1 client */
                if((k->timeofdoc > 0) && (data->set.timevalue > 0)) {
                  switch(data->set.timecondition) {
                  case TIMECOND_IFMODSINCE:
                  default:
                    if(k->timeofdoc < data->set.timevalue) {
                      infof(data,
                            "The requested document is not new enough\n");
                      *done = TRUE;
                      return CURLE_OK;
                    }
                    break;
                  case TIMECOND_IFUNMODSINCE:
                    if(k->timeofdoc > data->set.timevalue) {
                      infof(data,
                            "The requested document is not old enough\n");
                      *done = TRUE;
                      return CURLE_OK;
                    }
                    break;
                  } /* switch */
                } /* two valid time strings */
              } /* we have a time condition */

            } /* this is HTTP */
          } /* this is the first time we write a body part */
          k->bodywrites++;

          /* pass data to the debug function before it gets "dechunked" */
          if(data->set.verbose) {
            if(k->badheader) {
              Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff,
                         k->hbuflen);
              if(k->badheader == HEADER_PARTHEADER)
                Curl_debug(data, CURLINFO_DATA_IN, k->str, nread);
            }
            else
              Curl_debug(data, CURLINFO_DATA_IN, k->str, nread);
          }

          if(conn->bits.chunk) {
            /*