PROXYHEADER: send these headers in "normal" proxy requests too

Updated the docs to clarify and the code accordingly, with test 1528 to
verify:

When CURLHEADER_SEPARATE is set and libcurl is asked to send a request
to a proxy but it isn't CONNECT, then _both_ header lists
(CURLOPT_HTTPHEADER and CURLOPT_PROXYHEADER) will be used since the
single request is then made for both the proxy and the server.

docs/libcurl/curl_easy_setopt.3

@@ -1549,19 +1549,19 @@ There's an alternative option that sets or replaces headers only for requests
 that are sent with CONNECT to a proxy: \fICURLOPT_PROXYHEADER\fP. Use
 \fICURLOPT_HEADEROPT\fP to control the behavior.
 .IP CURLOPT_HEADEROPT
-Pass a long that is a bitmask of options of how to deal with headers. The
-available options are:
+Pass a long that is a bitmask of options of how to deal with headers. The two
+mutually exclusive options are:
 
 CURLHEADER_UNIFIED - keep working as before. This means CURLOPT_HTTPHEADER
-headers will be used in requests both to servers and in CONNECT requests. With
-this option enabled, \fICURLOPT_PROXYHEADER\fP will not have any effect.
+headers will be used in requests both to servers and proxies. With this option
+enabled, \fICURLOPT_PROXYHEADER\fP will not have any effect.
 
 CURLHEADER_SEPARATE - makes \fICURLOPT_HTTPHEADER\fP headers only get sent to
-a host and not to a proxy if CONNECT is being used. It has to be set to make
-\fICURLOPT_PROXYHEADER\fP get used.
-
-This behavior is set per request and an application can alter it between
-different invokes if desired.
+a server and not to a proxy. Proxy headers must be set with
+\fICURLOPT_PROXYHEADER\fP to get used. Note that if a non-CONNECT request is
+sent to a proxy, libcurl will send both server headers and proxy headers. When
+doing CONNECT, libcurl will send \fICURLOPT_PROXYHEADER\fP headers only do the
+proxy and then \fICURLOPT_HTTPHEADER\fP headers only to the server.
 
 (Added in 7.36.0)
 .IP CURLOPT_PROXYHEADER

lib/http.c

@@ -1544,80 +1544,120 @@ static CURLcode expect100(struct SessionHandle *data,
   return result;
 }
 
+enum proxy_use {
+  HEADER_SERVER,  /* direct to server */
+  HEADER_PROXY,   /* regular request to proxy */
+  HEADER_CONNECT  /* sending CONNECT to a proxy */
+};
+
 CURLcode Curl_add_custom_headers(struct connectdata *conn,
-                                 bool is_proxy,
+                                 bool is_connect,
                                  Curl_send_buffer *req_buffer)
 {
   char *ptr;
-  struct curl_slist *headers=
-    (is_proxy && conn->data->set.sep_headers)?
-    conn->data->set.proxyheaders:conn->data->set.headers;
-
-  while(headers) {
-    ptr = strchr(headers->data, ':');
-    if(ptr) {
-      /* we require a colon for this to be a true header */
+  struct curl_slist *h[2];
+  struct curl_slist *headers;
+  int numlists=1; /* by default */
+  struct SessionHandle *data = conn->data;
+  int i;
 
-      ptr++; /* pass the colon */
-      while(*ptr && ISSPACE(*ptr))
-        ptr++;
+  enum proxy_use proxy;
 
-      if(*ptr) {
-        /* only send this if the contents was non-blank */
+  if(is_connect)
+    proxy = HEADER_CONNECT;
+  else
+    proxy = conn->bits.httpproxy && !conn->bits.tunnel_proxy?
+      HEADER_PROXY:HEADER_SERVER;
 
-        if(conn->allocptr.host &&
-           /* a Host: header was sent already, don't pass on any custom Host:
-              header as that will produce *two* in the same request! */
-           checkprefix("Host:", headers->data))
-          ;
-        else if(conn->data->set.httpreq == HTTPREQ_POST_FORM &&
-                /* this header (extended by formdata.c) is sent later */
-                checkprefix("Content-Type:", headers->data))
-          ;
-        else if(conn->bits.authneg &&
-                /* while doing auth neg, don't allow the custom length since
-                   we will force length zero then */
-                checkprefix("Content-Length", headers->data))
-          ;
-        else if(conn->allocptr.te &&
-                /* when asking for Transfer-Encoding, don't pass on a custom
-                   Connection: */
-                checkprefix("Connection", headers->data))
-          ;
-        else {
-          CURLcode result = Curl_add_bufferf(req_buffer, "%s\r\n",
-                                             headers->data);
-          if(result)
-            return result;
-        }
-      }
+  switch(proxy) {
+  case HEADER_SERVER:
+    h[0] = data->set.headers;
+    break;
+  case HEADER_PROXY:
+    h[0] = data->set.headers;
+    if(data->set.sep_headers) {
+      h[1] = data->set.proxyheaders;
+      numlists++;
     }
-    else {
-      ptr = strchr(headers->data, ';');
+    break;
+  case HEADER_CONNECT:
+    if(data->set.sep_headers)
+      h[0] = data->set.proxyheaders;
+    else
+      h[0] = data->set.headers;
+    break;
+  }
+
+  /* loop through one or two lists */
+  for(i=0; i < numlists; i++) {
+    headers = h[i];
+
+    while(headers) {
+      ptr = strchr(headers->data, ':');
       if(ptr) {
+        /* we require a colon for this to be a true header */
 
-        ptr++; /* pass the semicolon */
+        ptr++; /* pass the colon */
         while(*ptr && ISSPACE(*ptr))
           ptr++;
 
         if(*ptr) {
-          /* this may be used for something else in the future */
-        }
-        else {
-          if(*(--ptr) == ';') {
-            CURLcode result;
-
-            /* send no-value custom header if terminated by semicolon */
-            *ptr = ':';
-            result = Curl_add_bufferf(req_buffer, "%s\r\n",
-                                             headers->data);
+          /* only send this if the contents was non-blank */
+
+          if(conn->allocptr.host &&
+             /* a Host: header was sent already, don't pass on any custom Host:
+                header as that will produce *two* in the same request! */
+             checkprefix("Host:", headers->data))
+            ;
+          else if(data->set.httpreq == HTTPREQ_POST_FORM &&
+                  /* this header (extended by formdata.c) is sent later */
+                  checkprefix("Content-Type:", headers->data))
+            ;
+          else if(conn->bits.authneg &&
+                  /* while doing auth neg, don't allow the custom length since
+                     we will force length zero then */
+                  checkprefix("Content-Length", headers->data))
+            ;
+          else if(conn->allocptr.te &&
+                  /* when asking for Transfer-Encoding, don't pass on a custom
+                     Connection: */
+                  checkprefix("Connection", headers->data))
+            ;
+          else {
+            CURLcode result = Curl_add_bufferf(req_buffer, "%s\r\n",
+                                               headers->data);
             if(result)
               return result;
           }
         }
       }
+      else {
+        ptr = strchr(headers->data, ';');
+        if(ptr) {
+
+          ptr++; /* pass the semicolon */
+          while(*ptr && ISSPACE(*ptr))
+            ptr++;
+
+          if(*ptr) {
+            /* this may be used for something else in the future */
+          }
+          else {
+            if(*(--ptr) == ';') {
+              CURLcode result;
+
+              /* send no-value custom header if terminated by semicolon */
+              *ptr = ':';
+              result = Curl_add_bufferf(req_buffer, "%s\r\n",
+                                        headers->data);
+              if(result)
+                return result;
+            }
+          }
+        }
+      }
+      headers = headers->next;
     }
-    headers = headers->next;
   }
   return CURLE_OK;
 }

tests/data/Makefile.am

@@ -129,7 +129,7 @@ test1500 test1501 test1502 test1503 test1504 test1505 test1506 test1507 \
 test1508 test1509 test1510 test1511 test1512 test1513 test1514 test1515 \
 test1516 \
 \
-test1525 test1526 test1527 \
+test1525 test1526 test1527 test1528 \
 \
 test1900 test1901 test1902 test1903 \
 \

tests/data/test1528

+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+HTTP CONNECT
+HTTP proxy
+proxytunnel
+</keywords>
+</info>
+
+# Server-side
+<reply>
+<connect>
+HTTP/1.1 200 OK
+We-are: good
+
+</connect>
+<data>
+HTTP/1.1 200 OK swsclose
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Content-Length: 5
+
+stop
+</data>
+
+</reply>
+# Client-side
+<client>
+<server>
+http
+http-proxy
+</server>
+<tool>
+lib1528
+</tool>
+ <name>
+Separately specified proxy/server headers sent in a proxy GET
+ </name>
+ <command>
+ http://the.old.moo:%HTTPPORT/1528 %HOSTIP:%PROXYPORT
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<proxy>
+GET http://the.old.moo:%HTTPPORT/1528 HTTP/1.1
+Host: the.old.moo:%HTTPPORT
+Accept: */*
+Proxy-Connection: Keep-Alive
+User-Agent: Http Agent
+Proxy-User-Agent: Http Agent2
+
+</proxy>
+</verify>
+</testcase>

tests/libtest/Makefile.inc

@@ -23,7 +23,7 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect                \
  lib1500 lib1501 lib1502 lib1503 lib1504 lib1505 lib1506 lib1507 lib1508 \
  lib1509 lib1510 lib1511 lib1512 lib1513 lib1514 lib1515 \
  lib1509 lib1510 lib1511 lib1512 lib1513 lib1514 lib1515 \
- lib1525 lib1526 lib1527 \
+ lib1525 lib1526 lib1527 lib1528 \
  lib1900 \
  lib2033
 
@@ -369,6 +369,10 @@ lib1527_SOURCES = lib1527.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1527_LDADD = $(TESTUTIL_LIBS)
 lib1527_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1527
 
+lib1528_SOURCES = lib1528.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
+lib1528_LDADD = $(TESTUTIL_LIBS)
+lib1528_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1528
+
 lib1900_SOURCES = lib1900.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1900_LDADD = $(TESTUTIL_LIBS)
 lib1900_CPPFLAGS = $(AM_CPPFLAGS)

tests/libtest/lib1528.c

+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2014, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at http://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+
+#include "test.h"
+
+#include "memdebug.h"
+
+int test(char *URL)
+{
+  CURL *curl = NULL;
+  CURLcode res = CURLE_FAILED_INIT;
+  /* http header list*/
+  struct curl_slist *hhl = NULL;
+  struct curl_slist *phl = NULL;
+
+  if(curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) {
+    fprintf(stderr, "curl_global_init() failed\n");
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  if((curl = curl_easy_init()) == NULL) {
+    fprintf(stderr, "curl_easy_init() failed\n");
+    curl_global_cleanup();
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  hhl = curl_slist_append(hhl, "User-Agent: Http Agent");
+  phl = curl_slist_append(phl, "Proxy-User-Agent: Http Agent2");
+
+  if (!hhl) {
+    goto test_cleanup;
+  }
+
+  test_setopt(curl, CURLOPT_URL, URL);
+  test_setopt(curl, CURLOPT_PROXY, libtest_arg2);
+  test_setopt(curl, CURLOPT_HTTPHEADER, hhl);
+  test_setopt(curl, CURLOPT_PROXYHEADER, phl);
+  test_setopt(curl, CURLOPT_HEADEROPT, CURLHEADER_SEPARATE);
+  test_setopt(curl, CURLOPT_VERBOSE, 1L);
+  test_setopt(curl, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
+  test_setopt(curl, CURLOPT_HEADER, 1L);
+
+  res = curl_easy_perform(curl);
+
+test_cleanup:
+
+  curl_easy_cleanup(curl);
+  curl_slist_free_all(hhl);
+  curl_slist_free_all(phl);
+  curl_global_cleanup();
+
+  return (int)res;
+}