CURLOPT_PIPEWAIT: added

By setting this option to 1 libcurl will wait for a connection to reveal
if it is possible to pipeline/multiplex on before it continues.

docs/libcurl/opts/CURLOPT_PIPEWAIT.3

+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at http://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_PIPEWAIT 3 "12 May 2015" "libcurl 7.43.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_PIPEWAIT \- wait for pipelining/multiplexing
+.SH SYNOPSIS
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_PIPEWAIT, long wait);
+.SH DESCRIPTION
+Set \fIwait\fP to 1L to tell libcurl to prefer to wait for a connection to
+confirm or deny that it can do pipelining or multiplexing before continuing.
+
+When about to perform a new transfer that allows pipelining or multiplexing,
+libcurl will check for existing connections to re-use and pipeline on. If no
+such connection exists it will immediately continue and create a fresh new
+connection to use.
+
+By setting this option to 1 - and having \fICURLMOPT_PIPELINE\fP enabled for
+the multi handle this transfer is associated with - libcurl will instead wait
+for the connection to reveal if it is possible to pipeline/multiplex on before
+it continues. This enables libcurl to much better keep the number of
+connections to a minimum when using pipelining or multiplexing protocols.
+
+The effect thus becomes that with this option set, libcurl prefers to wait and
+re-use an existing connection for pipelining rather than the opposite: prefer
+to open a new connection rather than waiting.
+
+The waiting time is as long as it takes for the connection to get up and for
+libcurl to get the necessary response back that informs it about its protocol
+and support level.
+.SH DEFAULT
+0 (off)
+.SH PROTOCOLS
+HTTP(S)
+.SH EXAMPLE
+.SH AVAILABILITY
+Added in 7.43.0
+.SH RETURN VALUE
+Returns CURLE_OK if the option is supported, and CURLE_UNKNOWN_OPTION if not.
+.SH "SEE ALSO"
+.BR CURLOPT_FORBID_REUSE "(3), " CURLOPT_FRESH_CONNECT "(3), "
+.BR CURLMOPT_PIPELINING "(3), " CURLMOPT_MAX_HOST_CONNECTIONS "(3), "

docs/libcurl/opts/Makefile.am

@@ -114,7 +114,7 @@ man_MANS = CURLOPT_ACCEPT_ENCODING.3 CURLOPT_ACCEPTTIMEOUT_MS.3		\
  CURLMOPT_SOCKETDATA.3 CURLMOPT_SOCKETFUNCTION.3 CURLMOPT_TIMERDATA.3	\
  CURLMOPT_TIMERFUNCTION.3 CURLOPT_UNIX_SOCKET_PATH.3			\
  CURLOPT_PATH_AS_IS.3 CURLOPT_PROXY_SERVICE_NAME.3			\
- CURLOPT_SERVICE_NAME.3
+ CURLOPT_SERVICE_NAME.3 CURLOPT_PIPEWAIT.3
 
 HTMLPAGES = CURLOPT_ACCEPT_ENCODING.html CURLOPT_ACCEPTTIMEOUT_MS.html	\
  CURLOPT_ADDRESS_SCOPE.html CURLOPT_APPEND.html				\
@@ -221,7 +221,8 @@ HTMLPAGES = CURLOPT_ACCEPT_ENCODING.html CURLOPT_ACCEPTTIMEOUT_MS.html	\
  CURLMOPT_SOCKETDATA.html CURLMOPT_SOCKETFUNCTION.html			\
  CURLMOPT_TIMERDATA.html CURLMOPT_TIMERFUNCTION.html			\
  CURLOPT_UNIX_SOCKET_PATH.html CURLOPT_PATH_AS_IS.html			\
- CURLOPT_PROXY_SERVICE_NAME.html CURLOPT_SERVICE_NAME.html
+ CURLOPT_PROXY_SERVICE_NAME.html CURLOPT_SERVICE_NAME.html		\
+ CURLOPT_PIPEWAIT.html
 
 PDFPAGES = CURLOPT_ACCEPT_ENCODING.pdf CURLOPT_ACCEPTTIMEOUT_MS.pdf	\
  CURLOPT_ADDRESS_SCOPE.pdf CURLOPT_APPEND.pdf CURLOPT_AUTOREFERER.pdf	\
@@ -326,7 +327,8 @@ PDFPAGES = CURLOPT_ACCEPT_ENCODING.pdf CURLOPT_ACCEPTTIMEOUT_MS.pdf	\
  CURLMOPT_SOCKETDATA.pdf CURLMOPT_SOCKETFUNCTION.pdf			\
  CURLMOPT_TIMERDATA.pdf CURLMOPT_TIMERFUNCTION.pdf			\
  CURLOPT_UNIX_SOCKET_PATH.pdf CURLOPT_PATH_AS_IS.pdf			\
- CURLOPT_PROXY_SERVICE_NAME.pdf CURLOPT_SERVICE_NAME.pdf
+ CURLOPT_PROXY_SERVICE_NAME.pdf CURLOPT_SERVICE_NAME.pdf		\
+ CURLOPT_PIPEWAIT.pdf
 
 CLEANFILES = $(HTMLPAGES) $(PDFPAGES)
 

include/curl/curl.h

@@ -1638,6 +1638,9 @@ typedef enum {
   /* Service Name */
   CINIT(SERVICE_NAME, OBJECTPOINT, 236),
 
+  /* Wait/don't wait for pipe/mutex to clarify */
+  CINIT(PIPEWAIT, LONG, 237),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

lib/url.c

@@ -2649,6 +2649,9 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option,
   case CURLOPT_PATH_AS_IS:
     data->set.path_as_is = (0 != va_arg(param, long))?TRUE:FALSE;
     break;
+  case CURLOPT_PIPEWAIT:
+    data->set.pipewait = (0 != va_arg(param, long))?TRUE:FALSE;
+    break;
   default:
     /* unknown tag and its companion, just ignore: */
     result = CURLE_UNKNOWN_OPTION;
@@ -3099,7 +3102,8 @@ static bool
 ConnectionExists(struct SessionHandle *data,
                  struct connectdata *needle,
                  struct connectdata **usethis,
-                 bool *force_reuse)
+                 bool *force_reuse,
+                 bool *waitpipe)
 {
   struct connectdata *check;
   struct connectdata *chosen = 0;
@@ -3112,6 +3116,7 @@ ConnectionExists(struct SessionHandle *data,
   struct connectbundle *bundle;
 
   *force_reuse = FALSE;
+  *waitpipe = FALSE;
 
   /* We can't pipe if the site is blacklisted */
   if(canPipeline && Curl_pipeline_site_blacklisted(data, needle)) {
@@ -3132,9 +3137,17 @@ ConnectionExists(struct SessionHandle *data,
           needle->host.name, (void *)bundle);
 
     /* We can't pipe if we don't know anything about the server */
-    if(canPipeline && (bundle->multiuse <= BUNDLE_UNKNOWN)) {
-      infof(data, "Server doesn't support multi-use (yet)\n");
-      canPipeline = FALSE;
+    if(canPipeline) {
+      if(bundle->multiuse <= BUNDLE_UNKNOWN) {
+        if((bundle->multiuse == BUNDLE_UNKNOWN) && data->set.pipewait) {
+          infof(data, "Server doesn't support multi-use yet, wait\n");
+          *waitpipe = TRUE;
+          return FALSE; /* no re-use */
+        }
+
+        infof(data, "Server doesn't support multi-use (yet)\n");
+        canPipeline = FALSE;
+      }
     }
 
     curr = bundle->conn_list->head;
@@ -5343,8 +5356,9 @@ static CURLcode create_conn(struct SessionHandle *data,
   bool reuse;
   char *proxy = NULL;
   bool prot_missing = FALSE;
-  bool no_connections_available = FALSE;
+  bool connections_available = TRUE;
   bool force_reuse = FALSE;
+  bool waitpipe = FALSE;
   size_t max_host_connections = Curl_multi_max_host_connections(data->multi);
   size_t max_total_connections = Curl_multi_max_total_connections(data->multi);
 
@@ -5684,7 +5698,7 @@ static CURLcode create_conn(struct SessionHandle *data,
   if(data->set.reuse_fresh && !data->state.this_is_a_follow)
     reuse = FALSE;
   else
-    reuse = ConnectionExists(data, conn, &conn_temp, &force_reuse);
+    reuse = ConnectionExists(data, conn, &conn_temp, &force_reuse, &waitpipe);
 
   /* If we found a reusable connection, we may still want to
      open a new connection if we are pipelining. */
@@ -5730,9 +5744,15 @@ static CURLcode create_conn(struct SessionHandle *data,
     /* We have decided that we want a new connection. However, we may not
        be able to do that if we have reached the limit of how many
        connections we are allowed to open. */
-    struct connectbundle *bundle;
+    struct connectbundle *bundle = NULL;
+
+    if(waitpipe)
+      /* There is a connection that *might* become usable for pipelining
+         "soon", and we wait for that */
+      connections_available = FALSE;
+    else
+      bundle = Curl_conncache_find_bundle(conn, data->state.conn_cache);
 
-    bundle = Curl_conncache_find_bundle(conn, data->state.conn_cache);
     if(max_host_connections > 0 && bundle &&
        (bundle->num_connections >= max_host_connections)) {
       struct connectdata *conn_candidate;
@@ -5748,11 +5768,12 @@ static CURLcode create_conn(struct SessionHandle *data,
       else {
         infof(data, "No more connections allowed to host: %d\n",
               max_host_connections);
-        no_connections_available = TRUE;
+        connections_available = FALSE;
       }
     }
 
-    if(max_total_connections > 0 &&
+    if(connections_available &&
+       (max_total_connections > 0) &&
        (data->state.conn_cache->num_connections >= max_total_connections)) {
       struct connectdata *conn_candidate;
 
@@ -5766,12 +5787,11 @@ static CURLcode create_conn(struct SessionHandle *data,
       }
       else {
         infof(data, "No connections available in cache\n");
-        no_connections_available = TRUE;
+        connections_available = FALSE;
       }
     }
 
-
-    if(no_connections_available) {
+    if(!connections_available) {
       infof(data, "No connections available.\n");
 
       conn_free(conn);

lib/urldata.h

@@ -1623,6 +1623,8 @@ struct UserDefined {
   bool ssl_enable_npn;  /* TLS NPN extension? */
   bool ssl_enable_alpn; /* TLS ALPN extension? */
   bool path_as_is;      /* allow dotdots? */
+  bool pipewait;        /* wait for pipe/multiplex status before starting a
+                           new connection */
   long expect_100_timeout; /* in milliseconds */
 };