http-proxy: do the HTTP CONNECT process entirely non-blocking (5113ad04) · Commits · CYBER - Cyber Security / TS 103 523 MSP / TLMSP / TLMSP curl

docs/libcurl/libcurl-multi.3

+1 −2

Original line number	Diff line number	Diff line
		@@ -5,7 +5,7 @@
		.\" * \| (__\| \|_\| \| _ <\| \|___
		.\" * \___\|\___/\|_\| \_\_____\|
		.\" *
		.\" * Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al.
		.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
		.\" *
		.\" * This software is licensed as described in the file COPYING, which
		.\" * you should have received as part of this distribution. The terms
		@@ -173,7 +173,6 @@ the future, you should be aware of the following current restrictions:

		.nf
		- Name resolves unless the c-ares or threaded-resolver backends are used
		- HTTP proxy CONNECT operations
		- SOCKS proxy handshakes
		- file:// transfers
		- TELNET transfers

lib/ftp.c

+2 −3

Original line number	Diff line number	Diff line
		@@ -272,7 +272,6 @@ static void close_secondarysocket(struct connectdata *conn)
		conn->sock[SECONDARYSOCKET] = CURL_SOCKET_BAD;
		}
		conn->bits.tcpconnect[SECONDARYSOCKET] = FALSE;
		conn->tunnel_state[SECONDARYSOCKET] = TUNNEL_INIT;
		}

		/*
		@@ -3585,7 +3584,7 @@ static CURLcode ftp_do_more(struct connectdata conn, int completep)

		/* if the second connection isn't done yet, wait for it */
		if(!conn->bits.tcpconnect[SECONDARYSOCKET]) {
		if(conn->tunnel_state[SECONDARYSOCKET] == TUNNEL_CONNECT) {
		if(Curl_connect_ongoing(conn)) {
		/* As we're in TUNNEL_CONNECT state now, we know the proxy name and port
		aren't used so we blank their arguments. TODO: make this nicer */
		result = Curl_proxyCONNECT(conn, SECONDARYSOCKET, NULL, 0);
		@@ -3617,7 +3616,7 @@ static CURLcode ftp_do_more(struct connectdata conn, int completep)
		return result;

		if(conn->bits.tunnel_proxy && conn->bits.httpproxy &&
		conn->tunnel_state[SECONDARYSOCKET] != TUNNEL_COMPLETE)
		Curl_connect_ongoing(conn))
		return result;

lib/http.c

+1 −1

Original line number	Diff line number	Diff line
		@@ -1369,7 +1369,7 @@ CURLcode Curl_http_connect(struct connectdata conn, bool done)
		if(CONNECT_FIRSTSOCKET_PROXY_SSL())
		return CURLE_OK; /* wait for HTTPS proxy SSL initialization to complete */

		if(conn->tunnel_state[FIRSTSOCKET] == TUNNEL_CONNECT)
		if(!Curl_connect_complete(conn))
		/* nothing else to do except wait right now - we're not done here. */
		return CURLE_OK;

lib/http_proxy.c

+134 −109

Original line number	Diff line number	Diff line
		@@ -135,7 +135,47 @@ CURLcode Curl_proxy_connect(struct connectdata *conn, int sockindex)
		return CURLE_OK;
		}

		#define CONNECT_BUFFER_SIZE 16384
		bool Curl_connect_complete(struct connectdata *conn)
		{
		return conn->connect_state &&
		(conn->connect_state->tunnel_state == TUNNEL_COMPLETE);
		}

		bool Curl_connect_ongoing(struct connectdata *conn)
		{
		return conn->connect_state &&
		(conn->connect_state->tunnel_state != TUNNEL_COMPLETE);
		}

		static CURLcode connect_init(struct connectdata *conn, bool reinit)
		{
		struct http_connect_state *s;
		if(!reinit) {
		DEBUGASSERT(!conn->connect_state);
		s = calloc(1, sizeof(struct http_connect_state));
		if(!s)
		return CURLE_OUT_OF_MEMORY;
		infof(conn->data, "allocate connect buffer!\n");
		conn->connect_state = s;
		}
		else {
		DEBUGASSERT(conn->connect_state);
		s = conn->connect_state;
		}
		s->tunnel_state = TUNNEL_INIT;
		s->keepon=TRUE;
		s->line_start = s->connect_buffer;
		s->ptr = s->line_start;
		s->cl=0;
		return CURLE_OK;
		}

		static void connect_done(struct connectdata *conn)
		{
		struct http_connect_state *s = conn->connect_state;
		s->tunnel_state = TUNNEL_COMPLETE;
		infof(conn->data, "CONNECT phase completed!\n");
		}

		static CURLcode CONNECT(struct connectdata *conn,
		int sockindex,
		@@ -147,23 +187,22 @@ static CURLcode CONNECT(struct connectdata *conn,
		struct SingleRequest *k = &data->req;
		CURLcode result;
		curl_socket_t tunnelsocket = conn->sock[sockindex];
		curl_off_t cl=0;
		bool closeConnection = FALSE;
		bool chunked_encoding = FALSE;
		time_t check;
		struct http_connect_state *s = conn->connect_state;

		#define SELECT_OK 0
		#define SELECT_ERROR 1
		#define SELECT_TIMEOUT 2
		int error = SELECT_OK;

		if(conn->tunnel_state[sockindex] == TUNNEL_COMPLETE)
		if(Curl_connect_complete(conn))
		return CURLE_OK; /* CONNECT is already completed */

		conn->bits.proxy_connect_closed = FALSE;

		do {
		if(TUNNEL_INIT == conn->tunnel_state[sockindex]) {
		if(TUNNEL_INIT == s->tunnel_state) {
		/* BEGIN CONNECT PHASE */
		char *host_port;
		Curl_send_buffer *req_buffer;
		@@ -271,65 +310,47 @@ static CURLcode CONNECT(struct connectdata *conn,
		if(result)
		return result;

		conn->tunnel_state[sockindex] = TUNNEL_CONNECT;
		s->tunnel_state = TUNNEL_CONNECT;
		} /* END CONNECT PHASE */

		check = Curl_timeleft(data, NULL, TRUE);
		if(check <= 0) {
		failf(data, "Proxy CONNECT aborted due to timeout");
		return CURLE_RECV_ERROR;
		return CURLE_OPERATION_TIMEDOUT;
		}

		if(!Curl_conn_data_pending(conn, sockindex))
		/* return so we'll be called again polling-style */
		return CURLE_OK;
		DEBUGF(infof(data, "Read response immediately from proxy CONNECT\n"));

		/* at this point, the tunnel_connecting phase is over. */

		{ /* READING RESPONSE PHASE */
		size_t nread; /* total size read */
		int perline; /* count bytes per line */
		int keepon=TRUE;
		ssize_t gotbytes;
		char *ptr;
		char *line_start;

		ptr = conn->connect_buffer;
		line_start = ptr;
		int error = SELECT_OK;

		nread = 0;
		perline = 0;
		s->perline = 0;

		while(nread < (size_t)CONNECT_BUFFER_SIZE && keepon && !error) {
		if(Curl_pgrsUpdate(conn))
		return CURLE_ABORTED_BY_CALLBACK;
		while(s->keepon && !error) {
		ssize_t gotbytes;

		if(ptr >= &conn->connect_buffer[CONNECT_BUFFER_SIZE]) {
		/* make sure we have space to read more data */
		if(s->ptr >= &s->connect_buffer[CONNECT_BUFFER_SIZE]) {
		failf(data, "CONNECT response too large!");
		return CURLE_RECV_ERROR;
		}

		check = Curl_timeleft(data, NULL, TRUE);
		if(check <= 0) {
		failf(data, "Proxy CONNECT aborted due to timeout");
		error = SELECT_TIMEOUT; /* already too little time */
		break;
		}

		/* Read one byte at a time to avoid a race condition. Wait at most one
		second before looping to ensure continuous pgrsUpdates. */
		result = Curl_read(conn, tunnelsocket, ptr, 1, &gotbytes);
		if(result == CURLE_AGAIN) {
		if(SOCKET_READABLE(tunnelsocket, check<1000L?check:1000) == -1) {
		error = SELECT_ERROR;
		failf(data, "Proxy CONNECT aborted due to select/poll error");
		break;
		}
		continue;
		}
		result = Curl_read(conn, tunnelsocket, s->ptr, 1, &gotbytes);
		if(result == CURLE_AGAIN)
		/* socket buffer drained, return */
		return CURLE_OK;

		if(Curl_pgrsUpdate(conn))
		return CURLE_ABORTED_BY_CALLBACK;

		if(result) {
		keepon = FALSE;
		s->keepon = FALSE;
		break;
		}
		else if(gotbytes <= 0) {
		@@ -343,24 +364,22 @@ static CURLcode CONNECT(struct connectdata *conn,
		error = SELECT_ERROR;
		failf(data, "Proxy CONNECT aborted");
		}
		keepon = FALSE;
		s->keepon = FALSE;
		break;
		}

		/* We got a byte of data */
		nread++;

		if(keepon > TRUE) {
		if(s->keepon > TRUE) {
		/* This means we are currently ignoring a response-body */

		nread = 0; /* make next read start over in the read buffer */
		ptr = conn->connect_buffer;
		if(cl) {
		s->ptr = s->connect_buffer;
		if(s->cl) {
		/* A Content-Length based body: simply count down the counter
		and make sure to break out of the loop when we're done! */
		cl--;
		if(cl <= 0) {
		keepon = FALSE;
		s->cl--;
		if(s->cl <= 0) {
		s->keepon = FALSE;
		s->tunnel_state = TUNNEL_COMPLETE;
		break;
		}
		}
		@@ -372,23 +391,23 @@ static CURLcode CONNECT(struct connectdata *conn,

		/* now parse the chunked piece of data so that we can
		properly tell when the stream ends */
		r = Curl_httpchunk_read(conn, ptr, 1, &tookcareof);
		r = Curl_httpchunk_read(conn, s->ptr, 1, &tookcareof);
		if(r == CHUNKE_STOP) {
		/* we're done reading chunks! */
		infof(data, "chunk reading DONE\n");
		keepon = FALSE;
		s->keepon = FALSE;
		/* we did the full CONNECT treatment, go COMPLETE */
		conn->tunnel_state[sockindex] = TUNNEL_COMPLETE;
		s->tunnel_state = TUNNEL_COMPLETE;
		}
		}
		continue;
		}

		perline++; /* amount of bytes in this line so far */
		s->perline++; /* amount of bytes in this line so far */

		/* if this is not the end of a header line then continue */
		if(*ptr != 0x0a) {
		ptr++;
		if(*s->ptr != 0x0a) {
		s->ptr++;
		continue;
		}

		@@ -401,7 +420,7 @@ static CURLcode CONNECT(struct connectdata *conn,
		/* output debug if that is requested */
		if(data->set.verbose)
		Curl_debug(data, CURLINFO_HEADER_IN,
		line_start, (size_t)perline, conn);
		s->line_start, (size_t)s->perline, conn);

		if(!data->set.suppress_connect_headers) {
		/* send the header to the callback */
		@@ -409,33 +428,32 @@ static CURLcode CONNECT(struct connectdata *conn,
		if(data->set.include_header)
		writetype \|= CLIENTWRITE_BODY;

		result = Curl_client_write(conn, writetype, line_start, perline);
		result = Curl_client_write(conn, writetype,
		s->line_start, s->perline);
		if(result)
		return result;
		}

		data->info.header_size += (long)perline;
		data->req.headerbytecount += (long)perline;
		data->info.header_size += (long)s->perline;
		data->req.headerbytecount += (long)s->perline;

		/* Newlines are CRLF, so the CR is ignored as the line isn't
		really terminated until the LF comes. Treat a following CR
		as end-of-headers as well.*/

		if(('\r' == line_start[0]) \|\|
		('\n' == line_start[0])) {
		if(('\r' == s->line_start[0]) \|\|
		('\n' == s->line_start[0])) {
		/* end of response-headers from the proxy */
		nread = 0; /* make next read start over in the read
		buffer */
		ptr = conn->connect_buffer;
		s->ptr = s->connect_buffer;
		if((407 == k->httpcode) && !data->state.authproblem) {
		/* If we get a 407 response code with content length
		when we have no auth problem, we must ignore the
		whole response-body */
		keepon = 2;
		s->keepon = 2;

		if(cl) {
		if(s->cl) {
		infof(data, "Ignore %" CURL_FORMAT_CURL_OFF_T
		" bytes of response-body\n", cl);
		" bytes of response-body\n", s->cl);
		}
		else if(chunked_encoding) {
		CHUNKcode r;
		@@ -448,46 +466,46 @@ static CURLcode CONNECT(struct connectdata *conn,
		function returns! */
		k->ignorebody = TRUE;

		if(line_start[1] == '\n') {
		if(s->line_start[1] == '\n') {
		/* this can only be a LF if the letter at index 0
		was a CR */
		line_start++;
		s->line_start++;
		}

		/* now parse the chunked piece of data so that we can
		properly tell when the stream ends */
		r = Curl_httpchunk_read(conn, line_start + 1, 1, &gotbytes);
		r = Curl_httpchunk_read(conn, s->line_start + 1, 1, &gotbytes);
		if(r == CHUNKE_STOP) {
		/* we're done reading chunks! */
		infof(data, "chunk reading DONE\n");
		keepon = FALSE;
		/* we did the full CONNECT treatment, go to
		COMPLETE */
		conn->tunnel_state[sockindex] = TUNNEL_COMPLETE;
		s->keepon = FALSE;
		/* we did the full CONNECT treatment, go to COMPLETE */
		s->tunnel_state = TUNNEL_COMPLETE;
		}
		}
		else {
		/* without content-length or chunked encoding, we
		can't keep the connection alive since the close is
		the end signal so we bail out at once instead */
		keepon = FALSE;
		s->keepon = FALSE;
		}
		}
		else
		keepon = FALSE;
		s->keepon = FALSE;
		if(!s->cl)
		/* we did the full CONNECT treatment, go to COMPLETE */
		conn->tunnel_state[sockindex] = TUNNEL_COMPLETE;
		s->tunnel_state = TUNNEL_COMPLETE;
		continue;
		}

		line_start[perline] = 0; /* zero terminate the buffer */
		if((checkprefix("WWW-Authenticate:", line_start) &&
		s->line_start[s->perline] = 0; /* zero terminate the buffer */
		if((checkprefix("WWW-Authenticate:", s->line_start) &&
		(401 == k->httpcode)) \|\|
		(checkprefix("Proxy-authenticate:", line_start) &&
		(checkprefix("Proxy-authenticate:", s->line_start) &&
		(407 == k->httpcode))) {

		bool proxy = (k->httpcode == 407) ? TRUE : FALSE;
		char *auth = Curl_copy_header_value(line_start);
		char *auth = Curl_copy_header_value(s->line_start);
		if(!auth)
		return CURLE_OUT_OF_MEMORY;

		@@ -498,7 +516,7 @@ static CURLcode CONNECT(struct connectdata *conn,
		if(result)
		return result;
		}
		else if(checkprefix("Content-Length:", line_start)) {
		else if(checkprefix("Content-Length:", s->line_start)) {
		if(k->httpcode/100 == 2) {
		/* A client MUST ignore any Content-Length or Transfer-Encoding
		header fields received in a successful response to CONNECT.
		@@ -507,13 +525,13 @@ static CURLcode CONNECT(struct connectdata *conn,
		k->httpcode);
		}
		else {
		cl = curlx_strtoofft(line_start +
		s->cl = curlx_strtoofft(s->line_start +
		strlen("Content-Length:"), NULL, 10);
		}
		}
		else if(Curl_compareheader(line_start, "Connection:", "close"))
		else if(Curl_compareheader(s->line_start, "Connection:", "close"))
		closeConnection = TRUE;
		else if(checkprefix("Transfer-Encoding:", line_start)) {
		else if(checkprefix("Transfer-Encoding:", s->line_start)) {
		if(k->httpcode/100 == 2) {
		/* A client MUST ignore any Content-Length or Transfer-Encoding
		header fields received in a successful response to CONNECT.
		@@ -521,7 +539,7 @@ static CURLcode CONNECT(struct connectdata *conn,
		infof(data, "Ignoring Transfer-Encoding in "
		"CONNECT %03d response\n", k->httpcode);
		}
		else if(Curl_compareheader(line_start,
		else if(Curl_compareheader(s->line_start,
		"Transfer-Encoding:", "chunked")) {
		infof(data, "CONNECT responded chunked\n");
		chunked_encoding = TRUE;
		@@ -529,18 +547,19 @@ static CURLcode CONNECT(struct connectdata *conn,
		Curl_httpchunk_init(conn);
		}
		}
		else if(Curl_compareheader(line_start, "Proxy-Connection:", "close"))
		else if(Curl_compareheader(s->line_start,
		"Proxy-Connection:", "close"))
		closeConnection = TRUE;
		else if(2 == sscanf(line_start, "HTTP/1.%d %d",
		else if(2 == sscanf(s->line_start, "HTTP/1.%d %d",
		&subversion,
		&k->httpcode)) {
		/* store the HTTP code from the proxy */
		data->info.httpproxycode = k->httpcode;
		}

		perline = 0; /* line starts over here */
		ptr = conn->connect_buffer;
		line_start = ptr;
		s->perline = 0; /* line starts over here */
		s->ptr = s->connect_buffer;
		s->line_start = s->ptr;
		} /* while there's buffer left and loop is requested */

		if(Curl_pgrsUpdate(conn))
		@@ -574,11 +593,8 @@ static CURLcode CONNECT(struct connectdata *conn,
		/* If we are supposed to continue and request a new URL, which basically
		* means the HTTP authentication is still going on so if the tunnel
		* is complete we start over in INIT state */
		if(data->req.newurl &&
		(TUNNEL_COMPLETE == conn->tunnel_state[sockindex])) {
		conn->tunnel_state[sockindex] = TUNNEL_INIT;
		infof(data, "TUNNEL_STATE switched to: %d\n",
		conn->tunnel_state[sockindex]);
		if(data->req.newurl && (TUNNEL_COMPLETE == s->tunnel_state)) {
		connect_init(conn, TRUE); /* reinit */
		}

		} while(data->req.newurl);
		@@ -587,6 +603,7 @@ static CURLcode CONNECT(struct connectdata *conn,
		if(closeConnection && data->req.newurl) {
		conn->bits.proxy_connect_closed = TRUE;
		infof(data, "Connect me again please\n");
		connect_done(conn);
		}
		else {
		free(data->req.newurl);
		@@ -598,7 +615,7 @@ static CURLcode CONNECT(struct connectdata *conn,
		}

		/* to back to init state */
		conn->tunnel_state[sockindex] = TUNNEL_INIT;
		s->tunnel_state = TUNNEL_INIT;

		if(conn->bits.proxy_connect_closed)
		/* this is not an error, just part of the connection negotiation */
		@@ -608,7 +625,7 @@ static CURLcode CONNECT(struct connectdata *conn,
		return CURLE_RECV_ERROR;
		}

		conn->tunnel_state[sockindex] = TUNNEL_COMPLETE;
		s->tunnel_state = TUNNEL_COMPLETE;

		/* If a proxy-authorization header was used for the proxy, then we should
		make sure that it isn't accidentally used for the document request
		@@ -625,6 +642,16 @@ static CURLcode CONNECT(struct connectdata *conn,
		return CURLE_OK;
		}

		void Curl_connect_free(struct Curl_easy *data)
		{
		struct connectdata *conn = data->easy_conn;
		struct http_connect_state *s = conn->connect_state;
		if(s) {
		free(s);
		conn->connect_state = NULL;
		}
		}

		/*
		* Curl_proxyCONNECT() requires that we're connected to a HTTP proxy. This
		* function will issue the necessary commands to get a seamless tunnel through
		@@ -637,17 +664,15 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,
		int remote_port)
		{
		CURLcode result;
		if(TUNNEL_INIT == conn->tunnel_state[sockindex]) {
		if(!conn->connect_buffer) {
		conn->connect_buffer = malloc(CONNECT_BUFFER_SIZE);
		if(!conn->connect_buffer)
		return CURLE_OUT_OF_MEMORY;
		}
		if(!conn->connect_state) {
		result = connect_init(conn, FALSE);
		if(result)
		return result;
		}
		result = CONNECT(conn, sockindex, hostname, remote_port);

		if(result \|\| (TUNNEL_COMPLETE == conn->tunnel_state[sockindex]))
		Curl_safefree(conn->connect_buffer);
		if(result \|\| Curl_connect_complete(conn))
		connect_done(conn);

		return result;
		}

lib/http_proxy.h

+4 −0

Original line number	Diff line number	Diff line
		@@ -33,6 +33,10 @@ CURLcode Curl_proxyCONNECT(struct connectdata *conn,

		CURLcode Curl_proxy_connect(struct connectdata *conn, int sockindex);

		bool Curl_connect_complete(struct connectdata *conn);
		bool Curl_connect_ongoing(struct connectdata *conn);
		void Curl_connect_free(struct Curl_easy *data);

		#else
		#define Curl_proxyCONNECT(x,y,z,w) CURLE_NOT_BUILT_IN
		#define Curl_proxy_connect(x,y) CURLE_OK

Admin message