HTTP: implement Brotli content encoding (11bf1796) · Commits · CYBER - Cyber Security / TS 103 523 MSP / TLMSP / TLMSP curl

configure.ac

+93 −0

Original line number	Original line	Diff line number	Diff line
	@@ -148,6 +148,7 @@ dnl initialize all the info variables
	curl_ssl_msg="no (--with-{ssl,gnutls,nss,polarssl,mbedtls,cyassl,axtls,winssl,darwinssl} )"		curl_ssl_msg="no (--with-{ssl,gnutls,nss,polarssl,mbedtls,cyassl,axtls,winssl,darwinssl} )"
	curl_ssh_msg="no (--with-libssh2)"		curl_ssh_msg="no (--with-libssh2)"
	curl_zlib_msg="no (--with-zlib)"		curl_zlib_msg="no (--with-zlib)"
			curl_brotli_msg="no (--with-brotli)"
	curl_gss_msg="no (--with-gssapi)"		curl_gss_msg="no (--with-gssapi)"
	curl_tls_srp_msg="no (--enable-tls-srp)"		curl_tls_srp_msg="no (--enable-tls-srp)"
	curl_res_msg="default (--enable-ares / --enable-threaded-resolver)"		curl_res_msg="default (--enable-ares / --enable-threaded-resolver)"
	@@ -975,6 +976,94 @@ dnl set variable for use in automakefile(s)
	AM_CONDITIONAL(HAVE_LIBZ, test x"$AMFIXLIB" = x1)		AM_CONDITIONAL(HAVE_LIBZ, test x"$AMFIXLIB" = x1)
	AC_SUBST(ZLIB_LIBS)		AC_SUBST(ZLIB_LIBS)

			dnl **********************************************************************
			dnl Check for the presence of BROTLI decoder libraries and headers
			dnl **********************************************************************

			dnl Brotli project home page: https://github.com/google/brotli

			dnl Default to compiler & linker defaults for BROTLI files & libraries.
			OPT_BROTLI=off
			AC_ARG_WITH(brotli,dnl
			AC_HELP_STRING([--with-brotli=PATH],[Where to look for brotli, PATH points to the BROTLI installation; when possible, set the PKG_CONFIG_PATH environment variable instead of using this option])
			AC_HELP_STRING([--without-brotli], [disable BROTLI]),
			OPT_BROTLI=$withval)

			if test X"$OPT_BROTLI" != Xno; then
			dnl backup the pre-brotli variables
			CLEANLDFLAGS="$LDFLAGS"
			CLEANCPPFLAGS="$CPPFLAGS"
			CLEANLIBS="$LIBS"

			case "$OPT_BROTLI" in
			yes)
			dnl --with-brotli (without path) used
			CURL_CHECK_PKGCONFIG(libbrotlidec)

			if test "$PKGCONFIG" != "no" ; then
			LIB_BROTLI=`$PKGCONFIG --libs-only-l libbrotlidec`
			LD_BROTLI=`$PKGCONFIG --libs-only-L libbrotlidec`
			CPP_BROTLI=`$PKGCONFIG --cflags-only-I libbrotlidec`
			version=`$PKGCONFIG --modversion libbrotlidec`
			DIR_BROTLI=`echo $LD_BROTLI \| $SED -e 's/-L//'`
			fi

			;;
			off)
			dnl no --with-brotli option given, just check default places
			;;
			*)
			dnl use the given --with-brotli spot
			PREFIX_BROTLI=$OPT_BROTLI
			;;
			esac

			dnl if given with a prefix, we set -L and -I based on that
			if test -n "$PREFIX_BROTLI"; then
			LIB_BROTLI="-lbrotlidec"
			LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff
			CPP_BROTLI=-I${PREFIX_BROTLI}/include
			DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff
			fi

			LDFLAGS="$LDFLAGS $LD_BROTLI"
			CPPFLAGS="$CPPFLAGS $CPP_BROTLI"
			LIBS="$LIB_BROTLI $LIBS"

			AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress)

			AC_CHECK_HEADERS(brotli/decode.h,
			curl_brotli_msg="enabled (libbrotlidec)"
			HAVE_BROTLI=1
			AC_DEFINE(HAVE_BROTLI, 1, [if BROTLI is in use])
			AC_SUBST(HAVE_BROTLI, [1])
			)

			if test X"$OPT_BROTLI" != Xoff &&
			test "$HAVE_BROTLI" != "1"; then
			AC_MSG_ERROR([BROTLI libs and/or directories were not found where specified!])
			fi

			if test "$HAVE_BROTLI" = "1"; then
			if test -n "$DIR_BROTLI"; then
			dnl when the brotli shared libs were found in a path that the run-time
			dnl linker doesn't search through, we need to add it to LD_LIBRARY_PATH
			dnl to prevent further configure tests to fail due to this

			if test "x$cross_compiling" != "xyes"; then
			LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$DIR_BROTLI"
			export LD_LIBRARY_PATH
			AC_MSG_NOTICE([Added $DIR_BROTLI to LD_LIBRARY_PATH])
			fi
			fi
			else
			dnl no brotli, revert back to clean variables
			LDFLAGS=$CLEANLDFLAGS
			CPPFLAGS=$CLEANCPPFLAGS
			LIBS=$CLEANLIBS
			fi
			fi

	dnl **********************************************************************		dnl **********************************************************************
	dnl Check for LDAP		dnl Check for LDAP
	dnl **********************************************************************		dnl **********************************************************************
	@@ -3786,6 +3875,9 @@ fi
	if test "x$HAVE_LIBZ" = "x1"; then		if test "x$HAVE_LIBZ" = "x1"; then
	SUPPORT_FEATURES="$SUPPORT_FEATURES libz"		SUPPORT_FEATURES="$SUPPORT_FEATURES libz"
	fi		fi
			if test "x$HAVE_BROTLI" = "x1"; then
			SUPPORT_FEATURES="$SUPPORT_FEATURES brotli"
			fi
	if test "x$USE_ARES" = "x1" -o "x$USE_THREADS_POSIX" = "x1" \		if test "x$USE_ARES" = "x1" -o "x$USE_THREADS_POSIX" = "x1" \
	-o "x$USE_THREADS_WIN32" = "x1"; then		-o "x$USE_THREADS_WIN32" = "x1"; then
	SUPPORT_FEATURES="$SUPPORT_FEATURES AsynchDNS"		SUPPORT_FEATURES="$SUPPORT_FEATURES AsynchDNS"
	@@ -4003,6 +4095,7 @@ AC_MSG_NOTICE([Configured to build curl/libcurl:
	SSL support: ${curl_ssl_msg}		SSL support: ${curl_ssl_msg}
	SSH support: ${curl_ssh_msg}		SSH support: ${curl_ssh_msg}
	zlib support: ${curl_zlib_msg}		zlib support: ${curl_zlib_msg}
			brotli support: ${curl_brotli_msg}
	GSS-API support: ${curl_gss_msg}		GSS-API support: ${curl_gss_msg}
	TLS-SRP support: ${curl_tls_srp_msg}		TLS-SRP support: ${curl_tls_srp_msg}
	resolver: ${curl_res_msg}		resolver: ${curl_res_msg}

docs/INTERNALS.md

+11 −10

docs/RESOURCES

+2 −0

Original line number	Original line	Diff line number	Diff line
	@@ -81,3 +81,5 @@ This document lists documents and standards used by curl.
	RFC 4616 - PLAIN authentication		RFC 4616 - PLAIN authentication

	RFC 4954 - SMTP Authentication		RFC 4954 - SMTP Authentication

			RFC 7932 - Brotli Compressed Data Format

docs/TODO

+4 −11

Original line number	Original line	Diff line number	Diff line
	@@ -64,9 +64,8 @@
	5.4 HTTP Digest using SHA-256		5.4 HTTP Digest using SHA-256
	5.5 auth= in URLs		5.5 auth= in URLs
	5.6 Refuse "downgrade" redirects		5.6 Refuse "downgrade" redirects
	5.7 Brotli compression		5.7 QUIC
	5.8 QUIC		5.8 Leave secure cookies alone
	5.9 Leave secure cookies alone

	6. TELNET		6. TELNET
	6.1 ditch stdin		6.1 ditch stdin
	@@ -514,13 +513,7 @@ This is not detailed in any FTP specification.
	Consider a way to tell curl to refuse to "downgrade" protocol with a redirect		Consider a way to tell curl to refuse to "downgrade" protocol with a redirect
	and/or possibly a bit that refuses redirect to change protocol completely.		and/or possibly a bit that refuses redirect to change protocol completely.

	5.7 Brotli compression		5.7 QUIC

	Brotli compression performs better than gzip and is being implemented by
	browsers and servers widely. The algorithm: https://github.com/google/brotli
	The Firefox bug: https://bugzilla.mozilla.org/show_bug.cgi?id=366559

	5.8 QUIC

	The standardization process of QUIC has been taken to the IETF and can be		The standardization process of QUIC has been taken to the IETF and can be
	followed on the [IETF QUIC Mailing		followed on the [IETF QUIC Mailing
	@@ -530,7 +523,7 @@ This is not detailed in any FTP specification.
	implemented. This, to allow other projects to benefit from the work and to		implemented. This, to allow other projects to benefit from the work and to
	thus broaden the interest and chance of others to participate.		thus broaden the interest and chance of others to participate.

	5.9 Leave secure cookies alone		5.8 Leave secure cookies alone

	Non-secure origins (HTTP sites) should not be allowed to set or modify		Non-secure origins (HTTP sites) should not be allowed to set or modify
	cookies with the 'secure' property:		cookies with the 'secure' property:

docs/libcurl/curl_version_info.3

+8 −0

Original line number	Original line	Diff line number	Diff line
	@@ -72,6 +72,12 @@ typedef struct {

	const char libssh_version; / human readable string */		const char libssh_version; / human readable string */

			/* when 'age' is 4 or higher (7.57.0 or later), the members below also
			exist */
			unsigned int brotli_ver_num; /* Numeric Brotli version
			(MAJOR << 24) \| (MINOR << 12) \| PATCH */
			const char brotli_version; / human readable string. */

	} curl_version_info_data;		} curl_version_info_data;
	.fi		.fi

	@@ -160,6 +166,8 @@ libcurl was built with support for HTTPS-proxy.
	libcurl was built with multiple SSL backends. For details, see		libcurl was built with multiple SSL backends. For details, see
	\fIcurl_global_sslset(3)\fP.		\fIcurl_global_sslset(3)\fP.
	(Added in 7.56.0)		(Added in 7.56.0)
			.IP CURL_VERSION_BROTLI
			supports HTTP Brotli content encoding using libbrotlidec (Added in 7.57.0)
	.RE		.RE
	\fIssl_version\fP is an ASCII string for the OpenSSL version used. If libcurl		\fIssl_version\fP is an ASCII string for the OpenSSL version used. If libcurl
	has no SSL support, this is NULL.		has no SSL support, this is NULL.

Original line number	Original line	Diff line number	Diff line
	@@ -644,9 +644,9 @@ Content Encoding
	[HTTP/1.1][4] specifies that a client may request that a server encode its		[HTTP/1.1][4] specifies that a client may request that a server encode its
	response. This is usually used to compress a response using one (or more)		response. This is usually used to compress a response using one (or more)
	encodings from a set of commonly available compression techniques. These		encodings from a set of commonly available compression techniques. These
	schemes include 'deflate' (the zlib algorithm), 'gzip' and 'compress'. A		schemes include 'deflate' (the zlib algorithm), 'gzip' 'br' (brotli) and
	client requests that the server perform an encoding by including an		'compress'. A client requests that the server perform an encoding by including
	Accept-Encoding header in the request document. The value of the header		an Accept-Encoding header in the request document. The value of the header
	should be one of the recognized tokens 'deflate', ... (there's a way to		should be one of the recognized tokens 'deflate', ... (there's a way to
	register new schemes/tokens, see sec 3.5 of the spec). A server MAY honor		register new schemes/tokens, see sec 3.5 of the spec). A server MAY honor
	the client's encoding request. When a response is encoded, the server		the client's encoding request. When a response is encoded, the server
	@@ -661,9 +661,10 @@ Content Encoding

	## Supported content encodings		## Supported content encodings

	The 'deflate' and 'gzip' content encodings are supported by libcurl. Both		The 'deflate', 'gzip' and 'br' content encodings are supported by libcurl.
	regular and chunked transfers work fine. The zlib library is required for		Both regular and chunked transfers work fine. The zlib library is required
	this feature.		for the 'deflate' and 'gzip' encodings, while the brotli decoding library is
			for the 'br' encoding.

	## The libcurl interface		## The libcurl interface

	@@ -674,10 +675,10 @@ Content Encoding
	where string is the intended value of the Accept-Encoding header.		where string is the intended value of the Accept-Encoding header.

	Currently, libcurl does support multiple encodings but only		Currently, libcurl does support multiple encodings but only
	understands how to process responses that use the "deflate" or "gzip"		understands how to process responses that use the "deflate", "gzip" and/or
	Content-Encoding, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5]		"br" content encodings, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5]
	that will work (besides "identity," which does nothing) are "deflate"		that will work (besides "identity," which does nothing) are "deflate",
	and "gzip". If a response is encoded using the "compress" or methods,		"gzip" and "br". If a response is encoded using the "compress" or methods,
	libcurl will return an error indicating that the response could		libcurl will return an error indicating that the response could
	not be decoded. If <string> is NULL no Accept-Encoding header is generated.		not be decoded. If <string> is NULL no Accept-Encoding header is generated.
	If <string> is a zero-length string, then an Accept-Encoding header		If <string> is a zero-length string, then an Accept-Encoding header

Admin message