Commit 11bf1796 authored by Patrick Monnerat's avatar Patrick Monnerat
Browse files

HTTP: implement Brotli content encoding 

This uses the brotli external library (https://github.com/google/brotli).
Brotli becomes a feature: additional curl_version_info() bit and
structure fields are provided for it and CURLVERSION_NOW bumped.

Tests 314 and 315 check Brotli content unencoding with correct and
erroneous data.

Some tests are updated to accomodate with the now configuration dependent
parameters of the Accept-Encoding header.
parent dbcced8e

configure.ac

+93 −0
Original line number Diff line number Diff line
@@ -148,6 +148,7 @@ dnl initialize all the info variables 
    curl_ssl_msg="no      (--with-{ssl,gnutls,nss,polarssl,mbedtls,cyassl,axtls,winssl,darwinssl} )"

    curl_ssh_msg="no      (--with-libssh2)"

   curl_zlib_msg="no      (--with-zlib)"

 curl_brotli_msg="no      (--with-brotli)"

    curl_gss_msg="no      (--with-gssapi)"

curl_tls_srp_msg="no      (--enable-tls-srp)"

    curl_res_msg="default (--enable-ares / --enable-threaded-resolver)"
@@ -975,6 +976,94 @@ dnl set variable for use in automakefile(s) 
AM_CONDITIONAL(HAVE_LIBZ, test x"$AMFIXLIB" = x1)

AC_SUBST(ZLIB_LIBS)



dnl **********************************************************************

dnl Check for the presence of BROTLI decoder libraries and headers

dnl **********************************************************************



dnl Brotli project home page: https://github.com/google/brotli



dnl Default to compiler & linker defaults for BROTLI files & libraries.

OPT_BROTLI=off

AC_ARG_WITH(brotli,dnl

AC_HELP_STRING([--with-brotli=PATH],[Where to look for brotli, PATH points to the BROTLI installation; when possible, set the PKG_CONFIG_PATH environment variable instead of using this option])

AC_HELP_STRING([--without-brotli], [disable BROTLI]),

  OPT_BROTLI=$withval)



if test X"$OPT_BROTLI" != Xno; then

  dnl backup the pre-brotli variables

  CLEANLDFLAGS="$LDFLAGS"

  CLEANCPPFLAGS="$CPPFLAGS"

  CLEANLIBS="$LIBS"



  case "$OPT_BROTLI" in

  yes)

    dnl --with-brotli (without path) used

    CURL_CHECK_PKGCONFIG(libbrotlidec)



    if test "$PKGCONFIG" != "no" ; then

      LIB_BROTLI=`$PKGCONFIG --libs-only-l libbrotlidec`

      LD_BROTLI=`$PKGCONFIG --libs-only-L libbrotlidec`

      CPP_BROTLI=`$PKGCONFIG --cflags-only-I libbrotlidec`

      version=`$PKGCONFIG --modversion libbrotlidec`

      DIR_BROTLI=`echo $LD_BROTLI | $SED -e 's/-L//'`

    fi



    ;;

  off)

    dnl no --with-brotli option given, just check default places

    ;;

  *)

    dnl use the given --with-brotli spot

    PREFIX_BROTLI=$OPT_BROTLI

    ;;

  esac



  dnl if given with a prefix, we set -L and -I based on that

  if test -n "$PREFIX_BROTLI"; then

    LIB_BROTLI="-lbrotlidec"

    LD_BROTLI=-L${PREFIX_BROTLI}/lib$libsuff

    CPP_BROTLI=-I${PREFIX_BROTLI}/include

    DIR_BROTLI=${PREFIX_BROTLI}/lib$libsuff

  fi



  LDFLAGS="$LDFLAGS $LD_BROTLI"

  CPPFLAGS="$CPPFLAGS $CPP_BROTLI"

  LIBS="$LIB_BROTLI $LIBS"



  AC_CHECK_LIB(brotlidec, BrotliDecoderDecompress)



  AC_CHECK_HEADERS(brotli/decode.h,

    curl_brotli_msg="enabled (libbrotlidec)"

    HAVE_BROTLI=1

    AC_DEFINE(HAVE_BROTLI, 1, [if BROTLI is in use])

    AC_SUBST(HAVE_BROTLI, [1])

  )



  if test X"$OPT_BROTLI" != Xoff &&

     test "$HAVE_BROTLI" != "1"; then

    AC_MSG_ERROR([BROTLI libs and/or directories were not found where specified!])

  fi



  if test "$HAVE_BROTLI" = "1"; then

    if test -n "$DIR_BROTLI"; then

       dnl when the brotli shared libs were found in a path that the run-time

       dnl linker doesn't search through, we need to add it to LD_LIBRARY_PATH

       dnl to prevent further configure tests to fail due to this



       if test "x$cross_compiling" != "xyes"; then

         LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$DIR_BROTLI"

         export LD_LIBRARY_PATH

         AC_MSG_NOTICE([Added $DIR_BROTLI to LD_LIBRARY_PATH])

       fi

    fi

  else

    dnl no brotli, revert back to clean variables

    LDFLAGS=$CLEANLDFLAGS

    CPPFLAGS=$CLEANCPPFLAGS

    LIBS=$CLEANLIBS

  fi

fi



dnl **********************************************************************

dnl Check for LDAP

dnl **********************************************************************
@@ -3786,6 +3875,9 @@ fi 
if test "x$HAVE_LIBZ" = "x1"; then

  SUPPORT_FEATURES="$SUPPORT_FEATURES libz"

fi

if test "x$HAVE_BROTLI" = "x1"; then

  SUPPORT_FEATURES="$SUPPORT_FEATURES brotli"

fi

if test "x$USE_ARES" = "x1" -o "x$USE_THREADS_POSIX" = "x1" \

                            -o "x$USE_THREADS_WIN32" = "x1"; then

  SUPPORT_FEATURES="$SUPPORT_FEATURES AsynchDNS"
@@ -4003,6 +4095,7 @@ AC_MSG_NOTICE([Configured to build curl/libcurl: 
  SSL support:      ${curl_ssl_msg}

  SSH support:      ${curl_ssh_msg}

  zlib support:     ${curl_zlib_msg}

  brotli support:   ${curl_brotli_msg}

  GSS-API support:  ${curl_gss_msg}

  TLS-SRP support:  ${curl_tls_srp_msg}

  resolver:         ${curl_res_msg}

docs/INTERNALS.md

+11 −10
Original line number Diff line number Diff line
@@ -644,9 +644,9 @@ Content Encoding 
 [HTTP/1.1][4] specifies that a client may request that a server encode its

 response. This is usually used to compress a response using one (or more)

 encodings from a set of commonly available compression techniques. These

 schemes include 'deflate' (the zlib algorithm), 'gzip' and 'compress'. A

 client requests that the server perform an encoding by including an

 Accept-Encoding header in the request document. The value of the header

 schemes include 'deflate' (the zlib algorithm), 'gzip' 'br' (brotli) and

 'compress'. A client requests that the server perform an encoding by including

 an Accept-Encoding header in the request document. The value of the header

 should be one of the recognized tokens 'deflate', ... (there's a way to

 register new schemes/tokens, see sec 3.5 of the spec). A server MAY honor

 the client's encoding request. When a response is encoded, the server
@@ -661,9 +661,10 @@ Content Encoding 


## Supported content encodings



 The 'deflate' and 'gzip' content encodings are supported by libcurl. Both

 regular and chunked transfers work fine.  The zlib library is required for

 this feature.

 The 'deflate', 'gzip' and 'br' content encodings are supported by libcurl.

 Both regular and chunked transfers work fine.  The zlib library is required

 for the 'deflate' and 'gzip' encodings, while the brotli decoding library is

 for the 'br' encoding.



## The libcurl interface
@@ -674,10 +675,10 @@ Content Encoding 
 where string is the intended value of the Accept-Encoding header.



 Currently, libcurl does support multiple encodings but only

 understands how to process responses that use the "deflate" or "gzip"

 Content-Encoding, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5]

 that will work (besides "identity," which does nothing) are "deflate"

 and "gzip". If a response is encoded using the "compress" or methods,

 understands how to process responses that use the "deflate", "gzip" and/or

 "br" content encodings, so the only values for [`CURLOPT_ACCEPT_ENCODING`][5]

 that will work (besides "identity," which does nothing) are "deflate",

 "gzip" and "br". If a response is encoded using the "compress" or methods,

 libcurl will return an error indicating that the response could

 not be decoded.  If <string> is NULL no Accept-Encoding header is generated.

 If <string> is a zero-length string, then an Accept-Encoding header

docs/RESOURCES

+2 −0
Original line number Diff line number Diff line
@@ -81,3 +81,5 @@ This document lists documents and standards used by curl. 
  RFC 4616 - PLAIN authentication



  RFC 4954 - SMTP Authentication



  RFC 7932 - Brotli Compressed Data Format

docs/TODO

+4 −11
Original line number Diff line number Diff line
@@ -64,9 +64,8 @@ 
 5.4 HTTP Digest using SHA-256

 5.5 auth= in URLs

 5.6 Refuse "downgrade" redirects

 5.7 Brotli compression

 5.8 QUIC

 5.9 Leave secure cookies alone

 5.7 QUIC

 5.8 Leave secure cookies alone



 6. TELNET

 6.1 ditch stdin
@@ -514,13 +513,7 @@ This is not detailed in any FTP specification. 
 Consider a way to tell curl to refuse to "downgrade" protocol with a redirect

 and/or possibly a bit that refuses redirect to change protocol completely.



5.7 Brotli compression



 Brotli compression performs better than gzip and is being implemented by

 browsers and servers widely. The algorithm: https://github.com/google/brotli

 The Firefox bug: https://bugzilla.mozilla.org/show_bug.cgi?id=366559



5.8 QUIC

5.7 QUIC



 The standardization process of QUIC has been taken to the IETF and can be

 followed on the [IETF QUIC Mailing
@@ -530,7 +523,7 @@ This is not detailed in any FTP specification. 
 implemented. This, to allow other projects to benefit from the work and to

 thus broaden the interest and chance of others to participate.



5.9 Leave secure cookies alone

5.8 Leave secure cookies alone



 Non-secure origins (HTTP sites) should not be allowed to set or modify

 cookies with the 'secure' property:

docs/libcurl/curl_version_info.3

+8 −0
Original line number Diff line number Diff line
@@ -72,6 +72,12 @@ typedef struct { 


  const char *libssh_version; /* human readable string */



  /* when 'age' is 4 or higher (7.57.0 or later), the members below also

     exist  */

  unsigned int brotli_ver_num; /* Numeric Brotli version

                                  (MAJOR << 24) | (MINOR << 12) | PATCH */

  const char *brotli_version; /* human readable string. */



} curl_version_info_data;

.fi
@@ -160,6 +166,8 @@ libcurl was built with support for HTTPS-proxy. 
libcurl was built with multiple SSL backends. For details, see

\fIcurl_global_sslset(3)\fP.

(Added in 7.56.0)

.IP CURL_VERSION_BROTLI

supports HTTP Brotli content encoding using libbrotlidec (Added in 7.57.0)

.RE

\fIssl_version\fP is an ASCII string for the OpenSSL version used. If libcurl

has no SSL support, this is NULL.