Commit 8541d02c authored by Patrick Monnerat's avatar Patrick Monnerat
Browse files

psl: use latest psl and refresh it periodically 

The latest psl is cached in the multi or share handle. It is refreshed
before use after 72 hours.
New share lock CURL_LOCK_DATA_PSL controls the psl cache sharing.
If the latest psl is not available, the builtin psl is used.

Reported-by: Yaakov Selkowitz
Fixes #2553
Closes #2601
parent 536e9f82

docs/libcurl/curl_share_setopt.3

+9 −0
Original line number Diff line number Diff line
@@ -87,6 +87,15 @@ existed before this. 


Note that when you use the multi interface, all easy handles added to the same

multi handle will share connection cache by default without using this option.

.IP CURL_LOCK_DATA_PSL

The Public Suffix List stored in the share object is made available to all

easy handle bound to the later. Since the Public Suffix List is periodically

refreshed, this avoids updates in too many different contexts.



\fBCURL_LOCK_DATA_PSL\fP exists since 7.61.0.



Note that when you use the multi interface, all easy handles added to the same

multi handle will share PSL cache by default without using this option.

.RE

.IP CURLSHOPT_UNSHARE

This option does the opposite of \fICURLSHOPT_SHARE\fP. It specifies that

docs/libcurl/symbols-in-versions

+1 −0
Original line number Diff line number Diff line
@@ -780,6 +780,7 @@ CURL_LOCK_DATA_CONNECT 7.10.3 
CURL_LOCK_DATA_COOKIE           7.10.3

CURL_LOCK_DATA_DNS              7.10.3

CURL_LOCK_DATA_NONE             7.10.3

CURL_LOCK_DATA_PSL              7.61.0

CURL_LOCK_DATA_SHARE            7.10.4

CURL_LOCK_DATA_SSL_SESSION      7.10.3

CURL_LOCK_TYPE_CONNECT          7.10          -           7.10.2

include/curl/curl.h

+1 −0
Original line number Diff line number Diff line
@@ -2582,6 +2582,7 @@ typedef enum { 
  CURL_LOCK_DATA_DNS,

  CURL_LOCK_DATA_SSL_SESSION,

  CURL_LOCK_DATA_CONNECT,

  CURL_LOCK_DATA_PSL,

  CURL_LOCK_DATA_LAST

} curl_lock_data;

lib/Makefile.inc

+2 −2
Original line number Diff line number Diff line
@@ -54,7 +54,7 @@ LIB_CFILES = file.c timeval.c base64.c hostip.c progress.c formdata.c \ 
  http_ntlm.c curl_ntlm_wb.c curl_ntlm_core.c curl_sasl.c rand.c        \

  curl_multibyte.c hostcheck.c conncache.c pipeline.c dotdot.c          \

  x509asn1.c http2.c smb.c curl_endian.c curl_des.c system_win32.c      \

  mime.c sha256.c setopt.c curl_path.c curl_ctype.c curl_range.c

  mime.c sha256.c setopt.c curl_path.c curl_ctype.c curl_range.c psl.c



LIB_HFILES = arpa_telnet.h netrc.h file.h timeval.h hostip.h progress.h \

  formdata.h cookie.h http.h sendf.h ftp.h url.h dict.h if2ip.h         \
@@ -74,7 +74,7 @@ LIB_HFILES = arpa_telnet.h netrc.h file.h timeval.h hostip.h progress.h \ 
  curl_setup_once.h multihandle.h setup-vms.h pipeline.h dotdot.h       \

  x509asn1.h http2.h sigpipe.h smb.h curl_endian.h curl_des.h           \

  curl_printf.h system_win32.h rand.h mime.h curl_sha256.h setopt.h     \

  curl_path.h curl_ctype.h curl_range.h

  curl_path.h curl_ctype.h curl_range.h psl.h



LIB_RCFILES = libcurl.rc

lib/cookie.c

+23 −22
Original line number Diff line number Diff line
@@ -84,12 +84,9 @@ Example set of cookies: 


#if !defined(CURL_DISABLE_HTTP) && !defined(CURL_DISABLE_COOKIES)



#ifdef USE_LIBPSL

# include <libpsl.h>

#endif



#include "urldata.h"

#include "cookie.h"

#include "psl.h"

#include "strtok.h"

#include "sendf.h"

#include "slist.h"
@@ -406,6 +403,12 @@ static void remove_expired(struct CookieInfo *cookies) 
  }

}



/* Make sure domain contains a dot or is localhost. */

static bool bad_domain(const char *domain)

{

  return !strchr(domain, '.') && !strcasecompare(domain, "localhost");

}



/****************************************************************************

 *

 * Curl_cookie_add()
@@ -442,10 +445,6 @@ Curl_cookie_add(struct Curl_easy *data, 
  bool badcookie = FALSE; /* cookies are good by default. mmmmm yummy */

  size_t myhash;



#ifdef USE_LIBPSL

  const psl_ctx_t *psl;

#endif



#ifdef CURL_DISABLE_VERBOSE_STRINGS

  (void)data;

#endif
@@ -585,13 +584,8 @@ Curl_cookie_add(struct Curl_easy *data, 
           * TLD or otherwise "protected" suffix. To reduce risk, we require a

           * dot OR the exact host name being "localhost".

           */

          {

            const char *dotp;

            /* check for more dots */

            dotp = strchr(whatptr, '.');

            if(!dotp && !strcasecompare("localhost", whatptr))

          if(bad_domain(whatptr))

            domain = ":";

          }

#endif



          is_ip = isip(domain ? domain : whatptr);
@@ -890,14 +884,21 @@ Curl_cookie_add(struct Curl_easy *data, 
    remove_expired(c);



#ifdef USE_LIBPSL

  /* Check if the domain is a Public Suffix and if yes, ignore the cookie.

     This needs a libpsl compiled with builtin data. */

  /* Check if the domain is a Public Suffix and if yes, ignore the cookie. */

  if(domain && co->domain && !isip(co->domain)) {

    psl = psl_builtin();

    if(psl && !psl_is_cookie_domain_acceptable(psl, domain, co->domain)) {

      infof(data,

            "cookie '%s' dropped, domain '%s' must not set cookies for '%s'\n",

            co->name, domain, co->domain);

    const psl_ctx_t *psl = Curl_psl_use(data);

    int acceptable;



    if(psl) {

      acceptable = psl_is_cookie_domain_acceptable(psl, domain, co->domain);

      Curl_psl_release(data);

    }

    else

      acceptable = !bad_domain(domain);



    if(!acceptable) {

      infof(data, "cookie '%s' dropped, domain '%s' must not "

                  "set cookies for '%s'\n", co->name, domain, co->domain);

      freecookie(co);

      return NULL;

    }