diff options
Diffstat (limited to 'content/fetchers/curl.c')
| -rw-r--r-- | content/fetchers/curl.c | 1257 |
1 files changed, 921 insertions, 336 deletions
diff --git a/content/fetchers/curl.c b/content/fetchers/curl.c index a6146edb3..680e60456 100644 --- a/content/fetchers/curl.c +++ b/content/fetchers/curl.c @@ -1,7 +1,7 @@ /* - * Copyright 2006 Daniel Silverstone <dsilvers@digital-scurf.org> + * Copyright 2006-2019 Daniel Silverstone <dsilvers@digital-scurf.org> + * Copyright 2010-2018 Vincent Sanders <vince@netsurf-browser.org> * Copyright 2007 James Bursa <bursa@users.sourceforge.net> - * Copyright 2003 Phil Mellor <monkeyson@users.sourceforge.net> * * This file is part of NetSurf. * @@ -38,12 +38,12 @@ #include <strings.h> #include <time.h> #include <sys/stat.h> -#include <openssl/ssl.h> #include <libwapcaplet/libwapcaplet.h> #include <nsutils/time.h> #include "utils/corestrings.h" +#include "utils/hashmap.h" #include "utils/nsoption.h" #include "utils/log.h" #include "utils/messages.h" @@ -61,11 +61,149 @@ #include "content/fetchers/curl.h" #include "content/urldb.h" -/** maximum number of progress notifications per second */ +/** + * maximum number of progress notifications per second + */ #define UPDATES_PER_SECOND 2 -/** maximum number of X509 certificates in chain for TLS connection */ -#define MAX_CERTS 10 +/** + * The ciphersuites the browser is prepared to use for TLS1.3 + */ +#define CIPHER_SUITES \ + "TLS_AES_256_GCM_SHA384:" \ + "TLS_CHACHA20_POLY1305_SHA256:" \ + "TLS_AES_128_GCM_SHA256" + +/** + * The ciphersuites the browser is prepared to use for TLS<1.3 + */ +#define CIPHER_LIST \ + /* disable everything */ \ + "-ALL:" \ + /* enable TLSv1.2 ECDHE AES GCM suites */ \ + "EECDH+AESGCM+TLSv1.2:" \ + /* enable ECDHE CHACHA20/POLY1305 suites */ \ + "EECDH+CHACHA20:" \ + /* Sort above by strength */ \ + "@STRENGTH:" \ + /* enable ECDHE (auth=RSA, mac=SHA1) AES CBC suites */ \ + "EECDH+aRSA+AES+SHA1" + +/** + * The legacy cipher suites the browser is prepared to use for TLS<1.3 + */ +#define CIPHER_LIST_LEGACY \ + /* as above */ \ + CIPHER_LIST":" \ + /* enable (non-PFS) RSA AES GCM suites */ \ + "RSA+AESGCM:" \ + /* enable (non-PFS) RSA (mac=SHA1) AES CBC suites */ \ + "RSA+AES+SHA1" + +/* Open SSL compatability for certificate handling */ +#ifdef WITH_OPENSSL + +#include <openssl/ssl.h> +#include <openssl/x509v3.h> + +#else /* WITH_OPENSSL */ + +typedef char X509; + +static void X509_free(X509 *cert) +{ + free(cert); +} + +#endif /* WITH_OPENSSL */ + +/* SSL certificate chain cache */ + +/* We're only interested in the hostname and port */ +static uint32_t +curl_fetch_ssl_key_hash(void *key) +{ + nsurl *url = key; + lwc_string *hostname = nsurl_get_component(url, NSURL_HOST); + lwc_string *port = nsurl_get_component(url, NSURL_PORT); + uint32_t hash; + + if (port == NULL) + port = lwc_string_ref(corestring_lwc_443); + + hash = lwc_string_hash_value(hostname) ^ lwc_string_hash_value(port); + + lwc_string_unref(hostname); + lwc_string_unref(port); + + return hash; +} + +/* We only compare the hostname and port */ +static bool +curl_fetch_ssl_key_eq(void *key1, void *key2) +{ + nsurl *url1 = key1; + nsurl *url2 = key2; + lwc_string *hostname1 = nsurl_get_component(url1, NSURL_HOST); + lwc_string *hostname2 = nsurl_get_component(url2, NSURL_HOST); + lwc_string *port1 = nsurl_get_component(url1, NSURL_PORT); + lwc_string *port2 = nsurl_get_component(url2, NSURL_PORT); + bool iseq = false; + + if (port1 == NULL) + port1 = lwc_string_ref(corestring_lwc_443); + if (port2 == NULL) + port2 = lwc_string_ref(corestring_lwc_443); + + if (lwc_string_isequal(hostname1, hostname2, &iseq) != lwc_error_ok || + iseq == false) + goto out; + + iseq = false; + if (lwc_string_isequal(port1, port2, &iseq) != lwc_error_ok) + goto out; + +out: + lwc_string_unref(hostname1); + lwc_string_unref(hostname2); + lwc_string_unref(port1); + lwc_string_unref(port2); + + return iseq; +} + +static void * +curl_fetch_ssl_value_alloc(void *key) +{ + struct cert_chain *out; + + if (cert_chain_alloc(0, &out) != NSERROR_OK) { + return NULL; + } + + return out; +} + +static void +curl_fetch_ssl_value_destroy(void *value) +{ + struct cert_chain *chain = value; + if (cert_chain_free(chain) != NSERROR_OK) { + NSLOG(netsurf, WARNING, "Problem freeing SSL certificate chain"); + } +} + +static hashmap_parameters_t curl_fetch_ssl_hashmap_parameters = { + .key_clone = (hashmap_key_clone_t)nsurl_ref, + .key_destroy = (hashmap_key_destroy_t)nsurl_unref, + .key_eq = curl_fetch_ssl_key_eq, + .key_hash = curl_fetch_ssl_key_hash, + .value_alloc = curl_fetch_ssl_value_alloc, + .value_destroy = curl_fetch_ssl_value_destroy, +}; + +static hashmap_t *curl_fetch_ssl_hashmap = NULL; /** SSL certificate info */ struct cert_info { @@ -73,15 +211,36 @@ struct cert_info { long err; /**< OpenSSL error code */ }; +#if LIBCURL_VERSION_NUM >= 0x072000 /* 7.32.0 depricated CURLOPT_PROGRESSFUNCTION*/ +#define NSCURLOPT_PROGRESS_FUNCTION CURLOPT_XFERINFOFUNCTION +#define NSCURLOPT_PROGRESS_DATA CURLOPT_XFERINFODATA +#define NSCURL_PROGRESS_T curl_off_t +#else +#define NSCURLOPT_PROGRESS_FUNCTION CURLOPT_PROGRESSFUNCTION +#define NSCURLOPT_PROGRESS_DATA CURLOPT_PROGRESSDATA +#define NSCURL_PROGRESS_T double +#endif + +#if LIBCURL_VERSION_NUM >= 0x073800 /* 7.56.0 depricated curl_formadd */ +#define NSCURL_POSTDATA_T curl_mime +#define NSCURL_POSTDATA_CURLOPT CURLOPT_MIMEPOST +#define NSCURL_POSTDATA_FREE(x) curl_mime_free(x) +#else +#define NSCURL_POSTDATA_T struct curl_httppost +#define NSCURL_POSTDATA_CURLOPT CURLOPT_HTTPPOST +#define NSCURL_POSTDATA_FREE(x) curl_formfree(x) +#endif + /** Information for a single fetch. */ struct curl_fetch_info { struct fetch *fetch_handle; /**< The fetch handle we're parented by. */ CURL * curl_handle; /**< cURL handle if being fetched, or 0. */ + bool sent_ssl_chain; /**< Have we tried to send the SSL chain */ bool had_headers; /**< Headers have been processed. */ bool abort; /**< Abort requested. */ bool stopped; /**< Download stopped on purpose. */ bool only_2xx; /**< Only HTTP 2xx responses acceptable. */ - bool downgrade_tls; /**< Downgrade to TLS <= 1.0 */ + bool downgrade_tls; /**< Downgrade to TLS 1.2 */ nsurl *url; /**< URL of this fetch. */ lwc_string *host; /**< The hostname of this fetch. */ struct curl_slist *headers; /**< List of request headers. */ @@ -89,12 +248,14 @@ struct curl_fetch_info { unsigned long content_length; /**< Response Content-Length, or 0. */ char *cookie_string; /**< Cookie string for this fetch */ char *realm; /**< HTTP Auth Realm */ - char *post_urlenc; /**< Url encoded POST string, or 0. */ + struct fetch_postdata *postdata; /**< POST data */ + NSCURL_POSTDATA_T *curl_postdata; /**< POST data in curl representation */ + long http_code; /**< HTTP result code from cURL. */ - struct curl_httppost *post_multipart; /**< Multipart post data, or 0. */ + uint64_t last_progress_update; /**< Time of last progress update */ int cert_depth; /**< deepest certificate in use */ - struct cert_info cert_data[MAX_CERTS]; /**< HTTPS certificate data */ + struct cert_info cert_data[MAX_CERT_DEPTH]; /**< HTTPS certificate data */ }; /** curl handle cache entry */ @@ -127,28 +288,9 @@ static char fetch_error_buffer[CURL_ERROR_SIZE]; /** Proxy authentication details. */ static char fetch_proxy_userpwd[100]; +/** Interlock to prevent initiation during callbacks */ +static bool inside_curl = false; -/* OpenSSL 1.0.x to 1.1.0 certificate reference counting changed - * LibreSSL declares its OpenSSL version as 2.1 but only supports the old way - */ -#if (defined(LIBRESSL_VERSION_NUMBER) || (OPENSSL_VERSION_NUMBER < 0x1010000fL)) -static int ns_X509_up_ref(X509 *cert) -{ - cert->references++; - return 1; -} - -static void ns_X509_free(X509 *cert) -{ - cert->references--; - if (cert->references == 0) { - X509_free(cert); - } -} -#else -#define ns_X509_up_ref X509_up_ref -#define ns_X509_free X509_free -#endif /** * Initialise a cURL fetcher. @@ -188,6 +330,10 @@ static void fetch_curl_finalise(lwc_string *scheme) "curl_multi_cleanup failed: ignoring"); curl_global_cleanup(); + + NSLOG(netsurf, DEBUG, "Cleaning up SSL cert chain hashmap"); + hashmap_destroy(curl_fetch_ssl_hashmap); + curl_fetch_ssl_hashmap = NULL; } /* Free anything remaining in the cached curl handle ring */ @@ -213,85 +359,95 @@ static bool fetch_curl_can_fetch(const nsurl *url) } + /** - * Convert a list of struct ::fetch_multipart_data to a list of - * struct curl_httppost for libcurl. + * allocate postdata */ -static struct curl_httppost * -fetch_curl_post_convert(const struct fetch_multipart_data *control) +static struct fetch_postdata * +fetch_curl_alloc_postdata(const char *post_urlenc, + const struct fetch_multipart_data *post_multipart) { - struct curl_httppost *post = 0, *last = 0; - CURLFORMcode code; - nserror ret; - - for (; control; control = control->next) { - if (control->file) { - char *leafname = NULL; - ret = guit->file->basename(control->value, &leafname, NULL); - if (ret != NSERROR_OK) { - continue; + struct fetch_postdata *postdata; + postdata = calloc(1, sizeof(struct fetch_postdata)); + if (postdata != NULL) { + + if (post_urlenc) { + postdata->type = FETCH_POSTDATA_URLENC; + postdata->data.urlenc = strdup(post_urlenc); + if (postdata->data.urlenc == NULL) { + free(postdata); + postdata = NULL; } - - /* We have to special case filenames of "", so curl - * a) actually attempts the fetch and - * b) doesn't attempt to open the file "" - */ - if (control->value[0] == '\0') { - /* dummy buffer - needs to be static so - * pointer's still valid when we go out - * of scope (not that libcurl should be - * attempting to access it, of course). - */ - static char buf; - - code = curl_formadd(&post, &last, - CURLFORM_COPYNAME, control->name, - CURLFORM_BUFFER, control->value, - /* needed, as basename("") == "." */ - CURLFORM_FILENAME, "", - CURLFORM_BUFFERPTR, &buf, - CURLFORM_BUFFERLENGTH, 0, - CURLFORM_CONTENTTYPE, - "application/octet-stream", - CURLFORM_END); - if (code != CURL_FORMADD_OK) - NSLOG(netsurf, INFO, - "curl_formadd: %d (%s)", code, - control->name); - } else { - char *mimetype = guit->fetch->mimetype(control->value); - code = curl_formadd(&post, &last, - CURLFORM_COPYNAME, control->name, - CURLFORM_FILE, control->rawfile, - CURLFORM_FILENAME, leafname, - CURLFORM_CONTENTTYPE, - (mimetype != 0 ? mimetype : "text/plain"), - CURLFORM_END); - if (code != CURL_FORMADD_OK) - NSLOG(netsurf, INFO, - "curl_formadd: %d (%s=%s)", - code, - control->name, - control->value); - free(mimetype); + } else if (post_multipart) { + postdata->type = FETCH_POSTDATA_MULTIPART; + postdata->data.multipart = fetch_multipart_data_clone(post_multipart); + if (postdata->data.multipart == NULL) { + free(postdata); + postdata = NULL; } - free(leafname); - } - else { - code = curl_formadd(&post, &last, - CURLFORM_COPYNAME, control->name, - CURLFORM_COPYCONTENTS, control->value, - CURLFORM_END); - if (code != CURL_FORMADD_OK) - NSLOG(netsurf, INFO, - "curl_formadd: %d (%s=%s)", code, - control->name, control->value); + } else { + postdata->type = FETCH_POSTDATA_NONE; } } + return postdata; +} - return post; +/** + * free postdata + */ +static void fetch_curl_free_postdata(struct fetch_postdata *postdata) +{ + if (postdata != NULL) { + switch (postdata->type) { + case FETCH_POSTDATA_NONE: + break; + case FETCH_POSTDATA_URLENC: + free(postdata->data.urlenc); + break; + case FETCH_POSTDATA_MULTIPART: + fetch_multipart_data_destroy(postdata->data.multipart); + break; + } + + free(postdata); + } } +/** + *construct a new fetch structure + */ +static struct curl_fetch_info *fetch_alloc(void) +{ + struct curl_fetch_info *fetch; + fetch = malloc(sizeof (*fetch)); + if (fetch == NULL) + return NULL; + + fetch->curl_handle = NULL; + fetch->sent_ssl_chain = false; + fetch->had_headers = false; + fetch->abort = false; + fetch->stopped = false; + fetch->only_2xx = false; + fetch->downgrade_tls = false; + fetch->headers = NULL; + fetch->url = NULL; + fetch->host = NULL; + fetch->location = NULL; + fetch->content_length = 0; + fetch->http_code = 0; + fetch->cookie_string = NULL; + fetch->realm = NULL; + fetch->last_progress_update = 0; + fetch->postdata = NULL; + fetch->curl_postdata = NULL; + + /* Clear certificate chain data */ + memset(fetch->cert_data, 0, sizeof(fetch->cert_data)); + fetch->cert_depth = -1; + + return fetch; +} /** * Start fetching data for the given URL. @@ -327,45 +483,22 @@ fetch_curl_setup(struct fetch *parent_fetch, struct curl_slist *slist; int i; - fetch = malloc(sizeof (*fetch)); + fetch = fetch_alloc(); if (fetch == NULL) - return 0; - - fetch->fetch_handle = parent_fetch; + return NULL; NSLOG(netsurf, INFO, "fetch %p, url '%s'", fetch, nsurl_access(url)); - /* construct a new fetch structure */ - fetch->curl_handle = NULL; - fetch->had_headers = false; - fetch->abort = false; - fetch->stopped = false; fetch->only_2xx = only_2xx; fetch->downgrade_tls = downgrade_tls; - fetch->headers = NULL; + fetch->fetch_handle = parent_fetch; fetch->url = nsurl_ref(url); fetch->host = nsurl_get_component(url, NSURL_HOST); - fetch->location = NULL; - fetch->content_length = 0; - fetch->http_code = 0; - fetch->cookie_string = NULL; - fetch->realm = NULL; - fetch->post_urlenc = NULL; - fetch->post_multipart = NULL; - if (post_urlenc) { - fetch->post_urlenc = strdup(post_urlenc); - } else if (post_multipart) { - fetch->post_multipart = fetch_curl_post_convert(post_multipart); + if (fetch->host == NULL) { + goto failed; } - fetch->last_progress_update = 0; - - /* TLS defaults */ - memset(fetch->cert_data, 0, sizeof(fetch->cert_data)); - fetch->cert_depth = -1; - - if ((fetch->host == NULL) || - (post_multipart != NULL && fetch->post_multipart == NULL) || - (post_urlenc != NULL && fetch->post_urlenc == NULL)) { + fetch->postdata = fetch_curl_alloc_postdata(post_urlenc, post_multipart); + if (fetch->postdata == NULL) { goto failed; } @@ -418,15 +551,141 @@ failed: lwc_string_unref(fetch->host); nsurl_unref(fetch->url); - free(fetch->post_urlenc); - if (fetch->post_multipart) - curl_formfree(fetch->post_multipart); + fetch_curl_free_postdata(fetch->postdata); curl_slist_free_all(fetch->headers); free(fetch); return NULL; } +#ifdef WITH_OPENSSL + +/** + * Retrieve the ssl cert chain for the fetch, creating a blank one if needed + */ +static struct cert_chain * +fetch_curl_get_cached_chain(struct curl_fetch_info *f) +{ + struct cert_chain *chain; + + chain = hashmap_lookup(curl_fetch_ssl_hashmap, f->url); + if (chain == NULL) { + chain = hashmap_insert(curl_fetch_ssl_hashmap, f->url); + } + + return chain; +} + +/** + * Report the certificate information in the fetch to the users + */ +static void +fetch_curl_store_certs_in_cache(struct curl_fetch_info *f) +{ + size_t depth; + BIO *mem; + BUF_MEM *buf[MAX_CERT_DEPTH]; + struct cert_chain chain, *cached_chain; + struct cert_info *certs; + + memset(&chain, 0, sizeof(chain)); + + certs = f->cert_data; + chain.depth = f->cert_depth + 1; /* 0 indexed certificate depth */ + + for (depth = 0; depth < chain.depth; depth++) { + if (certs[depth].cert == NULL) { + /* This certificate is missing, skip it */ + chain.certs[depth].err = SSL_CERT_ERR_CERT_MISSING; + continue; + } + + /* error code (if any) */ + switch (certs[depth].err) { + case X509_V_OK: + chain.certs[depth].err = SSL_CERT_ERR_OK; + break; + + case X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT: + /* fallthrough */ + case X509_V_ERR_UNABLE_TO_DECODE_ISSUER_PUBLIC_KEY: + chain.certs[depth].err = SSL_CERT_ERR_BAD_ISSUER; + break; + + case X509_V_ERR_UNABLE_TO_DECRYPT_CERT_SIGNATURE: + /* fallthrough */ + case X509_V_ERR_UNABLE_TO_DECRYPT_CRL_SIGNATURE: + /* fallthrough */ + case X509_V_ERR_CERT_SIGNATURE_FAILURE: + /* fallthrough */ + case X509_V_ERR_CRL_SIGNATURE_FAILURE: + chain.certs[depth].err = SSL_CERT_ERR_BAD_SIG; + break; + + case X509_V_ERR_CERT_NOT_YET_VALID: + /* fallthrough */ + case X509_V_ERR_CRL_NOT_YET_VALID: + chain.certs[depth].err = SSL_CERT_ERR_TOO_YOUNG; + break; + + case X509_V_ERR_CERT_HAS_EXPIRED: + /* fallthrough */ + case X509_V_ERR_CRL_HAS_EXPIRED: + chain.certs[depth].err = SSL_CERT_ERR_TOO_OLD; + break; + + case X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT: + chain.certs[depth].err = SSL_CERT_ERR_SELF_SIGNED; + break; + + case X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN: + chain.certs[depth].err = SSL_CERT_ERR_CHAIN_SELF_SIGNED; + break; + + case X509_V_ERR_CERT_REVOKED: + chain.certs[depth].err = SSL_CERT_ERR_REVOKED; + break; + + case X509_V_ERR_HOSTNAME_MISMATCH: + chain.certs[depth].err = SSL_CERT_ERR_HOSTNAME_MISMATCH; + break; + + default: + chain.certs[depth].err = SSL_CERT_ERR_UNKNOWN; + break; + } + + /* + * get certificate in Distinguished Encoding Rules (DER) format. + */ + mem = BIO_new(BIO_s_mem()); + i2d_X509_bio(mem, certs[depth].cert); + BIO_get_mem_ptr(mem, &buf[depth]); + (void) BIO_set_close(mem, BIO_NOCLOSE); + BIO_free(mem); + + chain.certs[depth].der = (uint8_t *)buf[depth]->data; + chain.certs[depth].der_length = buf[depth]->length; + } + + /* Now dup that chain into the cache */ + cached_chain = fetch_curl_get_cached_chain(f); + if (cert_chain_dup_into(&chain, cached_chain) != NSERROR_OK) { + /* Something went wrong storing the chain, give up */ + hashmap_remove(curl_fetch_ssl_hashmap, f->url); + } + + /* release the openssl memory buffer */ + for (depth = 0; depth < chain.depth; depth++) { + if (chain.certs[depth].err == SSL_CERT_ERR_CERT_MISSING) { + continue; + } + if (buf[depth] != NULL) { + BUF_MEM_free(buf[depth]); + } + } +} + /** * OpenSSL Certificate verification callback * @@ -455,24 +714,24 @@ fetch_curl_verify_callback(int verify_ok, X509_STORE_CTX *x509_ctx) depth = X509_STORE_CTX_get_error_depth(x509_ctx); fetch = X509_STORE_CTX_get_app_data(x509_ctx); - /* record the max depth */ - if (depth > fetch->cert_depth) { - fetch->cert_depth = depth; - } - /* certificate chain is excessively deep so fail verification */ - if (depth >= MAX_CERTS) { + if (depth >= MAX_CERT_DEPTH) { X509_STORE_CTX_set_error(x509_ctx, X509_V_ERR_CERT_CHAIN_TOO_LONG); return 0; } + /* record the max depth */ + if (depth > fetch->cert_depth) { + fetch->cert_depth = depth; + } + /* save the certificate by incrementing the reference count and * keeping a pointer. */ if (!fetch->cert_data[depth].cert) { fetch->cert_data[depth].cert = X509_STORE_CTX_get_current_cert(x509_ctx); - ns_X509_up_ref(fetch->cert_data[depth].cert); + X509_up_ref(fetch->cert_data[depth].cert); fetch->cert_data[depth].err = X509_STORE_CTX_get_error(x509_ctx); } @@ -506,16 +765,30 @@ fetch_curl_verify_callback(int verify_ok, X509_STORE_CTX *x509_ctx) */ static int fetch_curl_cert_verify_callback(X509_STORE_CTX *x509_ctx, void *parm) { + struct curl_fetch_info *f = (struct curl_fetch_info *) parm; int ok; + X509_VERIFY_PARAM *vparam; + + /* Configure the verification parameters to include hostname */ + vparam = X509_STORE_CTX_get0_param(x509_ctx); + X509_VERIFY_PARAM_set_hostflags(vparam, X509_CHECK_FLAG_NO_PARTIAL_WILDCARDS); + + ok = X509_VERIFY_PARAM_set1_host(vparam, + lwc_string_data(f->host), + lwc_string_length(f->host)); /* Store fetch struct in context for verify callback */ - ok = X509_STORE_CTX_set_app_data(x509_ctx, parm); + if (ok) { + ok = X509_STORE_CTX_set_app_data(x509_ctx, parm); + } /* verify the certificate chain using standard call */ if (ok) { ok = X509_verify_cert(x509_ctx); } + fetch_curl_store_certs_in_cache(f); + return ok; } @@ -533,7 +806,8 @@ fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, void *parm) { struct curl_fetch_info *f = (struct curl_fetch_info *) parm; SSL_CTX *sslctx = _sslctx; - long options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3; + long options = SSL_OP_ALL | SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | + SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1; /* set verify callback for each certificate in chain */ SSL_CTX_set_verify(sslctx, SSL_VERIFY_PEER, fetch_curl_verify_callback); @@ -544,12 +818,9 @@ fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, void *parm) parm); if (f->downgrade_tls) { - /* Disable TLS 1.1/1.2 if the server can't cope with them */ -#ifdef SSL_OP_NO_TLSv1_1 - options |= SSL_OP_NO_TLSv1_1; -#endif -#ifdef SSL_OP_NO_TLSv1_2 - options |= SSL_OP_NO_TLSv1_2; + /* Disable TLS 1.3 if the server can't cope with it */ +#ifdef SSL_OP_NO_TLSv1_3 + options |= SSL_OP_NO_TLSv1_3; #endif #ifdef SSL_MODE_SEND_FALLBACK_SCSV /* Ensure server rejects the connection if downgraded too far */ @@ -559,10 +830,334 @@ fetch_curl_sslctxfun(CURL *curl_handle, void *_sslctx, void *parm) SSL_CTX_set_options(sslctx, options); +#ifdef SSL_OP_NO_TICKET + SSL_CTX_clear_options(sslctx, SSL_OP_NO_TICKET); +#endif + return CURLE_OK; } +#endif /* WITH_OPENSSL */ + + +/** + * Report the certificate information in the fetch to the users + */ +static void +fetch_curl_report_certs_upstream(struct curl_fetch_info *f) +{ + fetch_msg msg; + struct cert_chain *chain; + + chain = hashmap_lookup(curl_fetch_ssl_hashmap, f->url); + + if (chain != NULL) { + msg.type = FETCH_CERTS; + msg.data.chain = chain; + + fetch_send_callback(&msg, f->fetch_handle); + } + + f->sent_ssl_chain = true; +} + +#if LIBCURL_VERSION_NUM >= 0x073800 /* 7.56.0 depricated curl_formadd */ + +/** + * curl mime data context + */ +struct curl_mime_ctx { + char *buffer; + curl_off_t size; + curl_off_t position; +}; + +static size_t mime_data_read_callback(char *buffer, size_t size, size_t nitems, void *arg) +{ + struct curl_mime_ctx *mctx = (struct curl_mime_ctx *) arg; + curl_off_t sz = mctx->size - mctx->position; + + nitems *= size; + if(sz > (curl_off_t)nitems) { + sz = nitems; + } + if(sz) { + memcpy(buffer, mctx->buffer + mctx->position, sz); + } + mctx->position += sz; + return sz; +} + +static int mime_data_seek_callback(void *arg, curl_off_t offset, int origin) +{ + struct curl_mime_ctx *mctx = (struct curl_mime_ctx *) arg; + + switch(origin) { + case SEEK_END: + offset += mctx->size; + break; + case SEEK_CUR: + offset += mctx->position; + break; + } + + if(offset < 0) { + return CURL_SEEKFUNC_FAIL; + } + mctx->position = offset; + return CURL_SEEKFUNC_OK; +} + +static void mime_data_free_callback(void *arg) +{ + struct curl_mime_ctx *mctx = (struct curl_mime_ctx *) arg; + free(mctx); +} + +/** + * Convert a POST data list to a libcurl curl_mime. + * + * \param chandle curl fetch handle. + * \param multipart limked list of struct ::fetch_multipart forming post data. + */ +static curl_mime * +fetch_curl_postdata_convert(CURL *chandle, + const struct fetch_multipart_data *multipart) +{ + curl_mime *cmime; + curl_mimepart *part; + CURLcode code = CURLE_OK; + size_t value_len; + + cmime = curl_mime_init(chandle); + if (cmime == NULL) { + NSLOG(netsurf, WARNING, "postdata conversion failed to curl mime context"); + return NULL; + } + + /* iterate post data */ + for (; multipart != NULL; multipart = multipart->next) { + part = curl_mime_addpart(cmime); + if (part == NULL) { + goto convert_failed; + } + + code = curl_mime_name(part, multipart->name); + if (code != CURLE_OK) { + goto convert_failed; + } + + value_len = strlen(multipart->value); + + if (multipart->file && value_len==0) { + /* file entries with no filename require special handling */ + code=curl_mime_data(part, multipart->value, value_len); + if (code != CURLE_OK) { + goto convert_failed; + } + + code = curl_mime_filename(part, ""); + if (code != CURLE_OK) { + goto convert_failed; + } + + code = curl_mime_type(part, "application/octet-stream"); + if (code != CURLE_OK) { + goto convert_failed; + } + + } else if(multipart->file) { + /* file entry */ + nserror ret; + char *leafname = NULL; + char *mimetype = NULL; + + code = curl_mime_filedata(part, multipart->rawfile); + if (code != CURLE_OK) { + goto convert_failed; + } + + ret = guit->file->basename(multipart->value, &leafname, NULL); + if (ret != NSERROR_OK) { + goto convert_failed; + } + code = curl_mime_filename(part, leafname); + free(leafname); + if (code != CURLE_OK) { + goto convert_failed; + } + + mimetype = guit->fetch->mimetype(multipart->value); + if (mimetype == NULL) { + mimetype=strdup("text/plain"); + } + if (mimetype == NULL) { + goto convert_failed; + } + code = curl_mime_type(part, mimetype); + free(mimetype); + if (code != CURLE_OK) { + goto convert_failed; + } + + } else { + /* make the curl mime reference the existing multipart + * data which requires use of a callback and context. + */ + struct curl_mime_ctx *cb_ctx; + cb_ctx = malloc(sizeof(struct curl_mime_ctx)); + if (cb_ctx == NULL) { + goto convert_failed; + } + cb_ctx->buffer = multipart->value; + cb_ctx->size = value_len; + cb_ctx->position = 0; + code = curl_mime_data_cb(part, + value_len, + mime_data_read_callback, + mime_data_seek_callback, + mime_data_free_callback, + cb_ctx); + if (code != CURLE_OK) { + free(cb_ctx); + goto convert_failed; + } + } + } + + return cmime; + +convert_failed: + NSLOG(netsurf, WARNING, "postdata conversion failed with curl code: %d", code); + curl_mime_free(cmime); + return NULL; +} + +#else /* LIBCURL_VERSION_NUM >= 0x073800 */ + +/** + * Convert a list of struct ::fetch_multipart_data to a list of + * struct curl_httppost for libcurl. + */ +static struct curl_httppost * +fetch_curl_postdata_convert(CURL *chandle, + const struct fetch_multipart_data *control) +{ + struct curl_httppost *post = NULL, *last = NULL; + CURLFORMcode code; + nserror ret; + + for (; control; control = control->next) { + if (control->file) { + char *leafname = NULL; + ret = guit->file->basename(control->value, &leafname, NULL); + if (ret != NSERROR_OK) { + continue; + } + + /* We have to special case filenames of "", so curl + * a) actually attempts the fetch and + * b) doesn't attempt to open the file "" + */ + if (control->value[0] == '\0') { + /* dummy buffer - needs to be static so + * pointer's still valid when we go out + * of scope (not that libcurl should be + * attempting to access it, of course). + */ + static char buf; + + code = curl_formadd(&post, &last, + CURLFORM_COPYNAME, control->name, + CURLFORM_BUFFER, control->value, + /* needed, as basename("") == "." */ + CURLFORM_FILENAME, "", + CURLFORM_BUFFERPTR, &buf, + CURLFORM_BUFFERLENGTH, 0, + CURLFORM_CONTENTTYPE, + "application/octet-stream", + CURLFORM_END); + if (code != CURL_FORMADD_OK) + NSLOG(netsurf, INFO, + "curl_formadd: %d (%s)", code, + control->name); + } else { + char *mimetype = guit->fetch->mimetype(control->value); + code = curl_formadd(&post, &last, + CURLFORM_COPYNAME, control->name, + CURLFORM_FILE, control->rawfile, + CURLFORM_FILENAME, leafname, + CURLFORM_CONTENTTYPE, + (mimetype != 0 ? mimetype : "text/plain"), + CURLFORM_END); + if (code != CURL_FORMADD_OK) + NSLOG(netsurf, INFO, + "curl_formadd: %d (%s=%s)", + code, + control->name, + control->value); + free(mimetype); + } + free(leafname); + } else { + code = curl_formadd(&post, &last, + CURLFORM_COPYNAME, control->name, + CURLFORM_COPYCONTENTS, control->value, + CURLFORM_END); + if (code != CURL_FORMADD_OK) + NSLOG(netsurf, INFO, + "curl_formadd: %d (%s=%s)", code, + control->name, control->value); + } + } + + return post; +} + +#endif /* LIBCURL_VERSION_NUM >= 0x073800 */ + +/** + * Setup multipart post data + */ +static CURLcode fetch_curl_set_postdata(struct curl_fetch_info *f) +{ + CURLcode code = CURLE_OK; + +#undef SETOPT +#define SETOPT(option, value) { \ + code = curl_easy_setopt(f->curl_handle, option, value); \ + if (code != CURLE_OK) \ + return code; \ + } + + switch (f->postdata->type) { + case FETCH_POSTDATA_NONE: + SETOPT(CURLOPT_POSTFIELDS, NULL); + SETOPT(NSCURL_POSTDATA_CURLOPT, NULL); + SETOPT(CURLOPT_HTTPGET, 1L); + break; + + case FETCH_POSTDATA_URLENC: + SETOPT(NSCURL_POSTDATA_CURLOPT, NULL); + SETOPT(CURLOPT_HTTPGET, 0L); + SETOPT(CURLOPT_POSTFIELDS, f->postdata->data.urlenc); + break; + + case FETCH_POSTDATA_MULTIPART: + SETOPT(CURLOPT_POSTFIELDS, NULL); + SETOPT(CURLOPT_HTTPGET, 0L); + if (f->curl_postdata == NULL) { + f->curl_postdata = + fetch_curl_postdata_convert(f->curl_handle, + f->postdata->data.multipart); + } + SETOPT(NSCURL_POSTDATA_CURLOPT, f->curl_postdata); + break; + } + return code; +} + /** * Set options specific for a fetch. * @@ -585,21 +1180,11 @@ static CURLcode fetch_curl_set_options(struct curl_fetch_info *f) SETOPT(CURLOPT_PRIVATE, f); SETOPT(CURLOPT_WRITEDATA, f); SETOPT(CURLOPT_WRITEHEADER, f); - SETOPT(CURLOPT_PROGRESSDATA, f); - SETOPT(CURLOPT_REFERER, fetch_get_referer_to_send(f->fetch_handle)); + SETOPT(NSCURLOPT_PROGRESS_DATA, f); SETOPT(CURLOPT_HTTPHEADER, f->headers); - if (f->post_urlenc) { - SETOPT(CURLOPT_HTTPPOST, NULL); - SETOPT(CURLOPT_HTTPGET, 0L); - SETOPT(CURLOPT_POSTFIELDS, f->post_urlenc); - } else if (f->post_multipart) { - SETOPT(CURLOPT_POSTFIELDS, NULL); - SETOPT(CURLOPT_HTTPGET, 0L); - SETOPT(CURLOPT_HTTPPOST, f->post_multipart); - } else { - SETOPT(CURLOPT_POSTFIELDS, NULL); - SETOPT(CURLOPT_HTTPPOST, NULL); - SETOPT(CURLOPT_HTTPGET, 1L); + code = fetch_curl_set_postdata(f); + if (code != CURLE_OK) { + return code; } f->cookie_string = urldb_get_cookie(f->url, true); @@ -610,7 +1195,7 @@ static CURLcode fetch_curl_set_options(struct curl_fetch_info *f) } if ((auth = urldb_get_auth_details(f->url, NULL)) != NULL) { - SETOPT(CURLOPT_HTTPAUTH, CURLAUTH_ANY); + SETOPT(CURLOPT_HTTPAUTH, CURLAUTH_BASIC); SETOPT(CURLOPT_USERPWD, auth); } else { SETOPT(CURLOPT_USERPWD, NULL); @@ -646,8 +1231,14 @@ static CURLcode fetch_curl_set_options(struct curl_fetch_info *f) SETOPT(CURLOPT_PROXY, NULL); } - /* Disable SSL session ID caching, as some servers can't cope. */ - SETOPT(CURLOPT_SSL_SESSIONID_CACHE, 0); + + if (curl_with_openssl) { + SETOPT(CURLOPT_SSL_CIPHER_LIST, + f->downgrade_tls ? CIPHER_LIST_LEGACY : CIPHER_LIST); + } + + /* Force-enable SSL session ID caching, as some distros are odd. */ + SETOPT(CURLOPT_SSL_SESSIONID_CACHE, 1); if (urldb_get_cert_permissions(f->url)) { /* Disable certificate verification */ @@ -661,10 +1252,12 @@ static CURLcode fetch_curl_set_options(struct curl_fetch_info *f) /* do verification */ SETOPT(CURLOPT_SSL_VERIFYPEER, 1L); SETOPT(CURLOPT_SSL_VERIFYHOST, 2L); +#ifdef WITH_OPENSSL if (curl_with_openssl) { SETOPT(CURLOPT_SSL_CTX_FUNCTION, fetch_curl_sslctxfun); SETOPT(CURLOPT_SSL_CTX_DATA, f); } +#endif } return CURLE_OK; @@ -689,6 +1282,9 @@ fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle) code = fetch_curl_set_options(fetch); if (code != CURLE_OK) { fetch->curl_handle = 0; + /* The handle maybe went bad, eat it */ + NSLOG(netsurf, WARNING, "cURL handle maybe went bad, retry later"); + curl_easy_cleanup(handle); return false; } @@ -726,6 +1322,10 @@ static CURL *fetch_curl_get_handle(lwc_string *host) static bool fetch_curl_start(void *vfetch) { struct curl_fetch_info *fetch = (struct curl_fetch_info*)vfetch; + if (inside_curl) { + NSLOG(netsurf, DEBUG, "Deferring fetch because we're inside cURL"); + return false; + } return fetch_curl_initiate_fetch(fetch, fetch_curl_get_handle(fetch->host)); } @@ -782,23 +1382,6 @@ static void fetch_curl_cache_handle(CURL *handle, lwc_string *host) /** - * Abort a fetch. - */ -static void fetch_curl_abort(void *vf) -{ - struct curl_fetch_info *f = (struct curl_fetch_info *)vf; - assert(f); - NSLOG(netsurf, INFO, "fetch %p, url '%s'", f, nsurl_access(f->url)); - if (f->curl_handle) { - f->abort = true; - } else { - fetch_remove_from_queues(f->fetch_handle); - fetch_free(f->fetch_handle); - } -} - - -/** * Clean up the provided fetch object and free it. * * Will prod the queue afterwards to allow pending requests to be initiated. @@ -825,6 +1408,30 @@ static void fetch_curl_stop(struct curl_fetch_info *f) /** + * Abort a fetch. + */ +static void fetch_curl_abort(void *vf) +{ + struct curl_fetch_info *f = (struct curl_fetch_info *)vf; + assert(f); + NSLOG(netsurf, INFO, "fetch %p, url '%s'", f, nsurl_access(f->url)); + if (f->curl_handle) { + if (inside_curl) { + NSLOG(netsurf, DEBUG, "Deferring cleanup"); + f->abort = true; + } else { + NSLOG(netsurf, DEBUG, "Immediate abort"); + fetch_curl_stop(f); + fetch_free(f->fetch_handle); + } + } else { + fetch_remove_from_queues(f->fetch_handle); + fetch_free(f->fetch_handle); + } +} + + +/** * Free a fetch structure and associated resources. */ static void fetch_curl_free(void *vf) @@ -843,13 +1450,14 @@ static void fetch_curl_free(void *vf) if (f->headers) { curl_slist_free_all(f->headers); } - free(f->post_urlenc); - if (f->post_multipart) { - curl_formfree(f->post_multipart); - } + fetch_curl_free_postdata(f->postdata); + NSCURL_POSTDATA_FREE(f->curl_postdata); - for (i = 0; i < MAX_CERTS && f->cert_data[i].cert; i++) { - ns_X509_free(f->cert_data[i].cert); + /* free certificate data */ + for (i = 0; i < MAX_CERT_DEPTH; i++) { + if (f->cert_data[i].cert != NULL) { + X509_free(f->cert_data[i].cert); + } } free(f); @@ -878,7 +1486,7 @@ static bool fetch_curl_process_headers(struct curl_fetch_info *f) http_code = f->http_code; NSLOG(netsurf, INFO, "HTTP status code %li", http_code); - if (http_code == 304 && !f->post_urlenc && !f->post_multipart) { + if ((http_code == 304) && (f->postdata->type==FETCH_POSTDATA_NONE)) { /* Not Modified && GET request */ msg.type = FETCH_NOTMODIFIED; fetch_send_callback(&msg, f->fetch_handle); @@ -917,114 +1525,6 @@ static bool fetch_curl_process_headers(struct curl_fetch_info *f) return false; } -/** - * setup callback to allow the user to examine certificates which have - * failed to validate during fetch. - */ -static void -curl_start_cert_validate(struct curl_fetch_info *f, - struct cert_info *certs) -{ - int depth; - BIO *mem; - BUF_MEM *buf; - struct ssl_cert_info ssl_certs[MAX_CERTS]; - fetch_msg msg; - - for (depth = 0; depth <= f->cert_depth; depth++) { - assert(certs[depth].cert != NULL); - - /* get certificate version */ - ssl_certs[depth].version = X509_get_version(certs[depth].cert); - - /* not before date */ - mem = BIO_new(BIO_s_mem()); - ASN1_TIME_print(mem, X509_get_notBefore(certs[depth].cert)); - BIO_get_mem_ptr(mem, &buf); - (void) BIO_set_close(mem, BIO_NOCLOSE); - BIO_free(mem); - memcpy(ssl_certs[depth].not_before, - buf->data, - min(sizeof(ssl_certs[depth].not_before) - 1, - (unsigned)buf->length)); - ssl_certs[depth].not_before[min(sizeof(ssl_certs[depth].not_before) - 1, - (unsigned)buf->length)] = 0; - BUF_MEM_free(buf); - - /* not after date */ - mem = BIO_new(BIO_s_mem()); - ASN1_TIME_print(mem, - X509_get_notAfter(certs[depth].cert)); - BIO_get_mem_ptr(mem, &buf); - (void) BIO_set_close(mem, BIO_NOCLOSE); - BIO_free(mem); - memcpy(ssl_certs[depth].not_after, - buf->data, - min(sizeof(ssl_certs[depth].not_after) - 1, - (unsigned)buf->length)); - ssl_certs[depth].not_after[min(sizeof(ssl_certs[depth].not_after) - 1, - (unsigned)buf->length)] = 0; - BUF_MEM_free(buf); - - /* signature type */ - ssl_certs[depth].sig_type = - X509_get_signature_type(certs[depth].cert); - - /* serial number */ - ssl_certs[depth].serial = - ASN1_INTEGER_get( - X509_get_serialNumber(certs[depth].cert)); - - /* issuer name */ - mem = BIO_new(BIO_s_mem()); - X509_NAME_print_ex(mem, - X509_get_issuer_name(certs[depth].cert), - 0, XN_FLAG_SEP_CPLUS_SPC | - XN_FLAG_DN_REV | XN_FLAG_FN_NONE); - BIO_get_mem_ptr(mem, &buf); - (void) BIO_set_close(mem, BIO_NOCLOSE); - BIO_free(mem); - memcpy(ssl_certs[depth].issuer, - buf->data, - min(sizeof(ssl_certs[depth].issuer) - 1, - (unsigned) buf->length)); - ssl_certs[depth].issuer[min(sizeof(ssl_certs[depth].issuer) - 1, - (unsigned) buf->length)] = 0; - BUF_MEM_free(buf); - - /* subject */ - mem = BIO_new(BIO_s_mem()); - X509_NAME_print_ex(mem, - X509_get_subject_name(certs[depth].cert), - 0, - XN_FLAG_SEP_CPLUS_SPC | - XN_FLAG_DN_REV | - XN_FLAG_FN_NONE); - BIO_get_mem_ptr(mem, &buf); - (void) BIO_set_close(mem, BIO_NOCLOSE); - BIO_free(mem); - memcpy(ssl_certs[depth].subject, - buf->data, - min(sizeof(ssl_certs[depth].subject) - 1, - (unsigned)buf->length)); - ssl_certs[depth].subject[min(sizeof(ssl_certs[depth].subject) - 1, - (unsigned) buf->length)] = 0; - BUF_MEM_free(buf); - - /* type of certificate */ - ssl_certs[depth].cert_type = - X509_certificate_type(certs[depth].cert, - X509_get_pubkey(certs[depth].cert)); - - /* and clean up */ - ns_X509_free(certs[depth].cert); - } - - msg.type = FETCH_CERT_ERR; - msg.data.cert_err.certs = ssl_certs; - msg.data.cert_err.num_certs = depth; - fetch_send_callback(&msg, f->fetch_handle); -} /** * Handle a completed fetch (CURLMSG_DONE from curl_multi_info_read()). @@ -1041,7 +1541,6 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result) struct curl_fetch_info *f; char **_hideous_hack = (char **) (void *) &f; CURLcode code; - struct cert_info certs[MAX_CERTS]; /* find the structure associated with this fetch */ /* For some reason, cURL thinks CURLINFO_PRIVATE should be a string?! */ @@ -1085,13 +1584,11 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result) */ ; } else if (result == CURLE_SSL_PEER_CERTIFICATE || - result == CURLE_SSL_CACERT) { - /* CURLE_SSL_PEER_CERTIFICATE renamed to - * CURLE_PEER_FAILED_VERIFICATION + result == CURLE_SSL_CACERT) { + /* Some kind of failure has occurred. If we don't know + * what happened, we'll have reported unknown errors up + * to the user already via the certificate chain error fields. */ - memset(certs, 0, sizeof(certs)); - memcpy(certs, f->cert_data, sizeof(certs)); - memset(f->cert_data, 0, sizeof(f->cert_data)); cert = true; } else { NSLOG(netsurf, INFO, "Unknown cURL response code %d", result); @@ -1100,6 +1597,10 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result) fetch_curl_stop(f); + if (f->sent_ssl_chain == false) { + fetch_curl_report_certs_upstream(f); + } + if (abort_fetch) { ; /* fetch was aborted: no callback */ } else if (finished) { @@ -1108,7 +1609,9 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result) fetch_send_callback(&msg, f->fetch_handle); } else if (cert) { /* user needs to validate certificate with issue */ - curl_start_cert_validate(f, certs); + fetch_msg msg; + msg.type = FETCH_CERT_ERR; + fetch_send_callback(&msg, f->fetch_handle); } else if (error) { fetch_msg msg; switch (result) { @@ -1158,7 +1661,7 @@ static void fetch_curl_poll(lwc_string *scheme_ignored) &exc_fd_set, &max_fd); assert(codem == CURLM_OK); - NSLOG(netsurf, INFO, + NSLOG(netsurf, DEEPDEBUG, "Curl file descriptor states (maxfd=%i):", max_fd); for (i = 0; i <= max_fd; i++) { bool read = false; @@ -1175,7 +1678,7 @@ static void fetch_curl_poll(lwc_string *scheme_ignored) error = true; } if (read || write || error) { - NSLOG(netsurf, INFO, " fd %i: %s %s %s", i, + NSLOG(netsurf, DEEPDEBUG, " fd %i: %s %s %s", i, read ? "read" : " ", write ? "write" : " ", error ? "error" : " "); @@ -1184,12 +1687,13 @@ static void fetch_curl_poll(lwc_string *scheme_ignored) } /* do any possible work on the current fetches */ + inside_curl = true; do { codem = curl_multi_perform(fetch_curl_multi, &running); if (codem != CURLM_OK && codem != CURLM_CALL_MULTI_PERFORM) { - NSLOG(netsurf, INFO, "curl_multi_perform: %i %s", + NSLOG(netsurf, WARNING, + "curl_multi_perform: %i %s", codem, curl_multi_strerror(codem)); - guit->misc->warning("MiscError", curl_multi_strerror(codem)); return; } } while (codem == CURLM_CALL_MULTI_PERFORM); @@ -1207,6 +1711,7 @@ static void fetch_curl_poll(lwc_string *scheme_ignored) } curl_msg = curl_multi_info_read(fetch_curl_multi, &queue); } + inside_curl = false; } @@ -1217,10 +1722,10 @@ static void fetch_curl_poll(lwc_string *scheme_ignored) */ static int fetch_curl_progress(void *clientp, - double dltotal, - double dlnow, - double ultotal, - double ulnow) + NSCURL_PROGRESS_T dltotal, + NSCURL_PROGRESS_T dlnow, + NSCURL_PROGRESS_T ultotal, + NSCURL_PROGRESS_T ulnow) { static char fetch_progress_buffer[256]; /**< Progress buffer for cURL */ struct curl_fetch_info *f = (struct curl_fetch_info *) clientp; @@ -1261,20 +1766,50 @@ fetch_curl_progress(void *clientp, /** - * Ignore everything given to it. - * - * Used to ignore cURL debug. + * Format curl debug for nslog */ -static int fetch_curl_ignore_debug(CURL *handle, - curl_infotype type, - char *data, - size_t size, - void *userptr) +static int +fetch_curl_debug(CURL *handle, + curl_infotype type, + char *data, + size_t size, + void *userptr) { + static const char s_infotype[CURLINFO_END][3] = { + "* ", "< ", "> ", "{ ", "} ", "{ ", "} " + }; + switch(type) { + case CURLINFO_TEXT: + case CURLINFO_HEADER_OUT: + case CURLINFO_HEADER_IN: + NSLOG(fetch, DEBUG, "%s%.*s", s_infotype[type], (int)size - 1, data); + break; + + default: + break; + } return 0; } +static curl_socket_t fetch_curl_socket_open(void *clientp, + curlsocktype purpose, struct curl_sockaddr *address) +{ + (void) clientp; + (void) purpose; + + return (curl_socket_t) guit->fetch->socket_open( + address->family, address->socktype, + address->protocol); +} + +static int fetch_curl_socket_close(void *clientp, curl_socket_t item) +{ + (void) clientp; + + return guit->fetch->socket_close((int) item); +} + /** * Callback function for cURL. */ @@ -1338,6 +1873,10 @@ fetch_curl_header(char *data, size_t size, size_t nmemb, void *_f) return 0; } + if (f->sent_ssl_chain == false) { + fetch_curl_report_certs_upstream(f); + } + msg.type = FETCH_HEADER; msg.data.header_or_data.buf = (const uint8_t *) data; msg.data.header_or_data.len = size; @@ -1441,6 +1980,18 @@ nserror fetch_curl_register(void) .finalise = fetch_curl_finalise }; +#if LIBCURL_VERSION_NUM >= 0x073800 + /* version 7.56.0 can select which SSL backend to use */ + CURLsslset setres; + + setres = curl_global_sslset(CURLSSLBACKEND_OPENSSL, NULL, NULL); + if (setres == CURLSSLSET_OK) { + curl_with_openssl = true; + } else { + curl_with_openssl = false; + } +#endif + NSLOG(netsurf, INFO, "curl_version %s", curl_version()); code = curl_global_init(CURL_GLOBAL_ALL); @@ -1465,8 +2016,10 @@ nserror fetch_curl_register(void) #undef SETOPT #define SETOPT(option, value) \ mcode = curl_multi_setopt(fetch_curl_multi, option, value); \ - if (mcode != CURLM_OK) \ - goto curl_multi_setopt_failed; + if (mcode != CURLM_OK) { \ + NSLOG(netsurf, ERROR, "attempting curl_multi_setopt(%s, ...)", #option); \ + goto curl_multi_setopt_failed; \ + } SETOPT(CURLMOPT_MAXCONNECTS, maxconnects); SETOPT(CURLMOPT_MAX_TOTAL_CONNECTIONS, maxconnects); @@ -1486,21 +2039,25 @@ nserror fetch_curl_register(void) #undef SETOPT #define SETOPT(option, value) \ code = curl_easy_setopt(fetch_blank_curl, option, value); \ - if (code != CURLE_OK) \ - goto curl_easy_setopt_failed; - - if (verbose_log) { - SETOPT(CURLOPT_VERBOSE, 1); - } else { - SETOPT(CURLOPT_VERBOSE, 0); + if (code != CURLE_OK) { \ + NSLOG(netsurf, ERROR, "attempting curl_easy_setopt(%s, ...)", #option); \ + goto curl_easy_setopt_failed; \ } + SETOPT(CURLOPT_ERRORBUFFER, fetch_error_buffer); + SETOPT(CURLOPT_DEBUGFUNCTION, fetch_curl_debug); if (nsoption_bool(suppress_curl_debug)) { - SETOPT(CURLOPT_DEBUGFUNCTION, fetch_curl_ignore_debug); + SETOPT(CURLOPT_VERBOSE, 0); + } else { + SETOPT(CURLOPT_VERBOSE, 1); } + + /* Currently we explode if curl uses HTTP2, so force 1.1. */ + SETOPT(CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1); + SETOPT(CURLOPT_WRITEFUNCTION, fetch_curl_data); SETOPT(CURLOPT_HEADERFUNCTION, fetch_curl_header); - SETOPT(CURLOPT_PROGRESSFUNCTION, fetch_curl_progress); + SETOPT(NSCURLOPT_PROGRESS_FUNCTION, fetch_curl_progress); SETOPT(CURLOPT_NOPROGRESS, 0); SETOPT(CURLOPT_USERAGENT, user_agent_string()); SETOPT(CURLOPT_ENCODING, "gzip"); @@ -1508,6 +2065,8 @@ nserror fetch_curl_register(void) SETOPT(CURLOPT_LOW_SPEED_TIME, 180L); SETOPT(CURLOPT_NOSIGNAL, 1L); SETOPT(CURLOPT_CONNECTTIMEOUT, nsoption_uint(curl_fetch_timeout)); + SETOPT(CURLOPT_OPENSOCKETFUNCTION, fetch_curl_socket_open); + SETOPT(CURLOPT_CLOSESOCKETFUNCTION, fetch_curl_socket_close); if (nsoption_charp(ca_bundle) && strcmp(nsoption_charp(ca_bundle), "")) { @@ -1520,12 +2079,32 @@ nserror fetch_curl_register(void) SETOPT(CURLOPT_CAPATH, nsoption_charp(ca_path)); } - /* Detect whether the SSL CTX function API works */ - curl_with_openssl = true; - code = curl_easy_setopt(fetch_blank_curl, - CURLOPT_SSL_CTX_FUNCTION, NULL); +#if LIBCURL_VERSION_NUM < 0x073800 + /* + * before 7.56.0 Detect openssl from whether the SSL CTX + * function API works + */ + code = curl_easy_setopt(fetch_blank_curl, CURLOPT_SSL_CTX_FUNCTION, NULL); if (code != CURLE_OK) { curl_with_openssl = false; + } else { + curl_with_openssl = true; + } +#endif + + if (curl_with_openssl) { + /* only set the cipher list with openssl otherwise the + * fetch fails with "Unknown cipher in list" + */ +#if LIBCURL_VERSION_NUM >= 0x073d00 + /* Need libcurl 7.61.0 or later built against OpenSSL with + * TLS1.3 support */ + code = curl_easy_setopt(fetch_blank_curl, + CURLOPT_TLS13_CIPHERS, CIPHER_SUITES); + if (code != CURLE_OK && code != CURLE_NOT_BUILT_IN) + goto curl_easy_setopt_failed; +#endif + SETOPT(CURLOPT_SSL_CIPHER_LIST, CIPHER_LIST); } NSLOG(netsurf, INFO, "cURL %slinked against openssl", @@ -1535,6 +2114,12 @@ nserror fetch_curl_register(void) data = curl_version_info(CURLVERSION_NOW); + curl_fetch_ssl_hashmap = hashmap_create(&curl_fetch_ssl_hashmap_parameters); + if (curl_fetch_ssl_hashmap == NULL) { + NSLOG(netsurf, CRITICAL, "Unable to initialise SSL certificate hashmap"); + return NSERROR_NOMEM; + } + for (i = 0; data->protocols[i]; i++) { if (strcmp(data->protocols[i], "http") == 0) { scheme = lwc_string_ref(corestring_lwc_http); |