Add support for retrying timed-out cURL fetches.

This is an attempt to amelioriate the situation found in #2384 where
we see the cURL connect() failing to complete.  Based on the pcap
from the bug log, we believe that RISC OS is likely failing to signal
the completion of the connection to cURL.  As such, cURL times out.

This change permits retries of timed out connections in the hope that
a fresh socket FD might subsequently function correctly.  The defaults
chosen mean that the previous behaviour of 30 seconds before timeout
is reported will remain the same, but in that time we will make 3 separate
attempts to connect the socket.

7 files changed, 60 insertions, 4 deletions

diff --git a/content/fetch.h b/content/fetch.h
index 529a800fa..3c1f1ccae 100644
--- a/content/fetch.h
+++ b/content/fetch.h
@@ -37,6 +37,7 @@ typedef enum {
 	FETCH_HEADER,
 	FETCH_DATA,
 	FETCH_FINISHED,
+	FETCH_TIMEDOUT,
 	FETCH_ERROR,
 	FETCH_REDIRECT,
 	FETCH_NOTMODIFIED,
diff --git a/content/fetchers/curl.c b/content/fetchers/curl.c
index a2c6f2eb4..624cdbf41 100644
--- a/content/fetchers/curl.c
+++ b/content/fetchers/curl.c
@@ -973,11 +973,19 @@ static void fetch_curl_done(CURL *curl_handle, CURLcode result)
 		msg.data.cert_err.num_certs = i;
 		fetch_send_callback(&msg, f->fetch_handle);
 	} else if (error) {
-		if (result != CURLE_SSL_CONNECT_ERROR) {
+		switch (result) {
+		case CURLE_SSL_CONNECT_ERROR:
+			msg.type = FETCH_SSL_ERR;
+			break;
+
+		case CURLE_OPERATION_TIMEDOUT:
+			msg.type = FETCH_TIMEDOUT;
+			msg.data.error = curl_easy_strerror(result);
+			break;
+
+		default:
 			msg.type = FETCH_ERROR;
 			msg.data.error = curl_easy_strerror(result);
-		} else {
-			msg.type = FETCH_SSL_ERR;
 		}
 
 		fetch_send_callback(&msg, f->fetch_handle);
@@ -1302,7 +1310,7 @@ nserror fetch_curl_register(void)
 	SETOPT(CURLOPT_LOW_SPEED_LIMIT, 1L);
 	SETOPT(CURLOPT_LOW_SPEED_TIME, 180L);
 	SETOPT(CURLOPT_NOSIGNAL, 1L);
-	SETOPT(CURLOPT_CONNECTTIMEOUT, 30L);
+	SETOPT(CURLOPT_CONNECTTIMEOUT, nsoption_uint(curl_fetch_timeout));
 
 	if (nsoption_charp(ca_bundle) &&
 	    strcmp(nsoption_charp(ca_bundle), "")) {
diff --git a/content/llcache.c b/content/llcache.c
index 851dbbb5f..e95eefbe6 100644
--- a/content/llcache.c
+++ b/content/llcache.c
@@ -120,6 +120,8 @@ typedef struct {
 
 	uint32_t redirect_count;	/**< Count of redirects followed */
 
+	uint32_t retries_remaining;     /**< Number of times to retry on timeout */
+
 	bool tried_with_auth;		/**< Whether we've tried with auth */
 
 	bool tried_with_tls_downgrade;	/**< Whether we've tried TLS <= 1.0 */
@@ -227,6 +229,9 @@ struct llcache_s {
 	/** The target upper bound for the RAM cache size */
 	uint32_t limit;
 
+	/** The number of fetch attempts we make when timing out */
+	uint32_t fetch_attempts;
+
 	/** Whether or not our users are caught up */
 	bool all_caught_up;
 
@@ -909,6 +914,7 @@ static nserror llcache_object_fetch(llcache_object *object, uint32_t flags,
 	object->fetch.referer = referer_clone;
 	object->fetch.post = post_clone;
 	object->fetch.redirect_count = redirect_count;
+	object->fetch.retries_remaining = llcache->fetch_attempts;
 
 	return llcache_object_refetch(object);
 }
@@ -2653,6 +2659,18 @@ static void llcache_fetch_callback(const fetch_msg *msg, void *p)
 		break;
 
 	/* Out-of-band information */
+	case FETCH_TIMEDOUT:
+		/* Timed out while trying to fetch. */
+		/* The fetch has already been cleaned up by the fetcher but
+		 * we would like to retry if we can. */
+		if (object->fetch.retries_remaining > 1) {
+			object->fetch.retries_remaining--;
+			error = llcache_object_refetch(object);
+			break;
+		}
+		/* Otherwise fall through to error, setting the message to
+		 * a timeout
+		 */
 	case FETCH_ERROR:
 		/* An error occurred while fetching */
 		/* The fetch has has already been cleaned up by the fetcher */
@@ -3303,6 +3321,7 @@ llcache_initialise(const struct llcache_parameters *prm)
 	llcache->minimum_bandwidth = prm->minimum_bandwidth;
 	llcache->maximum_bandwidth = prm->maximum_bandwidth;
 	llcache->time_quantum = prm->time_quantum;
+	llcache->fetch_attempts = prm->fetch_attempts;
 	llcache->all_caught_up = true;
 
 	LOG("llcache initialising with a limit of %d bytes", llcache->limit);
diff --git a/content/llcache.h b/content/llcache.h
index 1b7ba7dae..cce9a79bf 100644
--- a/content/llcache.h
+++ b/content/llcache.h
@@ -230,6 +230,9 @@ struct llcache_parameters {
 	 */
 	unsigned long time_quantum;
 
+	/** The number of fetches to attempt when timing out */
+	uint32_t fetch_attempts;
+
 	struct llcache_store_parameters store;
 };
 
diff --git a/desktop/netsurf.c b/desktop/netsurf.c
index 0fd7b7bbd..d6b5abe6e 100644
--- a/desktop/netsurf.c
+++ b/desktop/netsurf.c
@@ -164,6 +164,9 @@ nserror netsurf_init(const char *store_path)
 		LOG("Setting minimum memory cache size %zd", hlcache_parameters.llcache.limit);
 	} 
 
+	/* Set up the max attempts made to fetch a timing out resource */
+	hlcache_parameters.llcache.fetch_attempts = nsoption_uint(max_retried_fetches);
+
 	/* image cache is 25% of total memory cache size */
 	image_cache_parameters.limit = (hlcache_parameters.llcache.limit * 25) / 100;
 
diff --git a/desktop/options.h b/desktop/options.h
index f01261ee1..e734c5294 100644
--- a/desktop/options.h
+++ b/desktop/options.h
@@ -208,6 +208,14 @@ NSOPTION_INTEGER(max_fetchers_per_host, 5)
  */
 NSOPTION_INTEGER(max_cached_fetch_handles, 6)
 
+/** Number of times to retry timed-out fetches before giving up. */
+NSOPTION_UINT(max_retried_fetches, 3)
+
+/** Number of seconds to allow for a DNS-resolution+connect() before timing out
+ * the cURL socket.
+ */
+NSOPTION_UINT(curl_fetch_timeout, 10)
+
 /** Suppress debug output from cURL. */
 NSOPTION_BOOL(suppress_curl_debug, true)
 
diff --git a/utils/nsoption.c b/utils/nsoption.c
index 1309ed875..ccdd23ae9 100644
--- a/utils/nsoption.c
+++ b/utils/nsoption.c
@@ -181,6 +181,20 @@ static void nsoption_validate(struct nsoption_s *opts, struct nsoption_s *defs)
 			opts[cloop].value.c = defs[cloop].value.c;
 		}
 	}
+
+	/* To aid migration and ensure that timeouts don't go crazy,
+	 * ensure that (a) we allow at least 1 attempt and
+	 * (b) the total time that we spend should not exceed 60s
+	 */
+	if (opts[NSOPTION_max_retried_fetches].value.u == 0)
+		opts[NSOPTION_max_retried_fetches].value.u = 1;
+	if (opts[NSOPTION_curl_fetch_timeout].value.u < 5)
+		opts[NSOPTION_max_retried_fetches].value.u = 5;
+	if (opts[NSOPTION_curl_fetch_timeout].value.u > 60)
+		opts[NSOPTION_max_retried_fetches].value.u = 60;
+	while ((opts[NSOPTION_curl_fetch_timeout].value.u *
+		opts[NSOPTION_max_retried_fetches].value.u) > 60)
+		opts[NSOPTION_max_retried_fetches].value.u--;
 }
 
 /**