summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVincent Sanders <vince@kyllikki.org>2014-03-22 14:54:51 +0000
committerVincent Sanders <vince@kyllikki.org>2014-05-13 15:52:58 +0100
commit4a49ff5266aa1cc483b74fb084503cbc4c4cd1a2 (patch)
tree3edf231bd90132940837cf65281349a87dbf7811
parent657abbd245dc777f61314ad476deb821cff0b90a (diff)
downloadnetsurf-4a49ff5266aa1cc483b74fb084503cbc4c4cd1a2.tar.gz
netsurf-4a49ff5266aa1cc483b74fb084503cbc4c4cd1a2.tar.bz2
Extend low level source data cache with persistant storage
-rw-r--r--content/hlcache.c11
-rw-r--r--content/hlcache.h39
-rw-r--r--content/llcache.c993
-rw-r--r--content/llcache.h92
-rw-r--r--desktop/netsurf.c109
5 files changed, 1022 insertions, 222 deletions
diff --git a/content/hlcache.c b/content/hlcache.c
index 5a3cc8583..23fb79562 100644
--- a/content/hlcache.c
+++ b/content/hlcache.c
@@ -339,9 +339,10 @@ static nserror hlcache_migrate_ctx(hlcache_retrieval_ctx *ctx,
ctx->migrate_target = true;
- if (effective_type != NULL &&
- hlcache_type_is_acceptable(effective_type,
- ctx->accepted_types, &type)) {
+ if ((effective_type != NULL) &&
+ hlcache_type_is_acceptable(effective_type,
+ ctx->accepted_types,
+ &type)) {
error = hlcache_find_content(ctx, effective_type);
if (error != NSERROR_OK && error != NSERROR_NEED_DATA) {
if (ctx->handle->cb != NULL) {
@@ -524,9 +525,7 @@ hlcache_initialise(const struct hlcache_parameters *hlcache_parameters)
return NSERROR_NOMEM;
}
- ret = llcache_initialise(hlcache_parameters->cb,
- hlcache_parameters->cb_ctx,
- hlcache_parameters->limit);
+ ret = llcache_initialise(&hlcache_parameters->llcache);
if (ret != NSERROR_OK) {
free(hlcache);
hlcache = NULL;
diff --git a/content/hlcache.h b/content/hlcache.h
index 41f1ed6f4..746b3c866 100644
--- a/content/hlcache.h
+++ b/content/hlcache.h
@@ -23,11 +23,12 @@
#ifndef NETSURF_CONTENT_HLCACHE_H_
#define NETSURF_CONTENT_HLCACHE_H_
-#include "content/content.h"
-#include "content/llcache.h"
#include "utils/errors.h"
#include "utils/nsurl.h"
+#include "content/content.h"
+#include "content/llcache.h"
+
/** High-level cache handle */
typedef struct hlcache_handle hlcache_handle;
@@ -44,18 +45,10 @@ typedef struct {
} hlcache_event;
struct hlcache_parameters {
- llcache_query_callback cb; /**< Query handler for llcache */
- void *cb_ctx; /**< Pointer to llcache query handler data */
-
/** How frequently the background cache clean process is run (ms) */
unsigned int bg_clean_time;
- /** The target upper bound for the cache size */
- size_t limit;
-
- /** The hysteresis allowed round the target size */
- size_t hysteresis;
-
+ struct llcache_parameters llcache;
};
/**
@@ -67,13 +60,13 @@ struct hlcache_parameters {
* \return NSERROR_OK on success, appropriate error otherwise.
*/
typedef nserror (*hlcache_handle_callback)(hlcache_handle *handle,
- const hlcache_event *event, void *pw);
+ const hlcache_event *event, void *pw);
/** Flags for high-level cache object retrieval */
enum hlcache_retrieve_flag {
- /* Note: low-level cache retrieval flags occupy the bottom 16 bits of
- * the flags word. High-level cache flags occupy the top 16 bits.
- * To avoid confusion, high-level flags are allocated from bit 31 down.
+ /* Note: low-level cache retrieval flags occupy the bottom 16 bits of
+ * the flags word. High-level cache flags occupy the top 16 bits.
+ * To avoid confusion, high-level flags are allocated from bit 31 down.
*/
/** It's permitted to convert this request into a download */
HLCACHE_RETRIEVE_MAY_DOWNLOAD = (1 << 31),
@@ -84,7 +77,7 @@ enum hlcache_retrieve_flag {
/**
* Initialise the high-level cache, preparing the llcache also.
*
- * \param hlcache_parameters Settings to initialise cache with
+ * \param hlcache_parameters Settings to initialise cache with
* \return NSERROR_OK on success, appropriate error otherwise.
*/
nserror hlcache_initialise(const struct hlcache_parameters *hlcache_parameters);
@@ -133,7 +126,7 @@ nserror hlcache_poll(void);
nserror hlcache_handle_retrieve(nsurl *url, uint32_t flags,
nsurl *referer, llcache_post_data *post,
hlcache_handle_callback cb, void *pw,
- hlcache_child_context *child,
+ hlcache_child_context *child,
content_type accepted_types, hlcache_handle **result);
/**
@@ -169,13 +162,13 @@ nserror hlcache_handle_replace_callback(hlcache_handle *handle,
* \param handle Cache handle to dereference
* \return Pointer to content object, or NULL if there is none
*
- * \todo This may not be correct. Ideally, the client should never need to
- * directly access a content object. It may, therefore, be better to provide a
- * bunch of veneers here that take a hlcache_handle and invoke the
+ * \todo This may not be correct. Ideally, the client should never need to
+ * directly access a content object. It may, therefore, be better to provide a
+ * bunch of veneers here that take a hlcache_handle and invoke the
* corresponding content_ API. If there's no content object associated with the
- * hlcache_handle (e.g. because the source data is still being fetched, so it
- * doesn't exist yet), then these veneers would behave as a NOP. The important
- * thing being that the client need not care about this possibility and can
+ * hlcache_handle (e.g. because the source data is still being fetched, so it
+ * doesn't exist yet), then these veneers would behave as a NOP. The important
+ * thing being that the client need not care about this possibility and can
* just call the functions with impugnity.
*/
struct content *hlcache_handle_get_content(const hlcache_handle *handle);
diff --git a/content/llcache.c b/content/llcache.c
index 112a7fab9..653352d3f 100644
--- a/content/llcache.c
+++ b/content/llcache.c
@@ -17,26 +17,41 @@
*/
/** \file
- * Low-level resource cache (implementation)
+ * Low-level resource cache implementation
+ *
+ * This is the implementation of the low level cache. This cache
+ * stores source objects in memory and may use a persistant backing
+ * store to extend their lifetime.
+ *
+ * \todo fix writeout conditions and ordering.
+ *
+ * \todo support mmaped retrieve
+ *
+ * \todo instrument and (auto)tune
+ *
+ * \todo turn llcache debugging off
+ *
*/
#include <stdlib.h>
#include <string.h>
#include <time.h>
-
#include <curl/curl.h>
-#include "content/fetch.h"
-#include "content/llcache.h"
-#include "content/urldb.h"
#include "utils/corestrings.h"
#include "utils/log.h"
#include "utils/messages.h"
#include "utils/nsurl.h"
#include "utils/utils.h"
+#include "desktop/gui_factory.h"
+
+#include "content/fetch.h"
+#include "content/backing_store.h"
+#include "content/urldb.h"
/** Define to enable tracing of llcache operations. */
-#undef LLCACHE_TRACE
+//#undef LLCACHE_TRACE
+#define LLCACHE_TRACE 1
#ifdef LLCACHE_TRACE
#define LLCACHE_LOG(x) LOG(x)
@@ -44,6 +59,9 @@
#define LLCACHE_LOG(x)
#endif
+#define LLCACHE_MIN_DISC_LIFETIME 3600
+#define LLCACHE_MAX_DISC_BANDWIDTH (512*1024)
+
/** State of a low-level cache object fetch */
typedef enum {
LLCACHE_FETCH_INIT, /**< Initial state, before fetch */
@@ -96,19 +114,23 @@ typedef struct {
bool outstanding_query; /**< Waiting for a query response */
} llcache_fetch_ctx;
+/** validation control */
typedef enum {
LLCACHE_VALIDATE_FRESH, /**< Only revalidate if not fresh */
LLCACHE_VALIDATE_ALWAYS, /**< Always revalidate */
LLCACHE_VALIDATE_ONCE /**< Revalidate once only */
} llcache_validate;
+/** cache control value for invalid age */
+#define INVALID_AGE -1
+
/** Cache control data */
typedef struct {
time_t req_time; /**< Time of request */
time_t res_time; /**< Time of response */
+ time_t fin_time; /**< Time of request completion */
time_t date; /**< Date: response header */
time_t expires; /**< Expires: response header */
-#define INVALID_AGE -1
int age; /**< Age: response header */
int max_age; /**< Max-Age Cache-control parameter */
llcache_validate no_cache; /**< No-Cache Cache-control parameter */
@@ -122,31 +144,49 @@ typedef struct {
char *value; /**< Header value */
} llcache_header;
+/** Current status of objects data */
+typedef enum {
+ LLCACHE_STATE_RAM = 0, /**< source data is stored in RAM only */
+ LLCACHE_STATE_MMAP, /**< source data is mmaped (implies on disc too) */
+ LLCACHE_STATE_DISC, /**< source data is stored on disc */
+} llcache_store_state;
+
/** Low-level cache object */
/** \todo Consider whether a list is a sane container */
struct llcache_object {
- llcache_object *prev; /**< Previous in list */
- llcache_object *next; /**< Next in list */
+ llcache_object *prev; /**< Previous in list */
+ llcache_object *next; /**< Next in list */
- nsurl *url; /**< Post-redirect URL for object */
+ nsurl *url; /**< Post-redirect URL for object */
/** \todo We need a generic dynamic buffer object */
- uint8_t *source_data; /**< Source data for object */
- size_t source_len; /**< Byte length of source data */
- size_t source_alloc; /**< Allocated size of source buffer */
+ uint8_t *source_data; /**< Source data for object */
+ size_t source_len; /**< Byte length of source data */
+ size_t source_alloc; /**< Allocated size of source buffer */
+
+ llcache_store_state store_state; /**< where the data for the object is stored */
+
+ llcache_object_user *users; /**< List of users */
- llcache_object_user *users; /**< List of users */
+ llcache_fetch_ctx fetch; /**< Fetch context for object */
- llcache_fetch_ctx fetch; /**< Fetch context for object */
+ llcache_cache_control cache; /**< Cache control data for object */
+ llcache_object *candidate; /**< Object to use, if fetch determines
+ * that it is still fresh
+ */
+ uint32_t candidate_count; /**< Count of objects this is a
+ * candidate for
+ */
- llcache_cache_control cache; /**< Cache control data for object */
- llcache_object *candidate; /**< Object to use, if fetch determines
- * that it is still fresh */
- uint32_t candidate_count; /**< Count of objects this is a
- * candidate for */
+ llcache_header *headers; /**< Fetch headers */
+ size_t num_headers; /**< Number of fetch headers */
- llcache_header *headers; /**< Fetch headers */
- size_t num_headers; /**< Number of fetch headers */
+ /* Instrumentation. These elemnts are strictly for information
+ * to improve the cache performance and to provide performace
+ * metrics. The values are non-authorative and must not be used to
+ * determine object lifetime etc.
+ */
+ time_t last_used; /**< time the last user was removed from the object */
};
struct llcache_s {
@@ -162,7 +202,17 @@ struct llcache_s {
/** Head of the low-level uncached object list */
llcache_object *uncached_objects;
+ /** The target upper bound for the RAM cache size */
uint32_t limit;
+
+ /** The minimum lifetime to consider sending objects to
+ * backing store.
+ */
+ int minimum_lifetime;
+
+ /** The maximum bandwidth to allow the backing store to use. */
+ size_t bandwidth;
+
};
/** low level cache state */
@@ -261,6 +311,11 @@ static nserror llcache_object_remove_user(llcache_object *object,
user->next = user->prev = NULL;
+ /* record the time the last user was removed from the object */
+ if (object->users == NULL) {
+ object->last_used = time(NULL);
+ }
+
LLCACHE_LOG(("Removing user %p from %p", user, object));
return NSERROR_OK;
@@ -711,6 +766,7 @@ static nserror llcache_object_refetch(llcache_object *object)
/* Reset cache control data */
llcache_invalidate_cache_control_data(object);
object->cache.req_time = time(NULL);
+ object->cache.fin_time = object->cache.req_time;
/* Reset fetch state */
object->fetch.state = LLCACHE_FETCH_INIT;
@@ -878,7 +934,7 @@ llcache_object_rfc2616_remaining_lifetime(const llcache_cache_control *cd)
else
freshness_lifetime = 0;
- LLCACHE_LOG(("%d:%d", freshness_lifetime, current_age));
+ /* LLCACHE_LOG(("%d:%d", freshness_lifetime, current_age)); */
if ((cd->no_cache == LLCACHE_VALIDATE_FRESH) &&
(freshness_lifetime > current_age)) {
@@ -957,6 +1013,7 @@ static nserror llcache_object_clone_cache_data(llcache_object *source,
destination->cache.req_time = source->cache.req_time;
destination->cache.res_time = source->cache.res_time;
+ destination->cache.fin_time = source->cache.fin_time;
if (source->cache.date != 0)
destination->cache.date = source->cache.date;
@@ -980,6 +1037,367 @@ static nserror llcache_object_clone_cache_data(llcache_object *source,
}
/**
+ * Remove a low-level cache object from a cache list
+ *
+ * \param object Object to remove
+ * \param list List to remove from
+ * \return NSERROR_OK
+ */
+static nserror
+llcache_object_remove_from_list(llcache_object *object, llcache_object **list)
+{
+ if (object == *list)
+ *list = object->next;
+ else
+ object->prev->next = object->next;
+
+ if (object->next != NULL)
+ object->next->prev = object->prev;
+
+ return NSERROR_OK;
+}
+
+/**
+ * Retrieve source data for an object from persistant store if necessary.
+ *
+ * If an objects source data has been placed in the persistant store
+ * and the in memory copy freed this will attempt to retrive the
+ * source data.
+ *
+ * @param object the object to operate on.
+ * @return apropriate error code.
+ */
+static nserror llcache_persist_retrieve(llcache_object *object)
+{
+ enum backing_store_flags flags = BACKING_STORE_NONE;
+
+ /* ensure the source data is present if necessary */
+ if ((object->source_data != NULL) ||
+ (object->store_state != LLCACHE_STATE_DISC)) {
+ /* source data does not require retriving from
+ * persistant store.
+ */
+ return NSERROR_OK;
+ }
+
+ /* Source data for the object may be in the persiatant store */
+ return guit->llcache->fetch(object->url,
+ &flags,
+ &object->source_data,
+ &object->source_len);
+}
+
+/**
+ * Generate a serialised version of an objects metadata
+ *
+ * metadata includes object headers
+ */
+static nserror
+llcache_serialise_metadata(llcache_object *object,
+ uint8_t **data_out,
+ size_t *datasize_out)
+{
+ size_t allocsize;
+ int datasize;
+ uint8_t *data;
+ char *op;
+ unsigned int hloop;
+ int use;
+ struct tm *ltm;
+
+ allocsize = 10 + 1; /* object length */
+
+ allocsize += 10 + 1; /* request time */
+
+ allocsize += 10 + 1; /* response time */
+
+ allocsize += 10 + 1; /* completion time */
+
+ allocsize += 10 + 1; /* space for number of header entries */
+
+ allocsize += nsurl_length(object->url) + 1;
+
+ for (hloop = 0 ; hloop < object->num_headers ; hloop++) {
+ allocsize += strlen(object->headers[hloop].name) + 1;
+ allocsize += strlen(object->headers[hloop].value) + 1;
+ }
+
+ data = malloc(allocsize);
+ if (data == NULL) {
+ return NSERROR_NOMEM;
+ }
+
+ op = (char *)data;
+ datasize = allocsize;
+
+ /* the url, used for checking for collisions */
+ use = snprintf(op, datasize, "%s%c", nsurl_access(object->url), 0);
+ if (use > datasize)
+ goto overflow;
+ op += use;
+ datasize -= use;
+
+ /* object size */
+ use = snprintf(op, datasize, "%zu%c", object->source_len, 0);
+ if (use > datasize)
+ goto overflow;
+ op += use;
+ datasize -= use;
+
+ /* Time of request */
+ ltm = localtime(&object->cache.req_time);
+ use = strftime(op, datasize, "%s", ltm);
+ if (use == 0)
+ goto overflow;
+ use++; /* does not count the null */
+ op += use;
+ datasize -= use;
+
+ /* Time of response */
+ ltm = localtime(&object->cache.res_time);
+ use = strftime(op, datasize, "%s", ltm);
+ if (use == 0)
+ goto overflow;
+ use++; /* does not count the null */
+ op += use;
+ datasize -= use;
+
+ /* Time of completion */
+ ltm = localtime(&object->cache.fin_time);
+ use = strftime(op, datasize, "%s", ltm);
+ if (use == 0)
+ goto overflow;
+ use++; /* does not count the null */
+ op += use;
+ datasize -= use;
+
+ /* number of headers */
+ use = snprintf(op, datasize, "%zu%c", object->num_headers, 0);
+ if (use > datasize)
+ goto overflow;
+ op += use;
+ datasize -= use;
+
+ /* headers */
+ for (hloop = 0 ; hloop < object->num_headers ; hloop++) {
+ use = snprintf(op, datasize,
+ "%s:%s%c",
+ object->headers[hloop].name,
+ object->headers[hloop].value,
+ 0);
+ if (use > datasize)
+ goto overflow;
+ op += use;
+ datasize -= use;
+ }
+
+ LLCACHE_LOG(("Filled buffer with %d spare", datasize));
+
+ *data_out = data;
+ *datasize_out = allocsize - datasize;
+
+ return NSERROR_OK;
+
+overflow:
+ /* somehow we overflowed the buffer - hth? */
+ LOG(("Overflowed metadata buffer"));
+ free(data);
+ return NSERROR_INVALID;
+}
+
+/**
+ * un-serialise an objects metadata.
+ */
+static nserror
+llcache_process_metadata(llcache_object *object)
+{
+ nserror res;
+ uint8_t *metadata = NULL;
+ size_t metadatalen = 0;
+ nsurl *metadataurl;
+ unsigned int line;
+ uint8_t *end;
+ char *ln;
+ int lnsize;
+ size_t num_headers;
+ size_t hloop;
+ struct tm ltm;
+ enum backing_store_flags flags = BACKING_STORE_META;
+
+ LOG(("Retriving metadata"));
+
+ /* attempt to retrieve object metadata from the backing store */
+ res = guit->llcache->fetch(object->url,
+ &flags,
+ &metadata,
+ &metadatalen);
+ if (res != NSERROR_OK) {
+ return res;
+ }
+
+ end = metadata + metadatalen;
+
+ LOG(("Processing retrived data"));
+
+ /* metadata line 1 is the url the metadata referrs to */
+ line = 1;
+ ln = (char *)metadata;
+ lnsize = strlen(ln);
+
+ if (lnsize < 7)
+ goto format_error;
+
+ res = nsurl_create(ln, &metadataurl);
+ if (res != NSERROR_OK) {
+ free(metadata);
+ return res;
+ }
+
+ if (nsurl_compare(object->url, metadataurl, NSURL_COMPLETE) != true) {
+ /* backing store returned the wrong object for the
+ * request. This may occour if the backing store had
+ * a collision in its stoage method. We cope with this
+ * by simply skipping caching of this object.
+ */
+
+ LOG(("Got metadata for %s instead of %s",
+ nsurl_access(metadataurl),
+ nsurl_access(object->url)));
+
+ nsurl_unref(metadataurl);
+
+ free(metadata);
+
+ return NSERROR_BAD_URL;
+ }
+ nsurl_unref(metadataurl);
+
+
+ /* metadata line 2 is the objects length */
+ line = 2;
+ ln += lnsize + 1;
+ lnsize = strlen(ln);
+
+ if ((lnsize < 1) ||
+ (sscanf(ln, "%zu", &object->source_len) != 1))
+ goto format_error;
+ object->source_alloc = metadatalen;
+
+ /* metadata line 3 is the time of request */
+ line = 3;
+ ln += lnsize + 1;
+ lnsize = strlen(ln);
+
+ if ((lnsize < 1) ||
+ (strptime(ln, "%s", &ltm) == NULL))
+ goto format_error;
+ object->cache.req_time = mktime(&ltm);
+
+ /* metadata line 4 is the time of response */
+ line = 4;
+ ln += lnsize + 1;
+ lnsize = strlen(ln);
+
+ if ((lnsize < 1) ||
+ (strptime(ln, "%s", &ltm) == NULL))
+ goto format_error;
+ object->cache.res_time = mktime(&ltm);
+
+ /* metadata line 5 is the time of request completion */
+ line = 5;
+ ln += lnsize + 1;
+ lnsize = strlen(ln);
+
+ if ((lnsize < 1) ||
+ (strptime(ln, "%s", &ltm) == NULL))
+ goto format_error;
+ object->cache.fin_time = mktime(&ltm);
+
+
+ /* metadata line 6 is the number of headers */
+ line = 6;
+ ln += lnsize + 1;
+ lnsize = strlen(ln);
+
+ if ((lnsize < 1) ||
+ (sscanf(ln, "%zu", &num_headers) != 1))
+ goto format_error;
+
+
+ /* read headers */
+ for (hloop = 0 ; hloop < num_headers; hloop++) {
+ line++;
+ ln += lnsize + 1;
+ lnsize = strlen(ln);
+
+ res = llcache_fetch_process_header(object, (uint8_t *)ln, lnsize);
+ if (res != NSERROR_OK) {
+ free(metadata);
+ return res;
+ }
+ }
+
+ free(metadata);
+
+ /* object stored in backing store */
+ object->store_state = LLCACHE_STATE_DISC;
+
+ return NSERROR_OK;
+
+format_error:
+ LOG(("metadata error on line %d\n", line));
+ free(metadata);
+ return NSERROR_INVALID;
+
+}
+
+/**
+ * attempt to retrieve an object from persistant storage.
+ */
+static nserror
+llcache_object_fetch_persistant(llcache_object *object,
+ uint32_t flags,
+ nsurl *referer,
+ const llcache_post_data *post,
+ uint32_t redirect_count)
+{
+ nserror error;
+ nsurl *referer_clone = NULL;
+ llcache_post_data *post_clone = NULL;
+
+ object->cache.req_time = time(NULL);
+ object->cache.fin_time = object->cache.req_time;
+
+ /* retrieve and process metadata */
+ error = llcache_process_metadata(object);
+ if (error != NSERROR_OK) {
+ return error;
+ }
+
+ /* entry came out of cache - need to setup object state */
+ if (post != NULL) {
+ error = llcache_post_data_clone(post, &post_clone);
+ if (error != NSERROR_OK)
+ return error;
+ }
+
+ if (referer != NULL) {
+ referer_clone = nsurl_ref(referer);
+ }
+
+ object->fetch.flags = flags;
+ object->fetch.referer = referer_clone;
+ object->fetch.post = post_clone;
+ object->fetch.redirect_count = redirect_count;
+
+ /* fetch is "finished" */
+ object->fetch.state = LLCACHE_FETCH_COMPLETE;
+ object->fetch.fetch = NULL;
+
+ return NSERROR_OK;
+}
+
+/**
* Retrieve a potentially cached object
*
* \param url URL of object to retrieve
@@ -990,89 +1408,158 @@ static nserror llcache_object_clone_cache_data(llcache_object *source,
* \param result Pointer to location to recieve retrieved object
* \return NSERROR_OK on success, appropriate error otherwise
*/
-static nserror llcache_object_retrieve_from_cache(nsurl *url, uint32_t flags,
- nsurl *referer, const llcache_post_data *post,
- uint32_t redirect_count, llcache_object **result)
+static nserror
+llcache_object_retrieve_from_cache(nsurl *url,
+ uint32_t flags,
+ nsurl *referer,
+ const llcache_post_data *post,
+ uint32_t redirect_count,
+ llcache_object **result)
{
nserror error;
llcache_object *obj, *newest = NULL;
- LLCACHE_LOG(("Searching cache for %s (%x %p %p)",
- nsurl_access(url), flags, referer, post));
+ LLCACHE_LOG(("Searching cache for %s flags:%x referer:%s post:%p",
+ nsurl_access(url), flags, referer==NULL?"":nsurl_access(referer), post));
/* Search for the most recently fetched matching object */
for (obj = llcache->cached_objects; obj != NULL; obj = obj->next) {
if ((newest == NULL ||
- obj->cache.req_time > newest->cache.req_time) &&
- nsurl_compare(obj->url, url,
- NSURL_COMPLETE) == true) {
+ obj->cache.req_time > newest->cache.req_time) &&
+ nsurl_compare(obj->url, url,
+ NSURL_COMPLETE) == true) {
newest = obj;
}
}
- if (newest != NULL && llcache_object_is_fresh(newest)) {
- /* Found a suitable object, and it's still fresh, so use it */
- obj = newest;
+ /* No viable object found in cache create one and attempt to
+ * pull from persistant store.
+ */
+ if (newest == NULL) {
+ LLCACHE_LOG(("No viable object found in cache"));
- LLCACHE_LOG(("Found fresh %p", obj));
+ error = llcache_object_new(url, &obj);
+ if (error != NSERROR_OK)
+ return error;
+
+ /* attempt to retrieve object from persistant store */
+ error = llcache_object_fetch_persistant(obj, flags, referer, post, redirect_count);
+ if (error == NSERROR_OK) {
+ LLCACHE_LOG(("retrived object from persistant store"));
+
+ /* set object from persistant store as newest */
+ newest = obj;
+
+ /* Add new object to cached object list */
+ llcache_object_add_to_list(obj, &llcache->cached_objects);
+
+ }
+ /* else no object found and unretrivable from cache,
+ * fall through to start fetch
+ */
+ }
+
+ if ((newest != NULL) && (llcache_object_is_fresh(newest))) {
+ /* Found a suitable object, and it's still fresh */
+ LLCACHE_LOG(("Found fresh %p", newest));
/* The client needs to catch up with the object's state.
* This will occur the next time that llcache_poll is called.
*/
- } else if (newest != NULL) {
- /* Found a candidate object but it needs freshness validation */
- /* Create a new object */
- error = llcache_object_new(url, &obj);
- if (error != NSERROR_OK)
- return error;
+ /* ensure the source data is present */
+ error = llcache_persist_retrieve(newest);
+ if (error == NSERROR_OK) {
+ /* source data was sucessfully retrived from
+ * persistant store
+ */
+ *result = newest;
- LLCACHE_LOG(("Found candidate %p (%p)", obj, newest));
+ return NSERROR_OK;
+ }
+
+ /* retrival of source data from persistant store
+ * failed, destroy cache object and fall though to
+ * cache miss to re-retch
+ */
+ LLCACHE_LOG(("Persistant retrival failed for %p", newest));
+
+ llcache_object_remove_from_list(newest, &llcache->cached_objects);
+ llcache_object_destroy(newest);
- /* Clone candidate's cache data */
- error = llcache_object_clone_cache_data(newest, obj, true);
+ error = llcache_object_new(url, &obj);
if (error != NSERROR_OK) {
- llcache_object_destroy(obj);
return error;
}
+ } else if (newest != NULL) {
+ /* Found a candidate object but it needs freshness validation */
- /* Record candidate, so we can fall back if it is still fresh */
- newest->candidate_count++;
- obj->candidate = newest;
+ /* ensure the source data is present */
+ error = llcache_persist_retrieve(newest);
+ if (error == NSERROR_OK) {
- /* Attempt to kick-off fetch */
- error = llcache_object_fetch(obj, flags, referer, post,
- redirect_count);
- if (error != NSERROR_OK) {
- newest->candidate_count--;
- llcache_object_destroy(obj);
- return error;
+ /* Create a new object */
+ error = llcache_object_new(url, &obj);
+ if (error != NSERROR_OK)
+ return error;
+
+ LLCACHE_LOG(("Found candidate %p (%p)", obj, newest));
+
+ /* Clone candidate's cache data */
+ error = llcache_object_clone_cache_data(newest, obj, true);
+ if (error != NSERROR_OK) {
+ llcache_object_destroy(obj);
+ return error;
+ }
+
+ /* Record candidate, so we can fall back if it is still fresh */
+ newest->candidate_count++;
+ obj->candidate = newest;
+
+ /* Attempt to kick-off fetch */
+ error = llcache_object_fetch(obj, flags, referer, post,
+ redirect_count);
+ if (error != NSERROR_OK) {
+ newest->candidate_count--;
+ llcache_object_destroy(obj);
+ return error;
+ }
+
+ /* Add new object to cache */
+ llcache_object_add_to_list(obj, &llcache->cached_objects);
+
+ *result = obj;
+
+ return NSERROR_OK;
}
- /* Add new object to cache */
- llcache_object_add_to_list(obj, &llcache->cached_objects);
- } else {
- /* No object found; create a new one */
- /* Create new object */
- error = llcache_object_new(url, &obj);
- if (error != NSERROR_OK)
- return error;
+ LLCACHE_LOG(("Persistant retrival failed for %p", newest));
- LLCACHE_LOG(("Not found %p", obj));
+ /* retrival of source data from persistant store
+ * failed, destroy cache object and fall though to
+ * cache miss to re-retch
+ */
+ llcache_object_remove_from_list(newest,
+ &llcache->cached_objects);
+ llcache_object_destroy(newest);
- /* Attempt to kick-off fetch */
- error = llcache_object_fetch(obj, flags, referer, post,
- redirect_count);
+ error = llcache_object_new(url, &obj);
if (error != NSERROR_OK) {
- llcache_object_destroy(obj);
return error;
}
+ }
- /* Add new object to cache */
- llcache_object_add_to_list(obj, &llcache->cached_objects);
+ /* Attempt to kick-off fetch */
+ error = llcache_object_fetch(obj, flags, referer, post, redirect_count);
+ if (error != NSERROR_OK) {
+ llcache_object_destroy(obj);
+ return error;
}
+ /* Add new object to cache */
+ llcache_object_add_to_list(obj, &llcache->cached_objects);
+
*result = obj;
return NSERROR_OK;
@@ -1098,8 +1585,8 @@ static nserror llcache_object_retrieve(nsurl *url, uint32_t flags,
nsurl *defragmented_url;
bool uncachable = false;
- LLCACHE_LOG(("Retrieve %s (%x, %p, %p)",
- nsurl_access(url), flags, referer, post));
+ LLCACHE_LOG(("Retrieve %s (%x, %s, %p)", nsurl_access(url), flags,
+ referer==NULL?"":nsurl_access(referer), post));
/* Get rid of any url fragment */
@@ -1626,6 +2113,146 @@ static nserror llcache_fetch_ssl_error(llcache_object *object)
}
/**
+ * construct a sorted list of objects available for writeout operation
+ *
+ * The list contains fresh cacheable objects held in RAM with no
+ * pending fetches. Any objects with a remaining lifetime less than
+ * the configured minimum lifetime are simply not considered, they will
+ * become stale before pushing to backing store is worth the cost.
+ *
+ * \todo calculate useful cost metrics to improve sorting.
+ *
+ */
+static nserror
+build_candidate_list(struct llcache_object ***lst_out, int *lst_len_out)
+{
+ llcache_object *object, *next;
+ struct llcache_object **lst;
+ int lst_len = 0;
+ int remaining_lifetime;
+
+ lst = calloc(512, sizeof(struct llcache_object *));
+ if (lst == NULL)
+ return NSERROR_NOMEM;
+
+ for (object = llcache->cached_objects; object != NULL; object = next) {
+ next = object->next;
+
+ remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(&object->cache);
+
+ /* cacehable objects with no pending fetches, not
+ * already on disc and with sufficient lifetime to
+ * make disc cache worthwile
+ */
+ if ((object->candidate_count == 0) &&
+ (object->fetch.fetch == NULL) &&
+ (object->fetch.outstanding_query == false) &&
+ (object->store_state == LLCACHE_STATE_RAM) &&
+ (remaining_lifetime > llcache->minimum_lifetime)) {
+ lst[lst_len] = object;
+ lst_len++;
+ if (lst_len == 512)
+ break;
+ }
+ }
+
+ if (lst_len == 0) {
+ free(lst);
+ return NSERROR_NOT_FOUND;
+ }
+
+ /* sort list here */
+
+ *lst_len_out = lst_len;
+ *lst_out = lst;
+
+ return NSERROR_OK;
+}
+
+static nserror
+write_backing_store(struct llcache_object *object, size_t *written_out)
+{
+ nserror ret;
+ uint8_t *metadata;
+ size_t metadatasize;
+
+ /* put object data in backing store */
+ ret = guit->llcache->store(object->url,
+ BACKING_STORE_NONE,
+ object->source_data,
+ object->source_len);
+ if (ret != NSERROR_OK) {
+ /* unable to put source data in backing store */
+ return ret;
+ }
+
+ ret = llcache_serialise_metadata(object, &metadata, &metadatasize);
+ if (ret != NSERROR_OK) {
+ /* There has been a metadata serialisation error. Ensure the
+ * already written data object is invalidated.
+ */
+ guit->llcache->invalidate(object->url);
+ return ret;
+ }
+
+ ret = guit->llcache->store(object->url,
+ BACKING_STORE_META,
+ metadata,
+ metadatasize);
+ free(metadata);
+ if (ret != NSERROR_OK) {
+ /* There has been an error putting the metadata in the
+ * backing store. Ensure the data object is invalidated.
+ */
+ guit->llcache->invalidate(object->url);
+ return ret;
+ }
+ object->store_state = LLCACHE_STATE_DISC;
+
+ *written_out = object->source_len + metadatasize;
+
+ return NSERROR_OK;
+}
+
+/**
+ * possibly write objects data to backing store.
+ */
+static void llcache_persist(void *p)
+{
+ nserror ret;
+ size_t size_written;
+ size_t total_written = 0;
+ struct llcache_object **lst;
+ int lst_count;
+ int idx;
+
+ ret = build_candidate_list(&lst, &lst_count);
+ if (ret == NSERROR_OK) {
+ /* obtained a candidate list, make each object
+ * persistant in turn
+ */
+ for (idx = 0; idx < lst_count; idx++) {
+ ret = write_backing_store(lst[idx], &size_written);
+ if (ret != NSERROR_OK) {
+ break;
+ }
+ total_written += size_written;
+
+ if (total_written > llcache->bandwidth) {
+ /* The bandwidth limit has been reached.
+ * Writeout scheduled for the remaining objects
+ */
+ guit->browser->schedule(1000, llcache_persist, NULL);
+ break;
+ }
+ }
+
+ free(lst);
+ }
+}
+
+
+/**
* Handler for fetch events
*
* \param msg Fetch event
@@ -1724,6 +2351,11 @@ static void llcache_fetch_callback(const fetch_msg *msg, void *p)
}
llcache_object_cache_update(object);
+
+ /* record when the fetch finished */
+ object->cache.fin_time = time(NULL);
+
+ guit->browser->schedule(5000, llcache_persist, NULL);
}
break;
@@ -1833,26 +2465,6 @@ static llcache_object_user *llcache_object_find_user(const llcache_handle *handl
return user;
}
-/**
- * Remove a low-level cache object from a cache list
- *
- * \param object Object to remove
- * \param list List to remove from
- * \return NSERROR_OK
- */
-static nserror llcache_object_remove_from_list(llcache_object *object,
- llcache_object **list)
-{
- if (object == *list)
- *list = object->next;
- else
- object->prev->next = object->next;
-
- if (object->next != NULL)
- object->next->prev = object->prev;
-
- return NSERROR_OK;
-}
/**
* Determine if a low-level cache object resides in a given list
@@ -2110,8 +2722,8 @@ static nserror llcache_object_notify_users(llcache_object *object)
* \param snapshot Pointer to receive snapshot of \a object
* \return NSERROR_OK on success, appropriate error otherwise
*/
-static nserror llcache_object_snapshot(llcache_object *object,
- llcache_object **snapshot)
+static nserror
+llcache_object_snapshot(llcache_object *object, llcache_object **snapshot)
{
llcache_object *newobj;
nserror error;
@@ -2162,6 +2774,35 @@ static nserror llcache_object_snapshot(llcache_object *object,
return NSERROR_OK;
}
+/**
+ * total ram usage of object
+ */
+static inline uint32_t
+total_object_size(llcache_object *object)
+{
+ uint32_t tot;
+ size_t hdrc;
+
+ tot = sizeof(*object);
+ tot += nsurl_length(object->url);
+
+ if (object->source_data != NULL) {
+ tot += object->source_len;
+ }
+
+ tot += sizeof(llcache_header) * object->num_headers;
+
+ for (hdrc = 0; hdrc < object->num_headers; hdrc++) {
+ if (object->headers[hdrc].name != NULL) {
+ tot += strlen(object->headers[hdrc].name);
+ }
+ if (object->headers[hdrc].value != NULL) {
+ tot += strlen(object->headers[hdrc].value);
+ }
+ }
+
+ return tot;
+}
/******************************************************************************
* Public API *
@@ -2169,6 +2810,8 @@ static nserror llcache_object_snapshot(llcache_object *object,
/**
* Attempt to clean the cache
+ *
+ * The memory cache cleaning discards objects in order of increasing value.
*/
/* Exported interface documented in llcache.h */
void llcache_clean(void)
@@ -2179,15 +2822,10 @@ void llcache_clean(void)
LLCACHE_LOG(("Attempting cache clean"));
- /* Candidates for cleaning are (in order of priority):
- *
- * 1) Uncacheable objects with no users
- * 2) Stale cacheable objects with no users or pending fetches
- * 3) Fresh cacheable objects with no users or pending fetches
- */
-
- /* 1) Uncacheable objects with no users or fetches */
- for (object = llcache->uncached_objects; object != NULL; object = next) {
+ /* Uncacheable objects with no users or fetches */
+ for (object = llcache->uncached_objects;
+ object != NULL;
+ object = next) {
next = object->next;
/* The candidate count of uncacheable objects is always 0 */
@@ -2195,18 +2833,21 @@ void llcache_clean(void)
(object->candidate_count == 0) &&
(object->fetch.fetch == NULL) &&
(object->fetch.outstanding_query == false)) {
- LLCACHE_LOG(("Found victim %p", object));
+ LLCACHE_LOG(("Discarding uncachable object with no users (%p) %s", object, nsurl_access(object->url)));
llcache_object_remove_from_list(object,
&llcache->uncached_objects);
llcache_object_destroy(object);
} else {
- llcache_size += object->source_len + sizeof(*object);
+ llcache_size += total_object_size(object);
}
}
- /* 2) Stale cacheable objects with no users or pending fetches */
- for (object = llcache->cached_objects; object != NULL; object = next) {
+
+ /* Stale cacheable objects with no users or pending fetches */
+ for (object = llcache->cached_objects;
+ object != NULL;
+ object = next) {
next = object->next;
remaining_lifetime = llcache_object_rfc2616_remaining_lifetime(&object->cache);
@@ -2214,45 +2855,113 @@ void llcache_clean(void)
if ((object->users == NULL) &&
(object->candidate_count == 0) &&
(object->fetch.fetch == NULL) &&
- (object->fetch.outstanding_query == false)) {
-
- if (remaining_lifetime > 0) {
- /* object is fresh */
- llcache_size += object->source_len + sizeof(*object);
- } else {
- /* object is not fresh */
- LLCACHE_LOG(("Found stale cacheable object (%p) with no users or pending fetches", object));
+ (object->fetch.outstanding_query == false) &&
+ (remaining_lifetime <= 0)) {
+ /* object is stale */
+ LLCACHE_LOG(("discarding stale cacheable object with no users or pending fetches (%p) %s", object, nsurl_access(object->url)));
llcache_object_remove_from_list(object,
&llcache->cached_objects);
+
+ if (object->store_state == LLCACHE_STATE_DISC) {
+ guit->llcache->invalidate(object->url);
+ }
+
llcache_object_destroy(object);
- }
+
} else {
- llcache_size += object->source_len + sizeof(*object);
+ /* object has users so account for the storage */
+ llcache_size += total_object_size(object);
}
}
- /* 3) Fresh cacheable objects with no users or pending
- * fetches, only if the cache exceeds the configured size.
+ /* if the cache limit is exceeded try to make some objects
+ * persistant so their RAM can be reclaimed in the next
+ * step
*/
- if (llcache->limit < llcache_size) {
- for (object = llcache->cached_objects; object != NULL;
- object = next) {
- next = object->next;
+ if (llcache->limit < llcache_size) {
+ llcache_persist(NULL);
+ }
+
+ /* Source data of fresh cacheable objects with no users, no
+ * pending fetches and pushed to persistant store while the
+ * cache exceeds the configured size.
+ */
+ for (object = llcache->cached_objects;
+ ((llcache->limit < llcache_size) && (object != NULL));
+ object = next) {
+ next = object->next;
+ if ((object->users == NULL) &&
+ (object->candidate_count == 0) &&
+ (object->fetch.fetch == NULL) &&
+ (object->fetch.outstanding_query == false) &&
+ (object->store_state == LLCACHE_STATE_DISC)) {
+ free(object->source_data);
+ object->source_data = NULL;
- if ((object->users == NULL) &&
- (object->candidate_count == 0) &&
- (object->fetch.fetch == NULL) &&
- (object->fetch.outstanding_query == false)) {
- LLCACHE_LOG(("Found victim %p", object));
+ llcache_size -= object->source_len;
- llcache_size -=
- object->source_len + sizeof(*object);
+ LLCACHE_LOG(("Freeing source data for %p len:%d",
+ object,
+ object->source_len));
+ }
+ }
- llcache_object_remove_from_list(object,
+ /* Fresh cacheable objects with no users, no pending fetches
+ * and pushed to persistant store while the cache exceeds
+ * the configured size. Efectively just the object metadata.
+ */
+ for (object = llcache->cached_objects;
+ ((llcache->limit < llcache_size) && (object != NULL));
+ object = next) {
+ next = object->next;
+ if ((object->users == NULL) &&
+ (object->candidate_count == 0) &&
+ (object->fetch.fetch == NULL) &&
+ (object->fetch.outstanding_query == false) &&
+ (object->store_state == LLCACHE_STATE_DISC) &&
+ (object->source_data == NULL)) {
+ LLCACHE_LOG(("discarding backed object len:%d age:%d (%p) %s",
+ object->source_len,
+ time(NULL) - object->last_used,
+ object,
+ nsurl_access(object->url)));
+
+ llcache_size -= total_object_size(object);
+
+ llcache_object_remove_from_list(object,
&llcache->cached_objects);
- llcache_object_destroy(object);
- }
+ llcache_object_destroy(object);
+
+ }
+ }
+
+ /* Fresh cacheable objects with no users or pending fetches
+ * while the cache exceeds the configured size. These are the
+ * most valuble objects as replacing them is a full network
+ * fetch
+ */
+ for (object = llcache->cached_objects;
+ ((llcache->limit < llcache_size) && (object != NULL));
+ object = next) {
+ next = object->next;
+
+ if ((object->users == NULL) &&
+ (object->candidate_count == 0) &&
+ (object->fetch.fetch == NULL) &&
+ (object->fetch.outstanding_query == false) &&
+ (object->store_state == LLCACHE_STATE_RAM)) {
+ LLCACHE_LOG(("discarding fresh object len:%d age:%d (%p) %s",
+ object->source_len,
+ time(NULL) - object->last_used,
+ object,
+ nsurl_access(object->url)));
+
+ llcache_size -= object->source_len + sizeof(*object);
+
+ llcache_object_remove_from_list(object,
+ &llcache->cached_objects);
+ llcache_object_destroy(object);
}
}
@@ -2261,20 +2970,23 @@ void llcache_clean(void)
/* See llcache.h for documentation */
nserror
-llcache_initialise(llcache_query_callback cb, void *pw, uint32_t llcache_limit)
+llcache_initialise(const struct llcache_parameters *prm)
{
llcache = calloc(1, sizeof(struct llcache_s));
if (llcache == NULL) {
return NSERROR_NOMEM;
}
- llcache->query_cb = cb;
- llcache->query_cb_pw = pw;
- llcache->limit = llcache_limit;
+ llcache->query_cb = prm->cb;
+ llcache->query_cb_pw = prm->cb_ctx;
+ llcache->limit = prm->limit;
+ llcache->minimum_lifetime = prm->minimum_lifetime;
+ llcache->bandwidth = prm->bandwidth;
- LOG(("llcache initialised with a limit of %d bytes", llcache_limit));
+ LOG(("llcache initialising with a limit of %d bytes", llcache->limit));
- return NSERROR_OK;
+ /* backing store initialisation */
+ return guit->llcache->initialise(&prm->store);
}
/* See llcache.h for documentation */
@@ -2324,6 +3036,9 @@ void llcache_finalise(void)
llcache_object_destroy(object);
}
+ /* backing store finalisation */
+ guit->llcache->finalise();
+
free(llcache);
llcache = NULL;
}
diff --git a/content/llcache.h b/content/llcache.h
index 3d8232cae..a9ed1861a 100644
--- a/content/llcache.h
+++ b/content/llcache.h
@@ -76,7 +76,7 @@ typedef struct {
} data; /**< Event data */
} llcache_event;
-/**
+/**
* Client callback for low-level cache events
*
* \param handle Handle for which event is issued
@@ -84,18 +84,18 @@ typedef struct {
* \param pw Pointer to client-specific data
* \return NSERROR_OK on success, appropriate error otherwise.
*/
-typedef nserror (*llcache_handle_callback)(llcache_handle *handle,
+typedef nserror (*llcache_handle_callback)(llcache_handle *handle,
const llcache_event *event, void *pw);
/** Flags for low-level cache object retrieval */
enum llcache_retrieve_flag {
/* Note: We're permitted a maximum of 16 flags which must reside in the
- * bottom 16 bits of the flags word. See hlcache.h for further details.
+ * bottom 16 bits of the flags word. See hlcache.h for further details.
*/
/** Force a new fetch */
- LLCACHE_RETRIEVE_FORCE_FETCH = (1 << 0),
+ LLCACHE_RETRIEVE_FORCE_FETCH = (1 << 0),
/** Requested URL was verified */
- LLCACHE_RETRIEVE_VERIFIABLE = (1 << 1),
+ LLCACHE_RETRIEVE_VERIFIABLE = (1 << 1),
/**< No error pages */
LLCACHE_RETRIEVE_NO_ERROR_PAGES = (1 << 2),
/**< Stream data (implies that object is not cacheable) */
@@ -149,21 +149,89 @@ typedef nserror (*llcache_query_response)(bool proceed, void *cbpw);
* \param cbpw Opaque value to pass into \a cb
* \return NSERROR_OK on success, appropriate error otherwise
*
- * \note This callback should return immediately. Once a suitable answer to
- * the query has been obtained, the provided response callback should be
+ * \note This callback should return immediately. Once a suitable answer to
+ * the query has been obtained, the provided response callback should be
* called. This is intended to be an entirely asynchronous process.
*/
typedef nserror (*llcache_query_callback)(const llcache_query *query, void *pw,
llcache_query_response cb, void *cbpw);
/**
+ * Parameters to configure the low level cache backing store.
+ */
+struct llcache_store_parameters {
+ const char *path; /**< The path to the backing store */
+
+ size_t limit; /**< The backing store upper bound target size */
+ size_t hysteresis; /**< The hysteresis around the target size */
+
+ /** log2 of the default maximum number of entries the cache
+ * can track.
+ *
+ * If unset this defaults to 16 (65536 entries) The cache
+ * control file takes precedence so cache data remains
+ * portable between builds with differing defaults.
+ */
+ unsigned int entry_size;
+
+ /** log2 of the default number of entries in the mapping between
+ * the url and cache entries.
+ *
+ * @note This is exposing an internal implementation detail of
+ * the filesystem based default backing store implementation.
+ * However it is likely any backing store implementation will
+ * need some way to map url to cache entries so it is a
+ * generally useful configuration value.
+ *
+ * Too small a value will cause unecessary collisions and
+ * cache misses and larger values cause proportionaly larger
+ * amounts of memory to be used.
+ *
+ * The "birthday paradox" means that the hash will experience
+ * a collision in every 2^(address_size/2) urls the cache
+ * stores.
+ *
+ * A value of 20 means one object stored in every 1024 will
+ * cause a collion and a cache miss while using two megabytes
+ * of storage.
+ *
+ * If unset this defaults to 20 (1048576 entries using two
+ * megabytes) The cache control file takes precedence so cache
+ * data remains portable between builds with differing
+ * defaults.
+ */
+ unsigned int address_size;
+};
+
+/**
+ * Parameters to configure the low level cache.
+ */
+struct llcache_parameters {
+ llcache_query_callback cb; /**< Query handler for llcache */
+ void *cb_ctx; /**< Pointer to llcache query handler data */
+
+ size_t limit; /**< The target upper bound for the RAM cache size */
+ size_t hysteresis; /**< The hysteresis around the target size */
+
+ int minimum_lifetime; /**< The minimum lifetime to consider
+ * sending objects to backing store.
+ */
+
+ size_t bandwidth; /**< The maximum bandwidth to allow the
+ * backing store to use.
+ */
+
+ struct llcache_store_parameters store;
+};
+
+/**
* Initialise the low-level cache
*
* \param cb Query handler
* \param pw Pointer to query handler data
* \return NSERROR_OK on success, appropriate error otherwise.
*/
-nserror llcache_initialise(llcache_query_callback cb, void *pw, uint32_t llcache_limit);
+nserror llcache_initialise(const struct llcache_parameters *parameters);
/**
* Finalise the low-level cache
@@ -280,12 +348,12 @@ const uint8_t *llcache_handle_get_source_data(const llcache_handle *handle,
* \return Header value, or NULL if header does not exist
*
* \todo Make the key an enumeration, to avoid needless string comparisons
- * \todo Forcing the client to parse the header value seems wrong.
- * Better would be to return the actual value part and an array of
+ * \todo Forcing the client to parse the header value seems wrong.
+ * Better would be to return the actual value part and an array of
* key-value pairs for any additional parameters.
* \todo Deal with multiple headers of the same key (e.g. Set-Cookie)
*/
-const char *llcache_handle_get_header(const llcache_handle *handle,
+const char *llcache_handle_get_header(const llcache_handle *handle,
const char *key);
/**
@@ -295,7 +363,7 @@ const char *llcache_handle_get_header(const llcache_handle *handle,
* \param b Second handle
* \return True if handles reference the same object, false otherwise
*/
-bool llcache_handle_references_same_object(const llcache_handle *a,
+bool llcache_handle_references_same_object(const llcache_handle *a,
const llcache_handle *b);
#endif
diff --git a/desktop/netsurf.c b/desktop/netsurf.c
index 1c9d880f7..18ce1264f 100644
--- a/desktop/netsurf.c
+++ b/desktop/netsurf.c
@@ -67,11 +67,23 @@
*/
#define SPECULATE_SMALL 4096
-/* the time between cache clean runs in ms */
+/** the time between image cache clean runs in ms. */
#define IMAGE_CACHE_CLEAN_TIME (10 * 1000)
+/** default time between content cache cleans. */
#define HL_CACHE_CLEAN_TIME (2 * IMAGE_CACHE_CLEAN_TIME)
+/** default minimum object time before object is pushed to backing store. */
+#define LLCACHE_MIN_DISC_LIFETIME (60 * 30)
+
+/** default maximum bandwidth for backing store writeout. */
+#define LLCACHE_MAX_DISC_BANDWIDTH (128 * 1024)
+
+/** ensure there is a minimal amount of memory for source objetcs and
+ * decoded bitmaps.
+ */
+#define MINIMUM_MEMORY_CACHE_SIZE (2 * 1024 * 1024)
+
bool netsurf_quit = false;
static void netsurf_lwc_iterator(lwc_string *str, void *pw)
@@ -108,8 +120,6 @@ static nserror netsurf_llcache_query_handler(const llcache_query *query,
return NSERROR_OK;
}
-#define MINIMUM_MEMORY_CACHE_SIZE (2 * 1024 * 1024)
-
/* exported interface documented in desktop/netsurf.h */
nserror netsurf_register(struct netsurf_table *table)
{
@@ -120,12 +130,15 @@ nserror netsurf_register(struct netsurf_table *table)
/* exported interface documented in desktop/netsurf.h */
nserror netsurf_init(const char *messages)
{
- nserror error;
+ nserror ret;
struct utsname utsname;
- nserror ret = NSERROR_OK;
struct hlcache_parameters hlcache_parameters = {
.bg_clean_time = HL_CACHE_CLEAN_TIME,
- .cb = netsurf_llcache_query_handler,
+ .llcache = {
+ .cb = netsurf_llcache_query_handler,
+ .minimum_lifetime = LLCACHE_MIN_DISC_LIFETIME,
+ .bandwidth = LLCACHE_MAX_DISC_BANDWIDTH,
+ }
};
struct image_cache_parameters image_cache_parameters = {
.bg_clean_time = IMAGE_CACHE_CLEAN_TIME,
@@ -155,75 +168,87 @@ nserror netsurf_init(const char *messages)
messages_load(messages);
/* corestrings init */
- error = corestrings_init();
- if (error != NSERROR_OK)
- return error;
+ ret = corestrings_init();
+ if (ret != NSERROR_OK)
+ return ret;
/* set up cache limits based on the memory cache size option */
- hlcache_parameters.limit = nsoption_int(memory_cache_size);
+ hlcache_parameters.llcache.limit = nsoption_int(memory_cache_size);
- if (hlcache_parameters.limit < MINIMUM_MEMORY_CACHE_SIZE) {
- hlcache_parameters.limit = MINIMUM_MEMORY_CACHE_SIZE;
- LOG(("Setting minimum memory cache size to %d",
- hlcache_parameters.limit));
+ if (hlcache_parameters.llcache.limit < MINIMUM_MEMORY_CACHE_SIZE) {
+ hlcache_parameters.llcache.limit = MINIMUM_MEMORY_CACHE_SIZE;
+ LOG(("Setting minimum memory cache size %d",
+ hlcache_parameters.llcache.limit));
}
/* image cache is 25% of total memory cache size */
- image_cache_parameters.limit = (hlcache_parameters.limit * 25) / 100;
+ image_cache_parameters.limit = (hlcache_parameters.llcache.limit * 25) / 100;
/* image cache hysteresis is 20% of the image cache size */
image_cache_parameters.hysteresis = (image_cache_parameters.limit * 20) / 100;
/* account for image cache use from total */
- hlcache_parameters.limit -= image_cache_parameters.limit;
+ hlcache_parameters.llcache.limit -= image_cache_parameters.limit;
+
+ /* set backing store target limit */
+ hlcache_parameters.llcache.store.limit = nsoption_int(disc_cache_size);
+
+ /* set backing store hysterissi to 20% */
+ hlcache_parameters.llcache.store.hysteresis = (hlcache_parameters.llcache.store.limit * 20) / 100;;
+
+ /* set the path to the backing store */
+ /** \todo set the backing store path properly */
+ hlcache_parameters.llcache.store.path = "/tmp/ns";
/* image handler bitmap cache */
- error = image_cache_init(&image_cache_parameters);
- if (error != NSERROR_OK)
- return error;
+ ret = image_cache_init(&image_cache_parameters);
+ if (ret != NSERROR_OK)
+ return ret;
/* content handler initialisation */
- error = nscss_init();
- if (error != NSERROR_OK)
- return error;
+ ret = nscss_init();
+ if (ret != NSERROR_OK)
+ return ret;
- error = html_init();
- if (error != NSERROR_OK)
- return error;
+ ret = html_init();
+ if (ret != NSERROR_OK)
+ return ret;
- error = image_init();
- if (error != NSERROR_OK)
- return error;
+ ret = image_init();
+ if (ret != NSERROR_OK)
+ return ret;
- error = textplain_init();
- if (error != NSERROR_OK)
- return error;
+ ret = textplain_init();
+ if (ret != NSERROR_OK)
+ return ret;
- error = mimesniff_init();
- if (error != NSERROR_OK)
- return error;
+ ret = mimesniff_init();
+ if (ret != NSERROR_OK)
+ return ret;
url_init();
setlocale(LC_ALL, "C");
/* initialise the fetchers */
- error = fetch_init();
- if (error != NSERROR_OK)
- return error;
+ ret = fetch_init();
+ if (ret != NSERROR_OK)
+ return ret;
/* Initialise the hlcache and allow it to init the llcache for us */
- hlcache_initialise(&hlcache_parameters);
+ ret = hlcache_initialise(&hlcache_parameters);
+ if (ret != NSERROR_OK)
+ return ret;
/* Initialize system colours */
- error = ns_system_colour_init();
- if (error != NSERROR_OK)
- return error;
+ ret = ns_system_colour_init();
+ if (ret != NSERROR_OK)
+ return ret;
js_initialise();
- return ret;
+ return NSERROR_OK;
}