[project @ 2003-02-09 12:58:14 by bursa]

Reorganization and rewrite of fetch, cache, and content handling.

svn path=/import/netsurf/; revision=96

8 files changed, 1003 insertions, 0 deletions

diff --git a/content/cache.c b/content/cache.c
new file mode 100644
index 000000000..30a39053b
--- /dev/null
+++ b/content/cache.c
@@ -0,0 +1,245 @@
+/**
+ * $Id: cache.c,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#include <assert.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include "netsurf/content/cache.h"
+#include "netsurf/utils/utils.h"
+#include "netsurf/utils/log.h"
+
+#ifndef TEST
+#include "netsurf/desktop/browser.h"
+#else
+#include <unistd.h>
+struct content {
+	char *url;
+	struct cache_entry *cache;
+	unsigned long size;
+};
+void content_destroy(struct content *c);
+#endif
+
+
+/**
+ * internal structures and declarations
+ */
+
+struct cache_entry {
+	struct content *content;
+	unsigned int use_count;
+	time_t t;
+	struct cache_entry *next, *prev;
+};
+
+/* doubly-linked lists using a sentinel */
+/* TODO: replace with a structure which can be searched faster */
+static struct cache_entry inuse_list_sentinel  = {0, 0, 0, &inuse_list_sentinel,  &inuse_list_sentinel};
+static struct cache_entry unused_list_sentinel = {0, 0, 0, &unused_list_sentinel, &unused_list_sentinel};
+static struct cache_entry *inuse_list  = &inuse_list_sentinel;
+static struct cache_entry *unused_list = &unused_list_sentinel;
+
+static unsigned long max_size = 1024*1024;	/* TODO: make this configurable */
+static unsigned long current_size = 0;
+
+
+/**
+ * cache_init -- initialise the cache manager
+ */
+
+void cache_init(void)
+{
+}
+
+
+/**
+ * cache_quit -- terminate the cache manager
+ */
+
+void cache_quit(void)
+{
+}
+
+
+/**
+ * cache_get -- retrieve url from memory cache or disc cache
+ */
+
+struct content * cache_get(char * const url)
+{
+	struct cache_entry *e;
+
+	/* search inuse_list first */
+	for (e = inuse_list->next; e != inuse_list && strcmp(e->content->url, url) != 0; e = e->next)
+		;
+	if (e != inuse_list) {
+		LOG(("'%s' in inuse_list, content %p, use_count %u", url, e->content, e->use_count));
+		e->use_count++;
+		return e->content;
+	}
+
+	/* search unused_list if not found */
+	for (e = unused_list->next; e != unused_list && strcmp(e->content->url, url) != 0; e = e->next)
+		;
+	if (e != unused_list) {
+		LOG(("'%s' in unused_list, content %p", url, e->content));
+		/* move to inuse_list */
+		e->use_count = 1;
+		e->prev->next = e->next;
+		e->next->prev = e->prev;
+		e->prev = inuse_list->prev;
+		e->next = inuse_list;
+		inuse_list->prev->next = e;
+		inuse_list->prev = e;
+		return e->content;
+	}
+
+	LOG(("'%s' not in cache", url));
+	return 0;
+}
+
+
+/**
+ * cache_put -- place content in the memory cache
+ */
+
+void cache_put(struct content * content)
+{
+	struct cache_entry * e;
+	LOG(("content %p, url '%s'", content, content->url));
+
+	current_size += content->size;
+	/* clear old data from the usused_list until the size drops below max_size */
+	while (max_size < current_size && unused_list->next != unused_list) {
+		e = unused_list->next;
+		LOG(("size %lu, removing %p '%s'", current_size, e->content, e->content->url));
+		/* TODO: move to disc cache */
+		current_size -= e->content->size;
+		content_destroy(e->content);
+		unused_list->next = e->next;
+		e->next->prev = e->prev;
+		xfree(e);
+	}
+
+	/* add the new content to the inuse_list */
+	e = xcalloc(1, sizeof(struct cache_entry));
+	e->content = content;
+	e->use_count = 1;
+	e->prev = inuse_list->prev;
+	e->next = inuse_list;
+	inuse_list->prev->next = e;
+	inuse_list->prev = e;
+	content->cache = e;
+}
+
+
+/**
+ * cache_free -- free a cache object if it is no longer used
+ */
+
+void cache_free(struct content * content)
+{
+	struct cache_entry * e = content->cache;
+
+	assert(e != 0);
+	LOG(("content %p, url '%s', use_count %u", content, content->url, e->use_count));
+
+	assert(e->use_count != 0);
+	e->use_count--;
+	if (e->use_count == 0) {
+		/* move to unused_list or destroy if insufficient space */
+		e->use_count = 0;
+		e->t = time(0);
+		e->prev->next = e->next;
+		e->next->prev = e->prev;
+		if (max_size < current_size) {
+			LOG(("size %lu, removing", current_size));
+			/* TODO: move to disc cache */
+			current_size -= e->content->size;
+			content_destroy(e->content);
+			xfree(e);
+		} else {
+			LOG(("size %lu, moving to unused_list", current_size));
+			e->prev = unused_list->prev;
+			e->next = unused_list;
+			unused_list->prev->next = e;
+			unused_list->prev = e;
+		}
+	}
+}
+
+
+/**
+ * cache_dump -- dump contents of cache
+ */
+
+void cache_dump(void) {
+	struct cache_entry * e;
+	LOG(("size %lu", current_size));
+	LOG(("inuse_list:"));
+	for (e = inuse_list->next; e != inuse_list; e = e->next)
+		LOG(("  content %p, url '%s', use_count %u", e->content, e->content->url, e->use_count));
+	LOG(("unused_list (time now %lu):", time(0)));
+	for (e = unused_list->next; e != unused_list; e = e->next)
+		LOG(("  content %p, url '%s', t %lu", e->content, e->content->url, e->t));
+	LOG(("end"));
+}
+
+
+/**
+ * testing framework
+ */
+
+#ifdef TEST
+struct content test[] = {
+	{"aaa", 0, 200 * 1024},
+	{"bbb", 0, 100 * 1024},
+	{"ccc", 0, 400 * 1024},
+	{"ddd", 0, 600 * 1024},
+	{"eee", 0, 300 * 1024},
+	{"fff", 0, 500 * 1024},
+};
+
+#define TEST_COUNT (sizeof(test) / sizeof(test[0]))
+
+unsigned int test_state[TEST_COUNT];
+
+void content_destroy(struct content *c)
+{
+}
+
+int main(void)
+{
+	int i;
+	struct content *c;
+	for (i = 0; i != TEST_COUNT; i++)
+		test_state[i] = 0;
+
+	cache_init();
+
+	for (i = 0; i != 100; i++) {
+		int x = rand() % TEST_COUNT;
+		switch (rand() % 2) {
+			case 0:
+				c = cache_get(test[x].url);
+				if (c == 0) {
+					assert(test_state[x] == 0);
+					cache_put(&test[x]);
+				} else
+					assert(c == &test[x]);
+				test_state[x]++;
+				break;
+			case 1:
+				if (test_state[x] != 0) {
+					cache_free(&test[x]);
+					test_state[x]--;
+				}
+				break;
+		}
+	}
+	cache_dump();
+	return 0;
+}
+#endif
diff --git a/content/cache.h b/content/cache.h
new file mode 100644
index 000000000..2ee4e3b81
--- /dev/null
+++ b/content/cache.h
@@ -0,0 +1,38 @@
+/**
+ * $Id: cache.h,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+/**
+ * Using the cache:
+ *
+ *     cache_init();
+ *     ...
+ *     c = cache_get(url);
+ *     if (c == 0) {
+ *         ... (create c) ...
+ *         cache_put(c);
+ *     }
+ *     ...
+ *     cache_free(c);
+ *     ...
+ *     cache_quit();
+ *
+ * cache_free informs the cache that the content is no longer being used, so
+ * it can be deleted from the cache if necessary. There must be a call to
+ * cache_free for each cache_get or cache_put.
+ */
+
+#ifndef _NETSURF_DESKTOP_CACHE_H_
+#define _NETSURF_DESKTOP_CACHE_H_
+
+struct content;
+struct cache_entry;
+
+void cache_init(void);
+void cache_quit(void);
+struct content * cache_get(char * const url);
+void cache_put(struct content * content);
+void cache_free(struct content * content);
+void cache_dump(void);
+
+#endif
diff --git a/content/content.c b/content/content.c
new file mode 100644
index 000000000..c8601400d
--- /dev/null
+++ b/content/content.c
@@ -0,0 +1,145 @@
+/**
+ * $Id: content.c,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#include <assert.h>
+#include <string.h>
+#include <stdlib.h>
+#include "netsurf/content/content.h"
+#include "netsurf/render/html.h"
+#include "netsurf/render/textplain.h"
+#include "netsurf/utils/utils.h"
+
+
+/* mime_map must be in sorted order by mime_type */
+struct mime_entry {
+	char mime_type[16];
+	content_type type;
+};
+static const struct mime_entry mime_map[] = {
+/*	{"image/png", CONTENT_PNG},
+	{"text/css", CONTENT_CSS},*/
+	{"text/html", CONTENT_HTML},
+	{"text/plain", CONTENT_TEXTPLAIN},
+};
+#define MIME_MAP_COUNT (sizeof(mime_map) / sizeof(mime_map[0]))
+
+/* handler_map must be ordered as enum content_type */
+struct handler_entry {
+	void (*create)(struct content *c);
+	void (*process_data)(struct content *c, char *data, unsigned long size);
+	int (*convert)(struct content *c, unsigned int width, unsigned int height);
+	void (*revive)(struct content *c, unsigned int width, unsigned int height);
+	void (*reformat)(struct content *c, unsigned int width, unsigned int height);
+	void (*destroy)(struct content *c);
+};
+static const struct handler_entry handler_map[] = {
+	{html_create, html_process_data, html_convert, html_revive, html_reformat, html_destroy},
+	{textplain_create, textplain_process_data, textplain_convert,
+		textplain_revive, textplain_reformat, textplain_destroy},
+/*	{css_create, css_process_data, css_convert, css_revive, css_destroy},
+	{png_create, png_process_data, png_convert, png_revive, png_destroy},*/
+};
+
+
+/**
+ * content_lookup -- look up mime type
+ */
+
+content_type content_lookup(const char *mime_type)
+{
+	struct mime_entry *m;
+	m = bsearch(mime_type, mime_map, MIME_MAP_COUNT, sizeof(mime_map[0]),
+			(int (*)(const void *, const void *)) strcmp);
+	if (m == 0)
+		return CONTENT_OTHER;
+	return m->type;
+}
+
+
+/**
+ * content_create -- create a content structure of the specified mime type
+ */
+
+struct content * content_create(content_type type, char *url)
+{
+	struct content *c;
+	assert(type < CONTENT_OTHER);
+	c = xcalloc(1, sizeof(struct content));
+	c->url = xstrdup(url);
+	c->type = type;
+	c->status = CONTENT_LOADING;
+	c->size = sizeof(struct content);
+	handler_map[type].create(c);
+	return c;
+}
+
+
+/**
+ * content_process_data -- process a block source data
+ */
+
+void content_process_data(struct content *c, char *data, unsigned long size)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	assert(c->status == CONTENT_LOADING);
+	handler_map[c->type].process_data(c, data, size);
+}
+
+
+/**
+ * content_convert -- all data has arrived, complete the conversion
+ */
+
+int content_convert(struct content *c, unsigned long width, unsigned long height)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	assert(c->status == CONTENT_LOADING);
+	if (handler_map[c->type].convert(c, width, height))
+		return 1;
+	c->status = CONTENT_READY;
+	return 0;
+}
+
+
+/**
+ * content_revive -- fix content that has been loaded from the cache
+ *   eg. load dependencies, reformat to current width
+ */
+
+void content_revive(struct content *c, unsigned long width, unsigned long height)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	assert(c->status == CONTENT_READY);
+	handler_map[c->type].revive(c, width, height);
+}
+
+
+/**
+ * content_reformat -- reformat to new size
+ */
+
+void content_reformat(struct content *c, unsigned long width, unsigned long height)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	assert(c->status == CONTENT_READY);
+	handler_map[c->type].reformat(c, width, height);
+}
+
+
+/**
+ * content_destroy -- free content
+ */
+
+void content_destroy(struct content *c)
+{
+	assert(c != 0);
+	assert(c->type < CONTENT_OTHER);
+	handler_map[c->type].destroy(c);
+	xfree(c);
+}
+
diff --git a/content/content.h b/content/content.h
new file mode 100644
index 000000000..ed93e7c24
--- /dev/null
+++ b/content/content.h
@@ -0,0 +1,95 @@
+/**
+ * $Id: content.h,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#ifndef _NETSURF_DESKTOP_CONTENT_H_
+#define _NETSURF_DESKTOP_CONTENT_H_
+
+#include "libxml/HTMLparser.h"
+#include "netsurf/content/cache.h"
+#include "netsurf/render/css.h"
+#include "netsurf/render/box.h"
+#include "netsurf/riscos/font.h"
+
+
+/**
+ * A struct content corresponds to a single url.
+ *
+ * It is in one of the following states:
+ *   CONTENT_FETCHING - the data is being fetched and/or converted
+ *                      for use by the browser
+ *   CONTENT_READY - the content has been processed and is ready
+ *                      to display
+ *
+ * The converted data is stored in the cache, not the source data.
+ * Users of the structure are counted in use_count; when use_count = 0
+ * the content may be removed from the memory cache.
+ */
+
+typedef enum {CONTENT_HTML, CONTENT_TEXTPLAIN, CONTENT_CSS,
+	CONTENT_PNG, CONTENT_OTHER} content_type;
+
+struct box_position
+{
+  struct box* box;
+  int actual_box_x;
+  int actual_box_y;
+  int plot_index;
+  int pixel_offset;
+  int char_offset;
+};
+
+struct content
+{
+  char *url;
+  content_type type;
+  enum {CONTENT_LOADING, CONTENT_READY} status;
+
+  union
+  {
+    struct
+    {
+      htmlParserCtxt* parser;
+      xmlDoc* document;
+      xmlNode* markup;
+      struct box* layout;
+      struct css_stylesheet* stylesheet;
+      struct css_style* style;
+      struct {
+        struct box_position start;
+        struct box_position end;
+        enum {alter_UNKNOWN, alter_START, alter_END} altering;
+        int selected; /* 0 = unselected, 1 = selected */
+      } text_selection;
+      struct font_set* fonts;
+      struct page_elements elements;
+    } html;
+
+    struct
+    {
+      struct css_stylesheet * stylesheet;
+    } css;
+
+    struct
+    {
+      unsigned long width, height;
+      char * sprite;
+    } image;
+  
+  } data;
+
+  struct cache_entry *cache;
+  unsigned long size;
+  char *title;
+};
+
+
+content_type content_lookup(const char *mime_type);
+struct content * content_create(content_type type, char *url);
+void content_process_data(struct content *c, char *data, unsigned long size);
+int content_convert(struct content *c, unsigned long width, unsigned long height);
+void content_revive(struct content *c, unsigned long width, unsigned long height);
+void content_reformat(struct content *c, unsigned long width, unsigned long height);
+void content_destroy(struct content *c);
+
+#endif
diff --git a/content/fetch.c b/content/fetch.c
new file mode 100644
index 000000000..63283238a
--- /dev/null
+++ b/content/fetch.c
@@ -0,0 +1,295 @@
+/**
+ * $Id: fetch.c,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#include <assert.h>
+#include <time.h>
+#include "curl/curl.h"
+#include "netsurf/content/fetch.h"
+#include "netsurf/utils/utils.h"
+#include "netsurf/utils/log.h"
+
+#ifndef TEST
+#else
+#include <unistd.h>
+#endif
+
+struct fetch
+{
+	time_t start_time;
+	CURL * curl_handle;
+	void (*callback)(fetch_msg msg, void *p, char *data, unsigned long size);
+	int had_headers : 1;
+	int in_callback : 1;
+	int aborting : 1;
+	char *url;
+	char error_buffer[CURL_ERROR_SIZE];
+	void *p;
+};
+
+static const char * const user_agent = "NetSurf";
+static CURLM * curl_multi;
+
+static size_t fetch_curl_data(void * data, size_t size, size_t nmemb, struct fetch *f);
+
+
+/**
+ * fetch_init -- initialise the fetcher
+ */
+
+void fetch_init(void)
+{
+	CURLcode code;
+
+	code = curl_global_init(CURL_GLOBAL_ALL);
+	if (code != CURLE_OK)
+		die("curl_global_init failed");
+
+	curl_multi = curl_multi_init();
+	if (curl_multi == 0)
+		die("curl_multi_init failed");
+}
+
+
+/**
+ * fetch_quit -- clean up for quit
+ */
+
+void fetch_quit(void)
+{
+	CURLMcode codem;
+
+	codem = curl_multi_cleanup(curl_multi);
+	if (codem != CURLM_OK)
+		LOG(("curl_multi_cleanup failed: ignoring"));
+
+	curl_global_cleanup();
+}
+
+
+/**
+ * fetch_start -- start fetching data for the given url
+ *
+ * Returns immediately. The callback function will be called when
+ * something interesting happens.
+ */
+
+struct fetch * fetch_start(char *url, char *referer,
+                 void (*callback)(fetch_msg msg, void *p, char *data, unsigned long size), void *p)
+{
+	struct fetch* fetch = (struct fetch*) xcalloc(1, sizeof(struct fetch));
+	CURLcode code;
+	CURLMcode codem;
+
+	LOG(("fetch %p, url '%s'", fetch, url));
+  
+	fetch->start_time = time(&fetch->start_time);
+	fetch->callback = callback;
+	fetch->had_headers = 0;
+	fetch->in_callback = 0;
+	fetch->aborting = 0;
+	fetch->url = xstrdup(url);
+	fetch->p = p;
+
+	/* create the curl easy handle */
+	fetch->curl_handle = curl_easy_init();
+	assert(fetch->curl_handle != 0);  /* TODO: handle curl errors */
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_URL, fetch->url);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_PRIVATE, fetch);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_ERRORBUFFER, fetch->error_buffer);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_WRITEFUNCTION, fetch_curl_data);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_WRITEDATA, fetch);
+	assert(code == CURLE_OK);
+	code = curl_easy_setopt(fetch->curl_handle, CURLOPT_USERAGENT, user_agent);
+	assert(code == CURLE_OK);
+	if (referer != 0) {
+		code = curl_easy_setopt(fetch->curl_handle, CURLOPT_REFERER, referer);
+		assert(code == CURLE_OK);
+	} 
+
+	/* add to the global curl multi handle */
+	codem = curl_multi_add_handle(curl_multi, fetch->curl_handle);
+	assert(codem == CURLM_OK || codem == CURLM_CALL_MULTI_PERFORM);
+
+	/* do any possible work on the fetch */
+	while (codem == CURLM_CALL_MULTI_PERFORM) {
+		int running;
+		codem = curl_multi_perform(curl_multi, &running);
+		assert(codem == CURLM_OK || codem == CURLM_CALL_MULTI_PERFORM);
+	}
+
+	return fetch;
+}
+
+
+/**
+ * fetch_abort -- stop a fetch
+ */
+
+void fetch_abort(struct fetch *f)
+{
+	CURLMcode codem;
+
+	assert(f != 0);
+	LOG(("fetch %p, url '%s'", f, f->url));
+  
+	if (f->in_callback) {
+		LOG(("in callback: will abort later"));
+		f->aborting = 1;
+		return;
+	}
+  
+	/* remove from curl */
+	codem = curl_multi_remove_handle(curl_multi, f->curl_handle);
+	assert(codem == CURLM_OK);
+	curl_easy_cleanup(f->curl_handle);
+
+	xfree(f->url);
+	xfree(f);
+}
+
+
+/**
+ * fetch_poll -- do some work on current fetches
+ *
+ * Must return as soon as possible.
+ */
+
+void fetch_poll(void)
+{
+	CURLcode code;
+	CURLMcode codem;
+	int running, queue;
+	CURLMsg * curl_msg;
+	struct fetch *f;
+
+	/* do any possible work on the current fetches */
+	do {
+		codem = curl_multi_perform(curl_multi, &running);
+		assert(codem == CURLM_OK || codem == CURLM_CALL_MULTI_PERFORM);
+	} while (codem == CURLM_CALL_MULTI_PERFORM);
+
+	/* process curl results */
+	curl_msg = curl_multi_info_read(curl_multi, &queue);
+	while (curl_msg) {
+		switch (curl_msg->msg) {
+			case CURLMSG_DONE:
+				/* find the structure associated with this fetch */
+				code = curl_easy_getinfo(curl_msg->easy_handle, CURLINFO_PRIVATE, &f);
+				assert(code == CURLE_OK);
+
+				LOG(("CURLMSG_DONE, result %i", curl_msg->data.result));
+
+				/* inform the caller that the fetch is done */
+				if (curl_msg->data.result == CURLE_OK)
+					f->callback(FETCH_FINISHED, f->p, 0, 0);
+				else if (curl_msg->data.result != CURLE_WRITE_ERROR)
+					f->callback(FETCH_ERROR, f->p, f->error_buffer, 0);
+
+				/* clean up fetch */
+				fetch_abort(f);
+
+				break;
+
+			default:
+				assert(0);
+		}
+		curl_msg = curl_multi_info_read(curl_multi, &queue);
+	}
+}
+
+
+/**
+ * fetch_curl_data -- callback function for curl (internal)
+ */
+
+size_t fetch_curl_data(void * data, size_t size, size_t nmemb, struct fetch *f)
+{
+	f->in_callback = 1;
+
+	LOG(("fetch %p, size %lu", f, size * nmemb));
+
+	if (!f->had_headers) {
+		/* find the content type and inform the caller */
+		char *type;
+		CURLcode code;
+
+		code = curl_easy_getinfo(f->curl_handle, CURLINFO_CONTENT_TYPE, &type);
+		assert(code == CURLE_OK);
+
+		if (type == 0)
+			type = "text/html";  /* TODO: find type of file: urls */
+
+		LOG(("FETCH_TYPE, '%s'", type));
+		f->callback(FETCH_TYPE, f->p, type, 0);
+		if (f->aborting) {
+			f->in_callback = 0;
+			return 0;
+		}
+
+		f->had_headers = 1;
+	}
+
+	/* send data to the caller */
+	LOG(("FETCH_DATA"));
+	f->callback(FETCH_DATA, f->p, data, size * nmemb);
+
+	f->in_callback = 0;
+	return size * nmemb;
+}
+
+
+/**
+ * testing framework
+ */
+
+#ifdef TEST
+struct test {char *url; struct fetch *f;};
+
+void callback(fetch_msg msg, struct test *t, char *data, unsigned long size)
+{
+	printf("%s: ", t->url);
+	switch (msg) {
+		case FETCH_TYPE:
+			printf("FETCH_TYPE '%s'", data);
+			break;
+		case FETCH_DATA:
+			printf("FETCH_DATA %lu", size);
+			break;
+		case FETCH_FINISHED:
+			printf("FETCH_FINISHED");
+			break;
+		case FETCH_ERROR:
+			printf("FETCH_ERROR '%s'", data);
+			break;
+		default:
+			assert(0);
+	}
+	printf("\n");
+}
+
+struct test test[] = {
+	{"http://www.oxfordstudent.com/", 0},
+	{"http://www.google.co.uk/", 0},
+	{"http://doesnt.exist/", 0},
+	{"blah://blah", 0},
+};
+
+int main(void)
+{
+	int i;
+	fetch_init();
+	for (i = 0; i != sizeof(test) / sizeof(test[0]); i++)
+		test[i].f = fetch_start(test[i].url, 0, callback, &test[i]);
+	while (1) {
+		fetch_poll();
+		sleep(1);
+	}
+	return 0;
+}
+#endif
+
diff --git a/content/fetch.h b/content/fetch.h
new file mode 100644
index 000000000..171bf33cb
--- /dev/null
+++ b/content/fetch.h
@@ -0,0 +1,20 @@
+/**
+ * $Id: fetch.h,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#ifndef _NETSURF_DESKTOP_FETCH_H_
+#define _NETSURF_DESKTOP_FETCH_H_
+
+typedef enum {FETCH_TYPE, FETCH_DATA, FETCH_FINISHED, FETCH_ERROR} fetch_msg;
+
+struct content;
+struct fetch;
+
+void fetch_init(void);
+struct fetch * fetch_start(char *url, char *referer,
+                 void (*callback)(fetch_msg msg, void *p, char *data, unsigned long size), void *p);
+void fetch_abort(struct fetch *f);
+void fetch_poll(void);
+void fetch_quit(void);
+
+#endif
diff --git a/content/fetchcache.c b/content/fetchcache.c
new file mode 100644
index 000000000..cd41f62ff
--- /dev/null
+++ b/content/fetchcache.c
@@ -0,0 +1,149 @@
+/**
+ * $Id: fetchcache.c,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#include <assert.h>
+#include "netsurf/content/cache.h"
+#include "netsurf/content/fetchcache.h"
+#include "netsurf/content/fetch.h"
+#include "netsurf/utils/log.h"
+#include "netsurf/utils/utils.h"
+
+
+struct fetchcache {
+	void *url;
+	void (*callback)(fetchcache_msg msg, struct content *c, void *p, char *error);
+	void *p;
+	struct fetch *f;
+	struct content *c;
+	unsigned long width, height;
+};
+
+
+void fetchcache_free(struct fetchcache *fc);
+void fetchcache_callback(fetchcache_msg msg, struct fetchcache *fc, char *data, unsigned long size);
+
+
+void fetchcache(char *url, char *referer,
+		void (*callback)(fetchcache_msg msg, struct content *c, void *p, char *error),
+		void *p, unsigned long width, unsigned long height)
+{
+	struct content *c;
+	struct fetchcache *fc;
+
+	c = cache_get(url);
+	if (c != 0) {
+		content_revive(c, width, height);
+		callback(FETCHCACHE_OK, c, p, 0);
+		return;
+	}
+
+	fc = xcalloc(1, sizeof(struct fetchcache));
+	fc->url = xstrdup(url);
+	fc->callback = callback;
+	fc->p = p;
+	fc->c = 0;
+	fc->width = width;
+	fc->height = height;
+	fc->f = fetch_start(url, referer, fetchcache_callback, fc);
+}
+
+
+void fetchcache_free(struct fetchcache *fc)
+{
+	free(fc->url);
+	free(fc);
+}
+
+
+void fetchcache_callback(fetch_msg msg, struct fetchcache *fc, char *data, unsigned long size)
+{
+	content_type type;
+	switch (msg) {
+		case FETCH_TYPE:
+			type = content_lookup(data);
+			LOG(("FETCH_TYPE, type %u", type));
+			if (type == CONTENT_OTHER) {
+				fetch_abort(fc->f);
+				fc->callback(FETCHCACHE_BADTYPE, 0, fc->p, 0);
+				free(fc);
+			} else {
+				fc->c = content_create(type, fc->url);
+			}
+			break;
+		case FETCH_DATA:
+			LOG(("FETCH_DATA"));
+			assert(fc->c != 0);
+			content_process_data(fc->c, data, size);
+			break;
+		case FETCH_FINISHED:
+			LOG(("FETCH_FINISHED"));
+			assert(fc->c != 0);
+			if (content_convert(fc->c, fc->width, fc->height) == 0) {
+				cache_put(fc->c);
+				fc->callback(FETCHCACHE_OK, fc->c, fc->p, 0);
+			} else {
+				content_destroy(fc->c);
+				fc->callback(FETCHCACHE_ERROR, 0, fc->p, "Conversion failed");
+			}
+			free(fc);
+			break;
+		case FETCH_ERROR:
+			LOG(("FETCH_ERROR, '%s'", data));
+			if (fc->c != 0)
+				content_destroy(fc->c);
+			fc->callback(FETCHCACHE_ERROR, 0, fc->p, data);
+			free(fc);
+			break;
+		default:
+			assert(0);
+	}
+}
+
+
+#ifdef TEST
+
+#include <unistd.h>
+
+void callback(fetchcache_msg msg, struct content *c, void *p, char *error)
+{
+	switch (msg) {
+		case FETCHCACHE_OK:
+			LOG(("FETCHCACHE_OK, url '%s'", p));
+			break;
+		case FETCHCACHE_BADTYPE:
+			LOG(("FETCHCACHE_BADTYPE, url '%s'", p));
+			break;
+		case FETCHCACHE_ERROR:
+			LOG(("FETCHCACHE_ERROR, url '%s', error '%s'", p, error));
+			break;
+		default:
+			assert(0);
+	}
+}
+
+char *test[] = {"http://www.google.co.uk/", "http://www.ox.ac.uk/", "blah://blah/"};
+
+int main(void)
+{
+	int i;
+
+	cache_init();
+	fetch_init();
+
+	for (i = 0; i != sizeof(test) / sizeof(test[0]); i++)
+		fetchcache(test[i], 0, callback, test[i], 800, 0);
+	for (i = 0; i != 5; i++) {
+		fetch_poll();
+		sleep(1);
+	}
+	for (i = 0; i != sizeof(test) / sizeof(test[0]); i++)
+		fetchcache(test[i], 0, callback, test[i], 800, 0);
+	for (i = 0; i != 20; i++) {
+		fetch_poll();
+		sleep(1);
+	}
+	return 0;
+}
+
+#endif
diff --git a/content/fetchcache.h b/content/fetchcache.h
new file mode 100644
index 000000000..d212b56a5
--- /dev/null
+++ b/content/fetchcache.h
@@ -0,0 +1,16 @@
+/**
+ * $Id: fetchcache.h,v 1.1 2003/02/09 12:58:14 bursa Exp $
+ */
+
+#ifndef _NETSURF_DESKTOP_FETCHCACHE_H_
+#define _NETSURF_DESKTOP_FETCHCACHE_H_
+
+#include "netsurf/content/content.h"
+
+typedef enum {FETCHCACHE_OK, FETCHCACHE_BADTYPE, FETCHCACHE_ERROR} fetchcache_msg;
+
+void fetchcache(char *url, char *referer,
+		void (*callback)(fetchcache_msg msg, struct content *c, void *p, char *error),
+		void *p, unsigned long width, unsigned long height);
+
+#endif