From 58c28f9c1ab86da14f15cee44ae936c74d812a5f Mon Sep 17 00:00:00 2001 From: James Bursa Date: Thu, 17 Apr 2003 21:35:02 +0000 Subject: [project @ 2003-04-17 21:35:02 by bursa] Max one fetch from each host at once, fix multiple fetches of same url. svn path=/import/netsurf/; revision=127 --- content/fetchcache.c | 114 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 98 insertions(+), 16 deletions(-) (limited to 'content/fetchcache.c') diff --git a/content/fetchcache.c b/content/fetchcache.c index 8aa81f66f..34105c71d 100644 --- a/content/fetchcache.c +++ b/content/fetchcache.c @@ -1,5 +1,5 @@ /** - * $Id: fetchcache.c,v 1.7 2003/04/09 21:57:09 bursa Exp $ + * $Id: fetchcache.c,v 1.8 2003/04/17 21:35:02 bursa Exp $ */ #include @@ -20,8 +20,13 @@ struct fetchcache { unsigned long width, height; unsigned long size; content_type allowed; + struct fetchcache *next; + struct fetchcache *prev; + struct fetchcache *next_request; + int active; }; +static struct fetchcache *fetchcache_list = 0; static void fetchcache_free(struct fetchcache *fc); static void fetchcache_callback(fetchcache_msg msg, void *p, char *data, unsigned long size); @@ -33,7 +38,7 @@ void fetchcache(const char *url, char *referer, void *p, unsigned long width, unsigned long height, content_type allowed) { struct content *c; - struct fetchcache *fc; + struct fetchcache *fc, *fc_url; c = cache_get(url); if (c != 0) { @@ -59,7 +64,30 @@ void fetchcache(const char *url, char *referer, fc->height = height; fc->size = 0; fc->allowed = allowed; - fc->f = fetch_start(fc->url, referer, fetchcache_callback, fc); + fc->next = 0; + fc->prev = 0; + fc->next_request = 0; + fc->active = 1; + + /* check if we're already fetching this url */ + for (fc_url = fetchcache_list; + fc_url != 0 && strcmp(fc_url->url, url) != 0; + fc_url = fc_url->next) + ; + if (fc_url != 0) { + /* already fetching: add ourselves to list of requestors */ + LOG(("already fetching")); + fc->next_request = fc_url->next_request; + fc_url->next_request = fc; + + } else { + /* not fetching yet */ + if (fetchcache_list != 0) + fetchcache_list->prev = fc; + fc->next = fetchcache_list; + fetchcache_list = fc; + fc->f = fetch_start(fc->url, referer, fetchcache_callback, fc); + } } @@ -67,16 +95,24 @@ void fetchcache_free(struct fetchcache *fc) { free(fc->url); free(fc); + if (fc->prev == 0) + fetchcache_list = fc->next; + else + fc->prev->next = fc->next; + if (fc->next != 0) + fc->next->prev = fc->prev; } void fetchcache_callback(fetch_msg msg, void *p, char *data, unsigned long size) { - struct fetchcache *fc = p; + struct fetchcache *fc = p, *fc_url; content_type type; char *mime_type; char *semic; char status[40]; + int active = 0; + switch (msg) { case FETCH_TYPE: mime_type = strdup(data); @@ -84,46 +120,90 @@ void fetchcache_callback(fetch_msg msg, void *p, char *data, unsigned long size) *semic = 0; /* remove "; charset=..." */ type = content_lookup(mime_type); LOG(("FETCH_TYPE, type %u", type)); - if ((1 << type) & fc->allowed) { + + /* check if each request allows this type */ + for (fc_url = fc; fc_url != 0; fc_url = fc_url->next_request) { + if (!fc_url->active) + continue; + if ((1 << type) & fc_url->allowed) { + active++; + } else { + fc_url->active = 0; + fc_url->callback(FETCHCACHE_BADTYPE, 0, + fc_url->p, mime_type); + } + } + if (active != 0) { + /* someone is still interested */ fc->c = content_create(type, fc->url); fc->c->status_callback = status_callback; fc->c->status_p = fc; } else { + /* no request allows the type */ fetch_abort(fc->f); - fc->callback(FETCHCACHE_BADTYPE, 0, fc->p, mime_type); - free(fc); + for (; fc != 0; fc = fc_url) { + fc_url = fc->next_request; + fetchcache_free(fc); + } } + free(mime_type); break; + case FETCH_DATA: LOG(("FETCH_DATA")); assert(fc->c != 0); fc->size += size; sprintf(status, "Received %lu bytes", fc->size); - fc->callback(FETCHCACHE_STATUS, fc->c, fc->p, status); + for (fc_url = fc; fc_url != 0; fc_url = fc_url->next_request) + if (fc_url->active) + fc_url->callback(FETCHCACHE_STATUS, fc->c, + fc_url->p, status); content_process_data(fc->c, data, size); break; + case FETCH_FINISHED: LOG(("FETCH_FINISHED")); assert(fc->c != 0); sprintf(status, "Converting %lu bytes", fc->size); - fc->callback(FETCHCACHE_STATUS, fc->c, fc->p, status); + for (fc_url = fc; fc_url != 0; fc_url = fc_url->next_request) + if (fc_url->active) + fc_url->callback(FETCHCACHE_STATUS, fc->c, + fc_url->p, status); + if (content_convert(fc->c, fc->width, fc->height) == 0) { cache_put(fc->c); - fc->callback(FETCHCACHE_OK, fc->c, fc->p, 0); + for (fc_url = fc; fc_url != 0; fc_url = fc_url->next_request) + if (fc_url->active) + fc_url->callback(FETCHCACHE_OK, cache_get(fc->url), + fc_url->p, 0); + cache_free(fc->c); } else { content_destroy(fc->c); - fc->callback(FETCHCACHE_ERROR, 0, fc->p, "Conversion failed"); + for (fc_url = fc; fc_url != 0; fc_url = fc_url->next_request) + if (fc_url->active) + fc_url->callback(FETCHCACHE_ERROR, 0, + fc_url->p, "Conversion failed"); + } + for (; fc != 0; fc = fc_url) { + fc_url = fc->next_request; + fetchcache_free(fc); } - fetchcache_free(fc); break; + case FETCH_ERROR: LOG(("FETCH_ERROR, '%s'", data)); if (fc->c != 0) content_destroy(fc->c); - fc->callback(FETCHCACHE_ERROR, 0, fc->p, data); - fetchcache_free(fc); + for (fc_url = fc; fc_url != 0; fc_url = fc_url->next_request) + if (fc_url->active) + fc->callback(FETCHCACHE_ERROR, 0, fc_url->p, data); + for (; fc != 0; fc = fc_url) { + fc_url = fc->next_request; + fetchcache_free(fc); + } break; + default: assert(0); } @@ -132,8 +212,10 @@ void fetchcache_callback(fetch_msg msg, void *p, char *data, unsigned long size) void status_callback(void *p, const char *status) { - struct fetchcache *fc = p; - fc->callback(FETCHCACHE_STATUS, fc->c, fc->p, status); + struct fetchcache *fc = p, *fc_url; + for (fc_url = fc; fc_url != 0; fc_url = fc_url->next_request) + if (fc_url->active) + fc_url->callback(FETCHCACHE_STATUS, fc->c, fc_url->p, status); } -- cgit v1.2.3