From 4b2101ba6ab62ae26d82cc8b86e0e61e9c007156 Mon Sep 17 00:00:00 2001 From: Vincent Sanders Date: Thu, 19 Jun 2014 18:27:24 +0100 Subject: clean up the fetcher factory and improve its API --- content/fetch.c | 313 ++++++++++++++++++++++---------------------- content/fetch.h | 65 +-------- content/fetchers.h | 119 +++++++++++++++++ content/fetchers/about.c | 25 ++-- content/fetchers/about.h | 2 +- content/fetchers/curl.c | 34 ++--- content/fetchers/data.c | 25 ++-- content/fetchers/data.h | 2 +- content/fetchers/file.c | 25 ++-- content/fetchers/file.h | 2 +- content/fetchers/resource.c | 35 ++--- content/fetchers/resource.h | 2 +- content/hlcache.c | 9 -- content/hlcache.h | 8 -- content/llcache.c | 8 -- content/llcache.h | 7 - 16 files changed, 352 insertions(+), 329 deletions(-) create mode 100644 content/fetchers.h (limited to 'content') diff --git a/content/fetch.c b/content/fetch.c index 4736670ff..13c7c6542 100644 --- a/content/fetch.c +++ b/content/fetch.c @@ -19,12 +19,15 @@ */ /** \file - * Fetching of data from a URL (implementation). + * Implementation of fetching of data from a URL. + * + * The implementation is the fetch factory and the generic operations + * around the fetcher specific methods. * * Active fetches are held in the circular linked list ::fetch_ring. There may * be at most ::option_max_fetchers_per_host active requests per Host: header. * There may be at most ::option_max_fetchers active requests overall. Inactive - * fetchers are stored in the ::queue_ring waiting for use. + * fetches are stored in the ::queue_ring waiting for use. */ #include @@ -33,17 +36,9 @@ #include #include #include - #include #include "utils/config.h" -#include "content/fetch.h" -#include "content/fetchers/resource.h" -#include "content/fetchers/about.h" -#include "content/fetchers/curl.h" -#include "content/fetchers/data.h" -#include "content/fetchers/file.h" -#include "content/urldb.h" #include "desktop/netsurf.h" #include "utils/corestrings.h" #include "utils/nsoption.h" @@ -53,27 +48,34 @@ #include "utils/utils.h" #include "utils/ring.h" +#include "content/fetch.h" +#include "content/fetchers.h" +#include "content/fetchers/resource.h" +#include "content/fetchers/about.h" +#include "content/fetchers/curl.h" +#include "content/fetchers/data.h" +#include "content/fetchers/file.h" +#include "content/urldb.h" + /* Define this to turn on verbose fetch logging */ #undef DEBUG_FETCH_VERBOSE -bool fetch_active; /**< Fetches in progress, please call fetch_poll(). */ +/** The maximum number of fetchers that can be added */ +#define MAX_FETCHERS 8 -/** Information about a fetcher for a given scheme. */ +bool fetch_active; /**< Fetches in progress, please call fetch_poll(). */ + +/** + * Information about a fetcher for a given scheme. + */ typedef struct scheme_fetcher_s { - lwc_string *scheme_name; /**< The scheme. */ - fetcher_can_fetch can_fetch; /**< Ensure an URL can be fetched. */ - fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */ - fetcher_start_fetch start_fetch; /**< Start a fetch. */ - fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */ - fetcher_free_fetch free_fetch; /**< Free a fetch. */ - fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */ - fetcher_finalise finaliser; /**< Clean up this fetcher. */ - int refcount; /**< When zero, clean up the fetcher. */ - struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */ - struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */ + lwc_string *scheme; /**< The scheme. */ + + struct fetcher_operation_table ops; /**< The fetchers operations. */ + int refcount; /**< When zero the fetcher is no longer in use. */ } scheme_fetcher; -static scheme_fetcher *fetchers = NULL; +static scheme_fetcher fetchers[MAX_FETCHERS]; /** Information for a single fetch. */ struct fetch { @@ -85,43 +87,53 @@ struct fetch { void *p; /**< Private data for callback. */ lwc_string *host; /**< Host part of URL, interned */ long http_code; /**< HTTP response code, or 0. */ - scheme_fetcher *ops; /**< Fetcher operations for this fetch, - NULL if not set. */ + int fetcherd; /**< Fetcher descriptor for this fetch */ void *fetcher_handle; /**< The handle for the fetcher. */ bool fetch_is_active; /**< This fetch is active. */ struct fetch *r_prev; /**< Previous active fetch in ::fetch_ring. */ struct fetch *r_next; /**< Next active fetch in ::fetch_ring. */ }; -static struct fetch *fetch_ring = 0; /**< Ring of active fetches. */ -static struct fetch *queue_ring = 0; /**< Ring of queued fetches */ - -#define fetch_ref_fetcher(F) F->refcount++ +static struct fetch *fetch_ring = NULL; /**< Ring of active fetches. */ +static struct fetch *queue_ring = NULL; /**< Ring of queued fetches */ /****************************************************************************** * fetch internals * ******************************************************************************/ -static void fetch_unref_fetcher(scheme_fetcher *fetcher) +static inline void fetch_ref_fetcher(int fetcherd) { - if (--fetcher->refcount == 0) { - fetcher->finaliser(fetcher->scheme_name); - lwc_string_unref(fetcher->scheme_name); - if (fetcher == fetchers) { - fetchers = fetcher->next_fetcher; - if (fetchers) - fetchers->prev_fetcher = NULL; - } else { - fetcher->prev_fetcher->next_fetcher = - fetcher->next_fetcher; - if (fetcher->next_fetcher != NULL) - fetcher->next_fetcher->prev_fetcher = - fetcher->prev_fetcher; - } - free(fetcher); + fetchers[fetcherd].refcount++; +} + +static inline void fetch_unref_fetcher(int fetcherd) +{ + fetchers[fetcherd].refcount--; + if (fetchers[fetcherd].refcount == 0) { + fetchers[fetcherd].ops.finalise(fetchers[fetcherd].scheme); + lwc_string_unref(fetchers[fetcherd].scheme); } } +/** + * Find a suitable fetcher for a scheme. + */ +static int get_fetcher_for_scheme(lwc_string *scheme) +{ + int fetcherd; + bool match; + + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if ((fetchers[fetcherd].refcount > 0) && + (lwc_string_isequal(fetchers[fetcherd].scheme, + scheme, &match) == lwc_error_ok) && + (match == true)) { + return fetcherd; + } + } + return -1; +} + /** * Dispatch a single job */ @@ -132,7 +144,7 @@ static bool fetch_dispatch_job(struct fetch *fetch) LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch, fetch->fetcher_handle, nsurl_access(fetch->url))); #endif - if (!fetch->ops->start_fetch(fetch->fetcher_handle)) { + if (!fetchers[fetch->fetcherd].ops.start(fetch->fetcher_handle)) { RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */ return false; } else { @@ -236,7 +248,7 @@ static void fetch_dispatch_jobs(void) ******************************************************************************/ /* exported interface documented in content/fetch.h */ -nserror fetch_init(void) +nserror fetcher_init(void) { fetch_curl_register(); fetch_data_register(); @@ -249,76 +261,81 @@ nserror fetch_init(void) return NSERROR_OK; } -/* exported interface documented in content/fetch.h */ -void fetch_quit(void) +/* exported interface documented in content/fetchers.h */ +void fetcher_quit(void) { - while (fetchers != NULL) { - if (fetchers->refcount != 1) { - LOG(("Fetcher for scheme %s still active?!", - lwc_string_data(fetchers->scheme_name))); - /* We shouldn't do this, but... */ - fetchers->refcount = 1; + int fetcherd; /* fetcher index */ + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if (fetchers[fetcherd].refcount > 0) { + /* assert if the fetcher is active at quit */ + assert(fetchers[fetcherd].refcount == 1); + + fetch_unref_fetcher(fetcherd); } - fetch_unref_fetcher(fetchers); } } -/* exported interface documented in content/fetch.h */ -bool fetch_add_fetcher(lwc_string *scheme, - fetcher_initialise initialiser, - fetcher_can_fetch can_fetch, - fetcher_setup_fetch setup_fetch, - fetcher_start_fetch start_fetch, - fetcher_abort_fetch abort_fetch, - fetcher_free_fetch free_fetch, - fetcher_poll_fetcher poll_fetcher, - fetcher_finalise finaliser) +/* exported interface documented in content/fetchers.h */ +nserror +fetcher_add(lwc_string *scheme, const struct fetcher_operation_table *ops) { - scheme_fetcher *new_fetcher; - if (!initialiser(scheme)) - return false; - new_fetcher = malloc(sizeof(scheme_fetcher)); - if (new_fetcher == NULL) { - finaliser(scheme); - return false; + int fetcherd; + + /* find unused fetcher descriptor */ + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if (fetchers[fetcherd].refcount == 0) { + break; + } } - new_fetcher->scheme_name = scheme; - new_fetcher->refcount = 0; - new_fetcher->can_fetch = can_fetch; - new_fetcher->setup_fetch = setup_fetch; - new_fetcher->start_fetch = start_fetch; - new_fetcher->abort_fetch = abort_fetch; - new_fetcher->free_fetch = free_fetch; - new_fetcher->poll_fetcher = poll_fetcher; - new_fetcher->finaliser = finaliser; - new_fetcher->next_fetcher = fetchers; - fetchers = new_fetcher; - fetch_ref_fetcher(new_fetcher); - - return true; + if (fetcherd == MAX_FETCHERS) { + return NSERROR_INIT_FAILED; + } + + if (!ops->initialise(scheme)) { + return NSERROR_INIT_FAILED; + } + + fetchers[fetcherd].scheme = scheme; + fetchers[fetcherd].ops = *ops; + + fetch_ref_fetcher(fetcherd); + + return NSERROR_OK; } /* exported interface documented in content/fetch.h */ -struct fetch * fetch_start(nsurl *url, nsurl *referer, - fetch_callback callback, - void *p, bool only_2xx, const char *post_urlenc, - const struct fetch_multipart_data *post_multipart, - bool verifiable, bool downgrade_tls, - const char *headers[]) +struct fetch * +fetch_start(nsurl *url, + nsurl *referer, + fetch_callback callback, + void *p, + bool only_2xx, + const char *post_urlenc, + const struct fetch_multipart_data *post_multipart, + bool verifiable, + bool downgrade_tls, + const char *headers[]) { struct fetch *fetch; - scheme_fetcher *fetcher = fetchers; lwc_string *scheme; bool match; fetch = malloc(sizeof (*fetch)); - if (fetch == NULL) + if (fetch == NULL) { return NULL; + } /* The URL we're fetching must have a scheme */ scheme = nsurl_get_component(url, NSURL_SCHEME); assert(scheme != NULL); + /* try and obtain a fetcher for this scheme */ + fetch->fetcherd = get_fetcher_for_scheme(scheme); + if (fetch->fetcherd == -1) { + lwc_string_unref(scheme); + return NULL; + } + #ifdef DEBUG_FETCH_VERBOSE LOG(("fetch %p, url '%s'", fetch, nsurl_access(url))); #endif @@ -334,7 +351,6 @@ struct fetch * fetch_start(nsurl *url, nsurl *referer, fetch->referer = NULL; fetch->send_referer = false; fetch->fetcher_handle = NULL; - fetch->ops = NULL; fetch->fetch_is_active = false; fetch->host = nsurl_get_component(url, NSURL_HOST); @@ -378,53 +394,38 @@ struct fetch * fetch_start(nsurl *url, nsurl *referer, lwc_string_unref(ref_scheme); } - /* Pick the scheme ops */ - while (fetcher) { - if ((lwc_string_isequal(fetcher->scheme_name, scheme, - &match) == lwc_error_ok) && (match == true)) { - fetch->ops = fetcher; - break; - } - fetcher = fetcher->next_fetcher; - } + /* these aren't needed past here */ + lwc_string_unref(scheme); - if (fetch->ops == NULL) - goto failed; + /* try and set up the fetch */ + fetch->fetcher_handle = fetchers[fetch->fetcherd].ops.setup(fetch, url, + only_2xx, downgrade_tls, + post_urlenc, post_multipart, + headers); + if (fetch->fetcher_handle == NULL) { - /* Got a scheme fetcher, try and set up the fetch */ - fetch->fetcher_handle = fetch->ops->setup_fetch(fetch, url, - only_2xx, downgrade_tls, - post_urlenc, post_multipart, - headers); + if (fetch->host != NULL) + lwc_string_unref(fetch->host); - if (fetch->fetcher_handle == NULL) - goto failed; + if (fetch->url != NULL) + nsurl_unref(fetch->url); - /* Rah, got it, so ref the fetcher. */ - fetch_ref_fetcher(fetch->ops); + if (fetch->referer != NULL) + nsurl_unref(fetch->referer); - /* these aren't needed past here */ - lwc_string_unref(scheme); + free(fetch); + + return NULL; + } + + /* Rah, got it, so ref the fetcher. */ + fetch_ref_fetcher(fetch->fetcherd); /* Dump us in the queue and ask the queue to run. */ RING_INSERT(queue_ring, fetch); fetch_dispatch_jobs(); return fetch; - -failed: - lwc_string_unref(scheme); - - if (fetch->host != NULL) - lwc_string_unref(fetch->host); - if (fetch->url != NULL) - nsurl_unref(fetch->url); - if (fetch->referer != NULL) - nsurl_unref(fetch->referer); - - free(fetch); - - return NULL; } /* exported interface documented in content/fetch.h */ @@ -435,7 +436,7 @@ void fetch_abort(struct fetch *f) LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle, nsurl_access(f->url))); #endif - f->ops->abort_fetch(f->fetcher_handle); + fetchers[f->fetcherd].ops.abort(f->fetcher_handle); } /* exported interface documented in content/fetch.h */ @@ -444,8 +445,10 @@ void fetch_free(struct fetch *f) #ifdef DEBUG_FETCH_VERBOSE LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle)); #endif - f->ops->free_fetch(f->fetcher_handle); - fetch_unref_fetcher(f->ops); + fetchers[f->fetcherd].ops.free(f->fetcher_handle); + + fetch_unref_fetcher(f->fetcherd); + nsurl_unref(f->url); if (f->referer != NULL) nsurl_unref(f->referer); @@ -454,45 +457,37 @@ void fetch_free(struct fetch *f) free(f); } -/* exported interface documented in content/fetch.h */ -void fetch_poll(void) +/* exported interface documented in content/fetchers.h */ +void fetcher_poll(void) { - scheme_fetcher *fetcher = fetchers; - scheme_fetcher *next_fetcher; + int fetcherd; fetch_dispatch_jobs(); - if (!fetch_active) - return; /* No point polling, there's no fetch active. */ - while (fetcher != NULL) { - next_fetcher = fetcher->next_fetcher; - if (fetcher->poll_fetcher != NULL) { - /* LOG(("Polling fetcher for %s", - lwc_string_data(fetcher->scheme_name))); */ - fetcher->poll_fetcher(fetcher->scheme_name); + if (fetch_active) { + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if (fetchers[fetcherd].refcount > 0) { + /* fetcher present */ + fetchers[fetcherd].ops.poll(fetchers[fetcherd].scheme); + } } - fetcher = next_fetcher; } } /* exported interface documented in content/fetch.h */ bool fetch_can_fetch(const nsurl *url) { - scheme_fetcher *fetcher = fetchers; - bool match; lwc_string *scheme = nsurl_get_component(url, NSURL_SCHEME); + int fetcherd; - while (fetcher != NULL) { - if (lwc_string_isequal(fetcher->scheme_name, scheme, &match) == lwc_error_ok && match == true) { - break; - } + fetcherd = get_fetcher_for_scheme(scheme); + lwc_string_unref(scheme); - fetcher = fetcher->next_fetcher; + if (fetcherd == -1) { + return false; } - lwc_string_unref(scheme); - - return fetcher == NULL ? false : fetcher->can_fetch(url); + return fetchers[fetcherd].ops.acceptable(url); } /* exported interface documented in content/fetch.h */ diff --git a/content/fetch.h b/content/fetch.h index f20b4f2d2..37539ef2b 100644 --- a/content/fetch.h +++ b/content/fetch.h @@ -25,8 +25,6 @@ #include -#include - #include "utils/config.h" #include "utils/nsurl.h" @@ -95,17 +93,8 @@ struct ssl_cert_info { int cert_type; /**< Certificate type */ }; -extern bool fetch_active; - typedef void (*fetch_callback)(const fetch_msg *msg, void *p); -/** - * Initialise the fetcher. - * - * @return NSERROR_OK or error code - */ -nserror fetch_init(void); - /** * Start fetching data for the given URL. * @@ -137,19 +126,6 @@ struct fetch *fetch_start(nsurl *url, nsurl *referer, */ void fetch_abort(struct fetch *f); -/** - * Do some work on current fetches. - * - * Must be called regularly to make progress on fetches. - */ -void fetch_poll(void); - -/** - * Clean up for quit. - * - * Must be called before exiting. - */ -void fetch_quit(void); /** * Check if a URL's scheme can be fetched. @@ -162,9 +138,7 @@ bool fetch_can_fetch(const nsurl *url); /** * Change the callback function for a fetch. */ -void fetch_change_callback(struct fetch *fetch, - fetch_callback callback, - void *p); +void fetch_change_callback(struct fetch *fetch, fetch_callback callback, void *p); /** * Get the HTTP response code. @@ -225,41 +199,4 @@ const char *fetch_get_referer_to_send(struct fetch *fetch); void fetch_set_cookie(struct fetch *fetch, const char *data); -/* API for fetchers themselves */ - -typedef bool (*fetcher_initialise)(lwc_string *scheme); -typedef bool (*fetcher_can_fetch)(const nsurl *url); -typedef void *(*fetcher_setup_fetch)(struct fetch *parent_fetch, nsurl *url, - bool only_2xx, bool downgrade_tls, const char *post_urlenc, - const struct fetch_multipart_data *post_multipart, - const char **headers); -typedef bool (*fetcher_start_fetch)(void *fetch); -typedef void (*fetcher_abort_fetch)(void *fetch); -typedef void (*fetcher_free_fetch)(void *fetch); -typedef void (*fetcher_poll_fetcher)(lwc_string *scheme); -typedef void (*fetcher_finalise)(lwc_string *scheme); - -/** Register a fetcher for a scheme - * - * \param scheme scheme fetcher is for (caller relinquishes ownership) - * \param initialiser fetcher initialiser - * \param can_fetch fetcher can fetch function - * \param setup_fetch fetcher fetch setup function - * \param start_fetch fetcher fetch start function - * \param abort_fetch fetcher fetch abort function - * \param free_fetch fetcher fetch free function - * \param poll_fetcher fetcher poll function - * \param finaliser fetcher finaliser - * \return true iff success - */ -bool fetch_add_fetcher(lwc_string *scheme, - fetcher_initialise initialiser, - fetcher_can_fetch can_fetch, - fetcher_setup_fetch setup_fetch, - fetcher_start_fetch start_fetch, - fetcher_abort_fetch abort_fetch, - fetcher_free_fetch free_fetch, - fetcher_poll_fetcher poll_fetcher, - fetcher_finalise finaliser); - #endif diff --git a/content/fetchers.h b/content/fetchers.h new file mode 100644 index 000000000..6ca52245c --- /dev/null +++ b/content/fetchers.h @@ -0,0 +1,119 @@ +/* + * Copyright 2014 Vincent Sanders + * + * This file is part of NetSurf, http://www.netsurf-browser.org/ + * + * NetSurf is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; version 2 of the License. + * + * NetSurf is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/** \file content/fetchers.h + * Interface for fetchers factory. + */ + +#ifndef _NETSURF_DESKTOP_FETCHERS_H_ +#define _NETSURF_DESKTOP_FETCHERS_H_ + +#include + +struct nsurl; +struct fetch_multipart_data; +struct fetch; + +extern bool fetch_active; + +/** + * Fetcher operations API + * + * These are the operations a fetcher must implement. + */ +struct fetcher_operation_table { + /** + * The initialiser for the fetcher. + * + * Called once to initialise the fetcher. + */ + bool (*initialise)(lwc_string *scheme); + + /** + * can this fetcher accept a url. + * + * \param url the URL to check + * \return true if the fetcher can handle the url else false. + */ + bool (*acceptable)(const struct nsurl *url); + + /** + * Setup a fetch + */ + void *(*setup)(struct fetch *parent_fetch, struct nsurl *url, + bool only_2xx, bool downgrade_tls, const char *post_urlenc, + const struct fetch_multipart_data *post_multipart, + const char **headers); + + /** + * start a fetch. + */ + bool (*start)(void *fetch); + + /** + * abort a fetch. + */ + void (*abort)(void *fetch); + + /** + * free a fetch allocated through the setup method. + */ + void (*free)(void *fetch); + + /** + * poll a fetcher to let it make progress. + */ + void (*poll)(lwc_string *scheme); + + /** + * finalise the fetcher. + */ + void (*finalise)(lwc_string *scheme); +}; + +/** + * Register a fetcher for a scheme + * + * \param scheme The scheme fetcher is for (caller relinquishes ownership) + * \param ops The operations for the fetcher. + * \return NSERROR_OK or appropriate error code. + */ +nserror fetcher_add(lwc_string *scheme, const struct fetcher_operation_table *ops); + +/** + * Initialise the fetchers. + * + * @return NSERROR_OK or error code + */ +nserror fetcher_init(void); + +/** + * Clean up for quit. + * + * Must be called before exiting. + */ +void fetcher_quit(void); + +/** + * Do some work on current fetches. + * + * Must be called regularly to make progress on fetches. + */ +void fetcher_poll(void); + +#endif diff --git a/content/fetchers/about.c b/content/fetchers/about.c index 17f220932..8c4d29a91 100644 --- a/content/fetchers/about.c +++ b/content/fetchers/about.c @@ -45,6 +45,7 @@ #include "utils/config.h" #include "content/fetch.h" +#include "content/fetchers.h" #include "content/fetchers/about.h" #include "content/urldb.h" #include "desktop/netsurf.h" @@ -837,17 +838,19 @@ static void fetch_about_poll(lwc_string *scheme) } while ( (c = next) != ring && ring != NULL); } -void fetch_about_register(void) +nserror fetch_about_register(void) { lwc_string *scheme = lwc_string_ref(corestring_lwc_about); - - fetch_add_fetcher(scheme, - fetch_about_initialise, - fetch_about_can_fetch, - fetch_about_setup, - fetch_about_start, - fetch_about_abort, - fetch_about_free, - fetch_about_poll, - fetch_about_finalise); + const struct fetcher_operation_table fetcher_ops = { + .initialise = fetch_about_initialise, + .acceptable = fetch_about_can_fetch, + .setup = fetch_about_setup, + .start = fetch_about_start, + .abort = fetch_about_abort, + .free = fetch_about_free, + .poll = fetch_about_poll, + .finalise = fetch_about_finalise + }; + + return fetcher_add(scheme, &fetcher_ops); } diff --git a/content/fetchers/about.h b/content/fetchers/about.h index f22be6a5d..9544971a6 100644 --- a/content/fetchers/about.h +++ b/content/fetchers/about.h @@ -23,6 +23,6 @@ #ifndef NETSURF_CONTENT_FETCHERS_FETCH_ABOUT_H #define NETSURF_CONTENT_FETCHERS_FETCH_ABOUT_H -void fetch_about_register(void); +nserror fetch_about_register(void); #endif diff --git a/content/fetchers/curl.c b/content/fetchers/curl.c index 80ac5ec89..39c15b930 100644 --- a/content/fetchers/curl.c +++ b/content/fetchers/curl.c @@ -53,12 +53,10 @@ #include "utils/file.h" #include "content/fetch.h" +#include "content/fetchers.h" #include "content/fetchers/curl.h" #include "content/urldb.h" -/* uncomment this to use scheduler based calling -#define FETCHER_CURLL_SCHEDULED 1 -*/ /** SSL certificate info */ struct cert_info { @@ -160,6 +158,16 @@ void fetch_curl_register(void) curl_version_info_data *data; int i; lwc_string *scheme; + const struct fetcher_operation_table fetcher_ops = { + .initialise = fetch_curl_initialise, + .acceptable = fetch_curl_can_fetch, + .setup = fetch_curl_setup, + .start = fetch_curl_start, + .abort = fetch_curl_abort, + .free = fetch_curl_free, + .poll = fetch_curl_poll, + .finalise = fetch_curl_finalise + }; LOG(("curl_version %s", curl_version())); @@ -261,19 +269,7 @@ void fetch_curl_register(void) continue; } - if (!fetch_add_fetcher(scheme, - fetch_curl_initialise, - fetch_curl_can_fetch, - fetch_curl_setup, - fetch_curl_start, - fetch_curl_abort, - fetch_curl_free, -#ifdef FETCHER_CURLL_SCHEDULED - NULL, -#else - fetch_curl_poll, -#endif - fetch_curl_finalise)) { + if (fetcher_add(scheme, &fetcher_ops) != NSERROR_OK) { LOG(("Unable to register cURL fetcher for %s", data->protocols[i])); } @@ -835,12 +831,6 @@ void fetch_curl_poll(lwc_string *scheme_ignored) } curl_msg = curl_multi_info_read(fetch_curl_multi, &queue); } - -#ifdef FETCHER_CURLL_SCHEDULED - if (running != 0) { - guit->browser->schedule(10, fetch_curl_poll, fetch_curl_poll); - } -#endif } diff --git a/content/fetchers/data.c b/content/fetchers/data.c index f7ae17150..94ba63827 100644 --- a/content/fetchers/data.c +++ b/content/fetchers/data.c @@ -31,6 +31,7 @@ #include "utils/config.h" #include "content/fetch.h" +#include "content/fetchers.h" #include "content/fetchers/data.h" #include "content/urldb.h" #include "desktop/netsurf.h" @@ -324,17 +325,19 @@ static void fetch_data_poll(lwc_string *scheme) } while ( (c = next) != ring && ring != NULL); } -void fetch_data_register(void) +nserror fetch_data_register(void) { lwc_string *scheme = lwc_string_ref(corestring_lwc_data); - - fetch_add_fetcher(scheme, - fetch_data_initialise, - fetch_data_can_fetch, - fetch_data_setup, - fetch_data_start, - fetch_data_abort, - fetch_data_free, - fetch_data_poll, - fetch_data_finalise); + const struct fetcher_operation_table fetcher_ops = { + .initialise = fetch_data_initialise, + .acceptable = fetch_data_can_fetch, + .setup = fetch_data_setup, + .start = fetch_data_start, + .abort = fetch_data_abort, + .free = fetch_data_free, + .poll = fetch_data_poll, + .finalise = fetch_data_finalise + }; + + return fetcher_add(scheme, &fetcher_ops); } diff --git a/content/fetchers/data.h b/content/fetchers/data.h index 76f02cb3b..f6017e07a 100644 --- a/content/fetchers/data.h +++ b/content/fetchers/data.h @@ -23,6 +23,6 @@ #ifndef NETSURF_CONTENT_FETCHERS_FETCH_DATA_H #define NETSURF_CONTENT_FETCHERS_FETCH_DATA_H -void fetch_data_register(void); +nserror fetch_data_register(void); #endif diff --git a/content/fetchers/file.c b/content/fetchers/file.c index 7834b2702..f08be6288 100644 --- a/content/fetchers/file.c +++ b/content/fetchers/file.c @@ -55,6 +55,7 @@ #include "content/dirlist.h" #include "content/fetch.h" +#include "content/fetchers.h" #include "content/urldb.h" #include "content/fetchers/file.h" @@ -760,17 +761,19 @@ static void fetch_file_poll(lwc_string *scheme) } while ( (c = next) != ring && ring != NULL); } -void fetch_file_register(void) +nserror fetch_file_register(void) { lwc_string *scheme = lwc_string_ref(corestring_lwc_file); - - fetch_add_fetcher(scheme, - fetch_file_initialise, - fetch_file_can_fetch, - fetch_file_setup, - fetch_file_start, - fetch_file_abort, - fetch_file_free, - fetch_file_poll, - fetch_file_finalise); + const struct fetcher_operation_table fetcher_ops = { + .initialise = fetch_file_initialise, + .acceptable = fetch_file_can_fetch, + .setup = fetch_file_setup, + .start = fetch_file_start, + .abort = fetch_file_abort, + .free = fetch_file_free, + .poll = fetch_file_poll, + .finalise = fetch_file_finalise + }; + + return fetcher_add(scheme, &fetcher_ops); } diff --git a/content/fetchers/file.h b/content/fetchers/file.h index d1621b9ba..b3c39db9f 100644 --- a/content/fetchers/file.h +++ b/content/fetchers/file.h @@ -23,6 +23,6 @@ #ifndef NETSURF_CONTENT_FETCHERS_FETCH_FILE_H #define NETSURF_CONTENT_FETCHERS_FETCH_FILE_H -void fetch_file_register(void); +nserror fetch_file_register(void); #endif diff --git a/content/fetchers/resource.c b/content/fetchers/resource.c index 664c45720..18e302140 100644 --- a/content/fetchers/resource.c +++ b/content/fetchers/resource.c @@ -37,16 +37,19 @@ #include #include "utils/config.h" -#include "content/fetch.h" -#include "content/fetchers/resource.h" -#include "content/urldb.h" -#include "desktop/gui_factory.h" +#include "utils/errors.h" #include "utils/corestrings.h" #include "utils/nsoption.h" #include "utils/log.h" #include "utils/messages.h" #include "utils/utils.h" #include "utils/ring.h" +#include "desktop/gui_factory.h" + +#include "content/fetch.h" +#include "content/fetchers.h" +#include "content/fetchers/resource.h" +#include "content/urldb.h" struct fetch_resource_context; @@ -353,17 +356,19 @@ static void fetch_resource_poll(lwc_string *scheme) } while ( (c = next) != ring && ring != NULL); } -void fetch_resource_register(void) +nserror fetch_resource_register(void) { lwc_string *scheme = lwc_string_ref(corestring_lwc_resource); - - fetch_add_fetcher(scheme, - fetch_resource_initialise, - fetch_resource_can_fetch, - fetch_resource_setup, - fetch_resource_start, - fetch_resource_abort, - fetch_resource_free, - fetch_resource_poll, - fetch_resource_finalise); + const struct fetcher_operation_table fetcher_ops = { + .initialise = fetch_resource_initialise, + .acceptable = fetch_resource_can_fetch, + .setup = fetch_resource_setup, + .start = fetch_resource_start, + .abort = fetch_resource_abort, + .free = fetch_resource_free, + .poll = fetch_resource_poll, + .finalise = fetch_resource_finalise + }; + + return fetcher_add(scheme, &fetcher_ops); } diff --git a/content/fetchers/resource.h b/content/fetchers/resource.h index 79d8e37c4..cf4d6edac 100644 --- a/content/fetchers/resource.h +++ b/content/fetchers/resource.h @@ -35,6 +35,6 @@ * * should only be called from the fetch initialise */ -void fetch_resource_register(void); +nserror fetch_resource_register(void); #endif diff --git a/content/hlcache.c b/content/hlcache.c index e7c4cd84d..0d7996524 100644 --- a/content/hlcache.c +++ b/content/hlcache.c @@ -621,15 +621,6 @@ void hlcache_finalise(void) llcache_finalise(); } -/* See hlcache.h for documentation */ -nserror hlcache_poll(void) -{ - - llcache_poll(); - - return NSERROR_OK; -} - /* See hlcache.h for documentation */ nserror hlcache_handle_retrieve(nsurl *url, uint32_t flags, nsurl *referer, llcache_post_data *post, diff --git a/content/hlcache.h b/content/hlcache.h index 746b3c866..e0bf4161d 100644 --- a/content/hlcache.h +++ b/content/hlcache.h @@ -93,14 +93,6 @@ void hlcache_stop(void); */ void hlcache_finalise(void); -/** - * Drive the low-level cache poll loop, and attempt to clean the cache. - * No guarantee is made about what, if any, cache cleaning will occur. - * - * \return NSERROR_OK - */ -nserror hlcache_poll(void); - /** * Retrieve a high-level cache handle for an object * diff --git a/content/llcache.c b/content/llcache.c index b241ab22b..ad4d6d1f7 100644 --- a/content/llcache.c +++ b/content/llcache.c @@ -3087,14 +3087,6 @@ void llcache_finalise(void) llcache = NULL; } -/* See llcache.h for documentation */ -nserror llcache_poll(void) -{ - fetch_poll(); - - return NSERROR_OK; -} - /** * Catch up the cache users with state changes from fetchers. * diff --git a/content/llcache.h b/content/llcache.h index 4a3521637..d4ed5f095 100644 --- a/content/llcache.h +++ b/content/llcache.h @@ -238,13 +238,6 @@ nserror llcache_initialise(const struct llcache_parameters *parameters); */ void llcache_finalise(void); -/** - * Cause the low-level cache to emit any pending notifications. - * - * \return NSERROR_OK on success, appropriate error otherwise. - */ -nserror llcache_poll(void); - /** * Cause the low-level cache to attempt to perform cleanup. * -- cgit v1.2.3 From 1b7aa7ffe53843f072e3de5e28bdf06faa7980b9 Mon Sep 17 00:00:00 2001 From: Vincent Sanders Date: Thu, 26 Jun 2014 19:04:14 +0100 Subject: make fetchers scheduled --- content/fetch.c | 217 +++++++++++++++++++++++++++++++----------------- content/fetchers.h | 32 +++++-- content/fetchers/curl.c | 4 +- desktop/netsurf.c | 5 +- 4 files changed, 169 insertions(+), 89 deletions(-) (limited to 'content') diff --git a/content/fetch.c b/content/fetch.c index 13c7c6542..10cac9b27 100644 --- a/content/fetch.c +++ b/content/fetch.c @@ -37,9 +37,11 @@ #include #include #include +#include #include "utils/config.h" #include "desktop/netsurf.h" +#include "desktop/gui_factory.h" #include "utils/corestrings.h" #include "utils/nsoption.h" #include "utils/log.h" @@ -63,7 +65,11 @@ /** The maximum number of fetchers that can be added */ #define MAX_FETCHERS 8 -bool fetch_active; /**< Fetches in progress, please call fetch_poll(). */ +#ifdef DEBUG_FETCH_VERBOSE +#define FETCH_LOG(x) LOG(x) +#else +#define FETCH_LOG(x) +#endif /** * Information about a fetcher for a given scheme. @@ -129,7 +135,7 @@ static int get_fetcher_for_scheme(lwc_string *scheme) scheme, &match) == lwc_error_ok) && (match == true)) { return fetcherd; - } + } } return -1; } @@ -140,10 +146,9 @@ static int get_fetcher_for_scheme(lwc_string *scheme) static bool fetch_dispatch_job(struct fetch *fetch) { RING_REMOVE(queue_ring, fetch); -#ifdef DEBUG_FETCH_VERBOSE - LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch, + FETCH_LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch, fetch->fetcher_handle, nsurl_access(fetch->url))); -#endif + if (!fetchers[fetch->fetcherd].ops.start(fetch->fetcher_handle)) { RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */ return false; @@ -191,56 +196,76 @@ static bool fetch_choose_and_dispatch(void) return false; } -/** - * Dispatch as many jobs as we have room to dispatch. - */ -static void fetch_dispatch_jobs(void) +static void dump_rings(void) { - int all_active, all_queued; #ifdef DEBUG_FETCH_VERBOSE struct fetch *q; struct fetch *f; -#endif - - if (!queue_ring) - return; /* Nothing to do, the queue is empty */ - RING_GETSIZE(struct fetch, queue_ring, all_queued); - RING_GETSIZE(struct fetch, fetch_ring, all_active); - -#ifdef DEBUG_FETCH_VERBOSE - LOG(("queue_ring %i, fetch_ring %i", all_queued, all_active)); q = queue_ring; if (q) { do { - LOG(("queue_ring: %s", q->url)); + LOG(("queue_ring: %s", nsurl_access(q->url))); q = q->r_next; } while (q != queue_ring); } f = fetch_ring; if (f) { do { - LOG(("fetch_ring: %s", f->url)); + LOG(("fetch_ring: %s", nsurl_access(f->url))); f = f->r_next; } while (f != fetch_ring); } #endif +} + +/** + * Dispatch as many jobs as we have room to dispatch. + * + * @return true if there are active fetchers that require polling else false. + */ +static bool fetch_dispatch_jobs(void) +{ + int all_active; + int all_queued; + + RING_GETSIZE(struct fetch, queue_ring, all_queued); + RING_GETSIZE(struct fetch, fetch_ring, all_active); + + FETCH_LOG(("queue_ring %i, fetch_ring %i", all_queued, all_active)); + dump_rings(); - while ( all_queued && all_active < nsoption_int(max_fetchers) ) { - /*LOG(("%d queued, %d fetching", all_queued, all_active));*/ - if (fetch_choose_and_dispatch()) { + while ((all_queued != 0) && + (all_active < nsoption_int(max_fetchers)) && + fetch_choose_and_dispatch()) { all_queued--; all_active++; - } else { - /* Either a dispatch failed or we ran out. Just stop */ - break; + FETCH_LOG(("%d queued, %d fetching", + all_queued, all_active)); + } + + FETCH_LOG(("Fetch ring is now %d elements.", all_active)); + FETCH_LOG(("Queue ring is now %d elements.", all_queued)); + + return (all_active > 0); +} + +static void fetcher_poll(void *unused) +{ + int fetcherd; + + if (fetch_dispatch_jobs()) { + FETCH_LOG(("Polling fetchers")); + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if (fetchers[fetcherd].refcount > 0) { + /* fetcher present */ + fetchers[fetcherd].ops.poll(fetchers[fetcherd].scheme); + } } + + /* schedule active fetchers to run again in 10ms */ + guit->browser->schedule(10, fetcher_poll, NULL); } - fetch_active = (all_active > 0); -#ifdef DEBUG_FETCH_VERBOSE - LOG(("Fetch ring is now %d elements.", all_active)); - LOG(("Queue ring is now %d elements.", all_queued)); -#endif } /****************************************************************************** @@ -256,8 +281,6 @@ nserror fetcher_init(void) fetch_resource_register(); fetch_about_register(); - fetch_active = false; - return NSERROR_OK; } @@ -303,6 +326,63 @@ fetcher_add(lwc_string *scheme, const struct fetcher_operation_table *ops) return NSERROR_OK; } +/* exported interface documented in content/fetch.h */ +nserror fetcher_fdset(fd_set *read_fd_set, + fd_set *write_fd_set, + fd_set *except_fd_set, + int *maxfd_out) +{ + CURLMcode code; + int maxfd; + int fetcherd; /* fetcher index */ + + if (!fetch_dispatch_jobs()) { + FETCH_LOG(("No jobs")); + *maxfd_out = -1; + return NSERROR_OK; + } + + FETCH_LOG(("Polling fetchers")); + + for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { + if (fetchers[fetcherd].refcount > 0) { + /* fetcher present */ + fetchers[fetcherd].ops.poll(fetchers[fetcherd].scheme); + } + } + + FD_ZERO(read_fd_set); + FD_ZERO(write_fd_set); + FD_ZERO(except_fd_set); + code = curl_multi_fdset(fetch_curl_multi, + read_fd_set, + write_fd_set, + except_fd_set, + &maxfd); + assert(code == CURLM_OK); + + if (maxfd >= 0) { + /* change the scheduled poll to happen is a 1000ms as + * we assume fetching an fdset means the fetchers will + * be run by the client waking up on data available on + * the fd and re-calling fetcher_fdset() if this does + * not happen the fetch polling will continue as + * usual. + */ + /** @note adjusting the schedule time is only done for + * curl currently. This is because as it is assumed to + * be the only fetcher that can possibly have fd to + * select on. All the other fetchers continue to need + * polling frequently. + */ + guit->browser->schedule(1000, fetcher_poll, NULL); + } + + *maxfd_out = maxfd; + + return NSERROR_OK; +} + /* exported interface documented in content/fetch.h */ struct fetch * fetch_start(nsurl *url, @@ -336,9 +416,7 @@ fetch_start(nsurl *url, return NULL; } -#ifdef DEBUG_FETCH_VERBOSE - LOG(("fetch %p, url '%s'", fetch, nsurl_access(url))); -#endif + FETCH_LOG(("fetch %p, url '%s'", fetch, nsurl_access(url))); /* construct a new fetch structure */ fetch->callback = callback; @@ -421,9 +499,15 @@ fetch_start(nsurl *url, /* Rah, got it, so ref the fetcher. */ fetch_ref_fetcher(fetch->fetcherd); - /* Dump us in the queue and ask the queue to run. */ + /* Dump new fetch in the queue. */ RING_INSERT(queue_ring, fetch); - fetch_dispatch_jobs(); + + /* Ask the queue to run. */ + if (fetch_dispatch_jobs()) { + FETCH_LOG(("scheduling poll")); + /* schedule active fetchers to run again in 10ms */ + guit->browser->schedule(10, fetcher_poll, NULL); + } return fetch; } @@ -432,47 +516,31 @@ fetch_start(nsurl *url, void fetch_abort(struct fetch *f) { assert(f); -#ifdef DEBUG_FETCH_VERBOSE - LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle, + FETCH_LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle, nsurl_access(f->url))); -#endif fetchers[f->fetcherd].ops.abort(f->fetcher_handle); } /* exported interface documented in content/fetch.h */ void fetch_free(struct fetch *f) { -#ifdef DEBUG_FETCH_VERBOSE - LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle)); -#endif + FETCH_LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle)); + fetchers[f->fetcherd].ops.free(f->fetcher_handle); fetch_unref_fetcher(f->fetcherd); nsurl_unref(f->url); - if (f->referer != NULL) + if (f->referer != NULL) { nsurl_unref(f->referer); - if (f->host != NULL) + } + if (f->host != NULL) { lwc_string_unref(f->host); + } free(f); } -/* exported interface documented in content/fetchers.h */ -void fetcher_poll(void) -{ - int fetcherd; - - fetch_dispatch_jobs(); - if (fetch_active) { - for (fetcherd = 0; fetcherd < MAX_FETCHERS; fetcherd++) { - if (fetchers[fetcherd].refcount > 0) { - /* fetcher present */ - fetchers[fetcherd].ops.poll(fetchers[fetcherd].scheme); - } - } - } -} /* exported interface documented in content/fetch.h */ bool fetch_can_fetch(const nsurl *url) @@ -589,7 +657,7 @@ void fetch_multipart_data_destroy(struct fetch_multipart_data *list) free(list->name); free(list->value); if (list->file) { - LOG(("Freeing rawfile: %s", list->rawfile)); + FETCH_LOG(("Freeing rawfile: %s", list->rawfile)); free(list->rawfile); } free(list); @@ -607,12 +675,8 @@ fetch_send_callback(const fetch_msg *msg, struct fetch *fetch) /* exported interface documented in content/fetch.h */ void fetch_remove_from_queues(struct fetch *fetch) { - int all_active; - -#ifdef DEBUG_FETCH_VERBOSE - int all_queued; - LOG(("Fetch %p, fetcher %p can be freed", fetch, fetch->fetcher_handle)); -#endif + FETCH_LOG(("Fetch %p, fetcher %p can be freed", + fetch, fetch->fetcher_handle)); /* Go ahead and free the fetch properly now */ if (fetch->fetch_is_active) { @@ -621,15 +685,15 @@ void fetch_remove_from_queues(struct fetch *fetch) RING_REMOVE(queue_ring, fetch); } - RING_GETSIZE(struct fetch, fetch_ring, all_active); - - fetch_active = (all_active > 0); - #ifdef DEBUG_FETCH_VERBOSE - LOG(("Fetch ring is now %d elements.", all_active)); + int all_active; + int all_queued; + RING_GETSIZE(struct fetch, fetch_ring, all_active); RING_GETSIZE(struct fetch, queue_ring, all_queued); + LOG(("Fetch ring is now %d elements.", all_active)); + LOG(("Queue ring is now %d elements.", all_queued)); #endif } @@ -638,9 +702,8 @@ void fetch_remove_from_queues(struct fetch *fetch) /* exported interface documented in content/fetch.h */ void fetch_set_http_code(struct fetch *fetch, long http_code) { -#ifdef DEBUG_FETCH_VERBOSE - LOG(("Setting HTTP code to %ld", http_code)); -#endif + FETCH_LOG(("Setting HTTP code to %ld", http_code)); + fetch->http_code = http_code; } diff --git a/content/fetchers.h b/content/fetchers.h index 6ca52245c..3303379d2 100644 --- a/content/fetchers.h +++ b/content/fetchers.h @@ -23,14 +23,13 @@ #ifndef _NETSURF_DESKTOP_FETCHERS_H_ #define _NETSURF_DESKTOP_FETCHERS_H_ +#include #include struct nsurl; struct fetch_multipart_data; struct fetch; -extern bool fetch_active; - /** * Fetcher operations API * @@ -86,6 +85,7 @@ struct fetcher_operation_table { void (*finalise)(lwc_string *scheme); }; + /** * Register a fetcher for a scheme * @@ -95,6 +95,7 @@ struct fetcher_operation_table { */ nserror fetcher_add(lwc_string *scheme, const struct fetcher_operation_table *ops); + /** * Initialise the fetchers. * @@ -102,6 +103,7 @@ nserror fetcher_add(lwc_string *scheme, const struct fetcher_operation_table *op */ nserror fetcher_init(void); + /** * Clean up for quit. * @@ -109,11 +111,31 @@ nserror fetcher_init(void); */ void fetcher_quit(void); + /** - * Do some work on current fetches. + * Get the set of file descriptors the fetchers are currently using. + * + * This obtains the file descriptors the fetch system is using to + * obtain data. It will cause the fetchers to make progress, if + * possible, potentially completing fetches before requiring activity + * on file descriptors. + * + * If a set of descriptors is returned (maxfd is not -1) The caller is + * expected to wait on them (with select etc.) and continue to obtain + * the fdset with this call. This will switch the fetchers from polled + * mode to waiting for network activity which is much more efficient. + * + * \note If the caller does not subsequently obtain the fdset again + * the fetchers will fall back to the less efficient polled + * operation. The fallback to polled operation will only occour after + * a timeout which introduces additional delay. * - * Must be called regularly to make progress on fetches. + * \param read_fd_set[out] The fd set for read. + * \param write_fd_set[out] The fd set for write. + * \param except_fd_set[out] The fd set for exceptions. + * \param maxfd[out] The highest fd number in the set or -1 if no fd available. + * \return NSERROR_OK on success or appropriate error code. */ -void fetcher_poll(void); +nserror fetcher_fdset(fd_set *read_fd_set, fd_set *write_fd_set, fd_set *except_fd_set, int *maxfd); #endif diff --git a/content/fetchers/curl.c b/content/fetchers/curl.c index 39c15b930..32b7189db 100644 --- a/content/fetchers/curl.c +++ b/content/fetchers/curl.c @@ -500,9 +500,7 @@ bool fetch_curl_initiate_fetch(struct curl_fetch_info *fetch, CURL *handle) /* add to the global curl multi handle */ codem = curl_multi_add_handle(fetch_curl_multi, fetch->curl_handle); assert(codem == CURLM_OK || codem == CURLM_CALL_MULTI_PERFORM); - - guit->browser->schedule(10, (void *)fetch_curl_poll, NULL); - + return true; } diff --git a/desktop/netsurf.c b/desktop/netsurf.c index b0c338af6..579648bae 100644 --- a/desktop/netsurf.c +++ b/desktop/netsurf.c @@ -257,8 +257,7 @@ nserror netsurf_init(const char *messages, const char *store_path) int netsurf_main_loop(void) { while (!netsurf_quit) { - guit->browser->poll(fetch_active); - fetcher_poll(); + guit->browser->poll(false); } return 0; @@ -310,5 +309,3 @@ void netsurf_exit(void) LOG(("Exited successfully")); } - - -- cgit v1.2.3 From 8944edd649e74e4864f36d7293921385ba5ca2c7 Mon Sep 17 00:00:00 2001 From: Vincent Sanders Date: Thu, 26 Jun 2014 18:56:42 +0100 Subject: convert all frontends to scheduled fetch operation --- amiga/gui.c | 9 ------ atari/gui.c | 5 ++-- beos/gui.cpp | 34 ++++++++--------------- cocoa/gui.m | 2 +- content/fetch.c | 1 + framebuffer/gui.c | 4 --- gtk/gui.c | 69 +++++++++++++++++++--------------------------- monkey/poll.c | 82 +++++++++++++++++++++++-------------------------------- riscos/gui.c | 11 ++------ windows/gui.c | 5 ---- 10 files changed, 80 insertions(+), 142 deletions(-) (limited to 'content') diff --git a/amiga/gui.c b/amiga/gui.c index 756ed63e2..e2bb39c0c 100644 --- a/amiga/gui.c +++ b/amiga/gui.c @@ -2502,18 +2502,9 @@ void ami_get_msg(void) ami_quit_netsurf_delayed(); } -static void ami_gui_fetch_callback(void *p) -{ - /* This doesn't need to do anything - the scheduled event will - * send a message to trigger Wait() to return, thereby causing - * the event function to return, and NetSurf to call - * hlcache_poll() as part of the usual fetch/event loop. - */ -} static void gui_poll(bool active) { - if(active) ami_schedule(0, ami_gui_fetch_callback, NULL); ami_get_msg(); } diff --git a/atari/gui.c b/atari/gui.c index 7cac90c3b..608d0d859 100644 --- a/atari/gui.c +++ b/atari/gui.c @@ -121,7 +121,7 @@ static void gui_poll(bool active) aes_event_in.emi_tlow = schedule_run(); - if(active || rendering){ + if(rendering){ aes_event_in.emi_tlow = nsoption_int(atari_gui_poll_timeout); } @@ -130,11 +130,10 @@ static void gui_poll(bool active) printf("long poll!\n"); } - if( !active ) { if(input_window && input_window->root->redraw_slots.areas_used > 0) { window_process_redraws(input_window->root); } - } + graf_mkstate(&mx, &my, &dummy, &dummy); aes_event_in.emi_m1.g_x = mx; diff --git a/beos/gui.cpp b/beos/gui.cpp index db4374512..8518ba1ac 100644 --- a/beos/gui.cpp +++ b/beos/gui.cpp @@ -46,7 +46,7 @@ extern "C" { #include "content/content.h" #include "content/content_protected.h" #include "content/fetch.h" -#include "content/fetchers/curl.h" +#include "content/fetchers.h" #include "content/fetchers/resource.h" #include "content/hlcache.h" #include "content/urldb.h" @@ -702,32 +702,23 @@ void nsbeos_pipe_message_top(BMessage *message, BWindow *_this, struct beos_scaf static void gui_poll(bool active) { - CURLMcode code; fd_set read_fd_set, write_fd_set, exc_fd_set; - int max_fd = 0; + int max_fd; struct timeval timeout; unsigned int fd_count = 0; bool block = true; bigtime_t next_schedule = 0; - // handle early deadlines - schedule_run(); + /* get any active fetcher fd */ + fetcher_fdset(&read_fd_set, &write_fd_set, &exc_fd_set, &max_fd); - FD_ZERO(&read_fd_set); - FD_ZERO(&write_fd_set); - FD_ZERO(&exc_fd_set); - - if (active) { - code = curl_multi_fdset(fetch_curl_multi, - &read_fd_set, - &write_fd_set, - &exc_fd_set, - &max_fd); - assert(code == CURLM_OK); - } + /* run the scheduler */ + schedule_run(); // our own event pipe FD_SET(sEventPipe[0], &read_fd_set); + + /** @todo Check if this max_fd should have + 1 */ max_fd = MAX(max_fd, sEventPipe[0] + 1); // If there are pending events elsewhere, we should not be blocking @@ -741,8 +732,10 @@ static void gui_poll(bool active) if (next_schedule < 0) next_schedule = 0; - } else //we're not allowed to sleep, there is other activity going on. + } else {//we're not allowed to sleep, there is other activity going on. + nsbeos_window_process_reformats(); block = false; + } /* LOG(("gui_poll: browser_reformat_pending:%d earliest_callback_timeout:%Ld" @@ -767,11 +760,6 @@ static void gui_poll(bool active) nsbeos_dispatch_event(message); } } - - schedule_run(); - - if (browser_reformat_pending) - nsbeos_window_process_reformats(); } diff --git a/cocoa/gui.m b/cocoa/gui.m index 1cb19a756..713f456cd 100644 --- a/cocoa/gui.m +++ b/cocoa/gui.m @@ -53,7 +53,7 @@ static void gui_poll(bool active) { cocoa_autorelease(); - NSEvent *event = [NSApp nextEventMatchingMask: NSAnyEventMask untilDate: active ? nil : [NSDate distantFuture] + NSEvent *event = [NSApp nextEventMatchingMask: NSAnyEventMask untilDate: [NSDate distantFuture] inMode: NSDefaultRunLoopMode dequeue: YES]; if (nil != event) { diff --git a/content/fetch.c b/content/fetch.c index 10cac9b27..3d1183aa0 100644 --- a/content/fetch.c +++ b/content/fetch.c @@ -61,6 +61,7 @@ /* Define this to turn on verbose fetch logging */ #undef DEBUG_FETCH_VERBOSE +#define DEBUG_FETCH_VERBOSE /** The maximum number of fetchers that can be added */ #define MAX_FETCHERS 8 diff --git a/framebuffer/gui.c b/framebuffer/gui.c index 251326dc2..0632bf3c6 100644 --- a/framebuffer/gui.c +++ b/framebuffer/gui.c @@ -564,10 +564,6 @@ static void framebuffer_poll(bool active) /* run the scheduler and discover how long to wait for the next event */ timeout = schedule_run(); - /* if active do not wait for event, return immediately */ - if (active) - timeout = 0; - /* if redraws are pending do not wait for event, return immediately */ if (fbtk_get_redraw_pending(fbtk)) timeout = 0; diff --git a/gtk/gui.c b/gtk/gui.c index 7de448c0b..9cd89e627 100644 --- a/gtk/gui.c +++ b/gtk/gui.c @@ -33,13 +33,13 @@ #include #include #include -#include #include #include #include #include "content/content.h" #include "content/fetch.h" +#include "content/fetchers.h" #include "content/fetchers/curl.h" #include "content/fetchers/resource.h" #include "content/hlcache.h" @@ -485,51 +485,42 @@ static bool nslog_stream_configure(FILE *fptr) static void nsgtk_poll(bool active) { - CURLMcode code; fd_set read_fd_set, write_fd_set, exc_fd_set; int max_fd; GPollFD *fd_list[1000]; unsigned int fd_count = 0; bool block = true; + fetcher_fdset(&read_fd_set, &write_fd_set, &exc_fd_set, &max_fd); + for (int i = 0; i <= max_fd; i++) { + if (FD_ISSET(i, &read_fd_set)) { + GPollFD *fd = malloc(sizeof *fd); + fd->fd = i; + fd->events = G_IO_IN | G_IO_HUP | G_IO_ERR; + g_main_context_add_poll(0, fd, 0); + fd_list[fd_count++] = fd; + } + if (FD_ISSET(i, &write_fd_set)) { + GPollFD *fd = malloc(sizeof *fd); + fd->fd = i; + fd->events = G_IO_OUT | G_IO_ERR; + g_main_context_add_poll(0, fd, 0); + fd_list[fd_count++] = fd; + } + if (FD_ISSET(i, &exc_fd_set)) { + GPollFD *fd = malloc(sizeof *fd); + fd->fd = i; + fd->events = G_IO_ERR; + g_main_context_add_poll(0, fd, 0); + fd_list[fd_count++] = fd; + } + } + schedule_run(); - if (browser_reformat_pending) + if (browser_reformat_pending) { + nsgtk_window_process_reformats(); block = false; - - if (active) { - FD_ZERO(&read_fd_set); - FD_ZERO(&write_fd_set); - FD_ZERO(&exc_fd_set); - code = curl_multi_fdset(fetch_curl_multi, - &read_fd_set, - &write_fd_set, - &exc_fd_set, - &max_fd); - assert(code == CURLM_OK); - for (int i = 0; i <= max_fd; i++) { - if (FD_ISSET(i, &read_fd_set)) { - GPollFD *fd = malloc(sizeof *fd); - fd->fd = i; - fd->events = G_IO_IN | G_IO_HUP | G_IO_ERR; - g_main_context_add_poll(0, fd, 0); - fd_list[fd_count++] = fd; - } - if (FD_ISSET(i, &write_fd_set)) { - GPollFD *fd = malloc(sizeof *fd); - fd->fd = i; - fd->events = G_IO_OUT | G_IO_ERR; - g_main_context_add_poll(0, fd, 0); - fd_list[fd_count++] = fd; - } - if (FD_ISSET(i, &exc_fd_set)) { - GPollFD *fd = malloc(sizeof *fd); - fd->fd = i; - fd->events = G_IO_ERR; - g_main_context_add_poll(0, fd, 0); - fd_list[fd_count++] = fd; - } - } } gtk_main_iteration_do(block); @@ -539,10 +530,6 @@ static void nsgtk_poll(bool active) free(fd_list[i]); } - schedule_run(); - - if (browser_reformat_pending) - nsgtk_window_process_reformats(); } diff --git a/monkey/poll.c b/monkey/poll.c index 414d458bd..d5b49f059 100644 --- a/monkey/poll.c +++ b/monkey/poll.c @@ -22,7 +22,7 @@ #include "desktop/gui.h" #include "monkey/schedule.h" #include "monkey/browser.h" -#include "content/fetchers/curl.h" +#include "content/fetchers.h" #include "monkey/dispatch.h" #include "monkey/poll.h" @@ -90,58 +90,48 @@ monkey_prepare_input(void) void monkey_poll(bool active) { - CURLMcode code; fd_set read_fd_set, write_fd_set, exc_fd_set; int max_fd; GPollFD *fd_list[1000]; unsigned int fd_count = 0; bool block = true; - + + fetcher_fdset(&read_fd_set, &write_fd_set, &exc_fd_set, &max_fd); + for (int i = 0; i <= max_fd; i++) { + if (FD_ISSET(i, &read_fd_set)) { + GPollFD *fd = malloc(sizeof *fd); + fd->fd = i; + fd->events = G_IO_IN | G_IO_HUP | G_IO_ERR; + g_main_context_add_poll(0, fd, 0); + fd_list[fd_count++] = fd; + LOG(("Want to read %d", i)); + } + if (FD_ISSET(i, &write_fd_set)) { + GPollFD *fd = malloc(sizeof *fd); + fd->fd = i; + fd->events = G_IO_OUT | G_IO_ERR; + g_main_context_add_poll(0, fd, 0); + fd_list[fd_count++] = fd; + LOG(("Want to write %d", i)); + } + if (FD_ISSET(i, &exc_fd_set)) { + GPollFD *fd = malloc(sizeof *fd); + fd->fd = i; + fd->events = G_IO_ERR; + g_main_context_add_poll(0, fd, 0); + fd_list[fd_count++] = fd; + LOG(("Want to check %d", i)); + } + } + schedule_run(); - if (browser_reformat_pending) + if (browser_reformat_pending) { + monkey_window_process_reformats(); block = false; - - if (active) { - FD_ZERO(&read_fd_set); - FD_ZERO(&write_fd_set); - FD_ZERO(&exc_fd_set); - code = curl_multi_fdset(fetch_curl_multi, - &read_fd_set, - &write_fd_set, - &exc_fd_set, - &max_fd); - assert(code == CURLM_OK); - LOG(("maxfd from curl is %d", max_fd)); - for (int i = 0; i <= max_fd; i++) { - if (FD_ISSET(i, &read_fd_set)) { - GPollFD *fd = malloc(sizeof *fd); - fd->fd = i; - fd->events = G_IO_IN | G_IO_HUP | G_IO_ERR; - g_main_context_add_poll(0, fd, 0); - fd_list[fd_count++] = fd; - LOG(("Want to read %d", i)); - } - if (FD_ISSET(i, &write_fd_set)) { - GPollFD *fd = malloc(sizeof *fd); - fd->fd = i; - fd->events = G_IO_OUT | G_IO_ERR; - g_main_context_add_poll(0, fd, 0); - fd_list[fd_count++] = fd; - LOG(("Want to write %d", i)); - } - if (FD_ISSET(i, &exc_fd_set)) { - GPollFD *fd = malloc(sizeof *fd); - fd->fd = i; - fd->events = G_IO_ERR; - g_main_context_add_poll(0, fd, 0); - fd_list[fd_count++] = fd; - LOG(("Want to check %d", i)); - } - } } - - LOG(("Iterate %sactive %sblocking", active?"":"in", block?"":"non-")); + + LOG(("Iterate %sblocking", block?"":"non-")); if (block) { fprintf(stdout, "GENERIC POLL BLOCKING\n"); } @@ -152,9 +142,5 @@ monkey_poll(bool active) free(fd_list[i]); } - schedule_run(); - - if (browser_reformat_pending) - monkey_window_process_reformats(); } diff --git a/riscos/gui.c b/riscos/gui.c index 09f254bb4..00f6ebc11 100644 --- a/riscos/gui.c +++ b/riscos/gui.c @@ -1852,24 +1852,19 @@ static void ro_gui_handle_event(wimp_event_no event, wimp_block *block) /** - * Poll the OS for events (RISC OS). - * - * \param active return as soon as possible + * Poll the RISC OS wimp for events. */ static void riscos_poll(bool active) { wimp_event_no event; wimp_block block; - const wimp_poll_flags mask = wimp_MASK_LOSE | wimp_MASK_GAIN | - wimp_SAVE_FP; + const wimp_poll_flags mask = wimp_MASK_LOSE | wimp_MASK_GAIN | wimp_SAVE_FP; os_t track_poll_offset; /* Poll wimp. */ xhourglass_off(); track_poll_offset = ro_mouse_poll_interval(); - if (active) { - event = wimp_poll(mask, &block, 0); - } else if (sched_active || (track_poll_offset > 0) || + if (sched_active || (track_poll_offset > 0) || browser_reformat_pending) { os_t t = os_read_monotonic_time(); diff --git a/windows/gui.c b/windows/gui.c index 21eff0ef4..824abade7 100644 --- a/windows/gui.c +++ b/windows/gui.c @@ -107,10 +107,6 @@ static void win32_poll(bool active) /* run the scheduler and discover how long to wait for the next event */ timeout = schedule_run(); - /* if active set timeout so message is not waited for */ - if (active) - timeout = 0; - if (timeout == 0) { bRet = PeekMessage(&Msg, NULL, 0, 0, PM_REMOVE); } else { @@ -128,7 +124,6 @@ static void win32_poll(bool active) } } - if (bRet > 0) { TranslateMessage(&Msg); DispatchMessage(&Msg); -- cgit v1.2.3