summaryrefslogtreecommitdiff
path: root/content/fetch.c
diff options
context:
space:
mode:
Diffstat (limited to 'content/fetch.c')
-rw-r--r--content/fetch.c452
1 files changed, 243 insertions, 209 deletions
diff --git a/content/fetch.c b/content/fetch.c
index 2c3f63700..b69f50be1 100644
--- a/content/fetch.c
+++ b/content/fetch.c
@@ -48,16 +48,16 @@ bool fetch_active; /**< Fetches in progress, please call fetch_poll(). */
/** Information about a fetcher for a given scheme. */
typedef struct scheme_fetcher_s {
- char *scheme_name; /**< The scheme. */
- fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */
- fetcher_start_fetch start_fetch; /**< Start a fetch. */
- fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */
- fetcher_free_fetch free_fetch; /**< Free a fetch. */
- fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */
- fetcher_finalise finaliser; /**< Clean up this fetcher. */
- int refcount; /**< When zero, clean up the fetcher. */
- struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */
- struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */
+ char *scheme_name; /**< The scheme. */
+ fetcher_setup_fetch setup_fetch; /**< Set up a fetch. */
+ fetcher_start_fetch start_fetch; /**< Start a fetch. */
+ fetcher_abort_fetch abort_fetch; /**< Abort a fetch. */
+ fetcher_free_fetch free_fetch; /**< Free a fetch. */
+ fetcher_poll_fetcher poll_fetcher; /**< Poll this fetcher. */
+ fetcher_finalise finaliser; /**< Clean up this fetcher. */
+ int refcount; /**< When zero, clean up the fetcher. */
+ struct scheme_fetcher_s *next_fetcher; /**< Next fetcher in the list. */
+ struct scheme_fetcher_s *prev_fetcher; /**< Prev fetcher in the list. */
} scheme_fetcher;
static scheme_fetcher *fetchers = NULL;
@@ -67,15 +67,16 @@ struct fetch {
fetch_callback callback;/**< Callback function. */
bool abort; /**< Abort requested. */
bool stopped; /**< Download stopped on purpose. */
- char *url; /**< URL. */
- char *referer; /**< Referer URL. */
+ char *url; /**< URL. */
+ char *referer; /**< Referer URL. */
bool send_referer; /**< Valid to send the referer */
void *p; /**< Private data for callback. */
char *host; /**< Host part of URL. */
long http_code; /**< HTTP response code, or 0. */
- scheme_fetcher *ops; /**< Fetcher operations for this fetch. NULL if not set. */
- void *fetcher_handle; /**< The handle for the fetcher. */
- bool fetch_is_active; /**< This fetch is active. */
+ scheme_fetcher *ops; /**< Fetcher operations for this fetch,
+ NULL if not set. */
+ void *fetcher_handle; /**< The handle for the fetcher. */
+ bool fetch_is_active; /**< This fetch is active. */
struct fetch *r_prev; /**< Previous active fetch in ::fetch_ring. */
struct fetch *r_next; /**< Next active fetch in ::fetch_ring. */
};
@@ -83,65 +84,13 @@ struct fetch {
static struct fetch *fetch_ring = 0; /**< Ring of active fetches. */
static struct fetch *queue_ring = 0; /**< Ring of queued fetches */
-static void fetch_free(struct fetch *f);
+#define fetch_ref_fetcher(F) F->refcount++
+static void fetch_unref_fetcher(scheme_fetcher *fetcher);
static void fetch_dispatch_jobs(void);
+static bool fetch_choose_and_dispatch(void);
+static bool fetch_dispatch_job(struct fetch *fetch);
+static void fetch_free(struct fetch *f);
-#define fetch_ref_fetcher(F) F->refcount++;
-
-static void fetch_unref_fetcher(scheme_fetcher *fetcher)
-{
- if (--fetcher->refcount == 0) {
- fetcher->finaliser(fetcher->scheme_name);
- free(fetcher->scheme_name);
- if (fetcher == fetchers) {
- fetchers = fetcher->next_fetcher;
- if (fetchers)
- fetchers->prev_fetcher = NULL;
- } else {
- fetcher->prev_fetcher->next_fetcher = fetcher->next_fetcher;
- if (fetcher->next_fetcher != NULL)
- fetcher->next_fetcher->prev_fetcher = fetcher->prev_fetcher;
- }
- free(fetcher);
- }
-}
-
-bool
-fetch_add_fetcher(const char *scheme,
- fetcher_initialise initialiser,
- fetcher_setup_fetch setup_fetch,
- fetcher_start_fetch start_fetch,
- fetcher_abort_fetch abort_fetch,
- fetcher_free_fetch free_fetch,
- fetcher_poll_fetcher poll_fetcher,
- fetcher_finalise finaliser)
-{
- scheme_fetcher *new_fetcher;
- if (!initialiser(scheme))
- return false;
- new_fetcher = malloc(sizeof(scheme_fetcher));
- if (new_fetcher == NULL) {
- finaliser(scheme);
- return false;
- }
- new_fetcher->scheme_name = strdup(scheme);
- if (new_fetcher->scheme_name == NULL) {
- free(new_fetcher);
- finaliser(scheme);
- return false;
- }
- new_fetcher->refcount = 0;
- new_fetcher->setup_fetch = setup_fetch;
- new_fetcher->start_fetch = start_fetch;
- new_fetcher->abort_fetch = abort_fetch;
- new_fetcher->free_fetch = free_fetch;
- new_fetcher->poll_fetcher = poll_fetcher;
- new_fetcher->finaliser = finaliser;
- new_fetcher->next_fetcher = fetchers;
- fetchers = new_fetcher;
- fetch_ref_fetcher(new_fetcher);
- return true;
-}
/**
* Initialise the fetcher.
@@ -151,8 +100,8 @@ fetch_add_fetcher(const char *scheme,
void fetch_init(void)
{
- register_curl_fetchers();
- fetch_active = false;
+ fetch_curl_register();
+ fetch_active = false;
}
@@ -164,14 +113,73 @@ void fetch_init(void)
void fetch_quit(void)
{
- while (fetchers != NULL) {
- if (fetchers->refcount != 1) {
- LOG(("Fetcher for scheme %s still active?!", fetchers->scheme_name));
- /* We shouldn't do this, but... */
- fetchers->refcount = 1;
- }
- fetch_unref_fetcher(fetchers);
- }
+ while (fetchers != NULL) {
+ if (fetchers->refcount != 1) {
+ LOG(("Fetcher for scheme %s still active?!",
+ fetchers->scheme_name));
+ /* We shouldn't do this, but... */
+ fetchers->refcount = 1;
+ }
+ fetch_unref_fetcher(fetchers);
+ }
+}
+
+
+bool fetch_add_fetcher(const char *scheme,
+ fetcher_initialise initialiser,
+ fetcher_setup_fetch setup_fetch,
+ fetcher_start_fetch start_fetch,
+ fetcher_abort_fetch abort_fetch,
+ fetcher_free_fetch free_fetch,
+ fetcher_poll_fetcher poll_fetcher,
+ fetcher_finalise finaliser)
+{
+ scheme_fetcher *new_fetcher;
+ if (!initialiser(scheme))
+ return false;
+ new_fetcher = malloc(sizeof(scheme_fetcher));
+ if (new_fetcher == NULL) {
+ finaliser(scheme);
+ return false;
+ }
+ new_fetcher->scheme_name = strdup(scheme);
+ if (new_fetcher->scheme_name == NULL) {
+ free(new_fetcher);
+ finaliser(scheme);
+ return false;
+ }
+ new_fetcher->refcount = 0;
+ new_fetcher->setup_fetch = setup_fetch;
+ new_fetcher->start_fetch = start_fetch;
+ new_fetcher->abort_fetch = abort_fetch;
+ new_fetcher->free_fetch = free_fetch;
+ new_fetcher->poll_fetcher = poll_fetcher;
+ new_fetcher->finaliser = finaliser;
+ new_fetcher->next_fetcher = fetchers;
+ fetchers = new_fetcher;
+ fetch_ref_fetcher(new_fetcher);
+ return true;
+}
+
+
+void fetch_unref_fetcher(scheme_fetcher *fetcher)
+{
+ if (--fetcher->refcount == 0) {
+ fetcher->finaliser(fetcher->scheme_name);
+ free(fetcher->scheme_name);
+ if (fetcher == fetchers) {
+ fetchers = fetcher->next_fetcher;
+ if (fetchers)
+ fetchers->prev_fetcher = NULL;
+ } else {
+ fetcher->prev_fetcher->next_fetcher =
+ fetcher->next_fetcher;
+ if (fetcher->next_fetcher != NULL)
+ fetcher->next_fetcher->prev_fetcher =
+ fetcher->prev_fetcher;
+ }
+ free(fetcher);
+ }
}
@@ -196,16 +204,17 @@ void fetch_quit(void)
*/
struct fetch * fetch_start(const char *url, const char *referer,
- fetch_callback callback,
- void *p, bool only_2xx, const char *post_urlenc,
- struct form_successful_control *post_multipart,
- bool verifiable, const char *parent_url, char *headers[])
+ fetch_callback callback,
+ void *p, bool only_2xx, const char *post_urlenc,
+ struct form_successful_control *post_multipart,
+ bool verifiable, const char *parent_url,
+ char *headers[])
{
char *host;
struct fetch *fetch;
url_func_result res;
char *ref1 = 0, *ref2 = 0;
- scheme_fetcher *fetcher = fetchers;
+ scheme_fetcher *fetcher = fetchers;
fetch = malloc(sizeof (*fetch));
if (!fetch)
@@ -252,45 +261,45 @@ struct fetch * fetch_start(const char *url, const char *referer,
fetch->http_code = 0;
fetch->r_prev = 0;
fetch->r_next = 0;
- fetch->referer = 0;
- fetch->ops = 0;
- fetch->fetch_is_active = false;
-
- if (referer != NULL) {
- fetch->referer = strdup(referer);
- if (fetch->referer == NULL)
- goto failed;
+ fetch->referer = 0;
+ fetch->ops = 0;
+ fetch->fetch_is_active = false;
+
+ if (referer != NULL) {
+ fetch->referer = strdup(referer);
+ if (fetch->referer == NULL)
+ goto failed;
if (option_send_referer && ref1 && ref2 &&
strcasecmp(ref1, ref2) == 0)
fetch->send_referer = true;
- }
+ }
if (!fetch->url)
goto failed;
- /* Pick the scheme ops */
- while (fetcher) {
- if (strcmp(fetcher->scheme_name, ref1) == 0) {
- fetch->ops = fetcher;
- break;
- }
- fetcher = fetcher->next_fetcher;
- }
+ /* Pick the scheme ops */
+ while (fetcher) {
+ if (strcmp(fetcher->scheme_name, ref1) == 0) {
+ fetch->ops = fetcher;
+ break;
+ }
+ fetcher = fetcher->next_fetcher;
+ }
- if (fetch->ops == NULL)
- goto failed;
+ if (fetch->ops == NULL)
+ goto failed;
- /* Got a scheme fetcher, try and set up the fetch */
- fetch->fetcher_handle =
- fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc,
- post_multipart, verifiable, parent_url,
- (const char **)headers);
+ /* Got a scheme fetcher, try and set up the fetch */
+ fetch->fetcher_handle =
+ fetch->ops->setup_fetch(fetch, url, only_2xx, post_urlenc,
+ post_multipart, verifiable, parent_url,
+ (const char **)headers);
- if (fetch->fetcher_handle == NULL)
- goto failed;
+ if (fetch->fetcher_handle == NULL)
+ goto failed;
- /* Rah, got it, so ref the fetcher. */
- fetch_ref_fetcher(fetch->ops);
+ /* Rah, got it, so ref the fetcher. */
+ fetch_ref_fetcher(fetch->ops);
/* these aren't needed past here */
if (ref1) {
@@ -298,12 +307,12 @@ struct fetch * fetch_start(const char *url, const char *referer,
ref1 = 0;
}
- if (ref2) {
- free(ref2);
- ref2 = 0;
- }
+ if (ref2) {
+ free(ref2);
+ ref2 = 0;
+ }
- /* Dump us in the queue and ask the queue to run. */
+ /* Dump us in the queue and ask the queue to run. */
RING_INSERT(queue_ring, fetch);
fetch_dispatch_jobs();
return fetch;
@@ -313,37 +322,66 @@ failed:
if (ref1)
free(ref1);
free(fetch->url);
- if (fetch->referer)
- free(fetch->referer);
- free(fetch);
+ if (fetch->referer)
+ free(fetch->referer);
+ free(fetch);
return 0;
}
+
/**
- * Dispatch a single job
+ * Dispatch as many jobs as we have room to dispatch.
*/
-static bool fetch_dispatch_job(struct fetch *fetch)
+void fetch_dispatch_jobs(void)
{
- RING_REMOVE(queue_ring, fetch);
- LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch,
- fetch->fetcher_handle, fetch->url));
- if (!fetch->ops->start_fetch(fetch->fetcher_handle)) {
- RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */
- return false;
- } else {
- RING_INSERT(fetch_ring, fetch);
- fetch->fetch_is_active = true;
- return true;
+ int all_active, all_queued;
+
+ if (!queue_ring)
+ return; /* Nothing to do, the queue is empty */
+ RING_GETSIZE(struct fetch, queue_ring, all_queued);
+ RING_GETSIZE(struct fetch, fetch_ring, all_active);
+
+ LOG(("queue_ring %i, fetch_ring %i", all_queued, all_active));
+
+ struct fetch *q = queue_ring;
+ if (q) {
+ do {
+ LOG(("queue_ring: %s", q->url));
+ q = q->r_next;
+ } while (q != queue_ring);
}
+ struct fetch *f = fetch_ring;
+ if (f) {
+ do {
+ LOG(("fetch_ring: %s", f->url));
+ f = f->r_next;
+ } while (f != fetch_ring);
+ }
+
+ while ( all_queued && all_active < option_max_fetchers ) {
+ /*LOG(("%d queued, %d fetching", all_queued, all_active));*/
+ if (fetch_choose_and_dispatch()) {
+ all_queued--;
+ all_active++;
+ } else {
+ /* Either a dispatch failed or we ran out. Just stop */
+ break;
+ }
+ }
+ fetch_active = (all_active > 0);
+ LOG(("Fetch ring is now %d elements.", all_active));
+ LOG(("Queue ring is now %d elements.", all_queued));
}
+
/**
- * Choose and dispatch a single job. Return false if we failed to dispatch anything.
+ * Choose and dispatch a single job. Return false if we failed to dispatch
+ * anything.
*
* We don't check the overall dispatch size here because we're not called unless
* there is room in the fetch queue for us.
*/
-static bool fetch_choose_and_dispatch(void)
+bool fetch_choose_and_dispatch(void)
{
struct fetch *queueitem;
queueitem = queue_ring;
@@ -352,7 +390,8 @@ static bool fetch_choose_and_dispatch(void)
* fetch ring
*/
int countbyhost;
- RING_COUNTBYHOST(struct fetch, fetch_ring, countbyhost, queueitem->host);
+ RING_COUNTBYHOST(struct fetch, fetch_ring, countbyhost,
+ queueitem->host);
if (countbyhost < option_max_fetchers_per_host) {
/* We can dispatch this item in theory */
return fetch_dispatch_job(queueitem);
@@ -362,32 +401,26 @@ static bool fetch_choose_and_dispatch(void)
return false;
}
+
/**
- * Dispatch as many jobs as we have room to dispatch.
+ * Dispatch a single job
*/
-static void fetch_dispatch_jobs(void)
+bool fetch_dispatch_job(struct fetch *fetch)
{
- int all_active, all_queued;
-
- if (!queue_ring)
- return; /* Nothing to do, the queue is empty */
- RING_GETSIZE(struct fetch, queue_ring, all_queued);
- RING_GETSIZE(struct fetch, fetch_ring, all_active);
- while ( all_queued && all_active < option_max_fetchers ) {
- /*LOG(("%d queued, %d fetching", all_queued, all_active));*/
- if (fetch_choose_and_dispatch()) {
- all_queued--;
- all_active++;
- } else {
- /* Either a dispatch failed or we ran out. Just stop */
- break;
- }
+ RING_REMOVE(queue_ring, fetch);
+ LOG(("Attempting to start fetch %p, fetcher %p, url %s", fetch,
+ fetch->fetcher_handle, fetch->url));
+ if (!fetch->ops->start_fetch(fetch->fetcher_handle)) {
+ RING_INSERT(queue_ring, fetch); /* Put it back on the end of the queue */
+ return false;
+ } else {
+ RING_INSERT(fetch_ring, fetch);
+ fetch->fetch_is_active = true;
+ return true;
}
- fetch_active = (all_active > 0);
- LOG(("Fetch ring is now %d elements.", all_active));
- LOG(("Queue ring is now %d elements.", all_queued));
}
+
/**
* Abort a fetch.
*/
@@ -396,7 +429,7 @@ void fetch_abort(struct fetch *f)
{
assert(f);
LOG(("fetch %p, fetcher %p, url '%s'", f, f->fetcher_handle, f->url));
- f->ops->abort_fetch(f->fetcher_handle);
+ f->ops->abort_fetch(f->fetcher_handle);
}
@@ -406,13 +439,13 @@ void fetch_abort(struct fetch *f)
void fetch_free(struct fetch *f)
{
- LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle));
- f->ops->free_fetch(f->fetcher_handle);
- fetch_unref_fetcher(f->ops);
+ LOG(("Freeing fetch %p, fetcher %p", f, f->fetcher_handle));
+ f->ops->free_fetch(f->fetcher_handle);
+ fetch_unref_fetcher(f->ops);
free(f->url);
free(f->host);
- if (f->referer)
- free(f->referer);
+ if (f->referer)
+ free(f->referer);
free(f);
}
@@ -425,14 +458,17 @@ void fetch_free(struct fetch *f)
void fetch_poll(void)
{
- scheme_fetcher *fetcher = fetchers;
- if (!fetch_active)
- return; /* No point polling, there's no fetch active. */
- while (fetcher != NULL) {
- /* LOG(("Polling fetcher for %s", fetcher->scheme_name)); */
- fetcher->poll_fetcher(fetcher->scheme_name);
- fetcher = fetcher->next_fetcher;
- }
+ scheme_fetcher *fetcher = fetchers;
+
+ fetch_dispatch_jobs();
+
+ if (!fetch_active)
+ return; /* No point polling, there's no fetch active. */
+ while (fetcher != NULL) {
+ /* LOG(("Polling fetcher for %s", fetcher->scheme_name)); */
+ fetcher->poll_fetcher(fetcher->scheme_name);
+ fetcher = fetcher->next_fetcher;
+ }
}
@@ -447,18 +483,18 @@ bool fetch_can_fetch(const char *url)
{
const char *semi;
size_t len;
- scheme_fetcher *fetcher = fetchers;
+ scheme_fetcher *fetcher = fetchers;
if ((semi = strchr(url, ':')) == NULL)
return false;
len = semi - url;
- while (fetcher != NULL) {
- if (strlen(fetcher->scheme_name) == len &&
- strncmp(fetcher->scheme_name, url, len) == 0)
- return true;
- fetcher = fetcher->next_fetcher;
- }
+ while (fetcher != NULL) {
+ if (strlen(fetcher->scheme_name) == len &&
+ strncmp(fetcher->scheme_name, url, len) == 0)
+ return true;
+ fetcher = fetcher->next_fetcher;
+ }
return false;
}
@@ -469,8 +505,8 @@ bool fetch_can_fetch(const char *url)
*/
void fetch_change_callback(struct fetch *fetch,
- fetch_callback callback,
- void *p)
+ fetch_callback callback,
+ void *p)
{
assert(fetch);
fetch->callback = callback;
@@ -503,49 +539,47 @@ void
fetch_send_callback(fetch_msg msg, struct fetch *fetch, const void *data,
unsigned long size)
{
- LOG(("Fetcher sending callback. Fetch %p, fetcher %p data %p size %lu",
- fetch, fetch->fetcher_handle, data, size));
- fetch->callback(msg, fetch->p, data, size);
+ LOG(("Fetcher sending callback. Fetch %p, fetcher %p data %p size %lu",
+ fetch, fetch->fetcher_handle, data, size));
+ fetch->callback(msg, fetch->p, data, size);
}
void
fetch_can_be_freed(struct fetch *fetch)
{
- int all_active, all_queued;
-
- /* Go ahead and free the fetch properly now */
- LOG(("Fetch %p, fetcher %p can be freed", fetch, fetch->fetcher_handle));
-
- if (fetch->fetch_is_active) {
- RING_REMOVE(fetch_ring, fetch);
- } else {
- RING_REMOVE(queue_ring, fetch);
- }
-
+ int all_active, all_queued;
+
+ /* Go ahead and free the fetch properly now */
+ LOG(("Fetch %p, fetcher %p can be freed", fetch, fetch->fetcher_handle));
+
+ if (fetch->fetch_is_active) {
+ RING_REMOVE(fetch_ring, fetch);
+ } else {
+ RING_REMOVE(queue_ring, fetch);
+ }
+
RING_GETSIZE(struct fetch, fetch_ring, all_active);
RING_GETSIZE(struct fetch, queue_ring, all_queued);
-
- fetch_active = (all_active > 0);
-
- LOG(("Fetch ring is now %d elements.", all_active));
- LOG(("Queue ring is now %d elements.", all_queued));
-
- fetch_free(fetch);
-
- fetch_dispatch_jobs();
+
+ fetch_active = (all_active > 0);
+
+ LOG(("Fetch ring is now %d elements.", all_active));
+ LOG(("Queue ring is now %d elements.", all_queued));
+
+ fetch_free(fetch);
}
void
fetch_set_http_code(struct fetch *fetch, long http_code)
{
- LOG(("Setting HTTP code to %ld", http_code));
- fetch->http_code = http_code;
+ LOG(("Setting HTTP code to %ld", http_code));
+ fetch->http_code = http_code;
}
const char *
fetch_get_referer_to_send(struct fetch *fetch)
{
- if (fetch->send_referer)
- return fetch->referer;
- return NULL;
+ if (fetch->send_referer)
+ return fetch->referer;
+ return NULL;
}