From f58b5924a54fbef69de33dbb0340d1bf9f4fd237 Mon Sep 17 00:00:00 2001 From: Vincent Sanders Date: Mon, 13 Aug 2012 17:00:43 +0100 Subject: alter file fetcher to use mmap where available --- content/fetchers/file.c | 119 ++++++++++++++++++++++++++++++++++++++++-------- utils/config.h | 14 ++---- utils/utils.h | 12 +++++ 3 files changed, 116 insertions(+), 29 deletions(-) diff --git a/content/fetchers/file.c b/content/fetchers/file.c index 4deb93248..abedaa7b3 100644 --- a/content/fetchers/file.c +++ b/content/fetchers/file.c @@ -34,9 +34,14 @@ #include #include +#include "utils/config.h" + +#ifdef HAVE_MMAP +#include +#endif + #include -#include "utils/config.h" #include "content/dirlist.h" #include "content/fetch.h" #include "content/fetchers/file.h" @@ -249,6 +254,82 @@ fetch_file_process_error_aborted: static void fetch_file_process_plain(struct fetch_file_context *ctx, struct stat *fdstat) { +#ifdef HAVE_MMAP + fetch_msg msg; + char *buf; + size_t buf_size; + + int fd; /**< The file descriptor of the object */ + + /* Check if we can just return not modified */ + if (ctx->file_etag != 0 && ctx->file_etag == fdstat->st_mtime) { + fetch_set_http_code(ctx->fetchh, 304); + msg.type = FETCH_NOTMODIFIED; + fetch_file_send_callback(&msg, ctx); + return; + } + + fd = open(ctx->path, O_RDONLY); + if (fd < 0) { + /* process errors as appropriate */ + fetch_file_process_error(ctx, + fetch_file_errno_to_http_code(errno)); + return; + } + + /* set buffer size */ + buf_size = fdstat->st_size; + if (buf_size > FETCH_FILE_MAX_BUF_SIZE) + buf_size = FETCH_FILE_MAX_BUF_SIZE; + + /* allocate the buffer storage */ + buf = mmap(NULL, buf_size, PROT_READ, MAP_SHARED, fd, 0); + if (buf == MAP_FAILED) { + msg.type = FETCH_ERROR; + msg.data.error = "Unable to map memory for file data buffer"; + fetch_file_send_callback(&msg, ctx); + close(fd); + return; + } + + /* fetch is going to be successful */ + fetch_set_http_code(ctx->fetchh, 200); + + /* Any callback can result in the fetch being aborted. + * Therefore, we _must_ check for this after _every_ call to + * fetch_file_send_callback(). + */ + + /* content type */ + if (fetch_file_send_header(ctx, "Content-Type: %s", + fetch_filetype(ctx->path))) + goto fetch_file_process_aborted; + + /* content length */ + if (fetch_file_send_header(ctx, "Content-Length: %"SSIZET_FMT, fdstat->st_size)) + goto fetch_file_process_aborted; + + /* create etag */ + if (fetch_file_send_header(ctx, "ETag: \"%10" PRId64 "\"", + (int64_t) fdstat->st_mtime)) + goto fetch_file_process_aborted; + + + msg.type = FETCH_DATA; + msg.data.header_or_data.buf = (const uint8_t *) buf; + msg.data.header_or_data.len = buf_size; + fetch_file_send_callback(&msg, ctx); + + if (ctx->aborted == false) { + msg.type = FETCH_FINISHED; + fetch_file_send_callback(&msg, ctx); + } + +fetch_file_process_aborted: + + munmap(buf, buf_size); + close(fd); +#else fetch_msg msg; char *buf; size_t buf_size; @@ -256,7 +337,7 @@ static void fetch_file_process_plain(struct fetch_file_context *ctx, ssize_t tot_read = 0; ssize_t res; - int fd; /**< The file descriptor of the object */ + FILE *infile; /* Check if we can just return not modified */ if (ctx->file_etag != 0 && ctx->file_etag == fdstat->st_mtime) { @@ -266,8 +347,8 @@ static void fetch_file_process_plain(struct fetch_file_context *ctx, return; } - fd = open(ctx->path, O_RDONLY | O_BINARY); - if (fd < 0) { + infile = fopen(ctx->path, "rb"); + if (infile == NULL) { /* process errors as appropriate */ fetch_file_process_error(ctx, fetch_file_errno_to_http_code(errno)); @@ -286,7 +367,7 @@ static void fetch_file_process_plain(struct fetch_file_context *ctx, msg.data.error = "Unable to allocate memory for file data buffer"; fetch_file_send_callback(&msg, ctx); - close(fd); + fclose(infile); return; } @@ -314,21 +395,20 @@ static void fetch_file_process_plain(struct fetch_file_context *ctx, /* main data loop */ while (tot_read < fdstat->st_size) { - res = read(fd, buf, buf_size); - if (res == -1) { - msg.type = FETCH_ERROR; - msg.data.error = "Error reading file"; - fetch_file_send_callback(&msg, ctx); - goto fetch_file_process_aborted; - } - + res = fread(buf, 1, buf_size, infile); if (res == 0) { - msg.type = FETCH_ERROR; - msg.data.error = "Unexpected EOF reading file"; - fetch_file_send_callback(&msg, ctx); - goto fetch_file_process_aborted; + if (feof(infile)) { + msg.type = FETCH_ERROR; + msg.data.error = "Unexpected EOF reading file"; + fetch_file_send_callback(&msg, ctx); + goto fetch_file_process_aborted; + } else { + msg.type = FETCH_ERROR; + msg.data.error = "Error reading file"; + fetch_file_send_callback(&msg, ctx); + goto fetch_file_process_aborted; + } } - tot_read += res; msg.type = FETCH_DATA; @@ -345,8 +425,9 @@ static void fetch_file_process_plain(struct fetch_file_context *ctx, fetch_file_process_aborted: - close(fd); + fclose(infile); free(buf); +#endif return; } diff --git a/utils/config.h b/utils/config.h index 60b81967a..76142eeb6 100644 --- a/utils/config.h +++ b/utils/config.h @@ -90,6 +90,10 @@ char *realpath(const char *path, char *resolved_path); #undef HAVE_STDOUT #endif +#define HAVE_MMAP +#if (defined(_WIN32) || defined(riscos) || defined(__HAIKU__) || defined(__BEOS__) || defined(__amigaos4__) || defined(__AMIGA__)) +#undef HAVE_MMAP +#endif /* This section toggles build options on and off. * Simply undefine a symbol to turn the relevant feature off. @@ -127,14 +131,4 @@ char *realpath(const char *path, char *resolved_path); #define NO_IPV6 #endif -/* windows */ -#if (defined(_WIN32)) -#define SSIZET_FMT "Iu" -#elif (defined(riscos)) -#define SSIZET_FMT "zd" -#else -#define SSIZET_FMT "zd" -#define O_BINARY 0 -#endif - #endif diff --git a/utils/utils.h b/utils/utils.h index 02bf75fea..f3dba449f 100644 --- a/utils/utils.h +++ b/utils/utils.h @@ -32,26 +32,38 @@ #ifndef NOF_ELEMENTS #define NOF_ELEMENTS(array) (sizeof(array)/sizeof(*(array))) #endif + #ifndef ABS #define ABS(x) (((x)>0)?(x):(-(x))) #endif + #ifdef __MINT__ /* avoid using GCCs builtin min/max functions */ #undef min #undef max #endif + #ifndef min #define min(x,y) (((x)<(y))?(x):(y)) #endif + #ifndef max #define max(x,y) (((x)>(y))?(x):(y)) #endif + #ifndef PRIxPTR #define PRIxPTR "x" #endif + #ifndef PRId64 #define PRId64 "lld" #endif +#if defined(_WIN32) +#define SSIZET_FMT "Iu" +#else +#define SSIZET_FMT "zd" +#endif + #if defined(__GNUC__) && (__GNUC__ < 3) #define FLEX_ARRAY_LEN_DECL 0 #else -- cgit v1.2.3