support http method HEAD

http/server/FileCache.cpp

@@ -9,25 +9,24 @@
 
 #define ETAG_FMT    "\"%zx-%zx\""
 
-file_cache_t* FileCache::Open(const char* filepath, void* ctx) {
+file_cache_ptr FileCache::Open(const char* filepath, bool need_read, void* ctx) {
     std::lock_guard<std::mutex> locker(mutex_);
-    file_cache_t* fc = Get(filepath);
+    file_cache_ptr fc = Get(filepath);
     bool modified = false;
     if (fc) {
-        time_t tt;
-        time(&tt);
-        if (tt - fc->stat_time > file_stat_interval) {
-            time_t mtime = fc->st.st_mtime;
-            stat(filepath, &fc->st);
-            fc->stat_time = tt;
+        time_t now = time(NULL);
+        if (now - fc->stat_time > file_stat_interval) {
+            modified = fc->if_modified(filepath);
+            fc->stat_time = now;
             fc->stat_cnt++;
-            if (mtime != fc->st.st_mtime) {
-                modified = true;
-                fc->stat_cnt = 1;
+        }
+        if (need_read) {
+            if ((!modified) && fc->filebuf.len == fc->st.st_size) {
+                need_read = false;
             }
         }
     }
-    if (fc == NULL || modified) {
+    if (fc == NULL || modified || need_read) {
         int flags = O_RDONLY;
 #ifdef O_BINARY
         flags |= O_BINARY;
@@ -43,7 +42,7 @@ file_cache_t* FileCache::Open(const char* filepath, void* ctx) {
             if (S_ISREG(st.st_mode) ||
                 (S_ISDIR(st.st_mode) &&
                  filepath[strlen(filepath)-1] == '/')) {
-                fc = new file_cache_t;
+                fc.reset(new file_cache_t);
                 //fc->filepath = filepath;
                 fc->st = st;
                 time(&fc->open_time);
@@ -57,11 +56,17 @@ file_cache_t* FileCache::Open(const char* filepath, void* ctx) {
         }
         if (S_ISREG(fc->st.st_mode)) {
             // FILE
-            fc->resize_buf(fc->st.st_size);
-            int nread = read(fd, fc->filebuf.base, fc->filebuf.len);
-            if (nread != fc->filebuf.len) {
-                hloge("Too large file: %s", filepath);
-                return NULL;
+            if (need_read) {
+                if (fc->st.st_size > FILE_CACHE_MAX_SIZE) {
+                    hlogw("Too large file: %s", filepath);
+                    return NULL;
+                }
+                fc->resize_buf(fc->st.st_size);
+                int nread = read(fd, fc->filebuf.base, fc->filebuf.len);
+                if (nread != fc->filebuf.len) {
+                    hloge("Failed to read file: %s", filepath);
+                    return NULL;
+                }
             }
             const char* suffix = strrchr(filepath, '.');
             if (suffix) {
@@ -82,21 +87,47 @@ file_cache_t* FileCache::Open(const char* filepath, void* ctx) {
     return fc;
 }
 
-int FileCache::Close(const char* filepath) {
+bool FileCache::Close(const char* filepath) {
     std::lock_guard<std::mutex> locker(mutex_);
     auto iter = cached_files.find(filepath);
     if (iter != cached_files.end()) {
-        delete iter->second;
         iter = cached_files.erase(iter);
-        return 0;
+        return true;
     }
-    return -1;
+    return false;
 }
 
-file_cache_t* FileCache::Get(const char* filepath) {
+bool FileCache::Close(const file_cache_ptr& fc) {
+    std::lock_guard<std::mutex> locker(mutex_);
+    auto iter = cached_files.begin();
+    while (iter != cached_files.end()) {
+        if (iter->second == fc) {
+            iter = cached_files.erase(iter);
+            return true;
+        } else {
+            ++iter;
+        }
+    }
+    return false;
+}
+
+file_cache_ptr FileCache::Get(const char* filepath) {
     auto iter = cached_files.find(filepath);
     if (iter != cached_files.end()) {
         return iter->second;
     }
     return NULL;
 }
+
+void FileCache::RemoveExpiredFileCache() {
+    std::lock_guard<std::mutex> locker(mutex_);
+    time_t now = time(NULL);
+    auto iter = cached_files.begin();
+    while (iter != cached_files.end()) {
+        if (now - iter->second->stat_time > file_expired_time) {
+            iter = cached_files.erase(iter);
+        } else {
+            ++iter;
+        }
+    }
+}

http/server/FileCache.h

@@ -1,6 +1,7 @@
 #ifndef HV_FILE_CACHE_H_
 #define HV_FILE_CACHE_H_
 
+#include <memory>
 #include <map>
 #include <string>
 #include <mutex>
@@ -8,7 +9,8 @@
 #include "hbuf.h"
 #include "hstring.h"
 
-#define HTTP_HEADER_MAX_LENGTH      1024 // 1k
+#define HTTP_HEADER_MAX_LENGTH      1024        // 1K
+#define FILE_CACHE_MAX_SIZE         (1 << 30)   // 1G
 
 typedef struct file_cache_s {
     //std::string filepath;
@@ -28,6 +30,15 @@ typedef struct file_cache_s {
         content_type = NULL;
     }
 
+    bool if_modified(const char* filepath) {
+        time_t mtime = st.st_mtime;
+        stat(filepath, &st);
+        if (mtime == st.st_mtime) {
+            return false;
+        }
+        return true;
+    }
+
     void resize_buf(int filesize) {
         buf.resize(HTTP_HEADER_MAX_LENGTH + filesize);
         filebuf.base = buf.base + HTTP_HEADER_MAX_LENGTH;
@@ -42,34 +53,31 @@ typedef struct file_cache_s {
     }
 } file_cache_t;
 
-// filepath => file_cache_t
-typedef std::map<std::string, file_cache_t*> FileCacheMap;
+typedef std::shared_ptr<file_cache_t>           file_cache_ptr;
+// filepath => file_cache_ptr
+typedef std::map<std::string, file_cache_ptr>   FileCacheMap;
 
-#define DEFAULT_FILE_STAT_INTERVAL  10 // s
-#define DEFAULT_FILE_CACHED_TIME    60 // s
+#define DEFAULT_FILE_STAT_INTERVAL      10 // s
+#define DEFAULT_FILE_EXPIRED_TIME       60 // s
 class FileCache {
 public:
     int file_stat_interval;
-    int file_cached_time;
-    FileCacheMap cached_files;
-    std::mutex mutex_;
+    int file_expired_time;
+    FileCacheMap    cached_files;
+    std::mutex      mutex_;
 
     FileCache() {
-        file_stat_interval  = DEFAULT_FILE_STAT_INTERVAL;
-        file_cached_time    = DEFAULT_FILE_CACHED_TIME;
+        file_stat_interval = DEFAULT_FILE_STAT_INTERVAL;
+        file_expired_time  = DEFAULT_FILE_EXPIRED_TIME;
     }
 
-    ~FileCache() {
-        for (auto& pair : cached_files) {
-            delete pair.second;
-        }
-        cached_files.clear();
-    }
+    file_cache_ptr Open(const char* filepath, bool need_read = true, void* ctx = NULL);
+    bool Close(const char* filepath);
+    bool Close(const file_cache_ptr& fc);
+    void RemoveExpiredFileCache();
 
-    file_cache_t* Open(const char* filepath, void* ctx = NULL);
-    int Close(const char* filepath);
 protected:
-    file_cache_t* Get(const char* filepath);
+    file_cache_ptr Get(const char* filepath);
 };
 
 #endif // HV_FILE_CACHE_H_

http/server/HttpHandler.cpp

@@ -32,7 +32,8 @@ preprocessor:
             goto make_http_status_page;
         }
     }
-    else if (service->document_root.size() != 0 && req.method == HTTP_GET) {
+    else if (service->document_root.size() != 0 &&
+            (req.method == HTTP_GET || req.method == HTTP_HEAD)) {
         // web service
         // path safe check
         const char* req_path = req.path.c_str();
@@ -51,7 +52,8 @@ preprocessor:
             is_index_of = true;
         }
         if (!is_dir || is_index_of) {
-            fc = files->Open(filepath.c_str(), (void*)req_path);
+            bool need_read = req.method == HTTP_HEAD ? false : true;
+            fc = files->Open(filepath.c_str(), need_read, (void*)req_path);
         }
         if (fc == NULL) {
             // Not Found
@@ -90,7 +92,7 @@ make_http_status_page:
             std::string filepath = service->document_root;
             filepath += '/';
             filepath += service->error_page;
-            fc = files->Open(filepath.c_str(), NULL);
+            fc = files->Open(filepath.c_str(), true, NULL);
         }
         // status page
         if (fc == NULL && res.body.size() == 0) {
@@ -100,18 +102,19 @@ make_http_status_page:
     }
 
     if (fc) {
-        // link file cache
-        res.content = (unsigned char*)fc->filebuf.base;
         res.content_length = fc->filebuf.len;
         if (fc->content_type && *fc->content_type != '\0') {
             res.headers["Content-Type"] = fc->content_type;
-            res.FillContentType();
         }
-        char sz[64];
-        snprintf(sz, sizeof(sz), "%d", res.content_length);
-        res.headers["Content-Length"] = sz;
         res.headers["Last-Modified"] = fc->last_modified;
         res.headers["Etag"] = fc->etag;
+        if (req.method == HTTP_HEAD) {
+            res.headers["Accept-Ranges"] = "bytes";
+            res.content = NULL;
+            fc = NULL;
+        } else {
+            res.content = fc->filebuf.base;
+        }
     }
 
 postprocessor:

http/server/HttpHandler.h

@@ -61,7 +61,7 @@ public:
     // for http
     HttpService             *service;
     FileCache               *files;
-    file_cache_t            *fc;
+    file_cache_ptr          fc;
 
     HttpRequest             req;
     HttpResponse            res;
@@ -75,7 +75,6 @@ public:
         protocol = UNKNOWN;
         service = NULL;
         files = NULL;
-        fc = NULL;
         ws_cbs = NULL;
     }
 

http/server/HttpServer.cpp

@@ -225,21 +225,25 @@ static void on_recv(hio_t* io, void* _buf, int readbytes) {
     }
 
     if (req->http_major == 1) {
-        std::string header = res->Dump(true, false);
         hbuf_t sendbuf;
         bool send_in_one_packet = true;
+        std::string header = res->Dump(true, false);
         int content_length = res->ContentLength();
+        const char* content = (const char*)res->Content();
         if (handler->fc) {
-            // no copy filebuf, more efficient
+            // NOTE: no copy filebuf, more efficient
             handler->fc->prepend_header(header.c_str(), header.size());
             sendbuf = handler->fc->httpbuf;
         }
         else {
-            if (content_length > (1 << 20)) {
-                send_in_one_packet = false;
-            }
-            else if (content_length != 0) {
-                header.insert(header.size(), (const char*)res->Content(), content_length);
+            if (content) {
+                // NOTE: send header+body in one package if < 1M
+                if (content_length > (1 << 20)) {
+                    send_in_one_packet = false;
+                }
+                else if (content_length != 0) {
+                    header.append(content);
+                }
             }
             sendbuf.base = (char*)header.c_str();
             sendbuf.len = header.size();
@@ -248,7 +252,7 @@ static void on_recv(hio_t* io, void* _buf, int readbytes) {
         hio_write(io, sendbuf.base, sendbuf.len);
         if (send_in_one_packet == false) {
             // send body
-            hio_write(io, res->Content(), content_length);
+            hio_write(io, content, content_length);
         }
     }
     else if (req->http_major == 2) {
@@ -259,6 +263,13 @@ static void on_recv(hio_t* io, void* _buf, int readbytes) {
         }
     }
 
+    // NOTE: remove file cache if > 16M
+    if (handler->fc && handler->fc->filebuf.len > (1 << 24)) {
+        handler->files->Close(handler->fc);
+        handler->fc = NULL;
+    }
+
+    // LOG
     hloop_t* loop = hevent_loop(io);
     hlogi("[%ld-%ld][%s:%d][%s %s]=>[%d %s]",
         hloop_pid(loop), hloop_tid(loop),
@@ -282,6 +293,7 @@ static void on_recv(hio_t* io, void* _buf, int readbytes) {
         return;
     }
 
+    // keep-alive
     if (keepalive) {
         handler->Reset();
         parser->InitRequest(req);
@@ -332,23 +344,6 @@ static void on_accept(hio_t* io) {
     hevent_set_userdata(io, handler);
 }
 
-static void handle_cached_files(htimer_t* timer) {
-    FileCache* pfc = (FileCache*)hevent_userdata(timer);
-    file_cache_t* fc = NULL;
-    time_t tt = time(NULL);
-    std::lock_guard<std::mutex> locker(pfc->mutex_);
-    auto iter = pfc->cached_files.begin();
-    while (iter != pfc->cached_files.end()) {
-        fc = iter->second;
-        if (tt - fc->stat_time > pfc->file_cached_time) {
-            delete fc;
-            iter = pfc->cached_files.erase(iter);
-            continue;
-        }
-        ++iter;
-    }
-}
-
 static void loop_thread(void* userdata) {
     http_server_t* server = (http_server_t*)userdata;
     int listenfd = server->listenfd;
@@ -360,22 +355,23 @@ static void loop_thread(void* userdata) {
     if (server->ssl) {
         hio_enable_ssl(listenio);
     }
-    // fsync logfile when idle
-    hlog_disable_fsync();
-    hidle_add(hloop, [](hidle_t*) {
-        hlog_fsync();
-    }, INFINITE);
-    // timer handle_cached_files
-    FileCache* filecache = default_filecache();
-    htimer_t* timer = htimer_add(hloop, handle_cached_files, filecache->file_cached_time * 1000);
-    hevent_set_userdata(timer, filecache);
 
     HttpServerPrivdata* privdata = (HttpServerPrivdata*)server->privdata;
-    if (privdata) {
-        privdata->mutex_.lock();
-        privdata->loops.push_back(loop);
-        privdata->mutex_.unlock();
+    privdata->mutex_.lock();
+    if (privdata->loops.size() == 0) {
+        // NOTE: fsync logfile when idle
+        hlog_disable_fsync();
+        hidle_add(hloop, [](hidle_t*) {
+            hlog_fsync();
+        }, INFINITE);
+        // NOTE: add timer to remove expired file cache
+        htimer_add(hloop, [](htimer_t*) {
+            FileCache* filecache = default_filecache();
+            filecache->RemoveExpiredFileCache();
+        }, DEFAULT_FILE_EXPIRED_TIME * 1000);
     }
+    privdata->loops.push_back(loop);
+    privdata->mutex_.unlock();
 
     loop->run();
 }
@@ -389,6 +385,9 @@ int http_server_run(http_server_t* server, int wait) {
         server->service = default_http_service();
     }
 
+    HttpServerPrivdata* privdata = new HttpServerPrivdata;
+    server->privdata = privdata;
+
     if (server->worker_processes) {
         // multi-processes
         return master_workers_run(loop_thread, server, server->worker_processes, server->worker_threads, wait);
@@ -396,13 +395,7 @@ int http_server_run(http_server_t* server, int wait) {
     else {
         // multi-threads
         if (server->worker_threads == 0) server->worker_threads = 1;
-
-        // for SDK implement http_server_stop
-        HttpServerPrivdata* privdata = new HttpServerPrivdata;
-        server->privdata = privdata;
-
-        int i = wait ? 1 : 0;
-        for (; i < server->worker_threads; ++i) {
+        for (int i = wait ? 1 : 0; i < server->worker_threads; ++i) {
             hthread_t thrd = hthread_create((hthread_routine)loop_thread, server);
             privdata->threads.push_back(thrd);
         }